BITS BITS EVERYWHERE

src/main/java/com/volmit/iris/Iris.java

@@ -45,6 +45,7 @@ import com.volmit.iris.util.io.InstanceState;
 import com.volmit.iris.util.io.JarScanner;
 import com.volmit.iris.util.math.M;
 import com.volmit.iris.util.math.RNG;
+import com.volmit.iris.util.matter.MatterTest;
 import com.volmit.iris.util.parallel.MultiBurst;
 import com.volmit.iris.util.plugin.IrisService;
 import com.volmit.iris.util.plugin.Metrics;
@@ -447,6 +448,7 @@ public class Iris extends VolmitPlugin implements Listener {
         J.s(this::setupPapi);
         J.a(ServerConfigurator::configure, 20);
         splash();
+        J.a(MatterTest::test, 20);
         
         if (IrisSettings.get().getStudio().isAutoStartDefaultStudio()) {
             Iris.info("Starting up auto Studio!");

src/main/java/com/volmit/iris/engine/object/IrisCompat.java

@@ -53,7 +53,7 @@ public class IrisCompat {
                 try {
                     IO.writeAll(f, defa);
                 } catch (IOException e) {
-                    Iris.error("Failed to write to compat file");
+                    Iris.error("Failed to writeNodeData to compat file");
                     Iris.reportError(e);
                 }
             });

src/main/java/com/volmit/iris/util/collection/KMap.java

@@ -42,6 +42,19 @@ public class KMap<K, V> extends ConcurrentHashMap<K, V> {
         put(gMap);
     }
 
+    public K getKey(V value)
+    {
+        for(KeyPair<K,V> i : keypair())
+        {
+            if(i.getV().equals(value))
+            {
+                return i.getK();
+            }
+        }
+
+        return null;
+    }
+
     /**
      * Puts a value into a map-value-list based on the key such that if GMap<K,
      * GList<S>> where V is GList<S>

src/main/java/com/volmit/iris/util/data/Varint.java

@@ -45,7 +45,7 @@ public final class Varint {
      * {@link #writeUnsignedVarLong(long, DataOutput)} should be used.
      *
      * @param value value to encode
-     * @param out   to write bytes to
+     * @param out   to writeNodeData bytes to
      * @throws IOException if {@link DataOutput} throws {@link IOException}
      */
     public static void writeSignedVarLong(long value, DataOutput out) throws IOException {
@@ -61,7 +61,7 @@ public final class Varint {
      * instead. This method treats negative input as like a large unsigned value.
      *
      * @param value value to encode
-     * @param out   to write bytes to
+     * @param out   to writeNodeData bytes to
      * @throws IOException if {@link DataOutput} throws {@link IOException}
      */
     public static void writeUnsignedVarLong(long value, DataOutput out) throws IOException {

src/main/java/com/volmit/iris/util/data/palette/BitStorage.java

@@ -20,7 +20,6 @@ package com.volmit.iris.util.data.palette;
 
 import org.apache.commons.lang3.Validate;
 
-import java.util.concurrent.atomic.AtomicLong;
 import java.util.concurrent.atomic.AtomicLongArray;
 import java.util.function.IntConsumer;
 
@@ -141,6 +140,7 @@ public class BitStorage {
         int var2 = cellIndex(var0);
         long var3 = this.data.get(var2);
         int var5 = (var0 - var2 * this.valuesPerLong) * this.bits;
+
         this.data.set(var2, var3 & (this.mask << var5 ^ 0xFFFFFFFFFFFFFFFFL) | (var1 & this.mask) << var5);
     }
 

src/main/java/com/volmit/iris/util/data/palette/CrudeIncrementalIntIdentityHashBiMap.java

@@ -28,19 +28,12 @@ import java.util.concurrent.atomic.AtomicReferenceArray;
 
 public class CrudeIncrementalIntIdentityHashBiMap<K> implements IdMap<K> {
     public static final int NOT_FOUND = -1;
-
     private static final Object EMPTY_SLOT = null;
-
     private static final float LOADFACTOR = 0.8F;
-
     private AtomicReferenceArray<K> keys;
-
     private AtomicIntegerArray values;
-
     private AtomicReferenceArray<K> byId;
-
     private int nextId;
-
     private int size;
 
     public CrudeIncrementalIntIdentityHashBiMap(int var0) {

src/main/java/com/volmit/iris/util/data/palette/HashMapPalette.java

@@ -18,6 +18,8 @@
 
 package com.volmit.iris.util.data.palette;
 
+import com.volmit.iris.Iris;
+import com.volmit.iris.util.collection.KMap;
 import com.volmit.iris.util.nbt.tag.CompoundTag;
 import com.volmit.iris.util.nbt.tag.ListTag;
 
@@ -26,38 +28,40 @@ import java.util.function.Function;
 import java.util.function.Predicate;
 
 public class HashMapPalette<T> implements Palette<T> {
-    private final CrudeIncrementalIntIdentityHashBiMap<T> values;
+    private final KMap<T, Integer> values;
     private final PaletteResize<T> resizeHandler;
     private final int bits;
+    private int id;
 
     public HashMapPalette(int var1, PaletteResize<T> var2) {
         this.bits = var1;
         this.resizeHandler = var2;
-        this.values = new CrudeIncrementalIntIdentityHashBiMap<>(1 << var1);
+        this.values = new KMap<>();
+        id = 1;
     }
 
     public int idFor(T var0) {
-        int var1 = this.values.getId(var0);
+        if(var0 == null)
-        if (var1 == -1) {
+        {
-            var1 = this.values.add(var0);
+            return 0;
-            if (var1 >= 1 << this.bits)
-            {
-                var1 = this.resizeHandler.onResize(this.bits + 1, var0);
-            }
         }
-        return var1;
-    }
 
-    public boolean maybeHas(Predicate<T> var0) {
+        return this.values.computeIfAbsent(var0, (k) -> {
-        for (int var1 = 0; var1 < getSize(); var1++) {
+            int newId = id++;
-            if (var0.test(this.values.byId(var1)))
+
-                return true;
+            if (newId >= 1 << this.bits)
-        }
+            {
-        return false;
+                Iris.info(newId + " to...");
+                newId = this.resizeHandler.onResize(this.bits + 1, var0);
+                Iris.info(newId + "..");
+            }
+
+            return newId;
+        });
     }
 
     public T valueFor(int var0) {
-        return this.values.byId(var0);
+        return this.values.getKey(var0);
     }
 
     public int getSize() {
@@ -66,16 +70,11 @@ public class HashMapPalette<T> implements Palette<T> {
 
     @Override
     public void read(List<T> data) {
-        data.forEach(values::add);
+        data.forEach(this::idFor);
     }
 
     @Override
     public void write(List<T> toList) {
-        this.values.iterator().forEachRemaining(i -> {
+        toList.addAll(values.keySet());
-            if(i != null)
-            {
-                toList.add(i);
-            }
-        });
     }
 }

src/main/java/com/volmit/iris/util/data/palette/LinearPalette.java

@@ -56,17 +56,11 @@ public class LinearPalette<T> implements Palette<T> {
         return resizeHandler.onResize(bits + 1, var0);
     }
 
-    public boolean maybeHas(Predicate<T> var0) {
-        for (int var1 = 0; var1 < size; var1++) {
-            if (var0.test(values.get(var1)))
-                return true;
-        }
-        return false;
-    }
-
     public T valueFor(int var0) {
         if (var0 >= 0 && var0 < size)
+        {
             return this.values.get(var0);
+        }
         return null;
     }
 

src/main/java/com/volmit/iris/util/data/palette/Palette.java

@@ -27,8 +27,6 @@ import java.util.function.Predicate;
 public interface Palette<T> {
     int idFor(T paramT);
 
-    boolean maybeHas(Predicate<T> paramPredicate);
-
     T valueFor(int paramInt);
 
     int getSize();

src/main/java/com/volmit/iris/util/data/palette/PalettedContainer.java

@@ -18,6 +18,8 @@
 
 package com.volmit.iris.util.data.palette;
 
+import com.volmit.iris.Iris;
+import com.volmit.iris.util.math.M;
 import it.unimi.dsi.fastutil.ints.Int2IntOpenHashMap;
 import java.util.List;
 import java.util.function.Predicate;
@@ -64,8 +66,11 @@ public class PalettedContainer<T> implements PaletteResize<T> {
         for (int var4 = 0; var4 < var2.getSize(); var4++) {
             T var5 = var3.valueFor(var2.get(var4));
             if (var5 != null)
+            {
                 set(var4, var5);
+            }
         }
+
         return this.palette.idFor(var1);
     }
 
@@ -89,6 +94,12 @@ public class PalettedContainer<T> implements PaletteResize<T> {
 
     private void set(int var0, T var1) {
         int var2 = this.palette.idFor(var1);
+
+        if(M.r(0.003))
+        {
+            Iris.info("ID for " + var1 + " is " + var2 + " Palette: " + palette.getSize());
+        }
+
         this.storage.set(var0, var2);
     }
 
@@ -143,10 +154,6 @@ public class PalettedContainer<T> implements PaletteResize<T> {
         return var9.getRaw();
     }
 
-    public boolean maybeHas(Predicate<T> var0) {
-        return this.palette.maybeHas(var0);
-    }
-
     public void count(CountConsumer<T> var0) {
         Int2IntOpenHashMap int2IntOpenHashMap = new Int2IntOpenHashMap();
         this.storage.getAll(var1 -> int2IntOpenHashMap.put(var1, int2IntOpenHashMap.get(var1) + 1));

src/main/java/com/volmit/iris/util/function/Consumer2IO.java

@@ -0,0 +1,27 @@
+/*
+ * Iris is a World Generator for Minecraft Bukkit Servers
+ * Copyright (c) 2021 Arcane Arts (Volmit Software)
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.volmit.iris.util.function;
+
+import java.io.IOException;
+
+@SuppressWarnings({"hiding", "RedundantSuppression"})
+@FunctionalInterface
+public interface Consumer2IO<A, B> {
+    void accept(A a, B b) throws IOException;
+}

src/main/java/com/volmit/iris/util/hunk/bits/DataBits.java

@@ -0,0 +1,191 @@
+/*
+ * Iris is a World Generator for Minecraft Bukkit Servers
+ * Copyright (c) 2021 Arcane Arts (Volmit Software)
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.volmit.iris.util.hunk.bits;
+
+import com.volmit.iris.util.data.Varint;
+import org.apache.commons.lang3.Validate;
+
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLongArray;
+import java.util.function.IntConsumer;
+
+public class DataBits {
+    private static final int[] MAGIC = new int[]{
+            -1, -1, 0, Integer.MIN_VALUE, 0, 0, 1431655765, 1431655765, 0, Integer.MIN_VALUE,
+            0, 1, 858993459, 858993459, 0, 715827882, 715827882, 0, 613566756, 613566756,
+            0, Integer.MIN_VALUE, 0, 2, 477218588, 477218588, 0, 429496729, 429496729, 0,
+            390451572, 390451572, 0, 357913941, 357913941, 0, 330382099, 330382099, 0, 306783378,
+            306783378, 0, 286331153, 286331153, 0, Integer.MIN_VALUE, 0, 3, 252645135, 252645135,
+            0, 238609294, 238609294, 0, 226050910, 226050910, 0, 214748364, 214748364, 0,
+            204522252, 204522252, 0, 195225786, 195225786, 0, 186737708, 186737708, 0, 178956970,
+            178956970, 0, 171798691, 171798691, 0, 165191049, 165191049, 0, 159072862, 159072862,
+            0, 153391689, 153391689, 0, 148102320, 148102320, 0, 143165576, 143165576, 0,
+            138547332, 138547332, 0, Integer.MIN_VALUE, 0, 4, 130150524, 130150524, 0, 126322567,
+            126322567, 0, 122713351, 122713351, 0, 119304647, 119304647, 0, 116080197, 116080197,
+            0, 113025455, 113025455, 0, 110127366, 110127366, 0, 107374182, 107374182, 0,
+            104755299, 104755299, 0, 102261126, 102261126, 0, 99882960, 99882960, 0, 97612893,
+            97612893, 0, 95443717, 95443717, 0, 93368854, 93368854, 0, 91382282, 91382282,
+            0, 89478485, 89478485, 0, 87652393, 87652393, 0, 85899345, 85899345, 0,
+            84215045, 84215045, 0, 82595524, 82595524, 0, 81037118, 81037118, 0, 79536431,
+            79536431, 0, 78090314, 78090314, 0, 76695844, 76695844, 0, 75350303, 75350303,
+            0, 74051160, 74051160, 0, 72796055, 72796055, 0, 71582788, 71582788, 0,
+            70409299, 70409299, 0, 69273666, 69273666, 0, 68174084, 68174084, 0, Integer.MIN_VALUE,
+            0, 5};
+
+    private final AtomicLongArray data;
+    private final int bits;
+    private final long mask;
+    private final int size;
+    private final int valuesPerLong;
+    private final int divideMul;
+    private final int divideAdd;
+    private final int divideShift;
+
+    public DataBits(int bits, int length) {
+        this(bits, length, (AtomicLongArray) null);
+    }
+
+    public DataBits(int bits, int length, DataInputStream din) throws IOException
+    {
+        this(bits, length, longs(din, (length + ((char) (64 / bits)) - 1) / ((char) (64 / bits))));
+    }
+
+    public DataBits(int bits, int length, AtomicLongArray data) {
+        Validate.inclusiveBetween(1L, 32L, bits);
+        this.size = length;
+        this.bits = bits;
+        this.mask = (1L << bits) - 1L;
+        this.valuesPerLong = (char) (64 / bits);
+        int var3 = 3 * (valuesPerLong - 1);
+        this.divideMul = MAGIC[var3];
+        this.divideAdd = MAGIC[var3 + 1];
+        this.divideShift = MAGIC[var3 + 2];
+        int var4 = (length + valuesPerLong - 1) / valuesPerLong;
+        if (data != null) {
+            if (data.length() != var4)
+            {
+                throw new RuntimeException("NO!");
+            }
+            this.data = data;
+        } else {
+            this.data = new AtomicLongArray(var4);
+        }
+    }
+
+    private static AtomicLongArray longs(DataInputStream din, int len) throws IOException{
+        AtomicLongArray a = new AtomicLongArray(len);
+
+        for(int i = 0; i < len; i++)
+        {
+            a.set(i, din.readLong());
+        }
+
+        return a;
+    }
+
+    public DataBits setBits(int newBits)
+    {
+        if(bits != newBits)
+        {
+            DataBits newData = new DataBits(newBits, size);
+            AtomicInteger c = new AtomicInteger(0);
+            getAll((i) -> newData.set(c.incrementAndGet(), i));
+            return newData;
+        }
+
+        return this;
+    }
+
+    private int cellIndex(int var0) {
+        long var1 = Integer.toUnsignedLong(this.divideMul);
+        long var3 = Integer.toUnsignedLong(this.divideAdd);
+        return (int) (var0 * var1 + var3 >> 32L >> this.divideShift);
+    }
+
+    @SuppressWarnings("PointlessBitwiseExpression")
+    public int getAndSet(int var0, int var1) {
+        Validate.inclusiveBetween(0L, (this.size - 1), var0);
+        Validate.inclusiveBetween(0L, this.mask, var1);
+        int var2 = cellIndex(var0);
+        long var3 = this.data.get(var2);
+        int var5 = (var0 - var2 * this.valuesPerLong) * this.bits;
+        int var6 = (int) (var3 >> var5 & this.mask);
+        this.data.set(var2, var3 & (this.mask << var5 ^ 0xFFFFFFFFFFFFFFFFL) | (var1 & this.mask) << var5);
+        return var6;
+    }
+
+    @SuppressWarnings("PointlessBitwiseExpression")
+    public void set(int var0, int var1) {
+        Validate.inclusiveBetween(0L, (this.size - 1), var0);
+        Validate.inclusiveBetween(0L, this.mask, var1);
+        int var2 = cellIndex(var0);
+        long var3 = this.data.get(var2);
+        int var5 = (var0 - var2 * this.valuesPerLong) * this.bits;
+
+        this.data.set(var2, var3 & (this.mask << var5 ^ 0xFFFFFFFFFFFFFFFFL) | (var1 & this.mask) << var5);
+    }
+
+    public int get(int var0) {
+        Validate.inclusiveBetween(0L, (size - 1), var0);
+        int var1 = cellIndex(var0);
+        long var2 = this.data.get(var1);
+        int var4 = (var0 - var1 * valuesPerLong) * this.bits;
+        return (int) (var2 >> var4 & mask);
+    }
+
+    public AtomicLongArray getRaw() {
+        return data;
+    }
+
+    public int getSize() {
+        return size;
+    }
+
+    public int getBits() {
+        return bits;
+    }
+
+    public void getAll(IntConsumer var0) {
+        int var1 = 0;
+        for(int i = 0; i < data.length(); i++)
+        {
+            long var5 = data.get(i);
+            for (int var7 = 0; var7 < valuesPerLong; var7++) {
+                var0.accept((int) (var5 & mask));
+                var5 >>= bits;
+                if (++var1 >= size)
+                {
+                    return;
+                }
+            }
+        }
+    }
+
+    public void write(DataOutputStream dos) throws IOException {
+        dos.writeByte(bits);
+
+        for(int i = 0; i < data.length(); i++)
+        {
+            dos.writeLong(data.get(i));
+        }
+    }
+}

src/main/java/com/volmit/iris/util/hunk/bits/DataContainer.java

@@ -0,0 +1,146 @@
+/*
+ * Iris is a World Generator for Minecraft Bukkit Servers
+ * Copyright (c) 2021 Arcane Arts (Volmit Software)
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.volmit.iris.util.hunk.bits;
+
+import com.volmit.iris.util.data.Varint;
+
+import java.io.ByteArrayOutputStream;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
+
+public class DataContainer<T> {
+    protected static final int INITIAL_BITS = 3;
+    protected static final int LINEAR_BITS_LIMIT = 5;
+    protected static final int LINEAR_INITIAL_LENGTH = (int) Math.pow(2, LINEAR_BITS_LIMIT) + 1;
+    protected static final int[] BIT = computeBitLimits();
+    private final AtomicReference<Palette<T>> palette;
+    private final AtomicReference<DataBits> data;
+    private final AtomicInteger bits;
+    private final int length;
+    private final Writable<T> writer;
+
+    public DataContainer(Writable<T> writer, int length)
+    {
+        this.writer = writer;
+        this.length = length;
+        this.palette = new AtomicReference<>(newPalette(INITIAL_BITS));
+        this.data = new AtomicReference<>(new DataBits(INITIAL_BITS, length));
+        this.bits = new AtomicInteger(INITIAL_BITS);
+    }
+
+    public byte[] write() throws IOException
+    {
+        ByteArrayOutputStream boas = new ByteArrayOutputStream();
+        write(boas);
+        return boas.toByteArray();
+    }
+
+    public static <T> DataContainer<T> read(InputStream in, Writable<T> writer) throws IOException {
+        DataInputStream din = new DataInputStream(in);
+        return readDin(din, writer);
+    }
+
+    public static <T> DataContainer<T> readDin(DataInputStream in, Writable<T> writer) throws IOException {
+        DataInputStream din = new DataInputStream(in);
+        DataContainer<T> container = new DataContainer<>(writer, Varint.readUnsignedVarInt(din));
+        int paletteSize = Varint.readUnsignedVarInt(din);
+        container.palette.set(container.newPalette(BIT[paletteSize]).from(paletteSize, writer, din));
+        container.data.set(new DataBits(container.palette.get().bits(), container.length, din));
+        return container;
+    }
+
+    public void write(OutputStream out) throws IOException
+    {
+        DataOutputStream dos = new DataOutputStream(out);
+        writeDos(dos);
+    }
+
+    public void writeDos(DataOutputStream out) throws IOException
+    {
+        DataOutputStream dos = new DataOutputStream(out);
+        Varint.writeUnsignedVarInt(length);
+        Varint.writeUnsignedVarInt(palette.get().size());
+        palette.get().iterateIO((data, __) -> writer.writeNodeData(dos, data));
+        data.get().write(dos);
+    }
+
+    private Palette<T> newPalette(int bits)
+    {
+        if(bits <= LINEAR_BITS_LIMIT)
+        {
+            return new LinearPalette<>(LINEAR_INITIAL_LENGTH);
+        }
+
+        return new HashPalette<>();
+    }
+
+    public void set(int position, T t)
+    {
+        int id = palette.get().id(t);
+
+        if(id == -1)
+        {
+            id = palette.get().add(t);
+        }
+
+        data.get().set(position, id);
+    }
+
+    public T get(int position)
+    {
+        int id = data.get().get(position);
+
+        if(id <= 0)
+        {
+            return null;
+        }
+
+        return palette.get().get(id - 1);
+    }
+
+    public void setBits(int bits)
+    {
+        if(this.bits.get() != bits)
+        {
+            if(this.bits.get() <= LINEAR_BITS_LIMIT != bits <= LINEAR_BITS_LIMIT)
+            {
+                palette.set(newPalette(bits).from(palette.get()));
+            }
+
+            this.bits.set(bits);
+            data.set(data.get().setBits(bits));
+        }
+    }
+
+    private static int[] computeBitLimits() {
+        int[] m = new int[16];
+
+        for(int i = 0; i < m.length; i++)
+        {
+            m[i] = (int) Math.pow(2, i);
+        }
+
+        return m;
+    }
+}

src/main/java/com/volmit/iris/util/hunk/bits/HashPalette.java

@@ -0,0 +1,74 @@
+/*
+ * Iris is a World Generator for Minecraft Bukkit Servers
+ * Copyright (c) 2021 Arcane Arts (Volmit Software)
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.volmit.iris.util.hunk.bits;
+
+import com.volmit.iris.util.collection.KMap;
+import com.volmit.iris.util.function.Consumer2;
+
+import java.util.concurrent.atomic.AtomicInteger;
+
+public class HashPalette<T> implements Palette<T> {
+    private final KMap<T, Integer> palette;
+    private final KMap<Integer, T> lookup;
+    private final AtomicInteger size;
+
+    public HashPalette()
+    {
+        this.size = new AtomicInteger(0);
+        this.palette = new KMap<>();
+        this.lookup = new KMap<>();
+    }
+
+    @Override
+    public T get(int id) {
+        if(id < 0 || id >= size.get())
+        {
+            return null;
+        }
+
+        return lookup.get(id);
+    }
+
+    @Override
+    public int add(T t) {
+        int index = size.getAndIncrement();
+        palette.put(t, index);
+        lookup.put(index, t);
+        return index;
+    }
+
+    @Override
+    public int id(T t) {
+        Integer v = palette.get(t);
+        return v != null ? v : -1;
+    }
+
+    @Override
+    public int size() {
+        return size.get();
+    }
+
+    @Override
+    public void iterate(Consumer2<T, Integer> c) {
+        for(T i : palette.keySet())
+        {
+            c.accept(i, id(i));
+        }
+    }
+}

src/main/java/com/volmit/iris/util/hunk/bits/LinearPalette.java

@@ -0,0 +1,94 @@
+/*
+ * Iris is a World Generator for Minecraft Bukkit Servers
+ * Copyright (c) 2021 Arcane Arts (Volmit Software)
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.volmit.iris.util.hunk.bits;
+
+import com.volmit.iris.util.function.Consumer2;
+
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.concurrent.atomic.AtomicReferenceArray;
+
+public class LinearPalette<T> implements Palette<T> {
+    private final AtomicReference<AtomicReferenceArray<T>> palette;
+    private final AtomicInteger size;
+
+    public LinearPalette(int initialSize)
+    {
+        this.size = new AtomicInteger(0);
+        this.palette = new AtomicReference<>(new AtomicReferenceArray<>(initialSize));
+    }
+
+    @Override
+    public T get(int id) {
+        if(id < 0 || id >= size.get())
+        {
+            return null;
+        }
+
+        return palette.get().get(id);
+    }
+
+    @Override
+    public int add(T t) {
+        int index = size.getAndIncrement();
+        grow(index + 1);
+        palette.get().set(index, t);
+        return index;
+    }
+
+    private void grow(int newLength) {
+        if(newLength > palette.get().length())
+        {
+            AtomicReferenceArray<T> a = new AtomicReferenceArray<>(newLength + size.get());
+
+            for(int i = 0; i < palette.get().length(); i++)
+            {
+                a.set(i, palette.get().get(i));
+            }
+
+            palette.set(a);
+        }
+    }
+
+    @Override
+    public int id(T t) {
+        for(int i = 0; i < size(); i++)
+        {
+            if(t.equals(palette.get().get(i)))
+            {
+                return i;
+            }
+        }
+
+        return -1;
+    }
+
+    @Override
+    public int size() {
+        return size.get();
+    }
+
+    @Override
+    public void iterate(Consumer2<T, Integer> c) {
+        for(int i = 0; i < size(); i++)
+        {
+            c.accept(palette.get().get(i), i);
+        }
+    }
+}

src/main/java/com/volmit/iris/util/hunk/bits/Palette.java

@@ -0,0 +1,86 @@
+/*
+ * Iris is a World Generator for Minecraft Bukkit Servers
+ * Copyright (c) 2021 Arcane Arts (Volmit Software)
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.volmit.iris.util.hunk.bits;
+
+import com.volmit.iris.util.function.Consumer2;
+import com.volmit.iris.util.function.Consumer2IO;
+
+import java.io.DataInputStream;
+import java.io.IOException;
+
+public interface Palette<T> {
+    T get(int id);
+
+    int add(T t);
+
+    int id(T t);
+
+    int size();
+
+    default int bits()
+    {
+        return bits(DataContainer.INITIAL_BITS);
+    }
+
+    default int bits(int minBits)
+    {
+        if(size() <= DataContainer.BIT[minBits])
+        {
+            return minBits;
+        }
+
+        for(int i = 0; i <  DataContainer.BIT.length; i++)
+        {
+            if(DataContainer.BIT[i] >= size())
+            {
+                return i;
+            }
+        }
+
+        return DataContainer.BIT.length - 1;
+    }
+
+    void iterate(Consumer2<T, Integer> c);
+
+    default void iterateIO(Consumer2IO<T, Integer> c)
+    {
+        iterate((a,b)-> {
+            try {
+                c.accept(a,b);
+            } catch (IOException e) {
+                e.printStackTrace();
+            }
+        });
+    }
+
+    default Palette<T> from(int size, Writable<T> writable, DataInputStream in) throws IOException {
+        for(int i = 0; i < size; i++)
+        {
+            add(writable.readNodeData(in));
+        }
+
+        return this;
+    }
+
+    default Palette<T> from(Palette<T> oldPalette)
+    {
+        oldPalette.iterate((k,v) -> add(k));
+        return this;
+    }
+}

src/main/java/com/volmit/iris/util/hunk/bits/Writable.java

@@ -0,0 +1,29 @@
+/*
+ * Iris is a World Generator for Minecraft Bukkit Servers
+ * Copyright (c) 2021 Arcane Arts (Volmit Software)
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.volmit.iris.util.hunk.bits;
+
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+
+public interface Writable<T> {
+    T readNodeData(DataInputStream din) throws IOException;
+
+    void writeNodeData(DataOutputStream dos, T t) throws IOException;
+}

src/main/java/com/volmit/iris/util/hunk/storage/PaletteHunk.java

@@ -23,6 +23,8 @@ import com.volmit.iris.util.data.palette.PalettedContainer;
 import com.volmit.iris.util.function.Consumer4;
 import com.volmit.iris.util.function.Consumer4IO;
 import com.volmit.iris.util.hunk.Hunk;
+import com.volmit.iris.util.hunk.bits.DataContainer;
+import com.volmit.iris.util.hunk.bits.Writable;
 import lombok.Data;
 import lombok.EqualsAndHashCode;
 
@@ -33,17 +35,27 @@ import java.util.Map;
 @Data
 @EqualsAndHashCode(callSuper = false)
 public class PaletteHunk<T> extends StorageHunk<T> implements Hunk<T> {
-    private PalettedContainer<T> data;
+    private DataContainer<T> data;
 
-    public PaletteHunk() {
+    public PaletteHunk(int w, int h, int d, Writable<T> writer) {
-        super(16, 16, 16);
+        super(w,h,d);
-        data = new PalettedContainer<>();
+        data = new DataContainer<>(writer, w * h * d);
     }
 
+
+    public void setPalette(DataContainer<T> c) {
+        data = c;
+    }
+
+
     public boolean isMapped() {
         return false;
     }
 
+    private int index(int x, int y, int z) {
+        return (z * getWidth() * getHeight()) + (y * getWidth()) + x;
+    }
+
     @Override
     public synchronized Hunk<T> iterateSync(Consumer4<Integer, Integer, Integer, T> c) {
         for(int i = 0; i < getWidth(); i++)
@@ -84,11 +96,11 @@ public class PaletteHunk<T> extends StorageHunk<T> implements Hunk<T> {
 
     @Override
     public void setRaw(int x, int y, int z, T t) {
-        data.set(x,y,z,t);
+        data.set(index(x, y, z), t);
     }
 
     @Override
     public T getRaw(int x, int y, int z) {
-        return data.get(x,y,z);
+        return data.get(index(x, y, z));
     }
 }

src/main/java/com/volmit/iris/util/hunk/storage/PaletteOrHunk.java

@@ -22,6 +22,8 @@ import com.volmit.iris.util.data.palette.PalettedContainer;
 import com.volmit.iris.util.function.Consumer4;
 import com.volmit.iris.util.function.Consumer4IO;
 import com.volmit.iris.util.hunk.Hunk;
+import com.volmit.iris.util.hunk.bits.DataContainer;
+import com.volmit.iris.util.hunk.bits.Writable;
 
 import java.io.IOException;
 import java.util.Map;
@@ -29,26 +31,26 @@ import java.util.function.Supplier;
 
 public class PaletteOrHunk<T> extends StorageHunk<T> implements Hunk<T> {
     private final Hunk<T> hunk;
-    public PaletteOrHunk(int width, int height, int depth, boolean allow, Supplier<Hunk<T>> factory) {
+    public PaletteOrHunk(int width, int height, int depth, boolean allow, Writable<T> writable, Supplier<Hunk<T>> factory) {
         super(width, height, depth);
-        hunk = (width == 16 && height == 16 && depth == 16 && allow) ? new PaletteHunk<>() : factory.get();
+        hunk = (allow) ? new PaletteHunk<>(width, height, depth, writable) : factory.get();
     }
 
-    public PalettedContainer<T> palette()
+    public DataContainer<T> palette()
     {
         return isPalette() ? ((PaletteHunk<T>)hunk).getData() : null;
     }
 
-    public void palette(PalettedContainer<T> t)
+    public void setPalette(DataContainer<T> c) {
-    {
+        if(isPalette())
-        if(isPalette()){
+        {
-            ((PaletteHunk<T>)hunk).setData(t);
+            ((PaletteHunk<T>)hunk).setPalette(c);
         }
     }
 
     public boolean isPalette()
     {
-        return getWidth() == 16 && getHeight() == 16 && getDepth() == 16;
+        return hunk instanceof PaletteHunk;
     }
 
     @Override

src/main/java/com/volmit/iris/util/hunk/view/FunctionalHunkView.java

@@ -36,7 +36,7 @@ public class FunctionalHunkView<R, T> implements Hunk<T> {
     @Override
     public void setRaw(int x, int y, int z, T t) {
         if (backConverter == null) {
-            throw new UnsupportedOperationException("You cannot write to this hunk (Read Only)");
+            throw new UnsupportedOperationException("You cannot writeNodeData to this hunk (Read Only)");
         }
 
         src.setRaw(x, y, z, backConverter.apply(t));

src/main/java/com/volmit/iris/util/io/IO.java

@@ -25,6 +25,7 @@ import java.io.BufferedReader;
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
 import java.io.CharArrayWriter;
+import java.io.DataOutputStream;
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.FileNotFoundException;
@@ -162,6 +163,24 @@ public class IO {
         return new String(hexChars).toUpperCase();
     }
 
+    public static String longsToHex(long[] bytes) {
+        byte[] v = new byte[bytes.length * 8];
+
+        for(int i = 0; i < bytes.length; i++)
+        {
+            v[i * 8] = (byte)(bytes[i] >>> 56);
+            v[(i * 8) + 1] = (byte)(bytes[i] >>> 48);
+            v[(i * 8) + 2] = (byte)(bytes[i] >>> 40);
+            v[(i * 8) + 3] = (byte)(bytes[i] >>> 32);
+            v[(i * 8) + 4] = (byte)(bytes[i] >>> 24);
+            v[(i * 8) + 5] = (byte)(bytes[i] >>> 16);
+            v[(i * 8) + 6] = (byte)(bytes[i] >>>  8);
+            v[(i * 8) + 7] = (byte)(bytes[i] >>>  0);
+        }
+
+        return bytesToHex(v);
+    }
+
     /**
      * Transfers the length of the buffer amount of data from the input stream to
      * the output stream
@@ -239,7 +258,7 @@ public class IO {
      * buffer size. This does NOT close streams.
      *
      * @param in         the input stream to read from
-     * @param out        the output stream to write to
+     * @param out        the output stream to writeNodeData to
      * @param bufferSize the target buffer size
      * @return total size transfered
      * @throws IOException shit happens
@@ -915,7 +934,7 @@ public class IO {
         return new ByteArrayInputStream(bytes);
     }
 
-    // write byte[]
+    // writeNodeData byte[]
     // -----------------------------------------------------------------------
 
     /**
@@ -939,8 +958,8 @@ public class IO {
     /**
      * Writes bytes from a <code>byte[]</code> to an <code>OutputStream</code>.
      *
-     * @param data   the byte array to write, do not modify during output, null ignored
+     * @param data   the byte array to writeNodeData, do not modify during output, null ignored
-     * @param output the <code>OutputStream</code> to write to
+     * @param output the <code>OutputStream</code> to writeNodeData to
      * @throws NullPointerException if output is null
      * @throws IOException          if an I/O error occurs
      * @since Commons IO 1.1
@@ -957,8 +976,8 @@ public class IO {
      * <p>
      * This method uses {@link String#String(byte[])}.
      *
-     * @param data   the byte array to write, do not modify during output, null ignored
+     * @param data   the byte array to writeNodeData, do not modify during output, null ignored
-     * @param output the <code>Writer</code> to write to
+     * @param output the <code>Writer</code> to writeNodeData to
      * @throws NullPointerException if output is null
      * @throws IOException          if an I/O error occurs
      * @since Commons IO 1.1
@@ -969,7 +988,7 @@ public class IO {
         }
     }
 
-    // write char[]
+    // writeNodeData char[]
     // -----------------------------------------------------------------------
 
     /**
@@ -981,8 +1000,8 @@ public class IO {
      * <p>
      * This method uses {@link String#String(byte[], String)}.
      *
-     * @param data     the byte array to write, do not modify during output, null ignored
+     * @param data     the byte array to writeNodeData, do not modify during output, null ignored
-     * @param output   the <code>Writer</code> to write to
+     * @param output   the <code>Writer</code> to writeNodeData to
      * @param encoding the encoding to use, null means platform default
      * @throws NullPointerException if output is null
      * @throws IOException          if an I/O error occurs
@@ -1002,8 +1021,8 @@ public class IO {
      * Writes chars from a <code>char[]</code> to a <code>Writer</code> using the
      * default character encoding of the platform.
      *
-     * @param data   the char array to write, do not modify during output, null ignored
+     * @param data   the char array to writeNodeData, do not modify during output, null ignored
-     * @param output the <code>Writer</code> to write to
+     * @param output the <code>Writer</code> to writeNodeData to
      * @throws NullPointerException if output is null
      * @throws IOException          if an I/O error occurs
      * @since Commons IO 1.1
@@ -1020,8 +1039,8 @@ public class IO {
      * <p>
      * This method uses {@link String#String(char[])} and {@link String#getBytes()}.
      *
-     * @param data   the char array to write, do not modify during output, null ignored
+     * @param data   the char array to writeNodeData, do not modify during output, null ignored
-     * @param output the <code>OutputStream</code> to write to
+     * @param output the <code>OutputStream</code> to writeNodeData to
      * @throws NullPointerException if output is null
      * @throws IOException          if an I/O error occurs
      * @since Commons IO 1.1
@@ -1032,7 +1051,7 @@ public class IO {
         }
     }
 
-    // write String
+    // writeNodeData String
     // -----------------------------------------------------------------------
 
     /**
@@ -1045,8 +1064,8 @@ public class IO {
      * This method uses {@link String#String(char[])} and
      * {@link String#getBytes(String)}.
      *
-     * @param data     the char array to write, do not modify during output, null ignored
+     * @param data     the char array to writeNodeData, do not modify during output, null ignored
-     * @param output   the <code>OutputStream</code> to write to
+     * @param output   the <code>OutputStream</code> to writeNodeData to
      * @param encoding the encoding to use, null means platform default
      * @throws NullPointerException if output is null
      * @throws IOException          if an I/O error occurs
@@ -1065,8 +1084,8 @@ public class IO {
     /**
      * Writes chars from a <code>String</code> to a <code>Writer</code>.
      *
-     * @param data   the <code>String</code> to write, null ignored
+     * @param data   the <code>String</code> to writeNodeData, null ignored
-     * @param output the <code>Writer</code> to write to
+     * @param output the <code>Writer</code> to writeNodeData to
      * @throws NullPointerException if output is null
      * @throws IOException          if an I/O error occurs
      * @since Commons IO 1.1
@@ -1084,8 +1103,8 @@ public class IO {
      * <p>
      * This method uses {@link String#getBytes()}.
      *
-     * @param data   the <code>String</code> to write, null ignored
+     * @param data   the <code>String</code> to writeNodeData, null ignored
-     * @param output the <code>OutputStream</code> to write to
+     * @param output the <code>OutputStream</code> to writeNodeData to
      * @throws NullPointerException if output is null
      * @throws IOException          if an I/O error occurs
      * @since Commons IO 1.1
@@ -1096,7 +1115,7 @@ public class IO {
         }
     }
 
-    // write StringBuffer
+    // writeNodeData StringBuffer
     // -----------------------------------------------------------------------
 
     /**
@@ -1108,8 +1127,8 @@ public class IO {
      * <p>
      * This method uses {@link String#getBytes(String)}.
      *
-     * @param data     the <code>String</code> to write, null ignored
+     * @param data     the <code>String</code> to writeNodeData, null ignored
-     * @param output   the <code>OutputStream</code> to write to
+     * @param output   the <code>OutputStream</code> to writeNodeData to
      * @param encoding the encoding to use, null means platform default
      * @throws NullPointerException if output is null
      * @throws IOException          if an I/O error occurs
@@ -1128,8 +1147,8 @@ public class IO {
     /**
      * Writes chars from a <code>StringBuffer</code> to a <code>Writer</code>.
      *
-     * @param data   the <code>StringBuffer</code> to write, null ignored
+     * @param data   the <code>StringBuffer</code> to writeNodeData, null ignored
-     * @param output the <code>Writer</code> to write to
+     * @param output the <code>Writer</code> to writeNodeData to
      * @throws NullPointerException if output is null
      * @throws IOException          if an I/O error occurs
      * @since Commons IO 1.1
@@ -1147,8 +1166,8 @@ public class IO {
      * <p>
      * This method uses {@link String#getBytes()}.
      *
-     * @param data   the <code>StringBuffer</code> to write, null ignored
+     * @param data   the <code>StringBuffer</code> to writeNodeData, null ignored
-     * @param output the <code>OutputStream</code> to write to
+     * @param output the <code>OutputStream</code> to writeNodeData to
      * @throws NullPointerException if output is null
      * @throws IOException          if an I/O error occurs
      * @since Commons IO 1.1
@@ -1171,8 +1190,8 @@ public class IO {
      * <p>
      * This method uses {@link String#getBytes(String)}.
      *
-     * @param data     the <code>StringBuffer</code> to write, null ignored
+     * @param data     the <code>StringBuffer</code> to writeNodeData, null ignored
-     * @param output   the <code>OutputStream</code> to write to
+     * @param output   the <code>OutputStream</code> to writeNodeData to
      * @param encoding the encoding to use, null means platform default
      * @throws NullPointerException if output is null
      * @throws IOException          if an I/O error occurs
@@ -1193,9 +1212,9 @@ public class IO {
      * <code>OutputStream</code> line by line, using the default character encoding
      * of the platform and the specified line ending.
      *
-     * @param lines      the lines to write, null entries produce blank lines
+     * @param lines      the lines to writeNodeData, null entries produce blank lines
      * @param lineEnding the line separator to use, null is system default
-     * @param output     the <code>OutputStream</code> to write to, not null, not closed
+     * @param output     the <code>OutputStream</code> to writeNodeData to, not null, not closed
      * @throws NullPointerException if the output is null
      * @throws IOException          if an I/O error occurs
      * @since Commons IO 1.1
@@ -1224,9 +1243,9 @@ public class IO {
      * Character encoding names can be found at
      * <a href="http://www.iana.org/assignments/character-sets">IANA</a>.
      *
-     * @param lines      the lines to write, null entries produce blank lines
+     * @param lines      the lines to writeNodeData, null entries produce blank lines
      * @param lineEnding the line separator to use, null is system default
-     * @param output     the <code>OutputStream</code> to write to, not null, not closed
+     * @param output     the <code>OutputStream</code> to writeNodeData to, not null, not closed
      * @param encoding   the encoding to use, null means platform default
      * @throws NullPointerException if the output is null
      * @throws IOException          if an I/O error occurs
@@ -1259,9 +1278,9 @@ public class IO {
      * Writes the <code>toString()</code> value of each item in a collection to a
      * <code>Writer</code> line by line, using the specified line ending.
      *
-     * @param lines      the lines to write, null entries produce blank lines
+     * @param lines      the lines to writeNodeData, null entries produce blank lines
      * @param lineEnding the line separator to use, null is system default
-     * @param writer     the <code>Writer</code> to write to, not null, not closed
+     * @param writer     the <code>Writer</code> to writeNodeData to, not null, not closed
      * @throws NullPointerException if the input is null
      * @throws IOException          if an I/O error occurs
      * @since Commons IO 1.1
@@ -1294,7 +1313,7 @@ public class IO {
      * <code>copyLarge(InputStream, OutputStream)</code> method.
      *
      * @param input  the <code>InputStream</code> to read from
-     * @param output the <code>OutputStream</code> to write to
+     * @param output the <code>OutputStream</code> to writeNodeData to
      * @return the number of bytes copied
      * @throws NullPointerException if the input or output is null
      * @throws IOException          if an I/O error occurs
@@ -1317,7 +1336,7 @@ public class IO {
      * <code>BufferedInputStream</code>.
      *
      * @param input  the <code>InputStream</code> to read from
-     * @param output the <code>OutputStream</code> to write to
+     * @param output the <code>OutputStream</code> to writeNodeData to
      * @return the number of bytes copied
      * @throws NullPointerException if the input or output is null
      * @throws IOException          if an I/O error occurs
@@ -1344,7 +1363,7 @@ public class IO {
      * This method uses {@link InputStreamReader}.
      *
      * @param input  the <code>InputStream</code> to read from
-     * @param output the <code>Writer</code> to write to
+     * @param output the <code>Writer</code> to writeNodeData to
      * @throws NullPointerException if the input or output is null
      * @throws IOException          if an I/O error occurs
      * @since Commons IO 1.1
@@ -1370,7 +1389,7 @@ public class IO {
      * This method uses {@link InputStreamReader}.
      *
      * @param input    the <code>InputStream</code> to read from
-     * @param output   the <code>Writer</code> to write to
+     * @param output   the <code>Writer</code> to writeNodeData to
      * @param encoding the encoding to use, null means platform default
      * @throws NullPointerException if the input or output is null
      * @throws IOException          if an I/O error occurs
@@ -1397,7 +1416,7 @@ public class IO {
      * <code>copyLarge(Reader, Writer)</code> method.
      *
      * @param input  the <code>Reader</code> to read from
-     * @param output the <code>Writer</code> to write to
+     * @param output the <code>Writer</code> to writeNodeData to
      * @return the number of characters copied
      * @throws NullPointerException if the input or output is null
      * @throws IOException          if an I/O error occurs
@@ -1420,7 +1439,7 @@ public class IO {
      * <code>BufferedReader</code>.
      *
      * @param input  the <code>Reader</code> to read from
-     * @param output the <code>Writer</code> to write to
+     * @param output the <code>Writer</code> to writeNodeData to
      * @return the number of characters copied
      * @throws NullPointerException if the input or output is null
      * @throws IOException          if an I/O error occurs
@@ -1451,7 +1470,7 @@ public class IO {
      * This method uses {@link OutputStreamWriter}.
      *
      * @param input  the <code>Reader</code> to read from
-     * @param output the <code>OutputStream</code> to write to
+     * @param output the <code>OutputStream</code> to writeNodeData to
      * @throws NullPointerException if the input or output is null
      * @throws IOException          if an I/O error occurs
      * @since Commons IO 1.1
@@ -1483,7 +1502,7 @@ public class IO {
      * This method uses {@link OutputStreamWriter}.
      *
      * @param input    the <code>Reader</code> to read from
-     * @param output   the <code>OutputStream</code> to write to
+     * @param output   the <code>OutputStream</code> to writeNodeData to
      * @param encoding the encoding to use, null means platform default
      * @throws NullPointerException if the input or output is null
      * @throws IOException          if an I/O error occurs

src/main/java/com/volmit/iris/util/mantle/Mantle.java

@@ -52,7 +52,7 @@ import java.util.concurrent.atomic.AtomicBoolean;
 
 /**
  * The mantle can store any type of data slice anywhere and manage regions & IO on it's own.
- * This class is fully thread safe read & write
+ * This class is fully thread safe read & writeNodeData
  */
 public class Mantle {
     private final File dataFolder;
@@ -335,7 +335,7 @@ public class Mantle {
     }
 
     /**
-     * Closes the Mantle. By closing the mantle, you can no longer read or write
+     * Closes the Mantle. By closing the mantle, you can no longer read or writeNodeData
      * any data to the mantle or it's Tectonic Plates. Closing will also flush any
      * loaded regions to the disk in parallel.
      */

src/main/java/com/volmit/iris/util/mantle/TectonicPlate.java

@@ -163,7 +163,7 @@ public class TectonicPlate {
     /**
      * Write this tectonic plate to file
      *
-     * @param file the file to write it to
+     * @param file the file to writeNodeData it to
      * @throws IOException shit happens
      */
     public void write(File file) throws IOException {

src/main/java/com/volmit/iris/util/matter/MatterSlice.java

@@ -26,6 +26,8 @@ import com.volmit.iris.util.data.palette.Palette;
 import com.volmit.iris.util.data.palette.PaletteType;
 import com.volmit.iris.util.data.palette.PalettedContainer;
 import com.volmit.iris.util.hunk.Hunk;
+import com.volmit.iris.util.hunk.bits.DataContainer;
+import com.volmit.iris.util.hunk.bits.Writable;
 import com.volmit.iris.util.hunk.storage.PaletteOrHunk;
 import org.bukkit.Location;
 import org.bukkit.World;
@@ -39,7 +41,7 @@ import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
 
-public interface MatterSlice<T> extends Hunk<T>, PaletteType<T> {
+public interface MatterSlice<T> extends Hunk<T>, PaletteType<T>, Writable<T> {
     Class<T> getType();
 
     Palette<T> getGlobalPalette();
@@ -49,11 +51,21 @@ public interface MatterSlice<T> extends Hunk<T>, PaletteType<T> {
         writeNode(s, dos);
     }
 
+    @Override
+    default void writeNodeData(DataOutputStream dos, T s) throws IOException {
+        writeNode(s, dos);
+    }
+
     @Override
     default T readPaletteNode(DataInputStream din) throws IOException {
         return readNode(din);
     }
 
+    @Override
+    default T readNodeData(DataInputStream din) throws IOException {
+        return readNode(din);
+    }
+
     default void applyFilter(MatterFilter<T> filter) {
         updateSync(filter::update);
     }
@@ -154,22 +166,7 @@ public interface MatterSlice<T> extends Hunk<T>, PaletteType<T> {
         dos.writeUTF(getType().getCanonicalName());
        if((this instanceof PaletteOrHunk f && f.isPalette()))
        {
-           PalettedContainer<T> c = f.palette();
+           f.palette().writeDos(dos);
-           List<T> palette = new ArrayList<>();
-           long[] data = c.write(palette);
-
-           Varint.writeUnsignedVarInt(palette.size(), dos);
-           for(T i : palette)
-           {
-               writeNode(i, dos);
-           }
-
-           Varint.writeUnsignedVarInt(data.length, dos);
-           for(long i : data)
-           {
-               dos.writeLong(i);
-           }
-
            return;
        }
 
@@ -194,24 +191,7 @@ public interface MatterSlice<T> extends Hunk<T>, PaletteType<T> {
     default void read(DataInputStream din) throws IOException {
         if((this instanceof PaletteOrHunk f && f.isPalette()))
         {
-            PalettedContainer<T> c = new PalettedContainer<>();
+            f.setPalette(DataContainer.readDin(din, this));
-            List<T> palette = new ArrayList<>();
-            int ps = Varint.readUnsignedVarInt(din);
-
-            for(int i = 0; i < ps; i++)
-            {
-                palette.add(readNode(din));
-            }
-
-            int ds = Varint.readUnsignedVarInt(din);
-            long[] data = new long[ds];
-            for(int i = 0; i < ds; i++)
-            {
-                data[i] = din.readLong();
-            }
-
-            c.read(palette, data);
-
             return;
         }
 

src/main/java/com/volmit/iris/util/matter/MatterTest.java

@@ -0,0 +1,74 @@
+/*
+ * Iris is a World Generator for Minecraft Bukkit Servers
+ * Copyright (c) 2021 Arcane Arts (Volmit Software)
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.volmit.iris.util.matter;
+
+import com.volmit.iris.Iris;
+import com.volmit.iris.engine.object.NoiseStyle;
+import com.volmit.iris.util.collection.KList;
+import com.volmit.iris.util.data.palette.PalettedContainer;
+import com.volmit.iris.util.io.IO;
+import com.volmit.iris.util.math.RNG;
+import com.volmit.iris.util.noise.CNG;
+import org.checkerframework.checker.units.qual.K;
+
+import java.util.Arrays;
+import java.util.List;
+
+public class MatterTest {
+    public static void test()
+    {
+        CNG cng = NoiseStyle.STATIC.create(new RNG(1337));
+        PalettedContainer<Integer> p = new PalettedContainer<>();
+
+        for(int i = 0; i < 16; i++)
+        {
+            for(int j = 0; j < 16; j++)
+            {
+                for(int k = 0; k < 16; k++)
+                {
+                    p.set(i,j,k,cng.fit(1, 3, i,j,k));
+                }
+            }
+        }
+
+        KList<Integer> palette = new KList<>();
+        long[] data = p.write(palette);
+
+        Iris.info("RAW PALE: " + palette.toString(","));
+        Iris.info("RAW DATA: " + IO.longsToHex(data));
+
+        PalettedContainer<Integer> px = new PalettedContainer<>();
+        px.read(palette, data);
+
+        KList<Integer> palette2 = new KList<>();
+        long[] data2 = px.write(palette);
+
+        if(Arrays.equals(data, data2))
+        {
+            Iris.info("Correct! All data matches!");
+        }
+
+        else
+        {
+            Iris.warn("No match");
+            Iris.error("RAW PALE: " + palette2.toString(","));
+            Iris.error("RAW DATA: " + IO.longsToHex(data2));
+        }
+    }
+}

src/main/java/com/volmit/iris/util/matter/slices/RawMatter.java

@@ -37,7 +37,7 @@ public abstract class RawMatter<T> extends PaletteOrHunk<T> implements MatterSli
     private final Class<T> type;
 
     public RawMatter(int width, int height, int depth, Class<T> type) {
-        super(width, height, depth, true, () -> new MappedHunk<>(width, height, depth));
+        super(width, height, depth, false, this, () -> new MappedHunk<>(width, height, depth));
         writers = new KMap<>();
         readers = new KMap<>();
         this.type = type;

src/main/java/com/volmit/iris/util/nbt/mca/MCAFile.java

@@ -135,7 +135,7 @@ public class MCAFile {
     /**
      * Calls {@link MCAFile#serialize(RandomAccessFile, boolean)} without updating any timestamps.
      *
-     * @param raf The {@code RandomAccessFile} to write to.
+     * @param raf The {@code RandomAccessFile} to writeNodeData to.
      * @return The amount of chunks written to the file.
      * @throws IOException If something went wrong during serialization.
      * @see MCAFile#serialize(RandomAccessFile, boolean)
@@ -148,7 +148,7 @@ public class MCAFile {
      * Serializes this object to an .mca file.
      * This method does not perform any cleanups on the data.
      *
-     * @param raf              The {@code RandomAccessFile} to write to.
+     * @param raf              The {@code RandomAccessFile} to writeNodeData to.
      * @param changeLastUpdate Whether it should update all timestamps that show
      *                         when this file was last updated.
      * @return The amount of chunks written to the file.
@@ -190,7 +190,7 @@ public class MCAFile {
                 raf.writeByte(globalOffset & 0xFF);
                 raf.writeByte(sectors);
 
-                // write timestamp
+                // writeNodeData timestamp
                 raf.seek(index * 4L + 4096);
                 raf.writeInt(changeLastUpdate ? timestamp : chunk.getLastMCAUpdate());
 

src/main/java/com/volmit/iris/util/nbt/mca/MCAUtil.java

@@ -30,7 +30,7 @@ import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
 /**
- * Provides main and utility functions to read and write .mca files and
+ * Provides main and utility functions to read and writeNodeData .mca files and
  * to convert block, chunk and region coordinates.
  */
 public final class MCAUtil {
@@ -98,8 +98,8 @@ public final class MCAUtil {
     /**
      * Calls {@link MCAUtil#write(MCAFile, File, boolean)} without changing the timestamps.
      *
-     * @param file    The file to write to.
+     * @param file    The file to writeNodeData to.
-     * @param mcaFile The data of the MCA file to write.
+     * @param mcaFile The data of the MCA file to writeNodeData.
      * @return The amount of chunks written to the file.
      * @throws IOException If something goes wrong during serialization.
      * @see MCAUtil#write(MCAFile, File, boolean)
@@ -111,8 +111,8 @@ public final class MCAUtil {
     /**
      * Calls {@link MCAUtil#write(MCAFile, File, boolean)} without changing the timestamps.
      *
-     * @param file    The file to write to.
+     * @param file    The file to writeNodeData to.
-     * @param mcaFile The data of the MCA file to write.
+     * @param mcaFile The data of the MCA file to writeNodeData.
      * @return The amount of chunks written to the file.
      * @throws IOException If something goes wrong during serialization.
      * @see MCAUtil#write(MCAFile, File, boolean)
@@ -122,8 +122,8 @@ public final class MCAUtil {
     }
 
     /**
-     * @param file             The file to write to.
+     * @param file             The file to writeNodeData to.
-     * @param mcaFile          The data of the MCA file to write.
+     * @param mcaFile          The data of the MCA file to writeNodeData.
      * @param changeLastUpdate Whether to adjust the timestamps of when the file was saved.
      * @return The amount of chunks written to the file.
      * @throws IOException If something goes wrong during serialization.
@@ -139,8 +139,8 @@ public final class MCAUtil {
      * the value set by either loading an already existing MCA file or setting them manually.<br>
      * If the file already exists, it is completely overwritten by the new file (no modification).
      *
-     * @param file             The file to write to.
+     * @param file             The file to writeNodeData to.
-     * @param mcaFile          The data of the MCA file to write.
+     * @param mcaFile          The data of the MCA file to writeNodeData.
      * @param changeLastUpdate Whether to adjust the timestamps of when the file was saved.
      * @return The amount of chunks written to the file.
      * @throws IOException If something goes wrong during serialization.