Not worth,

plugins/Iris/cache/instance

@@ -1 +1 @@
--523376883
+1036166928

src/main/java/com/volmit/iris/Iris.java

@@ -48,6 +48,7 @@ import com.volmit.iris.util.io.InstanceState;
 import com.volmit.iris.util.io.JarScanner;
 import com.volmit.iris.util.math.M;
 import com.volmit.iris.util.math.RNG;
+import com.volmit.iris.util.matter.MatterTest;
 import com.volmit.iris.util.parallel.MultiBurst;
 import com.volmit.iris.util.plugin.IrisService;
 import com.volmit.iris.util.plugin.Metrics;

src/main/java/com/volmit/iris/core/IrisSettings.java

@@ -64,6 +64,7 @@ public class IrisSettings {
     @Data
     public static class IrisSettingsPerformance {
         public boolean trimMantleInStudio = false;
+        public boolean useExperimentalMantleMemoryCompression = false;
         public int mantleKeepAliveSeconds = 20;
         public int maxStreamCacheSize = 750_000;
         public int maxResourceLoaderCacheSize = 1_000;

src/main/java/com/volmit/iris/util/data/palette/CrudeIncrementalIntIdentityHashBiMap.java

@@ -130,6 +130,10 @@ public class CrudeIncrementalIntIdentityHashBiMap<K> implements IdMap<K> {
     private int indexOf(K var0, int var1) {
         int var2;
         for (var2 = var1; var2 < this.keys.length; var2++) {
+            if (this.keys[var2] == null)
+            {
+                return 0;
+            }
             if (this.keys[var2].equals(var0))
                 return var2;
             if (this.keys[var2] == EMPTY_SLOT)

src/main/java/com/volmit/iris/util/data/palette/HashMapPalette.java

@@ -71,6 +71,11 @@ public class HashMapPalette<T> implements Palette<T> {
 
     @Override
     public void write(List<T> toList) {
-        this.values.iterator().forEachRemaining(toList::add);
+        this.values.iterator().forEachRemaining(i -> {
+            if(i != null)
+            {
+                toList.add(i);
+            }
+        });
     }
 }

src/main/java/com/volmit/iris/util/data/palette/LinearPalette.java

@@ -43,7 +43,7 @@ public class LinearPalette<T> implements Palette<T> {
                 return var1;
             }
 
-            if (this.values[var1].equals(var0))
+            if (this.values[var1] != null && this.values[var1].equals(var0))
             {
                 return var1;
             }

src/main/java/com/volmit/iris/util/data/palette/PalettedContainer.java

@@ -34,7 +34,6 @@ public class PalettedContainer<T> implements PaletteResize<T> {
 
     public PalettedContainer() {
         setBits(4);
-
     }
 
     private static int getIndex(int var0, int var1, int var2) {

src/main/java/com/volmit/iris/util/hunk/storage/PaletteHunk.java

@@ -44,6 +44,44 @@ public class PaletteHunk<T> extends StorageHunk<T> implements Hunk<T> {
         return false;
     }
 
+    @Override
+    public synchronized Hunk<T> iterateSync(Consumer4<Integer, Integer, Integer, T> c) {
+        for(int i = 0; i < getWidth(); i++)
+        {
+            for(int j = 0; j < getHeight(); j++)
+            {
+                for(int k = 0; k < getDepth(); k++)
+                {
+                    T t = getRaw(i,j,k);
+                    if(t != null)
+                    {
+                        c.accept(i,j,k,t);
+                    }
+                }
+            }
+        }
+        return this;
+    }
+
+    @Override
+    public synchronized Hunk<T> iterateSyncIO(Consumer4IO<Integer, Integer, Integer, T> c) throws IOException {
+        for(int i = 0; i < getWidth(); i++)
+        {
+            for(int j = 0; j < getHeight(); j++)
+            {
+                for(int k = 0; k < getDepth(); k++)
+                {
+                    T t = getRaw(i,j,k);
+                    if(t != null)
+                    {
+                        c.accept(i,j,k,t);
+                    }
+                }
+            }
+        }
+        return this;
+    }
+
     @Override
     public void setRaw(int x, int y, int z, T t) {
         data.set(x,y,z,t);

src/main/java/com/volmit/iris/util/hunk/storage/PaletteOrHunk.java

@@ -18,6 +18,7 @@
 
 package com.volmit.iris.util.hunk.storage;
 
+import com.volmit.iris.util.data.palette.PalettedContainer;
 import com.volmit.iris.util.function.Consumer4;
 import com.volmit.iris.util.function.Consumer4IO;
 import com.volmit.iris.util.hunk.Hunk;
@@ -33,6 +34,23 @@ public class PaletteOrHunk<T> extends StorageHunk<T> implements Hunk<T> {
         hunk = width == 16 && height == 16 && depth == 16 ? new PaletteHunk<>() : factory.get();
     }
 
+    public PalettedContainer<T> palette()
+    {
+        return isPalette() ? ((PaletteHunk<T>)hunk).getData() : null;
+    }
+
+    public void palette(PalettedContainer<T> t)
+    {
+        if(isPalette()){
+            ((PaletteHunk<T>)hunk).setData(t);
+        }
+    }
+
+    public boolean isPalette()
+    {
+        return getWidth() == 16 && getHeight() == 16 && getDepth() == 16;
+    }
+
     @Override
     public void setRaw(int x, int y, int z, T t) {
         hunk.setRaw(x,y,z,t);

src/main/java/com/volmit/iris/util/matter/MatterSlice.java

@@ -18,11 +18,15 @@
 
 package com.volmit.iris.util.matter;
 
+import com.volmit.iris.Iris;
+import com.volmit.iris.core.IrisSettings;
 import com.volmit.iris.engine.data.cache.Cache;
 import com.volmit.iris.util.data.Varint;
 import com.volmit.iris.util.data.palette.Palette;
 import com.volmit.iris.util.data.palette.PaletteType;
+import com.volmit.iris.util.data.palette.PalettedContainer;
 import com.volmit.iris.util.hunk.Hunk;
+import com.volmit.iris.util.hunk.storage.PaletteOrHunk;
 import org.bukkit.Location;
 import org.bukkit.World;
 import org.bukkit.block.data.BlockData;
@@ -32,6 +36,8 @@ import org.bukkit.util.BlockVector;
 import java.io.DataInputStream;
 import java.io.DataOutputStream;
 import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
 
 public interface MatterSlice<T> extends Hunk<T>, PaletteType<T> {
     Class<T> getType();
@@ -145,9 +151,30 @@ public interface MatterSlice<T> extends Hunk<T>, PaletteType<T> {
     }
 
     default void write(DataOutputStream dos) throws IOException {
+        dos.writeUTF(getType().getCanonicalName());
+       if(IrisSettings.get().getPerformance().isUseExperimentalMantleMemoryCompression() && (this instanceof PaletteOrHunk f && f.isPalette()))
+       {
+           PalettedContainer<T> c = f.palette();
+           List<T> palette = new ArrayList<>();
+           long[] data = c.write(palette);
+
+           Varint.writeUnsignedVarInt(palette.size(), dos);
+           for(T i : palette)
+           {
+               writeNode(i, dos);
+           }
+
+           Varint.writeUnsignedVarInt(data.length, dos);
+           for(long i : data)
+           {
+               dos.writeLong(i);
+           }
+
+           return;
+       }
+
         int w = getWidth();
         int h = getHeight();
-        dos.writeUTF(getType().getCanonicalName());
         MatterPalette<T> palette = new MatterPalette<T>(this);
         iterateSync((x, y, z, b) -> palette.assign(b));
         palette.writePalette(dos);
@@ -165,6 +192,29 @@ public interface MatterSlice<T> extends Hunk<T>, PaletteType<T> {
     }
 
     default void read(DataInputStream din) throws IOException {
+        if(IrisSettings.get().getPerformance().isUseExperimentalMantleMemoryCompression() && (this instanceof PaletteOrHunk f && f.isPalette()))
+        {
+            PalettedContainer<T> c = new PalettedContainer<>();
+            List<T> palette = new ArrayList<>();
+            int ps = Varint.readUnsignedVarInt(din);
+
+            for(int i = 0; i < ps; i++)
+            {
+                palette.add(readNode(din));
+            }
+
+            int ds = Varint.readUnsignedVarInt(din);
+            long[] data = new long[ds];
+            for(int i = 0; i < ds; i++)
+            {
+                data[i] = din.readLong();
+            }
+
+            c.read(palette, data);
+
+            return;
+        }
+
         int w = getWidth();
         int h = getHeight();
         MatterPalette<T> palette = new MatterPalette<T>(this, din);

src/main/java/com/volmit/iris/util/matter/MatterTest.java

@@ -0,0 +1,99 @@
+/*
+ * Iris is a World Generator for Minecraft Bukkit Servers
+ * Copyright (c) 2021 Arcane Arts (Volmit Software)
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.volmit.iris.util.matter;
+
+import com.volmit.iris.Iris;
+import com.volmit.iris.core.IrisSettings;
+import com.volmit.iris.util.collection.KList;
+import com.volmit.iris.util.format.Form;
+import com.volmit.iris.util.io.IO;
+import com.volmit.iris.util.mantle.Mantle;
+import com.volmit.iris.util.mantle.TectonicPlate;
+import com.volmit.iris.util.math.RNG;
+import com.volmit.iris.util.nbt.tag.ByteArrayTag;
+import com.volmit.iris.util.scheduling.J;
+import com.volmit.iris.util.scheduling.PrecisionStopwatch;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.IOException;
+import java.util.List;
+import java.util.concurrent.ThreadLocalRandom;
+
+public class MatterTest {
+    public static long memorySample()
+    {
+        Runtime rt = Runtime.getRuntime();
+        System.gc();
+        System.gc();
+        return rt.totalMemory() - rt.freeMemory();
+    }
+
+    public static void test()
+    {
+        for(Thread i : Thread.getAllStackTraces().keySet())
+        {
+            if(i.getId() != Thread.currentThread().getId())
+            {
+                try {
+                    i.wait(10000);
+                } catch (Throwable ignored) {
+
+                }
+            }
+        }
+
+        System.gc();
+        System.gc();
+        J.sleep(250);
+
+        try {
+
+            double ms = 0;
+            long a = memorySample();
+            PrecisionStopwatch p = PrecisionStopwatch.start();
+            IrisSettings.get().getPerformance().setUseExperimentalMantleMemoryCompression(true);
+            Mantle mantle = new Mantle(new File("mantle-test/legacy"), 256);
+
+            for(int i = 0; i < 512; i++)
+            {
+                for(int j = 0; j < 255; j++)
+                {
+                    for(int k = 0; k < 512; k++)
+                    {
+                        mantle.set(i,j,k,RNG.r.chance(0.5));
+                        mantle.set(i,j,k,RNG.r.chance(0.5)?"a" : "b");
+                    }
+                }
+            }
+
+            ms += p.getMilliseconds();
+            long b = memorySample() - a;
+            Iris.info("Memory: " + Form.memSize(b, 0) + " (" + Form.f(b) + " bytes)");
+           p = PrecisionStopwatch.start();
+           mantle.saveAll();
+            mantle.close();
+            ms+=p.getMilliseconds();
+            Iris.info("Closed, done! took " + Form.duration(ms, 2));
+        } catch (Throwable e) {
+            e.printStackTrace();
+        }
+    }
+}

src/main/java/com/volmit/iris/util/matter/slices/RawMatter.java

@@ -20,6 +20,7 @@ package com.volmit.iris.util.matter.slices;
 
 import com.volmit.iris.util.collection.KMap;
 import com.volmit.iris.util.hunk.storage.MappedHunk;
+import com.volmit.iris.util.hunk.storage.PaletteOrHunk;
 import com.volmit.iris.util.matter.MatterReader;
 import com.volmit.iris.util.matter.MatterSlice;
 import com.volmit.iris.util.matter.MatterWriter;
@@ -29,14 +30,14 @@ import java.io.DataInputStream;
 import java.io.DataOutputStream;
 import java.io.IOException;
 
-public abstract class RawMatter<T> extends MappedHunk<T> implements MatterSlice<T> {
+public abstract class RawMatter<T> extends PaletteOrHunk<T> implements MatterSlice<T> {
     protected final KMap<Class<?>, MatterWriter<?, T>> writers;
     protected final KMap<Class<?>, MatterReader<?, T>> readers;
     @Getter
     private final Class<T> type;
 
     public RawMatter(int width, int height, int depth, Class<T> type) {
-        super(width, height, depth);
+        super(width, height, depth, () -> new MappedHunk<>(width, height, depth));
         writers = new KMap<>();
         readers = new KMap<>();
         this.type = type;