From 7f58e0413cba270231a2cfa678929cd997ac2305 Mon Sep 17 00:00:00 2001 From: cyberpwn Date: Wed, 22 Sep 2021 08:54:38 -0400 Subject: [PATCH] Not worth, --- plugins/Iris/cache/instance | 2 +- src/main/java/com/volmit/iris/Iris.java | 1 + .../com/volmit/iris/core/IrisSettings.java | 1 + .../CrudeIncrementalIntIdentityHashBiMap.java | 4 + .../util/data/palette/HashMapPalette.java | 7 +- .../iris/util/data/palette/LinearPalette.java | 2 +- .../util/data/palette/PalettedContainer.java | 1 - .../iris/util/hunk/storage/PaletteHunk.java | 38 +++++++ .../iris/util/hunk/storage/PaletteOrHunk.java | 18 ++++ .../volmit/iris/util/matter/MatterSlice.java | 52 +++++++++- .../volmit/iris/util/matter/MatterTest.java | 99 +++++++++++++++++++ .../iris/util/matter/slices/RawMatter.java | 5 +- 12 files changed, 223 insertions(+), 7 deletions(-) create mode 100644 src/main/java/com/volmit/iris/util/matter/MatterTest.java diff --git a/plugins/Iris/cache/instance b/plugins/Iris/cache/instance index 525366c54..4a40e9617 100644 --- a/plugins/Iris/cache/instance +++ b/plugins/Iris/cache/instance @@ -1 +1 @@ --523376883 +1036166928 diff --git a/src/main/java/com/volmit/iris/Iris.java b/src/main/java/com/volmit/iris/Iris.java index 78dfbf506..3de7ab2b9 100644 --- a/src/main/java/com/volmit/iris/Iris.java +++ b/src/main/java/com/volmit/iris/Iris.java @@ -48,6 +48,7 @@ import com.volmit.iris.util.io.InstanceState; import com.volmit.iris.util.io.JarScanner; import com.volmit.iris.util.math.M; import com.volmit.iris.util.math.RNG; +import com.volmit.iris.util.matter.MatterTest; import com.volmit.iris.util.parallel.MultiBurst; import com.volmit.iris.util.plugin.IrisService; import com.volmit.iris.util.plugin.Metrics; diff --git a/src/main/java/com/volmit/iris/core/IrisSettings.java b/src/main/java/com/volmit/iris/core/IrisSettings.java index 67ec21d68..42a6da891 100644 --- a/src/main/java/com/volmit/iris/core/IrisSettings.java +++ b/src/main/java/com/volmit/iris/core/IrisSettings.java @@ -64,6 +64,7 @@ public class IrisSettings { @Data public static class IrisSettingsPerformance { public boolean trimMantleInStudio = false; + public boolean useExperimentalMantleMemoryCompression = false; public int mantleKeepAliveSeconds = 20; public int maxStreamCacheSize = 750_000; public int maxResourceLoaderCacheSize = 1_000; diff --git a/src/main/java/com/volmit/iris/util/data/palette/CrudeIncrementalIntIdentityHashBiMap.java b/src/main/java/com/volmit/iris/util/data/palette/CrudeIncrementalIntIdentityHashBiMap.java index bdf0dfe02..7b6570d7f 100644 --- a/src/main/java/com/volmit/iris/util/data/palette/CrudeIncrementalIntIdentityHashBiMap.java +++ b/src/main/java/com/volmit/iris/util/data/palette/CrudeIncrementalIntIdentityHashBiMap.java @@ -130,6 +130,10 @@ public class CrudeIncrementalIntIdentityHashBiMap implements IdMap { private int indexOf(K var0, int var1) { int var2; for (var2 = var1; var2 < this.keys.length; var2++) { + if (this.keys[var2] == null) + { + return 0; + } if (this.keys[var2].equals(var0)) return var2; if (this.keys[var2] == EMPTY_SLOT) diff --git a/src/main/java/com/volmit/iris/util/data/palette/HashMapPalette.java b/src/main/java/com/volmit/iris/util/data/palette/HashMapPalette.java index a576e9004..066dc2c94 100644 --- a/src/main/java/com/volmit/iris/util/data/palette/HashMapPalette.java +++ b/src/main/java/com/volmit/iris/util/data/palette/HashMapPalette.java @@ -71,6 +71,11 @@ public class HashMapPalette implements Palette { @Override public void write(List toList) { - this.values.iterator().forEachRemaining(toList::add); + this.values.iterator().forEachRemaining(i -> { + if(i != null) + { + toList.add(i); + } + }); } } \ No newline at end of file diff --git a/src/main/java/com/volmit/iris/util/data/palette/LinearPalette.java b/src/main/java/com/volmit/iris/util/data/palette/LinearPalette.java index 54c89f169..6b2170e71 100644 --- a/src/main/java/com/volmit/iris/util/data/palette/LinearPalette.java +++ b/src/main/java/com/volmit/iris/util/data/palette/LinearPalette.java @@ -43,7 +43,7 @@ public class LinearPalette implements Palette { return var1; } - if (this.values[var1].equals(var0)) + if (this.values[var1] != null && this.values[var1].equals(var0)) { return var1; } diff --git a/src/main/java/com/volmit/iris/util/data/palette/PalettedContainer.java b/src/main/java/com/volmit/iris/util/data/palette/PalettedContainer.java index e2feb6a03..81d81f4f5 100644 --- a/src/main/java/com/volmit/iris/util/data/palette/PalettedContainer.java +++ b/src/main/java/com/volmit/iris/util/data/palette/PalettedContainer.java @@ -34,7 +34,6 @@ public class PalettedContainer implements PaletteResize { public PalettedContainer() { setBits(4); - } private static int getIndex(int var0, int var1, int var2) { diff --git a/src/main/java/com/volmit/iris/util/hunk/storage/PaletteHunk.java b/src/main/java/com/volmit/iris/util/hunk/storage/PaletteHunk.java index 771638131..fcdd8a131 100644 --- a/src/main/java/com/volmit/iris/util/hunk/storage/PaletteHunk.java +++ b/src/main/java/com/volmit/iris/util/hunk/storage/PaletteHunk.java @@ -44,6 +44,44 @@ public class PaletteHunk extends StorageHunk implements Hunk { return false; } + @Override + public synchronized Hunk iterateSync(Consumer4 c) { + for(int i = 0; i < getWidth(); i++) + { + for(int j = 0; j < getHeight(); j++) + { + for(int k = 0; k < getDepth(); k++) + { + T t = getRaw(i,j,k); + if(t != null) + { + c.accept(i,j,k,t); + } + } + } + } + return this; + } + + @Override + public synchronized Hunk iterateSyncIO(Consumer4IO c) throws IOException { + for(int i = 0; i < getWidth(); i++) + { + for(int j = 0; j < getHeight(); j++) + { + for(int k = 0; k < getDepth(); k++) + { + T t = getRaw(i,j,k); + if(t != null) + { + c.accept(i,j,k,t); + } + } + } + } + return this; + } + @Override public void setRaw(int x, int y, int z, T t) { data.set(x,y,z,t); diff --git a/src/main/java/com/volmit/iris/util/hunk/storage/PaletteOrHunk.java b/src/main/java/com/volmit/iris/util/hunk/storage/PaletteOrHunk.java index 955d0c109..867f3f3b7 100644 --- a/src/main/java/com/volmit/iris/util/hunk/storage/PaletteOrHunk.java +++ b/src/main/java/com/volmit/iris/util/hunk/storage/PaletteOrHunk.java @@ -18,6 +18,7 @@ package com.volmit.iris.util.hunk.storage; +import com.volmit.iris.util.data.palette.PalettedContainer; import com.volmit.iris.util.function.Consumer4; import com.volmit.iris.util.function.Consumer4IO; import com.volmit.iris.util.hunk.Hunk; @@ -33,6 +34,23 @@ public class PaletteOrHunk extends StorageHunk implements Hunk { hunk = width == 16 && height == 16 && depth == 16 ? new PaletteHunk<>() : factory.get(); } + public PalettedContainer palette() + { + return isPalette() ? ((PaletteHunk)hunk).getData() : null; + } + + public void palette(PalettedContainer t) + { + if(isPalette()){ + ((PaletteHunk)hunk).setData(t); + } + } + + public boolean isPalette() + { + return getWidth() == 16 && getHeight() == 16 && getDepth() == 16; + } + @Override public void setRaw(int x, int y, int z, T t) { hunk.setRaw(x,y,z,t); diff --git a/src/main/java/com/volmit/iris/util/matter/MatterSlice.java b/src/main/java/com/volmit/iris/util/matter/MatterSlice.java index 5cba77059..c35797b0c 100644 --- a/src/main/java/com/volmit/iris/util/matter/MatterSlice.java +++ b/src/main/java/com/volmit/iris/util/matter/MatterSlice.java @@ -18,11 +18,15 @@ package com.volmit.iris.util.matter; +import com.volmit.iris.Iris; +import com.volmit.iris.core.IrisSettings; import com.volmit.iris.engine.data.cache.Cache; import com.volmit.iris.util.data.Varint; import com.volmit.iris.util.data.palette.Palette; import com.volmit.iris.util.data.palette.PaletteType; +import com.volmit.iris.util.data.palette.PalettedContainer; import com.volmit.iris.util.hunk.Hunk; +import com.volmit.iris.util.hunk.storage.PaletteOrHunk; import org.bukkit.Location; import org.bukkit.World; import org.bukkit.block.data.BlockData; @@ -32,6 +36,8 @@ import org.bukkit.util.BlockVector; import java.io.DataInputStream; import java.io.DataOutputStream; import java.io.IOException; +import java.util.ArrayList; +import java.util.List; public interface MatterSlice extends Hunk, PaletteType { Class getType(); @@ -145,9 +151,30 @@ public interface MatterSlice extends Hunk, PaletteType { } default void write(DataOutputStream dos) throws IOException { + dos.writeUTF(getType().getCanonicalName()); + if(IrisSettings.get().getPerformance().isUseExperimentalMantleMemoryCompression() && (this instanceof PaletteOrHunk f && f.isPalette())) + { + PalettedContainer c = f.palette(); + List palette = new ArrayList<>(); + long[] data = c.write(palette); + + Varint.writeUnsignedVarInt(palette.size(), dos); + for(T i : palette) + { + writeNode(i, dos); + } + + Varint.writeUnsignedVarInt(data.length, dos); + for(long i : data) + { + dos.writeLong(i); + } + + return; + } + int w = getWidth(); int h = getHeight(); - dos.writeUTF(getType().getCanonicalName()); MatterPalette palette = new MatterPalette(this); iterateSync((x, y, z, b) -> palette.assign(b)); palette.writePalette(dos); @@ -165,6 +192,29 @@ public interface MatterSlice extends Hunk, PaletteType { } default void read(DataInputStream din) throws IOException { + if(IrisSettings.get().getPerformance().isUseExperimentalMantleMemoryCompression() && (this instanceof PaletteOrHunk f && f.isPalette())) + { + PalettedContainer c = new PalettedContainer<>(); + List palette = new ArrayList<>(); + int ps = Varint.readUnsignedVarInt(din); + + for(int i = 0; i < ps; i++) + { + palette.add(readNode(din)); + } + + int ds = Varint.readUnsignedVarInt(din); + long[] data = new long[ds]; + for(int i = 0; i < ds; i++) + { + data[i] = din.readLong(); + } + + c.read(palette, data); + + return; + } + int w = getWidth(); int h = getHeight(); MatterPalette palette = new MatterPalette(this, din); diff --git a/src/main/java/com/volmit/iris/util/matter/MatterTest.java b/src/main/java/com/volmit/iris/util/matter/MatterTest.java new file mode 100644 index 000000000..1cc0c2e32 --- /dev/null +++ b/src/main/java/com/volmit/iris/util/matter/MatterTest.java @@ -0,0 +1,99 @@ +/* + * Iris is a World Generator for Minecraft Bukkit Servers + * Copyright (c) 2021 Arcane Arts (Volmit Software) + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package com.volmit.iris.util.matter; + +import com.volmit.iris.Iris; +import com.volmit.iris.core.IrisSettings; +import com.volmit.iris.util.collection.KList; +import com.volmit.iris.util.format.Form; +import com.volmit.iris.util.io.IO; +import com.volmit.iris.util.mantle.Mantle; +import com.volmit.iris.util.mantle.TectonicPlate; +import com.volmit.iris.util.math.RNG; +import com.volmit.iris.util.nbt.tag.ByteArrayTag; +import com.volmit.iris.util.scheduling.J; +import com.volmit.iris.util.scheduling.PrecisionStopwatch; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.IOException; +import java.util.List; +import java.util.concurrent.ThreadLocalRandom; + +public class MatterTest { + public static long memorySample() + { + Runtime rt = Runtime.getRuntime(); + System.gc(); + System.gc(); + return rt.totalMemory() - rt.freeMemory(); + } + + public static void test() + { + for(Thread i : Thread.getAllStackTraces().keySet()) + { + if(i.getId() != Thread.currentThread().getId()) + { + try { + i.wait(10000); + } catch (Throwable ignored) { + + } + } + } + + System.gc(); + System.gc(); + J.sleep(250); + + try { + + double ms = 0; + long a = memorySample(); + PrecisionStopwatch p = PrecisionStopwatch.start(); + IrisSettings.get().getPerformance().setUseExperimentalMantleMemoryCompression(true); + Mantle mantle = new Mantle(new File("mantle-test/legacy"), 256); + + for(int i = 0; i < 512; i++) + { + for(int j = 0; j < 255; j++) + { + for(int k = 0; k < 512; k++) + { + mantle.set(i,j,k,RNG.r.chance(0.5)); + mantle.set(i,j,k,RNG.r.chance(0.5)?"a" : "b"); + } + } + } + + ms += p.getMilliseconds(); + long b = memorySample() - a; + Iris.info("Memory: " + Form.memSize(b, 0) + " (" + Form.f(b) + " bytes)"); + p = PrecisionStopwatch.start(); + mantle.saveAll(); + mantle.close(); + ms+=p.getMilliseconds(); + Iris.info("Closed, done! took " + Form.duration(ms, 2)); + } catch (Throwable e) { + e.printStackTrace(); + } + } +} diff --git a/src/main/java/com/volmit/iris/util/matter/slices/RawMatter.java b/src/main/java/com/volmit/iris/util/matter/slices/RawMatter.java index d2215217a..78e8433e8 100644 --- a/src/main/java/com/volmit/iris/util/matter/slices/RawMatter.java +++ b/src/main/java/com/volmit/iris/util/matter/slices/RawMatter.java @@ -20,6 +20,7 @@ package com.volmit.iris.util.matter.slices; import com.volmit.iris.util.collection.KMap; import com.volmit.iris.util.hunk.storage.MappedHunk; +import com.volmit.iris.util.hunk.storage.PaletteOrHunk; import com.volmit.iris.util.matter.MatterReader; import com.volmit.iris.util.matter.MatterSlice; import com.volmit.iris.util.matter.MatterWriter; @@ -29,14 +30,14 @@ import java.io.DataInputStream; import java.io.DataOutputStream; import java.io.IOException; -public abstract class RawMatter extends MappedHunk implements MatterSlice { +public abstract class RawMatter extends PaletteOrHunk implements MatterSlice { protected final KMap, MatterWriter> writers; protected final KMap, MatterReader> readers; @Getter private final Class type; public RawMatter(int width, int height, int depth, Class type) { - super(width, height, depth); + super(width, height, depth, () -> new MappedHunk<>(width, height, depth)); writers = new KMap<>(); readers = new KMap<>(); this.type = type;