From 779a56c3a14d90c37a1d79609f146dbad2191950 Mon Sep 17 00:00:00 2001 From: CrazyDev22 Date: Fri, 22 Dec 2023 15:50:27 +0100 Subject: [PATCH 1/5] add compression test command --- .../iris/core/commands/CommandDeveloper.java | 104 +++++++++++++++++- 1 file changed, 99 insertions(+), 5 deletions(-) diff --git a/core/src/main/java/com/volmit/iris/core/commands/CommandDeveloper.java b/core/src/main/java/com/volmit/iris/core/commands/CommandDeveloper.java index 2d2286fa6..969d74ade 100644 --- a/core/src/main/java/com/volmit/iris/core/commands/CommandDeveloper.java +++ b/core/src/main/java/com/volmit/iris/core/commands/CommandDeveloper.java @@ -19,23 +19,33 @@ package com.volmit.iris.core.commands; import com.volmit.iris.Iris; -import com.volmit.iris.core.IrisSettings; import com.volmit.iris.core.loader.IrisData; import com.volmit.iris.core.service.IrisEngineSVC; import com.volmit.iris.core.tools.IrisToolbelt; import com.volmit.iris.engine.framework.Engine; -import com.volmit.iris.engine.mantle.EngineMantle; import com.volmit.iris.util.decree.DecreeExecutor; import com.volmit.iris.util.decree.DecreeOrigin; import com.volmit.iris.util.decree.annotations.Decree; import com.volmit.iris.util.decree.annotations.Param; import com.volmit.iris.util.format.C; import com.volmit.iris.util.format.Form; -import com.volmit.iris.util.mantle.Mantle; +import com.volmit.iris.util.io.IO; +import com.volmit.iris.util.mantle.TectonicPlate; +import com.volmit.iris.util.plugin.VolmitSender; +import net.jpountz.lz4.LZ4BlockInputStream; +import net.jpountz.lz4.LZ4BlockOutputStream; +import net.jpountz.lz4.LZ4FrameInputStream; +import net.jpountz.lz4.LZ4FrameOutputStream; +import org.apache.commons.lang.RandomStringUtils; import org.bukkit.Bukkit; import org.bukkit.World; -import java.util.concurrent.atomic.AtomicInteger; +import java.io.*; +import java.util.concurrent.*; +import java.util.zip.GZIPInputStream; +import java.util.zip.GZIPOutputStream; +import java.util.zip.ZipInputStream; +import java.util.zip.ZipOutputStream; @Decree(name = "Developer", origin = DecreeOrigin.BOTH, description = "Iris World Manager", aliases = {"dev"}) public class CommandDeveloper implements DecreeExecutor { @@ -70,9 +80,93 @@ public class CommandDeveloper implements DecreeExecutor { } } @Decree(description = "Test", origin = DecreeOrigin.BOTH) - public void test(){ + public void test() { Iris.info("Test Developer CMD Executed"); } + + @Decree(description = "Test the compression algorithms") + public void compression( + @Param(description = "World") World world, + @Param(description = "File") String path, + @Param(description = "Algorithm") String algorithm, + @Param(description = "Amount of Tests") int amount) { + if (!IrisToolbelt.isIrisWorld(world)) { + sender().sendMessage(C.RED + "This is not an Iris world. Iris worlds: " + String.join(", ", Bukkit.getServer().getWorlds().stream().filter(IrisToolbelt::isIrisWorld).map(World::getName).toList())); + return; + } + + File file = new File(path); + if (!file.exists()) return; + + Engine engine = IrisToolbelt.access(world).getEngine(); + if(engine != null) { + int height = engine.getTarget().getHeight(); + ExecutorService service = Executors.newFixedThreadPool(1); + VolmitSender sender = sender(); + service.submit(() -> { + try { + DataInputStream raw = new DataInputStream(new FileInputStream(file)); + TectonicPlate plate = new TectonicPlate(height, raw); + raw.close(); + + double d1 = 0; + double d2 = 0; + long size = 0; + File folder = new File("tmp"); + folder.mkdirs(); + for (int i = 0; i < amount; i++) { + File tmp = new File(folder, RandomStringUtils.randomAlphanumeric(10) + "." + algorithm + ".bin"); + DataOutputStream dos = createOutput(tmp, algorithm); + long start = System.currentTimeMillis(); + plate.write(dos); + dos.close(); + d1 += System.currentTimeMillis() - start; + if (size == 0) + size = tmp.length(); + start = System.currentTimeMillis(); + DataInputStream din = createInput(tmp, algorithm); + new TectonicPlate(height, din); + din.close(); + d2 += System.currentTimeMillis() - start; + tmp.delete(); + } + IO.delete(folder); + sender.sendMessage(algorithm + " is " + Form.fileSize(size) + " big after compression"); + sender.sendMessage(algorithm + " Took " + d2/amount + "ms to read"); + sender.sendMessage(algorithm + " Took " + d1/amount + "ms to write"); + } catch (Throwable e) { + e.printStackTrace(); + } + }); + service.shutdown(); + } else { + Iris.info(C.RED + "Engine is null!"); + } + } + + private DataInputStream createInput(File file, String algorithm) throws Throwable { + FileInputStream in = new FileInputStream(file); + + return new DataInputStream(switch (algorithm) { + case "gzip" -> new GZIPInputStream(in); + case "zip" -> new ZipInputStream(in); + case "lz4f" -> new LZ4FrameInputStream(in); + case "lz4b" -> new LZ4BlockInputStream(in); + default -> throw new IllegalStateException("Unexpected value: " + algorithm); + }); + } + + private DataOutputStream createOutput(File file, String algorithm) throws Throwable { + FileOutputStream out = new FileOutputStream(file); + + return new DataOutputStream(switch (algorithm) { + case "gzip" -> new GZIPOutputStream(out); + case "zip" -> new ZipOutputStream(out); + case "lz4f" -> new LZ4FrameOutputStream(out); + case "lz4b" -> new LZ4BlockOutputStream(out); + default -> throw new IllegalStateException("Unexpected value: " + algorithm); + }); + } } From 4f888f16bcd9e492c5fb7b38d04607a91be98a0d Mon Sep 17 00:00:00 2001 From: CrazyDev22 Date: Fri, 22 Dec 2023 16:13:46 +0100 Subject: [PATCH 2/5] Add Compression Test Command --- .../com/volmit/iris/core/commands/CommandDeveloper.java | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/core/src/main/java/com/volmit/iris/core/commands/CommandDeveloper.java b/core/src/main/java/com/volmit/iris/core/commands/CommandDeveloper.java index 969d74ade..54450af8d 100644 --- a/core/src/main/java/com/volmit/iris/core/commands/CommandDeveloper.java +++ b/core/src/main/java/com/volmit/iris/core/commands/CommandDeveloper.java @@ -41,11 +41,10 @@ import org.bukkit.Bukkit; import org.bukkit.World; import java.io.*; -import java.util.concurrent.*; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; import java.util.zip.GZIPInputStream; import java.util.zip.GZIPOutputStream; -import java.util.zip.ZipInputStream; -import java.util.zip.ZipOutputStream; @Decree(name = "Developer", origin = DecreeOrigin.BOTH, description = "Iris World Manager", aliases = {"dev"}) public class CommandDeveloper implements DecreeExecutor { @@ -149,7 +148,6 @@ public class CommandDeveloper implements DecreeExecutor { return new DataInputStream(switch (algorithm) { case "gzip" -> new GZIPInputStream(in); - case "zip" -> new ZipInputStream(in); case "lz4f" -> new LZ4FrameInputStream(in); case "lz4b" -> new LZ4BlockInputStream(in); default -> throw new IllegalStateException("Unexpected value: " + algorithm); @@ -161,7 +159,6 @@ public class CommandDeveloper implements DecreeExecutor { return new DataOutputStream(switch (algorithm) { case "gzip" -> new GZIPOutputStream(out); - case "zip" -> new ZipOutputStream(out); case "lz4f" -> new LZ4FrameOutputStream(out); case "lz4b" -> new LZ4BlockOutputStream(out); default -> throw new IllegalStateException("Unexpected value: " + algorithm); From a28df78792997d9be1c508f9adc784ed224f9859 Mon Sep 17 00:00:00 2001 From: CrazyDev22 Date: Fri, 22 Dec 2023 16:13:57 +0100 Subject: [PATCH 3/5] switch to lz4 block compression --- core/src/main/java/com/volmit/iris/util/mantle/Mantle.java | 4 ++-- .../main/java/com/volmit/iris/util/mantle/TectonicPlate.java | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/core/src/main/java/com/volmit/iris/util/mantle/Mantle.java b/core/src/main/java/com/volmit/iris/util/mantle/Mantle.java index 011e9b47e..7b101cd2f 100644 --- a/core/src/main/java/com/volmit/iris/util/mantle/Mantle.java +++ b/core/src/main/java/com/volmit/iris/util/mantle/Mantle.java @@ -110,7 +110,7 @@ public class Mantle { * @return the file */ public static File fileForRegion(File folder, Long key) { - File f = new File(folder, "p." + key + ".ttp.lz4"); + File f = new File(folder, "p." + key + ".ttp.lz4b"); if (!f.getParentFile().exists()) { f.getParentFile().mkdirs(); } @@ -552,7 +552,7 @@ public class Mantle { File file = fileForRegion(dataFolder, x, z); if (!file.exists()) - file = new File(dataFolder, file.getName().substring(".lz4".length())); + file = new File(dataFolder, file.getName().substring(".lz4b".length())); if (file.exists()) { try { diff --git a/core/src/main/java/com/volmit/iris/util/mantle/TectonicPlate.java b/core/src/main/java/com/volmit/iris/util/mantle/TectonicPlate.java index a93cb9c4d..87d049b36 100644 --- a/core/src/main/java/com/volmit/iris/util/mantle/TectonicPlate.java +++ b/core/src/main/java/com/volmit/iris/util/mantle/TectonicPlate.java @@ -27,6 +27,7 @@ import com.volmit.iris.util.format.Form; import com.volmit.iris.util.scheduling.PrecisionStopwatch; import lombok.Getter; import net.jpountz.lz4.LZ4BlockInputStream; +import net.jpountz.lz4.LZ4BlockOutputStream; import net.jpountz.lz4.LZ4FrameInputStream; import net.jpountz.lz4.LZ4FrameOutputStream; @@ -87,7 +88,7 @@ public class TectonicPlate { GZIPInputStream gzi = new GZIPInputStream(fin); din = new DataInputStream(gzi); } else { - LZ4FrameInputStream lz4 = new LZ4FrameInputStream(fin); + LZ4BlockInputStream lz4 = new LZ4BlockInputStream(fin); din = new DataInputStream(lz4); } TectonicPlate p = new TectonicPlate(worldHeight, din); @@ -178,7 +179,7 @@ public class TectonicPlate { GZIPOutputStream gzo = new GZIPOutputStream(fos); dos = new DataOutputStream(gzo); } else { - LZ4FrameOutputStream lz4 = new LZ4FrameOutputStream(fos); + LZ4BlockOutputStream lz4 = new LZ4BlockOutputStream(fos); dos = new DataOutputStream(lz4); } write(dos); From 3f3f947f43962f8e3555e793eb4c6db9c56ac8fb Mon Sep 17 00:00:00 2001 From: CrazyDev22 Date: Fri, 22 Dec 2023 16:17:56 +0100 Subject: [PATCH 4/5] change the Param description --- .../com/volmit/iris/core/commands/CommandDeveloper.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/com/volmit/iris/core/commands/CommandDeveloper.java b/core/src/main/java/com/volmit/iris/core/commands/CommandDeveloper.java index 54450af8d..85e6f1644 100644 --- a/core/src/main/java/com/volmit/iris/core/commands/CommandDeveloper.java +++ b/core/src/main/java/com/volmit/iris/core/commands/CommandDeveloper.java @@ -85,9 +85,9 @@ public class CommandDeveloper implements DecreeExecutor { @Decree(description = "Test the compression algorithms") public void compression( - @Param(description = "World") World world, - @Param(description = "File") String path, - @Param(description = "Algorithm") String algorithm, + @Param(description = "base IrisWorld") World world, + @Param(description = "raw TectonicPlate File") String path, + @Param(description = "Algorithm to Test") String algorithm, @Param(description = "Amount of Tests") int amount) { if (!IrisToolbelt.isIrisWorld(world)) { sender().sendMessage(C.RED + "This is not an Iris world. Iris worlds: " + String.join(", ", Bukkit.getServer().getWorlds().stream().filter(IrisToolbelt::isIrisWorld).map(World::getName).toList())); From ee4eb7b3f064b68ea25de31409e55eca63ec6772 Mon Sep 17 00:00:00 2001 From: CrazyDev22 Date: Fri, 22 Dec 2023 16:34:39 +0100 Subject: [PATCH 5/5] fix --- .../main/java/com/volmit/iris/util/mantle/TectonicPlate.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/java/com/volmit/iris/util/mantle/TectonicPlate.java b/core/src/main/java/com/volmit/iris/util/mantle/TectonicPlate.java index f58380445..b8e854c7d 100644 --- a/core/src/main/java/com/volmit/iris/util/mantle/TectonicPlate.java +++ b/core/src/main/java/com/volmit/iris/util/mantle/TectonicPlate.java @@ -175,7 +175,7 @@ public class TectonicPlate { PrecisionStopwatch p = PrecisionStopwatch.start(); FileOutputStream fos = new FileOutputStream(file); DataOutputStream dos; - if (file.getName().endsWith("ttp.lz4")) { + if (file.getName().endsWith("ttp")) { GZIPOutputStream gzo = new GZIPOutputStream(fos); dos = new DataOutputStream(gzo); } else {