From fb0bc112e366294309a0850bf2a5a39fef360298 Mon Sep 17 00:00:00 2001 From: Julian Krings Date: Sun, 30 Mar 2025 14:31:11 +0200 Subject: [PATCH 1/3] increase worker threads on paper servers during pregen --- .../com/volmit/iris/core/IrisSettings.java | 6 ++- .../methods/AsyncPregenMethod.java | 45 +++++++++++++++++++ .../iris/core/tools/IrisPackBenchmarking.java | 41 ++--------------- 3 files changed, 54 insertions(+), 38 deletions(-) diff --git a/core/src/main/java/com/volmit/iris/core/IrisSettings.java b/core/src/main/java/com/volmit/iris/core/IrisSettings.java index 5381a7e39..eb1cb1d11 100644 --- a/core/src/main/java/com/volmit/iris/core/IrisSettings.java +++ b/core/src/main/java/com/volmit/iris/core/IrisSettings.java @@ -24,7 +24,6 @@ import com.volmit.iris.util.io.IO; import com.volmit.iris.util.json.JSONException; import com.volmit.iris.util.json.JSONObject; import com.volmit.iris.util.plugin.VolmitSender; -import com.volmit.iris.util.scheduling.ChronoLatch; import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; @@ -135,6 +134,11 @@ public class IrisSettings { @Data public static class IrisSettingsConcurrency { public int parallelism = -1; + public int worldGenParallelism = -1; + + public int getWorldGenThreads() { + return getThreadCount(worldGenParallelism); + } } @Data diff --git a/core/src/main/java/com/volmit/iris/core/pregenerator/methods/AsyncPregenMethod.java b/core/src/main/java/com/volmit/iris/core/pregenerator/methods/AsyncPregenMethod.java index 8496f924c..16c2fd8c1 100644 --- a/core/src/main/java/com/volmit/iris/core/pregenerator/methods/AsyncPregenMethod.java +++ b/core/src/main/java/com/volmit/iris/core/pregenerator/methods/AsyncPregenMethod.java @@ -19,6 +19,7 @@ package com.volmit.iris.core.pregenerator.methods; import com.volmit.iris.Iris; +import com.volmit.iris.core.IrisSettings; import com.volmit.iris.core.pregenerator.PregenListener; import com.volmit.iris.core.pregenerator.PregeneratorMethod; import com.volmit.iris.core.tools.IrisToolbelt; @@ -34,8 +35,10 @@ import org.bukkit.World; import java.util.ArrayList; import java.util.Map; import java.util.concurrent.Semaphore; +import java.util.concurrent.atomic.AtomicInteger; public class AsyncPregenMethod implements PregeneratorMethod { + private static final AtomicInteger THREAD_COUNT = new AtomicInteger(); private final World world; private final MultiBurst burst; private final Semaphore semaphore; @@ -92,6 +95,7 @@ public class AsyncPregenMethod implements PregeneratorMethod { @Override public void init() { unloadAndSaveAllChunks(); + increaseWorkerThreads(); } @Override @@ -104,6 +108,7 @@ public class AsyncPregenMethod implements PregeneratorMethod { semaphore.acquireUninterruptibly(256); unloadAndSaveAllChunks(); burst.close(); + resetWorkerThreads(); } @Override @@ -140,4 +145,44 @@ public class AsyncPregenMethod implements PregeneratorMethod { return null; } + + + public static void increaseWorkerThreads() { + THREAD_COUNT.updateAndGet(i -> { + if (i > 0) return 1; + try { + var field = Class.forName("ca.spottedleaf.moonrise.common.util.MoonriseCommon").getDeclaredField("WORKER_POOL"); + var pool = field.get(null); + var threads = ((Thread[]) pool.getClass().getDeclaredMethod("getCoreThreads").invoke(pool)).length; + var adjusted = IrisSettings.get().getConcurrency().getWorldGenThreads(); + if (threads >= adjusted) return 0; + + pool.getClass().getDeclaredMethod("adjustThreadCount", int.class).invoke(pool, adjusted); + return threads; + } catch (ClassNotFoundException ignored) { + } catch (Throwable e) { + Iris.error("Failed to increase worker threads"); + e.printStackTrace(); + } + return 0; + }); + } + + public static void resetWorkerThreads() { + THREAD_COUNT.updateAndGet(i -> { + if (i == 0) return 0; + try { + var field = Class.forName("ca.spottedleaf.moonrise.common.util.MoonriseCommon").getDeclaredField("WORKER_POOL"); + var pool = field.get(null); + var method = pool.getClass().getDeclaredMethod("adjustThreadCount", int.class); + method.invoke(pool, i); + return 0; + } catch (ClassNotFoundException ignored) { + } catch (Throwable e) { + Iris.error("Failed to reset worker threads"); + e.printStackTrace(); + } + return i; + }); + } } diff --git a/core/src/main/java/com/volmit/iris/core/tools/IrisPackBenchmarking.java b/core/src/main/java/com/volmit/iris/core/tools/IrisPackBenchmarking.java index 8667542a9..d9c6b0bd0 100644 --- a/core/src/main/java/com/volmit/iris/core/tools/IrisPackBenchmarking.java +++ b/core/src/main/java/com/volmit/iris/core/tools/IrisPackBenchmarking.java @@ -9,6 +9,7 @@ import com.volmit.iris.util.collection.KList; import com.volmit.iris.util.collection.KMap; import com.volmit.iris.util.exceptions.IrisException; import com.volmit.iris.util.format.Form; +import com.volmit.iris.util.io.IO; import com.volmit.iris.util.scheduling.J; import com.volmit.iris.util.scheduling.PrecisionStopwatch; import lombok.Getter; @@ -17,11 +18,6 @@ import org.bukkit.Bukkit; import java.io.File; import java.io.FileWriter; import java.io.IOException; -import java.nio.file.FileVisitResult; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.SimpleFileVisitor; -import java.nio.file.attribute.BasicFileAttributes; import java.time.Clock; import java.time.LocalDateTime; import java.util.Collections; @@ -50,10 +46,7 @@ public class IrisPackBenchmarking { .start(() -> { Iris.info("Setting up benchmark environment "); benchmarkInProgress = true; - File file = new File("benchmark"); - if (file.exists()) { - deleteDirectory(file.toPath()); - } + IO.delete(new File(Bukkit.getWorldContainer(), "benchmark")); createBenchmark(); while (!IrisToolbelt.isIrisWorld(Bukkit.getWorld("benchmark"))) { J.sleep(1000); @@ -72,7 +65,7 @@ public class IrisPackBenchmarking { public void finishedBenchmark(KList cps) { try { - String time = Form.duration(stopwatch.getMillis()); + String time = Form.duration((long) stopwatch.getMilliseconds()); Engine engine = IrisToolbelt.access(Bukkit.getWorld("benchmark")).getEngine(); Iris.info("-----------------"); Iris.info("Results:"); @@ -83,11 +76,7 @@ public class IrisPackBenchmarking { Iris.info(" - Lowest CPS: " + findLowest(cps)); Iris.info("-----------------"); Iris.info("Creating a report.."); - File profilers = new File("plugins" + File.separator + "Iris" + File.separator + "packbenchmarks"); - profilers.mkdir(); - - File results = new File(profilers, dimension.getName() + " " + LocalDateTime.now(Clock.systemDefaultZone()).toString().replace(':', '-') + ".txt"); - results.getParentFile().mkdirs(); + File results = Iris.instance.getDataFile("packbenchmarks", dimension.getName() + " " + LocalDateTime.now(Clock.systemDefaultZone()).toString().replace(':', '-') + ".txt"); KMap metrics = engine.getMetrics().pull(); try (FileWriter writer = new FileWriter(results)) { writer.write("-----------------\n"); @@ -178,26 +167,4 @@ public class IrisPackBenchmarking { private int findHighest(KList list) { return Collections.max(list); } - - private boolean deleteDirectory(Path dir) { - try { - Files.walkFileTree(dir, new SimpleFileVisitor<>() { - @Override - public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { - Files.delete(file); - return FileVisitResult.CONTINUE; - } - - @Override - public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException { - Files.delete(dir); - return FileVisitResult.CONTINUE; - } - }); - return true; - } catch (IOException e) { - e.printStackTrace(); - return false; - } - } } \ No newline at end of file From 566fca2b52604b2166009b5e6ac79bdd71b0ed39 Mon Sep 17 00:00:00 2001 From: Julian Krings Date: Mon, 7 Apr 2025 19:00:06 +0200 Subject: [PATCH 2/3] Add info message for failed increase of worker threads --- .../core/pregenerator/methods/AsyncPregenMethod.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/core/src/main/java/com/volmit/iris/core/pregenerator/methods/AsyncPregenMethod.java b/core/src/main/java/com/volmit/iris/core/pregenerator/methods/AsyncPregenMethod.java index 16c2fd8c1..7abeb2256 100644 --- a/core/src/main/java/com/volmit/iris/core/pregenerator/methods/AsyncPregenMethod.java +++ b/core/src/main/java/com/volmit/iris/core/pregenerator/methods/AsyncPregenMethod.java @@ -32,6 +32,7 @@ import io.papermc.lib.PaperLib; import org.bukkit.Chunk; import org.bukkit.World; +import java.lang.reflect.InvocationTargetException; import java.util.ArrayList; import java.util.Map; import java.util.concurrent.Semaphore; @@ -150,19 +151,19 @@ public class AsyncPregenMethod implements PregeneratorMethod { public static void increaseWorkerThreads() { THREAD_COUNT.updateAndGet(i -> { if (i > 0) return 1; + var adjusted = IrisSettings.get().getConcurrency().getWorldGenThreads(); try { var field = Class.forName("ca.spottedleaf.moonrise.common.util.MoonriseCommon").getDeclaredField("WORKER_POOL"); var pool = field.get(null); var threads = ((Thread[]) pool.getClass().getDeclaredMethod("getCoreThreads").invoke(pool)).length; - var adjusted = IrisSettings.get().getConcurrency().getWorldGenThreads(); if (threads >= adjusted) return 0; pool.getClass().getDeclaredMethod("adjustThreadCount", int.class).invoke(pool, adjusted); return threads; - } catch (ClassNotFoundException ignored) { } catch (Throwable e) { - Iris.error("Failed to increase worker threads"); - e.printStackTrace(); + Iris.warn("Failed to increase worker threads, please increase it manually to " + adjusted); + Iris.warn("For more information see https://docs.papermc.io/paper/reference/global-configuration#chunk_system_worker_threads"); + if (e instanceof InvocationTargetException) e.printStackTrace(); } return 0; }); @@ -177,7 +178,6 @@ public class AsyncPregenMethod implements PregeneratorMethod { var method = pool.getClass().getDeclaredMethod("adjustThreadCount", int.class); method.invoke(pool, i); return 0; - } catch (ClassNotFoundException ignored) { } catch (Throwable e) { Iris.error("Failed to reset worker threads"); e.printStackTrace(); From 976648340e963ef47e67f89edc0d2bff06e8bd11 Mon Sep 17 00:00:00 2001 From: Julian Krings Date: Fri, 16 May 2025 12:02:38 +0200 Subject: [PATCH 3/3] add pregen method that doesn't use waiting threads --- .../com/volmit/iris/core/IrisSettings.java | 3 + .../methods/AsyncPregenMethod.java | 104 ++++++++++++------ .../volmit/iris/core/tools/IrisToolbelt.java | 2 +- 3 files changed, 76 insertions(+), 33 deletions(-) diff --git a/core/src/main/java/com/volmit/iris/core/IrisSettings.java b/core/src/main/java/com/volmit/iris/core/IrisSettings.java index c26b09b3b..826746193 100644 --- a/core/src/main/java/com/volmit/iris/core/IrisSettings.java +++ b/core/src/main/java/com/volmit/iris/core/IrisSettings.java @@ -145,7 +145,10 @@ public class IrisSettings { @Data public static class IrisSettingsPregen { + public boolean useCacheByDefault = true; + public boolean useHighPriority = false; public boolean useVirtualThreads = false; + public boolean useTicketQueue = false; public int maxConcurrency = 256; } diff --git a/core/src/main/java/com/volmit/iris/core/pregenerator/methods/AsyncPregenMethod.java b/core/src/main/java/com/volmit/iris/core/pregenerator/methods/AsyncPregenMethod.java index c31bf5029..757dfcf8f 100644 --- a/core/src/main/java/com/volmit/iris/core/pregenerator/methods/AsyncPregenMethod.java +++ b/core/src/main/java/com/volmit/iris/core/pregenerator/methods/AsyncPregenMethod.java @@ -33,7 +33,6 @@ import org.bukkit.Chunk; import org.bukkit.World; import java.lang.reflect.InvocationTargetException; -import java.util.ArrayList; import java.util.Map; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -43,22 +42,22 @@ import java.util.concurrent.atomic.AtomicInteger; public class AsyncPregenMethod implements PregeneratorMethod { private static final AtomicInteger THREAD_COUNT = new AtomicInteger(); private final World world; - private final ExecutorService service; + private final Executor executor; private final Semaphore semaphore; private final int threads; + private final boolean urgent; private final Map lastUse; - public AsyncPregenMethod(World world, int threads) { + public AsyncPregenMethod(World world, int unusedThreads) { if (!PaperLib.isPaper()) { throw new UnsupportedOperationException("Cannot use PaperAsync on non paper!"); } this.world = world; - service = IrisSettings.get().getPregen().isUseVirtualThreads() ? - Executors.newVirtualThreadPerTaskExecutor() : - new MultiBurst("Iris Async Pregen", Thread.MIN_PRIORITY); + this.executor = IrisSettings.get().getPregen().isUseTicketQueue() ? new TicketExecutor() : new ServiceExecutor(); this.threads = IrisSettings.get().getPregen().getMaxConcurrency(); - semaphore = new Semaphore(threads); + this.semaphore = new Semaphore(this.threads, true); + this.urgent = IrisSettings.get().getPregen().useHighPriority; this.lastUse = new KMap<>(); } @@ -70,13 +69,18 @@ public class AsyncPregenMethod implements PregeneratorMethod { return; } - for (Chunk i : new ArrayList<>(lastUse.keySet())) { - Long lastUseTime = lastUse.get(i); - if (!i.isLoaded() || (lastUseTime != null && M.ms() - lastUseTime >= 10000)) { - i.unload(); - lastUse.remove(i); + long minTime = M.ms() - 10_000; + lastUse.entrySet().removeIf(i -> { + final Chunk chunk = i.getKey(); + final Long lastUseTime = i.getValue(); + if (!chunk.isLoaded() || lastUseTime == null) + return true; + if (lastUseTime < minTime) { + chunk.unload(); + return true; } - } + return false; + }); world.save(); }).get(); } catch (Throwable e) { @@ -84,21 +88,6 @@ public class AsyncPregenMethod implements PregeneratorMethod { } } - private void completeChunk(int x, int z, PregenListener listener) { - try { - PaperLib.getChunkAtAsync(world, x, z, true).thenAccept((i) -> { - lastUse.put(i, M.ms()); - listener.onChunkGenerated(x, z); - listener.onChunkCleaned(x, z); - }).get(); - } catch (InterruptedException ignored) { - } catch (Throwable e) { - e.printStackTrace(); - } finally { - semaphore.release(); - } - } - @Override public void init() { unloadAndSaveAllChunks(); @@ -114,7 +103,7 @@ public class AsyncPregenMethod implements PregeneratorMethod { public void close() { semaphore.acquireUninterruptibly(threads); unloadAndSaveAllChunks(); - service.shutdown(); + executor.shutdown(); resetWorkerThreads(); } @@ -141,7 +130,7 @@ public class AsyncPregenMethod implements PregeneratorMethod { } catch (InterruptedException e) { return; } - service.submit(() -> completeChunk(x, z, listener)); + executor.generate(x, z, listener); } @Override @@ -153,7 +142,6 @@ public class AsyncPregenMethod implements PregeneratorMethod { return null; } - public static void increaseWorkerThreads() { THREAD_COUNT.updateAndGet(i -> { if (i > 0) return 1; @@ -167,7 +155,7 @@ public class AsyncPregenMethod implements PregeneratorMethod { pool.getClass().getDeclaredMethod("adjustThreadCount", int.class).invoke(pool, adjusted); return threads; } catch (Throwable e) { - Iris.warn("Failed to increase worker threads, please increase it manually to " + adjusted); + Iris.warn("Failed to increase worker threads, if you are on paper or a fork of it please increase it manually to " + adjusted); Iris.warn("For more information see https://docs.papermc.io/paper/reference/global-configuration#chunk_system_worker_threads"); if (e instanceof InvocationTargetException) e.printStackTrace(); } @@ -191,4 +179,56 @@ public class AsyncPregenMethod implements PregeneratorMethod { return i; }); } + + private interface Executor { + void generate(int x, int z, PregenListener listener); + default void shutdown() {} + } + + private class ServiceExecutor implements Executor { + private final ExecutorService service = IrisSettings.get().getPregen().isUseVirtualThreads() ? + Executors.newVirtualThreadPerTaskExecutor() : + new MultiBurst("Iris Async Pregen", Thread.MIN_PRIORITY); + + public void generate(int x, int z, PregenListener listener) { + service.submit(() -> { + try { + PaperLib.getChunkAtAsync(world, x, z, true, urgent).thenAccept((i) -> { + listener.onChunkGenerated(x, z); + listener.onChunkCleaned(x, z); + if (i == null) return; + lastUse.put(i, M.ms()); + }).get(); + } catch (InterruptedException ignored) { + } catch (Throwable e) { + e.printStackTrace(); + } finally { + semaphore.release(); + } + }); + } + + @Override + public void shutdown() { + service.shutdown(); + } + } + + private class TicketExecutor implements Executor { + @Override + public void generate(int x, int z, PregenListener listener) { + PaperLib.getChunkAtAsync(world, x, z, true, urgent) + .exceptionally(e -> { + e.printStackTrace(); + return null; + }) + .thenAccept(i -> { + semaphore.release(); + listener.onChunkGenerated(x, z); + listener.onChunkCleaned(x, z); + if (i == null) return; + lastUse.put(i, M.ms()); + }); + } + } } diff --git a/core/src/main/java/com/volmit/iris/core/tools/IrisToolbelt.java b/core/src/main/java/com/volmit/iris/core/tools/IrisToolbelt.java index 15b3dc8d3..06651213c 100644 --- a/core/src/main/java/com/volmit/iris/core/tools/IrisToolbelt.java +++ b/core/src/main/java/com/volmit/iris/core/tools/IrisToolbelt.java @@ -142,7 +142,7 @@ public class IrisToolbelt { * @return the pregenerator job (already started) */ public static PregeneratorJob pregenerate(PregenTask task, PregeneratorMethod method, Engine engine) { - return pregenerate(task, method, engine, true); + return pregenerate(task, method, engine, IrisSettings.get().getPregen().useCacheByDefault); } /**