diff --git a/core/src/main/java/com/volmit/iris/core/IrisSettings.java b/core/src/main/java/com/volmit/iris/core/IrisSettings.java index c2c394c32..cfd830827 100644 --- a/core/src/main/java/com/volmit/iris/core/IrisSettings.java +++ b/core/src/main/java/com/volmit/iris/core/IrisSettings.java @@ -147,11 +147,36 @@ public class IrisSettings { @Data public static class IrisSettingsPregen { + private static final int REFERENCE_WORLD_HEIGHT = 384; + private static final int MIN_RESIDENT_TECTONIC_PLATES = 16; + private static final double MANTLE_HEAP_FRACTION = 0.6D; + private static final int REFERENCE_PLATE_MEGABYTES = 48; + public boolean useCacheByDefault = true; public boolean useHighPriority = false; public boolean useVirtualThreads = false; public boolean useTicketQueue = true; public int maxConcurrency = 256; + public int maxResidentTectonicPlates = 96; + public int mantleBackpressureWaitMs = 25; + public int mantleBackpressureTimeoutMs = 60_000; + + public int getEffectiveResidentTectonicPlates(int worldHeight) { + int normalizedHeight = Math.max(1, worldHeight); + int heightScaledCap = (int) Math.round((double) maxResidentTectonicPlates * REFERENCE_WORLD_HEIGHT / normalizedHeight); + double plateMegabytes = (double) REFERENCE_PLATE_MEGABYTES * normalizedHeight / REFERENCE_WORLD_HEIGHT; + int heapBudgetCap = (int) Math.floor(MANTLE_HEAP_FRACTION * getHardware.getProcessMemory() / plateMegabytes); + int effective = Math.min(heightScaledCap, heapBudgetCap); + return Math.max(MIN_RESIDENT_TECTONIC_PLATES, Math.min(maxResidentTectonicPlates, effective)); + } + + public int getMantleBackpressureWaitMs() { + return Math.max(5, Math.min(mantleBackpressureWaitMs, 1_000)); + } + + public int getMantleBackpressureTimeoutMs() { + return Math.max(5_000, Math.min(mantleBackpressureTimeoutMs, 600_000)); + } } @Data diff --git a/core/src/main/java/com/volmit/iris/core/pregenerator/IrisPregenerator.java b/core/src/main/java/com/volmit/iris/core/pregenerator/IrisPregenerator.java index 99fda1048..7b41e4e42 100644 --- a/core/src/main/java/com/volmit/iris/core/pregenerator/IrisPregenerator.java +++ b/core/src/main/java/com/volmit/iris/core/pregenerator/IrisPregenerator.java @@ -67,6 +67,7 @@ public class IrisPregenerator { private final ChronoLatch cl; private final ChronoLatch saveLatch = new ChronoLatch(30000); private final IrisPackBenchmarking benchmarking; + private final PregenMantleBackpressure backpressure; public IrisPregenerator(PregenTask task, PregeneratorMethod generator, PregenListener listener) { benchmarking = IrisPackBenchmarking.getInstance(); @@ -77,6 +78,7 @@ public class IrisPregenerator { this.paused = new AtomicBoolean(false); this.task = task; this.generator = generator; + this.backpressure = new PregenMantleBackpressure(this::getMantle, shutdown::get); retry = new KSet<>(); net = new KSet<>(); currentGeneratorMethod = new AtomicReference<>("Void"); @@ -246,6 +248,11 @@ public class IrisPregenerator { J.sleep(50); } + backpressure.apply(); + if (shutdown.get()) { + return; + } + generator.generateChunk(xx, zz, listener); }); } diff --git a/core/src/main/java/com/volmit/iris/core/pregenerator/MantleHeapPressure.java b/core/src/main/java/com/volmit/iris/core/pregenerator/MantleHeapPressure.java new file mode 100644 index 000000000..629e7d9d2 --- /dev/null +++ b/core/src/main/java/com/volmit/iris/core/pregenerator/MantleHeapPressure.java @@ -0,0 +1,229 @@ +package com.volmit.iris.core.pregenerator; + +import com.volmit.iris.Iris; + +import javax.management.MBeanServer; +import javax.management.ObjectName; +import java.lang.management.ManagementFactory; +import java.lang.management.MemoryUsage; +import java.util.function.BiConsumer; +import java.util.function.DoubleConsumer; +import java.util.function.LongSupplier; + +public final class MantleHeapPressure { + private static final double HIGH_WATER = 0.92D; + private static final double LOW_WATER = 0.82D; + private static final long MAXIMUM_HYSTERESIS_MS = 60_000L; + private static final ObjectName DIAGNOSTIC_COMMAND = diagnosticCommandName(); + private static final PanicGcReclaimer PANIC_RECLAIMER = new PanicGcReclaimer( + new PanicGcPolicy(10_000L, 60_000L, 60_000L, 15 * 60_000L), + new PanicGcActions( + System::currentTimeMillis, + System::gc, + () -> invokeHotSpotDiagnosticGc(ManagementFactory.getPlatformMBeanServer()), + fraction -> Iris.info("Iris heap remained at %.1f%% after a normal reclaim; running the JVM's diagnostic full GC to keep generation going.", + fraction * 100.0D), + (context, failure) -> { + Iris.warn("%s", context); + Iris.reportError(failure); + })); + private static final HeapPressureGate PRESSURE_GATE = new HeapPressureGate( + HIGH_WATER, + LOW_WATER, + MAXIMUM_HYSTERESIS_MS, + System::currentTimeMillis, + PANIC_RECLAIMER::resetEpisode); + + private MantleHeapPressure() { + } + + public static double usedFraction() { + MemoryUsage heap = ManagementFactory.getMemoryMXBean().getHeapMemoryUsage(); + long max = heap.getMax(); + if (max > 0L) { + return (double) heap.getUsed() / (double) max; + } + + Runtime runtime = Runtime.getRuntime(); + long runtimeMax = runtime.maxMemory(); + if (runtimeMax <= 0L) { + return 0.0D; + } + return (double) (runtime.totalMemory() - runtime.freeMemory()) / (double) runtimeMax; + } + + public static boolean overHighWater() { + return PRESSURE_GATE.update(usedFraction()); + } + + public static double reclaimUrgency(double fraction) { + if (!Double.isFinite(fraction) || fraction <= LOW_WATER) { + return 0D; + } + if (fraction >= HIGH_WATER) { + return 1D; + } + return (fraction - LOW_WATER) / (HIGH_WATER - LOW_WATER); + } + + // Paused generation barely allocates, so the collector may not run again and the last reading can stay high after + // most of it has become garbage. + public static void requestPanicReclaim() { + PANIC_RECLAIMER.request(usedFraction()); + } + + // Unlike System.gc(), still honoured by servers started with -XX:+DisableExplicitGC. + static void invokeHotSpotDiagnosticGc(MBeanServer server) throws Exception { + if (!server.isRegistered(DIAGNOSTIC_COMMAND)) { + throw new UnsupportedOperationException("HotSpot DiagnosticCommand MBean is not registered on this JVM"); + } + server.invoke(DIAGNOSTIC_COMMAND, "gcRun", new Object[0], new String[0]); + } + + private static ObjectName diagnosticCommandName() { + try { + return new ObjectName("com.sun.management:type=DiagnosticCommand"); + } catch (Exception failure) { + throw new ExceptionInInitializerError(failure); + } + } + + record PanicGcPolicy( + long diagnosticDelayMs, + long diagnosticCooldownMs, + long initialFailureBackoffMs, + long maximumFailureBackoffMs + ) { + } + + record PanicGcActions( + LongSupplier clock, + Runnable explicitGc, + DiagnosticGc diagnosticGc, + DoubleConsumer diagnosticStart, + BiConsumer failureSink + ) { + } + + @FunctionalInterface + interface DiagnosticGc { + void run() throws Exception; + } + + static final class HeapPressureGate { + private static final long NOT_BELOW_HIGH_WATER = Long.MIN_VALUE; + + private final double highWater; + private final double lowWater; + private final long maximumHysteresisMs; + private final LongSupplier clock; + private final Runnable releaseAction; + private boolean engaged; + private long belowHighWaterSince = NOT_BELOW_HIGH_WATER; + + HeapPressureGate(double highWater, double lowWater, long maximumHysteresisMs, LongSupplier clock, Runnable releaseAction) { + this.highWater = highWater; + this.lowWater = lowWater; + this.maximumHysteresisMs = maximumHysteresisMs; + this.clock = clock; + this.releaseAction = releaseAction; + } + + synchronized boolean update(double fraction) { + if (!Double.isFinite(fraction)) { + return engaged; + } + if (!engaged) { + engaged = fraction >= highWater; + return engaged; + } + if (fraction <= lowWater) { + release(); + return false; + } + if (fraction >= highWater) { + belowHighWaterSince = NOT_BELOW_HIGH_WATER; + return true; + } + + long now = clock.getAsLong(); + if (belowHighWaterSince == NOT_BELOW_HIGH_WATER) { + belowHighWaterSince = now; + return true; + } + // A collector that rarely shrinks the heap could otherwise keep generation paused between the two marks. + if (now - belowHighWaterSince < maximumHysteresisMs) { + return true; + } + + release(); + return false; + } + + private void release() { + engaged = false; + belowHighWaterSince = NOT_BELOW_HIGH_WATER; + releaseAction.run(); + } + } + + static final class PanicGcReclaimer { + private final PanicGcPolicy policy; + private final PanicGcActions actions; + private boolean panicEpisode; + private long explicitAttemptAt; + private long nextDiagnosticAllowedAt; + private long failureBackoffMs; + + PanicGcReclaimer(PanicGcPolicy policy, PanicGcActions actions) { + this.policy = policy; + this.actions = actions; + this.failureBackoffMs = policy.initialFailureBackoffMs(); + } + + synchronized void request(double fraction) { + if (!Double.isFinite(fraction) || fraction <= LOW_WATER) { + return; + } + + long now = actions.clock().getAsLong(); + if (!panicEpisode) { + beginEpisode(now); + return; + } + if (now - explicitAttemptAt < policy.diagnosticDelayMs() || now < nextDiagnosticAllowedAt) { + return; + } + + actions.diagnosticStart().accept(fraction); + try { + actions.diagnosticGc().run(); + failureBackoffMs = policy.initialFailureBackoffMs(); + nextDiagnosticAllowedAt = now + policy.diagnosticCooldownMs(); + } catch (Exception failure) { + actions.failureSink().accept( + "Iris could not run the JVM's diagnostic GC after a normal reclaim did not free enough memory; generation stays paused while the heap is full.", + failure); + nextDiagnosticAllowedAt = now + failureBackoffMs; + failureBackoffMs = Math.min(policy.maximumFailureBackoffMs(), failureBackoffMs * 2L); + } + } + + synchronized void resetEpisode() { + panicEpisode = false; + explicitAttemptAt = 0L; + } + + private void beginEpisode(long now) { + panicEpisode = true; + explicitAttemptAt = now; + try { + actions.explicitGc().run(); + } catch (RuntimeException failure) { + actions.failureSink().accept( + "Iris could not run a normal heap reclaim; the diagnostic GC will be tried if the heap stays full.", + failure); + } + } + } +} diff --git a/core/src/main/java/com/volmit/iris/core/pregenerator/PregenMantleBackpressure.java b/core/src/main/java/com/volmit/iris/core/pregenerator/PregenMantleBackpressure.java new file mode 100644 index 000000000..2d000496e --- /dev/null +++ b/core/src/main/java/com/volmit/iris/core/pregenerator/PregenMantleBackpressure.java @@ -0,0 +1,128 @@ +package com.volmit.iris.core.pregenerator; + +import com.volmit.iris.Iris; +import com.volmit.iris.core.IrisSettings; +import com.volmit.iris.util.mantle.Mantle; +import com.volmit.iris.util.math.M; + +import java.util.function.BooleanSupplier; +import java.util.function.Supplier; + +public final class PregenMantleBackpressure { + private static final long PROGRESS_LOG_INTERVAL_MS = 5_000L; + + private final Supplier mantleSupplier; + private final BooleanSupplier cancelled; + private final int waitMs; + private final long timeoutMs; + + public PregenMantleBackpressure(Supplier mantleSupplier, BooleanSupplier cancelled) { + IrisSettings.IrisSettingsPregen pregen = IrisSettings.get().getPregen(); + this.mantleSupplier = mantleSupplier; + this.cancelled = cancelled; + this.waitMs = pregen.getMantleBackpressureWaitMs(); + this.timeoutMs = pregen.getMantleBackpressureTimeoutMs(); + } + + public void apply() { + Mantle mantle = resolveMantle(); + if (mantle == null || mantle.isClosed()) { + return; + } + + int plateBudget = IrisSettings.get().getPregen().getEffectiveResidentTectonicPlates(mantle.getWorldHeight()); + enforcePlateBudget(mantle, plateBudget); + awaitHeapHeadroom(mantle, plateBudget); + } + + private void enforcePlateBudget(Mantle mantle, int plateBudget) { + int hardCap = plateBudget * 2; + long waitStart = M.ms(); + long lastLog = waitStart; + while (mantle.getLoadedRegionCount() > hardCap && !isCancelled()) { + int freed = evictIdlePlates(mantle); + int resident = mantle.getLoadedRegionCount(); + if (resident <= hardCap) { + return; + } + + long elapsed = M.ms() - waitStart; + if (elapsed >= timeoutMs) { + Iris.warn("Pregen mantle backpressure waited %dms with %d tectonic plates resident (hard cap %d); " + + "continuing to avoid a deadlock. Raise pregen.maxResidentTectonicPlates if this keeps happening.", + timeoutMs, resident, hardCap); + return; + } + if (M.ms() - lastLog >= PROGRESS_LOG_INTERVAL_MS) { + lastLog = M.ms(); + Iris.info("Pregen mantle backpressure: %d tectonic plates resident (hard cap %d), freed %d last pass, waited %dms.", + resident, hardCap, freed, elapsed); + } + if (!pause()) { + return; + } + } + } + + private void awaitHeapHeadroom(Mantle mantle, int plateBudget) { + long waitStart = M.ms(); + long lastLog = waitStart; + while (MantleHeapPressure.overHighWater() && !isCancelled()) { + if (mantle.getLoadedRegionCount() > plateBudget) { + evictIdlePlates(mantle); + } + MantleHeapPressure.requestPanicReclaim(); + + long elapsed = M.ms() - waitStart; + if (elapsed >= timeoutMs) { + Iris.warn("Pregen heap pressure wait exceeded %dms at %d%% heap; continuing to avoid a deadlock. " + + "Give the server more memory if this keeps happening.", timeoutMs, heapPercent()); + return; + } + if (M.ms() - lastLog >= PROGRESS_LOG_INTERVAL_MS) { + lastLog = M.ms(); + Iris.info("Pregen heap pressure: paused at %d%% heap with %d tectonic plates resident, waiting for headroom.", + heapPercent(), mantle.getLoadedRegionCount()); + } + if (!pause()) { + return; + } + } + } + + private int evictIdlePlates(Mantle mantle) { + try { + mantle.trim(0L, 0); + return mantle.unloadTectonicPlate(0); + } catch (Throwable e) { + Iris.reportError(e); + return 0; + } + } + + private boolean pause() { + try { + Thread.sleep(waitMs); + return true; + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return false; + } + } + + private static long heapPercent() { + return Math.round(MantleHeapPressure.usedFraction() * 100.0D); + } + + private boolean isCancelled() { + return Thread.currentThread().isInterrupted() || cancelled.getAsBoolean(); + } + + private Mantle resolveMantle() { + try { + return mantleSupplier.get(); + } catch (Throwable e) { + return null; + } + } +} diff --git a/core/src/main/java/com/volmit/iris/core/service/IrisEngineSVC.java b/core/src/main/java/com/volmit/iris/core/service/IrisEngineSVC.java index 36e4b6d98..9a79f73f6 100644 --- a/core/src/main/java/com/volmit/iris/core/service/IrisEngineSVC.java +++ b/core/src/main/java/com/volmit/iris/core/service/IrisEngineSVC.java @@ -4,6 +4,7 @@ import com.google.common.util.concurrent.AtomicDouble; import com.volmit.iris.Iris; import com.volmit.iris.core.IrisSettings; import com.volmit.iris.core.loader.ResourceLoader; +import com.volmit.iris.core.pregenerator.MantleHeapPressure; import com.volmit.iris.core.tools.IrisToolbelt; import com.volmit.iris.engine.framework.Engine; import com.volmit.iris.engine.platform.PlatformChunkGenerator; @@ -218,7 +219,7 @@ public class IrisEngineSVC implements IrisService { return; try { - engine.getMantle().trim(tectonicLimit()); + engine.getMantle().trim(heapScaledKeepAliveMillis(), tectonicLimit()); } catch (Throwable e) { Iris.reportError(e); Iris.error("EngineSVC: Failed to trim for " + name); @@ -235,7 +236,7 @@ public class IrisEngineSVC implements IrisService { try { long unloadStart = System.currentTimeMillis(); - int count = engine.getMantle().unloadTectonicPlate(IrisSettings.get().getPerformance().getEngineSVC().forceMulticoreWrite ? 0 : tectonicLimit()); + int count = engine.getMantle().unloadTectonicPlate(shouldUnloadMulticore() ? 0 : tectonicLimit()); if (count > 0) { Iris.debug(C.GOLD + "Unloaded " + C.YELLOW + count + " TectonicPlates in " + C.RED + Form.duration(System.currentTimeMillis() - unloadStart, 2)); } @@ -252,6 +253,19 @@ public class IrisEngineSVC implements IrisService { return tectonicLimit.get() / Math.max(worlds.size(), 1); } + private long heapScaledKeepAliveMillis() { + long keepAlive = TimeUnit.SECONDS.toMillis(IrisSettings.get().getPerformance().getMantleKeepAlive()); + return Math.round(keepAlive * (1D - currentReclaimUrgency())); + } + + private boolean shouldUnloadMulticore() { + return IrisSettings.get().getPerformance().getEngineSVC().forceMulticoreWrite || currentReclaimUrgency() >= 1D; + } + + private double currentReclaimUrgency() { + return MantleHeapPressure.reclaimUrgency(MantleHeapPressure.usedFraction()); + } + @Synchronized private void close() { if (closed) return; diff --git a/core/src/main/java/com/volmit/iris/util/mantle/Mantle.java b/core/src/main/java/com/volmit/iris/util/mantle/Mantle.java index 46185cc05..068e131aa 100644 --- a/core/src/main/java/com/volmit/iris/util/mantle/Mantle.java +++ b/core/src/main/java/com/volmit/iris/util/mantle/Mantle.java @@ -21,6 +21,7 @@ package com.volmit.iris.util.mantle; import com.google.common.util.concurrent.AtomicDouble; import com.volmit.iris.Iris; import com.volmit.iris.core.IrisSettings; +import com.volmit.iris.core.pregenerator.MantleHeapPressure; import com.volmit.iris.core.tools.IrisToolbelt; import com.volmit.iris.engine.data.cache.Cache; import com.volmit.iris.engine.framework.Engine; @@ -76,6 +77,7 @@ public class Mantle { private final IOWorker worker; private final AtomicDouble adjustedIdleDuration; private final KSet toUnload; + private final KMap> loadingRegions; /** * Create a new mantle @@ -96,6 +98,7 @@ public class Mantle { ioBurst = MultiBurst.ioBurst; adjustedIdleDuration = new AtomicDouble(0); toUnload = new KSet<>(); + loadingRegions = new KMap<>(); worker = new IOWorker(dataFolder, worldHeight); Iris.debug("Opened The Mantle " + C.DARK_AQUA + dataFolder.getAbsolutePath()); } @@ -484,9 +487,10 @@ public class Mantle { } double idleDuration = baseIdleDuration; + double minimumIdleDuration = Math.max(500D, Math.min(baseIdleDuration, 4000D)); if (loadedRegions.size() > tectonicLimit) { // todo update this correctly and maybe do something when its above a 100% - idleDuration = Math.max(idleDuration - (1000 * (((loadedRegions.size() - tectonicLimit) / (double) tectonicLimit) * 100) * 0.4), 4000); + idleDuration = Math.max(idleDuration - (1000 * (((loadedRegions.size() - tectonicLimit) / (double) tectonicLimit) * 100) * 0.4), minimumIdleDuration); } adjustedIdleDuration.set(idleDuration); @@ -616,6 +620,7 @@ public class Mantle { if (unload) ioTectonicUnload.release(); } + ensureRetryable(x, z); Iris.warn("Retrying to get " + x + " " + z + " Mantle Region"); return get(x, z); } @@ -644,6 +649,7 @@ public class Mantle { .thenCompose(p -> { release.apply(p); if (p != null) return CompletableFuture.completedFuture(p); + ensureRetryable(x, z); Iris.warn("Retrying to get " + x + " " + z + " Mantle Region"); return getFuture(x, z); }); @@ -667,9 +673,16 @@ public class Mantle { return fallback.get(); } + private void ensureRetryable(int x, int z) { + if (closed.get() || Thread.currentThread().isInterrupted()) { + throw new IllegalStateException("Stopped retrying Tectonic Plate " + x + " " + z + " because the Mantle closed or the thread was interrupted"); + } + } + /** * This retreives a future of the Tectonic Plate at the given coordinates. - * All methods accessing tectonic plates should go through this method + * All methods accessing tectonic plates should go through this method. Concurrent requests for a plate that is + * still loading share one load instead of each occupying an IO thread while they wait on the plate's lock. * * @param x the region x * @param z the region z @@ -677,7 +690,31 @@ public class Mantle { */ @RegionCoordinates protected CompletableFuture getSafe(int x, int z) { - return ioBurst.completableFuture(() -> hyperLock.withResult(x, z, () -> { + Long key = key(x, z); + CompletableFuture published = new CompletableFuture<>(); + CompletableFuture existing = loadingRegions.putIfAbsent(key, published); + if (existing != null) { + return existing; + } + + try { + loadRegion(x, z).whenComplete((region, failure) -> { + loadingRegions.remove(key, published); + if (failure == null) { + published.complete(region); + } else { + published.completeExceptionally(failure); + } + }); + } catch (Throwable failure) { + loadingRegions.remove(key, published); + published.completeExceptionally(failure); + } + return published; + } + + private CompletableFuture loadRegion(int x, int z) { + return CompletableFuture.supplyAsync(() -> hyperLock.withResult(x, z, () -> { Long k = key(x, z); use(k); TectonicPlate r = loadedRegions.get(k); @@ -719,7 +756,7 @@ public class Mantle { Iris.debug("Created new Tectonic Plate " + C.DARK_GREEN + x + " " + z); use(k); return region; - })); + }), ioBurst); } private void use(Long key) { @@ -768,6 +805,8 @@ public class Mantle { } public boolean shouldReduce(Engine engine) { - return !engine.isStudio() || IrisSettings.get().getPerformance().isTrimMantleInStudio(); + return !engine.isStudio() + || IrisSettings.get().getPerformance().isTrimMantleInStudio() + || MantleHeapPressure.overHighWater(); } } \ No newline at end of file