Hold pregen back when the mantle or the heap is full

Ported from Iris v4's mantle backpressure. Before each chunk the
pregenerator now:
- evicts idle tectonic plates and waits while more plates are loaded
  than the heap-scaled budget allows (pregen.maxResidentTectonicPlates)
- pauses between 92% and 82% heap, requesting a GC because a paused
  pregen allocates too little for the collector to run on its own; if
  the heap stays full, the JVM's diagnostic GC runs at most once a
  minute, which also works with -XX:+DisableExplicitGC
Both waits give up after pregen.mantleBackpressureTimeoutMs so a stuck
heap cannot hang the pregen. Unlike v4, a reclaim starts anywhere above
82% instead of only at 92%, since a pregen waiting at 89% otherwise
never triggers one.

Outside pregen, the engine service shortens the mantle keep-alive as
the heap fills and trims studio worlds once the heap is critical.

The mantle itself:
- concurrent requests for a plate that is still loading share one load
  instead of each holding an IO thread
- plate retries stop once the mantle is closed or the thread is
  interrupted, instead of recursing forever
- trim(0, 0) evicts plates idle for 500 ms instead of 4 s

Model used: Claude Opus 5.5
This commit is contained in:
piexel
2026-10-15 00:26:33 +02:00
parent 18e0834db8
commit 7f34bb917f
6 changed files with 449 additions and 7 deletions
@@ -147,11 +147,36 @@ public class IrisSettings {
@Data
public static class IrisSettingsPregen {
private static final int REFERENCE_WORLD_HEIGHT = 384;
private static final int MIN_RESIDENT_TECTONIC_PLATES = 16;
private static final double MANTLE_HEAP_FRACTION = 0.6D;
private static final int REFERENCE_PLATE_MEGABYTES = 48;
public boolean useCacheByDefault = true;
public boolean useHighPriority = false;
public boolean useVirtualThreads = false;
public boolean useTicketQueue = true;
public int maxConcurrency = 256;
public int maxResidentTectonicPlates = 96;
public int mantleBackpressureWaitMs = 25;
public int mantleBackpressureTimeoutMs = 60_000;
public int getEffectiveResidentTectonicPlates(int worldHeight) {
int normalizedHeight = Math.max(1, worldHeight);
int heightScaledCap = (int) Math.round((double) maxResidentTectonicPlates * REFERENCE_WORLD_HEIGHT / normalizedHeight);
double plateMegabytes = (double) REFERENCE_PLATE_MEGABYTES * normalizedHeight / REFERENCE_WORLD_HEIGHT;
int heapBudgetCap = (int) Math.floor(MANTLE_HEAP_FRACTION * getHardware.getProcessMemory() / plateMegabytes);
int effective = Math.min(heightScaledCap, heapBudgetCap);
return Math.max(MIN_RESIDENT_TECTONIC_PLATES, Math.min(maxResidentTectonicPlates, effective));
}
public int getMantleBackpressureWaitMs() {
return Math.max(5, Math.min(mantleBackpressureWaitMs, 1_000));
}
public int getMantleBackpressureTimeoutMs() {
return Math.max(5_000, Math.min(mantleBackpressureTimeoutMs, 600_000));
}
}
@Data
@@ -67,6 +67,7 @@ public class IrisPregenerator {
private final ChronoLatch cl;
private final ChronoLatch saveLatch = new ChronoLatch(30000);
private final IrisPackBenchmarking benchmarking;
private final PregenMantleBackpressure backpressure;
public IrisPregenerator(PregenTask task, PregeneratorMethod generator, PregenListener listener) {
benchmarking = IrisPackBenchmarking.getInstance();
@@ -77,6 +78,7 @@ public class IrisPregenerator {
this.paused = new AtomicBoolean(false);
this.task = task;
this.generator = generator;
this.backpressure = new PregenMantleBackpressure(this::getMantle, shutdown::get);
retry = new KSet<>();
net = new KSet<>();
currentGeneratorMethod = new AtomicReference<>("Void");
@@ -246,6 +248,11 @@ public class IrisPregenerator {
J.sleep(50);
}
backpressure.apply();
if (shutdown.get()) {
return;
}
generator.generateChunk(xx, zz, listener);
});
}
@@ -0,0 +1,229 @@
package com.volmit.iris.core.pregenerator;
import com.volmit.iris.Iris;
import javax.management.MBeanServer;
import javax.management.ObjectName;
import java.lang.management.ManagementFactory;
import java.lang.management.MemoryUsage;
import java.util.function.BiConsumer;
import java.util.function.DoubleConsumer;
import java.util.function.LongSupplier;
public final class MantleHeapPressure {
private static final double HIGH_WATER = 0.92D;
private static final double LOW_WATER = 0.82D;
private static final long MAXIMUM_HYSTERESIS_MS = 60_000L;
private static final ObjectName DIAGNOSTIC_COMMAND = diagnosticCommandName();
private static final PanicGcReclaimer PANIC_RECLAIMER = new PanicGcReclaimer(
new PanicGcPolicy(10_000L, 60_000L, 60_000L, 15 * 60_000L),
new PanicGcActions(
System::currentTimeMillis,
System::gc,
() -> invokeHotSpotDiagnosticGc(ManagementFactory.getPlatformMBeanServer()),
fraction -> Iris.info("Iris heap remained at %.1f%% after a normal reclaim; running the JVM's diagnostic full GC to keep generation going.",
fraction * 100.0D),
(context, failure) -> {
Iris.warn("%s", context);
Iris.reportError(failure);
}));
private static final HeapPressureGate PRESSURE_GATE = new HeapPressureGate(
HIGH_WATER,
LOW_WATER,
MAXIMUM_HYSTERESIS_MS,
System::currentTimeMillis,
PANIC_RECLAIMER::resetEpisode);
private MantleHeapPressure() {
}
public static double usedFraction() {
MemoryUsage heap = ManagementFactory.getMemoryMXBean().getHeapMemoryUsage();
long max = heap.getMax();
if (max > 0L) {
return (double) heap.getUsed() / (double) max;
}
Runtime runtime = Runtime.getRuntime();
long runtimeMax = runtime.maxMemory();
if (runtimeMax <= 0L) {
return 0.0D;
}
return (double) (runtime.totalMemory() - runtime.freeMemory()) / (double) runtimeMax;
}
public static boolean overHighWater() {
return PRESSURE_GATE.update(usedFraction());
}
public static double reclaimUrgency(double fraction) {
if (!Double.isFinite(fraction) || fraction <= LOW_WATER) {
return 0D;
}
if (fraction >= HIGH_WATER) {
return 1D;
}
return (fraction - LOW_WATER) / (HIGH_WATER - LOW_WATER);
}
// Paused generation barely allocates, so the collector may not run again and the last reading can stay high after
// most of it has become garbage.
public static void requestPanicReclaim() {
PANIC_RECLAIMER.request(usedFraction());
}
// Unlike System.gc(), still honoured by servers started with -XX:+DisableExplicitGC.
static void invokeHotSpotDiagnosticGc(MBeanServer server) throws Exception {
if (!server.isRegistered(DIAGNOSTIC_COMMAND)) {
throw new UnsupportedOperationException("HotSpot DiagnosticCommand MBean is not registered on this JVM");
}
server.invoke(DIAGNOSTIC_COMMAND, "gcRun", new Object[0], new String[0]);
}
private static ObjectName diagnosticCommandName() {
try {
return new ObjectName("com.sun.management:type=DiagnosticCommand");
} catch (Exception failure) {
throw new ExceptionInInitializerError(failure);
}
}
record PanicGcPolicy(
long diagnosticDelayMs,
long diagnosticCooldownMs,
long initialFailureBackoffMs,
long maximumFailureBackoffMs
) {
}
record PanicGcActions(
LongSupplier clock,
Runnable explicitGc,
DiagnosticGc diagnosticGc,
DoubleConsumer diagnosticStart,
BiConsumer<String, Throwable> failureSink
) {
}
@FunctionalInterface
interface DiagnosticGc {
void run() throws Exception;
}
static final class HeapPressureGate {
private static final long NOT_BELOW_HIGH_WATER = Long.MIN_VALUE;
private final double highWater;
private final double lowWater;
private final long maximumHysteresisMs;
private final LongSupplier clock;
private final Runnable releaseAction;
private boolean engaged;
private long belowHighWaterSince = NOT_BELOW_HIGH_WATER;
HeapPressureGate(double highWater, double lowWater, long maximumHysteresisMs, LongSupplier clock, Runnable releaseAction) {
this.highWater = highWater;
this.lowWater = lowWater;
this.maximumHysteresisMs = maximumHysteresisMs;
this.clock = clock;
this.releaseAction = releaseAction;
}
synchronized boolean update(double fraction) {
if (!Double.isFinite(fraction)) {
return engaged;
}
if (!engaged) {
engaged = fraction >= highWater;
return engaged;
}
if (fraction <= lowWater) {
release();
return false;
}
if (fraction >= highWater) {
belowHighWaterSince = NOT_BELOW_HIGH_WATER;
return true;
}
long now = clock.getAsLong();
if (belowHighWaterSince == NOT_BELOW_HIGH_WATER) {
belowHighWaterSince = now;
return true;
}
// A collector that rarely shrinks the heap could otherwise keep generation paused between the two marks.
if (now - belowHighWaterSince < maximumHysteresisMs) {
return true;
}
release();
return false;
}
private void release() {
engaged = false;
belowHighWaterSince = NOT_BELOW_HIGH_WATER;
releaseAction.run();
}
}
static final class PanicGcReclaimer {
private final PanicGcPolicy policy;
private final PanicGcActions actions;
private boolean panicEpisode;
private long explicitAttemptAt;
private long nextDiagnosticAllowedAt;
private long failureBackoffMs;
PanicGcReclaimer(PanicGcPolicy policy, PanicGcActions actions) {
this.policy = policy;
this.actions = actions;
this.failureBackoffMs = policy.initialFailureBackoffMs();
}
synchronized void request(double fraction) {
if (!Double.isFinite(fraction) || fraction <= LOW_WATER) {
return;
}
long now = actions.clock().getAsLong();
if (!panicEpisode) {
beginEpisode(now);
return;
}
if (now - explicitAttemptAt < policy.diagnosticDelayMs() || now < nextDiagnosticAllowedAt) {
return;
}
actions.diagnosticStart().accept(fraction);
try {
actions.diagnosticGc().run();
failureBackoffMs = policy.initialFailureBackoffMs();
nextDiagnosticAllowedAt = now + policy.diagnosticCooldownMs();
} catch (Exception failure) {
actions.failureSink().accept(
"Iris could not run the JVM's diagnostic GC after a normal reclaim did not free enough memory; generation stays paused while the heap is full.",
failure);
nextDiagnosticAllowedAt = now + failureBackoffMs;
failureBackoffMs = Math.min(policy.maximumFailureBackoffMs(), failureBackoffMs * 2L);
}
}
synchronized void resetEpisode() {
panicEpisode = false;
explicitAttemptAt = 0L;
}
private void beginEpisode(long now) {
panicEpisode = true;
explicitAttemptAt = now;
try {
actions.explicitGc().run();
} catch (RuntimeException failure) {
actions.failureSink().accept(
"Iris could not run a normal heap reclaim; the diagnostic GC will be tried if the heap stays full.",
failure);
}
}
}
}
@@ -0,0 +1,128 @@
package com.volmit.iris.core.pregenerator;
import com.volmit.iris.Iris;
import com.volmit.iris.core.IrisSettings;
import com.volmit.iris.util.mantle.Mantle;
import com.volmit.iris.util.math.M;
import java.util.function.BooleanSupplier;
import java.util.function.Supplier;
public final class PregenMantleBackpressure {
private static final long PROGRESS_LOG_INTERVAL_MS = 5_000L;
private final Supplier<Mantle> mantleSupplier;
private final BooleanSupplier cancelled;
private final int waitMs;
private final long timeoutMs;
public PregenMantleBackpressure(Supplier<Mantle> mantleSupplier, BooleanSupplier cancelled) {
IrisSettings.IrisSettingsPregen pregen = IrisSettings.get().getPregen();
this.mantleSupplier = mantleSupplier;
this.cancelled = cancelled;
this.waitMs = pregen.getMantleBackpressureWaitMs();
this.timeoutMs = pregen.getMantleBackpressureTimeoutMs();
}
public void apply() {
Mantle mantle = resolveMantle();
if (mantle == null || mantle.isClosed()) {
return;
}
int plateBudget = IrisSettings.get().getPregen().getEffectiveResidentTectonicPlates(mantle.getWorldHeight());
enforcePlateBudget(mantle, plateBudget);
awaitHeapHeadroom(mantle, plateBudget);
}
private void enforcePlateBudget(Mantle mantle, int plateBudget) {
int hardCap = plateBudget * 2;
long waitStart = M.ms();
long lastLog = waitStart;
while (mantle.getLoadedRegionCount() > hardCap && !isCancelled()) {
int freed = evictIdlePlates(mantle);
int resident = mantle.getLoadedRegionCount();
if (resident <= hardCap) {
return;
}
long elapsed = M.ms() - waitStart;
if (elapsed >= timeoutMs) {
Iris.warn("Pregen mantle backpressure waited %dms with %d tectonic plates resident (hard cap %d); "
+ "continuing to avoid a deadlock. Raise pregen.maxResidentTectonicPlates if this keeps happening.",
timeoutMs, resident, hardCap);
return;
}
if (M.ms() - lastLog >= PROGRESS_LOG_INTERVAL_MS) {
lastLog = M.ms();
Iris.info("Pregen mantle backpressure: %d tectonic plates resident (hard cap %d), freed %d last pass, waited %dms.",
resident, hardCap, freed, elapsed);
}
if (!pause()) {
return;
}
}
}
private void awaitHeapHeadroom(Mantle mantle, int plateBudget) {
long waitStart = M.ms();
long lastLog = waitStart;
while (MantleHeapPressure.overHighWater() && !isCancelled()) {
if (mantle.getLoadedRegionCount() > plateBudget) {
evictIdlePlates(mantle);
}
MantleHeapPressure.requestPanicReclaim();
long elapsed = M.ms() - waitStart;
if (elapsed >= timeoutMs) {
Iris.warn("Pregen heap pressure wait exceeded %dms at %d%% heap; continuing to avoid a deadlock. "
+ "Give the server more memory if this keeps happening.", timeoutMs, heapPercent());
return;
}
if (M.ms() - lastLog >= PROGRESS_LOG_INTERVAL_MS) {
lastLog = M.ms();
Iris.info("Pregen heap pressure: paused at %d%% heap with %d tectonic plates resident, waiting for headroom.",
heapPercent(), mantle.getLoadedRegionCount());
}
if (!pause()) {
return;
}
}
}
private int evictIdlePlates(Mantle mantle) {
try {
mantle.trim(0L, 0);
return mantle.unloadTectonicPlate(0);
} catch (Throwable e) {
Iris.reportError(e);
return 0;
}
}
private boolean pause() {
try {
Thread.sleep(waitMs);
return true;
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
return false;
}
}
private static long heapPercent() {
return Math.round(MantleHeapPressure.usedFraction() * 100.0D);
}
private boolean isCancelled() {
return Thread.currentThread().isInterrupted() || cancelled.getAsBoolean();
}
private Mantle resolveMantle() {
try {
return mantleSupplier.get();
} catch (Throwable e) {
return null;
}
}
}
@@ -4,6 +4,7 @@ import com.google.common.util.concurrent.AtomicDouble;
import com.volmit.iris.Iris;
import com.volmit.iris.core.IrisSettings;
import com.volmit.iris.core.loader.ResourceLoader;
import com.volmit.iris.core.pregenerator.MantleHeapPressure;
import com.volmit.iris.core.tools.IrisToolbelt;
import com.volmit.iris.engine.framework.Engine;
import com.volmit.iris.engine.platform.PlatformChunkGenerator;
@@ -218,7 +219,7 @@ public class IrisEngineSVC implements IrisService {
return;
try {
engine.getMantle().trim(tectonicLimit());
engine.getMantle().trim(heapScaledKeepAliveMillis(), tectonicLimit());
} catch (Throwable e) {
Iris.reportError(e);
Iris.error("EngineSVC: Failed to trim for " + name);
@@ -235,7 +236,7 @@ public class IrisEngineSVC implements IrisService {
try {
long unloadStart = System.currentTimeMillis();
int count = engine.getMantle().unloadTectonicPlate(IrisSettings.get().getPerformance().getEngineSVC().forceMulticoreWrite ? 0 : tectonicLimit());
int count = engine.getMantle().unloadTectonicPlate(shouldUnloadMulticore() ? 0 : tectonicLimit());
if (count > 0) {
Iris.debug(C.GOLD + "Unloaded " + C.YELLOW + count + " TectonicPlates in " + C.RED + Form.duration(System.currentTimeMillis() - unloadStart, 2));
}
@@ -252,6 +253,19 @@ public class IrisEngineSVC implements IrisService {
return tectonicLimit.get() / Math.max(worlds.size(), 1);
}
private long heapScaledKeepAliveMillis() {
long keepAlive = TimeUnit.SECONDS.toMillis(IrisSettings.get().getPerformance().getMantleKeepAlive());
return Math.round(keepAlive * (1D - currentReclaimUrgency()));
}
private boolean shouldUnloadMulticore() {
return IrisSettings.get().getPerformance().getEngineSVC().forceMulticoreWrite || currentReclaimUrgency() >= 1D;
}
private double currentReclaimUrgency() {
return MantleHeapPressure.reclaimUrgency(MantleHeapPressure.usedFraction());
}
@Synchronized
private void close() {
if (closed) return;
@@ -21,6 +21,7 @@ package com.volmit.iris.util.mantle;
import com.google.common.util.concurrent.AtomicDouble;
import com.volmit.iris.Iris;
import com.volmit.iris.core.IrisSettings;
import com.volmit.iris.core.pregenerator.MantleHeapPressure;
import com.volmit.iris.core.tools.IrisToolbelt;
import com.volmit.iris.engine.data.cache.Cache;
import com.volmit.iris.engine.framework.Engine;
@@ -76,6 +77,7 @@ public class Mantle {
private final IOWorker worker;
private final AtomicDouble adjustedIdleDuration;
private final KSet<Long> toUnload;
private final KMap<Long, CompletableFuture<TectonicPlate>> loadingRegions;
/**
* Create a new mantle
@@ -96,6 +98,7 @@ public class Mantle {
ioBurst = MultiBurst.ioBurst;
adjustedIdleDuration = new AtomicDouble(0);
toUnload = new KSet<>();
loadingRegions = new KMap<>();
worker = new IOWorker(dataFolder, worldHeight);
Iris.debug("Opened The Mantle " + C.DARK_AQUA + dataFolder.getAbsolutePath());
}
@@ -484,9 +487,10 @@ public class Mantle {
}
double idleDuration = baseIdleDuration;
double minimumIdleDuration = Math.max(500D, Math.min(baseIdleDuration, 4000D));
if (loadedRegions.size() > tectonicLimit) {
// todo update this correctly and maybe do something when its above a 100%
idleDuration = Math.max(idleDuration - (1000 * (((loadedRegions.size() - tectonicLimit) / (double) tectonicLimit) * 100) * 0.4), 4000);
idleDuration = Math.max(idleDuration - (1000 * (((loadedRegions.size() - tectonicLimit) / (double) tectonicLimit) * 100) * 0.4), minimumIdleDuration);
}
adjustedIdleDuration.set(idleDuration);
@@ -616,6 +620,7 @@ public class Mantle {
if (unload) ioTectonicUnload.release();
}
ensureRetryable(x, z);
Iris.warn("Retrying to get " + x + " " + z + " Mantle Region");
return get(x, z);
}
@@ -644,6 +649,7 @@ public class Mantle {
.thenCompose(p -> {
release.apply(p);
if (p != null) return CompletableFuture.completedFuture(p);
ensureRetryable(x, z);
Iris.warn("Retrying to get " + x + " " + z + " Mantle Region");
return getFuture(x, z);
});
@@ -667,9 +673,16 @@ public class Mantle {
return fallback.get();
}
private void ensureRetryable(int x, int z) {
if (closed.get() || Thread.currentThread().isInterrupted()) {
throw new IllegalStateException("Stopped retrying Tectonic Plate " + x + " " + z + " because the Mantle closed or the thread was interrupted");
}
}
/**
* This retreives a future of the Tectonic Plate at the given coordinates.
* All methods accessing tectonic plates should go through this method
* All methods accessing tectonic plates should go through this method. Concurrent requests for a plate that is
* still loading share one load instead of each occupying an IO thread while they wait on the plate's lock.
*
* @param x the region x
* @param z the region z
@@ -677,7 +690,31 @@ public class Mantle {
*/
@RegionCoordinates
protected CompletableFuture<TectonicPlate> getSafe(int x, int z) {
return ioBurst.completableFuture(() -> hyperLock.withResult(x, z, () -> {
Long key = key(x, z);
CompletableFuture<TectonicPlate> published = new CompletableFuture<>();
CompletableFuture<TectonicPlate> existing = loadingRegions.putIfAbsent(key, published);
if (existing != null) {
return existing;
}
try {
loadRegion(x, z).whenComplete((region, failure) -> {
loadingRegions.remove(key, published);
if (failure == null) {
published.complete(region);
} else {
published.completeExceptionally(failure);
}
});
} catch (Throwable failure) {
loadingRegions.remove(key, published);
published.completeExceptionally(failure);
}
return published;
}
private CompletableFuture<TectonicPlate> loadRegion(int x, int z) {
return CompletableFuture.supplyAsync(() -> hyperLock.withResult(x, z, () -> {
Long k = key(x, z);
use(k);
TectonicPlate r = loadedRegions.get(k);
@@ -719,7 +756,7 @@ public class Mantle {
Iris.debug("Created new Tectonic Plate " + C.DARK_GREEN + x + " " + z);
use(k);
return region;
}));
}), ioBurst);
}
private void use(Long key) {
@@ -768,6 +805,8 @@ public class Mantle {
}
public boolean shouldReduce(Engine engine) {
return !engine.isStudio() || IrisSettings.get().getPerformance().isTrimMantleInStudio();
return !engine.isStudio()
|| IrisSettings.get().getPerformance().isTrimMantleInStudio()
|| MantleHeapPressure.overHighWater();
}
}