Pregen failure handling

- Chunk generator reports generation failures to the running pregen job
- Pregen methods forward generation failures to the async method
- Async pregen fails every outstanding request through an in-flight ledger
- In-flight requests settle once and ignore late completions
- Pregen aborts with the failing chunk as its cause
- Admission drain waits with a 60 second deadline
- Closing the admission gate fails leftover requests
- Tests cover failure propagation, the ledger and the drain deadline

core/src/main/java/art/arcane/iris/platform/generation/BukkitChunkGenerator.java

@@ -30,6 +30,7 @@ import java.util.concurrent.Semaphore;
 import art.arcane.iris.spi.IrisServices;
 import art.arcane.iris.platform.bukkit.BukkitPlatform;
 import art.arcane.iris.configuration.IrisSettings;
+import art.arcane.iris.studio.view.PregeneratorJob;
 import art.arcane.iris.world.IrisWorlds;
 import art.arcane.iris.pack.loading.IrisData;
 import art.arcane.iris.platform.bukkit.nms.INMS;
@@ -1290,6 +1291,15 @@ public class BukkitChunkGenerator extends ChunkGenerator implements PlatformChun
 
     @Override
     public void generateNoise(@NotNull WorldInfo world, @NotNull Random random, int x, int z, @NotNull ChunkGenerator.ChunkData d) {
+        try {
+            generateNoiseChunk(world, random, x, z, d);
+        } catch (Throwable e) {
+            PregeneratorJob.reportChunkGenerationFailure(this.world.identity(), x, z, e);
+            throw e;
+        }
+    }
+
+    private void generateNoiseChunk(WorldInfo world, Random random, int x, int z, ChunkGenerator.ChunkData d) {
         if (closing) {
             throw new IllegalStateException("Iris chunk generation was rejected while the generator is closing.");
         }

core/src/main/java/art/arcane/iris/studio/view/PregeneratorJob.java

@@ -254,6 +254,14 @@ public class PregeneratorJob implements PregenListener, PregenRenderSource {
         return instance.get();
     }
 
+    public static void reportChunkGenerationFailure(String worldIdentity, int chunkX, int chunkZ, Throwable failure) {
+        PregeneratorJob inst = instance.get();
+        if (inst == null || worldIdentity == null || !worldIdentity.equals(inst.worldIdentity())) {
+            return;
+        }
+        inst.pregenerator.onChunkGenerationFailed(chunkX, chunkZ, failure);
+    }
+
     public static boolean pauseResume() {
         PregeneratorJob inst = instance.get();
         return inst != null && inst.togglePause();

core/src/main/java/art/arcane/iris/world/pregen/AsyncOrMedievalPregenMethod.java

@@ -85,6 +85,11 @@ public class AsyncOrMedievalPregenMethod implements PregeneratorMethod {
         method.onRegionBounds(minRegionX, minRegionZ, maxRegionX, maxRegionZ);
     }
 
+    @Override
+    public void onChunkGenerationFailed(int chunkX, int chunkZ, Throwable failure) {
+        method.onChunkGenerationFailed(chunkX, chunkZ, failure);
+    }
+
     @Override
     public void onPregenStart(int centerBlockX, int centerBlockZ) {
         method.onPregenStart(centerBlockX, centerBlockZ);

core/src/main/java/art/arcane/iris/world/pregen/AsyncPregenMethod.java

@@ -21,6 +21,7 @@ package art.arcane.iris.world.pregen;
 import art.arcane.iris.spi.IrisLogging;
 import art.arcane.iris.world.lifecycle.WorldLifecycleService;
 import art.arcane.volmlib.nativelib.terrain.NativeWorkerPool;
+import java.util.Map;
 import java.util.Objects;
 import art.arcane.iris.world.IrisPaperLikeBackendMode;
 import art.arcane.iris.world.IrisRuntimeSchedulerMode;
@@ -55,6 +56,7 @@ import java.util.concurrent.TimeoutException;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.atomic.AtomicReference;
 
 public class AsyncPregenMethod implements PregeneratorMethod {
     // THREAD_COUNT records the pool's original size while boosted; BOOST_HOLDERS counts live
@@ -64,7 +66,8 @@ public class AsyncPregenMethod implements PregeneratorMethod {
     private static final AtomicInteger BOOST_HOLDERS = new AtomicInteger();
     private static final int ADAPTIVE_SLOW_REQUEST_STEP = 3;
     private static final int ADAPTIVE_RECOVERY_INTERVAL = 8;
-    private static final long CLOSE_DRAIN_WARNING_SECONDS = 60L;
+    private static final long CLOSE_DRAIN_WARNING_SECONDS = 20L;
+    private static final long CLOSE_DRAIN_TIMEOUT_SECONDS = 60L;
     private static final long FLUSH_TIMEOUT_SECONDS = 120L;
     private static final long MANTLE_CLEANUP_DRAIN_SECONDS = 60L;
     private static final long CHUNK_FLUSH_DRAIN_SECONDS = 30L;
@@ -113,6 +116,8 @@ public class AsyncPregenMethod implements PregeneratorMethod {
     private final AtomicLong failed = new AtomicLong();
     private final AtomicLong lastProgressAt = new AtomicLong(M.ms());
     private final AtomicBoolean closing = new AtomicBoolean();
+    private final PregenInFlightRequests inFlightRequests = new PregenInFlightRequests();
+    private final AtomicReference<GenerationFailure> generationFailure = new AtomicReference<>();
     private final AtomicBoolean holdsWorkerBoost = new AtomicBoolean();
     private volatile Engine metricsEngine;
     private volatile Mantle cachedMantle;
@@ -449,10 +454,20 @@ public class AsyncPregenMethod implements PregeneratorMethod {
     }
 
     private void completeChunk(int x, int z, PregenListener listener, Chunk chunk, Throwable throwable) {
+        PregenListener owner = inFlightRequests.settle(PregenInFlightRequests.key(x, z));
+        if (owner == null) {
+            return;
+        }
+        settleChunk(x, z, owner, chunk, throwable, true);
+    }
+
+    private void settleChunk(int x, int z, PregenListener listener, Chunk chunk, Throwable throwable, boolean reportFailure) {
         boolean success = false;
         try {
             if (throwable != null) {
-                onChunkFutureFailure(x, z, throwable);
+                if (reportFailure) {
+                    onChunkFutureFailure(x, z, throwable);
+                }
                 onChunkFailedToLoad(x, z);
                 listener.onChunkFailed(x, z);
             } else if (chunk == null) {
@@ -652,11 +667,57 @@ public class AsyncPregenMethod implements PregeneratorMethod {
                 + " stalledForMs=" + stalledFor;
     }
 
-    private void markSubmitted() {
+    private void markSubmitted(int x, int z, PregenListener listener) {
+        inFlightRequests.add(PregenInFlightRequests.key(x, z), listener);
         submitted.incrementAndGet();
         inFlight.incrementAndGet();
     }
 
+    @Override
+    public void onChunkGenerationFailed(int chunkX, int chunkZ, Throwable failure) {
+        GenerationFailure recorded = new GenerationFailure(chunkX, chunkZ, failure);
+        if (generationFailure.compareAndSet(null, recorded)) {
+            IrisLogging.error("Async pregen aborting: Iris failed to generate chunk " + chunkX + "," + chunkZ
+                    + " and the server's chunk system will not complete the " + inFlightRequests.size()
+                    + " outstanding chunk request(s). " + metricsSnapshot());
+        }
+        failOutstandingRequests(failure);
+    }
+
+    private void failOutstandingRequests(Throwable failure) {
+        Map<Long, PregenListener> outstanding = inFlightRequests.drain();
+        for (Map.Entry<Long, PregenListener> entry : outstanding.entrySet()) {
+            long key = entry.getKey();
+            settleChunk(PregenInFlightRequests.chunkX(key), PregenInFlightRequests.chunkZ(key), entry.getValue(),
+                    null, failure, false);
+        }
+        admission.wake();
+        backpressure.signalProgress();
+    }
+
+    private void throwIfGenerationFailed() {
+        GenerationFailure failure = generationFailure.get();
+        if (failure != null) {
+            throw new IllegalStateException("Pregen aborted: Iris failed to generate chunk " + failure.chunkX() + ","
+                    + failure.chunkZ() + " in world " + world.getName()
+                    + " and the server's chunk system stopped completing requests.", failure.cause());
+        }
+    }
+
+    private void abandonOutstandingRequests(PregenAdmissionGate.Drain drain) {
+        Map<Long, PregenListener> abandoned = inFlightRequests.drain();
+        IrisLogging.error("Async pregen abandoned " + abandoned.size() + " chunk request(s) that did not complete within "
+                + CLOSE_DRAIN_TIMEOUT_SECONDS + "s of closing (" + drain.outstanding()
+                + " permit(s) still held). They are not cached and regenerate on the next run. " + metricsSnapshot());
+        IllegalStateException cause = new IllegalStateException(
+                "Async chunk request never completed before the pregen closed in world " + world.getName() + ".");
+        for (Map.Entry<Long, PregenListener> entry : abandoned.entrySet()) {
+            long key = entry.getKey();
+            settleChunk(PregenInFlightRequests.chunkX(key), PregenInFlightRequests.chunkZ(key), entry.getValue(),
+                    null, cause, false);
+        }
+    }
+
     private void markFinished(boolean success) {
         if (success) {
             completed.incrementAndGet();
@@ -774,11 +835,17 @@ public class AsyncPregenMethod implements PregeneratorMethod {
         // A stop request interrupts the pregen worker; shield the drain and flush so chunks still hit disk.
         boolean interrupted = Thread.interrupted();
         try {
-            interrupted |= admission.awaitDrain(
+            PregenAdmissionGate.Drain drain = admission.awaitDrain(
                     CLOSE_DRAIN_WARNING_SECONDS,
                     TimeUnit.SECONDS,
+                    CLOSE_DRAIN_TIMEOUT_SECONDS,
+                    TimeUnit.SECONDS,
                     () -> IrisLogging.warn("Async pregen is still draining outstanding chunks. " + metricsSnapshot())
             );
+            interrupted |= drain.interrupted();
+            if (!drain.drained()) {
+                abandonOutstandingRequests(drain);
+            }
 
             mantleCleanup.close(MANTLE_CLEANUP_DRAIN_SECONDS, TimeUnit.SECONDS);
             flushAllRemainingChunks();
@@ -808,7 +875,7 @@ public class AsyncPregenMethod implements PregeneratorMethod {
     }
 
     private boolean isCancelled() {
-        return closing.get() || Thread.currentThread().isInterrupted();
+        return closing.get() || generationFailure.get() != null || Thread.currentThread().isInterrupted();
     }
 
     @Override
@@ -835,6 +902,7 @@ public class AsyncPregenMethod implements PregeneratorMethod {
 
     @Override
     public void generateChunk(int x, int z, PregenListener listener) {
+        throwIfGenerationFailed();
         listener.onChunkGenerating(x, z);
         if (MantleHeapPressure.overHighWater()) {
             reclaimMemory();
@@ -865,9 +933,14 @@ public class AsyncPregenMethod implements PregeneratorMethod {
             recordPermitWait(waited.permitMs());
         }
 
+        throwIfGenerationFailed();
         regionPending.computeIfAbsent(rkey(x >> 5, z >> 5), k -> new AtomicInteger(1)).incrementAndGet();
-        markSubmitted();
+        markSubmitted(x, z, listener);
         executor.generate(x, z, listener);
+        GenerationFailure failure = generationFailure.get();
+        if (failure != null) {
+            failOutstandingRequests(failure.cause());
+        }
     }
 
     private void reclaimHeapPressure() {
@@ -1104,6 +1177,9 @@ public class AsyncPregenMethod implements PregeneratorMethod {
         }
     }
 
+    private record GenerationFailure(int chunkX, int chunkZ, Throwable cause) {
+    }
+
     private record ChunkAsyncMethodSelection(Method urgentMethod, Method standardMethod, String mode) {
     }
 }

core/src/main/java/art/arcane/iris/world/pregen/CachedPregenMethod.java

@@ -189,6 +189,11 @@ public class CachedPregenMethod implements PregeneratorMethod {
         method.onRegionBounds(minRegionX, minRegionZ, maxRegionX, maxRegionZ);
     }
 
+    @Override
+    public void onChunkGenerationFailed(int chunkX, int chunkZ, Throwable failure) {
+        method.onChunkGenerationFailed(chunkX, chunkZ, failure);
+    }
+
     @Override
     public void onPregenStart(int centerBlockX, int centerBlockZ) {
         method.onPregenStart(centerBlockX, centerBlockZ);

core/src/main/java/art/arcane/iris/world/pregen/HybridPregenMethod.java

@@ -78,6 +78,11 @@ public class HybridPregenMethod implements PregeneratorMethod {
         inWorld.onRegionBounds(minRegionX, minRegionZ, maxRegionX, maxRegionZ);
     }
 
+    @Override
+    public void onChunkGenerationFailed(int chunkX, int chunkZ, Throwable failure) {
+        inWorld.onChunkGenerationFailed(chunkX, chunkZ, failure);
+    }
+
     @Override
     public void onPregenStart(int centerBlockX, int centerBlockZ) {
         inWorld.onPregenStart(centerBlockX, centerBlockZ);

core/src/main/java/art/arcane/iris/world/pregen/IrisPregenerator.java

@@ -215,6 +215,10 @@ public class IrisPregenerator {
         signalState();
     }
 
+    public void onChunkGenerationFailed(int chunkX, int chunkZ, Throwable failure) {
+        generator.onChunkGenerationFailed(chunkX, chunkZ, failure);
+    }
+
     public void start() {
         PrecisionStopwatch p = PrecisionStopwatch.start();
         boolean completed = false;

core/src/main/java/art/arcane/iris/world/pregen/PregenAdmissionGate.java

@@ -111,14 +111,20 @@ public final class PregenAdmissionGate {
         }
     }
 
-    public boolean awaitDrain(long warningInterval, TimeUnit timeUnit, Runnable onWait) {
+    public Drain awaitDrain(long warningInterval, TimeUnit warningUnit, long deadline, TimeUnit deadlineUnit, Runnable onWait) {
+        long warningNanos = warningUnit.toNanos(warningInterval);
+        long deadlineAt = System.nanoTime() + deadlineUnit.toNanos(deadline);
         boolean interrupted = false;
         lock.lock();
         try {
             while (free < permits) {
+                long remaining = deadlineAt - System.nanoTime();
+                if (remaining <= 0L) {
+                    return new Drain(false, interrupted, permits - free);
+                }
                 boolean signalled = false;
                 try {
-                    signalled = changed.await(warningInterval, timeUnit);
+                    signalled = changed.awaitNanos(Math.min(warningNanos, remaining)) > 0L;
                 } catch (InterruptedException e) {
                     interrupted = true;
                     continue;
@@ -127,10 +133,12 @@ public final class PregenAdmissionGate {
                     onWait.run();
                 }
             }
+            return new Drain(true, interrupted, 0);
         } finally {
             lock.unlock();
         }
+    }
 
-        return interrupted;
+    public record Drain(boolean drained, boolean interrupted, int outstanding) {
     }
 }

core/src/main/java/art/arcane/iris/world/pregen/PregenInFlightRequests.java

@@ -0,0 +1,47 @@
+package art.arcane.iris.world.pregen;
+
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Objects;
+import java.util.concurrent.ConcurrentHashMap;
+
+public final class PregenInFlightRequests {
+    private final ConcurrentHashMap<Long, PregenListener> requests = new ConcurrentHashMap<>();
+
+    public static long key(int chunkX, int chunkZ) {
+        return ((long) chunkX << 32) | (chunkZ & 0xFFFFFFFFL);
+    }
+
+    public static int chunkX(long key) {
+        return (int) (key >> 32);
+    }
+
+    public static int chunkZ(long key) {
+        return (int) key;
+    }
+
+    public boolean add(long key, PregenListener listener) {
+        return requests.putIfAbsent(key, Objects.requireNonNull(listener, "listener")) == null;
+    }
+
+    public PregenListener settle(long key) {
+        return requests.remove(key);
+    }
+
+    public Map<Long, PregenListener> drain() {
+        HashMap<Long, PregenListener> drained = new HashMap<>();
+        Iterator<Map.Entry<Long, PregenListener>> iterator = requests.entrySet().iterator();
+        while (iterator.hasNext()) {
+            Map.Entry<Long, PregenListener> entry = iterator.next();
+            if (requests.remove(entry.getKey(), entry.getValue())) {
+                drained.put(entry.getKey(), entry.getValue());
+            }
+        }
+        return drained;
+    }
+
+    public int size() {
+        return requests.size();
+    }
+}

core/src/main/java/art/arcane/iris/world/pregen/PregeneratorMethod.java

@@ -83,6 +83,9 @@ public interface PregeneratorMethod {
     default void onRegionBounds(int minRegionX, int minRegionZ, int maxRegionX, int maxRegionZ) {
     }
 
+    default void onChunkGenerationFailed(int chunkX, int chunkZ, Throwable failure) {
+    }
+
     default void onPregenStart(int centerBlockX, int centerBlockZ) {
     }