Pregen failure handling

- Chunk generator reports generation failures to the running pregen job
- Pregen methods forward generation failures to the async method
- Async pregen fails every outstanding request through an in-flight ledger
- In-flight requests settle once and ignore late completions
- Pregen aborts with the failing chunk as its cause
- Admission drain waits with a 60 second deadline
- Closing the admission gate fails leftover requests
- Tests cover failure propagation, the ledger and the drain deadline

core/src/test/java/art/arcane/iris/world/pregen/AsyncPregenGenerationFailureTest.java

@@ -0,0 +1,147 @@
+package art.arcane.iris.world.pregen;
+
+import org.bukkit.Chunk;
+import org.bukkit.World;
+import org.junit.Test;
+
+import java.lang.reflect.Field;
+import java.lang.reflect.Method;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.atomic.AtomicReference;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertSame;
+import static org.junit.Assert.assertThrows;
+import static org.mockito.Mockito.CALLS_REAL_METHODS;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+public class AsyncPregenGenerationFailureTest {
+    @Test
+    public void aGenerationFailureFailsEveryOutstandingRequestAndReleasesTheirPermits() throws Exception {
+        Harness harness = Harness.withOutstanding(2);
+        IllegalStateException cause = new IllegalStateException("Accepted hydrology layer exceeds the mantle bounds");
+
+        harness.method.onChunkGenerationFailed(48, -4, cause);
+
+        assertEquals(2, harness.admission.availablePermits());
+        assertEquals(2, harness.failed.get());
+        assertEquals(0, harness.inFlight.get());
+        assertEquals(0, harness.requests.size());
+        verify(harness.listener).onChunkFailed(10, 0);
+        verify(harness.listener).onChunkFailed(11, 0);
+
+        IllegalStateException abort = assertThrows(IllegalStateException.class,
+                () -> harness.method.generateChunk(12, 0, harness.listener));
+        assertSame(cause, abort.getCause());
+        verify(harness.listener, never()).onChunkGenerating(12, 0);
+    }
+
+    @Test
+    public void aLateCompletionForASettledRequestIsIgnored() throws Exception {
+        Harness harness = Harness.withOutstanding(2);
+        harness.method.onChunkGenerationFailed(48, -4, new IllegalStateException("boom"));
+
+        harness.completeChunk(10, 0, null, null);
+        harness.completeChunk(11, 0, null, new IllegalStateException("late failure"));
+
+        assertEquals(2, harness.failed.get());
+        assertEquals(0, harness.completed.get());
+        assertEquals(0, harness.inFlight.get());
+        assertEquals(2, harness.admission.availablePermits());
+    }
+
+    @Test
+    public void closingAbandonsRequestsThatNeverCompleteAndReturnsTheirPermits() throws Exception {
+        Harness harness = Harness.withOutstanding(2);
+
+        harness.abandon(new PregenAdmissionGate.Drain(false, false, 2));
+
+        assertEquals(2, harness.admission.availablePermits());
+        assertEquals(2, harness.failed.get());
+        assertEquals(0, harness.requests.size());
+        verify(harness.listener).onChunkFailed(10, 0);
+        verify(harness.listener).onChunkFailed(11, 0);
+    }
+
+    private static final class Harness {
+        private final AsyncPregenMethod method;
+        private final PregenAdmissionGate admission;
+        private final PregenInFlightRequests requests;
+        private final PregenListener listener;
+        private final AtomicInteger inFlight;
+        private final AtomicLong failed;
+        private final AtomicLong completed;
+
+        private Harness(AsyncPregenMethod method, PregenAdmissionGate admission, PregenInFlightRequests requests,
+                        PregenListener listener, AtomicInteger inFlight, AtomicLong failed, AtomicLong completed) {
+            this.method = method;
+            this.admission = admission;
+            this.requests = requests;
+            this.listener = listener;
+            this.inFlight = inFlight;
+            this.failed = failed;
+            this.completed = completed;
+        }
+
+        static Harness withOutstanding(int outstanding) throws Exception {
+            AsyncPregenMethod method = mock(AsyncPregenMethod.class, CALLS_REAL_METHODS);
+            World world = mock(World.class);
+            when(world.getName()).thenReturn("qa");
+            PregenAdmissionGate admission = new PregenAdmissionGate(outstanding, 50L, System::currentTimeMillis);
+            PregenInFlightRequests requests = new PregenInFlightRequests();
+            PregenListener listener = mock(PregenListener.class);
+            AtomicInteger inFlight = new AtomicInteger(outstanding);
+            AtomicLong failed = new AtomicLong();
+            AtomicLong completed = new AtomicLong();
+            for (int index = 0; index < outstanding; index++) {
+                assertNotNull(admission.admit(() -> false, () -> false));
+                requests.add(PregenInFlightRequests.key(10 + index, 0), listener);
+            }
+            set(method, "world", world);
+            set(method, "admission", admission);
+            set(method, "backpressure", mock(PregenMantleBackpressure.class));
+            set(method, "mantleCleanup", mock(PregenSerialWorker.class));
+            set(method, "inFlightRequests", requests);
+            set(method, "generationFailure", new AtomicReference<>());
+            set(method, "regionPending", new ConcurrentHashMap<>());
+            set(method, "lastFailedReleaseLogAt", new AtomicLong());
+            set(method, "slowRequestWarnIntervalMs", 15_000);
+            set(method, "inFlight", inFlight);
+            set(method, "submitted", new AtomicLong(outstanding));
+            set(method, "completed", completed);
+            set(method, "failed", failed);
+            set(method, "lastProgressAt", new AtomicLong());
+            set(method, "adaptiveInFlightLimit", new AtomicInteger(outstanding));
+            set(method, "threads", outstanding);
+            set(method, "closing", new AtomicBoolean());
+            return new Harness(method, admission, requests, listener, inFlight, failed, completed);
+        }
+
+        void completeChunk(int x, int z, Chunk chunk, Throwable throwable) throws Exception {
+            Method complete = AsyncPregenMethod.class.getDeclaredMethod(
+                    "completeChunk", int.class, int.class, PregenListener.class, Chunk.class, Throwable.class);
+            complete.setAccessible(true);
+            complete.invoke(method, x, z, listener, chunk, throwable);
+        }
+
+        void abandon(PregenAdmissionGate.Drain drain) throws Exception {
+            Method abandon = AsyncPregenMethod.class.getDeclaredMethod(
+                    "abandonOutstandingRequests", PregenAdmissionGate.Drain.class);
+            abandon.setAccessible(true);
+            abandon.invoke(method, drain);
+        }
+
+        private static void set(AsyncPregenMethod method, String name, Object value) throws Exception {
+            Field field = AsyncPregenMethod.class.getDeclaredField(name);
+            field.setAccessible(true);
+            field.set(method, value);
+        }
+    }
+}

core/src/test/java/art/arcane/iris/world/pregen/AsyncPregenMethodConcurrencyCapTest.java

@@ -140,9 +140,11 @@ public class AsyncPregenMethodConcurrencyCapTest {
         AtomicInteger warnings = new AtomicInteger();
         ExecutorService executor = Executors.newSingleThreadExecutor();
         try {
-            Future<Boolean> drain = executor.submit(() -> gate.awaitDrain(
+            Future<PregenAdmissionGate.Drain> drain = executor.submit(() -> gate.awaitDrain(
                     10L,
                     TimeUnit.MILLISECONDS,
+                    5L,
+                    TimeUnit.SECONDS,
                     warnings::incrementAndGet
             ));
 
@@ -152,7 +154,9 @@ public class AsyncPregenMethodConcurrencyCapTest {
             gate.release();
             gate.release();
 
-            assertFalse(drain.get(1L, TimeUnit.SECONDS));
+            PregenAdmissionGate.Drain result = drain.get(1L, TimeUnit.SECONDS);
+            assertTrue(result.drained());
+            assertFalse(result.interrupted());
             assertTrue(warnings.get() > 0);
             assertEquals(2, gate.availablePermits());
         } finally {

core/src/test/java/art/arcane/iris/world/pregen/AsyncPregenServerSaveOwnershipTest.java

@@ -464,6 +464,8 @@ public class AsyncPregenServerSaveOwnershipTest {
             set("regionPending", pendingRegions);
             set("pendingEvictions", evictions);
             set("chunkIoExecutor", (Executor) Runnable::run);
+            set("generationFailure", new AtomicReference<>());
+            set("inFlightRequests", new PregenInFlightRequests());
             when(chunk.getX()).thenReturn(2);
             when(chunk.getZ()).thenReturn(3);
             when(binding.saveAndUnloadChunk(world, 2, 3)).thenReturn(true);

core/src/test/java/art/arcane/iris/world/pregen/AsyncPregenWorkerBoostTest.java

@@ -113,7 +113,8 @@ public class AsyncPregenWorkerBoostTest {
             AsyncPregenMethod method = mock(AsyncPregenMethod.class, CALLS_REAL_METHODS);
             PregenAdmissionGate admission = mock(PregenAdmissionGate.class);
             IllegalStateException failure = new IllegalStateException("drain failed");
-            when(admission.awaitDrain(anyLong(), any(TimeUnit.class), any(Runnable.class))).thenThrow(failure);
+            when(admission.awaitDrain(anyLong(), any(TimeUnit.class), anyLong(), any(TimeUnit.class), any(Runnable.class)))
+                    .thenThrow(failure);
             set(method, "closing", new AtomicBoolean());
             set(method, "holdsWorkerBoost", new AtomicBoolean(true));
             set(method, "admission", admission);

core/src/test/java/art/arcane/iris/world/pregen/PregenAdmissionGateTest.java

@@ -12,6 +12,7 @@ import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicLong;
 
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertTrue;
@@ -162,7 +163,8 @@ public class PregenAdmissionGateTest {
         AtomicInteger warnings = new AtomicInteger();
         ExecutorService executor = Executors.newSingleThreadExecutor();
         try {
-            Future<Boolean> drain = executor.submit(() -> gate.awaitDrain(5L, TimeUnit.MILLISECONDS, warnings::incrementAndGet));
+            Future<PregenAdmissionGate.Drain> drain = executor.submit(() -> gate.awaitDrain(
+                    5L, TimeUnit.MILLISECONDS, 5L, TimeUnit.SECONDS, warnings::incrementAndGet));
 
             while (warnings.get() < 2) {
                 Thread.onSpinWait();
@@ -170,13 +172,46 @@ public class PregenAdmissionGateTest {
             gate.release();
             gate.release();
 
-            assertEquals(Boolean.FALSE, drain.get(5, TimeUnit.SECONDS));
+            PregenAdmissionGate.Drain result = drain.get(5, TimeUnit.SECONDS);
+            assertTrue(result.drained());
+            assertFalse(result.interrupted());
+            assertEquals(0, result.outstanding());
             assertEquals(2, gate.availablePermits());
         } finally {
             executor.shutdownNow();
         }
     }
 
+    @Test
+    public void drainGivesUpAtItsDeadlineWhilePermitsStayHeld() throws Exception {
+        PregenAdmissionGate gate = new PregenAdmissionGate(2, GENEROUS_BOUND_MS, System::currentTimeMillis);
+        assertNotNull(gate.admit(() -> false, () -> false));
+        assertNotNull(gate.admit(() -> false, () -> false));
+        AtomicInteger warnings = new AtomicInteger();
+
+        PregenAdmissionGate.Drain result = gate.awaitDrain(
+                5L, TimeUnit.MILLISECONDS, 80L, TimeUnit.MILLISECONDS, warnings::incrementAndGet);
+
+        assertFalse(result.drained());
+        assertFalse(result.interrupted());
+        assertEquals(2, result.outstanding());
+        assertTrue(warnings.get() > 0);
+        assertEquals(0, gate.availablePermits());
+    }
+
+    @Test
+    public void drainReturnsAtOnceWhenNothingIsHeld() {
+        PregenAdmissionGate gate = new PregenAdmissionGate(2, GENEROUS_BOUND_MS, System::currentTimeMillis);
+        AtomicInteger warnings = new AtomicInteger();
+
+        PregenAdmissionGate.Drain result = gate.awaitDrain(
+                5L, TimeUnit.MILLISECONDS, 5L, TimeUnit.SECONDS, warnings::incrementAndGet);
+
+        assertTrue(result.drained());
+        assertEquals(0, result.outstanding());
+        assertEquals(0, warnings.get());
+    }
+
     @Test(expected = IllegalArgumentException.class)
     public void aGateWithoutPermitsIsRejected() {
         new PregenAdmissionGate(0, GENEROUS_BOUND_MS, System::currentTimeMillis);

core/src/test/java/art/arcane/iris/world/pregen/PregenInFlightRequestsTest.java

@@ -0,0 +1,62 @@
+package art.arcane.iris.world.pregen;
+
+import org.junit.Test;
+
+import java.util.Map;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertSame;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Mockito.mock;
+
+public class PregenInFlightRequestsTest {
+    @Test
+    public void settlingARequestReturnsItsListenerOnlyOnce() {
+        PregenInFlightRequests requests = new PregenInFlightRequests();
+        PregenListener listener = mock(PregenListener.class);
+        long key = PregenInFlightRequests.key(48, -4);
+
+        assertTrue(requests.add(key, listener));
+        assertEquals(1, requests.size());
+        assertSame(listener, requests.settle(key));
+        assertNull(requests.settle(key));
+        assertEquals(0, requests.size());
+    }
+
+    @Test
+    public void aDuplicateSubmissionKeepsTheOriginalRequest() {
+        PregenInFlightRequests requests = new PregenInFlightRequests();
+        PregenListener first = mock(PregenListener.class);
+        PregenListener second = mock(PregenListener.class);
+        long key = PregenInFlightRequests.key(1, 2);
+
+        assertTrue(requests.add(key, first));
+        assertFalse(requests.add(key, second));
+        assertSame(first, requests.settle(key));
+    }
+
+    @Test
+    public void drainingHandsBackEveryOutstandingRequestExactlyOnce() {
+        PregenInFlightRequests requests = new PregenInFlightRequests();
+        PregenListener listener = mock(PregenListener.class);
+        requests.add(PregenInFlightRequests.key(0, 0), listener);
+        requests.add(PregenInFlightRequests.key(1, 0), listener);
+        requests.add(PregenInFlightRequests.key(-7, 9), listener);
+
+        Map<Long, PregenListener> drained = requests.drain();
+
+        assertEquals(3, drained.size());
+        assertEquals(0, requests.size());
+        assertTrue(requests.drain().isEmpty());
+        assertNull(requests.settle(PregenInFlightRequests.key(1, 0)));
+    }
+
+    @Test
+    public void keysDistinguishNegativeCoordinates() {
+        assertTrue(PregenInFlightRequests.key(1, -1) != PregenInFlightRequests.key(-1, 1));
+        assertTrue(PregenInFlightRequests.key(0, -1) != PregenInFlightRequests.key(-1, 0));
+        assertEquals(PregenInFlightRequests.key(48, -4), PregenInFlightRequests.key(48, -4));
+    }
+}