Modded shutdown after levels close

- Stops downloads, protocol, pregen, tools and services at server stopping
- Keeps engines, mantle, generation history, the generation pool and burst pools open while the loader drains chunk generation
- Closes each engine when its level unloads and sweeps the rest at server stopped
- Logs a failed level unload instead of aborting the loader's level close loop

adapters/modded-common/src/main/java/art/arcane/iris/modded/ModdedEngineBootstrap.java

@@ -122,7 +122,7 @@ public final class ModdedEngineBootstrap {
         captureInitialSpawn(server);
         currentServer = server;
         bind();
-        // Pair of the stop() burst-pools stage. Load-bearing on integrated servers: once
+        // Pair of the stopped() burst-pools stage. Load-bearing on integrated servers: once
         // closed, MultiBurst falls back to a same-thread executor, so a second world load
         // without reopen() would silently run every burst inline.
         MultiBurst.burst.reopen();
@@ -180,15 +180,9 @@ public final class ModdedEngineBootstrap {
         try {
             generator.unbindEngine(world);
         } catch (Throwable exception) {
+            // Never propagate into the loader's unload loop: a throw there skips closing every later level. A
+            // runtime dimension removal retries the eviction itself and fails with the retained mapping.
             ModdedIrisLog.error("Iris engine unload failed for {}", world.name(), exception);
-            if (exception instanceof RuntimeException runtimeException) {
-                throw runtimeException;
-            }
-            if (exception instanceof Error fatalError) {
-                throw fatalError;
-            }
-            throw new IllegalStateException("Iris engine unload failed for "
-                    + world.name(), exception);
         }
     }
 
@@ -205,6 +199,16 @@ public final class ModdedEngineBootstrap {
         failure = runStopStage(failure, "studio commands", ModdedStudioCommands::clear);
         failure = runStopStage(failure, "gui host", ModdedGuiHost::clear);
         failure = runStopStage(failure, "services", () -> services().disableAll());
+        reportStopFailure("stopping", failure);
+    }
+
+    /**
+     * The loaders still drain queued chunk generation after server stopping and only close levels afterwards, so
+     * engines, their mantle and generation history, and the generation pools stay open until the server stopped.
+     * Engines of unloaded levels are already closed by {@link #levelUnloaded}; this closes whatever remains.
+     */
+    public static void stopped() {
+        Throwable failure = null;
         failure = runStopStage(failure, "world engines", ModdedWorldEngines::shutdown);
         failure = runStopStage(failure, "primary world router", ModdedPrimaryWorldRouter::clear);
         failure = runStopStage(failure, "dimension manager", ModdedDimensionManager::clear);
@@ -226,10 +230,14 @@ public final class ModdedEngineBootstrap {
             spawnCaptureServer = null;
             initialSpawnWasDefault = false;
         });
+        reportStopFailure("stopped", failure);
+    }
+
+    private static void reportStopFailure(String phase, Throwable failure) {
         if (failure != null) {
             // The shutdown path must not propagate: propagating aborts the remaining loader stop handlers and
             // can leave the level unsaved. Every stage already logged its own failure.
-            ModdedIrisLog.error("Iris modded shutdown completed with failures", failure);
+            ModdedIrisLog.error("Iris modded shutdown ({}) completed with failures", phase, failure);
         }
     }
 

adapters/modded-common/src/main/java/art/arcane/iris/modded/ModdedLifecycleCallbacks.java

@@ -16,6 +16,7 @@ public final class ModdedLifecycleCallbacks {
                 ModdedEngineBootstrap::start,
                 ModdedEngineBootstrap::serverStarted,
                 ModdedEngineBootstrap::stop,
+                ModdedEngineBootstrap::stopped,
                 ModdedEngineBootstrap::levelLoaded,
                 ModdedEngineBootstrap::levelUnloaded,
                 ModdedProtocolHandler::onPlayerJoin,

adapters/modded-common/src/main/java/art/arcane/iris/modded/ModdedScheduler.java

@@ -195,7 +195,7 @@ public final class ModdedScheduler implements PlatformScheduler {
         mainQueue.clear();
         delayedQueue.clear();
         mainThread = null;
-        // Shutdown stage that always runs (ModdedEngineBootstrap.stop): release the level snapshot with it.
+        // Shutdown stage that always runs (ModdedEngineBootstrap.stopped): release the level snapshot with it.
         ModdedServerLevels.forget();
     }
 

adapters/modded-common/src/main/java/art/arcane/iris/modded/ModdedWorldEngines.java

@@ -426,9 +426,9 @@ public final class ModdedWorldEngines {
             Engine engine = entry.getValue();
             try {
                 // Latch the generator's unloading flag BEFORE closing (unbindEngine sets it,
-                // then evicts): chunk-system drain work running after this stage would
+                // then evicts): late chunk work would otherwise see a closed engine and
-                // otherwise see a closed engine and silently rebuild a fresh engine + Mantle
+                // silently rebuild a fresh engine + Mantle that no teardown stage ever
-                // that no teardown stage ever closes, writing plates after the final save.
+                // closes, writing plates after the final save.
                 IrisModdedChunkGenerator generator = NativeWorldGenerators.find(level, IrisModdedChunkGenerator.class);
                 if (generator != null) {
                     generator.unbindEngine(level);

adapters/modded-common/src/test/java/art/arcane/iris/modded/ModdedEngineBootstrapShutdownTest.java

@@ -0,0 +1,89 @@
+package art.arcane.iris.modded;
+
+import art.arcane.iris.configuration.IrisSettings;
+import art.arcane.iris.generation.concurrent.MultiBurst;
+import art.arcane.iris.generation.runtime.Engine;
+import art.arcane.iris.modded.command.IrisModdedCommands;
+import art.arcane.volmlib.nativelib.terrain.NativeWorld;
+import net.minecraft.SharedConstants;
+import net.minecraft.server.Bootstrap;
+import net.minecraft.server.level.ServerChunkCache;
+import net.minecraft.server.level.ServerLevel;
+import net.minecraft.world.level.chunk.ChunkGenerator;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.util.concurrent.Callable;
+import java.util.concurrent.TimeUnit;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotSame;
+import static org.junit.Assert.assertSame;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+public class ModdedEngineBootstrapShutdownTest {
+    private IrisSettings previousSettings;
+
+    @BeforeClass
+    public static void bootstrap() {
+        SharedConstants.tryDetectVersion();
+        Bootstrap.bootStrap();
+    }
+
+    @Before
+    public void openRuntime() {
+        previousSettings = IrisSettings.settings;
+        IrisSettings.settings = new IrisSettings();
+        reopenBurstPools();
+    }
+
+    @After
+    public void restoreRuntime() {
+        reopenBurstPools();
+        IrisModdedCommands.openDownloadAdmission();
+        IrisSettings.settings = previousSettings;
+    }
+
+    @Test
+    public void serverStoppingKeepsEnginesAndWorkersOpenUntilServerStopped() throws Exception {
+        NativeWorld level = level();
+        Engine engine = mock(Engine.class);
+        ModdedWorldEngines.installReplacement(level, engine);
+        Callable<Thread> worker = Thread::currentThread;
+
+        ModdedEngineBootstrap.stop();
+
+        verify(engine, never()).close();
+        assertTrue(ModdedWorldEngines.activeEngines().contains(engine));
+        assertNotSame(Thread.currentThread(), MultiBurst.burst.submit(worker).get(10L, TimeUnit.SECONDS));
+
+        ModdedEngineBootstrap.stopped();
+
+        verify(engine).close();
+        assertFalse(ModdedWorldEngines.activeEngines().contains(engine));
+        assertSame(Thread.currentThread(), MultiBurst.burst.submit(worker).get(10L, TimeUnit.SECONDS));
+    }
+
+    private static NativeWorld level() {
+        ServerLevel serverLevel = mock(ServerLevel.class);
+        ServerChunkCache chunks = mock(ServerChunkCache.class);
+        when(serverLevel.getChunkSource()).thenReturn(chunks);
+        when(chunks.getGenerator()).thenReturn(mock(ChunkGenerator.class));
+        NativeWorld level = mock(NativeWorld.class);
+        when(level.nativeHandle()).thenReturn(serverLevel);
+        when(level.name()).thenReturn("example:shutdown");
+        return level;
+    }
+
+    private static void reopenBurstPools() {
+        MultiBurst.burst.reopen();
+        MultiBurst.hydrology.reopen();
+        MultiBurst.ioBurst.reopen();
+    }
+}