Modded teardown fallback

- Closes engines, pools and scheduler a previous server left open when the next server starts
- Flushes them at JVM exit once the stopping server thread has ended
- Corrects the scheduler shutdown comment

adapters/modded-common/src/main/java/art/arcane/iris/modded/ModdedEngineBootstrap.java

@@ -79,6 +79,8 @@ import art.arcane.iris.spi.IrisServices;
 import art.arcane.iris.generation.concurrent.MultiBurst;
 
 import java.util.ArrayDeque;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
 
 public final class ModdedEngineBootstrap {
     private static final String[] CORE_SELF_TEST_CLASSES = {
@@ -87,12 +89,17 @@ public final class ModdedEngineBootstrap {
         "art.arcane.iris.pack.loading.IrisData"
     };
     private static final Object LOCK = new Object();
+    private static final Object STOP_LOCK = new Object();
+    private static final AtomicBoolean EXIT_FLUSH_REGISTERED = new AtomicBoolean();
+    private static final long EXIT_FLUSH_SERVER_WAIT_SECONDS = 30L;
     private static final ModdedServiceManager UNBOUND_SERVICE_MANAGER = new ModdedServiceManager();
     private static volatile NativeModdedLoader loader;
     private static volatile BoundRuntime runtime;
     private static volatile NativeModdedServer currentServer;
     private static volatile NativeModdedServer spawnCaptureServer;
     private static volatile boolean initialSpawnWasDefault;
+    private static volatile boolean stoppedPending;
+    private static volatile Thread stoppingThread;
 
     private ModdedEngineBootstrap() {
     }
@@ -119,6 +126,8 @@ public final class ModdedEngineBootstrap {
     }
 
     public static void start(NativeModdedServer server) {
+        finishPreviousServer("server start");
+        registerExitFlush();
         captureInitialSpawn(server);
         currentServer = server;
         bind();
@@ -150,6 +159,7 @@ public final class ModdedEngineBootstrap {
     }
 
     public static void serverAboutToStart(NativeModdedServer server) {
+        finishPreviousServer("server start");
         captureInitialSpawn(server);
         ModdedStartup.prepareForStartup();
     }
@@ -188,6 +198,8 @@ public final class ModdedEngineBootstrap {
 
     public static void stop() {
         NativeModdedServer stoppingServer = currentServer;
+        stoppingThread = Thread.currentThread();
+        stoppedPending = true;
         Throwable failure = null;
         failure = runStopStage(failure, "pack downloads", IrisModdedCommands::shutdownDownloads);
         failure = runStopStage(failure, "world check", () -> ModdedWorldCheck.serverStopped(stoppingServer));
@@ -206,8 +218,54 @@ public final class ModdedEngineBootstrap {
      * The loaders still drain queued chunk generation after server stopping and only close levels afterwards, so
      * engines, their mantle and generation history, and the generation pools stay open until the server stopped.
      * Engines of unloaded levels are already closed by {@link #levelUnloaded}; this closes whatever remains.
+     * Fabric fires it at the tail of stopServer, so a throw there skips it: the next server start or the JVM
+     * exit runs it instead.
      */
     public static void stopped() {
+        synchronized (STOP_LOCK) {
+            stoppedPending = false;
+            stoppingThread = null;
+            closeStoppedServer();
+        }
+    }
+
+    private static void finishPreviousServer(String trigger) {
+        synchronized (STOP_LOCK) {
+            if (!stoppedPending) {
+                return;
+            }
+            ModdedIrisLog.warn("Iris closing the engines the previous server left open ({})", trigger);
+            stopped();
+        }
+    }
+
+    private static void registerExitFlush() {
+        if (EXIT_FLUSH_REGISTERED.compareAndSet(false, true)) {
+            Runtime.getRuntime().addShutdownHook(new Thread(ModdedEngineBootstrap::flushAtExit, "Iris Exit Flush"));
+        }
+    }
+
+    private static void flushAtExit() {
+        if (!stoppedPending) {
+            return;
+        }
+        Thread serverThread = stoppingThread;
+        if (serverThread != null && serverThread != Thread.currentThread()) {
+            try {
+                serverThread.join(TimeUnit.SECONDS.toMillis(EXIT_FLUSH_SERVER_WAIT_SECONDS));
+            } catch (InterruptedException interrupted) {
+                Thread.currentThread().interrupt();
+                return;
+            }
+            if (serverThread.isAlive()) {
+                ModdedIrisLog.warn("Iris skipped its exit flush: the server thread is still stopping");
+                return;
+            }
+        }
+        finishPreviousServer("JVM exit");
+    }
+
+    private static void closeStoppedServer() {
         Throwable failure = null;
         failure = runStopStage(failure, "world engines", ModdedWorldEngines::shutdown);
         failure = runStopStage(failure, "primary world router", ModdedPrimaryWorldRouter::clear);

adapters/modded-common/src/main/java/art/arcane/iris/modded/ModdedScheduler.java

@@ -195,7 +195,8 @@ public final class ModdedScheduler implements PlatformScheduler {
         mainQueue.clear();
         delayedQueue.clear();
         mainThread = null;
-        // Shutdown stage that always runs (ModdedEngineBootstrap.stopped): release the level snapshot with it.
+        // Runs from ModdedEngineBootstrap.stopped, or its fallback at the next server start or JVM exit when the
+        // loader skipped it: release the level snapshot with it.
         ModdedServerLevels.forget();
     }
 

adapters/modded-common/src/test/java/art/arcane/iris/modded/ModdedEngineBootstrapShutdownTest.java

@@ -4,6 +4,8 @@ import art.arcane.iris.configuration.IrisSettings;
 import art.arcane.iris.generation.concurrent.MultiBurst;
 import art.arcane.iris.generation.runtime.Engine;
 import art.arcane.iris.modded.command.IrisModdedCommands;
+import art.arcane.volmlib.nativelib.minecraft26_2.modded.NativeModdedServer;
+import art.arcane.volmlib.nativelib.minecraft26_2.modded.NativeServerSpawn;
 import art.arcane.volmlib.nativelib.terrain.NativeWorld;
 import net.minecraft.SharedConstants;
 import net.minecraft.server.Bootstrap;
@@ -14,6 +16,8 @@ import org.junit.After;
 import org.junit.Before;
 import org.junit.BeforeClass;
 import org.junit.Test;
+import org.mockito.MockedConstruction;
+import org.mockito.MockedStatic;
 
 import java.util.concurrent.Callable;
 import java.util.concurrent.TimeUnit;
@@ -23,6 +27,8 @@ import static org.junit.Assert.assertNotSame;
 import static org.junit.Assert.assertSame;
 import static org.junit.Assert.assertTrue;
 import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.mockConstruction;
+import static org.mockito.Mockito.mockStatic;
 import static org.mockito.Mockito.never;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
@@ -70,6 +76,22 @@ public class ModdedEngineBootstrapShutdownTest {
         assertSame(Thread.currentThread(), MultiBurst.burst.submit(worker).get(10L, TimeUnit.SECONDS));
     }
 
+    @Test
+    public void nextServerStartClosesTheEnginesOfAServerThatNeverReachedStopped() {
+        NativeWorld level = level();
+        Engine engine = mock(Engine.class);
+        ModdedWorldEngines.installReplacement(level, engine);
+        ModdedEngineBootstrap.stop();
+
+        try (MockedStatic<ModdedStartup> startup = mockStatic(ModdedStartup.class);
+             MockedConstruction<NativeServerSpawn> spawn = mockConstruction(NativeServerSpawn.class)) {
+            ModdedEngineBootstrap.serverAboutToStart(mock(NativeModdedServer.class));
+        }
+
+        verify(engine).close();
+        assertFalse(ModdedWorldEngines.activeEngines().contains(engine));
+    }
+
     private static NativeWorld level() {
         ServerLevel serverLevel = mock(ServerLevel.class);
         ServerChunkCache chunks = mock(ServerChunkCache.class);