From d454f25f4dd27965e79c6725ad3c40c18fa56180 Mon Sep 17 00:00:00 2001 From: Brian Neumann-Fopiano Date: Sun, 6 Sep 2026 04:17:40 -0400 Subject: [PATCH] Retained failed hotload complexes - Retain unpublished complex assemblies when cleanup fails - Keep published replacement runtimes owned after activation failure - Cover failed cleanup retries and successful hotload rollback --- .../arcane/iris/engine/EngineHotloader.java | 29 ++-- .../iris/engine/EngineShutdownDrainTest.java | 131 ++++++++++++++++++ 2 files changed, 148 insertions(+), 12 deletions(-) diff --git a/core/src/main/java/art/arcane/iris/engine/EngineHotloader.java b/core/src/main/java/art/arcane/iris/engine/EngineHotloader.java index f83a08557..baeff05cf 100644 --- a/core/src/main/java/art/arcane/iris/engine/EngineHotloader.java +++ b/core/src/main/java/art/arcane/iris/engine/EngineHotloader.java @@ -68,11 +68,12 @@ final class EngineHotloader { engine.lifecycleState = LifecycleState.HOTLOADING; EngineRuntime previous = engine.runtime; GenerationRuntime previousGeneration = previous.generation(); - IrisComplex nextComplex = null; + RuntimeAssembly assembly = null; + boolean published = false; try { engine.sealForTransition("complex hotload", false); engine.prepareRuntimeHotload(); - RuntimeAssembly assembly = new RuntimeAssembly( + assembly = new RuntimeAssembly( RuntimeAssembly.nextRuntimeId(), previousGeneration.target(), previousGeneration.mantleStorageDirectory(), @@ -82,7 +83,6 @@ final class EngineHotloader { EngineRuntime next; try (IrisContext.Scope ignored = IrisContext.open(engine, engine.getGenerationSessions().currentSessionId(), null)) { assembly.complex = assembly.runtimeKernel.createComplex(engine, assembly.transitionPlan); - nextComplex = assembly.complex; assembly.dimensionStackContext = assembly.runtimeKernel.createDimensionStackContext(engine); assembly.upperContext = assembly.runtimeKernel.createUpperContext(engine); BiomeMaxes biomeMaxes = engine.runtimeBuilder.computeBiomeMaxes(); @@ -98,25 +98,30 @@ final class EngineHotloader { } Throwable retirementFailure = runCleanup(null, previousGeneration.complex()::close); if (retirementFailure != null) { - retirementFailure = runCleanup(retirementFailure, nextComplex::close); - nextComplex = null; engine.lifecycleState = LifecycleState.FAILED; throw new IllegalStateException("Failed to retire the previous Iris biome complex.", retirementFailure); } engine.runtime = next; + published = true; engine.getGenerationSessions().activateNextSession(); engine.lifecycleState = LifecycleState.RUNNING; engine.getClosing().set(false); engine.backgroundTasks.openBackgroundTaskAdmission(); } catch (Throwable e) { - if (nextComplex != null && nextComplex != previousGeneration.complex()) { - Throwable cleanupFailure = runCleanup(null, nextComplex::close); - if (cleanupFailure != null) { - e.addSuppressed(cleanupFailure); + if (published) { + engine.lifecycleState = LifecycleState.FAILED; + engine.getClosing().set(true); + engine.backgroundTasks.closeBackgroundTaskAdmission(); + } else { + if (assembly != null && assembly.complex != previousGeneration.complex()) { + Throwable cleanupFailure = engine.shutdownSequence.closeAssembly(assembly, null); + if (cleanupFailure != null) { + e.addSuppressed(cleanupFailure); + } + } + if (engine.lifecycleState != LifecycleState.FAILED) { + engine.runtimeBuilder.restoreRuntimeAfterFailedTransition(previous); } - } - if (engine.lifecycleState != LifecycleState.FAILED) { - engine.runtimeBuilder.restoreRuntimeAfterFailedTransition(previous); } throw new IllegalStateException("Failed to rebuild the Iris biome complex.", e); } diff --git a/core/src/test/java/art/arcane/iris/engine/EngineShutdownDrainTest.java b/core/src/test/java/art/arcane/iris/engine/EngineShutdownDrainTest.java index 2c82e62c3..d44af2da6 100644 --- a/core/src/test/java/art/arcane/iris/engine/EngineShutdownDrainTest.java +++ b/core/src/test/java/art/arcane/iris/engine/EngineShutdownDrainTest.java @@ -11,6 +11,7 @@ import art.arcane.iris.engine.framework.EngineWorldManager; import art.arcane.iris.engine.framework.GenerationSessionException; import art.arcane.iris.engine.framework.GenerationSessionManager; import art.arcane.iris.engine.framework.NativeStructureOwnershipStore; +import art.arcane.iris.engine.history.GenerationKernelRegistry; import art.arcane.iris.engine.object.IrisWorld; import art.arcane.iris.engine.framework.EngineMode; import art.arcane.iris.engine.mantle.EngineMantle; @@ -25,6 +26,7 @@ import org.mockito.InOrder; import org.mockito.MockedStatic; import java.lang.reflect.Field; +import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicBoolean; @@ -35,12 +37,18 @@ import static org.junit.Assert.assertSame; import static org.junit.Assert.assertTrue; import static org.junit.Assert.assertThrows; import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.same; +import static org.mockito.ArgumentMatchers.isNull; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.inOrder; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.mockStatic; +import static org.mockito.Mockito.spy; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.never; import static org.mockito.Mockito.times; @@ -298,6 +306,114 @@ public class EngineShutdownDrainTest { verify(fixture.target).close(); } + @Test + public void failedComplexRetirementRetainsReplacementBeforeSharedStorageCanClose() throws Exception { + ShutdownFixture fixture = new ShutdownFixture(); + IrisComplex replacement = mock(IrisComplex.class); + fixture.prepareComplexHotload(replacement); + doThrow(new IllegalStateException("Previous planner still active")).when(fixture.complex).close(); + doThrow(new IllegalStateException("Replacement planner still active")).when(replacement).close(); + + assertThrows(IllegalStateException.class, () -> new EngineHotloader(fixture.engine).hotloadComplex()); + + assertSame(fixture.runtime, fixture.engine.runtime); + assertSame(IrisEngine.LifecycleState.FAILED, fixture.engine.lifecycleState); + doNothing().when(fixture.complex).close(); + try (MockedStatic ignored = mockStatic(NativeStructureOwnershipStore.class)) { + assertThrows(IllegalStateException.class, fixture.shutdown::close); + verify(replacement, times(2)).close(); + verify(fixture.mantle, never()).close(); + verify(fixture.target, never()).close(); + doNothing().when(replacement).close(); + fixture.shutdown.close(); + } + assertTrue(fixture.engine.closed); + verify(replacement, times(3)).close(); + verify(fixture.mantle).close(); + } + + @Test + public void failedComplexBuildDrainBlocksRollbackUntilShutdownRetry() throws Exception { + ShutdownFixture fixture = new ShutdownFixture(); + IrisComplex replacement = mock(IrisComplex.class); + GenerationKernelRegistry.RuntimeKernel kernel = fixture.prepareComplexHotload(replacement); + IllegalStateException buildFailure = new IllegalStateException("Dimension stack failed"); + when(kernel.createDimensionStackContext(fixture.engine)).thenThrow(buildFailure); + IllegalStateException drainFailure = new IllegalStateException("Replacement planner still active"); + doThrow(drainFailure).when(replacement).close(); + + IllegalStateException failure = assertThrows(IllegalStateException.class, + () -> new EngineHotloader(fixture.engine).hotloadComplex()); + + assertSame(buildFailure, failure.getCause()); + assertSame(drainFailure, buildFailure.getSuppressed()[0]); + assertSame(IrisEngine.LifecycleState.FAILED, fixture.engine.lifecycleState); + assertSame(fixture.runtime, fixture.engine.runtime); + verify(fixture.sessions, never()).activateNextSession(); + verify(fixture.complex, never()).close(); + doNothing().when(replacement).close(); + try (MockedStatic ignored = mockStatic(NativeStructureOwnershipStore.class)) { + fixture.shutdown.close(); + } + assertTrue(fixture.engine.closed); + verify(replacement, times(2)).close(); + verify(fixture.mantle).close(); + } + + @Test + public void successfullyDrainedFailedComplexBuildRestoresPreviousRuntime() throws Exception { + ShutdownFixture fixture = new ShutdownFixture(); + IrisComplex replacement = mock(IrisComplex.class); + GenerationKernelRegistry.RuntimeKernel kernel = fixture.prepareComplexHotload(replacement); + when(kernel.createDimensionStackContext(fixture.engine)) + .thenThrow(new IllegalStateException("Dimension stack failed")); + + assertThrows(IllegalStateException.class, () -> new EngineHotloader(fixture.engine).hotloadComplex()); + + assertSame(fixture.runtime, fixture.engine.runtime); + assertSame(IrisEngine.LifecycleState.RUNNING, fixture.engine.lifecycleState); + verify(replacement).close(); + verify(fixture.complex, never()).close(); + verify(fixture.mantle, never()).close(); + verify(fixture.sessions).activateNextSession(); + } + + @Test + public void successfulComplexHotloadPublishesReplacementWithoutClosingIt() throws Exception { + ShutdownFixture fixture = new ShutdownFixture(); + IrisComplex replacement = mock(IrisComplex.class); + fixture.prepareComplexHotload(replacement); + + new EngineHotloader(fixture.engine).hotloadComplex(); + + assertSame(replacement, fixture.engine.runtime.generation().complex()); + assertSame(IrisEngine.LifecycleState.RUNNING, fixture.engine.lifecycleState); + verify(fixture.complex).close(); + verify(replacement, never()).close(); + verify(fixture.mantle, never()).close(); + } + + @Test + public void failedComplexActivationKeepsPublishedReplacementOwnedForShutdown() throws Exception { + ShutdownFixture fixture = new ShutdownFixture(); + IrisComplex replacement = mock(IrisComplex.class); + fixture.prepareComplexHotload(replacement); + doThrow(new IllegalStateException("Session activation failed")).when(fixture.sessions).activateNextSession(); + + assertThrows(IllegalStateException.class, () -> new EngineHotloader(fixture.engine).hotloadComplex()); + + assertSame(replacement, fixture.engine.runtime.generation().complex()); + assertSame(IrisEngine.LifecycleState.FAILED, fixture.engine.lifecycleState); + assertTrue(fixture.engine.getClosing().get()); + verify(fixture.complex).close(); + verify(replacement, never()).close(); + try (MockedStatic ignored = mockStatic(NativeStructureOwnershipStore.class)) { + fixture.shutdown.close(); + } + assertTrue(fixture.engine.closed); + verify(replacement).close(); + } + private static EngineTarget target() { EngineTarget target = mock(EngineTarget.class); when(target.getData()).thenReturn(mock(IrisData.class)); @@ -360,6 +476,21 @@ public class EngineShutdownDrainTest { private final GenerationSessionManager sessions = mock(GenerationSessionManager.class); private final EngineShutdownSequence shutdown = new EngineShutdownSequence(engine); + private GenerationKernelRegistry.RuntimeKernel prepareComplexHotload(IrisComplex replacement) throws Exception { + GenerationKernelRegistry.RuntimeKernel kernel = mock(GenerationKernelRegistry.RuntimeKernel.class); + when(kernel.createComplex(engine, null)).thenReturn(replacement); + when(generation.runtimeKernel()).thenReturn(kernel); + GenerationRuntime nextGeneration = generation(target, mantle); + when(nextGeneration.complex()).thenReturn(replacement); + when(generation.withComplex(anyInt(), same(replacement), isNull(), isNull(), any())) + .thenReturn(nextGeneration); + EngineRuntimeBuilder builder = spy(new EngineRuntimeBuilder(engine)); + doReturn(new GenerationRuntime.BiomeMaxes(0D, 0D, 0D)).when(builder).computeBiomeMaxes(); + setField(engine, "runtimeBuilder", builder); + when(engine.getGenerationHistoryRuntimeRouter()).thenReturn(Optional.empty()); + return kernel; + } + private ShutdownFixture() throws Exception { EngineBackgroundTasks background = mock(EngineBackgroundTasks.class); when(background.drainBackgroundTasks(anyString()))