Isolated chunk update failures

- Treats only loading saved biome information as an engine failure
- Leaves a container without loot and warns once when its saved biome cannot be recovered
- Keeps a failed chunk unetched and retries it without stopping the rest of the level's update pass
- Ends a level's pass only when its engine or mantle is closed
- Covers the deferred, permanent and isolated update paths through updateChunk and the server tick

adapters/modded-common/src/main/java/art/arcane/iris/modded/service/ModdedChunkUpdateService.java

@@ -152,7 +152,7 @@ public final class ModdedChunkUpdateService implements ModdedTickableService {
             int centerZ = player.z() >> 4;
             for (int dx = -PLAYER_CHUNK_RADIUS; dx <= PLAYER_CHUNK_RADIUS; dx++) {
                 for (int dz = -PLAYER_CHUNK_RADIUS; dz <= PLAYER_CHUNK_RADIUS; dz++) {
-                    updateChunk(engine, level, centerX + dx, centerZ + dz);
+                    updatePassChunk(engine, level, centerX + dx, centerZ + dz);
                 }
             }
         });
@@ -160,15 +160,29 @@ public final class ModdedChunkUpdateService implements ModdedTickableService {
 
     private void updateForcedChunks(Engine engine, NativeWorld level) {
         new NativeWorldMaintenance(level).forEachForcedChunk(chunkKey ->
-                updateChunk(engine, level, (int) chunkKey, (int) (chunkKey >> 32)));
+                updatePassChunk(engine, level, (int) chunkKey, (int) (chunkKey >> 32)));
     }
 
     public void updateRegeneratedChunk(Engine engine, NativeWorld level, int chunkX, int chunkZ) {
         updateChunk(engine, level, chunkX, chunkZ, false);
     }
 
-    private void updateChunk(Engine engine, NativeWorld level, int chunkX, int chunkZ) {
+    /**
-        updateChunk(engine, level, chunkX, chunkZ, true);
+     * One chunk of a level pass. A failed chunk keeps its flags and retries on a later pass without stopping the
+     * chunks after it; only a closed engine ends the level's pass.
+     */
+    private void updatePassChunk(Engine engine, NativeWorld level, int chunkX, int chunkZ) {
+        try {
+            updateChunk(engine, level, chunkX, chunkZ, true);
+        } catch (Throwable failure) {
+            if (engine.isClosed() || engine.getMantle().getMantle().isClosed()) {
+                throw failure;
+            }
+            if (IrisLogging.errorOnce("chunk-update:" + level.name() + ":" + chunkX + "," + chunkZ + ":" + failure,
+                    "Iris chunk update failed at " + chunkX + "," + chunkZ + " in " + level.name() + ", retrying each pass: " + failure)) {
+                IrisLogging.reportError(failure);
+            }
+        }
     }
 
     private void updateChunk(Engine engine, NativeWorld level, int chunkX, int chunkZ, boolean requireNeighbors) {
@@ -370,6 +384,10 @@ public final class ModdedChunkUpdateService implements ModdedTickableService {
             }
             try {
                 ModdedLootApplier.apply(engine, level, pos, state, chunk);
+            } catch (SavedBiomeUnavailableException unavailable) {
+                GenerationFailures.rethrowEngineFailure(unavailable);
+                IrisLogging.warnOnce("chunk-update-loot:" + level.name() + ":" + unavailable.getMessage(),
+                        "Iris left the container at " + pos + " in " + level.name() + " without loot: " + unavailable.getMessage());
             } catch (Throwable e) {
                 GenerationFailures.rethrowEngineFailure(e);
                 IrisLogging.reportError(e);

adapters/modded-common/src/test/java/art/arcane/iris/modded/service/ModdedChunkUpdateServiceTest.java

@@ -2,14 +2,25 @@ package art.arcane.iris.modded.service;
 
 import art.arcane.volmlib.nativelib.minecraft26_2.modded.ModdedPlatformWorld;
 
+import art.arcane.iris.configuration.IrisSettings;
+import art.arcane.iris.generation.mantle.EngineMantle;
 import art.arcane.iris.generation.runtime.Engine;
 import art.arcane.iris.generation.runtime.EngineMetrics;
+import art.arcane.iris.modded.IrisModdedChunkGenerator;
 import art.arcane.iris.modded.ModdedLootApplier;
+import art.arcane.iris.spi.IrisLogging;
 import art.arcane.iris.world.IrisWorld;
 import art.arcane.iris.world.history.SavedBiomeUnavailableException;
 import art.arcane.iris.world.storage.matter.TileWrapper;
+import art.arcane.volmlib.nativelib.minecraft26_2.modded.NativeModdedServer;
+import art.arcane.volmlib.nativelib.minecraft26_2.modded.NativeWorldGenerators;
+import art.arcane.volmlib.nativelib.minecraft26_2.modded.NativeWorldMaintenance;
 import art.arcane.volmlib.nativelib.terrain.NativeBlockPoint;
+import art.arcane.volmlib.nativelib.terrain.NativeBlockState;
+import art.arcane.volmlib.nativelib.terrain.NativeWorld;
 import art.arcane.volmlib.util.function.Consumer4;
+import art.arcane.volmlib.util.mantle.flag.MantleFlag;
+import art.arcane.volmlib.util.mantle.runtime.Mantle;
 import art.arcane.volmlib.util.mantle.runtime.MantleChunk;
 import art.arcane.volmlib.util.mantle.runtime.MantleDataAdapter;
 import art.arcane.volmlib.util.mantle.runtime.MantleHooks;
@@ -28,11 +39,15 @@ import net.minecraft.world.level.block.state.BlockState;
 import net.minecraft.world.level.material.FlowingFluid;
 import org.junit.Test;
 import org.mockito.InOrder;
+import org.mockito.MockedConstruction;
 import org.mockito.MockedStatic;
+import org.mockito.invocation.InvocationOnMock;
 
 import java.lang.reflect.Proxy;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.LongConsumer;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
@@ -41,9 +56,13 @@ import static org.junit.Assert.assertThrows;
 import static org.junit.Assert.assertTrue;
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.ArgumentMatchers.anyInt;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.ArgumentMatchers.contains;
 import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.doAnswer;
 import static org.mockito.Mockito.inOrder;
 import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.mockConstruction;
 import static org.mockito.Mockito.mockStatic;
 import static org.mockito.Mockito.never;
 import static org.mockito.Mockito.verify;
@@ -99,7 +118,7 @@ public class ModdedChunkUpdateServiceTest {
                 "Saved biome information is loading at chunk 0,0. Try again shortly.", true);
 
         try (MockedStatic<ModdedLootApplier> loot = mockStatic(ModdedLootApplier.class)) {
-            loot.when(() -> ModdedLootApplier.apply(any(), any(), any(), any(), any())).thenAnswer(invocation -> {
+            loot.when(() -> ModdedLootApplier.apply(any(), any(), any(), any(), any())).thenAnswer((InvocationOnMock invocation) -> {
                 NativeBlockPoint position = invocation.getArgument(2);
                 if (position.x() == 2) {
                     throw loading;
@@ -118,6 +137,110 @@ public class ModdedChunkUpdateServiceTest {
         verify(chunk.slice, never()).set(2, 5, 2, null);
     }
 
+    @Test
+    public void loadingSavedBiomeLeavesTheChunkUnetchedWithoutAnErrorAndRetriesOnTheNextPass() {
+        ChestUpdateMantleChunk chunk = new ChestUpdateMantleChunk();
+        Engine engine = updateEngine(chunk);
+        NativeWorld level = chestLevel();
+        AtomicInteger loadingAttempts = new AtomicInteger();
+        ModdedChunkUpdateService service = new ModdedChunkUpdateService();
+
+        try (MockedStatic<ModdedLootApplier> loot = mockStatic(ModdedLootApplier.class);
+             MockedStatic<IrisLogging> logging = mockStatic(IrisLogging.class)) {
+            loot.when(() -> ModdedLootApplier.apply(any(), any(), any(), any(), any())).thenAnswer((InvocationOnMock invocation) -> {
+                NativeBlockPoint position = invocation.getArgument(2);
+                if (position.x() == 2 && loadingAttempts.getAndIncrement() == 0) {
+                    throw new SavedBiomeUnavailableException("Saved biome information is loading at chunk 0,0.", true);
+                }
+                return null;
+            });
+
+            service.updateRegeneratedChunk(engine, level, 0, 0);
+
+            assertFalse(chunk.isFlagged(MantleFlag.ETCHED));
+            assertFalse(chunk.isFlagged(MantleFlag.UPDATE));
+            assertFalse(chunk.deleted());
+            logging.verify(() -> IrisLogging.reportError(any(Throwable.class)), never());
+            logging.verify(() -> IrisLogging.reportError(anyString(), any(Throwable.class)), never());
+
+            service.updateRegeneratedChunk(engine, level, 0, 0);
+
+            assertTrue(chunk.isFlagged(MantleFlag.ETCHED));
+            assertTrue(chunk.deleted());
+        }
+    }
+
+    @Test
+    public void permanentSavedBiomeUnavailabilityCompletesThePassWithAWarning() {
+        ChestUpdateMantleChunk chunk = new ChestUpdateMantleChunk();
+        Engine engine = updateEngine(chunk);
+        NativeWorld level = chestLevel();
+        String reason = "This chunk has no exact saved Iris biome assignment for generation 3.";
+
+        try (MockedStatic<ModdedLootApplier> loot = mockStatic(ModdedLootApplier.class);
+             MockedStatic<IrisLogging> logging = mockStatic(IrisLogging.class)) {
+            loot.when(() -> ModdedLootApplier.apply(any(), any(), any(), any(), any())).thenAnswer((InvocationOnMock invocation) -> {
+                NativeBlockPoint position = invocation.getArgument(2);
+                if (position.x() == 2) {
+                    throw new SavedBiomeUnavailableException(reason, false);
+                }
+                return null;
+            });
+
+            new ModdedChunkUpdateService().updateRegeneratedChunk(engine, level, 0, 0);
+
+            assertTrue(chunk.isFlagged(MantleFlag.ETCHED));
+            assertTrue(chunk.deleted());
+            logging.verify(() -> IrisLogging.warnOnce(anyString(), contains(reason)));
+            logging.verify(() -> IrisLogging.reportError(any(Throwable.class)), never());
+            logging.verify(() -> IrisLogging.reportError(anyString(), any(Throwable.class)), never());
+        }
+    }
+
+    @Test
+    public void failedChunkDoesNotStopTheLevelPassForItsOtherChunks() {
+        IrisSettings previousSettings = IrisSettings.settings;
+        IrisSettings.settings = new IrisSettings();
+        FailingMantleChunk failing = new FailingMantleChunk();
+        ChestUpdateMantleChunk healthy = new ChestUpdateMantleChunk();
+        @SuppressWarnings("unchecked")
+        Mantle<Matter> mantle = mock(Mantle.class);
+        when(mantle.isChunkLoaded(anyInt(), anyInt())).thenReturn(true);
+        when(mantle.useChunk(0, 0)).thenAnswer((InvocationOnMock invocation) -> failing.use());
+        when(mantle.useChunk(4, 0)).thenAnswer((InvocationOnMock invocation) -> healthy.use());
+        Engine engine = engineWith(mantle);
+        NativeWorld level = chestLevel();
+        NativeModdedServer server = mock(NativeModdedServer.class);
+        when(server.worlds()).thenReturn(List.of(level));
+        IrisModdedChunkGenerator generator = mock(IrisModdedChunkGenerator.class);
+        when(generator.engineIfBound()).thenReturn(engine);
+
+        try (MockedStatic<NativeWorldGenerators> generators = mockStatic(NativeWorldGenerators.class);
+             MockedStatic<ModdedLootApplier> loot = mockStatic(ModdedLootApplier.class);
+             MockedStatic<IrisLogging> logging = mockStatic(IrisLogging.class);
+             MockedConstruction<NativeWorldMaintenance> maintenance = mockConstruction(NativeWorldMaintenance.class,
+                     (NativeWorldMaintenance mocked, MockedConstruction.Context context) -> {
+                         when(mocked.hasForcedChunks()).thenReturn(true);
+                         doAnswer((InvocationOnMock invocation) -> {
+                             LongConsumer action = invocation.getArgument(0);
+                             action.accept(0L);
+                             action.accept(4L);
+                             return null;
+                         }).when(mocked).forEachForcedChunk(any(LongConsumer.class));
+                     })) {
+            generators.when(() -> NativeWorldGenerators.find(level, IrisModdedChunkGenerator.class)).thenReturn(generator);
+
+            new ModdedChunkUpdateService().onServerTick(server);
+
+            assertFalse(failing.isFlagged(MantleFlag.ETCHED));
+            assertTrue(healthy.isFlagged(MantleFlag.ETCHED));
+            assertTrue(healthy.deleted());
+            logging.verify(() -> IrisLogging.errorOnce(anyString(), anyString()));
+        } finally {
+            IrisSettings.settings = previousSettings;
+        }
+    }
+
     @Test
     public void scansWhenPlayersArePresent() {
         assertTrue(ModdedChunkUpdateService.hasUpdateTargets(true, false));
@@ -254,6 +377,54 @@ public class ModdedChunkUpdateServiceTest {
         }
     }
 
+    private static Engine updateEngine(ChestUpdateMantleChunk chunk) {
+        @SuppressWarnings("unchecked")
+        Mantle<Matter> mantle = mock(Mantle.class);
+        when(mantle.isChunkLoaded(0, 0)).thenReturn(true);
+        when(mantle.hasFlag(0, 0, MantleFlag.ETCHED)).thenAnswer((InvocationOnMock invocation) -> chunk.isFlagged(MantleFlag.ETCHED));
+        when(mantle.useChunk(0, 0)).thenAnswer((InvocationOnMock invocation) -> chunk.use());
+        return engineWith(mantle);
+    }
+
+    private static Engine engineWith(Mantle<Matter> mantle) {
+        IrisWorld world = mock(IrisWorld.class);
+        when(world.minHeight()).thenReturn(-64);
+        EngineMantle engineMantle = mock(EngineMantle.class);
+        when(engineMantle.getMantle()).thenReturn(mantle);
+        Engine engine = mock(Engine.class);
+        when(engine.getWorld()).thenReturn(world);
+        when(engine.getMetrics()).thenReturn(new EngineMetrics(8));
+        when(engine.getMantle()).thenReturn(engineMantle);
+        return engine;
+    }
+
+    private static NativeWorld chestLevel() {
+        NativeBlockState chest = mock(NativeBlockState.class);
+        when(chest.isStorage()).thenReturn(true);
+        when(chest.isStorageChest()).thenReturn(true);
+        NativeWorld level = mock(NativeWorld.class);
+        when(level.name()).thenReturn("example:updates");
+        when(level.minHeight()).thenReturn(-64);
+        when(level.maxHeight()).thenReturn(320);
+        when(level.isChunkLoaded(anyInt(), anyInt())).thenReturn(true);
+        when(level.getBlock(anyInt(), anyInt(), anyInt())).thenReturn(chest);
+        return level;
+    }
+
+    private static final class FailingMantleChunk extends MantleChunk<Matter> {
+        private FailingMantleChunk() {
+            super(1, 0, 0, emptyAdapter(), MantleHooks.NONE);
+        }
+
+        @Override
+        public <T> void iterate(
+                Class<T> type,
+                Consumer4<Integer, Integer, Integer, T> iterator
+        ) {
+            throw new IllegalStateException("corrupt " + type.getSimpleName() + " slice");
+        }
+    }
+
     @SuppressWarnings("unchecked")
     private static MantleDataAdapter<Matter> emptyAdapter() {
         return (MantleDataAdapter<Matter>) Proxy.newProxyInstance(

core/src/main/java/art/arcane/iris/generation/runtime/GenerationFailures.java

@@ -31,7 +31,7 @@ public final class GenerationFailures {
                     || current instanceof GenerationClosedException
                     || current instanceof MantleClosedException
                     || current instanceof WrongEngineBroException
-                    || current instanceof SavedBiomeUnavailableException
+                    || current instanceof SavedBiomeUnavailableException unavailable && unavailable.isLoading()
                     || current instanceof InterruptedException
                     || current instanceof InterruptedIOException
                     || current instanceof ClosedChannelException

core/src/test/java/art/arcane/iris/generation/runtime/GenerationFailuresTest.java

@@ -34,6 +34,14 @@ public class GenerationFailuresTest {
         assertFalse(GenerationFailures.isEngineFailure(new IllegalStateException("unrelated")));
     }
 
+    @Test
+    public void permanentSavedBiomeUnavailabilityIsAContentFailure() {
+        assertFalse(GenerationFailures.isEngineFailure(new SavedBiomeUnavailableException(
+                "This chunk has no exact saved Iris biome assignment for generation 3.", false)));
+        assertFalse(GenerationFailures.isEngineFailure(new SavedBiomeUnavailableException(
+                "Unable to read the saved biome at chunk 0,0.", new IllegalArgumentException("bad header"))));
+    }
+
     @Test
     public void anyFailureOnAnInterruptedThreadIsAnEngineFailure() {
         Thread.currentThread().interrupt();