diff --git a/adapters/bukkit/plugin/src/main/java/art/arcane/iris/world/BukkitWorldReconciler.java b/adapters/bukkit/plugin/src/main/java/art/arcane/iris/world/BukkitWorldReconciler.java index 735f382ac..91bba32e6 100644 --- a/adapters/bukkit/plugin/src/main/java/art/arcane/iris/world/BukkitWorldReconciler.java +++ b/adapters/bukkit/plugin/src/main/java/art/arcane/iris/world/BukkitWorldReconciler.java @@ -27,6 +27,11 @@ import art.arcane.iris.world.lifecycle.BukkitWorldConfiguration; import art.arcane.iris.world.lifecycle.LifecycleOperationCoordinator; import art.arcane.iris.platform.bukkit.nms.INMS; import art.arcane.iris.generation.terrain.IrisDimension; +import art.arcane.iris.generation.runtime.IrisEngine; +import art.arcane.iris.platform.generation.BukkitChunkGenerator; +import art.arcane.iris.world.history.GenerationEpoch; +import art.arcane.iris.world.history.GenerationHistory; +import art.arcane.iris.world.history.GenerationHistoryRuntimeRouter; import art.arcane.iris.platform.bukkit.BukkitEnvironment; import art.arcane.iris.localization.C; import art.arcane.iris.platform.bootstrap.ServerProperties; @@ -182,7 +187,7 @@ public final class BukkitWorldReconciler { }); } - private CompletableFuture loadConfiguredWorld( + CompletableFuture loadConfiguredWorld( File configurationFile, String configuredWorldName, NamespacedKey worldKey, @@ -196,6 +201,28 @@ public final class BukkitWorldReconciler { return CompletableFuture.completedFuture(LoadResult.busy(worldKey, failure)); } + try { + IrisStartupValidation.requireWorldCreationReady(); + Optional loaded = backend.loadedWorld(worldKey); + if (loaded.isPresent() && backend.validateLoadedRuntime( + loaded.get(), worldKey, dimension, seed) == LoadedRuntimeStatus.READY) { + ReconciliationResult result = verifyLoadedWorld(worldKey, loaded.get(), true); + BukkitWorldConfiguration.Registration registration = BukkitWorldConfiguration.Registration.UNCHANGED; + if (result.succeeded()) { + try { + registration = BukkitWorldConfiguration.register(configurationFile, configuredWorldName, dimension, seed); + } catch (Throwable failure) { + lease.close(); + return CompletableFuture.completedFuture(LoadResult.configurationFailure(worldKey, failure)); + } + } + lease.close(); + return CompletableFuture.completedFuture(new LoadResult(result, registration, false, true, null)); + } + } catch (Throwable failure) { + lease.close(); + return CompletableFuture.completedFuture(LoadResult.dimensionFailure(worldKey, failure)); + } return loadWithLease(configurationFile, configuredWorldName, worldKey, dimension, seed, lease); } @@ -434,11 +461,19 @@ public final class BukkitWorldReconciler { boolean isIrisWorld(World world); + LoadedRuntimeStatus validateLoadedRuntime(World world, NamespacedKey worldKey, String dimension, Long seed) + throws IOException; + DimensionResolution resolveDimension(NamespacedKey worldKey); void requireDimensionLoadable(NamespacedKey worldKey, String dimension); } + enum LoadedRuntimeStatus { + READY, + UNAVAILABLE + } + public enum ReconciliationStatus { LOADED, ALREADY_LOADED, @@ -717,6 +752,55 @@ public final class BukkitWorldReconciler { return IrisToolbelt.isIrisWorld(world); } + @Override + public LoadedRuntimeStatus validateLoadedRuntime( + World world, NamespacedKey worldKey, String dimension, Long seed) throws IOException { + if (!(world.getGenerator() instanceof BukkitChunkGenerator generator) + || generator.isStudio() + || generator.isClosing() + || generator.getInitializationFailure() != null + || !generator.getStartupReady().isDone() + || generator.getStartupReady().isCompletedExceptionally() + || !(generator.getEngine() instanceof IrisEngine engine) + || engine.isClosed() || engine.isClosing()) { + return LoadedRuntimeStatus.UNAVAILABLE; + } + GenerationHistory history = generator.getGenerationHistory(); + GenerationHistoryRuntimeRouter router = engine.getGenerationHistoryRuntimeRouter().orElse(null); + if (history == null || router == null || router.history() != history) { + return LoadedRuntimeStatus.UNAVAILABLE; + } + GenerationEpoch active = history.activeEpoch(); + if (!worldKey.equals(WorldIdentity.key(world)) + || !worldKey.toString().equals(engine.getWorld().identity()) + || !dimension.equals(generator.getDimensionKey()) + || !dimension.equals(active.dimensionContract().dimensionKey()) + || !dimension.equals(engine.getDimension().getLoadKey()) + || seed != null && seed.longValue() != active.worldSeed()) { + throw new IOException("Loaded Iris runtime does not match configured world " + worldKey + "."); + } + Path expectedRoot = WorldCreatorCompat.persistentDimensionRoot(worldKey).toPath().toAbsolutePath().normalize(); + File levelRoot = IrisWorldStorage.levelRoot(); + Path storedRoot = IrisWorldStorage.requireFrozenDimensionRoot( + Bukkit.getWorldContainer(), levelRoot, configuredWorldName(worldKey), worldKey) + .toPath().toAbsolutePath().normalize(); + if (!expectedRoot.equals(storedRoot) || !storedRoot.equals(history.paths().dimensionRoot()) + || !storedRoot.equals(engine.getWorld().worldFolder().toPath().toAbsolutePath().normalize()) + || engine.getData().isClosed()) { + throw new IOException("Loaded Iris runtime storage does not match world " + worldKey + "."); + } + GenerationHistory.PackInspection inspection = GenerationHistory.inspectPacks(storedRoot); + if (!active.equals(inspection.manifest().activeEpoch()) + || !history.activeActivation().equals(inspection.manifest().activeActivation()) + || !history.pendingActivation().equals(inspection.manifest().pendingActivation()) + || !inspection.activePackRoot().equals(engine.getData().getDataFolder().toPath().toAbsolutePath().normalize()) + || !Files.isDirectory(storedRoot.resolve("iris/generation/semantics"), LinkOption.NOFOLLOW_LINKS) + || !Files.isRegularFile(storedRoot.resolve("iris/generation/semantics/index.isix"), LinkOption.NOFOLLOW_LINKS)) { + throw new IOException("Loaded Iris runtime history does not match saved world " + worldKey + "."); + } + return LoadedRuntimeStatus.READY; + } + @Override public DimensionResolution resolveDimension(NamespacedKey worldKey) { File dimensionsDirectory = new File(snapshotRoot(worldKey), "dimensions"); diff --git a/adapters/bukkit/plugin/src/main/java/art/arcane/iris/world/IrisWorldGeneratorResolver.java b/adapters/bukkit/plugin/src/main/java/art/arcane/iris/world/IrisWorldGeneratorResolver.java index 19d1a2a3d..6d7e88aae 100644 --- a/adapters/bukkit/plugin/src/main/java/art/arcane/iris/world/IrisWorldGeneratorResolver.java +++ b/adapters/bukkit/plugin/src/main/java/art/arcane/iris/world/IrisWorldGeneratorResolver.java @@ -383,8 +383,9 @@ public final class IrisWorldGeneratorResolver { IrisDimension dimension = null; if (dimensionRoot != null) { long worldSeed = requireStoredWorldSeed(configuredWorldName, dimensionRoot); - GenerationHistory history = requireGenerationHistory(dimensionRoot, id, worldSeed); - File pack = requireActivePack(history); + LoadedHistory loaded = requireGenerationHistory(dimensionRoot, id, worldSeed); + GenerationHistory history = loaded.history(); + File pack = loaded.activePack(); requireSnapshotLoadable(pack); dimension = requireHistoricalDimension(history, pack, id); } @@ -615,8 +616,9 @@ public final class IrisWorldGeneratorResolver { + worldKey + "."); } long worldSeed = requireStoredWorldSeed(worldName, dimensionRoot); - GenerationHistory history = requireGenerationHistory(dimensionRoot, id, worldSeed); - File snapshotRoot = requireActivePack(history); + LoadedHistory loaded = requireGenerationHistory(dimensionRoot, id, worldSeed); + GenerationHistory history = loaded.history(); + File snapshotRoot = loaded.activePack(); PackValidationResult validation = validateSnapshot(snapshotRoot); if (hasPendingExternalContent(validation, IrisServices.getOrNull(ExternalDataSVC.class))) { return deferFrozenWorldGenerator(worldName, worldKey, dimensionRoot, snapshotRoot, id, history); @@ -705,7 +707,7 @@ public final class IrisWorldGeneratorResolver { } } - private static GenerationHistory requireGenerationHistory( + private static LoadedHistory requireGenerationHistory( File dimensionRoot, String dimensionKey, long worldSeed @@ -724,7 +726,7 @@ public final class IrisWorldGeneratorResolver { history.requireRegistryDefinitions( GenerationRegistryContractFactory.captureRequiredDefinitions(retainedContracts) ); - return history; + return new LoadedHistory(history, history.paths().packRoot(history.activeEpoch().epochId()).toFile()); } catch (IOException failure) { throw new IllegalStateException("Iris generation history is unusable at " + root + ".", failure); } @@ -825,12 +827,7 @@ public final class IrisWorldGeneratorResolver { ); } - private static File requireActivePack(GenerationHistory history) { - try { - return history.activePackRoot().toFile(); - } catch (IOException failure) { - throw new IllegalStateException("Active Iris generation pack is unusable.", failure); - } + private record LoadedHistory(GenerationHistory history, File activePack) { } private static long requireStoredWorldSeed(String worldName, File dimensionRoot) { diff --git a/adapters/bukkit/plugin/src/test/java/art/arcane/iris/world/BukkitWorldReconcilerRuntimeTest.java b/adapters/bukkit/plugin/src/test/java/art/arcane/iris/world/BukkitWorldReconcilerRuntimeTest.java new file mode 100644 index 000000000..bf4f7c9d3 --- /dev/null +++ b/adapters/bukkit/plugin/src/test/java/art/arcane/iris/world/BukkitWorldReconcilerRuntimeTest.java @@ -0,0 +1,148 @@ +package art.arcane.iris.world; + +import art.arcane.iris.Iris; +import art.arcane.iris.generation.runtime.IrisEngine; +import art.arcane.iris.generation.terrain.IrisDimension; +import art.arcane.iris.pack.loading.IrisData; +import art.arcane.iris.platform.generation.BukkitChunkGenerator; +import art.arcane.iris.world.history.GenerationActivation; +import art.arcane.iris.world.history.GenerationEpoch; +import art.arcane.iris.world.history.GenerationHistory; +import art.arcane.iris.world.history.GenerationHistoryPaths; +import art.arcane.iris.world.history.GenerationHistoryRuntimeRouter; +import art.arcane.iris.world.history.GenerationManifest; +import java.io.IOException; +import java.lang.reflect.Constructor; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Optional; +import java.util.concurrent.CompletableFuture; +import org.bukkit.Bukkit; +import org.bukkit.NamespacedKey; +import org.bukkit.World; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.mockito.MockedStatic; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.mockStatic; +import static org.mockito.Mockito.when; + +public class BukkitWorldReconcilerRuntimeTest { + @Rule + public TemporaryFolder temporaryFolder = new TemporaryFolder(); + + @Test + public void incompleteOrDetachedRuntimeRequiresFullValidation() throws Exception { + RuntimeFixture fixture = fixture(); + when(fixture.generator().getStartupReady()).thenReturn(new CompletableFuture<>()); + assertEquals(BukkitWorldReconciler.LoadedRuntimeStatus.UNAVAILABLE, + fixture.backend().validateLoadedRuntime(fixture.world(), fixture.key(), "overworld", 1337L)); + when(fixture.generator().getStartupReady()).thenReturn(CompletableFuture.completedFuture(null)); + when(fixture.engine().getGenerationHistoryRuntimeRouter()).thenReturn(Optional.empty()); + assertEquals(BukkitWorldReconciler.LoadedRuntimeStatus.UNAVAILABLE, + fixture.backend().validateLoadedRuntime(fixture.world(), fixture.key(), "overworld", 1337L)); + } + + @Test + public void configuredDimensionAndSeedMustMatchLiveRuntime() throws Exception { + RuntimeFixture fixture = fixture(); + assertThrows(IOException.class, () -> fixture.backend().validateLoadedRuntime( + fixture.world(), fixture.key(), "other", 1337L)); + assertThrows(IOException.class, () -> fixture.backend().validateLoadedRuntime( + fixture.world(), fixture.key(), "overworld", 7L)); + } + + @Test + public void validatedLiveRuntimeStillRequiresMatchingSavedStorage() throws Exception { + RuntimeFixture fixture = fixture(); + Path levelRoot = fixture.root().getParent().getParent().getParent(); + Path container = levelRoot.getParent(); + GenerationManifest manifest = mock(GenerationManifest.class); + GenerationEpoch activeEpoch = fixture.history().activeEpoch(); + GenerationActivation activeActivation = fixture.history().activeActivation(); + when(manifest.activeEpoch()).thenReturn(activeEpoch); + when(manifest.activeActivation()).thenReturn(activeActivation); + when(manifest.pendingActivation()).thenReturn(Optional.empty()); + GenerationHistory.PackInspection inspection = new GenerationHistory.PackInspection( + fixture.history().paths(), manifest, fixture.pack(), Optional.empty()); + try (MockedStatic bukkit = mockStatic(Bukkit.class); + MockedStatic creator = mockStatic(WorldCreatorCompat.class); + MockedStatic storage = mockStatic(IrisWorldStorage.class); + MockedStatic history = mockStatic(GenerationHistory.class)) { + bukkit.when(Bukkit::getWorldContainer).thenReturn(container.toFile()); + creator.when(() -> WorldCreatorCompat.persistentDimensionRoot(fixture.key())).thenReturn(fixture.root().toFile()); + storage.when(IrisWorldStorage::levelRoot).thenReturn(levelRoot.toFile()); + storage.when(() -> IrisWorldStorage.configuredWorldName(fixture.key(), "world")).thenReturn("world_iris_probe"); + storage.when(() -> IrisWorldStorage.requireFrozenDimensionRoot( + container.toFile(), levelRoot.toFile(), "world_iris_probe", fixture.key())).thenReturn(fixture.root().toFile()); + history.when(() -> GenerationHistory.inspectPacks(fixture.root())).thenReturn(inspection); + + assertEquals(BukkitWorldReconciler.LoadedRuntimeStatus.READY, fixture.backend().validateLoadedRuntime( + fixture.world(), fixture.key(), "overworld", 1337L)); + + GenerationActivation differentActivation = mock(GenerationActivation.class); + when(manifest.activeActivation()).thenReturn(differentActivation); + assertThrows(IOException.class, () -> fixture.backend().validateLoadedRuntime( + fixture.world(), fixture.key(), "overworld", 1337L)); + when(manifest.activeActivation()).thenReturn(activeActivation); + Files.delete(fixture.root().resolve("iris/generation/semantics/index.isix")); + assertThrows(IOException.class, () -> fixture.backend().validateLoadedRuntime( + fixture.world(), fixture.key(), "overworld", 1337L)); + } + } + + private RuntimeFixture fixture() throws Exception { + Path container = temporaryFolder.newFolder().toPath(); + Path root = container.resolve("world/dimensions/iris/probe"); + Path pack = root.resolve("iris/generation/epochs/" + "a".repeat(64) + "/pack"); + Files.createDirectories(pack); + Path semantics = root.resolve("iris/generation/semantics"); + Files.createDirectories(semantics); + Files.createFile(semantics.resolve("index.isix")); + NamespacedKey key = new NamespacedKey("iris", "probe"); + World world = mock(World.class); + BukkitChunkGenerator generator = mock(BukkitChunkGenerator.class); + IrisEngine engine = mock(IrisEngine.class); + IrisDimension dimension = mock(IrisDimension.class); + IrisData data = mock(IrisData.class); + GenerationHistory history = mock(GenerationHistory.class); + GenerationHistoryRuntimeRouter router = mock(GenerationHistoryRuntimeRouter.class); + GenerationEpoch epoch = mock(GenerationEpoch.class); + GenerationActivation activation = mock(GenerationActivation.class); + GenerationEpoch.DimensionContract contract = mock(GenerationEpoch.DimensionContract.class); + when(world.getGenerator()).thenReturn(generator); + when(world.getKey()).thenReturn(key); + when(generator.getStartupReady()).thenReturn(CompletableFuture.completedFuture(null)); + when(generator.getEngine()).thenReturn(engine); + when(generator.getGenerationHistory()).thenReturn(history); + when(generator.getDimensionKey()).thenReturn("overworld"); + when(engine.getGenerationHistoryRuntimeRouter()).thenReturn(Optional.of(router)); + when(router.history()).thenReturn(history); + when(history.paths()).thenReturn(GenerationHistoryPaths.forDimension(root)); + when(history.activeEpoch()).thenReturn(epoch); + when(history.activeActivation()).thenReturn(activation); + when(history.pendingActivation()).thenReturn(Optional.empty()); + when(epoch.worldSeed()).thenReturn(1337L); + when(epoch.dimensionContract()).thenReturn(contract); + when(contract.dimensionKey()).thenReturn("overworld"); + when(engine.getWorld()).thenReturn(IrisWorld.builder().platformIdentity(key.toString()).worldFolder(root.toFile()).seed(1337L).build()); + when(engine.getDimension()).thenReturn(dimension); + when(dimension.getLoadKey()).thenReturn("overworld"); + when(engine.getData()).thenReturn(data); + when(data.getDataFolder()).thenReturn(pack.toFile()); + Class backendClass = Class.forName(BukkitWorldReconciler.class.getName() + "$BukkitBackend"); + Constructor constructor = backendClass.getDeclaredConstructor(Iris.class); + constructor.setAccessible(true); + BukkitWorldReconciler.Backend backend = (BukkitWorldReconciler.Backend) constructor.newInstance(mock(Iris.class)); + return new RuntimeFixture(backend, world, key, generator, engine, history, root, pack); + } + + private record RuntimeFixture(BukkitWorldReconciler.Backend backend, World world, NamespacedKey key, + BukkitChunkGenerator generator, IrisEngine engine, GenerationHistory history, + Path root, Path pack) { + } +} diff --git a/adapters/bukkit/plugin/src/test/java/art/arcane/iris/world/BukkitWorldReconcilerTest.java b/adapters/bukkit/plugin/src/test/java/art/arcane/iris/world/BukkitWorldReconcilerTest.java index 2940acb71..accc615c3 100644 --- a/adapters/bukkit/plugin/src/test/java/art/arcane/iris/world/BukkitWorldReconcilerTest.java +++ b/adapters/bukkit/plugin/src/test/java/art/arcane/iris/world/BukkitWorldReconcilerTest.java @@ -71,6 +71,98 @@ public class BukkitWorldReconcilerTest { assertTrue(coordinator.isIdle()); } + @Test + public void startupBatchReusesValidatedLoadedRuntime() throws Exception { + File configuration = temporaryFolder.newFile("loaded.yml"); + BukkitWorldConfiguration.register(configuration, "probe", "overworld", 1337L); + FakeBackend backend = new FakeBackend(); + backend.loaded = Optional.of(world(backend.worldKey)); + backend.runtimeStatus = BukkitWorldReconciler.LoadedRuntimeStatus.READY; + backend.validationFailure = new IllegalStateException("unexpected disk reopen"); + LifecycleOperationCoordinator coordinator = coordinator(); + BukkitWorldReconciler reconciler = new BukkitWorldReconciler(backend, coordinator); + + BukkitWorldReconciler.LoadResult result = reconciler.loadConfiguredWorld( + configuration, "probe", backend.worldKey, "overworld", 1337L).join(); + + assertEquals(BukkitWorldReconciler.ReconciliationStatus.ALREADY_LOADED, result.status()); + assertEquals(BukkitWorldConfiguration.Registration.UNCHANGED, result.registration()); + assertEquals(1, backend.runtimeValidationCount); + assertEquals(0, backend.validationCount); + assertEquals(0, backend.createCount.get()); + assertTrue(coordinator.isIdle()); + } + + @Test + public void startupBatchFallsBackWhenLoadedRuntimeIsUnavailable() throws Exception { + File configuration = temporaryFolder.newFile("unavailable.yml"); + FakeBackend backend = new FakeBackend(); + backend.loaded = Optional.of(world(backend.worldKey)); + LifecycleOperationCoordinator coordinator = coordinator(); + BukkitWorldReconciler reconciler = new BukkitWorldReconciler(backend, coordinator); + + BukkitWorldReconciler.LoadResult result = reconciler.loadConfiguredWorld( + configuration, "probe", backend.worldKey, "overworld", null).join(); + + assertEquals(BukkitWorldReconciler.ReconciliationStatus.ALREADY_LOADED, result.status()); + assertEquals(1, backend.runtimeValidationCount); + assertEquals(1, backend.validationCount); + assertTrue(coordinator.isIdle()); + } + + @Test + public void startupBatchRejectsChangedRuntimeStorage() throws Exception { + File configuration = temporaryFolder.newFile("changed-storage.yml"); + FakeBackend backend = new FakeBackend(); + backend.loaded = Optional.of(world(backend.worldKey)); + backend.runtimeFailure = new IllegalStateException("saved storage changed"); + LifecycleOperationCoordinator coordinator = coordinator(); + BukkitWorldReconciler reconciler = new BukkitWorldReconciler(backend, coordinator); + + BukkitWorldReconciler.LoadResult result = reconciler.loadConfiguredWorld( + configuration, "probe", backend.worldKey, "overworld", null).join(); + + assertEquals(BukkitWorldReconciler.ReconciliationStatus.DIMENSION_UNRESOLVED, result.status()); + assertEquals(0, backend.validationCount); + assertEquals(0, backend.createCount.get()); + assertEquals(0L, configuration.length()); + assertTrue(coordinator.isIdle()); + } + + @Test + public void startupBatchKeepsConfigurationConflictChecks() throws Exception { + File configuration = temporaryFolder.newFile("changed-config.yml"); + BukkitWorldConfiguration.register(configuration, "probe", "other", 7L); + FakeBackend backend = new FakeBackend(); + backend.loaded = Optional.of(world(backend.worldKey)); + backend.runtimeStatus = BukkitWorldReconciler.LoadedRuntimeStatus.READY; + LifecycleOperationCoordinator coordinator = coordinator(); + BukkitWorldReconciler reconciler = new BukkitWorldReconciler(backend, coordinator); + + BukkitWorldReconciler.LoadResult result = reconciler.loadConfiguredWorld( + configuration, "probe", backend.worldKey, "overworld", 1337L).join(); + + assertEquals(BukkitWorldReconciler.ReconciliationStatus.CONFIGURATION_FAILED, result.status()); + assertEquals("Iris:other", YamlConfiguration.loadConfiguration(configuration).getString("worlds.probe.generator")); + assertTrue(coordinator.isIdle()); + } + + @Test + public void explicitLoadStillPerformsFullValidationForLoadedWorlds() throws Exception { + File configuration = temporaryFolder.newFile("explicit-loaded.yml"); + FakeBackend backend = new FakeBackend(); + backend.loaded = Optional.of(world(backend.worldKey)); + backend.runtimeStatus = BukkitWorldReconciler.LoadedRuntimeStatus.READY; + backend.validationFailure = new IllegalStateException("pack changed"); + BukkitWorldReconciler reconciler = new BukkitWorldReconciler(backend, coordinator()); + + BukkitWorldReconciler.LoadResult result = reconciler.loadWorld(configuration, backend.worldKey.toString()).join(); + + assertEquals(BukkitWorldReconciler.ReconciliationStatus.DIMENSION_UNRESOLVED, result.status()); + assertEquals(0, backend.runtimeValidationCount); + assertEquals(1, backend.validationCount); + } + @Test public void failedCreationRollsBackOnlyNewRegistration() throws Exception { File configuration = temporaryFolder.newFile("bukkit.yml"); @@ -356,6 +448,10 @@ public class BukkitWorldReconcilerTest { private Long configuredSeed; private BukkitWorldReconciler.DimensionResolution dimensionResolution; private RuntimeException validationFailure; + private BukkitWorldReconciler.LoadedRuntimeStatus runtimeStatus = BukkitWorldReconciler.LoadedRuntimeStatus.UNAVAILABLE; + private RuntimeException runtimeFailure; + private int validationCount; + private int runtimeValidationCount; private FakeBackend() { worldKey = new NamespacedKey("iris", "probe"); @@ -404,6 +500,16 @@ public class BukkitWorldReconcilerTest { return irisWorld; } + @Override + public BukkitWorldReconciler.LoadedRuntimeStatus validateLoadedRuntime( + World world, NamespacedKey requestedWorldKey, String dimension, Long seed) { + runtimeValidationCount++; + if (runtimeFailure != null) { + throw runtimeFailure; + } + return runtimeStatus; + } + @Override public BukkitWorldReconciler.DimensionResolution resolveDimension(NamespacedKey requestedWorldKey) { return dimensionResolution; @@ -411,6 +517,7 @@ public class BukkitWorldReconcilerTest { @Override public void requireDimensionLoadable(NamespacedKey requestedWorldKey, String dimension) { + validationCount++; if (validationFailure != null) { throw validationFailure; } diff --git a/core/build.gradle b/core/build.gradle index a0e3d5509..1864cb8fd 100644 --- a/core/build.gradle +++ b/core/build.gradle @@ -150,6 +150,7 @@ dependencies { slim(libs.commons.lang3) slim(libs.oshi) slim(libs.lz4) + slim('com.fulcrumgenomics:jlibdeflate:0.2.0') slim(libs.fastutil) slim(libs.zip) { exclude(group: 'org.slf4j', module: 'slf4j-api') diff --git a/core/src/main/java/art/arcane/iris/pack/datapack/IrisDatapackCompiler.java b/core/src/main/java/art/arcane/iris/pack/datapack/IrisDatapackCompiler.java index 09697b9c0..05c58959c 100644 --- a/core/src/main/java/art/arcane/iris/pack/datapack/IrisDatapackCompiler.java +++ b/core/src/main/java/art/arcane/iris/pack/datapack/IrisDatapackCompiler.java @@ -198,12 +198,13 @@ public final class IrisDatapackCompiler { Objects.requireNonNull(fixer, "fixer"); LinkedHashMap requirements = new LinkedHashMap<>(); Map> biomeTags = new TreeMap<>(); + Map histories = new LinkedHashMap<>(); for (File packRoot : packRoots) { PackDirectoryResolver.requireSafePackTree(packRoot); if (!hasDimensions(packRoot.toPath())) { continue; } - BiomeAliasContext aliasContext = biomeAliasContext(packRoot.toPath()); + BiomeAliasContext aliasContext = biomeAliasContext(packRoot.toPath(), histories); IrisData data = IrisData.openDatapackCompiler(packRoot); try { if (aliasContext.retainedContract() != null) { @@ -230,7 +231,7 @@ public final class IrisDatapackCompiler { data.close(); } } - for (GenerationRegistryContract contract : retainedRegistryContracts(packRoots)) { + for (GenerationRegistryContract contract : retainedRegistryContracts(packRoots, histories)) { for (PhysicalResourceKey key : contract.generatedSources().keySet()) { putRegistryRequirement(requirements, registryPath(key) + "/" + key.resourceKey(), fingerprintContent(GenerationRegistryContractFactory.renderGeneratedSource(contract, key, fixer))); @@ -453,6 +454,7 @@ public final class IrisDatapackCompiler { DimensionHeight height = new DimensionHeight(fixer); Map> biomes = new LinkedHashMap<>(); Map> dimensions = new LinkedHashMap<>(); + Map histories = new LinkedHashMap<>(); int packCount = 0; int dimensionCount = 0; for (File packRoot : packRoots) { @@ -460,7 +462,7 @@ public final class IrisDatapackCompiler { if (!hasDimensions(packRoot.toPath())) { continue; } - BiomeAliasContext aliasContext = biomeAliasContext(packRoot.toPath()); + BiomeAliasContext aliasContext = biomeAliasContext(packRoot.toPath(), histories); IrisData data = IrisData.openDatapackCompiler(packRoot); try { if (aliasContext.retainedContract() != null) { @@ -512,7 +514,7 @@ public final class IrisDatapackCompiler { } IrisDimension.writeShared(datapackRoots, height, adjustVanillaHeight); - installRetainedRegistrySources(packRoots, datapackRoots, fixer); + installRetainedRegistrySources(packRoots, datapackRoots, fixer, histories); installLevelStemBindings(normalizedBindings, dimensions, datapackRoots); validateOutputs(datapackRoots, dimensionCount, normalizedBindings.size()); return new CompilationResult(packCount, dimensionCount, countBiomes(biomes)); @@ -520,9 +522,15 @@ public final class IrisDatapackCompiler { public static void installRetainedRegistrySources(List packRoots, Collection datapackRoots, IDataFixer fixer) throws IOException { + installRetainedRegistrySources(packRoots, datapackRoots, fixer, new LinkedHashMap<>()); + } + + private static void installRetainedRegistrySources(List packRoots, Collection datapackRoots, + IDataFixer fixer, + Map histories) throws IOException { Map sources = new TreeMap<>(); Map> tags = new TreeMap<>(); - for (GenerationRegistryContract contract : retainedRegistryContracts(packRoots)) { + for (GenerationRegistryContract contract : retainedRegistryContracts(packRoots, histories)) { for (PhysicalResourceKey key : contract.generatedSources().keySet()) { String source = GenerationRegistryContractFactory.renderGeneratedSource(contract, key, fixer); String previous = sources.putIfAbsent(key, source); @@ -588,6 +596,11 @@ public final class IrisDatapackCompiler { } private static List retainedRegistryContracts(List packRoots) throws IOException { + return retainedRegistryContracts(packRoots, new LinkedHashMap<>()); + } + + private static List retainedRegistryContracts( + List packRoots, Map histories) throws IOException { Map contracts = new TreeMap<>(); Set loadedWorlds = new LinkedHashSet<>(); for (File pack : packRoots) { @@ -595,7 +608,7 @@ public final class IrisDatapackCompiler { if (dimensionRoot == null || !loadedWorlds.add(dimensionRoot)) { continue; } - for (GenerationEpoch epoch : GenerationHistory.open(dimensionRoot).manifest().epochs()) { + for (GenerationEpoch epoch : inspectHistory(dimensionRoot, histories).manifest().epochs()) { contracts.putIfAbsent(epoch.registryContract().fingerprint(), epoch.registryContract()); } } @@ -765,11 +778,9 @@ public final class IrisDatapackCompiler { if (!Files.isSymbolicLink(irisRoot) && (Files.exists(generationRoot, LinkOption.NOFOLLOW_LINKS) || Files.isSymbolicLink(generationRoot))) { - GenerationHistory history = GenerationHistory.open(worldRoot); + GenerationHistory.PackInspection history = GenerationHistory.inspectPacks(worldRoot); candidates.add(history.activePackRoot()); - if (history.pendingActivation().isPresent()) { - candidates.add(history.packRoot(history.pendingActivation().orElseThrow().activationId())); - } + history.pendingPackRoot().ifPresent(candidates::add); return true; } Path legacyPack = irisRoot.resolve("pack"); @@ -816,7 +827,8 @@ public final class IrisDatapackCompiler { roots.putIfAbsent(identity, normalized.toFile()); } - private static BiomeAliasContext biomeAliasContext(Path packRoot) throws IOException { + private static BiomeAliasContext biomeAliasContext( + Path packRoot, Map histories) throws IOException { Path normalizedRoot = packRoot.toAbsolutePath().normalize(); Path rootParent = normalizedRoot.getParent(); if (rootParent == null || !"pack".equals(normalizedRoot.getFileName().toString())) { @@ -841,7 +853,7 @@ public final class IrisDatapackCompiler { } String epochId = rootParent.getFileName().toString(); - GenerationHistory history = GenerationHistory.open(dimensionRoot); + GenerationHistory.PackInspection history = inspectHistory(dimensionRoot, histories); GenerationEpoch epoch = history.manifest().epoch(epochId).orElseThrow( () -> new IOException("Retained Iris pack is not referenced by its generation history: " + normalizedRoot) @@ -853,6 +865,17 @@ public final class IrisDatapackCompiler { return BiomeAliasContext.retained(epoch.registryContract()); } + private static GenerationHistory.PackInspection inspectHistory( + Path dimensionRoot, Map histories) throws IOException { + Path root = dimensionRoot.toAbsolutePath().normalize(); + GenerationHistory.PackInspection history = histories.get(root); + if (history == null) { + history = GenerationHistory.inspectPacks(root); + histories.put(root, history); + } + return history; + } + private static boolean hasDimensions(Path root) { Path dimensions = root.resolve("dimensions"); if (Files.isSymbolicLink(dimensions) diff --git a/core/src/main/java/art/arcane/iris/platform/generation/BukkitChunkGenerator.java b/core/src/main/java/art/arcane/iris/platform/generation/BukkitChunkGenerator.java index 540021909..556dcce12 100644 --- a/core/src/main/java/art/arcane/iris/platform/generation/BukkitChunkGenerator.java +++ b/core/src/main/java/art/arcane/iris/platform/generation/BukkitChunkGenerator.java @@ -581,16 +581,11 @@ public class BukkitChunkGenerator extends ChunkGenerator implements PlatformChun } private void setupEngine() { - try (GenerationAdmission.RuntimeLease startupAdmission = generationHistory == null ? null : generationHistory.retainRuntime()) { + try (GenerationAdmission.RuntimeLease startupAdmission = generationHistory == null ? null : generationHistory.retainRuntime(); + GenerationHistory.StartupPreparation preparation = generationHistory == null ? null + : prepareGenerationHistoryTarget(IrisSettings.get().getGenerator().getGenerationTransitionWidthBlocks())) { lastMode = StudioMode.NORMAL; lastJigsawStudioRequestId = null; - if (generationHistory != null) { - try { - prepareGenerationHistoryTarget(IrisSettings.get().getGenerator().getGenerationTransitionWidthBlocks()); - } catch (IOException failure) { - throw new IllegalStateException("Unable to prepare saved Iris terrain for the current generator.", failure); - } - } EngineTarget engineTarget = getTarget(); String packKey = engineTarget.getDimension().getLoadKey(); JigsawStudioActivation.Request request = studio @@ -605,24 +600,9 @@ public class BukkitChunkGenerator extends ChunkGenerator implements PlatformChun objectStudioActive = studio && ObjectStudioActivation.isActive(packKey); IrisEngine createdEngine = createEngine(engineTarget); if (generationHistory != null) { - long initialActivationId = generationHistory.activeActivation().activationId(); - int transitionWidthBlocks = IrisSettings.get() - .getGenerator() - .getGenerationTransitionWidthBlocks(); try { - createdEngine.attachGenerationHistory( - generationHistory, - IrisBoundarySignatureSampler.INSTANCE, - transitionWidthBlocks); - if (generationHistory.activeActivation().activationId() != initialActivationId) { - createdEngine.close(); - targetCache.reset(); - createdEngine = createEngine(loadActiveGenerationHistoryTarget()); - createdEngine.attachGenerationHistory( - generationHistory, - IrisBoundarySignatureSampler.INSTANCE, - transitionWidthBlocks); - } + GenerationHistoryRuntimeRouter.attachPrepared( + createdEngine, preparation, IrisBoundarySignatureSampler.INSTANCE); } catch (Throwable failure) { try { createdEngine.close(); @@ -641,6 +621,8 @@ public class BukkitChunkGenerator extends ChunkGenerator implements PlatformChun publishInitializedEngine(createdEngine); populators.clear(); targetCache.reset(); + } catch (IOException failure) { + throw new IllegalStateException("Unable to prepare saved Iris terrain for the current generator.", failure); } } @@ -651,25 +633,31 @@ public class BukkitChunkGenerator extends ChunkGenerator implements PlatformChun engine = initializedEngine; } - void prepareGenerationHistoryTarget(int transitionWidthBlocks) throws IOException { - generationHistory.prepareCurrentGenerator(transitionWidthBlocks); - EngineTarget cachedTarget = targetCache.getIfPresent(); - if (cachedTarget == null) { - return; - } - Path activePackRoot = generationHistory.activePackRoot(); - IrisData cachedData = cachedTarget.getData(); - if (!cachedData.isClosed() && cachedData.getDataFolder().toPath().equals(activePackRoot)) { - return; - } - targetCache.reset(); - EngineTarget pendingTarget = startupTarget; - Engine activeEngine = engine; - if (!cachedData.isClosed() - && cachedData.getEngines().isEmpty() - && (pendingTarget == null || pendingTarget.getData() != cachedData) - && (activeEngine == null || activeEngine.getData() != cachedData)) { - cachedData.close(); + GenerationHistory.StartupPreparation prepareGenerationHistoryTarget(int transitionWidthBlocks) throws IOException { + GenerationHistory.StartupPreparation preparation = generationHistory.prepareStartup(transitionWidthBlocks); + try { + EngineTarget cachedTarget = targetCache.getIfPresent(); + if (cachedTarget == null) { + return preparation; + } + Path activePackRoot = generationHistory.activePackRoot(); + IrisData cachedData = cachedTarget.getData(); + if (!cachedData.isClosed() && cachedData.getDataFolder().toPath().equals(activePackRoot)) { + return preparation; + } + targetCache.reset(); + EngineTarget pendingTarget = startupTarget; + Engine activeEngine = engine; + if (!cachedData.isClosed() + && cachedData.getEngines().isEmpty() + && (pendingTarget == null || pendingTarget.getData() != cachedData) + && (activeEngine == null || activeEngine.getData() != cachedData)) { + cachedData.close(); + } + return preparation; + } catch (IOException | RuntimeException | Error failure) { + preparation.close(); + throw failure; } } diff --git a/core/src/main/java/art/arcane/iris/world/history/ChunkGenerationSemantics.java b/core/src/main/java/art/arcane/iris/world/history/ChunkGenerationSemantics.java index 0fc1e0f5d..093e3bc6f 100644 --- a/core/src/main/java/art/arcane/iris/world/history/ChunkGenerationSemantics.java +++ b/core/src/main/java/art/arcane/iris/world/history/ChunkGenerationSemantics.java @@ -249,23 +249,23 @@ public final class ChunkGenerationSemantics { throw new IllegalArgumentException("Generation semantic resource key contains an invalid character"); } } - if (requiredKey.getBytes(StandardCharsets.UTF_8).length > MAX_KEY_BYTES) { + if (requiredKey.length() > MAX_KEY_BYTES / 3 + && requiredKey.getBytes(StandardCharsets.UTF_8).length > MAX_KEY_BYTES) { throw new IllegalArgumentException("Generation semantic resource key is too long"); } return requiredKey; } private static Set immutableKeys(Collection keys, String kind) { + if (keys.isEmpty()) { + return Set.of(); + } if (keys.size() > MAX_KEYS_PER_KIND) { throw new IllegalArgumentException( "A chunk cannot contain more than " + MAX_KEYS_PER_KIND + " " + kind + " keys" ); } - TreeSet validated = new TreeSet<>(); - for (String key : keys) { - validated.add(requireResourceKey(key)); - } - return Collections.unmodifiableSet(validated); + return Collections.unmodifiableSet(new TreeSet<>(keys)); } private static Set immutableStructures(Collection structures) { diff --git a/core/src/main/java/art/arcane/iris/world/history/DiskBoundaryCapture.java b/core/src/main/java/art/arcane/iris/world/history/DiskBoundaryCapture.java index 30f4d8ed8..a96f46c8c 100644 --- a/core/src/main/java/art/arcane/iris/world/history/DiskBoundaryCapture.java +++ b/core/src/main/java/art/arcane/iris/world/history/DiskBoundaryCapture.java @@ -9,13 +9,13 @@ import java.util.Objects; public final class DiskBoundaryCapture implements TerrainBoundarySignatureStore.SignatureSampler { private static final int MAXIMUM_CACHED_CHUNKS = 32; - private final Path dimensionRoot; + private final SavedTerrainChunkReader.StatusReader reader; private final int minimumY; private final int height; private final LinkedHashMap chunks = new LinkedHashMap<>(32, 0.75F, true); public DiskBoundaryCapture(Path dimensionRoot, int minimumY, int height) { - this.dimensionRoot = Objects.requireNonNull(dimensionRoot, "dimension root"); + this.reader = new SavedTerrainChunkReader.StatusReader(Objects.requireNonNull(dimensionRoot, "dimension root")); this.minimumY = minimumY; this.height = height; } @@ -27,7 +27,7 @@ public final class DiskBoundaryCapture implements TerrainBoundarySignatureStore. long key = ChunkGenerationOwnership.packChunk(chunkX, chunkZ); SavedTerrainChunk chunk = chunks.get(key); if (chunk == null) { - chunk = SavedTerrainChunk.read(dimensionRoot, chunkX, chunkZ, minimumY, height); + chunk = reader.readChunk(chunkX, chunkZ, minimumY, height); chunks.put(key, chunk); if (chunks.size() > MAXIMUM_CACHED_CHUNKS) { Map.Entry oldest = chunks.firstEntry(); @@ -38,7 +38,8 @@ public final class DiskBoundaryCapture implements TerrainBoundarySignatureStore. } @Override - public void close() { + public void close() throws IOException { chunks.clear(); + reader.close(); } } diff --git a/core/src/main/java/art/arcane/iris/world/history/GenerationHistory.java b/core/src/main/java/art/arcane/iris/world/history/GenerationHistory.java index 1c2865088..7fef0061d 100644 --- a/core/src/main/java/art/arcane/iris/world/history/GenerationHistory.java +++ b/core/src/main/java/art/arcane/iris/world/history/GenerationHistory.java @@ -67,7 +67,7 @@ public final class GenerationHistory { this.admission = new GenerationAdmission(paths.dimensionRoot()); this.boundaryCache = boundedCache(MAXIMUM_CACHED_BOUNDARIES); this.terrainSignatureCache = boundedCache(MAXIMUM_CACHED_TERRAIN_SIGNATURES); - validateReferencedState(publication); + validateReferencedState(publication, true); } public GenerationAdmission.RuntimeLease retainRuntime() { @@ -148,6 +148,26 @@ public final class GenerationHistory { return open(dimensionRoot, GenerationKernelRegistry.standard()); } + public static PackInspection inspectPacks(Path dimensionRoot) throws IOException { + GenerationHistoryPaths paths = GenerationHistoryPaths.forDimension(dimensionRoot); + requireSafeStatePaths(paths); + GenerationManifest manifest = GenerationHistoryStore.open(paths.generationRoot()).manifest(); + for (GenerationEpoch epoch : manifest.epochs()) { + GenerationPackFingerprint.requireSupported(epoch.packFingerprintVersion()); + } + GenerationPackRepository packs = new GenerationPackRepository(paths.dimensionRoot()); + GenerationEpoch active = manifest.activeEpoch(); + Path activePack = packs.requireExactPack( + active.epochId(), active.packFingerprint(), active.packFingerprintVersion()); + Optional pendingPack = Optional.empty(); + if (manifest.pendingEpoch().isPresent()) { + GenerationEpoch pending = manifest.pendingEpoch().orElseThrow(); + pendingPack = Optional.of(pending.epochId().equals(active.epochId()) ? activePack + : packs.requireExactPack(pending.epochId(), pending.packFingerprint(), pending.packFingerprintVersion())); + } + return new PackInspection(paths, manifest, activePack, pendingPack); + } + public static GenerationHistory open(Path dimensionRoot, long expectedWorldSeed) throws IOException { GenerationHistory history = open(dimensionRoot, GenerationKernelRegistry.standard()); history.requireWorldSeed(expectedWorldSeed); @@ -393,7 +413,7 @@ public final class GenerationHistory { ); try (GenerationAdmission.CutoverLease ignored = admission.beginStartupCutover()) { synchronized (this) { - return promotePendingLocked(requiredCapture); + return promotePendingLocked(requiredCapture, true, true); } } } @@ -414,7 +434,7 @@ public final class GenerationHistory { GenerationActivation promote(BoundarySignatureCapture signatureCapture) throws IOException { requireOpen(); synchronized (GenerationHistory.this) { - return promotePendingLocked(Objects.requireNonNull(signatureCapture, "signature capture")); + return promotePendingLocked(Objects.requireNonNull(signatureCapture, "signature capture"), false, true); } } @@ -433,10 +453,10 @@ public final class GenerationHistory { } private GenerationActivation promotePendingLocked( - BoundarySignatureCapture signatureCapture + BoundarySignatureCapture signatureCapture, boolean startupRecovery, boolean validateSemantics ) throws IOException { - WorldChunkInventory inventory = recoverUnstoredClaims(); - validateReferencedState(Optional.empty()); + WorldChunkInventory inventory = recoverUnstoredClaims(startupRecovery); + validateReferencedState(Optional.empty(), validateSemantics); Optional pending = store.pendingActivation(); if (pending.isEmpty()) { return store.activeActivation(); @@ -498,7 +518,7 @@ public final class GenerationHistory { ) throws IOException { GenerationEpoch active = store.activeEpoch(); return stageUpdate( - activePackRoot(), + store.pendingActivation().isPresent() ? activePackRoot() : packs.requireSafePackRoot(active.epochId()), active.packFingerprint(), active.packFingerprintVersion(), active.dimensionContract(), @@ -509,11 +529,17 @@ public final class GenerationHistory { } public void prepareCurrentGenerator(int transitionWidthBlocks) throws IOException { - try (GenerationAdmission.CutoverLease ignored = admission.beginStartupCutover()) { + try (StartupPreparation ignored = prepareStartup(transitionWidthBlocks)) { + } + } + + public StartupPreparation prepareStartup(int transitionWidthBlocks) throws IOException { + GenerationAdmission.CutoverLease lease = admission.beginStartupCutover(); + try { synchronized (this) { if (store.pendingActivation().isEmpty() && usesCurrentGenerator()) { - recoverUnstoredClaims(); - validateReferencedState(Optional.empty()); + recoverUnstoredClaims(true); + validateReferencedState(Optional.empty(), false); } else { if (store.pendingActivation().isEmpty()) { stageCurrentKernel(transitionWidthBlocks); @@ -524,27 +550,80 @@ public final class GenerationHistory { promoteSavedBoundary(); } } + return new StartupPreparation(lease, store.manifest()); + } + } catch (IOException | RuntimeException | Error failure) { + lease.close(); + throw failure; + } + } + + public final class StartupPreparation implements AutoCloseable { + private final GenerationAdmission.CutoverLease lease; + private final GenerationManifest preparedManifest; + private final Thread owner = Thread.currentThread(); + private boolean consumed; + private boolean closed; + + private StartupPreparation(GenerationAdmission.CutoverLease lease, GenerationManifest preparedManifest) { + this.lease = lease; + this.preparedManifest = preparedManifest; + } + + GenerationHistory history() { + return GenerationHistory.this; + } + + void consume() { + requireOwner(); + if (closed || consumed) { + throw new IllegalStateException("Generation startup preparation is closed or already attached."); + } + if (store.manifest() != preparedManifest) { + throw new IllegalStateException("Generation history changed after startup preparation."); + } + consumed = true; + } + + @Override + public void close() { + requireOwner(); + if (!closed) { + closed = true; + lease.close(); + } + } + + private void requireOwner() { + if (Thread.currentThread() != owner) { + throw new IllegalStateException("Generation startup preparation belongs to another thread."); } } } private void promoteSavedBoundary() throws IOException { GenerationEpoch.DimensionContract contract = activeEpoch().dimensionContract(); - promotePendingLocked(boundary -> new DiskBoundaryCapture(paths.dimensionRoot(), contract.minHeight(), contract.height())); + promotePendingLocked(boundary -> new DiskBoundaryCapture(paths.dimensionRoot(), contract.minHeight(), contract.height()), true, false); } - private WorldChunkInventory recoverUnstoredClaims() throws IOException { + private WorldChunkInventory recoverUnstoredClaims(boolean startupRecovery) throws IOException { long outgoing = store.activeActivation().activationId(); Set selected = store.pendingActivation() .map(pending -> Set.of(outgoing, pending.activationId())) .orElseGet(() -> Set.of(outgoing)); - WorldChunkInventory inventory = WorldChunkInventory.scan(paths.dimensionRoot()).filter((chunkX, chunkZ) -> { - if (ownership.isExplicitlyAssigned(chunkX, chunkZ) - && !selected.contains(ownership.resolve(chunkX, chunkZ, outgoing))) { - return true; + WorldChunkInventory allocated = WorldChunkInventory.scan(paths.dimensionRoot()); + WorldChunkInventory.ChunkPredicate retained = (chunkX, chunkZ) -> + ownership.isExplicitlyAssigned(chunkX, chunkZ) + && !selected.contains(ownership.resolve(chunkX, chunkZ, outgoing)); + WorldChunkInventory inventory; + if (startupRecovery) { + inventory = SavedTerrainRecoveryScan.scan(paths.dimensionRoot(), allocated, retained); + } else { + try (SavedTerrainChunkReader.StatusReader reader = new SavedTerrainChunkReader.StatusReader(paths.dimensionRoot())) { + inventory = allocated.filter((chunkX, chunkZ) -> retained.test(chunkX, chunkZ) + || SavedTerrainChunk.hasTerrain(reader.readStatus(chunkX, chunkZ))); } - return SavedTerrainChunk.hasTerrain(SavedTerrainChunk.readStatus(paths.dimensionRoot(), chunkX, chunkZ)); - }); + } semantics.discardUnstoredClaims(inventory, selected); savedBiomes.discardUnstoredClaims(inventory, selected); ownership.discardUnstoredClaims(inventory, selected); @@ -808,7 +887,7 @@ public final class GenerationHistory { return ownership.explicitChunkCount(); } - private void validateReferencedState(Optional publication) throws IOException { + private void validateReferencedState(Optional publication, boolean validateSemantics) throws IOException { validateRuntimeVersions(); GenerationEpoch active = store.activeEpoch(); if (publication.isEmpty() @@ -833,7 +912,9 @@ public final class GenerationHistory { } } validateOwnershipReferences(); - validateSemanticReferences(); + if (validateSemantics) { + validateSemanticReferences(); + } } private static void requireTransitionSnapshotIdentities( @@ -1175,6 +1256,10 @@ public final class GenerationHistory { }; } + public record PackInspection(GenerationHistoryPaths paths, GenerationManifest manifest, + Path activePackRoot, Optional pendingPackRoot) { + } + private record FreshPublication(Path pack, GenerationEpoch epoch) { } diff --git a/core/src/main/java/art/arcane/iris/world/history/GenerationHistoryRuntimeRouter.java b/core/src/main/java/art/arcane/iris/world/history/GenerationHistoryRuntimeRouter.java index c307845a4..84d1fb949 100644 --- a/core/src/main/java/art/arcane/iris/world/history/GenerationHistoryRuntimeRouter.java +++ b/core/src/main/java/art/arcane/iris/world/history/GenerationHistoryRuntimeRouter.java @@ -117,6 +117,27 @@ public final class GenerationHistoryRuntimeRouter implements AutoCloseable { ); } + public static GenerationHistoryRuntimeRouter attachPrepared( + IrisEngine engine, + GenerationHistory.StartupPreparation preparation, + GenerationBoundarySignatureSampler signatureSampler + ) throws IOException { + return attachPrepared(engine, preparation, signatureSampler, new DefaultActivationRuntimeFactory()); + } + + static GenerationHistoryRuntimeRouter attachPrepared( + IrisEngine engine, + GenerationHistory.StartupPreparation preparation, + GenerationBoundarySignatureSampler signatureSampler, + ActivationRuntimeFactory runtimeFactory + ) throws IOException { + GenerationHistory history = Objects.requireNonNull(preparation, "startup preparation").history(); + synchronized (history) { + preparation.consume(); + return attach(engine, history, signatureSampler, runtimeFactory); + } + } + public static GenerationHistoryRuntimeRouter attachAndPromotePending( IrisEngine engine, GenerationHistory history, diff --git a/core/src/main/java/art/arcane/iris/world/history/GenerationPackRepository.java b/core/src/main/java/art/arcane/iris/world/history/GenerationPackRepository.java index e2477fc71..c5f6c0d01 100644 --- a/core/src/main/java/art/arcane/iris/world/history/GenerationPackRepository.java +++ b/core/src/main/java/art/arcane/iris/world/history/GenerationPackRepository.java @@ -118,17 +118,22 @@ public final class GenerationPackRepository { String packFingerprint, int packFingerprintVersion ) throws IOException { + Path pack = requireSafePackRoot(epochId); + requireFingerprint( + pack, + requireDigest(packFingerprint, "packFingerprint"), + packFingerprintVersion + ); + return pack; + } + + Path requireSafePackRoot(String epochId) throws IOException { String requiredEpochId = requireDigest(epochId, "epochId"); validateExistingAncestors(requiredEpochId); Path epoch = epochRoot(requiredEpochId); requireSafeDirectory(epoch, "Generation epoch path is not a safe directory"); Path pack = epoch.resolve("pack"); requireSafeDirectory(pack, "Generation epoch pack is missing or unsafe"); - requireFingerprint( - pack, - requireDigest(packFingerprint, "packFingerprint"), - packFingerprintVersion - ); return pack; } @@ -198,7 +203,7 @@ public final class GenerationPackRepository { public static void copyPackTree(Path source, Path target) throws IOException { Executor executor = IrisPlatforms.isBound() ? MultiBurst.ioBurst : ForkJoinPool.commonPool(); - int workers = Math.max(1, Math.min(MAXIMUM_COPY_WORKERS, Runtime.getRuntime().availableProcessors() / 2)); + int workers = Math.max(1, Math.min(MAXIMUM_COPY_WORKERS, Runtime.getRuntime().availableProcessors())); copyPackTree(source, target, executor, workers); } @@ -272,7 +277,7 @@ public final class GenerationPackRepository { static void forceDirectoryTree(Path root, FileAction force) throws IOException { Executor executor = IrisPlatforms.isBound() ? MultiBurst.ioBurst : ForkJoinPool.commonPool(); - int workers = Math.max(1, Math.min(MAXIMUM_COPY_WORKERS, Runtime.getRuntime().availableProcessors() / 2)); + int workers = Math.max(1, Math.min(MAXIMUM_COPY_WORKERS, Runtime.getRuntime().availableProcessors())); forceDirectoryTree(root, executor, workers, force); } diff --git a/core/src/main/java/art/arcane/iris/world/history/GenerationSemanticIndex.java b/core/src/main/java/art/arcane/iris/world/history/GenerationSemanticIndex.java index a50584b77..13ec5d5dd 100644 --- a/core/src/main/java/art/arcane/iris/world/history/GenerationSemanticIndex.java +++ b/core/src/main/java/art/arcane/iris/world/history/GenerationSemanticIndex.java @@ -1013,13 +1013,13 @@ public final class GenerationSemanticIndex { } RegionShard cachedRegion = regions.get(regionKey); if (cachedRegion != null) { - RegionSummary summary = RegionSummary.from(cachedRegion); + RegionSummary summary = cachedRegion.summary(); cacheSummary(regionKey, summary); return summary; } if (journalEntries.containsKey(regionKey)) { RegionShard region = loadRegionLocked(regionKey, regionX(regionKey), regionZ(regionKey)); - return RegionSummary.from(region); + return region.summary(); } String shardHash = shardHashes.get(regionKey); if (shardHash == null) { @@ -1051,7 +1051,7 @@ public final class GenerationSemanticIndex { private void cacheRegion(long regionKey, RegionShard region) { regions.put(regionKey, region); - cacheSummary(regionKey, RegionSummary.from(region)); + cacheSummary(regionKey, region.summary()); while (regions.size() > MAXIMUM_CACHED_REGIONS) { Iterator> entries = regions.entrySet().iterator(); entries.next(); @@ -1663,7 +1663,10 @@ public final class GenerationSemanticIndex { EnumSet riverTypes = EnumSet.noneOf(HydrologyFeatureType.class); LongOpenHashSet activations = new LongOpenHashSet(); LongOpenHashSet sealedActivations = new LongOpenHashSet(); - for (ChunkGenerationSemantics semantics : region.records()) { + for (ChunkGenerationSemantics semantics : region.records) { + if (semantics == null) { + continue; + } activations.add(semantics.activationId()); mutableKeys(keys, SemanticKind.SURFACE_BIOME).addAll(semantics.surfaceBiomeKeys()); mutableKeys(keys, SemanticKind.CAVE_BIOME).addAll(semantics.caveBiomeKeys()); @@ -2298,6 +2301,7 @@ public final class GenerationSemanticIndex { private final int regionZ; private final ChunkGenerationSemantics[] records; private int recordCount; + private RegionSummary summary; private RegionShard(int regionX, int regionZ) { this.regionX = regionX; @@ -2496,7 +2500,7 @@ public final class GenerationSemanticIndex { if (input.available() != 0) { throw invalid(source, "unexpected trailing data"); } - if (!RegionSummary.from(region).equals(storedSummary)) { + if (!region.summary().equals(storedSummary)) { throw invalid(source, "summary does not match semantic records"); } return region; @@ -2519,6 +2523,13 @@ public final class GenerationSemanticIndex { return recordCount; } + private RegionSummary summary() { + if (summary == null) { + summary = RegionSummary.from(this); + } + return summary; + } + private ChunkGenerationSemantics get(int chunkX, int chunkZ) { if ((chunkX >> 5) != regionX || (chunkZ >> 5) != regionZ) { return null; @@ -2560,6 +2571,7 @@ public final class GenerationSemanticIndex { } records[index] = semantics; recordCount++; + summary = null; } private List records() { @@ -2632,7 +2644,7 @@ public final class GenerationSemanticIndex { } ByteArrayOutputStream bodyBytes = new ByteArrayOutputStream(8_192); - byte[] summary = RegionSummary.from(this).encode(); + byte[] summary = summary().encode(); try (DataOutputStream output = new DataOutputStream(bodyBytes)) { output.writeInt(MAGIC); output.writeShort(FORMAT_VERSION); diff --git a/core/src/main/java/art/arcane/iris/world/history/SavedBiomeStore.java b/core/src/main/java/art/arcane/iris/world/history/SavedBiomeStore.java index bf45eb238..ad4a1357a 100644 --- a/core/src/main/java/art/arcane/iris/world/history/SavedBiomeStore.java +++ b/core/src/main/java/art/arcane/iris/world/history/SavedBiomeStore.java @@ -1,5 +1,6 @@ package art.arcane.iris.world.history; +import java.io.BufferedInputStream; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.DataInputStream; @@ -8,6 +9,7 @@ import java.io.IOException; import java.io.File; import java.io.RandomAccessFile; import java.nio.ByteBuffer; +import java.nio.channels.Channels; import java.nio.channels.FileChannel; import java.nio.file.DirectoryStream; import java.nio.file.Files; @@ -356,8 +358,10 @@ public final class SavedBiomeStore { if (!Files.isRegularFile(region.path, LinkOption.NOFOLLOW_LINKS)) { throw new IOException("Saved biome region is not a regular file: " + region.path); } - try (RandomAccessFile input = new RandomAccessFile(region.path.toFile(), "rw")) { - long fileLength = input.length(); + try (RandomAccessFile file = new RandomAccessFile(region.path.toFile(), "rw"); + DataInputStream input = new DataInputStream(new BufferedInputStream( + Channels.newInputStream(file.getChannel()), 65536))) { + long fileLength = file.length(); if (fileLength < HEADER_BYTES || fileLength > MAXIMUM_REGION_BYTES) { throw new IOException("Invalid saved biome region size: " + region.path); } @@ -371,21 +375,22 @@ public final class SavedBiomeStore { } byte[] buffer = new byte[8192]; int records = 0; - while (input.getFilePointer() < fileLength) { - long offset = input.getFilePointer(); + long offset = HEADER_BYTES; + CRC32 crc = new CRC32(); + while (offset < fileLength) { if (fileLength - offset < Integer.BYTES) { - truncateTail(input, offset); + truncateTail(file, offset); break; } int length = input.readInt(); if (length < 12 || length > MAXIMUM_RECORD_BYTES) { throw new IOException("Invalid saved biome record size: " + region.path); } - if (fileLength - input.getFilePointer() < length + 4L) { - truncateTail(input, offset); + if (fileLength - offset - Integer.BYTES < length + 4L) { + truncateTail(file, offset); break; } - CRC32 crc = new CRC32(); + crc.reset(); input.readFully(buffer, 0, 8); crc.update(buffer, 0, 8); ByteBuffer coordinates = ByteBuffer.wrap(buffer, 0, 8); @@ -406,6 +411,7 @@ public final class SavedBiomeStore { } region.offsets[slot] = offset; region.lengths[slot] = length; + offset += length + 2L * Integer.BYTES; } } } diff --git a/core/src/main/java/art/arcane/iris/world/history/SavedTerrainChunkReader.java b/core/src/main/java/art/arcane/iris/world/history/SavedTerrainChunkReader.java index 3e9ab8a5f..5eee1643b 100644 --- a/core/src/main/java/art/arcane/iris/world/history/SavedTerrainChunkReader.java +++ b/core/src/main/java/art/arcane/iris/world/history/SavedTerrainChunkReader.java @@ -15,6 +15,8 @@ import art.arcane.volmlib.util.nbt.tag.NumberTag; import art.arcane.volmlib.util.nbt.tag.StringTag; import art.arcane.volmlib.util.nbt.tag.Tag; import net.jpountz.lz4.LZ4BlockInputStream; +import com.fulcrumgenomics.jlibdeflate.LibdeflateDecompressor; +import com.fulcrumgenomics.jlibdeflate.LibdeflateException; import java.io.ByteArrayInputStream; import java.io.IOException; @@ -23,6 +25,7 @@ import java.io.RandomAccessFile; import java.nio.file.Files; import java.nio.file.LinkOption; import java.nio.file.Path; +import java.nio.file.attribute.BasicFileAttributes; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; @@ -30,7 +33,8 @@ import java.util.List; import java.util.Map; import java.util.TreeMap; import java.util.zip.GZIPInputStream; -import java.util.zip.InflaterInputStream; +import java.util.zip.Inflater; +import java.util.zip.DataFormatException; final class SavedTerrainChunkReader { private static final int SECTOR_BYTES = 4096; @@ -153,7 +157,9 @@ final class SavedTerrainChunkReader { } static String readStatus(Path dimensionRoot, int chunkX, int chunkZ) throws IOException { - return string(readRoot(dimensionRoot, chunkX, chunkZ), "Status"); + try (StatusReader reader = new StatusReader(dimensionRoot)) { + return reader.readStatus(chunkX, chunkZ); + } } private static Section requireSection(Map sections, int y) throws IOException { @@ -245,77 +251,9 @@ final class SavedTerrainChunkReader { } private static CompoundTag readRoot(Path dimensionRoot, int chunkX, int chunkZ) throws IOException { - Path regionDirectory = dimensionRoot.toAbsolutePath().normalize().resolve("region"); - Path region = regionDirectory.resolve("r." + Math.floorDiv(chunkX, 32) + "." - + Math.floorDiv(chunkZ, 32) + ".mca"); - if (!Files.isRegularFile(region, LinkOption.NOFOLLOW_LINKS)) { - throw new IOException("Saved chunk region is unavailable: " + region); + try (StatusReader reader = new StatusReader(dimensionRoot)) { + return reader.readRoot(chunkX, chunkZ); } - byte[] compressed; - int compression; - try (RandomAccessFile input = new RandomAccessFile(region.toFile(), "r")) { - int slot = Math.floorMod(chunkZ, 32) * 32 + Math.floorMod(chunkX, 32); - input.seek((long) slot * 4); - int allocation = input.readInt(); - long offset = (long) (allocation >>> 8) * SECTOR_BYTES; - int sectors = allocation & 255; - if (offset < 2L * SECTOR_BYTES || sectors == 0 || offset + 5 > input.length()) { - throw new IOException("Saved chunk allocation is unavailable: " + chunkX + "," + chunkZ); - } - input.seek(offset); - int length = input.readInt(); - compression = input.readUnsignedByte(); - if ((compression & 128) != 0) { - if (length != 1) { - throw new IOException("External saved chunk has an invalid allocation length"); - } - Path external = regionDirectory.resolve("c." + chunkX + "." + chunkZ + ".mcc"); - if (!Files.isRegularFile(external, LinkOption.NOFOLLOW_LINKS) - || Files.size(external) > MAX_CHUNK_BYTES) { - throw new IOException("External saved chunk is unavailable or too large: " + external); - } - compressed = Files.readAllBytes(external); - compression &= 127; - } else { - if (length < 1 || length > MAX_CHUNK_BYTES || (long) length + 4 > (long) sectors * SECTOR_BYTES - || offset + 4L + length > input.length()) { - throw new IOException("Saved chunk payload is truncated or too large: " + chunkX + "," + chunkZ); - } - compressed = new byte[length - 1]; - input.readFully(compressed); - } - input.seek((long) slot * 4); - if (input.readInt() != allocation) { - throw new IOException("Saved chunk allocation changed during boundary capture: " + chunkX + "," + chunkZ); - } - } - try (InputStream decoded = decompress(compressed, compression)) { - byte[] bytes = decoded.readNBytes(MAX_CHUNK_BYTES + 1); - if (bytes.length > MAX_CHUNK_BYTES) { - throw new IOException("Saved chunk decompressed payload exceeds the size limit"); - } - Tag tag = NBTUtil.read(new ByteArrayInputStream(bytes), false).getTag(); - if (!(tag instanceof CompoundTag root)) { - throw new IOException("Saved chunk root is not a compound"); - } - if (number(root, "xPos") != chunkX || number(root, "zPos") != chunkZ) { - throw new IOException("Saved chunk coordinates do not match its region allocation"); - } - return root; - } catch (IllegalArgumentException | ClassCastException exception) { - throw new IOException("Invalid saved chunk data at " + chunkX + "," + chunkZ, exception); - } - } - - private static InputStream decompress(byte[] encoded, int compression) throws IOException { - InputStream input = new ByteArrayInputStream(encoded); - return switch (compression) { - case 1 -> new GZIPInputStream(input); - case 2 -> new InflaterInputStream(input); - case 3 -> input; - case 4 -> new LZ4BlockInputStream(input); - default -> throw new IOException("Unsupported saved chunk compression " + compression); - }; } private static long[] packedData(CompoundTag tag) throws IOException { @@ -357,6 +295,268 @@ final class SavedTerrainChunkReader { return value.asInt(); } + static final class StatusReader implements AutoCloseable { + private static final boolean NATIVE_AVAILABLE = nativeAvailable(); + + private final Path regionDirectory; + private final Inflater inflater = new Inflater(); + private final LibdeflateDecompressor nativeInflater = NATIVE_AVAILABLE ? new LibdeflateDecompressor() : null; + private byte[] compressed = new byte[65536]; + private byte[] decoded = new byte[65536]; + private Path region; + private RandomAccessFile input; + private Object regionFileKey; + private int regionX; + private int regionZ; + private boolean closed; + + StatusReader(Path dimensionRoot) { + regionDirectory = dimensionRoot.toAbsolutePath().normalize().resolve("region"); + } + + SavedTerrainChunk readChunk(int chunkX, int chunkZ, int minimumY, int height) throws IOException { + return SavedTerrainChunkReader.readRoot(readRoot(chunkX, chunkZ), chunkX, chunkZ, minimumY, height); + } + + String readStatus(int chunkX, int chunkZ) throws IOException { + try { + Payload payload = readPayload(chunkX, chunkZ); + return SavedTerrainStatusNbt.read(payload.bytes(), payload.offset(), payload.length(), chunkX, chunkZ); + } catch (IOException exception) { + throw new IOException("Cannot read saved chunk status at " + chunkX + "," + chunkZ + + " in " + regionDirectory, exception); + } + } + + String readStatus(byte[] bytes, SavedTerrainRegionSnapshot.Payload stored, int chunkX, int chunkZ) + throws IOException { + if (closed) { + throw new IOException("Saved chunk status reader is closed"); + } + if (stored.external()) { + return readStatus(chunkX, chunkZ); + } + try { + Payload payload = decode(stored.compression(), bytes, stored.offset(), stored.length()); + return SavedTerrainStatusNbt.read(payload.bytes(), payload.offset(), payload.length(), chunkX, chunkZ); + } catch (IOException exception) { + throw new IOException("Cannot read saved chunk status at " + chunkX + "," + chunkZ + + " in " + regionDirectory, exception); + } + } + + @Override + public void close() throws IOException { + if (closed) { + return; + } + closed = true; + try { + if (input != null) { + input.close(); + } + } finally { + inflater.end(); + if (nativeInflater != null) { + nativeInflater.close(); + } + } + } + + private CompoundTag readRoot(int chunkX, int chunkZ) throws IOException { + try { + Payload payload = readPayload(chunkX, chunkZ); + Tag tag = NBTUtil.read( + new ByteArrayInputStream(payload.bytes(), payload.offset(), payload.length()), false).getTag(); + if (!(tag instanceof CompoundTag root)) { + throw new IOException("Saved chunk root is not a compound"); + } + if (number(root, "xPos") != chunkX || number(root, "zPos") != chunkZ) { + throw new IOException("Saved chunk coordinates do not match its region allocation"); + } + return root; + } catch (IllegalArgumentException | ClassCastException exception) { + throw new IOException("Invalid saved chunk data at " + chunkX + "," + chunkZ, exception); + } + } + + private Payload readPayload(int chunkX, int chunkZ) throws IOException { + openRegion(chunkX, chunkZ); + int slot = Math.floorMod(chunkZ, 32) * 32 + Math.floorMod(chunkX, 32); + input.seek((long) slot * 4); + int allocation = input.readInt(); + long offset = (long) (allocation >>> 8) * SECTOR_BYTES; + int sectors = allocation & 255; + if (offset < 2L * SECTOR_BYTES || sectors == 0 || offset + 5 > input.length()) { + throw new IOException("Saved chunk allocation is unavailable: " + chunkX + "," + chunkZ); + } + input.seek(offset); + int length = input.readInt(); + int compression = input.readUnsignedByte(); + int compressedLength; + if ((compression & 128) != 0) { + if (length != 1) { + throw new IOException("External saved chunk has an invalid allocation length"); + } + compressedLength = readExternal(chunkX, chunkZ); + compression &= 127; + } else { + if (length < 1 || length > MAX_CHUNK_BYTES || (long) length + 4 > (long) sectors * SECTOR_BYTES + || offset + 4L + length > input.length()) { + throw new IOException("Saved chunk payload is truncated or too large: " + chunkX + "," + chunkZ); + } + compressedLength = length - 1; + ensureCompressedCapacity(compressedLength); + input.readFully(compressed, 0, compressedLength); + } + input.seek((long) slot * 4); + if (input.readInt() != allocation) { + throw new IOException("Saved chunk allocation changed during boundary capture: " + chunkX + "," + chunkZ); + } + return decode(compression, compressed, 0, compressedLength); + } + + private void openRegion(int chunkX, int chunkZ) throws IOException { + if (closed) { + throw new IOException("Saved chunk status reader is closed"); + } + int nextRegionX = Math.floorDiv(chunkX, 32); + int nextRegionZ = Math.floorDiv(chunkZ, 32); + boolean sameRegion = input != null && regionX == nextRegionX && regionZ == nextRegionZ; + Path nextRegion = sameRegion ? region + : regionDirectory.resolve("r." + nextRegionX + "." + nextRegionZ + ".mca"); + BasicFileAttributes attributes = Files.readAttributes(nextRegion, BasicFileAttributes.class, + LinkOption.NOFOLLOW_LINKS); + if (!attributes.isRegularFile()) { + throw new IOException("Saved chunk region is unavailable: " + nextRegion); + } + Object nextFileKey = attributes.fileKey(); + if (sameRegion && nextFileKey != null && nextFileKey.equals(regionFileKey)) { + return; + } + if (input != null) { + input.close(); + input = null; + } + input = new RandomAccessFile(nextRegion.toFile(), "r"); + region = nextRegion; + regionFileKey = nextFileKey; + regionX = nextRegionX; + regionZ = nextRegionZ; + } + + private int readExternal(int chunkX, int chunkZ) throws IOException { + Path external = regionDirectory.resolve("c." + chunkX + "." + chunkZ + ".mcc"); + if (!Files.isRegularFile(external, LinkOption.NOFOLLOW_LINKS)) { + throw new IOException("External saved chunk is unavailable: " + external); + } + long length = Files.size(external); + if (length > MAX_CHUNK_BYTES) { + throw new IOException("External saved chunk is too large: " + external); + } + int size = (int) length; + ensureCompressedCapacity(size); + try (InputStream externalInput = Files.newInputStream(external)) { + if (externalInput.readNBytes(compressed, 0, size) != size || externalInput.read() != -1) { + throw new IOException("External saved chunk length changed during capture: " + external); + } + } + return size; + } + + private void ensureCompressedCapacity(int size) { + if (size > compressed.length) { + compressed = new byte[Math.max(size, Math.min(MAX_CHUNK_BYTES, compressed.length * 2))]; + } + } + + private Payload decode(int compression, byte[] bytes, int offset, int length) throws IOException { + if (compression == 2) { + int size = inflate(bytes, offset, length); + return new Payload(decoded, 0, size); + } + if (compression == 3) { + return new Payload(bytes, offset, length); + } + InputStream encoded = new ByteArrayInputStream(bytes, offset, length); + try (InputStream stream = switch (compression) { + case 1 -> new GZIPInputStream(encoded); + case 4 -> new LZ4BlockInputStream(encoded); + default -> throw new IOException("Unsupported saved chunk compression " + compression); + }) { + int size = 0; + int count; + do { + ensureDecodedCapacity(size); + count = stream.read(decoded, size, decoded.length - size); + if (count > 0) { + size += count; + checkDecodedSize(size); + } + } while (count != -1); + return new Payload(decoded, 0, size); + } + } + + private int inflate(byte[] bytes, int offset, int length) throws IOException { + if (nativeInflater != null) { + try { + int size = nativeInflater.zlibDecompressEx(bytes, offset, length, decoded, 0, decoded.length) + .outputBytesProduced(); + checkDecodedSize(size); + return size; + } catch (LibdeflateException exception) { + return inflateJava(bytes, offset, length); + } + } + return inflateJava(bytes, offset, length); + } + + private static boolean nativeAvailable() { + try (LibdeflateDecompressor ignored = new LibdeflateDecompressor()) { + return true; + } catch (LinkageError | SecurityException exception) { + return false; + } + } + + private int inflateJava(byte[] bytes, int offset, int length) throws IOException { + inflater.reset(); + inflater.setInput(bytes, offset, length); + int size = 0; + try { + while (!inflater.finished()) { + ensureDecodedCapacity(size); + int count = inflater.inflate(decoded, size, decoded.length - size); + size += count; + checkDecodedSize(size); + if (count == 0 && !inflater.finished()) { + throw new IOException("Saved chunk zlib payload is truncated or requires a dictionary"); + } + } + } catch (DataFormatException exception) { + throw new IOException("Invalid saved chunk zlib payload", exception); + } + return size; + } + + private void ensureDecodedCapacity(int size) throws IOException { + checkDecodedSize(size); + if (size == decoded.length) { + decoded = Arrays.copyOf(decoded, Math.min(MAX_CHUNK_BYTES + 1, decoded.length * 2)); + } + } + + private void checkDecodedSize(int size) throws IOException { + if (size > MAX_CHUNK_BYTES) { + throw new IOException("Saved chunk decompressed payload exceeds the size limit"); + } + } + } + + private record Payload(byte[] bytes, int offset, int length) { + } + private record Section(Palette blocks, Palette biomes) { } diff --git a/core/src/main/java/art/arcane/iris/world/history/SavedTerrainRecoveryScan.java b/core/src/main/java/art/arcane/iris/world/history/SavedTerrainRecoveryScan.java new file mode 100644 index 000000000..e3fba4542 --- /dev/null +++ b/core/src/main/java/art/arcane/iris/world/history/SavedTerrainRecoveryScan.java @@ -0,0 +1,219 @@ +package art.arcane.iris.world.history; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.BitSet; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; + +final class SavedTerrainRecoveryScan implements AutoCloseable { + private static final long WORKER_HEAP_BYTES = 768L * 1024 * 1024; + private static final int MAX_REGION_BYTES = 64 * 1024 * 1024; + private static final int MIN_PARALLEL_CHUNKS = 128; + + private final Path world; + private final int regionByteLimit; + private final SavedTerrainChunkReader.StatusReader[] readers; + private final ExecutorService executor; + private byte[] regionBuffer = new byte[0]; + + SavedTerrainRecoveryScan(Path world, Limits limits) { + this.world = world; + regionByteLimit = limits.regionBytes(); + readers = new SavedTerrainChunkReader.StatusReader[limits.workers()]; + for (int index = 0; index < readers.length; index++) { + readers[index] = new SavedTerrainChunkReader.StatusReader(world); + } + executor = readers.length == 1 ? null : Executors.newFixedThreadPool(readers.length, + Thread.ofPlatform().daemon().name("Iris terrain recovery ", 1).factory()); + } + + static WorldChunkInventory scan(Path world, WorldChunkInventory inventory, + WorldChunkInventory.ChunkPredicate retained) throws IOException { + Runtime runtime = Runtime.getRuntime(); + try (SavedTerrainRecoveryScan scan = new SavedTerrainRecoveryScan(world, + Limits.forRuntime(runtime.availableProcessors(), runtime.maxMemory()))) { + return scan.filter(inventory, retained); + } + } + + WorldChunkInventory filter(WorldChunkInventory inventory, WorldChunkInventory.ChunkPredicate retained) + throws IOException { + return inventory.filterRegions((regionX, regionZ, allocated) -> + filterRegion(new Region(regionX, regionZ, allocated), retained)); + } + + @Override + public void close() throws IOException { + if (executor != null) { + executor.close(); + } + IOException failure = null; + for (SavedTerrainChunkReader.StatusReader reader : readers) { + try { + reader.close(); + } catch (IOException exception) { + if (failure == null) { + failure = exception; + } else { + failure.addSuppressed(exception); + } + } + } + if (failure != null) { + throw failure; + } + } + + private BitSet filterRegion(Region region, WorldChunkInventory.ChunkPredicate retained) throws IOException { + requireNotInterrupted(); + BitSet accepted = new BitSet(1024); + BitSet pending = (BitSet) region.allocated().clone(); + for (int slot = pending.nextSetBit(0); slot >= 0; slot = pending.nextSetBit(slot + 1)) { + if (retained.test(region.chunkX(slot), region.chunkZ(slot))) { + accepted.set(slot); + pending.clear(slot); + } + } + if (pending.isEmpty()) { + return accepted; + } + Path file = world.resolve("region/r." + region.x() + "." + region.z() + ".mca"); + int snapshotLimit = Math.min(regionByteLimit, pending.cardinality() * 64 * 1024); + Optional captured = SavedTerrainRegionSnapshot.read(file, snapshotLimit, regionBuffer); + if (captured.isEmpty()) { + for (int slot = pending.nextSetBit(0); slot >= 0; slot = pending.nextSetBit(slot + 1)) { + requireNotInterrupted(); + if (SavedTerrainChunk.hasTerrain(readers[0].readStatus(region.chunkX(slot), region.chunkZ(slot)))) { + accepted.set(slot); + } + } + return accepted; + } + SavedTerrainRegionSnapshot snapshot = captured.get(); + regionBuffer = snapshot.bytes(); + for (int slot = pending.nextSetBit(0); slot >= 0; slot = pending.nextSetBit(slot + 1)) { + requireNotInterrupted(); + if (snapshot.payload(slot).external()) { + if (SavedTerrainChunk.hasTerrain(readers[0].readStatus(region.chunkX(slot), region.chunkZ(slot)))) { + accepted.set(slot); + } + pending.clear(slot); + } + } + if (executor == null || pending.cardinality() < MIN_PARALLEL_CHUNKS) { + accepted.or(decode(new DecodeRegion(region, pending, snapshot), 0, 1)); + } else { + List> tasks = new ArrayList<>(readers.length); + DecodeRegion input = new DecodeRegion(region, pending, snapshot); + for (int index = 0; index < readers.length; index++) { + int worker = index; + tasks.add(executor.submit(() -> decode(input, worker, readers.length))); + } + accepted.or(await(tasks)); + } + snapshot.validateUnchanged(); + return accepted; + } + + private BitSet decode(DecodeRegion input, int worker, int stride) throws IOException { + BitSet accepted = new BitSet(1024); + int ordinal = 0; + for (int slot = input.pending().nextSetBit(0); slot >= 0; slot = input.pending().nextSetBit(slot + 1)) { + if (ordinal++ % stride != worker) { + continue; + } + requireNotInterrupted(); + String status = readers[worker].readStatus(input.snapshot().bytes(), input.snapshot().payload(slot), + input.region().chunkX(slot), input.region().chunkZ(slot)); + if (SavedTerrainChunk.hasTerrain(status)) { + accepted.set(slot); + } + } + return accepted; + } + + private BitSet await(List> tasks) throws IOException { + BitSet accepted = new BitSet(1024); + Throwable failure = null; + boolean interrupted = false; + try { + for (Future task : tasks) { + boolean completed = false; + while (!completed) { + try { + accepted.or(task.get()); + completed = true; + } catch (InterruptedException exception) { + interrupted = true; + if (failure == null) { + failure = new IOException("Interrupted while reading saved terrain", exception); + } + } catch (ExecutionException exception) { + if (failure == null) { + failure = exception.getCause(); + } else if (failure != exception.getCause()) { + failure.addSuppressed(exception.getCause()); + } + completed = true; + } + } + } + } finally { + if (interrupted) { + Thread.currentThread().interrupt(); + } + } + if (failure instanceof IOException exception) { + throw exception; + } + if (failure instanceof RuntimeException exception) { + throw exception; + } + if (failure instanceof Error error) { + throw error; + } + if (failure != null) { + throw new IOException("Saved terrain recovery failed", failure); + } + return accepted; + } + + private void requireNotInterrupted() throws IOException { + if (Thread.currentThread().isInterrupted()) { + throw new IOException("Saved terrain recovery interrupted"); + } + } + + record Limits(int workers, int regionBytes) { + Limits { + if (workers < 1 || workers > 4 || regionBytes < 8192 || regionBytes > MAX_REGION_BYTES) { + throw new IllegalArgumentException("Invalid saved terrain recovery limits"); + } + } + + static Limits forRuntime(int processors, long heapBytes) { + int workers = (int) Math.max(1, Math.min(Math.min(processors, 4), heapBytes / WORKER_HEAP_BYTES)); + int regionBytes = (int) Math.max(8192, Math.min(MAX_REGION_BYTES, heapBytes / 16)); + return new Limits(workers, regionBytes); + } + } + + private record Region(int x, int z, BitSet allocated) { + int chunkX(int slot) { + return x * 32 + (slot & 31); + } + + int chunkZ(int slot) { + return z * 32 + (slot >> 5); + } + } + + private record DecodeRegion(Region region, BitSet pending, SavedTerrainRegionSnapshot snapshot) { + } +} diff --git a/core/src/main/java/art/arcane/iris/world/history/SavedTerrainRegionSnapshot.java b/core/src/main/java/art/arcane/iris/world/history/SavedTerrainRegionSnapshot.java new file mode 100644 index 000000000..9b1493cc7 --- /dev/null +++ b/core/src/main/java/art/arcane/iris/world/history/SavedTerrainRegionSnapshot.java @@ -0,0 +1,138 @@ +package art.arcane.iris.world.history; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; +import java.nio.file.Files; +import java.nio.file.LinkOption; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.nio.file.attribute.BasicFileAttributes; +import java.util.Arrays; +import java.util.Objects; +import java.util.Optional; + +final class SavedTerrainRegionSnapshot { + private static final int SECTOR_BYTES = 4_096; + private static final int HEADER_BYTES = 2 * SECTOR_BYTES; + private static final int MAX_CHUNK_BYTES = 64 * 1024 * 1024; + + private final Path file; + private final BasicFileAttributes attributes; + private final byte[] bytes; + private final int byteLength; + + private SavedTerrainRegionSnapshot(SnapshotData data) { + file = data.file(); + attributes = data.attributes(); + bytes = data.bytes(); + byteLength = Math.toIntExact(attributes.size()); + } + + static Optional read(Path file, int maxBytes, byte[] reusable) throws IOException { + Objects.requireNonNull(file, "file"); + Objects.requireNonNull(reusable, "reusable"); + if (maxBytes < 0) { + throw new IllegalArgumentException("Snapshot byte limit must not be negative"); + } + Path normalized = file.toAbsolutePath().normalize(); + BasicFileAttributes attributes = readAttributes(normalized); + if (attributes.size() < HEADER_BYTES) { + throw new IOException("Truncated saved chunk region: " + normalized); + } + if (attributes.size() > maxBytes || attributes.fileKey() == null) { + return Optional.empty(); + } + int byteLength = (int) attributes.size(); + byte[] bytes = reusable.length >= byteLength ? reusable : new byte[byteLength]; + try (FileChannel input = FileChannel.open(normalized, StandardOpenOption.READ, LinkOption.NOFOLLOW_LINKS)) { + readFully(input, ByteBuffer.wrap(bytes, 0, byteLength), normalized); + if (input.size() != byteLength) { + throw new IOException("Saved chunk region length changed during capture: " + normalized); + } + } + SavedTerrainRegionSnapshot snapshot = new SavedTerrainRegionSnapshot( + new SnapshotData(normalized, attributes, bytes)); + snapshot.validateAttributes(); + return Optional.of(snapshot); + } + + byte[] bytes() { + return bytes; + } + + Payload payload(int index) throws IOException { + if (index < 0 || index >= 1024) { + throw new IOException("Saved chunk allocation index is invalid: " + index); + } + int allocation = readInt(bytes, index * Integer.BYTES); + long offset = (long) (allocation >>> Byte.SIZE) * SECTOR_BYTES; + int sectors = allocation & 255; + if (offset < HEADER_BYTES || sectors == 0 || offset + 5 > byteLength) { + throw new IOException("Saved chunk allocation is unavailable in " + file + " at " + index); + } + int start = (int) offset; + int length = readInt(bytes, start); + int compression = bytes[start + Integer.BYTES] & 255; + if (length < 1 || length > MAX_CHUNK_BYTES || (long) length + Integer.BYTES > (long) sectors * SECTOR_BYTES + || offset + Integer.BYTES + length > byteLength) { + throw new IOException("Saved chunk payload is truncated or too large in " + file + " at " + index); + } + boolean external = (compression & 128) != 0; + if (external && length != 1) { + throw new IOException("External saved chunk has an invalid allocation length in " + file); + } + return new Payload(start + 5, length - 1, compression & 127, external); + } + + void validateUnchanged() throws IOException { + validateAttributes(); + byte[] header = new byte[HEADER_BYTES]; + try (FileChannel input = FileChannel.open(file, StandardOpenOption.READ, LinkOption.NOFOLLOW_LINKS)) { + readFully(input, ByteBuffer.wrap(header), file); + if (input.size() != byteLength) { + throw new IOException("Saved chunk region length changed during capture: " + file); + } + } + if (!Arrays.equals(bytes, 0, HEADER_BYTES, header, 0, HEADER_BYTES)) { + throw new IOException("Saved chunk region header changed during capture: " + file); + } + validateAttributes(); + } + + private static BasicFileAttributes readAttributes(Path file) throws IOException { + BasicFileAttributes attributes = Files.readAttributes(file, BasicFileAttributes.class, + LinkOption.NOFOLLOW_LINKS); + if (!attributes.isRegularFile()) { + throw new IOException("Saved chunk region is not a regular file: " + file); + } + return attributes; + } + + private static void readFully(FileChannel input, ByteBuffer buffer, Path file) throws IOException { + while (buffer.hasRemaining()) { + if (input.read(buffer) < 0) { + throw new IOException("Truncated saved chunk region: " + file); + } + } + } + + private static int readInt(byte[] bytes, int offset) { + return (bytes[offset] & 255) << 24 | (bytes[offset + 1] & 255) << 16 + | (bytes[offset + 2] & 255) << 8 | bytes[offset + 3] & 255; + } + + private void validateAttributes() throws IOException { + BasicFileAttributes current = readAttributes(file); + if (!Objects.equals(attributes.fileKey(), current.fileKey()) || attributes.size() != current.size() + || !attributes.lastModifiedTime().equals(current.lastModifiedTime())) { + throw new IOException("Saved chunk region changed during capture: " + file); + } + } + + record Payload(int offset, int length, int compression, boolean external) { + } + + private record SnapshotData(Path file, BasicFileAttributes attributes, byte[] bytes) { + } +} diff --git a/core/src/main/java/art/arcane/iris/world/history/SavedTerrainStatusNbt.java b/core/src/main/java/art/arcane/iris/world/history/SavedTerrainStatusNbt.java new file mode 100644 index 000000000..98bb98fc9 --- /dev/null +++ b/core/src/main/java/art/arcane/iris/world/history/SavedTerrainStatusNbt.java @@ -0,0 +1,225 @@ +package art.arcane.iris.world.history; + +import art.arcane.volmlib.util.nbt.tag.Tag; + +import java.io.ByteArrayInputStream; +import java.io.DataInputStream; +import java.io.EOFException; +import java.io.IOException; + +final class SavedTerrainStatusNbt { + private final byte[] bytes; + private final int limit; + private int position; + + private SavedTerrainStatusNbt(byte[] bytes, int length) { + this.bytes = bytes; + this.limit = length; + } + + static String read(byte[] bytes, int offset, int length, int chunkX, int chunkZ) throws IOException { + if (offset < 0 || length < 0 || offset > bytes.length - length) { + throw new IOException("Invalid saved chunk NBT payload bounds"); + } + SavedTerrainStatusNbt input = new SavedTerrainStatusNbt(bytes, offset + length); + input.position = offset; + if (input.readUnsignedByte() != 10) { + throw new IOException("Saved chunk root is not a compound"); + } + input.readUtf(); + Integer savedX = null; + Integer savedZ = null; + String status = null; + for (int type = input.readUnsignedByte(); type != 0; type = input.readUnsignedByte()) { + int start = input.position + 2; + int size = input.readUtf(); + boolean xName = input.matches(start, size, "xPos"); + boolean zName = input.matches(start, size, "zPos"); + boolean statusName = input.matches(start, size, "Status"); + if (xName || zName) { + Integer value = null; + if (type >= 1 && type <= 6) { + value = input.number(type); + } else { + input.skip(type, Tag.DEFAULT_MAX_DEPTH - 1); + } + if (xName) { + savedX = value; + } else { + savedZ = value; + } + } else if (statusName) { + if (type == 8) { + int valueStart = input.position; + int valueLength = input.readUtf(); + status = new DataInputStream(new ByteArrayInputStream(bytes, valueStart, valueLength + 2)).readUTF(); + } else { + input.skip(type, Tag.DEFAULT_MAX_DEPTH - 1); + status = null; + } + } else { + input.skip(type, Tag.DEFAULT_MAX_DEPTH - 1); + } + } + if (savedX == null || savedZ == null || savedX != chunkX || savedZ != chunkZ) { + throw new IOException("Saved chunk coordinates do not match its region allocation"); + } + if (status == null || status.isBlank()) { + throw new IOException("Saved chunk field Status is not a nonempty string"); + } + return status; + } + + private boolean matches(int start, int size, String name) { + if (size < name.length() || size > name.length() * 3) { + return false; + } + if (size == name.length()) { + for (int index = 0; index < name.length(); index++) { + if (bytes[start + index] != name.charAt(index)) { + return false; + } + } + return true; + } + int end = start + size; + for (int index = 0; index < name.length(); index++) { + if (start == end) { + return false; + } + int value = bytes[start++] & 255; + if ((value & 224) == 192) { + value = (value & 31) << 6 | bytes[start++] & 63; + } else if ((value & 240) == 224) { + value = (value & 15) << 12 | (bytes[start++] & 63) << 6 | bytes[start++] & 63; + } + if (value != name.charAt(index)) { + return false; + } + } + return start == end; + } + + private int number(int type) throws IOException { + return switch (type) { + case 1 -> (byte) readUnsignedByte(); + case 2 -> (short) readUnsignedShort(); + case 3 -> readInt(); + case 4 -> (int) readLong(); + case 5 -> (int) Float.intBitsToFloat(readInt()); + case 6 -> (int) Double.longBitsToDouble(readLong()); + default -> throw new IOException("Not numeric"); + }; + } + + private void require(long count) throws EOFException { + if (count < 0 || count > limit - position) { + throw new EOFException("Truncated saved chunk NBT payload"); + } + } + + private void advance(long count) throws IOException { + require(count); + position += (int) count; + } + + private int readUnsignedByte() throws IOException { + require(1); + return bytes[position++] & 255; + } + + private int readUnsignedShort() throws IOException { + require(2); + int result = (bytes[position] & 255) << 8 | bytes[position + 1] & 255; + position += 2; + return result; + } + + private int readInt() throws IOException { + require(4); + int result = (bytes[position] & 255) << 24 | (bytes[position + 1] & 255) << 16 + | (bytes[position + 2] & 255) << 8 | bytes[position + 3] & 255; + position += 4; + return result; + } + + private long readLong() throws IOException { + return (long) readInt() << 32 | Integer.toUnsignedLong(readInt()); + } + + private int readUtf() throws IOException { + int length = readUnsignedShort(); + require(length); + int end = position + length; + while (position < end) { + int first = bytes[position++] & 255; + if (first < 128) { + continue; + } + if ((first & 224) == 192) { + if (position >= end || (bytes[position++] & 192) != 128) { + throw new IOException("Invalid saved chunk modified UTF-8"); + } + } else if ((first & 240) == 224) { + if (end - position < 2 || (bytes[position++] & 192) != 128 || (bytes[position++] & 192) != 128) { + throw new IOException("Invalid saved chunk modified UTF-8"); + } + } else { + throw new IOException("Invalid modified UTF-8"); + } + } + return length; + } + + private void skip(int type, int depth) throws IOException { + if (depth < 0) { + throw new IOException("Saved chunk NBT nesting limit exceeded"); + } + switch (type) { + case 0 -> { + } + case 1 -> advance(1); + case 2 -> advance(2); + case 3, 5 -> advance(4); + case 4, 6 -> advance(8); + case 7, 11, 12 -> { + int length = readInt(); + if (length < 0) { + throw new IOException("Negative saved chunk NBT array length"); + } + advance((long) length * (type == 7 ? 1 : type == 11 ? 4 : 8)); + } + case 8 -> readUtf(); + case 9 -> { + int child = readUnsignedByte(); + int length = readInt(); + if (length > 0 && child > 12) { + throw new IOException("Invalid saved chunk NBT list type"); + } + if (length > 0 && depth == 0) { + throw new IOException("Saved chunk NBT nesting limit exceeded"); + } + if (child >= 1 && child <= 6) { + int width = switch (child) { + case 1 -> 1; + case 2 -> 2; + case 3, 5 -> 4; + default -> 8; + }; + advance((long) Math.max(0, length) * width); + } else if (child != 0) { + for (int index = 0; index < length; index++) { + skip(child, depth - 1); + } + } + } + case 10 -> { + for (int child = readUnsignedByte(); child != 0; child = readUnsignedByte()) { + readUtf(); + skip(child, depth - 1); + } + } + default -> throw new IOException("Invalid NBT tag type"); + } + } +} diff --git a/core/src/main/java/art/arcane/iris/world/history/TerrainBoundarySignatureStore.java b/core/src/main/java/art/arcane/iris/world/history/TerrainBoundarySignatureStore.java index cfcdbe4fe..f91df067c 100644 --- a/core/src/main/java/art/arcane/iris/world/history/TerrainBoundarySignatureStore.java +++ b/core/src/main/java/art/arcane/iris/world/history/TerrainBoundarySignatureStore.java @@ -174,6 +174,7 @@ public final class TerrainBoundarySignatureStore { } private Snapshot publishCatalog(long activationId, PublishedCatalog catalog) throws IOException { + forceDirectory(directory); Path target = snapshotPath(activationId); if (Files.exists(target, LinkOption.NOFOLLOW_LINKS)) { Snapshot existing = load(activationId); @@ -186,6 +187,7 @@ public final class TerrainBoundarySignatureStore { return existing; } publishAtomic(target, catalog.encoded(), "terrain signature catalog"); + forceDirectory(directory); return load(activationId); } @@ -533,7 +535,6 @@ public final class TerrainBoundarySignatureStore { } catch (AtomicMoveNotSupportedException error) { throw new IOException("Terrain boundary " + kind + " publication requires an atomic move", error); } - forceDirectory(directory); } finally { Files.deleteIfExists(temporary); } diff --git a/core/src/main/java/art/arcane/iris/world/history/WorldChunkInventory.java b/core/src/main/java/art/arcane/iris/world/history/WorldChunkInventory.java index 922913596..db0dedaf9 100644 --- a/core/src/main/java/art/arcane/iris/world/history/WorldChunkInventory.java +++ b/core/src/main/java/art/arcane/iris/world/history/WorldChunkInventory.java @@ -87,6 +87,30 @@ public final class WorldChunkInventory { boolean test(int chunkX, int chunkZ) throws IOException; } + WorldChunkInventory filterRegions(RegionPredicate predicate) throws IOException { + Objects.requireNonNull(predicate, "region predicate"); + Long2ObjectOpenHashMap selected = new Long2ObjectOpenHashMap<>(regionMasks.size()); + for (long regionKey : regionKeys) { + BitSet source = regionMasks.get(regionKey); + BitSet accepted = predicate.test(ChunkGenerationOwnership.chunkX(regionKey), + ChunkGenerationOwnership.chunkZ(regionKey), (BitSet) source.clone()); + BitSet unexpected = (BitSet) accepted.clone(); + unexpected.andNot(source); + if (!unexpected.isEmpty()) { + throw new IOException("Filtered world region contains unallocated chunks"); + } + if (!accepted.isEmpty()) { + selected.put(regionKey, accepted); + } + } + return new WorldChunkInventory(selected); + } + + @FunctionalInterface + interface RegionPredicate { + BitSet test(int regionX, int regionZ, BitSet allocated) throws IOException; + } + public static boolean isDurablyAllocated(Path worldDirectory, int chunkX, int chunkZ) throws IOException { Path regionDirectory = Objects.requireNonNull(worldDirectory, "worldDirectory") .toAbsolutePath() diff --git a/core/src/test/java/art/arcane/iris/platform/generation/BukkitChunkGeneratorHistoryTargetTest.java b/core/src/test/java/art/arcane/iris/platform/generation/BukkitChunkGeneratorHistoryTargetTest.java index 7ed14b429..4ddbe7583 100644 --- a/core/src/test/java/art/arcane/iris/platform/generation/BukkitChunkGeneratorHistoryTargetTest.java +++ b/core/src/test/java/art/arcane/iris/platform/generation/BukkitChunkGeneratorHistoryTargetTest.java @@ -25,7 +25,6 @@ import static org.junit.Assert.assertNull; import static org.junit.Assert.assertSame; import static org.junit.Assert.assertThrows; import static org.mockito.Mockito.CALLS_REAL_METHODS; -import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; @@ -63,7 +62,7 @@ public class BukkitChunkGeneratorHistoryTargetTest { fixture.generator.prepareGenerationHistoryTarget(128); assertSame(fixture.target, fixture.generator.getTarget()); - verify(fixture.history).prepareCurrentGenerator(128); + verify(fixture.history).prepareStartup(128); verify(fixture.history).activePackRoot(); verify(fixture.data, never()).close(); verify(fixture.data, never()).dump(); @@ -102,7 +101,8 @@ public class BukkitChunkGeneratorHistoryTargetTest { assertSame(failure, assertThrows(IOException.class, () -> fixture.generator.prepareGenerationHistoryTarget(128))); - verify(fixture.history).prepareCurrentGenerator(128); + verify(fixture.history).prepareStartup(128); + verify(fixture.preparation).close(); verify(fixture.data, never()).close(); } @@ -110,7 +110,7 @@ public class BukkitChunkGeneratorHistoryTargetTest { public void preparationFailureCannotBeHiddenByAnExistingTarget() throws Exception { Fixture fixture = new Fixture(); IOException failure = new IOException("Pending activation is invalid"); - doThrow(failure).when(fixture.history).prepareCurrentGenerator(128); + when(fixture.history.prepareStartup(128)).thenThrow(failure); assertSame(failure, assertThrows(IOException.class, () -> fixture.generator.prepareGenerationHistoryTarget(128))); @@ -167,12 +167,14 @@ public class BukkitChunkGeneratorHistoryTargetTest { private static final class Fixture { private final BukkitChunkGenerator generator = mock(BukkitChunkGenerator.class, CALLS_REAL_METHODS); private final GenerationHistory history = mock(GenerationHistory.class); + private final GenerationHistory.StartupPreparation preparation = mock(GenerationHistory.StartupPreparation.class); private final AtomicCache targets = new AtomicCache<>(); private final EngineTarget target = mock(EngineTarget.class); private final IrisData data = mock(IrisData.class); private Fixture() throws Exception { Path packRoot = Path.of("generation", "packs", "current"); + when(history.prepareStartup(128)).thenReturn(preparation); when(history.activePackRoot()).thenReturn(packRoot); when(target.getData()).thenReturn(data); when(data.getDataFolder()).thenReturn(packRoot.toFile()); diff --git a/core/src/test/java/art/arcane/iris/world/history/BukkitStartupRecoveryScanTest.java b/core/src/test/java/art/arcane/iris/world/history/BukkitStartupRecoveryScanTest.java new file mode 100644 index 000000000..cfd8ba45e --- /dev/null +++ b/core/src/test/java/art/arcane/iris/world/history/BukkitStartupRecoveryScanTest.java @@ -0,0 +1,97 @@ +package art.arcane.iris.world.history; + +import art.arcane.iris.configuration.IrisSettings; +import art.arcane.iris.generation.runtime.EngineTarget; +import art.arcane.iris.generation.runtime.IrisEngine; +import art.arcane.iris.generation.terrain.IrisDimension; +import art.arcane.iris.pack.loading.IrisData; +import art.arcane.iris.platform.generation.BukkitChunkGenerator; +import art.arcane.iris.world.IrisWorld; +import art.arcane.volmlib.util.cache.AtomicCache; +import art.arcane.volmlib.util.collection.KList; +import org.bukkit.generator.BlockPopulator; +import org.junit.Test; +import org.mockito.ArgumentCaptor; +import org.mockito.MockedConstruction; +import org.mockito.MockedStatic; + +import java.lang.reflect.Field; +import java.lang.reflect.Method; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.concurrent.atomic.AtomicBoolean; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertSame; +import static org.mockito.Mockito.CALLS_REAL_METHODS; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.mockConstruction; +import static org.mockito.Mockito.mockStatic; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public final class BukkitStartupRecoveryScanTest extends GenerationHistoryRuntimeRouterSupport { + @Test + public void unchangedBukkitStartupScansSavedChunksOnce() throws Exception { + Path world = temporaryFolder.newFolder("startup-recovery-world").toPath(); + GenerationHistory history = createHistory(world, createPack("startup-recovery-pack", "alpha")); + Path region = Files.createDirectories(world.resolve("region")).resolve("r.0.0.mca"); + writeRegion(region, new int[][]{{0, 0}, {1, 0}}); + byte[] originalRegion = Files.readAllBytes(region); + GenerationEpoch epoch = history.activeEpoch(); + EngineTarget target = mock(EngineTarget.class); + IrisData data = mock(IrisData.class); + IrisDimension dimension = mock(IrisDimension.class); + IrisWorld irisWorld = mock(IrisWorld.class); + when(target.getData()).thenReturn(data); + when(target.getDimension()).thenReturn(dimension); + when(target.getWorld()).thenReturn(irisWorld); + when(data.getDataFolder()).thenReturn(history.activePackRoot().toFile()); + when(dimension.getLoadKey()).thenReturn(epoch.dimensionContract().dimensionKey()); + when(irisWorld.getRawWorldSeed()).thenReturn(epoch.worldSeed()); + IrisEngine.GenerationRuntimeBinding binding = new FakeRuntimeFactory().binding(history, history.activeActivation()); + when(binding.target()).thenReturn(target); + BukkitChunkGenerator generator = mock(BukkitChunkGenerator.class, CALLS_REAL_METHODS); + AtomicCache targets = new AtomicCache<>(); + targets.aquireOrThrow(() -> target); + setField(generator, "generationHistory", history); + setField(generator, "targetCache", targets); + setField(generator, "studioEntryBootstrapActive", new AtomicBoolean()); + setField(generator, "populators", new KList()); + IrisSettings settings = new IrisSettings(); + + try (MockedStatic configuration = mockStatic(IrisSettings.class); + MockedStatic contracts = mockStatic(GenerationEpochContractFactory.class); + MockedStatic inventory = mockStatic(WorldChunkInventory.class, CALLS_REAL_METHODS); + MockedConstruction engines = mockConstruction(IrisEngine.class, (engine, context) -> { + when(engine.getActiveGenerationRuntimeBinding()).thenReturn(binding); + })) { + configuration.when(IrisSettings::get).thenReturn(settings); + contracts.when(() -> GenerationEpochContractFactory.createForEpoch( + dimension, epoch.dimensionContract().dimensionTypeKey(), epoch)).thenReturn(epoch.dimensionContract()); + Method setup = BukkitChunkGenerator.class.getDeclaredMethod("setupEngine"); + setup.setAccessible(true); + setup.invoke(generator); + + assertEquals(1, engines.constructed().size()); + IrisEngine engine = engines.constructed().getFirst(); + ArgumentCaptor attached = ArgumentCaptor.forClass(GenerationHistoryRuntimeRouter.class); + verify(engine).attachGenerationHistoryRuntimeRouter(attached.capture()); + try (GenerationHistoryRuntimeRouter router = attached.getValue()) { + assertSame(history, router.history()); + assertSame(engine, generator.getEngine()); + assertEquals(1L, history.activeActivation().activationId()); + inventory.verify(() -> WorldChunkInventory.scan(world), times(1)); + assertArrayEquals(originalRegion, Files.readAllBytes(region)); + } + } + } + + private static void setField(BukkitChunkGenerator generator, String name, Object value) throws Exception { + Field field = BukkitChunkGenerator.class.getDeclaredField(name); + field.setAccessible(true); + field.set(generator, value); + } +} diff --git a/core/src/test/java/art/arcane/iris/world/history/GenerationHistoryStartupValidationTest.java b/core/src/test/java/art/arcane/iris/world/history/GenerationHistoryStartupValidationTest.java new file mode 100644 index 000000000..a607175f3 --- /dev/null +++ b/core/src/test/java/art/arcane/iris/world/history/GenerationHistoryStartupValidationTest.java @@ -0,0 +1,170 @@ +package art.arcane.iris.world.history; + +import org.junit.Test; + +import java.io.IOException; +import java.lang.reflect.Field; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.verify; + +public final class GenerationHistoryStartupValidationTest extends GenerationHistorySupport { + @Test + public void startupRetainsSavedClaimsWithoutRepeatingReferenceTraversal() throws Exception { + Path world = temporaryFolder.newFolder("startup-reference-reuse").toPath(); + createHistory(world, createPack("startup-reference-pack", "alpha")); + Path regions = Files.createDirectories(world.resolve("region")); + SavedTerrainTestRegion.write(regions.resolve("r.0.0.mca"), new int[][]{{0, 0}}, "minecraft:noise"); + SavedTerrainTestRegion.write(regions.resolve("r.1.0.mca"), new int[][]{{32, 0}}, "minecraft:biomes"); + GenerationSemanticIndex stored = GenerationSemanticIndex.loadRequired(world); + for (int chunkX : new int[]{0, 32, 64}) { + stored.recordAndPersist(ChunkGenerationSemantics.builder(chunkX, 0, 1L).seal().build()); + } + GenerationHistory history = GenerationHistory.open(world); + GenerationSemanticIndex observed = observeSemantics(history); + + try (GenerationHistory.StartupPreparation ignored = history.prepareStartup(64)) { + assertEquals(1L, history.activeActivation().activationId()); + } + + verify(observed, never()).forEachRecord(any()); + assertTrue(history.semantics(0, 0).isPresent()); + assertTrue(history.semantics(32, 0).isEmpty()); + assertTrue(history.semantics(64, 0).isEmpty()); + GenerationHistory reopened = GenerationHistory.open(world); + assertTrue(reopened.semantics(0, 0).isPresent()); + assertTrue(reopened.semantics(32, 0).isEmpty()); + assertTrue(reopened.semantics(64, 0).isEmpty()); + } + + @Test + public void startupPromotionRecoversOrphansWithoutRepeatingReferenceTraversal() throws Exception { + Path world = temporaryFolder.newFolder("startup-promotion-reference-reuse").toPath(); + createHistory(world, createPack("startup-promotion-pack-a", "alpha")); + GenerationSemanticIndex stored = GenerationSemanticIndex.loadRequired(world); + stored.recordAndPersist(ChunkGenerationSemantics.builder(0, 0, 1L).seal().build()); + GenerationHistory history = GenerationHistory.open(world); + stage(history, createPack("startup-promotion-pack-b", "beta")); + GenerationSemanticIndex observed = observeSemantics(history); + + try (GenerationHistory.StartupPreparation ignored = history.prepareStartup(64)) { + assertEquals(2L, history.activeActivation().activationId()); + } + + verify(observed, never()).forEachRecord(any()); + assertTrue(history.semantics(0, 0).isEmpty()); + assertEquals(2L, GenerationHistory.open(world).activeActivation().activationId()); + } + + @Test + public void standalonePromotionStillValidatesSemanticReferences() throws Exception { + Path world = temporaryFolder.newFolder("standalone-reference-validation").toPath(); + GenerationHistory history = createHistory(world, createPack("standalone-reference-pack", "alpha")); + semantics(history).recordAndPersist(ChunkGenerationSemantics.builder(0, 0, 99L).seal().build()); + + IOException failure = assertThrows(IOException.class, () -> history.promotePending(List.of())); + + assertTrue(failure.getMessage().contains("missing generation activation 99")); + assertEquals(1L, history.activeActivation().activationId()); + } + + @Test + public void livePromotionStillValidatesSemanticReferences() throws Exception { + Path world = temporaryFolder.newFolder("live-reference-validation").toPath(); + GenerationHistory history = createHistory(world, createPack("live-reference-pack", "alpha")); + try (GenerationHistory.GenerationStage ignored = history.openStage(0, 0)) { + } + semantics(history).recordAndPersist(ChunkGenerationSemantics.builder(0, 0, 99L).seal().build()); + + try (GenerationHistory.LiveCutover cutover = history.beginLiveCutover()) { + IOException failure = assertThrows(IOException.class, + () -> cutover.promote(boundary -> (blockX, blockZ) -> null)); + assertTrue(failure.getMessage().contains("missing generation activation 99")); + } + assertEquals(1L, history.activeActivation().activationId()); + } + + @Test + public void startupValidationCannotBeReusedAfterGenerationAdmission() throws Exception { + Path world = temporaryFolder.newFolder("startup-admission-guard").toPath(); + GenerationHistory history = createHistory(world, createPack("startup-admission-pack", "alpha")); + try (GenerationHistory.GenerationStage generation = history.openStage(0, 0)) { + history.claimGeneratedSemantics(generation, + ChunkGenerationSemantics.builder(0, 0, 1L).seal().build()); + } + + assertThrows(IllegalStateException.class, () -> history.prepareStartup(64)); + assertFalse(history.semantics(0, 0).isEmpty()); + } + + @Test + public void openingHistoryStillRejectsInvalidSemanticReferences() throws Exception { + Path world = temporaryFolder.newFolder("open-reference-validation").toPath(); + createHistory(world, createPack("open-reference-pack", "alpha")); + GenerationSemanticIndex stored = GenerationSemanticIndex.loadRequired(world); + stored.recordAndPersist(ChunkGenerationSemantics.builder(0, 0, 99L).seal().build()); + + IOException failure = assertThrows(IOException.class, () -> GenerationHistory.open(world)); + + assertTrue(failure.getMessage().contains("missing generation activation 99")); + } + + @Test + public void kernelStagingRejectsTamperedActivePackWithOrWithoutPendingActivation() throws Exception { + GenerationKernelRegistry.Version oldVersion = new GenerationKernelRegistry.Version(101, 1, 1); + GenerationKernelRegistry.Version currentVersion = new GenerationKernelRegistry.Version(102, 1, 1); + for (boolean pending : new boolean[]{false, true}) { + Path world = temporaryFolder.newFolder("kernel-tamper-" + pending).toPath(); + Path pack = createPack("kernel-tamper-pack-" + pending, "alpha"); + GenerationHistory.create(world, pack, fingerprint(pack), 42L, + contract(), GenerationRegistryContract.empty(), oldVersion, singleKernel(oldVersion, "a")); + GenerationHistory history = GenerationHistory.open(world, singleKernel(currentVersion, "b")); + Path activePack = history.activePackRoot(); + if (pending) { + history.stageCurrentKernel(32); + } + Files.writeString(activePack.resolve("dimensions/main.json"), "changed"); + + assertThrows(IOException.class, () -> history.stageCurrentKernel(32)); + + assertEquals(pending, history.pendingActivation().isPresent()); + assertEquals(1L, history.activeActivation().activationId()); + } + } + + @Test + public void kernelStagingRejectsSymlinkedActivePackRoots() throws Exception { + Path world = temporaryFolder.newFolder("kernel-symlink").toPath(); + GenerationHistory history = createHistory(world, createPack("kernel-symlink-pack", "alpha")); + Path activePack = history.activePackRoot(); + Path moved = temporaryFolder.getRoot().toPath().resolve("moved-kernel-pack"); + Files.move(activePack, moved); + Files.createSymbolicLink(activePack, moved); + + assertThrows(IOException.class, () -> history.stageCurrentKernel(32)); + assertTrue(history.pendingActivation().isEmpty()); + } + + private static GenerationSemanticIndex observeSemantics(GenerationHistory history) throws Exception { + GenerationSemanticIndex observed = spy(semantics(history)); + Field field = GenerationHistory.class.getDeclaredField("semantics"); + field.setAccessible(true); + field.set(history, observed); + return observed; + } + + private static GenerationSemanticIndex semantics(GenerationHistory history) throws Exception { + Field field = GenerationHistory.class.getDeclaredField("semantics"); + field.setAccessible(true); + return (GenerationSemanticIndex) field.get(history); + } +} diff --git a/core/src/test/java/art/arcane/iris/world/history/GenerationPackInspectionTest.java b/core/src/test/java/art/arcane/iris/world/history/GenerationPackInspectionTest.java new file mode 100644 index 000000000..67090251a --- /dev/null +++ b/core/src/test/java/art/arcane/iris/world/history/GenerationPackInspectionTest.java @@ -0,0 +1,81 @@ +package art.arcane.iris.world.history; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.concurrent.atomic.AtomicInteger; +import org.junit.Test; +import org.mockito.MockedStatic; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertThrows; +import static org.mockito.Mockito.mockStatic; + +public class GenerationPackInspectionTest extends GenerationHistorySupport { + @Test + public void inspectionVerifiesEachSelectedPackOnce() throws Exception { + Path world = temporaryFolder.newFolder("world").toPath(); + Path pack = createPack("pack", "alpha"); + Path update = createPack("update", "beta"); + GenerationHistory history = createHistory(world, pack); + GenerationActivation pending = stage(history, update); + AtomicInteger hashes = new AtomicInteger(); + try (MockedStatic ignored = mockStatic(GenerationPackFingerprint.class, invocation -> { + if (invocation.getMethod().getName().equals("compute")) { + hashes.incrementAndGet(); + } + return invocation.callRealMethod(); + })) { + GenerationHistory.PackInspection inspection = GenerationHistory.inspectPacks(world); + assertEquals(history.activeEpoch(), inspection.manifest().activeEpoch()); + assertEquals(history.paths().packRoot(history.activeEpoch().epochId()), inspection.activePackRoot()); + assertEquals(history.paths().packRoot(pending.epochId()), inspection.pendingPackRoot().orElseThrow()); + assertEquals(2, hashes.get()); + } + } + + @Test + public void inspectionDoesNotReplaceRuntimeSemanticValidation() throws Exception { + Path world = temporaryFolder.newFolder("world").toPath(); + GenerationHistory history = createHistory(world, createPack("pack", "alpha")); + Files.delete(history.paths().generationRoot().resolve("semantics/index.isix")); + + GenerationHistory.PackInspection inspection = GenerationHistory.inspectPacks(world); + + assertEquals(history.activeEpoch(), inspection.manifest().activeEpoch()); + assertFalse(inspection.pendingPackRoot().isPresent()); + assertThrows(IOException.class, () -> GenerationHistory.open(world)); + } + + @Test + public void laterInspectionsRejectModifiedActivePacks() throws Exception { + Path world = temporaryFolder.newFolder("world").toPath(); + GenerationHistory history = createHistory(world, createPack("pack", "alpha")); + GenerationHistory.PackInspection inspection = GenerationHistory.inspectPacks(world); + Files.writeString(inspection.activePackRoot().resolve("dimensions/main.json"), "changed"); + + assertThrows(IOException.class, () -> GenerationHistory.inspectPacks(world)); + assertThrows(IOException.class, history::activePackRoot); + assertThrows(IOException.class, () -> GenerationHistory.open(world)); + } + + @Test + public void inspectionRejectsModifiedPendingPacks() throws Exception { + Path world = temporaryFolder.newFolder("world").toPath(); + GenerationHistory history = createHistory(world, createPack("pack", "alpha")); + GenerationActivation pending = stage(history, createPack("update", "beta")); + Files.writeString(history.paths().packRoot(pending.epochId()).resolve("dimensions/main.json"), "changed"); + + assertThrows(IOException.class, () -> GenerationHistory.inspectPacks(world)); + } + + @Test + public void inspectionRejectsMalformedManifest() throws Exception { + Path world = temporaryFolder.newFolder("world").toPath(); + GenerationHistory history = createHistory(world, createPack("pack", "alpha")); + Files.writeString(history.paths().manifest(), "{}"); + + assertThrows(IOException.class, () -> GenerationHistory.inspectPacks(world)); + } +} diff --git a/core/src/test/java/art/arcane/iris/world/history/GenerationPackRepositoryTest.java b/core/src/test/java/art/arcane/iris/world/history/GenerationPackRepositoryTest.java index 22046a8dc..a2c5e0bf5 100644 --- a/core/src/test/java/art/arcane/iris/world/history/GenerationPackRepositoryTest.java +++ b/core/src/test/java/art/arcane/iris/world/history/GenerationPackRepositoryTest.java @@ -297,6 +297,9 @@ public class GenerationPackRepositoryTest { String fingerprint = fingerprint(source); GenerationPackRepository repository = new GenerationPackRepository(dimensionRoot); + assertThrows(IOException.class, () -> repository.requireSafePackRoot(epochId)); + assertThrows(IOException.class, () -> repository.requireExactPack( + epochId, fingerprint, GenerationPackFingerprint.CURRENT_VERSION)); assertThrows(IOException.class, () -> repository.publish( epochId, fingerprint, diff --git a/core/src/test/java/art/arcane/iris/world/history/GenerationSemanticIndexTest.java b/core/src/test/java/art/arcane/iris/world/history/GenerationSemanticIndexTest.java index f0d9b4de5..08770aa91 100644 --- a/core/src/test/java/art/arcane/iris/world/history/GenerationSemanticIndexTest.java +++ b/core/src/test/java/art/arcane/iris/world/history/GenerationSemanticIndexTest.java @@ -18,6 +18,7 @@ import java.util.Arrays; import java.util.HexFormat; import java.util.List; import java.util.Optional; +import java.util.Set; import java.util.stream.Stream; import java.util.zip.CRC32; @@ -178,6 +179,33 @@ public final class GenerationSemanticIndexTest { assertEquals(Optional.of(expected), GenerationSemanticIndex.load(dimensionRoot).get(4, 5)); } + @Test + public void summariesFollowLoadedShardUpdatesAndOrphanRemoval() throws Exception { + Path root = temporaryFolder.newFolder("summary-updates").toPath(); + GenerationSemanticIndex initial = GenerationSemanticIndex.initialize(root); + initial.recordAndPersist(ChunkGenerationSemantics.builder(-33, -1, 1L).addObject("iris:old").build()); + GenerationSemanticIndex loaded = GenerationSemanticIndex.load(root); + assertEquals("iris:old", requiredMatch(loaded, GenerationSemanticIndex.SemanticKind.OBJECT, "iris:old").key()); + + loaded.claimAndPersist(ChunkGenerationSemantics.builder(-33, -1, 1L).addObject("iris:new").seal().build()); + loaded.claimAndPersist(ChunkGenerationSemantics.builder(-34, -1, 1L).addObject("iris:orphan").seal().build()); + assertEquals("iris:new", requiredMatch(loaded, GenerationSemanticIndex.SemanticKind.OBJECT, "iris:new").key()); + assertEquals(2, sealedClaims(loaded, 1L).size()); + loaded.compactJournals(); + + GenerationSemanticIndex reopened = GenerationSemanticIndex.load(root); + assertEquals("iris:new", requiredMatch(reopened, GenerationSemanticIndex.SemanticKind.OBJECT, "iris:new").key()); + assertEquals(1, reopened.discardUnstoredClaims( + WorldChunkInventory.ofPackedChunks(ChunkGenerationOwnership.packChunk(-33, -1)), Set.of(1L))); + GenerationSemanticIndex recovered = GenerationSemanticIndex.load(root); + assertEquals(1, sealedClaims(recovered, 1L).size()); + assertEquals("iris:old", requiredMatch(recovered, GenerationSemanticIndex.SemanticKind.OBJECT, "iris:old").key()); + assertEquals("iris:new", requiredMatch(recovered, GenerationSemanticIndex.SemanticKind.OBJECT, "iris:new").key()); + assertTrue(recovered.findNearest(GenerationSemanticIndex.Query.acrossActivations( + GenerationSemanticIndex.SemanticKind.OBJECT, "iris:orphan", + new ChunkGenerationSemantics.BlockPosition(-520, 80, -8), 4)).isEmpty()); + } + @Test public void conflictingActivationsAndFactsAfterSealAreRejectedWithoutDiskChanges() throws Exception { Path dimensionRoot = temporaryFolder.newFolder("sealed").toPath(); @@ -627,6 +655,22 @@ public final class GenerationSemanticIndexTest { assertTrue(error.getCause().getMessage().contains("unsupported semantic record flags 2")); } + @Test + public void rejectsSummaryMismatchEvenWhenTheRecordAndChecksumAreValid() throws Exception { + Path root = temporaryFolder.newFolder("summary-mismatch").toPath(); + GenerationSemanticIndex index = GenerationSemanticIndex.initialize(root); + index.recordAndPersist(ChunkGenerationSemantics.builder(0, 0, 1L).seal().build()); + Path shard = onlyShard(index.storageDirectory()); + byte[] encoded = Files.readAllBytes(shard); + encoded[firstRecordFlagsOffset(encoded)] = 0; + rewriteChecksum(encoded); + replaceReferencedShard(root, shard, encoded); + + GenerationSemanticIndex loaded = GenerationSemanticIndex.load(root); + IllegalStateException error = assertThrows(IllegalStateException.class, () -> loaded.get(0, 0)); + assertTrue(error.getCause().getMessage().contains("summary does not match semantic records")); + } + @Test public void symbolicLinkStorageAncestorsAreRejected() throws Exception { Path dimensionRoot = temporaryFolder.newFolder("symlink-dimension").toPath(); diff --git a/core/src/test/java/art/arcane/iris/world/history/GenerationSemanticKeyValidationTest.java b/core/src/test/java/art/arcane/iris/world/history/GenerationSemanticKeyValidationTest.java index 81864cace..873d4e7e5 100644 --- a/core/src/test/java/art/arcane/iris/world/history/GenerationSemanticKeyValidationTest.java +++ b/core/src/test/java/art/arcane/iris/world/history/GenerationSemanticKeyValidationTest.java @@ -2,6 +2,7 @@ package art.arcane.iris.world.history; import org.junit.Test; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -9,8 +10,44 @@ import java.util.Set; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; public final class GenerationSemanticKeyValidationTest { + @Test + public void resourceKeyLimitCountsUtf8BytesAcrossCharacterWidths() { + for (String unit : List.of("a", "é", "漢", "\ud801\udc00", "\ud801")) { + int unitBytes = unit.getBytes(StandardCharsets.UTF_8).length; + int repetitions = ChunkGenerationSemantics.MAX_KEY_BYTES / unitBytes; + int remainder = ChunkGenerationSemantics.MAX_KEY_BYTES % unitBytes; + String accepted = unit.repeat(repetitions) + "a".repeat(remainder); + assertEquals(ChunkGenerationSemantics.MAX_KEY_BYTES, accepted.getBytes(StandardCharsets.UTF_8).length); + assertEquals(accepted, ChunkGenerationSemantics.requireResourceKey(accepted)); + assertThrows(IllegalArgumentException.class, + () -> ChunkGenerationSemantics.requireResourceKey(accepted + "a")); + } + for (int length : new int[]{ChunkGenerationSemantics.MAX_KEY_BYTES / 3, + ChunkGenerationSemantics.MAX_KEY_BYTES / 3 + 1}) { + String accepted = "a".repeat(length); + assertEquals(accepted, ChunkGenerationSemantics.requireResourceKey(accepted)); + } + } + + @Test + public void emptyAndPopulatedRecordsRemainImmutableWhenTheirBuilderChanges() { + ChunkGenerationSemantics.Builder builder = ChunkGenerationSemantics.builder(0, 0, 1L); + ChunkGenerationSemantics empty = builder.build(); + builder.addObject("iris:b").addObject("iris:a"); + ChunkGenerationSemantics populated = builder.build(); + builder.addObject("iris:c"); + + assertTrue(empty.objectKeys().isEmpty()); + assertEquals(List.of("iris:a", "iris:b"), List.copyOf(populated.objectKeys())); + assertThrows(UnsupportedOperationException.class, () -> empty.objectKeys().add("iris:a")); + assertThrows(UnsupportedOperationException.class, () -> populated.objectKeys().add("iris:c")); + assertThrows(IllegalArgumentException.class, () -> builder.addObject(" iris:d")); + assertEquals(List.of("iris:a", "iris:b", "iris:c"), List.copyOf(builder.build().objectKeys())); + } + @Test public void duplicateKeysRetainValidationAndCanonicalOrdering() { ChunkGenerationSemantics.Builder builder = ChunkGenerationSemantics.builder(-2, 3, 5L); diff --git a/core/src/test/java/art/arcane/iris/world/history/GenerationStartupPreparationTest.java b/core/src/test/java/art/arcane/iris/world/history/GenerationStartupPreparationTest.java new file mode 100644 index 000000000..b56f36e15 --- /dev/null +++ b/core/src/test/java/art/arcane/iris/world/history/GenerationStartupPreparationTest.java @@ -0,0 +1,215 @@ +package art.arcane.iris.world.history; + +import art.arcane.iris.generation.runtime.IrisEngine; +import art.arcane.iris.testsupport.Await; +import org.junit.Test; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Duration; +import java.util.concurrent.FutureTask; +import java.util.concurrent.TimeUnit; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public final class GenerationStartupPreparationTest extends GenerationHistoryRuntimeRouterSupport { + @Test + public void preparationAttachesThePreparedActivationOnlyOnce() throws Exception { + Path world = temporaryFolder.newFolder("prepared-startup-world").toPath(); + GenerationHistory history = createHistory(world, createPack("prepared-startup-pack", "alpha")); + FakeRuntimeFactory runtimes = new FakeRuntimeFactory(); + IrisEngine engine = preparedEngine(history, runtimes); + + try (GenerationAdmission.RuntimeLease runtime = history.retainRuntime(); + GenerationHistory.StartupPreparation preparation = history.prepareStartup(64); + GenerationHistoryRuntimeRouter router = attach(engine, preparation, runtimes)) { + assertSame(history, router.history()); + assertSame(engine, router.engine()); + assertEquals(1L, history.activeActivation().activationId()); + assertEquals(0, runtimes.loadCount(1L)); + verify(engine).attachGenerationHistoryRuntimeRouter(router); + assertThrows(IllegalStateException.class, () -> attach(engine, preparation, runtimes)); + } + } + + @Test + public void preparationPromotesPendingStateBeforeEngineAttachment() throws Exception { + Path world = temporaryFolder.newFolder("prepared-pending-world").toPath(); + GenerationHistory history = createHistory(world, createPack("prepared-pending-a", "alpha")); + stage(history, createPack("prepared-pending-b", "beta")); + FakeRuntimeFactory runtimes = new FakeRuntimeFactory(); + + try (GenerationAdmission.RuntimeLease runtime = history.retainRuntime(); + GenerationHistory.StartupPreparation preparation = history.prepareStartup(64)) { + assertEquals(2L, history.activeActivation().activationId()); + assertFalse(history.pendingActivation().isPresent()); + IrisEngine engine = preparedEngine(history, runtimes); + try (GenerationHistoryRuntimeRouter router = attach(engine, preparation, runtimes)) { + assertEquals(2L, router.history().activeActivation().activationId()); + assertEquals(0, runtimes.loadCount(2L)); + } + } + } + + @Test + public void changedManifestRejectsPreparationBeforeAttachingAnEngine() throws Exception { + Path world = temporaryFolder.newFolder("prepared-stale-world").toPath(); + GenerationHistory history = createHistory(world, createPack("prepared-stale-a", "alpha")); + FakeRuntimeFactory runtimes = new FakeRuntimeFactory(); + IrisEngine engine = preparedEngine(history, runtimes); + + try (GenerationAdmission.RuntimeLease runtime = history.retainRuntime(); + GenerationHistory.StartupPreparation preparation = history.prepareStartup(64)) { + stage(history, createPack("prepared-stale-b", "beta")); + assertThrows(IllegalStateException.class, () -> attach(engine, preparation, runtimes)); + verify(engine, never()).attachGenerationHistoryRuntimeRouter(any()); + assertEquals(1L, history.activeActivation().activationId()); + assertEquals(2L, history.pendingActivation().orElseThrow().activationId()); + } + } + + @Test + public void closedPreparationCannotAttach() throws Exception { + Path world = temporaryFolder.newFolder("prepared-closed-world").toPath(); + GenerationHistory history = createHistory(world, createPack("prepared-closed-pack", "alpha")); + FakeRuntimeFactory runtimes = new FakeRuntimeFactory(); + IrisEngine engine = preparedEngine(history, runtimes); + + try (GenerationAdmission.RuntimeLease runtime = history.retainRuntime()) { + GenerationHistory.StartupPreparation preparation = history.prepareStartup(64); + preparation.close(); + assertThrows(IllegalStateException.class, () -> attach(engine, preparation, runtimes)); + verify(engine, never()).attachGenerationHistoryRuntimeRouter(any()); + } + } + + @Test + public void anotherThreadCannotConsumePreparation() throws Exception { + Path world = temporaryFolder.newFolder("prepared-thread-world").toPath(); + GenerationHistory history = createHistory(world, createPack("prepared-thread-pack", "alpha")); + FakeRuntimeFactory runtimes = new FakeRuntimeFactory(); + IrisEngine engine = preparedEngine(history, runtimes); + + try (GenerationAdmission.RuntimeLease runtime = history.retainRuntime(); + GenerationHistory.StartupPreparation preparation = history.prepareStartup(64)) { + FutureTask attempt = new FutureTask<>(() -> { + assertThrows(IllegalStateException.class, () -> attach(engine, preparation, runtimes)); + return null; + }); + Thread thread = Thread.ofPlatform().daemon().start(attempt); + try { + attempt.get(5L, TimeUnit.SECONDS); + verify(engine, never()).attachGenerationHistoryRuntimeRouter(any()); + try (GenerationHistoryRuntimeRouter router = attach(engine, preparation, runtimes)) { + assertSame(history, router.history()); + } + } finally { + thread.join(5_000L); + } + } + } + + @Test + public void failedAttachmentConsumesPreparationAndFreshPreparationCanRetry() throws Exception { + Path world = temporaryFolder.newFolder("prepared-retry-world").toPath(); + GenerationHistory history = createHistory(world, createPack("prepared-retry-pack", "alpha")); + FakeRuntimeFactory runtimes = new FakeRuntimeFactory(); + IrisEngine failedEngine = preparedEngine(history, runtimes); + IllegalStateException failure = new IllegalStateException("Engine attachment failed"); + doThrow(failure).when(failedEngine).attachGenerationHistoryRuntimeRouter(any()); + + try (GenerationAdmission.RuntimeLease runtime = history.retainRuntime()) { + try (GenerationHistory.StartupPreparation preparation = history.prepareStartup(64)) { + assertSame(failure, assertThrows(IllegalStateException.class, + () -> attach(failedEngine, preparation, runtimes))); + IrisEngine replacement = preparedEngine(history, runtimes); + assertThrows(IllegalStateException.class, () -> attach(replacement, preparation, runtimes)); + verify(replacement, never()).attachGenerationHistoryRuntimeRouter(any()); + } + IrisEngine replacement = preparedEngine(history, runtimes); + try (GenerationHistory.StartupPreparation retry = history.prepareStartup(64); + GenerationHistoryRuntimeRouter router = attach(replacement, retry, runtimes)) { + assertSame(history, router.history()); + verify(replacement).attachGenerationHistoryRuntimeRouter(router); + } + } + } + + @Test(timeout = 10_000L) + public void failedPreparationReleasesAdmissionForARetry() throws Exception { + Path world = temporaryFolder.newFolder("prepared-invalid-world").toPath(); + GenerationHistory history = createHistory(world, createPack("prepared-invalid-pack", "alpha")); + Path dimension = history.activePackRoot().resolve("dimensions/main.json"); + byte[] original = Files.readAllBytes(dimension); + FakeRuntimeFactory runtimes = new FakeRuntimeFactory(); + IrisEngine engine = preparedEngine(history, runtimes); + + try (GenerationAdmission.RuntimeLease runtime = history.retainRuntime()) { + Files.writeString(dimension, "changed"); + assertThrows(IOException.class, () -> history.prepareStartup(64)); + Files.write(dimension, original); + try (GenerationHistory.StartupPreparation preparation = history.prepareStartup(64); + GenerationHistoryRuntimeRouter router = attach(engine, preparation, runtimes)) { + assertSame(history, router.history()); + } + } + } + + @Test + public void generationAdmissionWaitsUntilPreparedAttachmentScopeCloses() throws Exception { + Path world = temporaryFolder.newFolder("prepared-admission-world").toPath(); + GenerationHistory history = createHistory(world, createPack("prepared-admission-pack", "alpha")); + FakeRuntimeFactory runtimes = new FakeRuntimeFactory(); + IrisEngine engine = preparedEngine(history, runtimes); + GenerationAdmission admission = new GenerationAdmission(world); + FutureTask stage = new FutureTask<>(() -> { + try (GenerationAdmission.StageLease ignored = admission.enterStage()) { + } + return null; + }); + Thread thread = Thread.ofPlatform().daemon().unstarted(stage); + + try (GenerationAdmission.RuntimeLease runtime = history.retainRuntime()) { + try { + try (GenerationHistory.StartupPreparation preparation = history.prepareStartup(64); + GenerationHistoryRuntimeRouter router = attach(engine, preparation, runtimes)) { + thread.start(); + Await.reached("generation to wait for startup preparation", Duration.ofSeconds(5L), + () -> thread.getState() == Thread.State.WAITING || stage.isDone()); + assertFalse(stage.isDone()); + assertSame(history, router.history()); + } + stage.get(5L, TimeUnit.SECONDS); + assertThrows(IllegalStateException.class, () -> history.prepareStartup(64)); + } finally { + thread.join(5_000L); + } + } + } + + private static IrisEngine preparedEngine(GenerationHistory history, FakeRuntimeFactory runtimes) throws Exception { + IrisEngine engine = mock(IrisEngine.class); + IrisEngine.GenerationRuntimeBinding binding = runtimes.binding(history, history.activeActivation()); + when(engine.getActiveGenerationRuntimeBinding()).thenReturn(binding); + return engine; + } + + private static GenerationHistoryRuntimeRouter attach( + IrisEngine engine, + GenerationHistory.StartupPreparation preparation, + FakeRuntimeFactory runtimes + ) throws Exception { + return GenerationHistoryRuntimeRouter.attachPrepared(engine, preparation, + (ignored, blockX, blockZ) -> signature(blockX, blockZ), runtimes); + } +} diff --git a/core/src/test/java/art/arcane/iris/world/history/SavedBiomeStoreTest.java b/core/src/test/java/art/arcane/iris/world/history/SavedBiomeStoreTest.java index aa6bf022c..e4abafbf8 100644 --- a/core/src/test/java/art/arcane/iris/world/history/SavedBiomeStoreTest.java +++ b/core/src/test/java/art/arcane/iris/world/history/SavedBiomeStoreTest.java @@ -194,6 +194,28 @@ public class SavedBiomeStoreTest { assertThrows(IOException.class, () -> SavedBiomeStore.open(root).get(0, 0)); } + @Test + public void indexesLargeRegionsAndTruncatesOnlyIncompleteRecordLength() throws Exception { + Path root = temporaryFolder.newFolder().toPath(); + SavedBiomeStore writer = SavedBiomeStore.open(root); + for (int index = 0; index < 512; index++) { + writer.claimAndPersist(chunk(index & 31, index >>> 5, 3L)); + } + Path path = regionPath(root, 0, 0); + long completeLength = Files.size(path); + assertTrue(completeLength > 65536); + Files.write(path, new byte[]{0, 1}, StandardOpenOption.APPEND); + + SavedBiomeStore reader = SavedBiomeStore.open(root); + assertEquals(Optional.of(chunk(31, 15, 3L)), reader.get(31, 15)); + assertEquals(completeLength, Files.size(path)); + assertEquals(Optional.of(chunk(0, 0, 3L)), reader.get(0, 0)); + assertEquals(Optional.of(chunk(0, 8, 3L)), reader.get(0, 8)); + SavedBiomeChunk appended = chunk(0, 16, 3L); + assertTrue(reader.claimAndPersist(appended)); + assertEquals(Optional.of(appended), SavedBiomeStore.open(root).get(0, 16)); + } + @Test public void rejectsUnknownFormatEvenWithValidHeaderChecksum() throws Exception { Path root = temporaryFolder.newFolder().toPath(); diff --git a/core/src/test/java/art/arcane/iris/world/history/SavedTerrainChunkTest.java b/core/src/test/java/art/arcane/iris/world/history/SavedTerrainChunkTest.java index 6dda1546a..ffb5e8f33 100644 --- a/core/src/test/java/art/arcane/iris/world/history/SavedTerrainChunkTest.java +++ b/core/src/test/java/art/arcane/iris/world/history/SavedTerrainChunkTest.java @@ -123,6 +123,49 @@ public final class SavedTerrainChunkTest { } } + @Test + public void reusesFullChunkReaderAcrossCodecsAndPreservesValidationAfterFailures() throws Exception { + Path world = temporaryFolder.newFolder("reused-full-reader").toPath(); + try (SavedTerrainChunkReader.StatusReader reader = new SavedTerrainChunkReader.StatusReader(world)) { + for (int compression = 1; compression <= 4; compression++) { + for (boolean external : new boolean[]{false, true}) { + CompoundTag saved = root("minecraft:noise"); + saved.putByteArray("padding", new byte[196_608]); + writeChunk(world, saved, compression, external); + SavedTerrainChunk captured = reader.readChunk(-1, -2, -16, 16); + assertEquals("minecraft:noise", captured.nativeStatus()); + assertEquals("minecraft:stone", captured.column(-16, -32).geometry().voxelAt(-16).stateKey()); + assertEquals("minecraft:water[level=0]", captured.column(-16, -32).geometry().voxelAt(-15).stateKey()); + assertEquals("example:saved_biome", captured.column(-16, -32).biomeAtSample(0)); + saved.putInt("xPos", 0); + writeChunk(world, saved, compression, external); + assertThrows(IOException.class, () -> reader.readChunk(-1, -2, -16, 16)); + writeChunk(world, root("minecraft:empty"), compression, external); + assertThrows(IOException.class, () -> reader.readChunk(-1, -2, -16, 16)); + saved = root("minecraft:full"); + saved.remove("sections"); + writeChunk(world, saved, compression, external); + assertThrows(IOException.class, () -> reader.readChunk(-1, -2, -16, 16)); + writeChunk(world, root("minecraft:full"), compression, external); + assertEquals("minecraft:full", reader.readChunk(-1, -2, -16, 16).nativeStatus()); + } + } + } + } + + @Test + public void diskBoundaryCaptureClosesItsReaderAndClearsCachedChunks() throws Exception { + Path world = temporaryFolder.newFolder("closed-disk-boundary").toPath(); + writeChunk(world, root("minecraft:full"), 2, false); + DiskBoundaryCapture capture = new DiskBoundaryCapture(world, -16, 16); + try (capture) { + assertEquals("minecraft:stone", capture.sample(-16, -32).geometry().voxelAt(-16).stateKey()); + assertEquals("minecraft:water[level=0]", capture.sample(-16, -31).geometry().voxelAt(-15).stateKey()); + } + capture.close(); + assertThrows(IOException.class, () -> capture.sample(-16, -32)); + } + @Test public void externalChunkAndPartialNativeStagePreserveAvailableTerrain() throws Exception { Path world = temporaryFolder.newFolder("external").toPath(); diff --git a/core/src/test/java/art/arcane/iris/world/history/SavedTerrainRecoveryScanTest.java b/core/src/test/java/art/arcane/iris/world/history/SavedTerrainRecoveryScanTest.java new file mode 100644 index 000000000..58e1f2966 --- /dev/null +++ b/core/src/test/java/art/arcane/iris/world/history/SavedTerrainRecoveryScanTest.java @@ -0,0 +1,164 @@ +package art.arcane.iris.world.history; + +import art.arcane.volmlib.util.nbt.io.NBTUtil; +import art.arcane.volmlib.util.nbt.tag.CompoundTag; +import net.jpountz.lz4.LZ4BlockOutputStream; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.io.RandomAccessFile; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.zip.DeflaterOutputStream; +import java.util.zip.GZIPOutputStream; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; + +public final class SavedTerrainRecoveryScanTest { + @Rule + public final TemporaryFolder temporaryFolder = new TemporaryFolder(); + + @Test + public void bufferedAndDirectScansMatchAcrossCodecsCoordinatesAndExternalChunks() throws Exception { + Path world = temporaryFolder.newFolder().toPath(); + for (int index = 191; index >= 0; index--) { + int x = index % 32 - 32; + int z = index / 32 - 32; + write(world, new Chunk(x, z, index % 4 + 1, index % 17 == 0, + encode(x, z, index % 3 == 0 ? "minecraft:biomes" : "minecraft:noise"))); + } + write(world, new Chunk(33, 3, 3, false, encode(33, 3, "minecraft:full"))); + WorldChunkInventory inventory = WorldChunkInventory.scan(world); + WorldChunkInventory expected; + try (SavedTerrainChunkReader.StatusReader reader = new SavedTerrainChunkReader.StatusReader(world)) { + expected = inventory.filter((x, z) -> SavedTerrainChunk.hasTerrain(reader.readStatus(x, z))); + } + for (int workers : new int[]{1, 2, 4}) { + for (int regionBytes : new int[]{8192, 4 * 1024 * 1024}) { + try (SavedTerrainRecoveryScan scan = new SavedTerrainRecoveryScan(world, + new SavedTerrainRecoveryScan.Limits(workers, regionBytes))) { + WorldChunkInventory actual = scan.filter(inventory, (x, z) -> false); + assertEquals(expected.size(), actual.size()); + inventory.forEach((x, z) -> assertEquals(expected.contains(x, z), actual.contains(x, z))); + } + } + } + } + + @Test + public void doesNotDecodeRetainedHistoricalChunks() throws Exception { + Path world = temporaryFolder.newFolder().toPath(); + write(world, new Chunk(0, 0, 2, false, new byte[]{10})); + write(world, new Chunk(1, 0, 2, false, encode(1, 0, "minecraft:full"))); + try (SavedTerrainRecoveryScan scan = new SavedTerrainRecoveryScan(world, + new SavedTerrainRecoveryScan.Limits(2, 1024 * 1024))) { + WorldChunkInventory actual = scan.filter(WorldChunkInventory.scan(world), (x, z) -> x == 0); + assertEquals(2, actual.size()); + } + } + + @Test + public void propagatesWorkerCorruptionAndCanScanAgain() throws Exception { + Path world = temporaryFolder.newFolder().toPath(); + for (int x = 0; x < 16; x++) { + for (int z = 0; z < 16; z++) { + write(world, new Chunk(x, z, 2, false, encode(x, z, "minecraft:full"))); + } + } + write(world, new Chunk(1, 1, 2, false, new byte[]{10})); + try (SavedTerrainRecoveryScan scan = new SavedTerrainRecoveryScan(world, + new SavedTerrainRecoveryScan.Limits(2, 4 * 1024 * 1024))) { + IOException failure = assertThrows(IOException.class, + () -> scan.filter(WorldChunkInventory.scan(world), (x, z) -> false)); + assertTrue(failure.getMessage().contains("1,1")); + write(world, new Chunk(1, 1, 2, false, encode(1, 1, "minecraft:full"))); + assertEquals(256, scan.filter(WorldChunkInventory.scan(world), (x, z) -> false).size()); + } + assertEquals(256, SavedTerrainRecoveryScan.scan(world, WorldChunkInventory.scan(world), (x, z) -> false).size()); + } + + @Test + public void retainsInterruptStatusWhenRecoveryIsInterrupted() throws Exception { + Path world = temporaryFolder.newFolder().toPath(); + write(world, new Chunk(0, 0, 2, false, encode(0, 0, "minecraft:full"))); + WorldChunkInventory inventory = WorldChunkInventory.scan(world); + try (SavedTerrainRecoveryScan scan = new SavedTerrainRecoveryScan(world, + new SavedTerrainRecoveryScan.Limits(1, 1024 * 1024))) { + Thread.currentThread().interrupt(); + try { + assertThrows(IOException.class, () -> scan.filter(inventory, (x, z) -> false)); + assertTrue(Thread.currentThread().isInterrupted()); + } finally { + Thread.interrupted(); + } + } + } + + @Test + public void capsWorkersAndRegionMemoryForTheAvailableResources() { + assertEquals(1, SavedTerrainRecoveryScan.Limits.forRuntime(1, 8L << 30).workers()); + assertEquals(1, SavedTerrainRecoveryScan.Limits.forRuntime(64, 512L << 20).workers()); + assertEquals(2, SavedTerrainRecoveryScan.Limits.forRuntime(64, 2L << 30).workers()); + assertEquals(4, SavedTerrainRecoveryScan.Limits.forRuntime(64, 8L << 30).workers()); + assertEquals(32 << 20, SavedTerrainRecoveryScan.Limits.forRuntime(64, 512L << 20).regionBytes()); + assertEquals(64 << 20, SavedTerrainRecoveryScan.Limits.forRuntime(64, 8L << 30).regionBytes()); + } + + @Test + public void emptyInventoriesDoNotNeedARegionDirectory() throws Exception { + Path world = temporaryFolder.newFolder().toPath(); + assertTrue(SavedTerrainRecoveryScan.scan(world, WorldChunkInventory.empty(), (x, z) -> false).isEmpty()); + assertFalse(Files.exists(world.resolve("region"))); + } + + private static byte[] encode(int x, int z, String status) throws IOException { + CompoundTag root = new CompoundTag(); + root.putInt("xPos", x); + root.putInt("zPos", z); + root.putString("Status", status); + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + NBTUtil.write(root, bytes, false); + return bytes.toByteArray(); + } + + private static void write(Path world, Chunk chunk) throws IOException { + Path directory = Files.createDirectories(world.resolve("region")); + ByteArrayOutputStream encoded = new ByteArrayOutputStream(); + try (OutputStream stream = switch (chunk.compression()) { + case 1 -> new GZIPOutputStream(encoded); + case 2 -> new DeflaterOutputStream(encoded); + case 4 -> new LZ4BlockOutputStream(encoded); + default -> encoded; + }) { + stream.write(chunk.nbt()); + } + byte[] payload = encoded.toByteArray(); + Path path = directory.resolve("r." + Math.floorDiv(chunk.x(), 32) + "." + Math.floorDiv(chunk.z(), 32) + ".mca"); + try (RandomAccessFile file = new RandomAccessFile(path.toFile(), "rw")) { + int sector = Math.max(2, Math.toIntExact(Math.ceilDiv(file.length(), 4096))); + int sectors = chunk.external() ? 1 : Math.ceilDiv(payload.length + 5, 4096); + file.setLength((long) (sector + sectors) * 4096); + file.seek((Math.floorMod(chunk.z(), 32) * 32L + Math.floorMod(chunk.x(), 32)) * 4); + file.writeInt(sector << 8 | sectors); + file.seek((long) sector * 4096); + file.writeInt(chunk.external() ? 1 : payload.length + 1); + file.writeByte(chunk.compression() | (chunk.external() ? 128 : 0)); + if (!chunk.external()) { + file.write(payload); + } + } + if (chunk.external()) { + Files.write(directory.resolve("c." + chunk.x() + "." + chunk.z() + ".mcc"), payload); + } + } + + private record Chunk(int x, int z, int compression, boolean external, byte[] nbt) { + } +} diff --git a/core/src/test/java/art/arcane/iris/world/history/SavedTerrainRegionSnapshotTest.java b/core/src/test/java/art/arcane/iris/world/history/SavedTerrainRegionSnapshotTest.java new file mode 100644 index 000000000..0f4d3b3c4 --- /dev/null +++ b/core/src/test/java/art/arcane/iris/world/history/SavedTerrainRegionSnapshotTest.java @@ -0,0 +1,210 @@ +package art.arcane.iris.world.history; + +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.io.IOException; +import java.io.RandomAccessFile; +import java.nio.ByteBuffer; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.nio.file.attribute.FileTime; +import java.util.Arrays; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; + +public final class SavedTerrainRegionSnapshotTest { + private static final int SECTOR_BYTES = 4_096; + private static final int HEADER_BYTES = 2 * SECTOR_BYTES; + + @Rule + public final TemporaryFolder temporaryFolder = new TemporaryFolder(); + + @Test + public void extractsInternalAndExternalPayloadDescriptors() throws Exception { + byte[] bytes = new byte[4 * SECTOR_BYTES]; + ByteBuffer buffer = ByteBuffer.wrap(bytes); + buffer.putInt(0, 2 << 8 | 1); + buffer.putInt(1023 * Integer.BYTES, 3 << 8 | 1); + buffer.putInt(HEADER_BYTES, 4); + bytes[HEADER_BYTES + 4] = 2; + bytes[HEADER_BYTES + 5] = 12; + bytes[HEADER_BYTES + 6] = 34; + bytes[HEADER_BYTES + 7] = 56; + buffer.putInt(3 * SECTOR_BYTES, 1); + bytes[3 * SECTOR_BYTES + 4] = (byte) 132; + Path file = writeRegion(bytes); + + SavedTerrainRegionSnapshot snapshot = SavedTerrainRegionSnapshot.read(file, bytes.length, new byte[0]).orElseThrow(); + SavedTerrainRegionSnapshot.Payload internal = snapshot.payload(0); + assertEquals(2, internal.compression()); + assertFalse(internal.external()); + assertEquals(3, internal.length()); + assertArrayEquals(new byte[]{12, 34, 56}, Arrays.copyOfRange(snapshot.bytes(), + internal.offset(), internal.offset() + internal.length())); + SavedTerrainRegionSnapshot.Payload external = snapshot.payload(1023); + assertTrue(external.external()); + assertEquals(4, external.compression()); + assertEquals(0, external.length()); + snapshot.validateUnchanged(); + } + + @Test + public void skipsOversizedRegionsWithoutReadingTheirPayloads() throws Exception { + Path file = temporaryFolder.newFile().toPath(); + try (RandomAccessFile output = new RandomAccessFile(file.toFile(), "rw")) { + output.setLength(128L * 1024 * 1024); + } + assertTrue(SavedTerrainRegionSnapshot.read(file, 64 * 1024, new byte[0]).isEmpty()); + } + + @Test + public void acceptsAnExactUnpaddedPayload() throws Exception { + byte[] bytes = region(2 << 8 | 1, 3, 3, HEADER_BYTES + 7); + bytes[HEADER_BYTES + 5] = 10; + bytes[HEADER_BYTES + 6] = 0; + Path file = writeRegion(bytes); + SavedTerrainRegionSnapshot snapshot = SavedTerrainRegionSnapshot.read(file, bytes.length, new byte[0]).orElseThrow(); + assertEquals(2, snapshot.payload(0).length()); + snapshot.validateUnchanged(); + } + + @Test + public void reusesLargerBuffersWithoutAcceptingStalePayloadBytes() throws Exception { + byte[] larger = region(2 << 8 | 1, 5, 2, 4 * SECTOR_BYTES); + ByteBuffer.wrap(larger).putInt(3 * SECTOR_BYTES, 1); + larger[3 * SECTOR_BYTES + Integer.BYTES] = 2; + Path largerFile = writeRegion(larger); + SavedTerrainRegionSnapshot original = SavedTerrainRegionSnapshot.read( + largerFile, larger.length, new byte[0]).orElseThrow(); + byte[] reusable = original.bytes(); + original.validateUnchanged(); + + byte[] smaller = region(2 << 8 | 1, 1, 3, HEADER_BYTES + 5); + Path smallerFile = writeRegion(smaller); + SavedTerrainRegionSnapshot snapshot = SavedTerrainRegionSnapshot.read( + smallerFile, smaller.length, reusable).orElseThrow(); + assertSame(reusable, snapshot.bytes()); + assertEquals(larger.length, snapshot.bytes().length); + assertEquals(0, snapshot.payload(0).length()); + assertEquals(3, snapshot.payload(0).compression()); + snapshot.validateUnchanged(); + assertTrue(SavedTerrainRegionSnapshot.read(largerFile, smaller.length, reusable).isEmpty()); + + for (int[] entry : new int[][]{{3 << 8 | 1, 1}, {2 << 8 | 1, 5}}) { + Path malformed = writeRegion(region(entry[0], entry[1], 2, smaller.length)); + SavedTerrainRegionSnapshot invalid = SavedTerrainRegionSnapshot.read( + malformed, smaller.length, reusable).orElseThrow(); + assertSame(reusable, invalid.bytes()); + assertThrows(IOException.class, () -> invalid.payload(0)); + invalid.validateUnchanged(); + } + } + + @Test + public void rejectsAtomicReplacementWithMatchingSizeAndTimestamp() throws Exception { + byte[] bytes = region(2 << 8 | 1, 1, 2, 3 * SECTOR_BYTES); + Path file = writeRegion(bytes); + SavedTerrainRegionSnapshot snapshot = SavedTerrainRegionSnapshot.read(file, bytes.length, new byte[0]).orElseThrow(); + FileTime timestamp = Files.getLastModifiedTime(file); + Path replacement = writeRegion(bytes); + Files.setLastModifiedTime(replacement, timestamp); + Files.move(replacement, file, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); + assertThrows(IOException.class, snapshot::validateUnchanged); + } + + @Test + public void rejectsSameFilePayloadModificationAndTruncation() throws Exception { + byte[] bytes = region(2 << 8 | 1, 2, 2, 3 * SECTOR_BYTES); + Path file = writeRegion(bytes); + SavedTerrainRegionSnapshot snapshot = SavedTerrainRegionSnapshot.read(file, bytes.length, new byte[0]).orElseThrow(); + FileTime timestamp = Files.getLastModifiedTime(file); + try (RandomAccessFile output = new RandomAccessFile(file.toFile(), "rw")) { + output.seek(HEADER_BYTES + 5); + output.writeByte(1); + } + Files.setLastModifiedTime(file, FileTime.fromMillis(timestamp.toMillis() + 2_000)); + assertThrows(IOException.class, snapshot::validateUnchanged); + + SavedTerrainRegionSnapshot rewritten = SavedTerrainRegionSnapshot.read(file, bytes.length, new byte[0]).orElseThrow(); + try (RandomAccessFile output = new RandomAccessFile(file.toFile(), "rw")) { + output.setLength(HEADER_BYTES - 1); + } + assertThrows(IOException.class, rewritten::validateUnchanged); + } + + @Test + public void validatesBothHeaderSectorsEvenWhenTimestampIsRestored() throws Exception { + for (int offset : new int[]{0, SECTOR_BYTES}) { + byte[] bytes = region(2 << 8 | 1, 1, 2, 3 * SECTOR_BYTES); + Path file = writeRegion(bytes); + SavedTerrainRegionSnapshot snapshot = SavedTerrainRegionSnapshot.read(file, bytes.length, new byte[0]).orElseThrow(); + FileTime timestamp = Files.getLastModifiedTime(file); + try (RandomAccessFile output = new RandomAccessFile(file.toFile(), "rw")) { + output.seek(offset); + output.writeInt(7); + } + Files.setLastModifiedTime(file, timestamp); + assertThrows(IOException.class, snapshot::validateUnchanged); + } + } + + @Test + public void rejectsInvalidAllocationsAndPayloadLengths() throws Exception { + int[][] cases = { + {0, 1, 2, 3 * SECTOR_BYTES}, + {1 << 8 | 1, 1, 2, 3 * SECTOR_BYTES}, + {2 << 8, 1, 2, 3 * SECTOR_BYTES}, + {3 << 8 | 1, 1, 2, 3 * SECTOR_BYTES}, + {0xFFFFFF01, 1, 2, 3 * SECTOR_BYTES}, + {2 << 8 | 1, 0, 2, 3 * SECTOR_BYTES}, + {2 << 8 | 1, -1, 2, 3 * SECTOR_BYTES}, + {2 << 8 | 1, Integer.MAX_VALUE, 2, 3 * SECTOR_BYTES}, + {2 << 8 | 1, SECTOR_BYTES, 2, 4 * SECTOR_BYTES}, + {2 << 8 | 1, 2, 130, 3 * SECTOR_BYTES}, + {2 << 8 | 1, 2, 2, HEADER_BYTES + 5} + }; + for (int[] entry : cases) { + byte[] bytes = region(entry[0], entry[1], entry[2], entry[3]); + Path file = writeRegion(bytes); + SavedTerrainRegionSnapshot snapshot = SavedTerrainRegionSnapshot.read(file, bytes.length, new byte[0]).orElseThrow(); + assertThrows(IOException.class, () -> snapshot.payload(0)); + } + } + + @Test + public void rejectsTruncatedRegionsDirectoriesAndSymlinks() throws Exception { + for (int size : new int[]{0, HEADER_BYTES - 1}) { + Path file = writeRegion(new byte[size]); + assertThrows(IOException.class, () -> SavedTerrainRegionSnapshot.read(file, 0, new byte[0])); + } + Path directory = temporaryFolder.newFolder().toPath(); + assertThrows(IOException.class, () -> SavedTerrainRegionSnapshot.read(directory, HEADER_BYTES, new byte[0])); + Path file = writeRegion(new byte[HEADER_BYTES]); + Path link = directory.resolve("region-link"); + Files.createSymbolicLink(link, file); + assertThrows(IOException.class, () -> SavedTerrainRegionSnapshot.read(link, HEADER_BYTES, new byte[0])); + } + + private Path writeRegion(byte[] bytes) throws IOException { + Path file = temporaryFolder.newFile().toPath(); + Files.write(file, bytes); + return file; + } + + private static byte[] region(int allocation, int length, int compression, int size) { + byte[] bytes = new byte[size]; + ByteBuffer buffer = ByteBuffer.wrap(bytes); + buffer.putInt(0, allocation); + buffer.putInt(HEADER_BYTES, length); + bytes[HEADER_BYTES + Integer.BYTES] = (byte) compression; + return bytes; + } +} diff --git a/core/src/test/java/art/arcane/iris/world/history/SavedTerrainStatusNbtTest.java b/core/src/test/java/art/arcane/iris/world/history/SavedTerrainStatusNbtTest.java new file mode 100644 index 000000000..4f5dcf7b6 --- /dev/null +++ b/core/src/test/java/art/arcane/iris/world/history/SavedTerrainStatusNbtTest.java @@ -0,0 +1,269 @@ +package art.arcane.iris.world.history; + +import art.arcane.volmlib.util.nbt.io.NBTUtil; +import art.arcane.volmlib.util.nbt.tag.CompoundTag; +import art.arcane.volmlib.util.nbt.tag.NumberTag; +import art.arcane.volmlib.util.nbt.tag.StringTag; +import art.arcane.volmlib.util.nbt.tag.Tag; +import org.junit.Test; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.util.Arrays; + +import static org.junit.Assert.assertTrue; + +public final class SavedTerrainStatusNbtTest { + + @Test + public void validatesEveryTagAndListType() throws Exception { + for (int type = 1; type <= 12; type++) { + int tagType = type; + check("tag-" + type, root(out -> field(out, tagType, "extra", child -> payload(child, tagType))), 7, -9); + } + for (int type = 0; type <= 12; type++) { + int tagType = type; + check("list-" + type, root(out -> field(out, 9, "extra", child -> { + child.writeByte(tagType); + child.writeInt(2); + payload(child, tagType); + payload(child, tagType); + })), 7, -9); + } + } + + @Test + public void preservesNumericCoordinateConversions() throws Exception { + for (int type = 1; type <= 6; type++) { + int tagType = type; + check("coordinates-" + type, root(out -> { + field(out, tagType, "xPos", child -> numeric(child, tagType, 7)); + field(out, tagType, "zPos", child -> numeric(child, tagType, -9)); + }), 7, -9); + } + double[] special = {Double.NaN, Double.POSITIVE_INFINITY, Double.NEGATIVE_INFINITY, -9.75}; + for (double value : special) { + check("double-coordinate-" + value, root(out -> field(out, 6, "xPos", child -> child.writeDouble(value))), (int) value, -9); + } + check("long-overflow-coordinate", root(out -> field(out, 4, "xPos", child -> child.writeLong(0x100000007L))), 7, -9); + } + + @Test + public void validatesModifiedUtfNamesAndValues() throws Exception { + String[] strings = {"minecraft:full", "nul\u0000tail", "\u00e9\u4e00", "\ud83d\ude00", "\ud800", "\udc00", "\t \n", ""}; + for (int index = 0; index < strings.length; index++) { + String value = strings[index]; + check("status-utf-" + index, root(out -> field(out, 8, "Status", child -> child.writeUTF(value))), 7, -9); + } + byte[][] malformed = {{(byte) 0x80}, {(byte) 0xc2}, {(byte) 0xe1, (byte) 0x80}, {(byte) 0xf0, (byte) 0x90, (byte) 0x80, (byte) 0x80}, {(byte) 0xc2, 0x41}}; + for (int index = 0; index < malformed.length; index++) { + byte[] value = malformed[index]; + check("malformed-tail-utf-" + index, root(out -> field(out, 8, "unused", child -> rawUtf(child, value))), 7, -9); + } + check("overlong-status-name-2", namedStatus(2), 7, -9); + check("overlong-status-name-3", namedStatus(3), 7, -9); + check("overlong-x-name", root(out -> { + out.writeByte(3); + rawUtf(out, new byte[]{(byte) 0xc1, (byte) 0xb8, 'P', 'o', 's'}); + out.writeInt(11); + }), 11, -9); + } + + @Test + public void preservesDuplicateAndNestedFieldSemantics() throws Exception { + check("duplicate-status-last-wins", root(out -> field(out, 8, "Status", child -> child.writeUTF("minecraft:noise"))), 7, -9); + check("duplicate-status-wrong-type", root(out -> field(out, 3, "Status", child -> child.writeInt(4))), 7, -9); + check("duplicate-x-last-wins", root(out -> field(out, 3, "xPos", child -> child.writeInt(11))), 11, -9); + check("duplicate-x-wrong-type", root(out -> field(out, 8, "xPos", child -> child.writeUTF("7"))), 7, -9); + check("duplicate-z-last-wins", root(out -> field(out, 3, "zPos", child -> child.writeInt(11))), 7, 11); + check("nested-fields-ignored", root(out -> field(out, 10, "extra", child -> { + field(child, 3, "xPos", value -> value.writeInt(33)); + field(child, 8, "Status", value -> value.writeUTF("minecraft:empty")); + child.writeByte(0); + })), 7, -9); + } + + @Test + public void validatesNestingAndArrayLengths() throws Exception { + for (int depth : new int[]{510, 511, 512, 513}) { + check("compound-depth-" + depth, root(out -> nested(out, depth, false)), 7, -9); + check("list-depth-" + depth, root(out -> nested(out, depth, true)), 7, -9); + } + for (int type : new int[]{7, 11, 12}) { + check("negative-array-" + type, root(out -> field(out, type, "extra", child -> child.writeInt(-1))), 7, -9); + check("truncated-array-" + type, root(out -> field(out, type, "extra", child -> child.writeInt(65536))), 7, -9); + } + for (int length : new int[]{-1, 0, 1}) { + check("invalid-list-type-length-" + length, root(out -> field(out, 9, "extra", child -> { + child.writeByte(99); + child.writeInt(length); + })), 7, -9); + } + check("negative-list-length", root(out -> field(out, 9, "extra", child -> { + child.writeByte(3); + child.writeInt(-1); + })), 7, -9); + } + + @Test + public void validatesTruncatedAndMalformedPayloads() throws Exception { + byte[] valid = root(out -> field(out, 8, "tail", child -> child.writeUTF("tailvalue"))); + for (int cut : new int[]{0, 1, 2, 3, valid.length - 1, valid.length - 2, valid.length - 8}) { + check("truncated-at-" + cut, Arrays.copyOf(valid, cut), 7, -9); + } + check("invalid-tail-tag", root(out -> { out.writeByte(99); out.writeUTF("tail"); }), 7, -9); + check("trailing-garbage", Arrays.copyOf(valid, valid.length + 3), 7, -9); + check("coordinates-mismatch", valid, 8, -9); + check("raw-zero-string", root(out -> field(out, 8, "Status", child -> rawUtf(child, new byte[]{0}))), 7, -9); + check("overlong-value", root(out -> field(out, 8, "Status", child -> rawUtf(child, new byte[]{(byte) 0xc1, (byte) 0x93}))), 7, -9); + check("non-ascii-root-key", root(out -> field(out, 8, "xP\u00f6s", child -> child.writeUTF("ignored"))), 7, -9); + check("maximum-utf-name", root(out -> field(out, 1, "k".repeat(65535), child -> child.writeByte(3))), 7, -9); + check("root-not-compound", new byte[]{3, 0, 0, 0, 0, 0, 1}, 7, -9); + check("empty-compound-missing-fields", new byte[]{10, 0, 0, 0}, 7, -9); + check("root-malformed-utf", new byte[]{10, 0, 1, (byte) 0x80, 0}, 7, -9); + check("maximum-empty-list-type", root(out -> field(out, 9, "extra", child -> { + child.writeByte(255); + child.writeInt(0); + })), 7, -9); + check("empty-array", root(out -> field(out, 12, "extra", child -> child.writeInt(0))), 7, -9); + check("truncated-primitive-tail", root(out -> field(out, 6, "extra", child -> child.writeInt(4))), 7, -9); + check("malformed-utf-name-tail", root(out -> { + out.writeByte(1); + rawUtf(out, new byte[]{(byte) 0xc2}); + out.writeByte(0); + }), 7, -9); + for (int depth : new int[]{511, 512}) { + check("nonempty-depth-" + depth, root(out -> { + for (int index = 0; index < depth; index++) { + out.writeByte(10); + out.writeUTF("nested"); + } + field(out, 1, "leaf", child -> child.writeByte(1)); + for (int index = 0; index < depth; index++) { + out.writeByte(0); + } + }), 7, -9); + } + check("float-nan-coordinate", root(out -> field(out, 5, "xPos", child -> child.writeFloat(Float.NaN))), 0, -9); + + } + + private static byte[] namedStatus(int width) throws IOException { + return root(out -> { + out.writeByte(8); + if (width == 2) { + rawUtf(out, new byte[]{(byte) 0xc1, (byte) 0x93, 't', 'a', 't', 'u', 's'}); + } else { + rawUtf(out, new byte[]{(byte) 0xe0, (byte) 0x81, (byte) 0x93, 't', 'a', 't', 'u', 's'}); + } + out.writeUTF("minecraft:noise"); + }); + } + + private static void check(String label, byte[] data, int x, int z) { + Outcome tree = tree(data, x, z); + Outcome cursor; + try { + cursor = new Outcome(true, SavedTerrainStatusNbt.read(data, 0, data.length, x, z), ""); + } catch (IOException | RuntimeException failure) { + cursor = new Outcome(false, null, failure.getClass().getSimpleName()); + } + boolean same = tree.accepted == cursor.accepted && (!tree.accepted || tree.status.equals(cursor.status)); + assertTrue(label + " tree=" + tree + " cursor=" + cursor, same); + } + + private static Outcome tree(byte[] bytes, int x, int z) { + try { + Tag parsed = NBTUtil.read(new ByteArrayInputStream(bytes), false).getTag(); + if (!(parsed instanceof CompoundTag root) + || !(root.get("xPos") instanceof NumberTag savedX) || savedX.asInt() != x + || !(root.get("zPos") instanceof NumberTag savedZ) || savedZ.asInt() != z + || !(root.get("Status") instanceof StringTag status) || status.getValue().isBlank()) { + throw new IOException("Invalid chunk fields"); + } + return new Outcome(true, status.getValue(), ""); + } catch (IOException | RuntimeException failure) { + return new Outcome(false, null, failure.getClass().getSimpleName()); + } + } + + private static byte[] root(Writer extra) throws IOException { + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + DataOutputStream out = new DataOutputStream(bytes); + out.writeByte(10); + out.writeUTF("\u0000\ud800root"); + field(out, 3, "xPos", child -> child.writeInt(7)); + field(out, 3, "zPos", child -> child.writeInt(-9)); + field(out, 8, "Status", child -> child.writeUTF("minecraft:full")); + extra.write(out); + out.writeByte(0); + return bytes.toByteArray(); + } + + private static void field(DataOutputStream out, int type, String key, Writer value) throws IOException { + out.writeByte(type); + out.writeUTF(key); + value.write(out); + } + + private static void rawUtf(DataOutputStream out, byte[] bytes) throws IOException { + out.writeShort(bytes.length); + out.write(bytes); + } + + private static void nested(DataOutputStream out, int depth, boolean lists) throws IOException { + out.writeByte(lists ? 9 : 10); + out.writeUTF("nested"); + for (int index = 1; index < depth; index++) { + out.writeByte(lists ? 9 : 10); + if (lists) { + out.writeInt(1); + } else { + out.writeUTF("nested"); + } + } + if (lists) { + out.writeByte(0); + out.writeInt(0); + } else { + for (int index = 0; index < depth; index++) { + out.writeByte(0); + } + } + } + + private static void payload(DataOutputStream out, int type) throws IOException { + switch (type) { + case 0 -> { } + case 1, 2, 3, 4, 5, 6 -> numeric(out, type, 7); + case 7 -> { out.writeInt(3); out.write(new byte[]{1, 2, 3}); } + case 8 -> out.writeUTF("\u0000\ud83d\ude00\u00e9\u4e00"); + case 9 -> { out.writeByte(3); out.writeInt(1); out.writeInt(42); } + case 10 -> { field(out, 3, "nested", child -> child.writeInt(42)); out.writeByte(0); } + case 11 -> { out.writeInt(2); out.writeInt(42); out.writeInt(-42); } + case 12 -> { out.writeInt(2); out.writeLong(42); out.writeLong(-42); } + default -> throw new IOException("Unknown type"); + } + } + + private static void numeric(DataOutputStream out, int type, int value) throws IOException { + switch (type) { + case 1 -> out.writeByte(value); + case 2 -> out.writeShort(value); + case 3 -> out.writeInt(value); + case 4 -> out.writeLong(value); + case 5 -> out.writeFloat(value); + case 6 -> out.writeDouble(value); + default -> throw new IOException("Unknown numeric type"); + } + } + + private record Outcome(boolean accepted, String status, String failure) { + } + + private interface Writer { + void write(DataOutputStream out) throws IOException; + } +} diff --git a/core/src/test/java/art/arcane/iris/world/history/SavedTerrainStatusReaderTest.java b/core/src/test/java/art/arcane/iris/world/history/SavedTerrainStatusReaderTest.java new file mode 100644 index 000000000..101068478 --- /dev/null +++ b/core/src/test/java/art/arcane/iris/world/history/SavedTerrainStatusReaderTest.java @@ -0,0 +1,257 @@ +package art.arcane.iris.world.history; + +import art.arcane.volmlib.util.nbt.io.NBTUtil; +import art.arcane.volmlib.util.nbt.tag.CompoundTag; +import net.jpountz.lz4.LZ4BlockOutputStream; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.io.ByteArrayOutputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.io.RandomAccessFile; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.util.Arrays; +import java.util.Random; +import java.util.zip.DeflaterOutputStream; +import java.util.zip.GZIPOutputStream; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; + +public final class SavedTerrainStatusReaderTest { + @Rule + public final TemporaryFolder temporaryFolder = new TemporaryFolder(); + + @Test + public void reusesReaderAcrossCodecsRegionsAndExternalChunks() throws Exception { + Path world = temporaryFolder.newFolder().toPath(); + try (SavedTerrainChunkReader.StatusReader reader = new SavedTerrainChunkReader.StatusReader(world)) { + for (int compression = 1; compression <= 4; compression++) { + for (boolean external : new boolean[]{false, true}) { + int x = compression * 32 - 160; + writeChunk(world, x, -2, encode(root(x, -2, "minecraft:noise")), compression, external); + assertEquals("minecraft:noise", reader.readStatus(x, -2)); + assertEquals("minecraft:noise", SavedTerrainChunk.readStatus(world, x, -2)); + writeChunk(world, x, -2, encode(root(x, -2, "minecraft:full")), compression, external); + assertEquals("minecraft:full", reader.readStatus(x, -2)); + } + } + } + } + + @Test + public void observesAtomicReplacementOfAnOpenRegion() throws Exception { + Path world = temporaryFolder.newFolder().toPath(); + Path replacement = temporaryFolder.newFolder().toPath(); + writeChunk(world, -1, -2, orderedRoot(), 2, false); + writeChunk(replacement, -1, -2, encode(root(-1, -2, "minecraft:noise")), 2, false); + try (SavedTerrainChunkReader.StatusReader reader = new SavedTerrainChunkReader.StatusReader(world)) { + assertEquals("minecraft:full", reader.readStatus(-1, -2)); + Files.move(replacement.resolve("region/r.-1.-1.mca"), world.resolve("region/r.-1.-1.mca"), + StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); + assertEquals("minecraft:noise", reader.readStatus(-1, -2)); + } + } + + @Test + public void identifiesTheChunkWhenDecodingFails() throws Exception { + Path world = temporaryFolder.newFolder().toPath(); + writeChunk(world, -1, -2, new byte[]{10}, 2, false); + try (SavedTerrainChunkReader.StatusReader reader = new SavedTerrainChunkReader.StatusReader(world)) { + IOException failure = assertThrows(IOException.class, () -> reader.readStatus(-1, -2)); + assertTrue(failure.getMessage().contains("-1,-2")); + assertTrue(failure.getCause() instanceof IOException); + } + } + + @Test + public void rejectsReadsAfterCloseAndCanCloseTwice() throws Exception { + Path world = temporaryFolder.newFolder().toPath(); + writeChunk(world, -1, -2, encode(root(-1, -2, "minecraft:full")), 2, false); + SavedTerrainChunkReader.StatusReader reader = new SavedTerrainChunkReader.StatusReader(world); + assertEquals("minecraft:full", reader.readStatus(-1, -2)); + reader.close(); + reader.close(); + assertThrows(IOException.class, () -> reader.readStatus(-1, -2)); + } + + @Test + public void rejectsCorruptionAfterRequiredFieldsAndRecoversForNextChunk() throws Exception { + Path world = temporaryFolder.newFolder().toPath(); + byte[] valid = orderedRoot(); + byte[] malformed = Arrays.copyOf(valid, valid.length + 3); + malformed[valid.length - 1] = 99; + byte[][] failures = {Arrays.copyOf(valid, valid.length - 1), malformed, + encode(root(1, -2, "minecraft:full")), encode(root(-1, -2, " "))}; + try (SavedTerrainChunkReader.StatusReader reader = new SavedTerrainChunkReader.StatusReader(world)) { + for (byte[] failure : failures) { + writeChunk(world, -1, -2, failure, 2, false); + assertThrows(IOException.class, () -> reader.readStatus(-1, -2)); + writeChunk(world, -1, -2, valid, 2, false); + assertEquals("minecraft:full", reader.readStatus(-1, -2)); + } + } + } + + @Test + public void validatesCompressionChecksumsAndTruncationBeforeReturningStatus() throws Exception { + Path world = temporaryFolder.newFolder().toPath(); + try (SavedTerrainChunkReader.StatusReader reader = new SavedTerrainChunkReader.StatusReader(world)) { + for (int compression : new int[]{1, 2, 4}) { + writeChunk(world, -1, -2, orderedRoot(), compression, true); + Path external = world.resolve("region/c.-1.-2.mcc"); + byte[] encoded = Files.readAllBytes(external); + Files.write(external, Arrays.copyOf(encoded, encoded.length - 3)); + assertThrows(IOException.class, () -> reader.readStatus(-1, -2)); + writeChunk(world, -1, -2, orderedRoot(), compression, true); + byte[] corrupted = Files.readAllBytes(external); + corrupted[compression == 4 ? 17 : corrupted.length - 5] ^= 1; + Files.write(external, corrupted); + assertThrows(IOException.class, () -> reader.readStatus(-1, -2)); + } + } + } + + @Test + public void rejectsOversizedInflationAndRetainsOnlyCurrentPayloadLength() throws Exception { + Path world = temporaryFolder.newFolder().toPath(); + try (SavedTerrainChunkReader.StatusReader reader = new SavedTerrainChunkReader.StatusReader(world)) { + writeChunk(world, -1, -2, orderedRoot(), 2, true); + Path external = world.resolve("region/c.-1.-2.mcc"); + try (OutputStream output = new DeflaterOutputStream(Files.newOutputStream(external))) { + byte[] padding = new byte[65536]; + for (int index = 0; index < 1024; index++) { + output.write(padding); + } + output.write(0); + } + assertThrows(IOException.class, () -> reader.readStatus(-1, -2)); + writeChunk(world, -1, -2, orderedRoot(), 2, false); + assertEquals("minecraft:full", reader.readStatus(-1, -2)); + byte[] truncated = Arrays.copyOf(orderedRoot(), orderedRoot().length - 1); + writeChunk(world, -1, -2, truncated, 2, false); + assertThrows(IOException.class, () -> reader.readStatus(-1, -2)); + } + } + + @Test + public void growsBuffersAndReusesReaderAcrossLargeDirectAndSliceReads() throws Exception { + Path world = temporaryFolder.newFolder().toPath(); + Random random = new Random(729L); + try (SavedTerrainChunkReader.StatusReader reader = new SavedTerrainChunkReader.StatusReader(world)) { + for (int size : new int[]{196_608, 64, 131_072, 196_608}) { + byte[] padding = new byte[size]; + random.nextBytes(padding); + CompoundTag root = root(-1, -2, "minecraft:full"); + root.putByteArray("padding", padding); + byte[] nbt = encode(root); + writeChunk(world, -1, -2, nbt, 2, false); + assertEquals("minecraft:full", reader.readStatus(-1, -2)); + byte[] compressed = compress(nbt); + byte[] slice = new byte[compressed.length + 96]; + Arrays.fill(slice, (byte) 127); + System.arraycopy(compressed, 0, slice, 37, compressed.length); + assertEquals("minecraft:full", reader.readStatus(slice, + new SavedTerrainRegionSnapshot.Payload(37, compressed.length, 2, false), -1, -2)); + } + } + } + + @Test + public void rejectsCorruptAndTruncatedSlicesAndRecoversForValidInput() throws Exception { + Path world = temporaryFolder.newFolder().toPath(); + byte[] compressed = compress(orderedRoot()); + byte[] slice = new byte[compressed.length + 64]; + System.arraycopy(compressed, 0, slice, 31, compressed.length); + SavedTerrainRegionSnapshot.Payload stored = new SavedTerrainRegionSnapshot.Payload( + 31, compressed.length, 2, false); + try (SavedTerrainChunkReader.StatusReader reader = new SavedTerrainChunkReader.StatusReader(world)) { + assertEquals("minecraft:full", reader.readStatus(slice, stored, -1, -2)); + SavedTerrainRegionSnapshot.Payload truncated = new SavedTerrainRegionSnapshot.Payload( + 31, compressed.length - 1, 2, false); + assertThrows(IOException.class, () -> reader.readStatus(slice, truncated, -1, -2)); + assertEquals("minecraft:full", reader.readStatus(slice, stored, -1, -2)); + byte[] corrupt = slice.clone(); + corrupt[31 + compressed.length - 1] ^= 1; + assertThrows(IOException.class, () -> reader.readStatus(corrupt, stored, -1, -2)); + assertEquals("minecraft:full", reader.readStatus(slice, stored, -1, -2)); + } + } + + private static byte[] compress(byte[] bytes) throws IOException { + ByteArrayOutputStream compressed = new ByteArrayOutputStream(); + try (OutputStream output = new DeflaterOutputStream(compressed)) { + output.write(bytes); + } + return compressed.toByteArray(); + } + + private static CompoundTag root(int x, int z, String status) { + CompoundTag root = new CompoundTag(); + root.putInt("xPos", x); + root.putInt("zPos", z); + root.putString("Status", status); + return root; + } + + private static byte[] orderedRoot() throws IOException { + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + DataOutputStream output = new DataOutputStream(bytes); + output.writeByte(10); + output.writeUTF(""); + output.writeByte(3); + output.writeUTF("xPos"); + output.writeInt(-1); + output.writeByte(3); + output.writeUTF("zPos"); + output.writeInt(-2); + output.writeByte(8); + output.writeUTF("Status"); + output.writeUTF("minecraft:full"); + output.writeByte(0); + return bytes.toByteArray(); + } + + private static byte[] encode(CompoundTag root) throws IOException { + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + NBTUtil.write(root, bytes, false); + return bytes.toByteArray(); + } + + private static void writeChunk(Path world, int x, int z, byte[] nbt, int compression, boolean external) + throws IOException { + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + try (OutputStream output = switch (compression) { + case 1 -> new GZIPOutputStream(bytes); + case 2 -> new DeflaterOutputStream(bytes); + case 4 -> new LZ4BlockOutputStream(bytes); + default -> bytes; + }) { + output.write(nbt); + } + byte[] payload = bytes.toByteArray(); + Path directory = Files.createDirectories(world.resolve("region")); + Path region = directory.resolve("r." + Math.floorDiv(x, 32) + "." + Math.floorDiv(z, 32) + ".mca"); + try (RandomAccessFile file = new RandomAccessFile(region.toFile(), "rw")) { + int sectors = external ? 1 : Math.ceilDiv(payload.length + 5, 4096); + file.setLength((2L + sectors) * 4096); + file.seek((Math.floorMod(z, 32) * 32L + Math.floorMod(x, 32)) * 4); + file.writeInt((2 << 8) | sectors); + file.seek(8192); + file.writeInt(external ? 1 : payload.length + 1); + file.writeByte(compression | (external ? 128 : 0)); + if (!external) { + file.write(payload); + } + } + if (external) { + Files.write(directory.resolve("c." + x + "." + z + ".mcc"), payload); + } + } +} diff --git a/core/src/test/java/art/arcane/iris/world/history/TerrainBoundarySignaturePublicationTest.java b/core/src/test/java/art/arcane/iris/world/history/TerrainBoundarySignaturePublicationTest.java new file mode 100644 index 000000000..9e8910f32 --- /dev/null +++ b/core/src/test/java/art/arcane/iris/world/history/TerrainBoundarySignaturePublicationTest.java @@ -0,0 +1,205 @@ +package art.arcane.iris.world.history; + +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.mockito.MockedStatic; + +import java.io.DataInputStream; +import java.io.File; +import java.io.IOException; +import java.nio.channels.FileChannel; +import java.nio.file.DirectoryStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.OptionalInt; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; +import static org.junit.Assume.assumeTrue; +import static org.mockito.AdditionalAnswers.delegatesTo; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.mockStatic; + +public final class TerrainBoundarySignaturePublicationTest { + @Rule + public final TemporaryFolder temporaryFolder = new TemporaryFolder(); + + @Test + public void syncsAllShardFilesThenDirectoryBeforePublishingEitherCatalogPath() throws Exception { + assumeTrue(File.separatorChar != '\\'); + GenerationBoundary boundary = GenerationBoundary.freeze("a".repeat(64), List.of( + new GenerationBoundary.ChunkCoordinate(0, 0), + new GenerationBoundary.ChunkCoordinate(4, 0), + new GenerationBoundary.ChunkCoordinate(8, 0))); + List signatures = new ArrayList<>(); + boundary.forEachExposedBlockColumn((x, z) -> signatures.add(signature(x, z))); + for (boolean streamed : new boolean[]{false, true}) { + TerrainBoundarySignatureStore store = store(); + List events = new ArrayList<>(); + try (MockedStatic ignored = channels(path -> { + if (path.equals(store.directory())) { + assertEquals(3, shards(store).size()); + events.add(Files.exists(store.snapshotPath(1L)) ? "catalog-directory" : "shard-directory"); + } else if (path.getFileName().toString().endsWith(".tmp")) { + events.add(fileKind(path)); + } + })) { + if (streamed) { + store.publish(1L, boundary, TerrainBoundarySignaturePublicationTest::signature); + } else { + store.publish(1L, signatures); + } + } + assertEquals(List.of("shard", "shard", "shard", "shard-directory", "catalog", "catalog-directory"), + events); + assertEquals(signatures.size(), store.load(1L).size()); + } + } + + @Test + public void failedShardDirectoryBarrierDoesNotPublishCatalogAndRetrySyncsExistingShards() throws Exception { + assumeTrue(File.separatorChar != '\\'); + TerrainBoundarySignatureStore store = store(); + List signatures = List.of(signature(0, 0), signature(64, 0)); + IOException failure = new IOException("Shard directory force failed"); + try (MockedStatic ignored = channels(path -> { + if (path.equals(store.directory())) { + throw failure; + } + })) { + assertSame(failure, assertThrows(IOException.class, () -> store.publish(1L, signatures))); + } + assertFalse(Files.exists(store.snapshotPath(1L))); + assertEquals(2, shards(store).size()); + List events = new ArrayList<>(); + try (MockedStatic ignored = channels(path -> { + if (path.equals(store.directory())) { + events.add(Files.exists(store.snapshotPath(1L)) ? "catalog-directory" : "shard-directory"); + } else if (path.getFileName().toString().endsWith(".tmp")) { + events.add(fileKind(path)); + } + })) { + assertEquals(2, store.publish(1L, signatures).size()); + } + assertEquals(List.of("shard-directory", "catalog", "catalog-directory"), events); + } + + @Test + public void failedCatalogDirectoryBarrierIsRetriedBeforeExistingCatalogReturns() throws Exception { + assumeTrue(File.separatorChar != '\\'); + TerrainBoundarySignatureStore store = store(); + List signatures = List.of(signature(0, 0)); + IOException failure = new IOException("Catalog directory force failed"); + try (MockedStatic ignored = channels(path -> { + if (path.equals(store.directory()) && Files.exists(store.snapshotPath(1L))) { + throw failure; + } + })) { + assertSame(failure, assertThrows(IOException.class, () -> store.publish(1L, signatures))); + } + assertTrue(Files.exists(store.snapshotPath(1L))); + AtomicInteger directoryForces = new AtomicInteger(); + try (MockedStatic ignored = channels(path -> { + if (path.equals(store.directory())) { + directoryForces.incrementAndGet(); + } + assertFalse(path.getFileName().toString().endsWith(".tmp")); + })) { + assertEquals(1, store.publish(1L, signatures).size()); + } + assertEquals(1, directoryForces.get()); + } + + @Test + public void shardForceFailureCannotPublishShardOrCatalog() throws Exception { + TerrainBoundarySignatureStore store = store(); + IOException failure = new IOException("Shard file force failed"); + try (MockedStatic ignored = channels(path -> { + if (path.getFileName().toString().endsWith(".tmp") && fileKind(path).equals("shard")) { + throw failure; + } + })) { + assertSame(failure, assertThrows(IOException.class, + () -> store.publish(1L, List.of(signature(0, 0))))); + } + assertTrue(shards(store).isEmpty()); + assertFalse(Files.exists(store.snapshotPath(1L))); + } + + @Test + public void existingCorruptShardCannotReachCatalogPublication() throws Exception { + TerrainBoundarySignatureStore store = store(); + List signatures = List.of(signature(0, 0)); + store.publish(1L, signatures); + Path shard = shards(store).getFirst(); + byte[] bytes = Files.readAllBytes(shard); + bytes[bytes.length - 1] ^= 1; + Files.write(shard, bytes); + assertThrows(IOException.class, () -> store.publish(2L, signatures)); + assertFalse(Files.exists(store.snapshotPath(2L))); + } + + private TerrainBoundarySignatureStore store() throws IOException { + TerrainBoundarySignatureStore store = new TerrainBoundarySignatureStore(temporaryFolder.newFolder().toPath()); + Files.createDirectories(store.directory()); + return store; + } + + private static List shards(TerrainBoundarySignatureStore store) throws IOException { + List shards = new ArrayList<>(); + try (DirectoryStream entries = Files.newDirectoryStream(store.directory(), "*.irtm")) { + for (Path entry : entries) { + shards.add(entry); + } + } + return shards; + } + + private static String fileKind(Path path) throws IOException { + try (DataInputStream input = new DataInputStream(Files.newInputStream(path))) { + return switch (input.readInt()) { + case 0x4952544D -> "shard"; + case 0x49525453 -> "catalog"; + default -> throw new IOException("Unexpected terrain boundary file"); + }; + } + } + + private static MockedStatic channels(ForceObserver observer) { + return mockStatic(FileChannel.class, invocation -> { + FileChannel source = (FileChannel) invocation.callRealMethod(); + if (invocation.getMethod().getParameterCount() != 2) { + return source; + } + Path path = invocation.getArgument(0); + FileChannel intercepted = mock(FileChannel.class, delegatesTo(source)); + doAnswer(force -> { + observer.force(path); + source.force(true); + return null; + }).when(intercepted).force(true); + return intercepted; + }); + } + + private static TerrainBoundarySignature signature(int x, int z) { + return new TerrainBoundarySignature( + new TerrainBoundarySignature.Column(x, z, 80 + x, 60, OptionalInt.empty(), OptionalInt.empty()), + new TerrainBoundarySignature.Samples(new TerrainBoundarySignature.VerticalLayout(-64, 32, 1), + new TerrainBoundarySignature.BiomeEncoding(List.of("iris:plains"), new short[]{0})), + BoundaryColumnGeometry.empty()); + } + + @FunctionalInterface + private interface ForceObserver { + void force(Path path) throws IOException; + } +}