From 21f8440299e393601e121335bd6163677f46208b Mon Sep 17 00:00:00 2001 From: Brian Neumann-Fopiano Date: Mon, 28 Sep 2026 02:13:17 -0400 Subject: [PATCH] Stabilized hydrology pregeneration - Preserve exact river owner plans across cache eviction and persistence. - Retry required hydrology plans without generating riverless terrain. - Bound object plans and pregeneration residency by available heap. - Release completed native chunks and avoid empty reclamation flushes. - Cover persistence, admission, cancellation, and ownership regressions. --- .../iris/configuration/IrisSettings.java | 9 +- .../hydrology/CrossTileResolutionContext.java | 3 +- .../hydrology/HydrologyCacheWeights.java | 3 + .../hydrology/HydrologyOwnerDraft.java | 9 + .../hydrology/HydrologyPlanner.java | 73 +- .../generation/hydrology/HydrologyTile.java | 34 + .../hydrology/HydrologyTileCache.java | 132 ++-- .../hydrology/HydrologyTileCodec.java | 706 ++++++++++++++++++ .../hydrology/PreparedHydrologyTileStore.java | 258 +------ .../hydrology/cave/HydrologyCavePlan.java | 84 ++- .../generation/mantle/MatterGenerator.java | 48 +- .../mantle/ObjectDestinationTransaction.java | 9 - .../generation/mantle/ObjectSourcePlan.java | 53 +- .../mantle/ObjectSourcePlanCache.java | 31 +- .../pregen/AsyncOrMedievalPregenMethod.java | 5 + .../iris/world/pregen/AsyncPregenMethod.java | 115 +-- .../iris/world/pregen/CachedPregenMethod.java | 5 + .../iris/world/pregen/HybridPregenMethod.java | 5 + .../iris/world/pregen/IrisPregenerator.java | 6 + .../pregen/PregenMantleBackpressure.java | 55 +- .../iris/world/pregen/PregeneratorMethod.java | 3 + .../IrisSettingsDefaultsTest.java | 10 + ...CrossTileCaveAdmissionIntegrationTest.java | 36 + .../hydrology/HydrologyPlannerTest.java | 44 ++ ...ologyPlanningAdmissionIntegrationTest.java | 1 - .../hydrology/HydrologyRegionalFallsTest.java | 32 +- .../HydrologyRegionalPlannerTest.java | 1 + .../hydrology/HydrologyTileCacheTest.java | 247 ++++-- .../hydrology/HydrologyTileTest.java | 65 ++ .../PreparedHydrologyTileStoreTest.java | 231 +++++- .../MatterGeneratorConcurrencyTest.java | 97 ++- .../ObjectDestinationTransactionTest.java | 2 +- .../mantle/ObjectSourcePlanCacheTest.java | 61 +- .../mantle/ObjectSourcePlanTest.java | 12 +- .../AsyncPregenMethodConcurrencyCapTest.java | 10 + .../AsyncPregenServerSaveOwnershipTest.java | 166 +++- .../CachedPregenMethodCompletionTest.java | 11 +- .../pregen/IrisPregeneratorInitTest.java | 48 ++ .../pregen/PregenMantleBackpressureTest.java | 117 ++- .../PregenSilentFailureContractTest.java | 1 - 40 files changed, 2218 insertions(+), 620 deletions(-) create mode 100644 core/src/main/java/art/arcane/iris/generation/hydrology/HydrologyTileCodec.java create mode 100644 core/src/test/java/art/arcane/iris/generation/hydrology/HydrologyTileTest.java diff --git a/core/src/main/java/art/arcane/iris/configuration/IrisSettings.java b/core/src/main/java/art/arcane/iris/configuration/IrisSettings.java index e5fd81a6c..ed692a86a 100644 --- a/core/src/main/java/art/arcane/iris/configuration/IrisSettings.java +++ b/core/src/main/java/art/arcane/iris/configuration/IrisSettings.java @@ -27,7 +27,6 @@ import art.arcane.iris.spi.IrisPlatforms; import art.arcane.volmlib.util.io.IO; import art.arcane.volmlib.util.json.JSONException; import art.arcane.volmlib.util.json.JSONObject; -import art.arcane.iris.platform.bootstrap.getHardware; import art.arcane.iris.platform.bukkit.plugin.VolmitSender; import lombok.Data; @@ -226,7 +225,7 @@ public class IrisSettings { public static class IrisSettingsPregen { private static final int REFERENCE_WORLD_HEIGHT = 384; private static final int MIN_RESIDENT_TECTONIC_PLATES = 16; - private static final double MANTLE_HEAP_FRACTION = 0.6D; + private static final double MANTLE_HEAP_FRACTION = 0.25D; private static final int REFERENCE_PLATE_MEGABYTES = 48; public IrisRuntimeSchedulerMode runtimeSchedulerMode = IrisRuntimeSchedulerMode.AUTO; public IrisPaperLikeBackendMode paperLikeBackendMode = IrisPaperLikeBackendMode.AUTO; @@ -256,10 +255,14 @@ public class IrisSettings { } public int getEffectiveResidentTectonicPlates(int worldHeight) { + return effectiveResidentTectonicPlates(worldHeight, Runtime.getRuntime().maxMemory()); + } + + int effectiveResidentTectonicPlates(int worldHeight, long maximumHeapBytes) { int baseCap = getMaxResidentTectonicPlates(); int normalizedHeight = Math.max(1, worldHeight); int heightScaledCap = (int) Math.round((double) baseCap * REFERENCE_WORLD_HEIGHT / (double) normalizedHeight); - long maxHeapMegabytes = getHardware.getProcessMemory(); + long maxHeapMegabytes = maximumHeapBytes / (1024L * 1024L); double plateMegabytes = (double) REFERENCE_PLATE_MEGABYTES * (double) normalizedHeight / (double) REFERENCE_WORLD_HEIGHT; int byteBudgetCap = (int) Math.floor(MANTLE_HEAP_FRACTION * (double) maxHeapMegabytes / plateMegabytes); int effective = Math.min(heightScaledCap, byteBudgetCap); diff --git a/core/src/main/java/art/arcane/iris/generation/hydrology/CrossTileResolutionContext.java b/core/src/main/java/art/arcane/iris/generation/hydrology/CrossTileResolutionContext.java index 5061b9030..d74ac8d93 100644 --- a/core/src/main/java/art/arcane/iris/generation/hydrology/CrossTileResolutionContext.java +++ b/core/src/main/java/art/arcane/iris/generation/hydrology/CrossTileResolutionContext.java @@ -39,7 +39,8 @@ final class CrossTileResolutionContext { CrossTileResolvedOwner existing = resolved.putIfAbsent(key, owner); if (existing != null && !existing.withoutFootprintCompiler().equals(owner.withoutFootprintCompiler())) { - throw new IllegalStateException("Cross-tile owner resolution produced inconsistent cached results."); + throw new IllegalStateException("Cross-tile owner resolution produced inconsistent cached results for " + + key.tileX() + "," + key.tileZ() + " while planning " + root.tileX() + "," + root.tileZ() + "."); } if (resolved.size() > maximumOwners) { throw new IllegalStateException("Cross-tile owner dependencies exceeded their color-ranked count bound."); diff --git a/core/src/main/java/art/arcane/iris/generation/hydrology/HydrologyCacheWeights.java b/core/src/main/java/art/arcane/iris/generation/hydrology/HydrologyCacheWeights.java index 3202b868a..512357c32 100644 --- a/core/src/main/java/art/arcane/iris/generation/hydrology/HydrologyCacheWeights.java +++ b/core/src/main/java/art/arcane/iris/generation/hydrology/HydrologyCacheWeights.java @@ -24,6 +24,9 @@ final class HydrologyCacheWeights { long bytes = network(tile.nodes(), tile.edges(), tile.outlets(), tile.courses(), tile.cavePlans()); bytes += diagnostics(tile.localDiagnosticCandidates()); bytes += (long) tile.regionalCourseIds().size() * 64L; + if (tile.resolvedOwner() != null) { + bytes += owner(tile.resolvedOwner()); + } RiverFootprint footprint = tile.footprint(); if (footprint != null) { bytes += (long) footprint.size() * 64L; diff --git a/core/src/main/java/art/arcane/iris/generation/hydrology/HydrologyOwnerDraft.java b/core/src/main/java/art/arcane/iris/generation/hydrology/HydrologyOwnerDraft.java index 3380d1016..89704ad52 100644 --- a/core/src/main/java/art/arcane/iris/generation/hydrology/HydrologyOwnerDraft.java +++ b/core/src/main/java/art/arcane/iris/generation/hydrology/HydrologyOwnerDraft.java @@ -1,6 +1,7 @@ package art.arcane.iris.generation.hydrology; import java.util.List; +import java.util.Objects; record HydrologyOwnerDraft( HydrologyTileKey key, @@ -8,6 +9,14 @@ record HydrologyOwnerDraft( List diagnostics, HydrologyFootprintCompiler footprintCompiler ) { + HydrologyOwnerDraft { + Objects.requireNonNull(key, "key"); + Objects.requireNonNull(result, "result"); + result = new HydrologyCaveCourseFilter.Result(List.copyOf(result.nodes()), List.copyOf(result.edges()), + List.copyOf(result.outlets()), List.copyOf(result.courses()), List.copyOf(result.cavePlans())); + diagnostics = List.copyOf(diagnostics); + } + HydrologyOwnerDraft withoutFootprintCompiler() { if (footprintCompiler == null) { return this; diff --git a/core/src/main/java/art/arcane/iris/generation/hydrology/HydrologyPlanner.java b/core/src/main/java/art/arcane/iris/generation/hydrology/HydrologyPlanner.java index 5f7392bd0..042ca17b1 100644 --- a/core/src/main/java/art/arcane/iris/generation/hydrology/HydrologyPlanner.java +++ b/core/src/main/java/art/arcane/iris/generation/hydrology/HydrologyPlanner.java @@ -214,33 +214,12 @@ public final class HydrologyPlanner { return settings; } - /** - * A tile with the same identity as a planned tile but no rivers, used when planning the tile - * failed and terrain there must still generate. - */ - public HydrologyTile emptyTile(HydrologyTileKey key) { - Objects.requireNonNull(key, "key"); - return new HydrologyTile( - key, - worldSeed, - settings.fingerprint(), - settings.routing().tileSize(), - List.of(), - List.of(), - List.of(), - List.of(), - Set.of(), - List.of(), - List.of(), - RiverFootprint.empty() - ); - } - public HydrologyTile plan(HydrologyTileKey key) { long started = System.nanoTime(); CrossTileResolution resolution = resolveCrossTileOwner(key); long materializeStarted = System.nanoTime(); - HydrologyTile tile = crossTile.materializeAcceptedTile(resolution); + HydrologyTile tile = crossTile.materializeAcceptedTile(resolution) + .withResolvedOwner(new CrossTileResolvedOwner(resolution.draft(), resolution.observedRejections())); long finished = System.nanoTime(); IrisLogging.debug( "Hydrology tile %d,%d planned in %dms: owners=%d resolve=%dms materialize=%dms courses=%d on %s", @@ -277,32 +256,10 @@ public final class HydrologyPlanner { || tile.tileSize() != settings.routing().tileSize()) { throw new IllegalArgumentException("Hydrology tile does not match this planner."); } - ArrayList localCourses = new ArrayList<>(tile.courses().size()); - for (RiverCourse course : tile.courses()) { - if (!tile.regionalCourseIds().contains(course.id())) { - localCourses.add(course); - } + CrossTileResolvedOwner owner = tile.resolvedOwner(); + if (owner != null) { + resolvedOwners.asMap().putIfAbsent(tile.key(), owner); } - ArrayList localPlans = new ArrayList<>(tile.cavePlans().size()); - for (HydrologyCavePlan plan : tile.cavePlans()) { - if (!tile.regionalCourseIds().contains(plan.source().sourceId())) { - localPlans.add(plan); - } - } - HydrologyCaveCourseFilter.Result result = new HydrologyCaveCourseFilter.Result( - tile.nodes(), - tile.edges(), - tile.outlets(), - localCourses, - localPlans - ); - HydrologyOwnerDraft draft = new HydrologyOwnerDraft( - tile.key(), - result, - tile.localDiagnosticCandidates(), - null - ); - resolvedOwners.put(tile.key(), new CrossTileResolvedOwner(draft, List.of())); } CrossTileResolution resolveCrossTileOwner(HydrologyTileKey key) { @@ -386,6 +343,11 @@ public final class HydrologyPlanner { if (candidateKeys.size() < 2 || !IrisPlatforms.isBound()) { ArrayList resolved = new ArrayList<>(candidateKeys.size()); for (HydrologyTileKey candidateKey : candidateKeys) { + CrossTileResolvedOwner local = context.resolved(candidateKey); + if (local != null) { + resolved.add(local); + continue; + } HydrologyForkJoin.Task prepared = preparedOwners.get(candidateKey); CrossTileResolvedOwner owner = prepared == null ? resolveCrossTileOwner(candidateKey, context, false) : prepared.await(); @@ -397,16 +359,25 @@ public final class HydrologyPlanner { // Neighbour drafts use the current pool (or the burst pool from outside any pool). // Waiting workers claim their own tasks without helping unrelated owner drafts. ArrayList> tasks = new ArrayList<>(candidateKeys.size()); + ArrayList resolved = new ArrayList<>(candidateKeys.size()); for (HydrologyTileKey candidateKey : candidateKeys) { + CrossTileResolvedOwner local = context.resolved(candidateKey); + resolved.add(local); + if (local != null) { + continue; + } HydrologyForkJoin.Task prepared = preparedOwners.get(candidateKey); tasks.add(prepared == null ? () -> resolveIndependentOwner(candidateKey) : prepared::await); } List owners = HydrologyForkJoin.invokeAll(tasks, MultiBurst.burst); - ArrayList resolved = new ArrayList<>(candidateKeys.size()); + int completed = 0; for (int index = 0; index < candidateKeys.size(); index++) { - CrossTileResolvedOwner owner = owners.get(index); + if (resolved.get(index) != null) { + continue; + } + CrossTileResolvedOwner owner = owners.get(completed++); context.remember(candidateKeys.get(index), owner); - resolved.add(owner); + resolved.set(index, owner); } return List.copyOf(resolved); } diff --git a/core/src/main/java/art/arcane/iris/generation/hydrology/HydrologyTile.java b/core/src/main/java/art/arcane/iris/generation/hydrology/HydrologyTile.java index 262cc882b..121ddbfa0 100644 --- a/core/src/main/java/art/arcane/iris/generation/hydrology/HydrologyTile.java +++ b/core/src/main/java/art/arcane/iris/generation/hydrology/HydrologyTile.java @@ -34,6 +34,7 @@ public final class HydrologyTile { private final Map outletsById; private final Map cavePlansByCourseId; private final List features; + private final CrossTileResolvedOwner resolvedOwner; public HydrologyTile( HydrologyTileKey key, @@ -76,6 +77,39 @@ public final class HydrologyTile { validateGraph(); validateCourses(); this.features = collectFeatures(footprint); + this.resolvedOwner = null; + } + + private HydrologyTile(HydrologyTile tile, CrossTileResolvedOwner resolvedOwner) { + this.key = tile.key; + this.worldSeed = tile.worldSeed; + this.settingsFingerprint = tile.settingsFingerprint; + this.tileSize = tile.tileSize; + this.nodes = tile.nodes; + this.edges = tile.edges; + this.outlets = tile.outlets; + this.courses = tile.courses; + this.regionalCourseIds = tile.regionalCourseIds; + this.cavePlans = tile.cavePlans; + this.localDiagnosticCandidates = tile.localDiagnosticCandidates; + this.footprint = tile.footprint; + this.nodesById = tile.nodesById; + this.outletsById = tile.outletsById; + this.cavePlansByCourseId = tile.cavePlansByCourseId; + this.features = tile.features; + this.resolvedOwner = resolvedOwner; + } + + HydrologyTile withResolvedOwner(CrossTileResolvedOwner owner) { + Objects.requireNonNull(owner, "owner"); + if (!key.equals(owner.draft().key())) { + throw new IllegalArgumentException("Resolved hydrology owner does not match its tile."); + } + return new HydrologyTile(this, owner.withoutFootprintCompiler()); + } + + CrossTileResolvedOwner resolvedOwner() { + return resolvedOwner; } public HydrologyTileKey key() { diff --git a/core/src/main/java/art/arcane/iris/generation/hydrology/HydrologyTileCache.java b/core/src/main/java/art/arcane/iris/generation/hydrology/HydrologyTileCache.java index 2dce0754e..29461755c 100644 --- a/core/src/main/java/art/arcane/iris/generation/hydrology/HydrologyTileCache.java +++ b/core/src/main/java/art/arcane/iris/generation/hydrology/HydrologyTileCache.java @@ -316,7 +316,7 @@ public final class HydrologyTileCache implements AutoCloseable { } try { present = tiles.getIfPresent(key); - HydrologyTile tile = present == null ? planOrEmpty(key, epoch) : present; + HydrologyTile tile = present == null ? planRequired(key, epoch) : present; synchronized (publicationLock) { if (!closed.get() && cacheEpoch.get() == epoch) { tiles.put(key, tile); @@ -332,69 +332,107 @@ public final class HydrologyTileCache implements AutoCloseable { } } - /** - * A tile whose planning throws is published without rivers instead of failing the chunks that - * asked for it: one bad column must not take the chunk system down. The failure is logged in - * full so the cause can be traced, and the empty tile is cached like any other so the session - * stays consistent. A plan that was interrupted is not a bad tile: it is rethrown without being - * cached so the next request plans it again. - */ - private HydrologyTile planOrEmpty(HydrologyTileKey key, long planningEpoch) { + private HydrologyTile planRequired(HydrologyTileKey key, long planningEpoch) { SharedTileKey sharedKey = sharedCacheScope == null ? null : new SharedTileKey(sharedCacheScope, key); - HydrologyTile shared = sharedKey == null ? null : SHARED_TILES.getIfPresent(sharedKey); - if (shared != null && validSharedTile(shared, sharedKey)) { - PreparedHydrologyTileStore store = persistentStore; - if (store != null && store.load(key).isEmpty()) { - persistTile(store, shared); - } - planner.reuseResolvedTile(shared); - IrisLogging.debug("Reused shared prepared hydrology tile %d,%d", key.tileX(), key.tileZ()); - return shared; - } - if (shared != null) { - SHARED_TILES.invalidate(sharedKey); - } PreparedHydrologyTileStore store = persistentStore; - try (HydrologyPlanningAdmission.Permit ignored = HydrologyPlanningAdmission.acquireRoot(closed::get)) { - return planAdmitted(key, planningEpoch, sharedKey, store); + long attempt = 0L; + long retryDelayMillis = 250L; + long nextFailureLog = 0L; + while (true) { + requirePlanningActive(); + try { + if (attempt == 0L && sharedKey != null) { + HydrologyTile shared = SHARED_TILES.getIfPresent(sharedKey); + if (shared != null && validSharedTile(shared, sharedKey)) { + planner.reuseResolvedTile(shared); + if (store != null && !store.contains(key)) { + persistTile(store, shared); + } + IrisLogging.debug("Reused shared prepared hydrology tile %d,%d", key.tileX(), key.tileZ()); + return shared; + } + if (shared != null) { + SHARED_TILES.invalidate(sharedKey); + } + } + try (HydrologyPlanningAdmission.Permit ignored = HydrologyPlanningAdmission.acquireRoot( + closed::get)) { + return planAdmitted(key, planningEpoch, sharedKey, store, attempt == 0L); + } + } catch (RuntimeException failure) { + if (failure instanceof CancellationException || interrupted(failure)) { + throw failure; + } + requirePlanningActive(); + attempt++; + long now = System.nanoTime(); + if (nextFailureLog == 0L || now - nextFailureLog >= 0L) { + IrisLogging.error("Hydrology tile " + key.tileX() + "," + key.tileZ() + + " failed planning attempt " + attempt + "; retrying in " + retryDelayMillis + + " ms. Generation is waiting for its river plan: " + failure.getMessage()); + IrisLogging.reportError(failure); + nextFailureLog = now + TimeUnit.SECONDS.toNanos(30L); + } + if (sharedKey != null) { + SHARED_TILES.invalidate(sharedKey); + } + planner.clearOwnerDrafts(); + awaitPlanningRetry(retryDelayMillis); + retryDelayMillis = Math.min(5000L, retryDelayMillis * 2L); + } + } + } + + private void requirePlanningActive() { + requireOpen(); + if (Thread.currentThread().isInterrupted()) { + throw new CancellationException("Hydrology plan retry interrupted."); + } + } + + private void awaitPlanningRetry(long delayMillis) { + long deadline = System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(delayMillis); + while (true) { + requirePlanningActive(); + long remaining = deadline - System.nanoTime(); + if (remaining <= 0L) { + return; + } + try { + TimeUnit.NANOSECONDS.sleep(Math.min(remaining, TimeUnit.MILLISECONDS.toNanos(100L))); + } catch (InterruptedException interruption) { + Thread.currentThread().interrupt(); + throw new CancellationException("Hydrology plan retry interrupted."); + } } } private HydrologyTile planAdmitted(HydrologyTileKey key, long planningEpoch, - SharedTileKey sharedKey, PreparedHydrologyTileStore store) { - if (store != null) { + SharedTileKey sharedKey, PreparedHydrologyTileStore store, + boolean reusePrepared) { + if (store != null && reusePrepared) { HydrologyTile persisted = store.load(key).orElse(null); if (persisted != null) { - SHARED_TILES.put(sharedKey, persisted); planner.reuseResolvedTile(persisted); + SHARED_TILES.put(sharedKey, persisted); IrisLogging.debug("Loaded persisted prepared hydrology tile %d,%d", key.tileX(), key.tileZ()); return persisted; } } prepareTerrain(); - try { - HydrologyTile planned = planner.plan(key); - synchronized (publicationLock) { - if (sharedKey != null && !closed.get() && planningEpoch == cacheEpoch.get()) { - HydrologyTile existing = SHARED_TILES.asMap().putIfAbsent(sharedKey, planned); - if (existing == null) { - IrisLogging.debug("Published shared prepared hydrology tile %d,%d", key.tileX(), key.tileZ()); - } + HydrologyTile planned = planner.plan(key); + synchronized (publicationLock) { + if (sharedKey != null && !closed.get() && planningEpoch == cacheEpoch.get()) { + HydrologyTile existing = SHARED_TILES.asMap().putIfAbsent(sharedKey, planned); + if (existing == null) { + IrisLogging.debug("Published shared prepared hydrology tile %d,%d", key.tileX(), key.tileZ()); } } - if (store != null && !closed.get() && planningEpoch == cacheEpoch.get()) { - persistTile(store, planned); - } - return planned; - } catch (RuntimeException failure) { - if (failure instanceof CancellationException || interrupted(failure)) { - throw failure; - } - IrisLogging.error("Hydrology tile " + key.tileX() + "," + key.tileZ() - + " failed to plan and generates without rivers: " + failure.getMessage()); - IrisLogging.reportError(failure); - return planner.emptyTile(key); } + if (store != null && !closed.get() && planningEpoch == cacheEpoch.get()) { + persistTile(store, planned); + } + return planned; } private void persistTile(PreparedHydrologyTileStore store, HydrologyTile tile) { diff --git a/core/src/main/java/art/arcane/iris/generation/hydrology/HydrologyTileCodec.java b/core/src/main/java/art/arcane/iris/generation/hydrology/HydrologyTileCodec.java new file mode 100644 index 000000000..481bbf9d7 --- /dev/null +++ b/core/src/main/java/art/arcane/iris/generation/hydrology/HydrologyTileCodec.java @@ -0,0 +1,706 @@ +package art.arcane.iris.generation.hydrology; + +import art.arcane.iris.generation.hydrology.cave.CavePosition; +import art.arcane.iris.generation.hydrology.cave.HydrologyCaveMode; +import art.arcane.iris.generation.hydrology.cave.HydrologyCavePlan; +import art.arcane.iris.generation.hydrology.cave.HydrologyCaveRejection; +import art.arcane.iris.generation.hydrology.cave.HydrologyCaveSource; +import art.arcane.iris.generation.hydrology.policy.SurfaceRiverPolicy; + +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.IdentityHashMap; +import java.util.List; +import java.util.Map; +import java.util.OptionalLong; +import java.util.Set; + +final class HydrologyTileCodec { + private static final int MAGIC = 0x49524854; + private static final int FORMAT = 2; + private static final long MAXIMUM_ALLOCATION_BYTES = Math.max(16L * 1024L * 1024L, + Math.min(512L * 1024L * 1024L, Runtime.getRuntime().maxMemory() / 8L)); + private static final int MAXIMUM_COLLECTION_SIZE = 2_000_000; + private static final HydrologyFeatureType[] HYDROLOGY_FEATURE_TYPE_VALUES = HydrologyFeatureType.values(); + private static final RiverCourseType[] RIVER_COURSE_TYPE_VALUES = RiverCourseType.values(); + private static final HydrologyCandidateKind[] HYDROLOGY_CANDIDATE_KIND_VALUES = HydrologyCandidateKind.values(); + private static final HydrologyCandidateRejection[] HYDROLOGY_CANDIDATE_REJECTION_VALUES = HydrologyCandidateRejection.values(); + private static final HydrologyCaveMode[] HYDROLOGY_CAVE_MODE_VALUES = HydrologyCaveMode.values(); + private static final HydrologyCaveRejection[] HYDROLOGY_CAVE_REJECTION_VALUES = HydrologyCaveRejection.values(); + + private final Map strings = new HashMap<>(); + private long remainingAllocation = MAXIMUM_ALLOCATION_BYTES; + + void write(DataOutputStream output, HydrologyTile tile, String scopeIdentity) throws IOException { + output.writeInt(MAGIC); + output.writeInt(FORMAT); + output.writeUTF(scopeIdentity); + output.writeInt(tile.key().tileX()); + output.writeInt(tile.key().tileZ()); + output.writeLong(tile.worldSeed()); + output.writeLong(tile.settingsFingerprint()); + output.writeInt(tile.tileSize()); + writeList(output, tile.nodes(), this::writeDrainageNode); + writeList(output, tile.edges(), this::writeDrainageEdge); + writeList(output, tile.outlets(), this::writeRiverOutlet); + writeList(output, tile.courses(), this::writeRiverCourse); + writeCount(output, tile.regionalCourseIds().size(), 64); + for (long id : tile.regionalCourseIds()) { + output.writeLong(id); + } + writeList(output, tile.cavePlans(), this::writeCavePlan); + writeList(output, tile.localDiagnosticCandidates(), this::writeHydrologyDiagnosticCandidate); + writeCount(output, tile.footprint().columns().size(), 256); + for (HydrologyColumnSample column : tile.footprint().columns().values()) { + writeHydrologyColumnSample(output, column); + } + output.writeBoolean(tile.resolvedOwner() != null); + if (tile.resolvedOwner() != null) { + writeResolvedOwner(output, tile); + } + } + + HydrologyTile read(DataInputStream input, String scopeIdentity, HydrologyTileKey key, + HydrologyTileCache.SharedCacheScope scope, int expectedTileSize) throws IOException { + if (input.readInt() != MAGIC || input.readInt() != FORMAT || !scopeIdentity.equals(input.readUTF()) + || input.readInt() != key.tileX() || input.readInt() != key.tileZ() + || input.readLong() != scope.worldSeed() || input.readLong() != scope.settingsFingerprint() + || input.readInt() != expectedTileSize) { + throw new IOException("Prepared hydrology tile header does not match its scope."); + } + List nodes = readList(input, this::readDrainageNode); + List edges = readList(input, this::readDrainageEdge); + List outlets = readList(input, this::readRiverOutlet); + List courses = readList(input, this::readRiverCourse); + int regionalCount = readCount(input, 64); + Set regionalCourseIds = HashSet.newHashSet(regionalCount); + for (int index = 0; index < regionalCount; index++) { + if (!regionalCourseIds.add(input.readLong())) { + throw new IOException("Duplicate regional hydrology course."); + } + } + List cavePlans = readList(input, this::readCavePlan); + List candidates = readList(input, this::readHydrologyDiagnosticCandidate); + int columnCount = readCount(input, 256); + Map columns = HashMap.newHashMap(columnCount); + for (int index = 0; index < columnCount; index++) { + HydrologyColumnSample column = readHydrologyColumnSample(input); + if (columns.put(RiverFootprint.pack(column.x(), column.z()), column) != null) { + throw new IOException("Duplicate hydrology footprint column."); + } + } + HydrologyTile tile = new HydrologyTile(key, scope.worldSeed(), scope.settingsFingerprint(), expectedTileSize, + nodes, edges, outlets, courses, regionalCourseIds, cavePlans, candidates, + new RiverFootprint(columns)); + if (input.readBoolean()) { + tile = tile.withResolvedOwner(readResolvedOwner(input, tile)); + } + if (input.read() != -1) { + throw new IOException("Unexpected data after prepared hydrology tile."); + } + return tile; + } + + private void writeResolvedOwner(DataOutputStream output, HydrologyTile tile) throws IOException { + allocate(256L); + CrossTileResolvedOwner owner = tile.resolvedOwner(); + HydrologyCaveCourseFilter.Result result = owner.draft().result(); + writeOwnerList(output, result.nodes(), tile.nodes(), this::writeDrainageNode); + writeOwnerList(output, result.edges(), tile.edges(), this::writeDrainageEdge); + writeOwnerList(output, result.outlets(), tile.outlets(), this::writeRiverOutlet); + writeOwnerList(output, result.courses(), tile.courses(), this::writeRiverCourse); + writeOwnerList(output, result.cavePlans(), tile.cavePlans(), this::writeCavePlan); + writeOwnerList(output, owner.draft().diagnostics(), tile.localDiagnosticCandidates(), + this::writeHydrologyDiagnosticCandidate); + writeList(output, owner.observedRejections(), this::writeRejectedCourse); + } + + private CrossTileResolvedOwner readResolvedOwner(DataInputStream input, HydrologyTile tile) throws IOException { + allocate(256L); + HydrologyCaveCourseFilter.Result result = new HydrologyCaveCourseFilter.Result( + readOwnerList(input, tile.nodes(), this::readDrainageNode), + readOwnerList(input, tile.edges(), this::readDrainageEdge), + readOwnerList(input, tile.outlets(), this::readRiverOutlet), + readOwnerList(input, tile.courses(), this::readRiverCourse), + readOwnerList(input, tile.cavePlans(), this::readCavePlan)); + List diagnostics = readOwnerList(input, tile.localDiagnosticCandidates(), + this::readHydrologyDiagnosticCandidate); + return new CrossTileResolvedOwner(new HydrologyOwnerDraft(tile.key(), result, diagnostics, null), + readList(input, this::readRejectedCourse)); + } + + private void writeRejectedCourse(DataOutputStream output, CrossTileRejectedCourse rejection) throws IOException { + writeRiverCourse(output, rejection.course()); + output.writeLong(rejection.winnerSourceId()); + } + + private CrossTileRejectedCourse readRejectedCourse(DataInputStream input) throws IOException { + return new CrossTileRejectedCourse(readRiverCourse(input), input.readLong()); + } + + private void writeOwnerList(DataOutputStream output, List values, List published, + ElementWriter writer) throws IOException { + writeCount(output, values.size(), 16); + Map indexes = new IdentityHashMap<>(published.size()); + for (int index = 0; index < published.size(); index++) { + indexes.put(published.get(index), index); + } + for (T value : values) { + Integer index = indexes.get(value); + output.writeInt(index == null ? -1 : index); + if (index == null) { + allocate(256L); + writer.write(output, value); + } + } + } + + private List readOwnerList(DataInputStream input, List published, ElementReader reader) throws IOException { + int count = readCount(input, 16); + List values = new ArrayList<>(Math.min(count, 4096)); + for (int index = 0; index < count; index++) { + int reference = input.readInt(); + if (reference == -1) { + allocate(256L); + values.add(reader.read(input)); + } else if (reference >= 0 && reference < published.size()) { + values.add(published.get(reference)); + } else { + throw new IOException("Invalid prepared hydrology owner reference."); + } + } + return List.copyOf(values); + } + + private void writeCavePlan(DataOutputStream output, HydrologyCavePlan plan) throws IOException { + writeHydrologyCaveSource(output, plan.source()); + output.writeByte(plan.rejection().ordinal()); + writeOptionalLong(output, plan.arbitrationWinnerSourceId()); + validateCount(plan.baselinePreconditions().size(), 96); + plan.writeCells(output); + } + + private HydrologyCavePlan readCavePlan(DataInputStream input) throws IOException { + HydrologyCaveSource source = readHydrologyCaveSource(input); + HydrologyCaveRejection rejection = readEnum(input, HYDROLOGY_CAVE_REJECTION_VALUES); + OptionalLong winner = readOptionalLong(input); + int cells = readCount(input, 96); + return HydrologyCavePlan.readCells(source, rejection, winner, input, cells); + } + + private void writeHydraulicChannelProfile(DataOutputStream output, HydraulicChannelProfile profile) throws IOException { + writeCount(output, profile.size(), 32); + for (int station = 0; station < profile.size(); station++) { + output.writeDouble(profile.widthAt(station)); + output.writeDouble(profile.depthAt(station)); + } + } + + private HydraulicChannelProfile readHydraulicChannelProfile(DataInputStream input) throws IOException { + int count = readCount(input, 32); + double[] widths = new double[count]; + double[] depths = new double[count]; + for (int station = 0; station < count; station++) { + widths[station] = input.readDouble(); + depths[station] = input.readDouble(); + } + return new HydraulicChannelProfile(widths, depths); + } + + private void writeDrainageNode(DataOutputStream output, DrainageNode value) throws IOException { + output.writeLong(value.id()); + output.writeInt(value.x()); + output.writeInt(value.z()); + writeHydrologyTerrainSample(output, value.terrain()); + output.writeDouble(value.potential()); + output.writeLong(value.outletId()); + } + + private DrainageNode readDrainageNode(DataInputStream input) throws IOException { + return new DrainageNode( + input.readLong(), + input.readInt(), + input.readInt(), + readHydrologyTerrainSample(input), + input.readDouble(), + input.readLong() + ); + } + + private void writeDrainageEdge(DataOutputStream output, DrainageEdge value) throws IOException { + output.writeLong(value.id()); + output.writeLong(value.upstreamNodeId()); + output.writeLong(value.downstreamNodeId()); + output.writeLong(value.outletId()); + output.writeDouble(value.cost()); + output.writeInt(value.contributingSurfaceSources()); + output.writeInt(value.contributingUndergroundSources()); + writeList(output, value.centerline(), this::writeHydrologyPoint); + } + + private DrainageEdge readDrainageEdge(DataInputStream input) throws IOException { + return new DrainageEdge( + input.readLong(), + input.readLong(), + input.readLong(), + input.readLong(), + input.readDouble(), + input.readInt(), + input.readInt(), + readList(input, this::readHydrologyPoint) + ); + } + + private void writeHydrologyTerrainSample(DataOutputStream output, HydrologyTerrainSample value) throws IOException { + output.writeInt(value.naturalHeight()); + output.writeDouble(value.slope()); + output.writeBoolean(value.ocean()); + output.writeBoolean(value.caveAvailable()); + output.writeInt(value.caveFloorY()); + output.writeInt(value.caveFluidY()); + output.writeBoolean(value.transitAllowed()); + output.writeBoolean(value.outletAllowed()); + output.writeBoolean(value.surfaceSourceAllowed()); + output.writeBoolean(value.surfaceSourceRequired()); + output.writeBoolean(value.undergroundSourceAllowed()); + output.writeBoolean(value.undergroundSourceRequired()); + output.writeDouble(value.routingCost()); + output.writeDouble(value.surfaceSourceWeight()); + output.writeDouble(value.undergroundSourceWeight()); + output.writeDouble(value.widthMultiplier()); + output.writeDouble(value.depthMultiplier()); + output.writeDouble(value.incisionMultiplier()); + output.writeDouble(value.routingMultiplier()); + output.writeDouble(value.bankMultiplier()); + writeString(output, value.parentBiomeKey()); + writeString(output, value.surfaceBiomeKey()); + writeString(output, value.mouthBiomeKey()); + writeString(output, value.shoreBiomeKey()); + writeString(output, value.bankBiomeKey()); + writeString(output, value.floodedCaveBiomeKey()); + writeList(output, value.preferredProfileKeys(), this::writeString); + writeList(output, value.surfacePoolKeys(), this::writeString); + output.writeDouble(value.shoreBiomeWidth()); + writeString(output, value.confinesKey()); + output.writeDouble(value.shoreWidth()); + output.writeBoolean(value.erosion()); + writeSurfaceRiverPolicy(output, value.surfacePolicy()); + } + + private HydrologyTerrainSample readHydrologyTerrainSample(DataInputStream input) throws IOException { + return new HydrologyTerrainSample( + input.readInt(), + input.readDouble(), + input.readBoolean(), + input.readBoolean(), + input.readInt(), + input.readInt(), + input.readBoolean(), + input.readBoolean(), + input.readBoolean(), + input.readBoolean(), + input.readBoolean(), + input.readBoolean(), + input.readDouble(), + input.readDouble(), + input.readDouble(), + input.readDouble(), + input.readDouble(), + input.readDouble(), + input.readDouble(), + input.readDouble(), + readString(input), + readString(input), + readString(input), + readString(input), + readString(input), + readString(input), + readList(input, this::readString), + readList(input, this::readString), + input.readDouble(), + readString(input), + input.readDouble(), + input.readBoolean(), + readSurfaceRiverPolicy(input) + ); + } + + private void writeRiverOutlet(DataOutputStream output, RiverOutlet value) throws IOException { + output.writeLong(value.id()); + output.writeByte(value.type().ordinal()); + output.writeLong(value.drainageNodeId()); + writeHydrologyPoint(output, value.landwardPoint()); + writeHydrologyPoint(output, value.connectionPoint()); + output.writeInt(value.seaLevel()); + output.writeBoolean(value.directOcean()); + } + + private RiverOutlet readRiverOutlet(DataInputStream input) throws IOException { + return new RiverOutlet( + input.readLong(), + readEnum(input, HYDROLOGY_FEATURE_TYPE_VALUES), + input.readLong(), + readHydrologyPoint(input), + readHydrologyPoint(input), + input.readInt(), + input.readBoolean() + ); + } + + private void writeRiverCourse(DataOutputStream output, RiverCourse value) throws IOException { + output.writeLong(value.id()); + output.writeByte(value.type().ordinal()); + writeOptionalLong(output, value.sourceNodeId()); + writeOptionalLong(output, value.outletId()); + writeString(output, value.profileKey()); + output.writeInt(value.discharge()); + writeList(output, value.drainageEdges(), this::writeDrainageEdge); + writeList(output, value.segments(), this::writeHydraulicSegment); + } + + private RiverCourse readRiverCourse(DataInputStream input) throws IOException { + return new RiverCourse( + input.readLong(), + readEnum(input, RIVER_COURSE_TYPE_VALUES), + readOptionalLong(input), + readOptionalLong(input), + readString(input), + input.readInt(), + readList(input, this::readDrainageEdge), + readList(input, this::readHydraulicSegment) + ); + } + + private void writeHydraulicSegment(DataOutputStream output, HydraulicSegment value) throws IOException { + output.writeLong(value.id()); + output.writeLong(value.courseId()); + output.writeByte(value.type().ordinal()); + output.writeInt(value.upstreamHeadY()); + output.writeInt(value.downstreamHeadY()); + output.writeInt(value.width()); + output.writeInt(value.depth()); + output.writeBoolean(value.fallingFluid()); + output.writeBoolean(value.receivingPool()); + writeList(output, value.centerline(), this::writeHydrologyPoint); + writeHydraulicChannelProfile(output, value.channelProfile()); + } + + private HydraulicSegment readHydraulicSegment(DataInputStream input) throws IOException { + return new HydraulicSegment( + input.readLong(), + input.readLong(), + readEnum(input, HYDROLOGY_FEATURE_TYPE_VALUES), + input.readInt(), + input.readInt(), + input.readInt(), + input.readInt(), + input.readBoolean(), + input.readBoolean(), + readList(input, this::readHydrologyPoint), + readHydraulicChannelProfile(input) + ); + } + + private void writeHydrologyDiagnosticCandidate(DataOutputStream output, HydrologyDiagnosticCandidate value) throws IOException { + output.writeLong(value.id()); + output.writeByte(value.kind().ordinal()); + output.writeByte(value.projectedType().ordinal()); + writeHydrologyPoint(output, value.point()); + output.writeByte(value.rejection().ordinal()); + output.writeInt(value.detail()); + } + + private HydrologyDiagnosticCandidate readHydrologyDiagnosticCandidate(DataInputStream input) throws IOException { + return new HydrologyDiagnosticCandidate( + input.readLong(), + readEnum(input, HYDROLOGY_CANDIDATE_KIND_VALUES), + readEnum(input, HYDROLOGY_FEATURE_TYPE_VALUES), + readHydrologyPoint(input), + readEnum(input, HYDROLOGY_CANDIDATE_REJECTION_VALUES), + input.readInt() + ); + } + + private void writeHydrologyColumnSample(DataOutputStream output, HydrologyColumnSample value) throws IOException { + output.writeInt(value.x()); + output.writeInt(value.z()); + output.writeInt(value.naturalHeight()); + output.writeInt(value.seaLevel()); + output.writeBoolean(value.ocean()); + writeString(output, value.parentBiomeKey()); + writeList(output, value.layers(), this::writeHydrologyColumnLayer); + } + + private HydrologyColumnSample readHydrologyColumnSample(DataInputStream input) throws IOException { + return new HydrologyColumnSample( + input.readInt(), + input.readInt(), + input.readInt(), + input.readInt(), + input.readBoolean(), + readString(input), + readList(input, this::readHydrologyColumnLayer) + ); + } + + private void writeHydrologyColumnLayer(DataOutputStream output, HydrologyColumnLayer value) throws IOException { + writeHydrologyFeatureRef(output, value.feature()); + output.writeInt(value.bedY()); + output.writeInt(value.fluidHeadY()); + output.writeInt(value.ceilingY()); + output.writeBoolean(value.channel()); + output.writeBoolean(value.shore()); + output.writeBoolean(value.grading()); + output.writeBoolean(value.connectedFluid()); + output.writeBoolean(value.fallingFluid()); + output.writeBoolean(value.receivingPool()); + output.writeBoolean(value.terrainOwned()); + output.writeBoolean(value.fluidOwned()); + output.writeBoolean(value.oceanApron()); + writeString(output, value.profileKey()); + writeString(output, value.surfaceBiomeKey()); + writeString(output, value.mouthBiomeKey()); + writeString(output, value.shoreBiomeKey()); + writeString(output, value.bankBiomeKey()); + writeString(output, value.floodedCaveBiomeKey()); + } + + private HydrologyColumnLayer readHydrologyColumnLayer(DataInputStream input) throws IOException { + return new HydrologyColumnLayer( + readHydrologyFeatureRef(input), + input.readInt(), + input.readInt(), + input.readInt(), + input.readBoolean(), + input.readBoolean(), + input.readBoolean(), + input.readBoolean(), + input.readBoolean(), + input.readBoolean(), + input.readBoolean(), + input.readBoolean(), + input.readBoolean(), + readString(input), + readString(input), + readString(input), + readString(input), + readString(input), + readString(input) + ); + } + + private void writeHydrologyFeatureRef(DataOutputStream output, HydrologyFeatureRef value) throws IOException { + output.writeLong(value.id()); + output.writeByte(value.type().ordinal()); + output.writeLong(value.courseId()); + output.writeLong(value.segmentId()); + output.writeInt(value.x()); + output.writeInt(value.y()); + output.writeInt(value.z()); + output.writeInt(value.flowDeltaX()); + output.writeInt(value.flowDeltaZ()); + output.writeBoolean(value.source()); + } + + private HydrologyFeatureRef readHydrologyFeatureRef(DataInputStream input) throws IOException { + return new HydrologyFeatureRef( + input.readLong(), + readEnum(input, HYDROLOGY_FEATURE_TYPE_VALUES), + input.readLong(), + input.readLong(), + input.readInt(), + input.readInt(), + input.readInt(), + input.readInt(), + input.readInt(), + input.readBoolean() + ); + } + + private void writeHydrologyPoint(DataOutputStream output, HydrologyPoint value) throws IOException { + output.writeInt(value.x()); + output.writeInt(value.y()); + output.writeInt(value.z()); + } + + private HydrologyPoint readHydrologyPoint(DataInputStream input) throws IOException { + return new HydrologyPoint( + input.readInt(), + input.readInt(), + input.readInt() + ); + } + + private void writeHydrologyCaveSource(DataOutputStream output, HydrologyCaveSource value) throws IOException { + output.writeLong(value.sourceId()); + writeCavePosition(output, value.entry()); + writeCavePosition(output, value.target()); + output.writeInt(value.waterHeadY()); + output.writeByte(value.mode().ordinal()); + } + + private HydrologyCaveSource readHydrologyCaveSource(DataInputStream input) throws IOException { + return new HydrologyCaveSource( + input.readLong(), + readCavePosition(input), + readCavePosition(input), + input.readInt(), + readEnum(input, HYDROLOGY_CAVE_MODE_VALUES) + ); + } + + private void writeCavePosition(DataOutputStream output, CavePosition value) throws IOException { + output.writeInt(value.x()); + output.writeInt(value.y()); + output.writeInt(value.z()); + } + + private CavePosition readCavePosition(DataInputStream input) throws IOException { + return new CavePosition( + input.readInt(), + input.readInt(), + input.readInt() + ); + } + + private void writeSurfaceRiverPolicy(DataOutputStream output, SurfaceRiverPolicy value) throws IOException { + writeString(output, value.areaKey()); + writeDouble(output, value.sourceDensity()); + writeInteger(output, value.sourceSpacing()); + writeInteger(output, value.tributaries()); + writeInteger(output, value.inlandOutlets()); + writeInteger(output, value.coastalOutlets()); + writeInteger(output, value.minimumCourseLength()); + writeInteger(output, value.maximumIncision()); + } + + private SurfaceRiverPolicy readSurfaceRiverPolicy(DataInputStream input) throws IOException { + return new SurfaceRiverPolicy( + readString(input), + readDouble(input), + readInteger(input), + readInteger(input), + readInteger(input), + readInteger(input), + readInteger(input), + readInteger(input) + ); + } + + private void writeString(DataOutputStream output, String value) throws IOException { + output.writeBoolean(value != null); + if (value != null) { + allocate(40L + (long) value.length() * Character.BYTES); + output.writeUTF(value); + } + } + + private String readString(DataInputStream input) throws IOException { + if (!input.readBoolean()) { + return null; + } + String value = input.readUTF(); + allocate(40L + (long) value.length() * Character.BYTES); + if (strings.size() >= 4096) { + return value; + } + String existing = strings.putIfAbsent(value, value); + return existing == null ? value : existing; + } + + private void writeInteger(DataOutputStream output, Integer value) throws IOException { + output.writeBoolean(value != null); + if (value != null) { + output.writeInt(value); + } + } + + private Integer readInteger(DataInputStream input) throws IOException { + return input.readBoolean() ? input.readInt() : null; + } + + private void writeDouble(DataOutputStream output, Double value) throws IOException { + output.writeBoolean(value != null); + if (value != null) { + output.writeDouble(value); + } + } + + private Double readDouble(DataInputStream input) throws IOException { + return input.readBoolean() ? input.readDouble() : null; + } + + private void writeOptionalLong(DataOutputStream output, OptionalLong value) throws IOException { + output.writeBoolean(value.isPresent()); + if (value.isPresent()) { + output.writeLong(value.getAsLong()); + } + } + + private OptionalLong readOptionalLong(DataInputStream input) throws IOException { + return input.readBoolean() ? OptionalLong.of(input.readLong()) : OptionalLong.empty(); + } + + private void writeList(DataOutputStream output, List values, ElementWriter writer) throws IOException { + writeCount(output, values.size(), 256); + for (T value : values) { + writer.write(output, value); + } + } + + private List readList(DataInputStream input, ElementReader reader) throws IOException { + int count = readCount(input, 256); + List values = new ArrayList<>(Math.min(count, 4096)); + for (int index = 0; index < count; index++) { + values.add(reader.read(input)); + } + return values; + } + + private int readCount(DataInputStream input, int bytesPerElement) throws IOException { + int count = input.readInt(); + validateCount(count, bytesPerElement); + return count; + } + + private void writeCount(DataOutputStream output, int count, int bytesPerElement) throws IOException { + validateCount(count, bytesPerElement); + output.writeInt(count); + } + + private void validateCount(int count, int bytesPerElement) throws IOException { + if (count < 0 || count > MAXIMUM_COLLECTION_SIZE) { + throw new IOException("Invalid hydrology collection length: " + count); + } + allocate((long) count * bytesPerElement); + } + + private void allocate(long bytes) throws IOException { + if (bytes > remainingAllocation) { + throw new IOException("Prepared hydrology tile exceeds its allocation budget."); + } + remainingAllocation -= bytes; + } + + private > T readEnum(DataInputStream input, T[] values) throws IOException { + int ordinal = input.readUnsignedByte(); + if (ordinal >= values.length) { + throw new IOException("Invalid hydrology enum value: " + ordinal); + } + return values[ordinal]; + } + + @FunctionalInterface + private interface ElementWriter { + void write(DataOutputStream output, T value) throws IOException; + } + + @FunctionalInterface + private interface ElementReader { + T read(DataInputStream input) throws IOException; + } +} diff --git a/core/src/main/java/art/arcane/iris/generation/hydrology/PreparedHydrologyTileStore.java b/core/src/main/java/art/arcane/iris/generation/hydrology/PreparedHydrologyTileStore.java index 960724b76..cf064199a 100644 --- a/core/src/main/java/art/arcane/iris/generation/hydrology/PreparedHydrologyTileStore.java +++ b/core/src/main/java/art/arcane/iris/generation/hydrology/PreparedHydrologyTileStore.java @@ -1,24 +1,13 @@ package art.arcane.iris.generation.hydrology; -import art.arcane.iris.generation.hydrology.cave.CavePosition; -import art.arcane.iris.generation.hydrology.cave.CaveVoxelPrecondition; -import art.arcane.iris.generation.hydrology.cave.HydrologyCaveAction; -import art.arcane.iris.generation.hydrology.cave.HydrologyCavePlan; -import art.arcane.iris.generation.hydrology.cave.HydrologyCaveRejection; -import art.arcane.iris.generation.hydrology.cave.HydrologyCaveSource; -import com.google.gson.Gson; -import com.google.gson.GsonBuilder; - import java.io.BufferedInputStream; import java.io.BufferedOutputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; import java.io.FilterInputStream; import java.io.IOException; import java.io.InputStream; -import java.io.InputStreamReader; import java.io.OutputStream; -import java.io.OutputStreamWriter; -import java.io.Reader; -import java.io.Writer; import java.nio.charset.StandardCharsets; import java.nio.file.AtomicMoveNotSupportedException; import java.nio.file.Files; @@ -27,29 +16,15 @@ import java.nio.file.Path; import java.nio.file.StandardCopyOption; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; -import java.util.ArrayList; -import java.util.Comparator; -import java.util.HashMap; import java.util.HexFormat; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; import java.util.Objects; import java.util.Optional; -import java.util.OptionalLong; -import java.util.Set; import java.util.zip.GZIPInputStream; import java.util.zip.GZIPOutputStream; final class PreparedHydrologyTileStore { - private static final int SCHEMA_VERSION = 6; private static final long MAXIMUM_COMPRESSED_BYTES = 128L * 1024L * 1024L; private static final long MAXIMUM_DECOMPRESSED_BYTES = 512L * 1024L * 1024L; - private static final Gson GSON = new GsonBuilder() - .disableHtmlEscaping() - .serializeSpecialFloatingPointValues() - .create(); - private final Path directory; private final String scopeIdentity; private final HydrologyTileCache.SharedCacheScope scope; @@ -68,32 +43,27 @@ final class PreparedHydrologyTileStore { .resolve(scopeIdentity); } - Optional load(HydrologyTileKey key) { + boolean contains(HydrologyTileKey key) { Path file = file(key); try { - if (!Files.isRegularFile(file, LinkOption.NOFOLLOW_LINKS) - || Files.isSymbolicLink(file) - || Files.size(file) > MAXIMUM_COMPRESSED_BYTES) { - return Optional.empty(); - } + return Files.isRegularFile(file, LinkOption.NOFOLLOW_LINKS) + && !Files.isSymbolicLink(file) + && Files.size(file) <= MAXIMUM_COMPRESSED_BYTES; } catch (IOException failure) { + return false; + } + } + + Optional load(HydrologyTileKey key) { + if (!contains(key)) { return Optional.empty(); } - try (InputStream raw = Files.newInputStream(file); + try (InputStream raw = Files.newInputStream(file(key)); InputStream buffered = new BufferedInputStream(raw); - InputStream compressed = new GZIPInputStream(buffered); + InputStream compressed = new GZIPInputStream(buffered, 32768); InputStream bounded = new LimitedInputStream(compressed, MAXIMUM_DECOMPRESSED_BYTES); - Reader reader = new InputStreamReader(bounded, StandardCharsets.UTF_8)) { - PersistedTile persisted = GSON.fromJson(reader, PersistedTile.class); - if (persisted == null - || persisted.schemaVersion() != SCHEMA_VERSION - || !scopeIdentity.equals(persisted.scopeIdentity()) - || !key.equals(persisted.key()) - || persisted.worldSeed() != scope.worldSeed() - || persisted.settingsFingerprint() != scope.settingsFingerprint()) { - return Optional.empty(); - } - HydrologyTile tile = persisted.toTile(); + DataInputStream input = new DataInputStream(new BufferedInputStream(bounded, 32768))) { + HydrologyTile tile = new HydrologyTileCodec().read(input, scopeIdentity, key, scope, expectedTileSize); return valid(tile, key) ? Optional.of(tile) : Optional.empty(); } catch (IOException | RuntimeException failure) { return Optional.empty(); @@ -110,9 +80,9 @@ final class PreparedHydrologyTileStore { try { try (OutputStream raw = Files.newOutputStream(staged); OutputStream buffered = new BufferedOutputStream(raw); - OutputStream compressed = new GZIPOutputStream(buffered); - Writer writer = new OutputStreamWriter(compressed, StandardCharsets.UTF_8)) { - GSON.toJson(PersistedTile.from(tile, scopeIdentity), writer); + OutputStream compressed = new GZIPOutputStream(buffered, 32768); + DataOutputStream output = new DataOutputStream(new BufferedOutputStream(compressed, 32768))) { + new HydrologyTileCodec().write(output, tile, scopeIdentity); } if (Files.size(staged) > MAXIMUM_COMPRESSED_BYTES) { throw new IOException("Prepared hydrology tile cache entry exceeds the size limit."); @@ -128,7 +98,7 @@ final class PreparedHydrologyTileStore { } Path file(HydrologyTileKey key) { - return directory.resolve("tile-" + key.tileX() + "-" + key.tileZ() + ".json.gz"); + return directory.resolve("tile-" + key.tileX() + "-" + key.tileZ() + ".bin.gz"); } private boolean valid(HydrologyTile tile, HydrologyTileKey key) { @@ -162,194 +132,6 @@ final class PreparedHydrologyTileStore { digest.update(bytes); } - private record PersistedTile( - int schemaVersion, - String scopeIdentity, - HydrologyTileKey key, - long worldSeed, - long settingsFingerprint, - int tileSize, - List nodes, - List edges, - List outlets, - List courses, - List regionalCourseIds, - List cavePlans, - List localDiagnosticCandidates, - List columns - ) { - private static PersistedTile from(HydrologyTile tile, String scopeIdentity) { - ArrayList courses = new ArrayList<>(tile.courses().size()); - for (RiverCourse course : tile.courses()) { - courses.add(PersistedCourse.from(course)); - } - ArrayList regionalCourseIds = new ArrayList<>(tile.regionalCourseIds()); - regionalCourseIds.sort(Long::compareTo); - ArrayList cavePlans = new ArrayList<>(tile.cavePlans().size()); - for (HydrologyCavePlan cavePlan : tile.cavePlans()) { - cavePlans.add(PersistedCavePlan.from(cavePlan)); - } - return new PersistedTile( - SCHEMA_VERSION, - scopeIdentity, - tile.key(), - tile.worldSeed(), - tile.settingsFingerprint(), - tile.tileSize(), - tile.nodes(), - tile.edges(), - tile.outlets(), - courses, - regionalCourseIds, - cavePlans, - tile.localDiagnosticCandidates(), - new ArrayList<>(tile.footprint().columns().values()) - ); - } - - private HydrologyTile toTile() { - requireCollection(nodes, "nodes"); - requireCollection(edges, "edges"); - requireCollection(outlets, "outlets"); - requireCollection(courses, "courses"); - requireCollection(regionalCourseIds, "regionalCourseIds"); - requireCollection(cavePlans, "cavePlans"); - requireCollection(localDiagnosticCandidates, "localDiagnosticCandidates"); - requireCollection(columns, "columns"); - ArrayList restoredCourses = new ArrayList<>(courses.size()); - for (PersistedCourse course : courses) { - restoredCourses.add(Objects.requireNonNull(course, "course").toCourse()); - } - ArrayList restoredCavePlans = new ArrayList<>(cavePlans.size()); - for (PersistedCavePlan cavePlan : cavePlans) { - restoredCavePlans.add(Objects.requireNonNull(cavePlan, "cavePlan").toPlan()); - } - HashMap restoredColumns = HashMap.newHashMap(columns.size()); - for (HydrologyColumnSample column : columns) { - HydrologyColumnSample required = Objects.requireNonNull(column, "column"); - HydrologyColumnSample existing = restoredColumns.put( - RiverFootprint.pack(required.x(), required.z()), required); - if (existing != null) { - throw new IllegalArgumentException("Duplicate hydrology footprint column."); - } - } - return new HydrologyTile( - key, - worldSeed, - settingsFingerprint, - tileSize, - nodes, - edges, - outlets, - restoredCourses, - Set.copyOf(regionalCourseIds), - restoredCavePlans, - localDiagnosticCandidates, - new RiverFootprint(restoredColumns) - ); - } - - private static void requireCollection(List values, String name) { - Objects.requireNonNull(values, name); - } - } - - private record PersistedCourse( - long id, - RiverCourseType type, - Long sourceNodeId, - Long outletId, - String profileKey, - int discharge, - List drainageEdges, - List segments - ) { - private static PersistedCourse from(RiverCourse course) { - return new PersistedCourse( - course.id(), - course.type(), - course.sourceNodeId().isPresent() ? course.sourceNodeId().getAsLong() : null, - course.outletId().isPresent() ? course.outletId().getAsLong() : null, - course.profileKey(), - course.discharge(), - course.drainageEdges(), - course.segments() - ); - } - - private RiverCourse toCourse() { - return new RiverCourse( - id, - type, - sourceNodeId == null ? OptionalLong.empty() : OptionalLong.of(sourceNodeId), - outletId == null ? OptionalLong.empty() : OptionalLong.of(outletId), - profileKey, - discharge, - drainageEdges, - segments - ); - } - } - - private record PersistedCavePlan( - HydrologyCaveSource source, - HydrologyCaveRejection rejection, - List cells, - Long arbitrationWinnerSourceId - ) { - private static PersistedCavePlan from(HydrologyCavePlan plan) { - ArrayList cells = new ArrayList<>(plan.baselinePreconditions().size()); - plan.forEachPrecondition((CavePosition position, CaveVoxelPrecondition precondition) -> cells.add( - new PersistedCaveCell(position, plan.actions().get(position), precondition))); - cells.sort(Comparator.comparingInt((PersistedCaveCell cell) -> cell.position().x()) - .thenComparingInt(cell -> cell.position().z()) - .thenComparingInt(cell -> cell.position().y())); - return new PersistedCavePlan( - plan.source(), - plan.rejection(), - cells, - plan.arbitrationWinnerSourceId().isPresent() - ? plan.arbitrationWinnerSourceId().getAsLong() - : null - ); - } - - private HydrologyCavePlan toPlan() { - LinkedHashMap actions = new LinkedHashMap<>(); - LinkedHashMap preconditions = new LinkedHashMap<>(); - for (PersistedCaveCell cell : Objects.requireNonNull(cells, "cells")) { - PersistedCaveCell required = Objects.requireNonNull(cell, "cell"); - CaveVoxelPrecondition existing = preconditions.put(required.position(), required.precondition()); - if (existing != null) { - throw new IllegalArgumentException("Duplicate hydrology cave cell."); - } - if (required.action() != null) { - actions.put(required.position(), required.action()); - } - } - return new HydrologyCavePlan( - source, - rejection, - actions, - preconditions, - arbitrationWinnerSourceId == null - ? OptionalLong.empty() - : OptionalLong.of(arbitrationWinnerSourceId) - ); - } - } - - private record PersistedCaveCell( - CavePosition position, - HydrologyCaveAction action, - CaveVoxelPrecondition precondition - ) { - private PersistedCaveCell { - Objects.requireNonNull(position, "position"); - Objects.requireNonNull(precondition, "precondition"); - } - } - private static final class LimitedInputStream extends FilterInputStream { private final long limit; private long count; diff --git a/core/src/main/java/art/arcane/iris/generation/hydrology/cave/HydrologyCavePlan.java b/core/src/main/java/art/arcane/iris/generation/hydrology/cave/HydrologyCavePlan.java index 56e3675b2..d1ecb57e7 100644 --- a/core/src/main/java/art/arcane/iris/generation/hydrology/cave/HydrologyCavePlan.java +++ b/core/src/main/java/art/arcane/iris/generation/hydrology/cave/HydrologyCavePlan.java @@ -5,6 +5,9 @@ import it.unimi.dsi.fastutil.HashCommon; import it.unimi.dsi.fastutil.longs.Long2ObjectMap; import it.unimi.dsi.fastutil.longs.Long2ObjectOpenHashMap; +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; import java.util.AbstractCollection; import java.util.AbstractList; import java.util.AbstractMap; @@ -49,7 +52,11 @@ public record HydrologyCavePlan( && (!requestedActions.isEmpty() || !requestedPreconditions.isEmpty())) { throw new IllegalArgumentException("Rejected cave plans cannot contain mutations or preconditions"); } - if (!requestedPreconditions.keySet().containsAll(requestedActions.keySet())) { + boolean packedInput = requestedActions instanceof ActionMap actionMap + && requestedPreconditions instanceof PreconditionMap preconditionMap + && actionMap.packed == preconditionMap.packed + && actionMap.spatialIndex == preconditionMap.spatialIndex; + if (!packedInput && !requestedPreconditions.keySet().containsAll(requestedActions.keySet())) { throw new IllegalArgumentException("Every cave action requires a baseline precondition"); } if (rejection != HydrologyCaveRejection.OVERLAPPING_SOURCE && arbitrationWinnerSourceId.isPresent()) { @@ -58,7 +65,7 @@ public record HydrologyCavePlan( if (requestedPreconditions.isEmpty()) { actions = Map.of(); baselinePreconditions = Map.of(); - } else { + } else if (!packedInput) { PackedPositions packed = new PackedPositions(requestedActions, requestedPreconditions); PositionSpatialIndex spatialIndex = new PositionSpatialIndex(packed); actions = new ActionMap(packed, requestedActions.size(), spatialIndex); @@ -66,6 +73,53 @@ public record HydrologyCavePlan( } } + public static HydrologyCavePlan readCells(HydrologyCaveSource source, HydrologyCaveRejection rejection, + OptionalLong winner, DataInput input, int count) throws IOException { + if (count < 0 || count > 2_000_000) { + throw new IOException("Invalid hydrology cave cell count: " + count); + } + int actionCount = input.readInt(); + if (actionCount < 0 || actionCount > count) { + throw new IOException("Invalid hydrology cave action count: " + actionCount); + } + if (count == 0) { + return new HydrologyCavePlan(source, rejection, Map.of(), Map.of(), winner); + } + PackedPositions packed = new PackedPositions(input, count, actionCount); + int[] order = new int[count]; + boolean[] seen = new boolean[count]; + for (int index = 0; index < count; index++) { + int row = input.readInt(); + if (row < 0 || row >= count || seen[row]) { + throw new IOException("Invalid hydrology cave precondition order."); + } + seen[row] = true; + order[index] = row; + } + PositionSpatialIndex spatialIndex = new PositionSpatialIndex(packed); + return new HydrologyCavePlan(source, rejection, + new ActionMap(packed, actionCount, spatialIndex), + new PreconditionMap(packed, new PositionOrder(packed, order), spatialIndex), winner); + } + + public void writeCells(DataOutput output) throws IOException { + output.writeInt(baselinePreconditions.size()); + output.writeInt(actions.size()); + if (!(baselinePreconditions instanceof PreconditionMap preconditions)) { + return; + } + PackedPositions packed = preconditions.packed; + for (int row = 0; row < packed.size(); row++) { + output.writeInt(packed.x[row]); + output.writeInt(packed.y[row]); + output.writeInt(packed.z[row]); + output.writeByte(packed.flags[row]); + } + for (int row : preconditions.order.rows) { + output.writeInt(row); + } + } + public boolean accepted() { return rejection == HydrologyCaveRejection.NONE; } @@ -204,6 +258,32 @@ public record HydrologyCavePlan( private final int mask; private int count; + private PackedPositions(DataInput input, int size, int actionCount) throws IOException { + x = new int[size]; + y = new int[size]; + z = new int[size]; + flags = new byte[size]; + int required = (int) Math.max(2L, ((long) size * 4L + 2L) / 3L); + int capacity = Integer.highestOneBit(required - 1) << 1; + slots = new int[capacity]; + mask = capacity - 1; + for (int row = 0; row < size; row++) { + CavePosition position = new CavePosition(input.readInt(), input.readInt(), input.readInt()); + byte value = input.readByte(); + int unsigned = Byte.toUnsignedInt(value); + if ((unsigned & 0x80) != 0 || (unsigned & VOXEL_MASK) >>> VOXEL_SHIFT >= VOXELS.length + || hasAction(value) != (row < actionCount) + || (hasAction(value) && (unsigned & ACTION_MASK) >= ACTIONS.length) + || (!hasAction(value) && (unsigned & ACTION_MASK) != 0)) { + throw new IOException("Invalid hydrology cave cell flags."); + } + if (row(position) >= 0) { + throw new IOException("Duplicate hydrology cave cell."); + } + append(position, value); + } + } + private PackedPositions(Map actions, Map preconditions) { int size = preconditions.size(); diff --git a/core/src/main/java/art/arcane/iris/generation/mantle/MatterGenerator.java b/core/src/main/java/art/arcane/iris/generation/mantle/MatterGenerator.java index e68aeba59..4595f4afc 100644 --- a/core/src/main/java/art/arcane/iris/generation/mantle/MatterGenerator.java +++ b/core/src/main/java/art/arcane/iris/generation/mantle/MatterGenerator.java @@ -577,6 +577,9 @@ public interface MatterGenerator { private volatile Future submission; private volatile Throwable failure; private boolean started; + private boolean slowWarningIssued; + private Thread runningThread; + private long startedNanos; MatterComponentTask(MatterTaskKey key) { this.key = key; @@ -587,6 +590,8 @@ public interface MatterGenerator { return false; } started = true; + runningThread = Thread.currentThread(); + startedNanos = System.nanoTime(); return true; } @@ -612,6 +617,7 @@ public interface MatterGenerator { } synchronized void finish(Throwable cause) { + runningThread = null; if (failure == null) { failure = cause; } else if (cause != null && cause != failure) { @@ -644,13 +650,14 @@ public interface MatterGenerator { Future submitted = submission; boolean dropped = submitted != null && submitted.isDone(); long waited = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start); - if (!dropped && waited < timeoutMillis) { - continue; - } - IllegalStateException timeoutFailure = new IllegalStateException("Mantle component " + key - + (dropped ? " was dropped by the dispatcher" : " did not complete in " + waited + "ms")); - if (cancel(timeoutFailure)) { - IrisLogging.error(timeoutFailure.getMessage()); + if (dropped) { + IllegalStateException dispatchFailure = new IllegalStateException( + "Mantle component " + key + " was dropped by the dispatcher"); + if (cancel(dispatchFailure)) { + IrisLogging.reportError(dispatchFailure.getMessage(), dispatchFailure); + } + } else if (waited >= timeoutMillis) { + warnSlowComponent(waited); } } catch (ExecutionException executionFailure) { Throwable cause = executionFailure.getCause(); @@ -663,6 +670,33 @@ public interface MatterGenerator { } } } + + private void warnSlowComponent(long waited) { + Thread worker; + long runningMillis; + synchronized (this) { + if (slowWarningIssued || future.isDone() || failure != null) { + return; + } + slowWarningIssued = true; + worker = runningThread; + runningMillis = started ? TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startedNanos) : 0L; + } + StringBuilder diagnostic = new StringBuilder("Mantle component ") + .append(key).append(" is still pending after ").append(waited) + .append("ms; waiting for completion. "); + if (worker == null) { + diagnostic.append("Task is queued for the dispatcher."); + } else { + diagnostic.append("Worker=").append(worker.getName()) + .append(" state=").append(worker.getState()) + .append(" runningForMs=").append(runningMillis); + for (StackTraceElement frame : worker.getStackTrace()) { + diagnostic.append("\n\tat ").append(frame); + } + } + IrisLogging.warn(diagnostic.toString()); + } } record MatterPassPlan(MantlePass pass, int passChunkRadius, int downstreamBlockRadius) { diff --git a/core/src/main/java/art/arcane/iris/generation/mantle/ObjectDestinationTransaction.java b/core/src/main/java/art/arcane/iris/generation/mantle/ObjectDestinationTransaction.java index 7e10c48b4..045e353b2 100644 --- a/core/src/main/java/art/arcane/iris/generation/mantle/ObjectDestinationTransaction.java +++ b/core/src/main/java/art/arcane/iris/generation/mantle/ObjectDestinationTransaction.java @@ -424,10 +424,6 @@ final class ObjectDestinationTransaction implements ObjectPassPlacer { DataKey key(); void apply(IObjectPlacer placer); - - default int weight() { - return 1; - } } record SetMutation(DataKey key, Object value) implements Mutation { @@ -462,10 +458,5 @@ final class ObjectDestinationTransaction implements ObjectPassPlacer { public void apply(IObjectPlacer placer) { placer.set(key.x(), key.y(), key.z(), state); } - - @Override - public int weight() { - return 2; - } } } diff --git a/core/src/main/java/art/arcane/iris/generation/mantle/ObjectSourcePlan.java b/core/src/main/java/art/arcane/iris/generation/mantle/ObjectSourcePlan.java index ef409292d..d763cc533 100644 --- a/core/src/main/java/art/arcane/iris/generation/mantle/ObjectSourcePlan.java +++ b/core/src/main/java/art/arcane/iris/generation/mantle/ObjectSourcePlan.java @@ -1,5 +1,10 @@ package art.arcane.iris.generation.mantle; +import art.arcane.iris.generation.decoration.tree.TreeBlockMaterial; +import art.arcane.iris.generation.hydrology.cave.HydrologyCaveCell; +import art.arcane.iris.integration.Identifier; +import art.arcane.volmlib.nativelib.terrain.NativeBlockState; +import art.arcane.volmlib.util.matter.MatterCavern; import it.unimi.dsi.fastutil.longs.Long2ObjectMap; import it.unimi.dsi.fastutil.longs.Long2ObjectOpenHashMap; @@ -7,29 +12,35 @@ import java.util.ArrayList; import java.util.List; final class ObjectSourcePlan { - private static final int DESTINATION_INDEX_WEIGHT = 8; + private static final int PLAN_BYTES = 256; + private static final int DESTINATION_INDEX_BYTES = 96; + private static final int MUTATION_BYTES = 96; private final Long2ObjectOpenHashMap> destinations; - private final int mutationWeight; + private final int estimatedRetainedBytes; ObjectSourcePlan(List mutations) { Long2ObjectOpenHashMap> grouped = new Long2ObjectOpenHashMap<>(2); - long weight = 1L; + long weight = PLAN_BYTES; for (ObjectDestinationTransaction.Mutation mutation : mutations) { long key = destinationKey(mutation.x() >> 4, mutation.z() >> 4); ArrayList local = grouped.get(key); if (local == null) { local = new ArrayList<>(); grouped.put(key, local); - weight += DESTINATION_INDEX_WEIGHT; + weight += DESTINATION_INDEX_BYTES; } local.add(mutation); - weight += mutation.weight(); + Object payload = switch (mutation) { + case ObjectDestinationTransaction.SetMutation set -> set.value(); + case ObjectDestinationTransaction.CustomBlockMutation custom -> custom.state(); + }; + weight = Math.min(Integer.MAX_VALUE, weight + MUTATION_BYTES + payloadBytes(payload)); } destinations = new Long2ObjectOpenHashMap<>(grouped.size()); for (Long2ObjectMap.Entry> entry : grouped.long2ObjectEntrySet()) { destinations.put(entry.getLongKey(), List.copyOf(entry.getValue())); } - this.mutationWeight = weight >= Integer.MAX_VALUE ? Integer.MAX_VALUE : (int) weight; + this.estimatedRetainedBytes = weight >= Integer.MAX_VALUE ? Integer.MAX_VALUE : (int) weight; } List mutationsFor(int chunkX, int chunkZ) { @@ -37,8 +48,34 @@ final class ObjectSourcePlan { return mutations == null ? List.of() : mutations; } - int mutationWeight() { - return mutationWeight; + int estimatedRetainedBytes() { + return estimatedRetainedBytes; + } + + private static long payloadBytes(Object value) { + return switch (value) { + case null -> 0L; + case String text -> stringBytes(text); + case TreeBlockMaterial material -> 24L + stringBytes(material.materialKey()); + case Identifier identifier -> 32L + stringBytes(identifier.namespace()) + stringBytes(identifier.key()); + case NativeBlockState ignored -> 64L; + case MatterCavern cavern -> 32L + stringBytes(cavern.getCustomBiome()); + case HydrologyCaveCell cell -> 64L + stringBytes(cell.fluidProfileKey()) + stringBytes(cell.floodedBiomeKey()); + case Byte ignored -> 24L; + case Short ignored -> 24L; + case Integer ignored -> 24L; + case Long ignored -> 24L; + case Float ignored -> 24L; + case Double ignored -> 24L; + case Boolean ignored -> 24L; + case Character ignored -> 24L; + case Enum ignored -> 0L; + default -> Integer.MAX_VALUE; + }; + } + + private static long stringBytes(String value) { + return value == null ? 0L : 48L + 2L * value.length(); } private static long destinationKey(int chunkX, int chunkZ) { diff --git a/core/src/main/java/art/arcane/iris/generation/mantle/ObjectSourcePlanCache.java b/core/src/main/java/art/arcane/iris/generation/mantle/ObjectSourcePlanCache.java index 800dec793..4ea851ab9 100644 --- a/core/src/main/java/art/arcane/iris/generation/mantle/ObjectSourcePlanCache.java +++ b/core/src/main/java/art/arcane/iris/generation/mantle/ObjectSourcePlanCache.java @@ -13,21 +13,25 @@ import java.util.function.Supplier; import static art.arcane.iris.generation.cache.Cache.key; final class ObjectSourcePlanCache { - static final long MAXIMUM_MUTATION_WEIGHT = 4_194_304L; + private static final long MAXIMUM_RETAINED_BYTES = 256L * 1024L * 1024L; - private final long maximumMutationWeight; + private final long maximumRetainedBytes; private volatile State state; ObjectSourcePlanCache() { - this(MAXIMUM_MUTATION_WEIGHT); + this(retainedByteBudget(Runtime.getRuntime().maxMemory())); } - ObjectSourcePlanCache(long maximumMutationWeight) { - if (maximumMutationWeight <= 0L) { - throw new IllegalArgumentException("Maximum mutation weight must be positive"); + ObjectSourcePlanCache(long maximumRetainedBytes) { + if (maximumRetainedBytes <= 0L) { + throw new IllegalArgumentException("Maximum retained bytes must be positive"); } - this.maximumMutationWeight = maximumMutationWeight; - state = new State(maximumMutationWeight); + this.maximumRetainedBytes = maximumRetainedBytes; + state = new State(maximumRetainedBytes); + } + + static long retainedByteBudget(long maximumHeapBytes) { + return Math.max(1L, Math.min(MAXIMUM_RETAINED_BYTES, maximumHeapBytes / 64L)); } ObjectSourcePlan get(int sourceChunkX, int sourceChunkZ, Supplier builder) { @@ -47,7 +51,8 @@ final class ObjectSourcePlanCache { ObjectSourcePlan plan = current.plans.getIfPresent(source); if (plan == null) { plan = builder.get(); - if (plan != null) { + if (plan != null && plan.estimatedRetainedBytes() < Integer.MAX_VALUE + && plan.estimatedRetainedBytes() <= maximumRetainedBytes) { current.plans.put(source, plan); } } @@ -63,7 +68,7 @@ final class ObjectSourcePlanCache { } void clear() { - state = new State(maximumMutationWeight); + state = new State(maximumRetainedBytes); } long estimatedSize() { @@ -76,10 +81,10 @@ final class ObjectSourcePlanCache { private final Cache plans; private final ConcurrentHashMap pending = new ConcurrentHashMap<>(); - private State(long maximumMutationWeight) { + private State(long maximumRetainedBytes) { plans = Caffeine.newBuilder() - .maximumWeight(maximumMutationWeight) - .weigher((Long key, ObjectSourcePlan plan) -> plan.mutationWeight()) + .maximumWeight(maximumRetainedBytes) + .weigher((Long key, ObjectSourcePlan plan) -> plan.estimatedRetainedBytes()) .build(); } } diff --git a/core/src/main/java/art/arcane/iris/world/pregen/AsyncOrMedievalPregenMethod.java b/core/src/main/java/art/arcane/iris/world/pregen/AsyncOrMedievalPregenMethod.java index cc726e177..0a729c1fc 100644 --- a/core/src/main/java/art/arcane/iris/world/pregen/AsyncOrMedievalPregenMethod.java +++ b/core/src/main/java/art/arcane/iris/world/pregen/AsyncOrMedievalPregenMethod.java @@ -95,6 +95,11 @@ public class AsyncOrMedievalPregenMethod implements PregeneratorMethod { method.onRegionSubmitted(regionX, regionZ); } + @Override + public void reclaimMemory() { + method.reclaimMemory(); + } + @Override public Mantle getMantle() { return method.getMantle(); diff --git a/core/src/main/java/art/arcane/iris/world/pregen/AsyncPregenMethod.java b/core/src/main/java/art/arcane/iris/world/pregen/AsyncPregenMethod.java index 02d21c486..b470b333d 100644 --- a/core/src/main/java/art/arcane/iris/world/pregen/AsyncPregenMethod.java +++ b/core/src/main/java/art/arcane/iris/world/pregen/AsyncPregenMethod.java @@ -31,7 +31,6 @@ import art.arcane.iris.platform.bukkit.nms.INMSBinding; import art.arcane.iris.generation.runtime.Engine; import art.arcane.iris.generation.hydrology.HydrologyTileCache; import art.arcane.iris.platform.bukkit.BukkitPlatform; -import art.arcane.volmlib.util.collection.KSet; import art.arcane.volmlib.util.mantle.runtime.Mantle; import art.arcane.volmlib.util.math.M; import art.arcane.iris.generation.concurrent.MultiBurst; @@ -97,9 +96,6 @@ public class AsyncPregenMethod implements PregeneratorMethod { private final ConcurrentHashMap regionPending; private final ConcurrentHashMap> regionChunks; private final Queue> pendingEvictions; - private final KSet drainedRegions; - private final KSet evictedRegions; - private volatile int evictionWindowRegions; private volatile int boundsMinRegionX; private volatile int boundsMinRegionZ; private volatile int boundsMaxRegionX; @@ -164,6 +160,7 @@ public class AsyncPregenMethod implements PregeneratorMethod { int configuredThreads = foliaRuntime ? computeFoliaRecommendedCap(workerThreadsForCap) : computePaperLikeRecommendedCap(workerThreadsForCap); + configuredThreads = heapLimitedConcurrency(configuredThreads, Runtime.getRuntime().maxMemory()); this.threads = selectConcurrencyCap(configuredThreads, strictSerial); this.workerPoolThreads = detectedWorkerPoolThreads; this.runtimeCpuThreads = detectedCpuThreads; @@ -183,9 +180,6 @@ public class AsyncPregenMethod implements PregeneratorMethod { this.regionPending = new ConcurrentHashMap<>(); this.regionChunks = new ConcurrentHashMap<>(); this.pendingEvictions = new ConcurrentLinkedQueue<>(); - this.drainedRegions = new KSet<>(); - this.evictedRegions = new KSet<>(); - this.evictionWindowRegions = -1; this.boundsMinRegionX = Integer.MIN_VALUE; this.boundsMinRegionZ = Integer.MIN_VALUE; this.boundsMaxRegionX = Integer.MAX_VALUE; @@ -232,28 +226,6 @@ public class AsyncPregenMethod implements PregeneratorMethod { return (((long) rx) << 32) | (rz & 0xFFFFFFFFL); } - private int evictionWindow() { - int cached = evictionWindowRegions; - if (cached > 0) { - return cached; - } - - Engine engine = resolveMetricsEngine(); - if (engine == null) { - return 2; - } - - try { - int radius = engine.getMantle().getRadius(); - int resolved = radius > 0 ? Math.max(1, (int) Math.ceil(radius / 32.0)) : 2; - evictionWindowRegions = resolved; - return resolved; - } catch (Throwable e) { - PregenDiagnostics.probeFailed("mantle radius for eviction window", e); - return 2; - } - } - private void onChunkCompleted(int x, int z, Chunk chunk) { if (chunk == null) { return; @@ -261,10 +233,16 @@ public class AsyncPregenMethod implements PregeneratorMethod { try { long rk = rkey(x >> 5, z >> 5); - regionChunks.computeIfAbsent(rk, k -> new ConcurrentLinkedQueue<>()).add(chunk); + regionChunks.compute(rk, (key, existing) -> { + Queue chunks = existing == null ? new ConcurrentLinkedQueue<>() : existing; + chunks.add(chunk); + return chunks; + }); AtomicInteger pending = regionPending.get(rk); if (pending != null && pending.decrementAndGet() == 0) { onRegionDrained(rk); + } else if (MantleHeapPressure.overHighWater()) { + evictRegion(rk); } } catch (Throwable e) { IrisLogging.reportError(e); @@ -317,10 +295,6 @@ public class AsyncPregenMethod implements PregeneratorMethod { } } - private boolean inBounds(int rx, int rz) { - return rx >= boundsMinRegionX && rx <= boundsMaxRegionX && rz >= boundsMinRegionZ && rz <= boundsMaxRegionZ; - } - @Override public void onRegionSubmitted(int regionX, int regionZ) { try { @@ -335,58 +309,14 @@ public class AsyncPregenMethod implements PregeneratorMethod { } private void onRegionDrained(long rk) { - if (!drainedRegions.add(rk)) { - return; - } - - int w = evictionWindow(); - int rx = (int) (rk >> 32); - int rz = (int) rk; - for (int dx = -w; dx <= w; dx++) { - for (int dz = -w; dz <= w; dz++) { - int cx = rx + dx; - int cz = rz + dz; - long candidate = rkey(cx, cz); - if (evictedRegions.contains(candidate)) { - continue; - } - if (!drainedRegions.contains(candidate)) { - continue; - } - if (allNeighborsDrained(cx, cz, w)) { - evictRegion(candidate); - } - } - } - } - - private boolean allNeighborsDrained(int rx, int rz, int w) { - for (int dx = -w; dx <= w; dx++) { - for (int dz = -w; dz <= w; dz++) { - int nx = rx + dx; - int nz = rz + dz; - if (!inBounds(nx, nz)) { - continue; - } - - if (!drainedRegions.contains(rkey(nx, nz))) { - return false; - } - } - } - - return true; + regionPending.remove(rk); + evictRegion(rk); } private CompletableFuture evictRegion(long c) { if (IrisToolbelt.isServerStopping()) { return CompletableFuture.completedFuture(null); } - if (!evictedRegions.add(c)) { - return CompletableFuture.completedFuture(null); - } - - regionPending.remove(c); Queue chunks = regionChunks.remove(c); if (chunks == null || chunks.isEmpty()) { return CompletableFuture.completedFuture(null); @@ -651,6 +581,11 @@ public class AsyncPregenMethod implements PregeneratorMethod { return strictSerial ? 1 : Math.max(1, recommendedCap); } + static int heapLimitedConcurrency(int recommendedCap, long maximumHeapBytes) { + long heapCap = Math.max(1L, maximumHeapBytes / (256L * 1024L * 1024L)); + return (int) Math.min(Math.max(1, recommendedCap), heapCap); + } + static int computeInitialInFlightLimit(int concurrencyCap, int workerThreads) { long initial = Math.max(1L, workerThreads) * 8L; return (int) Math.min(Math.max(1, concurrencyCap), initial); @@ -880,6 +815,14 @@ public class AsyncPregenMethod implements PregeneratorMethod { public void save() { } + @Override + public void reclaimMemory() { + if (regionChunks.isEmpty() && pendingEvictions.isEmpty()) { + return; + } + flushAllRemainingChunks(); + } + @Override public boolean supportsRegions(int x, int z, PregenListener listener) { return false; @@ -893,8 +836,11 @@ public class AsyncPregenMethod implements PregeneratorMethod { @Override public void generateChunk(int x, int z, PregenListener listener) { listener.onChunkGenerating(x, z); + if (MantleHeapPressure.overHighWater()) { + reclaimMemory(); + } backpressure.enforceMantleBudget(); - backpressure.awaitHeapHeadroom(); + backpressure.awaitHeapHeadroom(MantleHeapPressure::overHighWater, this::reclaimHeapPressure); if (isCancelled()) { return; } @@ -924,6 +870,13 @@ public class AsyncPregenMethod implements PregeneratorMethod { executor.generate(x, z, listener); } + private void reclaimHeapPressure() { + if (!regionChunks.isEmpty() || !pendingEvictions.isEmpty()) { + reclaimMemory(); + } + MantleHeapPressure.requestPanicReclaim(); + } + private CompletableFuture requestChunkAsync(int x, int z) { Throwable failure = null; diff --git a/core/src/main/java/art/arcane/iris/world/pregen/CachedPregenMethod.java b/core/src/main/java/art/arcane/iris/world/pregen/CachedPregenMethod.java index b0f435c27..783906055 100644 --- a/core/src/main/java/art/arcane/iris/world/pregen/CachedPregenMethod.java +++ b/core/src/main/java/art/arcane/iris/world/pregen/CachedPregenMethod.java @@ -203,6 +203,11 @@ public class CachedPregenMethod implements PregeneratorMethod { PregenSavedChunkStatus savedStatus) { } + @Override + public void reclaimMemory() { + method.reclaimMemory(); + } + @Override public Mantle getMantle() { return method.getMantle(); diff --git a/core/src/main/java/art/arcane/iris/world/pregen/HybridPregenMethod.java b/core/src/main/java/art/arcane/iris/world/pregen/HybridPregenMethod.java index 770e56faa..7e7498bf8 100644 --- a/core/src/main/java/art/arcane/iris/world/pregen/HybridPregenMethod.java +++ b/core/src/main/java/art/arcane/iris/world/pregen/HybridPregenMethod.java @@ -88,6 +88,11 @@ public class HybridPregenMethod implements PregeneratorMethod { inWorld.onRegionSubmitted(regionX, regionZ); } + @Override + public void reclaimMemory() { + inWorld.reclaimMemory(); + } + @Override public Mantle getMantle() { return inWorld.getMantle(); diff --git a/core/src/main/java/art/arcane/iris/world/pregen/IrisPregenerator.java b/core/src/main/java/art/arcane/iris/world/pregen/IrisPregenerator.java index ed8b31e38..61f45c3e4 100644 --- a/core/src/main/java/art/arcane/iris/world/pregen/IrisPregenerator.java +++ b/core/src/main/java/art/arcane/iris/world/pregen/IrisPregenerator.java @@ -438,6 +438,12 @@ public class IrisPregenerator { return; } + try { + generator.reclaimMemory(); + } catch (Throwable e) { + IrisLogging.reportError("Pregen could not reclaim completed server chunks under heap pressure.", e); + } + try { Mantle mantle = getMantle(); if (mantle != null) { diff --git a/core/src/main/java/art/arcane/iris/world/pregen/PregenMantleBackpressure.java b/core/src/main/java/art/arcane/iris/world/pregen/PregenMantleBackpressure.java index b87eebbf6..146294f66 100644 --- a/core/src/main/java/art/arcane/iris/world/pregen/PregenMantleBackpressure.java +++ b/core/src/main/java/art/arcane/iris/world/pregen/PregenMantleBackpressure.java @@ -29,6 +29,8 @@ import java.util.function.BooleanSupplier; import java.util.function.Supplier; public final class PregenMantleBackpressure { + private static final long LOG_INTERVAL_MS = 5_000L; + private final Supplier mantleSupplier; private final int maxResidentTectonicPlates; private final int waitMs; @@ -46,7 +48,7 @@ public final class PregenMantleBackpressure { public PregenMantleBackpressure(Supplier mantleSupplier, int maxResidentTectonicPlates, int waitMs, long timeoutMs, Runnable onBudgetTimeout, Supplier diagnostics, BooleanSupplier cancelled) { this.mantleSupplier = mantleSupplier; this.maxResidentTectonicPlates = maxResidentTectonicPlates; - this.waitMs = waitMs; + this.waitMs = Math.max(1, waitMs); this.timeoutMs = timeoutMs; this.onBudgetTimeout = onBudgetTimeout; this.diagnostics = diagnostics; @@ -69,14 +71,13 @@ public final class PregenMantleBackpressure { return; } - int hardCap = cap * 2; - if (mantle.getLoadedRegionCount() <= hardCap) { + if (mantle.getLoadedRegionCount() <= cap) { return; } long waitStart = M.ms(); long lastLog = 0L; - while (mantle.getLoadedRegionCount() > hardCap) { + while (mantle.getLoadedRegionCount() > cap) { if (isCancelled()) { return; } @@ -87,29 +88,34 @@ public final class PregenMantleBackpressure { mantle.trim(0L); freed = mantle.unloadTectonicPlate(0); resident = mantle.getLoadedRegionCount(); + if (resident > cap && mantle.saveOldestIdleTectonicPlate()) { + freed++; + resident = mantle.getLoadedRegionCount(); + } } catch (Throwable e) { IrisLogging.reportError(e); break; } - if (resident <= hardCap) { + if (resident <= cap) { break; } long elapsed = M.ms() - waitStart; - if (elapsed >= timeoutMs) { + if (elapsed >= timeoutMs && freed == 0) { IrisLogging.warn("Pregen mantle backpressure exceeded " + timeoutMs + "ms with " + resident - + " tectonic plates resident (hard cap " + hardCap + "); proceeding to avoid deadlock. " - + "Raise pregen.maxResidentTectonicPlates if this persists. " + diagnostics.get()); + + " tectonic plates resident (residency target " + cap + "); no idle plate could be evicted. " + + "Allowing the active dependency working set to exceed the residency target; " + + "heap pressure still blocks new chunk submissions. " + diagnostics.get()); onBudgetTimeout.run(); return; } long logNow = M.ms(); - if (logNow - lastLog >= 5_000L) { + if (logNow - lastLog >= LOG_INTERVAL_MS) { lastLog = logNow; // Pausing to stay under the plate cap is the design working, and it is reported every five // seconds for the whole duration of a large pregen. Only exceeding the budget is a warning. - IrisLogging.info("Pregen mantle backpressure: " + resident + " tectonic plates resident (hard cap " + hardCap + IrisLogging.info("Pregen mantle backpressure: " + resident + " tectonic plates resident (residency target " + cap + "), freed " + freed + " last pass, waited " + elapsed + "ms."); } @@ -127,6 +133,9 @@ public final class PregenMantleBackpressure { Mantle mantle = resolveMantle(); long waitStart = M.ms(); long lastLog = 0L; + long nextWarningElapsed = Math.max(0L, timeoutMs); + long nextEvictionErrorLog = 0L; + long warningInterval = Math.max(LOG_INTERVAL_MS, timeoutMs); while (heapPressure.getAsBoolean()) { if (isCancelled()) { return; @@ -137,22 +146,30 @@ public final class PregenMantleBackpressure { mantle.saveOldestIdleTectonicPlate(); } } catch (Throwable e) { - IrisLogging.reportError(e); + long now = M.ms(); + if (now >= nextEvictionErrorLog) { + nextEvictionErrorLog = now + warningInterval; + IrisLogging.reportError(e); + } } panicReclaim.run(); - - long elapsed = M.ms() - waitStart; - if (elapsed >= timeoutMs) { - IrisLogging.warn("Pregen heap pressure wait exceeded " + timeoutMs + "ms at " - + Math.round(MantleHeapPressure.usedFraction() * 100.0D) + "% heap; proceeding to avoid deadlock. " - + diagnostics.get()); - onBudgetTimeout.run(); + if (isCancelled() || !heapPressure.getAsBoolean()) { return; } + long elapsed = M.ms() - waitStart; long logNow = M.ms(); - if (logNow - lastLog >= 5_000L) { + if (elapsed >= nextWarningElapsed) { + nextWarningElapsed = elapsed + warningInterval; + lastLog = logNow; + IrisLogging.warn("Pregen heap pressure is still blocking new chunk submissions after " + elapsed + "ms at " + + Math.round(MantleHeapPressure.usedFraction() * 100.0D) + "% heap; waiting for headroom. " + + diagnostics.get()); + onBudgetTimeout.run(); + } + + if (logNow - lastLog >= LOG_INTERVAL_MS) { lastLog = logNow; IrisLogging.info("Pregen heap pressure: pausing generation at " + Math.round(MantleHeapPressure.usedFraction() * 100.0D) + "% heap; evicting tectonic plates and waiting for headroom" diff --git a/core/src/main/java/art/arcane/iris/world/pregen/PregeneratorMethod.java b/core/src/main/java/art/arcane/iris/world/pregen/PregeneratorMethod.java index 4d62e8775..151ef6a15 100644 --- a/core/src/main/java/art/arcane/iris/world/pregen/PregeneratorMethod.java +++ b/core/src/main/java/art/arcane/iris/world/pregen/PregeneratorMethod.java @@ -89,5 +89,8 @@ public interface PregeneratorMethod { default void onRegionSubmitted(int regionX, int regionZ) { } + default void reclaimMemory() { + } + Mantle getMantle(); } diff --git a/core/src/test/java/art/arcane/iris/configuration/IrisSettingsDefaultsTest.java b/core/src/test/java/art/arcane/iris/configuration/IrisSettingsDefaultsTest.java index e5b134757..42e19ec60 100644 --- a/core/src/test/java/art/arcane/iris/configuration/IrisSettingsDefaultsTest.java +++ b/core/src/test/java/art/arcane/iris/configuration/IrisSettingsDefaultsTest.java @@ -12,6 +12,16 @@ import static org.junit.Assert.assertTrue; import static org.junit.Assert.assertThrows; public class IrisSettingsDefaultsTest { + @Test + public void regionBudgetLeavesHeapForHydrologyAndServerChunks() { + IrisSettings.IrisSettingsPregen settings = new IrisSettings.IrisSettingsPregen(); + assertEquals(21, settings.effectiveResidentTectonicPlates(384, 4L << 30)); + assertEquals(42, settings.effectiveResidentTectonicPlates(384, 8L << 30)); + assertEquals(16, settings.effectiveResidentTectonicPlates(768, 4L << 30)); + settings.setMaxResidentTectonicPlates(16); + assertEquals(16, settings.effectiveResidentTectonicPlates(384, 16L << 30)); + } + @Test public void concurrencyIsNotSerializedIntoIrisJson() { String json = new Gson().toJson(new IrisSettings()); diff --git a/core/src/test/java/art/arcane/iris/generation/hydrology/HydrologyPlannerCrossTileCaveAdmissionIntegrationTest.java b/core/src/test/java/art/arcane/iris/generation/hydrology/HydrologyPlannerCrossTileCaveAdmissionIntegrationTest.java index a25f2cedc..e1bf9bbc8 100644 --- a/core/src/test/java/art/arcane/iris/generation/hydrology/HydrologyPlannerCrossTileCaveAdmissionIntegrationTest.java +++ b/core/src/test/java/art/arcane/iris/generation/hydrology/HydrologyPlannerCrossTileCaveAdmissionIntegrationTest.java @@ -23,6 +23,7 @@ import java.util.Map; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertSame; import static org.junit.Assert.assertTrue; import static org.mockito.Mockito.mock; @@ -107,6 +108,41 @@ public class HydrologyPlannerCrossTileCaveAdmissionIntegrationTest { assertEquals(lazy, parallel); } + @Test + public void repeatedDependencyRequestsKeepTheirContextOwnersAfterCacheEviction() { + HydrologyPlanner planner = planner(); + List keys = List.of(new HydrologyTileKey(0, 0), new HydrologyTileKey(-1, 0)); + CrossTileResolutionContext context = new CrossTileResolutionContext(new HydrologyTileKey(-1, -1), 64L, 4096); + Map original = new LinkedHashMap<>(); + Map> prepared = new LinkedHashMap<>(); + for (HydrologyTileKey key : keys) { + CrossTileResolvedOwner owner = planner.resolveIndependentOwner(key); + original.put(key, owner); + context.remember(key, owner); + prepared.put(key, new HydrologyForkJoin.Task<>(() -> { + throw new AssertionError("A remembered owner must not be resolved again"); + })); + } + planner.clearOwnerDrafts(); + IrisSettings previousSettings = IrisSettings.settings; + IrisSettings.settings = new IrisSettings(); + try { + for (boolean bound : List.of(false, true)) { + IrisPlatforms.unbind(); + if (bound) { + IrisPlatforms.bind(mock(IrisPlatform.class)); + } + List owners = planner.resolveLowerRankOwners(keys, context, prepared); + for (int index = 0; index < keys.size(); index++) { + assertSame(original.get(keys.get(index)), owners.get(index)); + } + } + } finally { + IrisPlatforms.unbind(); + IrisSettings.settings = previousSettings; + } + } + @Test public void colorRankBoundsTheCompleteOwnerDependencyGeometry() { HydrologyPlanner planner = planner(); diff --git a/core/src/test/java/art/arcane/iris/generation/hydrology/HydrologyPlannerTest.java b/core/src/test/java/art/arcane/iris/generation/hydrology/HydrologyPlannerTest.java index 094175c68..17fd5d818 100644 --- a/core/src/test/java/art/arcane/iris/generation/hydrology/HydrologyPlannerTest.java +++ b/core/src/test/java/art/arcane/iris/generation/hydrology/HydrologyPlannerTest.java @@ -113,6 +113,50 @@ public class HydrologyPlannerTest { assertTileContentsEqual(EARLY_OWNER_BASELINE, dependencyPlanner.plan(EARLY_OWNER_TILE)); } + @Test + public void restoredTilePreservesOwnerOrderingAndRejectedCoursesAfterEviction() { + HydrologyPlanner planner = new HydrologyPlanner(77L, EARLY_OWNER_SETTINGS, EARLY_OWNER_TERRAIN); + HydrologyTile tile = planner.plan(EARLY_OWNER_TILE); + CrossTileResolvedOwner original = tile.resolvedOwner(); + assertNotNull(original); + assertNull(original.draft().footprintCompiler()); + assertFalse(original.observedRejections().isEmpty()); + CrossTileResolutionContext context = new CrossTileResolutionContext(EARLY_OWNER_TILE, 64L, 4096); + context.remember(EARLY_OWNER_TILE, original); + planner.clearOwnerDrafts(); + + planner.reuseResolvedTile(tile); + CrossTileResolvedOwner restored = planner.resolveIndependentOwner(EARLY_OWNER_TILE); + + context.remember(EARLY_OWNER_TILE, restored); + assertEquals(original, restored); + assertTileContentsEqual(tile, planner.plan(EARLY_OWNER_TILE)); + } + + @Test + public void restoringPreparedTileKeepsExistingCanonicalOwner() { + HydrologyPlanner planner = new HydrologyPlanner(77L, EARLY_OWNER_SETTINGS, EARLY_OWNER_TERRAIN); + HydrologyTile tile = planner.plan(EARLY_OWNER_TILE); + CrossTileResolvedOwner existing = planner.resolveIndependentOwner(EARLY_OWNER_TILE); + + planner.reuseResolvedTile(tile); + + assertSame(existing, planner.resolveIndependentOwner(EARLY_OWNER_TILE)); + } + + @Test + public void finalTileWithoutOwnerMetadataCannotInventOwnerDraft() { + HydrologyPlanner planner = new HydrologyPlanner(77L, EARLY_OWNER_SETTINGS, EARLY_OWNER_TERRAIN); + + HydrologyTile tile = new HydrologyTile(EARLY_OWNER_TILE, 77L, EARLY_OWNER_SETTINGS.fingerprint(), + EARLY_OWNER_SETTINGS.routing().tileSize(), List.of(), List.of(), List.of(), List.of(), + Set.of(), List.of(), List.of(), RiverFootprint.empty()); + planner.reuseResolvedTile(tile); + + assertNull(planner.resolvedOwners.getIfPresent(EARLY_OWNER_TILE)); + assertTileContentsEqual(EARLY_OWNER_BASELINE, planner.plan(EARLY_OWNER_TILE)); + } + @Test public void collidingRoutingContextKeysCompileIndependently() throws Exception { CountDownLatch entered = new CountDownLatch(1); diff --git a/core/src/test/java/art/arcane/iris/generation/hydrology/HydrologyPlanningAdmissionIntegrationTest.java b/core/src/test/java/art/arcane/iris/generation/hydrology/HydrologyPlanningAdmissionIntegrationTest.java index f5f8cc352..05b0acdff 100644 --- a/core/src/test/java/art/arcane/iris/generation/hydrology/HydrologyPlanningAdmissionIntegrationTest.java +++ b/core/src/test/java/art/arcane/iris/generation/hydrology/HydrologyPlanningAdmissionIntegrationTest.java @@ -99,7 +99,6 @@ public class HydrologyPlanningAdmissionIntegrationTest { () -> pending.get(2L, TimeUnit.SECONDS)); assertTrue(failure.getCause() instanceof CancellationException); verify(planner, never()).plan(any(HydrologyTileKey.class)); - verify(planner, never()).emptyTile(any(HydrologyTileKey.class)); assertEquals(0, cache.size()); releaseRoots(held); HydrologyTile tile = mock(HydrologyTile.class); diff --git a/core/src/test/java/art/arcane/iris/generation/hydrology/HydrologyRegionalFallsTest.java b/core/src/test/java/art/arcane/iris/generation/hydrology/HydrologyRegionalFallsTest.java index 46ab117e7..9cf79e993 100644 --- a/core/src/test/java/art/arcane/iris/generation/hydrology/HydrologyRegionalFallsTest.java +++ b/core/src/test/java/art/arcane/iris/generation/hydrology/HydrologyRegionalFallsTest.java @@ -44,19 +44,33 @@ public final class HydrologyRegionalFallsTest { .materializeFinalHydrology(new HydrologyCaveCourseFilter.Result(network.nodes(), network.edges(), network.outlets(), network.courses(), plans), new ArrayList<>(), compiler, true); HydrologyCaveCourseFilter.Result clipped = HydrologyCrossTileResolver.clipRegionalPlans(materialized.result(), network, bounds); - RiverCourse local = new RiverCourse(101L, RiverCourseType.SURFACE, regional.sourceNodeId(), regional.outletId(), + DrainageNode localNode = new DrainageNode(103L, 0, 0, + HydrologyTerrainSample.openLand(89, 0D, "parent"), 100D, 104L); + RiverOutlet localOutlet = new RiverOutlet(104L, HydrologyFeatureType.MOUTH, 103L, + new HydrologyPoint(0, 72, 0), new HydrologyPoint(1, 63, 0), 63, true); + RiverCourse local = new RiverCourse(101L, RiverCourseType.SURFACE, OptionalLong.of(103L), OptionalLong.of(104L), "water", 1, List.of(), List.of(new HydraulicSegment(102L, 101L, HydrologyFeatureType.SURFACE_POOL, 72, 72, 4, 2, false, false, List.of(new HydrologyPoint(0, 72, 0), new HydrologyPoint(1, 72, 0)), HydraulicChannelProfile.uniform(4, 2)))); - HydrologyTile original = new HydrologyTile(new HydrologyTileKey(0, 0), 1L, SETTINGS.fingerprint(), - SETTINGS.routing().tileSize(), network.nodes(), network.edges(), network.outlets(), - List.of(regional, local), Set.of(regional.id()), clipped.cavePlans(), List.of(), materialized.footprint()); + HydrologyTileKey key = new HydrologyTileKey(0, 0); + CrossTileResolvedOwner owner = new CrossTileResolvedOwner(new HydrologyOwnerDraft(key, + new HydrologyCaveCourseFilter.Result(List.of(localNode), List.of(), List.of(localOutlet), + List.of(local), List.of()), List.of(), null), List.of()); + ArrayList publishedNodes = new ArrayList<>(network.nodes()); + publishedNodes.add(localNode); + ArrayList publishedOutlets = new ArrayList<>(network.outlets()); + publishedOutlets.add(localOutlet); + HydrologyTile original = new HydrologyTile(key, 1L, SETTINGS.fingerprint(), + SETTINGS.routing().tileSize(), publishedNodes, network.edges(), publishedOutlets, + List.of(regional, local), Set.of(regional.id()), clipped.cavePlans(), List.of(), materialized.footprint()) + .withResolvedOwner(owner); PreparedHydrologyTileStore store = new PreparedHydrologyTileStore(temporaryFolder.newFolder().toPath(), new HydrologyTileCache.SharedCacheScope("regional-ownership", 1L, 128, "overworld", SETTINGS.fingerprint()), SETTINGS.routing().tileSize()); store.save(original); HydrologyTile restored = store.load(original.key()).orElseThrow(); assertEquals(original, restored); + assertEquals(owner, restored.resolvedOwner()); assertFalse(restored.cavePlans().isEmpty()); HydrologyPlanner fresh = new HydrologyPlanner(1L, SETTINGS, (x, z) -> { throw new AssertionError("Restoring final ownership must not sample terrain"); @@ -64,12 +78,14 @@ public final class HydrologyRegionalFallsTest { fresh.reuseResolvedTile(restored); - HydrologyCaveCourseFilter.Result owned = fresh.resolvedOwners.getIfPresent(restored.key()).draft().result(); + CrossTileResolvedOwner restoredOwner = fresh.resolveIndependentOwner(restored.key()); + assertEquals(owner, restoredOwner); + HydrologyCaveCourseFilter.Result owned = restoredOwner.draft().result(); assertEquals(List.of(local), owned.courses()); assertTrue(owned.cavePlans().isEmpty()); - assertEquals(restored.nodes(), owned.nodes()); - assertEquals(restored.edges(), owned.edges()); - assertEquals(restored.outlets(), owned.outlets()); + assertEquals(List.of(localNode), owned.nodes()); + assertTrue(owned.edges().isEmpty()); + assertEquals(List.of(localOutlet), owned.outlets()); } @Test diff --git a/core/src/test/java/art/arcane/iris/generation/hydrology/HydrologyRegionalPlannerTest.java b/core/src/test/java/art/arcane/iris/generation/hydrology/HydrologyRegionalPlannerTest.java index d0259e01c..758af88d6 100644 --- a/core/src/test/java/art/arcane/iris/generation/hydrology/HydrologyRegionalPlannerTest.java +++ b/core/src/test/java/art/arcane/iris/generation/hydrology/HydrologyRegionalPlannerTest.java @@ -39,6 +39,7 @@ public class HydrologyRegionalPlannerTest { }); restored.reuseResolvedTile(original); + assertEquals(original.resolvedOwner(), restored.resolveIndependentOwner(key)); planning.set(true); assertEquals(expected, restored.plan(adjacent)); diff --git a/core/src/test/java/art/arcane/iris/generation/hydrology/HydrologyTileCacheTest.java b/core/src/test/java/art/arcane/iris/generation/hydrology/HydrologyTileCacheTest.java index 87f9233a4..c0800e0ae 100644 --- a/core/src/test/java/art/arcane/iris/generation/hydrology/HydrologyTileCacheTest.java +++ b/core/src/test/java/art/arcane/iris/generation/hydrology/HydrologyTileCacheTest.java @@ -56,6 +56,32 @@ public class HydrologyTileCacheTest { @Rule public TemporaryFolder preparedFolder = new TemporaryFolder(); + @Test + public void sharedTileReuseDoesNotDecodeThePersistedCopy() throws Exception { + HydrologyPlanner firstPlanner = mock(HydrologyPlanner.class); + HydrologyPlanner secondPlanner = mock(HydrologyPlanner.class); + when(firstPlanner.settings()).thenReturn(emptySettings()); + when(secondPlanner.settings()).thenReturn(emptySettings()); + HydrologyTileCache.SharedCacheScope scope = sharedScope(); + HydrologyTileKey key = new HydrologyTileKey(10, -15); + HydrologyTile tile = preparedTile(key, scope); + when(firstPlanner.plan(key)).thenReturn(tile); + PreparedHydrologyTileStore store = mock(PreparedHydrologyTileStore.class); + when(store.contains(key)).thenReturn(true); + try (HydrologyTileCache first = new HydrologyTileCache(firstPlanner, 4, null, null, scope); + HydrologyTileCache second = new HydrologyTileCache(secondPlanner, 4, null, null, scope)) { + assertSame(tile, first.get(key)); + Field persistentStore = HydrologyTileCache.class.getDeclaredField("persistentStore"); + persistentStore.setAccessible(true); + persistentStore.set(second, store); + assertSame(tile, second.get(key)); + verify(store).contains(key); + verify(store, never()).load(key); + verify(store, never()).save(tile); + verify(secondPlanner, never()).plan(key); + } + } + @Test public void preparedPlansRestoreFarFromOriginWithoutTerrainPlanning() throws Exception { HydrologyPlanner planner = mock(HydrologyPlanner.class); @@ -740,26 +766,56 @@ public class HydrologyTileCacheTest { assertEquals(1, terrainPreparations.get()); } - @Test - public void failedPlansDoNotEnterTheSharedStudioCache() { - HydrologyPlanner failingPlanner = mock(HydrologyPlanner.class); - when(failingPlanner.settings()).thenReturn(emptySettings()); - HydrologyPlanner succeedingPlanner = mock(HydrologyPlanner.class); - when(succeedingPlanner.settings()).thenReturn(emptySettings()); - HydrologyTile emptyTile = mock(HydrologyTile.class); - HydrologyTile plannedTile = mock(HydrologyTile.class); - HydrologyTileKey key = new HydrologyTileKey(-4, 5); + @Test(timeout = 10000L) + public void failedPlansRetryBeforePublishingToMemorySharedCacheOrDisk() throws Exception { + HydrologyPlanner planner = mock(HydrologyPlanner.class); + when(planner.settings()).thenReturn(emptySettings()); HydrologyTileCache.SharedCacheScope scope = sharedScope(); - when(failingPlanner.plan(key)).thenThrow(new IllegalStateException("test failure")); - when(failingPlanner.emptyTile(key)).thenReturn(emptyTile); - when(succeedingPlanner.plan(key)).thenReturn(plannedTile); - HydrologyTileCache first = new HydrologyTileCache(failingPlanner, 4, null, null, scope); - HydrologyTileCache second = new HydrologyTileCache(succeedingPlanner, 4, null, null, scope); - - assertSame(emptyTile, first.get(key)); - assertSame(plannedTile, second.get(key)); - - verify(succeedingPlanner, times(1)).plan(key); + HydrologyTileKey key = new HydrologyTileKey(-4, 5); + HydrologyTile tile = preparedTile(key, scope); + AtomicInteger attempts = new AtomicInteger(); + CountDownLatch rebuilding = new CountDownLatch(1); + CountDownLatch recovered = new CountDownLatch(1); + when(planner.plan(key)).thenAnswer(invocation -> { + if (attempts.incrementAndGet() <= 3) { + throw new IllegalStateException("Inconsistent owner metadata"); + } + rebuilding.countDown(); + assertTrue(recovered.await(5L, TimeUnit.SECONDS)); + return tile; + }); + PreparedHydrologyTileStore store = mock(PreparedHydrologyTileStore.class); + ExecutorService callers = Executors.newFixedThreadPool(2); + try (HydrologyTileCache cache = new HydrologyTileCache(planner, 4, null, null, scope)) { + Field persistentStore = HydrologyTileCache.class.getDeclaredField("persistentStore"); + persistentStore.setAccessible(true); + persistentStore.set(cache, store); + Future first = callers.submit(() -> cache.get(key)); + Future second = callers.submit(() -> cache.get(key)); + assertTrue(rebuilding.await(5L, TimeUnit.SECONDS)); + assertFalse(first.isDone()); + assertFalse(second.isDone()); + assertEquals(0, cache.size()); + verify(store, never()).save(any(HydrologyTile.class)); + recovered.countDown(); + assertSame(tile, first.get(5L, TimeUnit.SECONDS)); + assertSame(tile, second.get(5L, TimeUnit.SECONDS)); + assertSame(tile, cache.get(key)); + verify(planner, times(4)).plan(key); + verify(planner, times(3)).clearOwnerDrafts(); + verify(store, times(1)).load(key); + verify(store, times(1)).save(tile); + HydrologyPlanner sharedPlanner = mock(HydrologyPlanner.class); + when(sharedPlanner.settings()).thenReturn(emptySettings()); + try (HydrologyTileCache shared = new HydrologyTileCache(sharedPlanner, 4, null, null, scope)) { + assertSame(tile, shared.get(key)); + verify(sharedPlanner, never()).plan(key); + } + } finally { + recovered.countDown(); + callers.shutdownNow(); + assertTrue(callers.awaitTermination(5L, TimeUnit.SECONDS)); + } } @Test @@ -1243,35 +1299,139 @@ public class HydrologyTileCacheTest { ); } - @Test - public void planningFailureFallsBackToTheEmptyTileAndIsCached() { + @Test(timeout = 10000L) + public void planningFailureRetriesAndCachesTheRecoveredTile() { HydrologyPlanner planner = mock(HydrologyPlanner.class); - HydrologyTileKey key = new HydrologyTileKey(3, -2); - HydrologyTile empty = new HydrologyTile(key, 7L, 11L, 1024, List.of(), List.of(), List.of(), List.of(), Set.of(), List.of(), List.of(), RiverFootprint.empty()); - when(planner.settings()).thenReturn(emptySettings()); - when(planner.plan(key)).thenThrow(new IllegalStateException("Hydrology natural height was not finite at -66,-641")); - when(planner.emptyTile(key)).thenReturn(empty); - HydrologyTileCache cache = new HydrologyTileCache(planner, 4); - - assertSame(empty, cache.get(key)); - assertSame(empty, cache.get(key)); - - verify(planner, times(1)).plan(key); - assertTrue(cache.get(key).courses().isEmpty()); + HydrologyPlanner canonical = new HydrologyPlanner(811L, featureSettings(), this::featureTerrain); + HydrologyTileKey key = new HydrologyTileKey(0, 0); + HydrologyTile tile = canonical.plan(key); + HydrologyColumnSample sample = tile.footprint().columns().values().stream() + .filter(HydrologyColumnSample::present).findFirst().orElseThrow(); + when(planner.settings()).thenReturn(featureSettings()); + when(planner.plan(any(HydrologyTileKey.class))).thenAnswer(invocation -> + canonical.plan(invocation.getArgument(0))); + when(planner.plan(key)).thenThrow(new IllegalStateException("Inconsistent owner metadata")) + .thenReturn(tile); + try (HydrologyTileCache cache = new HydrologyTileCache(planner, 8)) { + assertSame(tile, cache.get(key)); + assertSame(tile, cache.get(key)); + assertFalse(tile.courses().isEmpty()); + assertTrue(tile.footprint().size() > 0); + HydrologyColumnSample published = cache.columnAt(sample.x(), sample.z()).orElseThrow(); + assertEquals(sample.x(), published.x()); + assertEquals(sample.z(), published.z()); + assertTrue(published.layers().containsAll(sample.layers())); + verify(planner, times(2)).plan(key); + verify(planner).clearOwnerDrafts(); + } } - @Test - public void plannerEmptyTileCarriesTheTileIdentityAndNoContent() { - HydrologyPlanner planner = new HydrologyPlanner(811L, featureSettings(), this::featureTerrain); - HydrologyTileKey key = new HydrologyTileKey(-4, 9); + @Test(timeout = 10000L) + public void clearingDuringRetryCompletesTheSnapshotWithoutPublishing() throws Exception { + HydrologyPlanner planner = mock(HydrologyPlanner.class); + when(planner.settings()).thenReturn(emptySettings()); + HydrologyTileCache.SharedCacheScope scope = sharedScope(); + HydrologyTileKey key = new HydrologyTileKey(3, -2); + HydrologyTile tile = preparedTile(key, scope); + CountDownLatch failed = new CountDownLatch(1); + doAnswer(invocation -> { + failed.countDown(); + return null; + }).when(planner).clearOwnerDrafts(); + when(planner.plan(key)).thenThrow(new IllegalStateException("Inconsistent owner metadata")) + .thenReturn(tile); + PreparedHydrologyTileStore store = mock(PreparedHydrologyTileStore.class); + ExecutorService caller = Executors.newSingleThreadExecutor(); + try (HydrologyTileCache cache = new HydrologyTileCache(planner, 4, null, null, scope)) { + Field persistentStore = HydrologyTileCache.class.getDeclaredField("persistentStore"); + persistentStore.setAccessible(true); + persistentStore.set(cache, store); + Future pending = caller.submit(() -> cache.get(key)); + assertTrue(failed.await(5L, TimeUnit.SECONDS)); + cache.clear(); + assertSame(tile, pending.get(5L, TimeUnit.SECONDS)); + assertEquals(0, cache.size()); + verify(store, never()).save(any(HydrologyTile.class)); + HydrologyPlanner freshPlanner = mock(HydrologyPlanner.class); + when(freshPlanner.settings()).thenReturn(emptySettings()); + when(freshPlanner.plan(key)).thenReturn(tile); + try (HydrologyTileCache fresh = new HydrologyTileCache(freshPlanner, 4, null, null, scope)) { + assertSame(tile, fresh.get(key)); + verify(freshPlanner).plan(key); + } + } finally { + caller.shutdownNow(); + assertTrue(caller.awaitTermination(5L, TimeUnit.SECONDS)); + } + } - HydrologyTile tile = planner.emptyTile(key); + @Test(timeout = 10000L) + public void closingDuringRetryCancelsWithoutPublishing() throws Exception { + HydrologyPlanner planner = mock(HydrologyPlanner.class); + when(planner.settings()).thenReturn(emptySettings()); + HydrologyTileKey key = new HydrologyTileKey(3, -2); + CountDownLatch failed = new CountDownLatch(1); + doAnswer(invocation -> { + failed.countDown(); + return null; + }).when(planner).clearOwnerDrafts(); + when(planner.plan(key)).thenThrow(new IllegalStateException("Inconsistent owner metadata")); + ExecutorService caller = Executors.newSingleThreadExecutor(); + HydrologyTileCache cache = new HydrologyTileCache(planner, 4); + try { + Future pending = caller.submit(() -> cache.get(key)); + assertTrue(failed.await(5L, TimeUnit.SECONDS)); + cache.close(); + ExecutionException failure = assertThrows(ExecutionException.class, + () -> pending.get(2L, TimeUnit.SECONDS)); + assertTrue(failure.getCause() instanceof CancellationException); + assertEquals(0, cache.size()); + verify(planner, times(1)).plan(key); + } finally { + caller.shutdownNow(); + assertTrue(caller.awaitTermination(5L, TimeUnit.SECONDS)); + cache.close(); + } + } - assertEquals(key, tile.key()); - assertEquals(featureSettings().routing().tileSize(), tile.tileSize()); - assertTrue(tile.courses().isEmpty()); - assertTrue(tile.outlets().isEmpty()); - assertTrue(tile.columnAt(0, 0).isEmpty()); + @Test(timeout = 10000L) + public void interruptionDuringRetryPreservesInterruptAndPublishesNothing() throws Exception { + HydrologyPlanner planner = mock(HydrologyPlanner.class); + when(planner.settings()).thenReturn(emptySettings()); + HydrologyTileKey key = new HydrologyTileKey(3, -2); + AtomicReference planningThread = new AtomicReference<>(); + CountDownLatch failed = new CountDownLatch(1); + AtomicBoolean interruptPreserved = new AtomicBoolean(); + when(planner.plan(key)).thenAnswer(invocation -> { + planningThread.set(Thread.currentThread()); + throw new IllegalStateException("Inconsistent owner metadata"); + }); + doAnswer(invocation -> { + failed.countDown(); + return null; + }).when(planner).clearOwnerDrafts(); + ExecutorService caller = Executors.newSingleThreadExecutor(); + try (HydrologyTileCache cache = new HydrologyTileCache(planner, 4)) { + Future pending = caller.submit(() -> { + try { + return cache.get(key); + } catch (CancellationException failure) { + interruptPreserved.set(Thread.currentThread().isInterrupted()); + throw failure; + } + }); + assertTrue(failed.await(5L, TimeUnit.SECONDS)); + planningThread.get().interrupt(); + ExecutionException failure = assertThrows(ExecutionException.class, + () -> pending.get(2L, TimeUnit.SECONDS)); + assertTrue(failure.getCause() instanceof CancellationException); + assertTrue(interruptPreserved.get()); + assertEquals(0, cache.size()); + verify(planner, times(1)).plan(key); + } finally { + caller.shutdownNow(); + assertTrue(caller.awaitTermination(5L, TimeUnit.SECONDS)); + } } @Test @@ -1794,7 +1954,6 @@ public class HydrologyTileCacheTest { assertEquals(0, cache.size()); assertSame(tile, cache.get(key)); assertEquals(2, plans.get()); - verify(planner, org.mockito.Mockito.never()).emptyTile(any(HydrologyTileKey.class)); } @Test diff --git a/core/src/test/java/art/arcane/iris/generation/hydrology/HydrologyTileTest.java b/core/src/test/java/art/arcane/iris/generation/hydrology/HydrologyTileTest.java new file mode 100644 index 000000000..5ba1247e5 --- /dev/null +++ b/core/src/test/java/art/arcane/iris/generation/hydrology/HydrologyTileTest.java @@ -0,0 +1,65 @@ +package art.arcane.iris.generation.hydrology; + +import art.arcane.iris.generation.hydrology.cave.CavePosition; +import art.arcane.iris.generation.hydrology.cave.HydrologyCaveMode; +import art.arcane.iris.generation.hydrology.cave.HydrologyCavePlan; +import art.arcane.iris.generation.hydrology.cave.HydrologyCaveRejection; +import art.arcane.iris.generation.hydrology.cave.HydrologyCaveSource; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.OptionalLong; +import java.util.Set; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; + +public class HydrologyTileTest { + @Test + public void retainedOwnerFreezesMutableDraftCollections() { + HydrologyTileKey key = new HydrologyTileKey(0, 0); + HydrologyPoint point = new HydrologyPoint(0, 72, 0); + HydraulicSegment segment = new HydraulicSegment(3L, 4L, HydrologyFeatureType.SURFACE_POOL, + 72, 72, 2, 1, false, false, List.of(point), HydraulicChannelProfile.uniform(2D, 1D)); + RiverCourse course = new RiverCourse(4L, RiverCourseType.SURFACE_POOL, OptionalLong.empty(), + OptionalLong.empty(), "water", 1, List.of(), List.of(segment)); + HydrologyDiagnosticCandidate candidate = new HydrologyDiagnosticCandidate(9L, + HydrologyCandidateKind.SOURCE, HydrologyFeatureType.SURFACE_POOL, point, + HydrologyCandidateRejection.SOURCE_SPACING, 16); + List nodes = new ArrayList<>(); + List edges = new ArrayList<>(); + List outlets = new ArrayList<>(); + List courses = new ArrayList<>(List.of(course)); + List plans = new ArrayList<>(); + List diagnostics = new ArrayList<>(List.of(candidate)); + HydrologyOwnerDraft draft = new HydrologyOwnerDraft(key, + new HydrologyCaveCourseFilter.Result(nodes, edges, outlets, courses, plans), diagnostics, null); + HydrologyTile tile = new HydrologyTile(key, 91L, 17L, 64, List.of(), List.of(), List.of(), List.of(), + Set.of(), List.of(), List.of(), RiverFootprint.empty()) + .withResolvedOwner(new CrossTileResolvedOwner(draft, List.of())); + + courses.clear(); + diagnostics.clear(); + nodes.add(new DrainageNode(1L, 0, 0, HydrologyTerrainSample.openLand(80, 0D, "plains"), 10D, 2L)); + edges.add(new DrainageEdge(7L, 8L, 1L, 2L, 1D, 1, 0, List.of(point, new HydrologyPoint(1, 72, 0)))); + outlets.add(new RiverOutlet(2L, HydrologyFeatureType.MOUTH, 1L, point, + new HydrologyPoint(1, 63, 0), 63, true)); + CavePosition position = new CavePosition(0, 20, 0); + plans.add(new HydrologyCavePlan(new HydrologyCaveSource(4L, position, position, 20, + HydrologyCaveMode.CLOSED_COMPONENT), HydrologyCaveRejection.NONE, + Map.of(), Map.of(), OptionalLong.empty())); + + HydrologyOwnerDraft retained = tile.resolvedOwner().draft(); + assertEquals(List.of(course), retained.result().courses()); + assertEquals(List.of(candidate), retained.diagnostics()); + assertTrue(retained.result().nodes().isEmpty()); + assertTrue(retained.result().edges().isEmpty()); + assertTrue(retained.result().outlets().isEmpty()); + assertTrue(retained.result().cavePlans().isEmpty()); + assertThrows(UnsupportedOperationException.class, () -> retained.result().courses().clear()); + assertThrows(UnsupportedOperationException.class, () -> retained.diagnostics().clear()); + } +} diff --git a/core/src/test/java/art/arcane/iris/generation/hydrology/PreparedHydrologyTileStoreTest.java b/core/src/test/java/art/arcane/iris/generation/hydrology/PreparedHydrologyTileStoreTest.java index 9d8ea4cb1..611393ee6 100644 --- a/core/src/test/java/art/arcane/iris/generation/hydrology/PreparedHydrologyTileStoreTest.java +++ b/core/src/test/java/art/arcane/iris/generation/hydrology/PreparedHydrologyTileStoreTest.java @@ -9,15 +9,19 @@ import art.arcane.iris.generation.hydrology.cave.HydrologyCaveMode; import art.arcane.iris.generation.hydrology.cave.HydrologyCavePlan; import art.arcane.iris.generation.hydrology.cave.HydrologyCaveRejection; import art.arcane.iris.generation.hydrology.cave.HydrologyCaveSource; -import com.google.gson.JsonObject; -import com.google.gson.JsonParser; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; @@ -26,12 +30,16 @@ import java.util.Set; import java.util.Map; import java.util.OptionalLong; import java.util.ArrayList; -import java.util.Comparator; +import java.util.Arrays; +import java.util.Collections; import java.util.LinkedHashMap; +import java.util.stream.Stream; import java.util.zip.GZIPInputStream; import java.util.zip.GZIPOutputStream; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertSame; import static org.junit.Assert.assertThrows; import static org.junit.Assert.assertTrue; @@ -58,9 +66,73 @@ public class PreparedHydrologyTileStoreTest { assertEquals(original.cavePlans(), restored.cavePlans()); assertEquals(original.localDiagnosticCandidates(), restored.localDiagnosticCandidates()); assertEquals(original.footprint(), restored.footprint()); + assertNull(restored.resolvedOwner()); assertThrows(UnsupportedOperationException.class, () -> restored.regionalCourseIds().clear()); } + @Test + public void roundTripsExactOwnerOrderingAndUnpublishedCourses() throws Exception { + HydrologyTile original = tile(); + HydrologyTerrainSample terrain = original.nodes().getFirst().terrain(); + DrainageNode unpublishedNode = new DrainageNode(18L, 4, 0, terrain, 30D, 2L); + RiverCourse unpublishedCourse = surfaceCourse(44L); + HydrologyDiagnosticCandidate unpublishedDiagnostic = new HydrologyDiagnosticCandidate(42L, + HydrologyCandidateKind.SOURCE, HydrologyFeatureType.SURFACE_POOL, + new HydrologyPoint(4, 80, 0), HydrologyCandidateRejection.SOURCE_SPACING, 16); + HydrologyCaveCourseFilter.Result result = new HydrologyCaveCourseFilter.Result( + List.of(unpublishedNode, original.nodes().getLast(), original.nodes().getFirst()), + original.edges(), original.outlets(), List.of(unpublishedCourse, original.courses().getFirst()), + List.of()); + HydrologyFootprintCompiler compiler = new HydrologyFootprintCompiler( + HydrologyPlannerSettings.defaults(), (x, z) -> terrain); + CrossTileResolvedOwner owner = new CrossTileResolvedOwner(new HydrologyOwnerDraft(original.key(), + result, List.of(unpublishedDiagnostic, original.localDiagnosticCandidates().getFirst()), compiler), + List.of(new CrossTileRejectedCourse(surfaceCourse(60L), 44L), + new CrossTileRejectedCourse(surfaceCourse(50L), 4L))); + HydrologyTile attached = original.withResolvedOwner(owner); + assertEquals(original, attached); + assertEquals(original.hashCode(), attached.hashCode()); + assertSame(original.nodes(), attached.nodes()); + assertSame(original.courses(), attached.courses()); + assertSame(original.footprint(), attached.footprint()); + assertSame(original.features(), attached.features()); + assertSame(original.node(1L).orElseThrow(), attached.node(1L).orElseThrow()); + assertNull(attached.resolvedOwner().draft().footprintCompiler()); + assertEquals(HydrologyCacheWeights.tile(original) + HydrologyCacheWeights.owner(owner), + HydrologyCacheWeights.tile(attached)); + + PreparedHydrologyTileStore store = store(); + store.save(attached); + HydrologyTile restored = store.load(original.key()).orElseThrow(); + assertEquals(original, restored); + assertEquals(owner.withoutFootprintCompiler(), restored.resolvedOwner()); + assertSame(restored.nodes().getLast(), restored.resolvedOwner().draft().result().nodes().get(1)); + assertSame(restored.courses().getFirst(), restored.resolvedOwner().draft().result().courses().getLast()); + assertThrows(UnsupportedOperationException.class, + () -> restored.resolvedOwner().draft().result().nodes().clear()); + assertThrows(UnsupportedOperationException.class, + () -> restored.resolvedOwner().draft().diagnostics().clear()); + CrossTileResolutionContext context = new CrossTileResolutionContext(original.key(), 1L, 9); + context.remember(original.key(), owner); + context.remember(restored.key(), restored.resolvedOwner()); + } + + @Test + public void invalidOwnerReferencesAreRejected() throws Exception { + HydrologyTile original = tile(); + PreparedHydrologyTileStore store = store(); + store.save(original); + int ownerOffset = readPayload(store.file(original.key())).length; + CrossTileResolvedOwner owner = new CrossTileResolvedOwner(new HydrologyOwnerDraft(original.key(), + new HydrologyCaveCourseFilter.Result(original.nodes(), original.edges(), original.outlets(), + original.courses(), original.cavePlans()), original.localDiagnosticCandidates(), null), List.of()); + store.save(original.withResolvedOwner(owner)); + byte[] bytes = readPayload(store.file(original.key())); + ByteBuffer.wrap(bytes).putInt(ownerOffset + Integer.BYTES, Integer.MAX_VALUE); + writePayload(store.file(original.key()), bytes); + assertTrue(store.load(original.key()).isEmpty()); + } + @Test public void roundTripsNonemptyCompactCavePlan() throws Exception { CavePosition wet = new CavePosition(0, 20, 0); @@ -93,44 +165,141 @@ public class PreparedHydrologyTileStoreTest { HydrologyTile original = new HydrologyTile(new HydrologyTileKey(0, 0), 91L, 17L, 64, List.of(), List.of(), List.of(), List.of(course), Set.of(), List.of(plan), List.of(), new RiverFootprint(Map.of(RiverFootprint.pack(0, 0), column))); + original = original.withResolvedOwner(new CrossTileResolvedOwner(new HydrologyOwnerDraft(original.key(), + new HydrologyCaveCourseFilter.Result(original.nodes(), original.edges(), original.outlets(), + original.courses(), original.cavePlans()), original.localDiagnosticCandidates(), null), List.of())); PreparedHydrologyTileStore store = store(); store.save(original); HydrologyTile restored = store.load(original.key()).orElseThrow(); assertEquals(original, restored); + assertEquals(original.resolvedOwner(), restored.resolvedOwner()); + assertSame(restored.cavePlans().getFirst(), restored.resolvedOwner().draft().result().cavePlans().getFirst()); HydrologyCavePlan restoredPlan = restored.cavePlans().getFirst(); assertEquals(actions, restoredPlan.actions()); assertEquals(preconditions, restoredPlan.baselinePreconditions()); - Comparator persistedOrder = Comparator.comparingInt(CavePosition::x) - .thenComparingInt(CavePosition::z).thenComparingInt(CavePosition::y); - ArrayList expectedActions = new ArrayList<>(actions.keySet()); - expectedActions.sort(persistedOrder); - ArrayList expectedPreconditions = new ArrayList<>(preconditions.keySet()); - expectedPreconditions.sort(persistedOrder); - assertEquals(expectedActions, new ArrayList<>(restoredPlan.actions().keySet())); - assertEquals(expectedPreconditions, new ArrayList<>(restoredPlan.baselinePreconditions().keySet())); + assertEquals(new ArrayList<>(actions.keySet()), new ArrayList<>(restoredPlan.actions().keySet())); + assertEquals(new ArrayList<>(preconditions.keySet()), new ArrayList<>(restoredPlan.baselinePreconditions().keySet())); assertThrows(UnsupportedOperationException.class, () -> restoredPlan.actions().clear()); assertThrows(UnsupportedOperationException.class, () -> restoredPlan.baselinePreconditions().remove(boundary)); ArrayList published = new ArrayList<>(); restoredPlan.forEachActionIn(0, 0, 16, 16, (position, action) -> published.add(position)); - assertEquals(List.of(wet, dry), published); + assertEquals(List.of(dry, wet), published); + ByteArrayOutputStream buffer = new ByteArrayOutputStream(); + plan.writeCells(new DataOutputStream(buffer)); + byte[] valid = buffer.toByteArray(); + byte[] flags = valid.clone(); + flags[20] = (byte) 0x80; + byte[] duplicatePosition = valid.clone(); + System.arraycopy(duplicatePosition, 8, duplicatePosition, 21, 12); + byte[] duplicateOrder = valid.clone(); + System.arraycopy(duplicateOrder, 60, duplicateOrder, 64, 4); + byte[] invalidActionCount = valid.clone(); + ByteBuffer.wrap(invalidActionCount).putInt(4, 5); + for (byte[] invalid : List.of(flags, duplicatePosition, duplicateOrder, invalidActionCount)) { + DataInputStream input = new DataInputStream(new ByteArrayInputStream(invalid)); + int count = input.readInt(); + assertThrows(IOException.class, () -> HydrologyCavePlan.readCells( + plan.source(), plan.rejection(), plan.arbitrationWinnerSourceId(), input, count)); + } } @Test - public void entryRequiresRegionalOwnershipMetadata() throws Exception { + public void largeCavePlanUsesFixedSizeCellsWithoutLosingPreconditions() throws Exception { + int count = 100_000; + Map actions = new LinkedHashMap<>(); + Map preconditions = new LinkedHashMap<>(); + CaveVoxelPrecondition baseline = new CaveVoxelPrecondition(CaveVoxel.SOLID, false); + for (int index = 0; index < count; index++) { + CavePosition position = new CavePosition(index % 256, index / 65536, index / 256 % 256); + preconditions.put(position, baseline); + if (index % 2 == 0) { + actions.put(position, HydrologyCaveAction.SEAL_GUARD); + } + } + HydrologyCaveSource source = new HydrologyCaveSource(4L, new CavePosition(0, 0, 0), + new CavePosition(1, 0, 0), 20, HydrologyCaveMode.CLOSED_COMPONENT); + HydrologyCavePlan original = new HydrologyCavePlan(source, HydrologyCaveRejection.NONE, + actions, preconditions, OptionalLong.empty()); + ByteArrayOutputStream buffer = new ByteArrayOutputStream(); + original.writeCells(new DataOutputStream(buffer)); + assertEquals(8 + 17 * count, buffer.size()); + DataInputStream input = new DataInputStream(new ByteArrayInputStream(buffer.toByteArray())); + HydrologyCavePlan restored = HydrologyCavePlan.readCells(source, HydrologyCaveRejection.NONE, + OptionalLong.empty(), input, input.readInt()); + assertEquals(original, restored); + assertEquals(-1, input.read()); + } + + @Test(timeout = 2000) + public void invalidCollectionLengthsFailBeforeAllocation() throws Exception { HydrologyTile original = tile(); PreparedHydrologyTileStore store = store(); store.save(original); Path file = store.file(original.key()); - JsonObject persisted; - try (InputStream input = new GZIPInputStream(Files.newInputStream(file))) { - persisted = JsonParser.parseString(new String(input.readAllBytes(), StandardCharsets.UTF_8)).getAsJsonObject(); - } - persisted.remove("regionalCourseIds"); - try (OutputStream output = new GZIPOutputStream(Files.newOutputStream(file))) { - output.write(persisted.toString().getBytes(StandardCharsets.UTF_8)); + byte[] bytes = readPayload(file); + DataInputStream input = new DataInputStream(new ByteArrayInputStream(bytes)); + input.readInt(); + input.readInt(); + input.readUTF(); + input.skipNBytes(28); + int countOffset = bytes.length - input.available(); + for (int count : new int[]{-1, Integer.MAX_VALUE, 1_000_000}) { + byte[] invalid = Arrays.copyOf(bytes, countOffset + Integer.BYTES); + ByteBuffer.wrap(invalid).putInt(countOffset, count); + writePayload(file, invalid); + assertTrue(store.load(original.key()).isEmpty()); } + } + @Test + public void truncatedAndTrailingPayloadsAreRejected() throws Exception { + HydrologyTile original = tile(); + PreparedHydrologyTileStore store = store(); + store.save(original); + Path file = store.file(original.key()); + byte[] bytes = readPayload(file); + writePayload(file, Arrays.copyOf(bytes, bytes.length - 1)); assertTrue(store.load(original.key()).isEmpty()); + writePayload(file, Arrays.copyOf(bytes, bytes.length + 1)); + assertTrue(store.load(original.key()).isEmpty()); + } + + @Test + public void presenceCheckDoesNotDecodeTileContents() throws Exception { + HydrologyTile original = tile(); + PreparedHydrologyTileStore store = store(); + store.save(original); + Files.writeString(store.file(original.key()), "invalid", StandardCharsets.UTF_8); + assertTrue(store.contains(original.key())); + assertTrue(store.load(original.key()).isEmpty()); + } + + @Test + public void allocationLimitDoesNotReplaceAnExistingTile() throws Exception { + HydrologyTile original = tile(); + PreparedHydrologyTileStore store = store(); + store.save(original); + HydrologyTile oversized = new HydrologyTile(original.key(), original.worldSeed(), + original.settingsFingerprint(), original.tileSize(), original.nodes(), original.edges(), + original.outlets(), original.courses(), original.regionalCourseIds(), original.cavePlans(), + Collections.nCopies(600_000, original.localDiagnosticCandidates().getFirst()), original.footprint()); + assertThrows(IOException.class, () -> store.save(oversized)); + assertEquals(original, store.load(original.key()).orElseThrow()); + try (Stream files = Files.list(store.file(original.key()).getParent())) { + assertEquals(List.of(store.file(original.key())), files.toList()); + } + } + + private byte[] readPayload(Path file) throws Exception { + try (InputStream input = new GZIPInputStream(Files.newInputStream(file))) { + return input.readAllBytes(); + } + } + + private void writePayload(Path file, byte[] bytes) throws Exception { + try (OutputStream output = new GZIPOutputStream(Files.newOutputStream(file))) { + output.write(bytes); + } } @Test @@ -191,6 +360,12 @@ public class PreparedHydrologyTileStoreTest { HydrologyTerrainSample terrain = HydrologyTerrainSample.openLand(80, 0.25D, "plains") .withSurfacePolicy(new SurfaceRiverPolicy("region:tropical", 8D, 160, 3, 3, 4, 128, 24)); DrainageNode node = new DrainageNode(1L, 0, 0, terrain, 10D, 2L); + DrainageNode upstream = new DrainageNode(8L, 2, 0, terrain, 20D, 2L); + DrainageEdge edge = new DrainageEdge(7L, 8L, 1L, 2L, 1D, 1, 0, + List.of(new HydrologyPoint(2, 80, 0), new HydrologyPoint(0, 80, 0))); + HydrologyDiagnosticCandidate candidate = new HydrologyDiagnosticCandidate(9L, + HydrologyCandidateKind.SOURCE, HydrologyFeatureType.SURFACE_POOL, + new HydrologyPoint(5, 80, 0), HydrologyCandidateRejection.SOURCE_SPACING, 16); RiverOutlet outlet = new RiverOutlet( 2L, HydrologyFeatureType.MOUTH, @@ -220,7 +395,7 @@ public class PreparedHydrologyTileStoreTest { OptionalLong.of(2L), "default", 1, - List.of(), + List.of(edge), List.of(segment) ); HydrologyFeatureRef feature = new HydrologyFeatureRef( @@ -270,14 +445,22 @@ public class PreparedHydrologyTileStoreTest { 91L, 17L, 64, - List.of(node), - List.of(), + List.of(node, upstream), + List.of(edge), List.of(outlet), List.of(course), Set.of(course.id()), List.of(), - List.of(), + List.of(candidate), new RiverFootprint(Map.of(RiverFootprint.pack(0, 0), column)) ); } + + private RiverCourse surfaceCourse(long id) { + HydraulicSegment segment = new HydraulicSegment(id + 1L, id, HydrologyFeatureType.SURFACE_POOL, + 72, 72, 4, 2, false, false, List.of(new HydrologyPoint(0, 72, 0)), + HydraulicChannelProfile.uniform(4D, 2D)); + return new RiverCourse(id, RiverCourseType.SURFACE, OptionalLong.of(1L), OptionalLong.of(2L), + "default", 1, List.of(), List.of(segment)); + } } diff --git a/core/src/test/java/art/arcane/iris/generation/mantle/MatterGeneratorConcurrencyTest.java b/core/src/test/java/art/arcane/iris/generation/mantle/MatterGeneratorConcurrencyTest.java index 117460431..d1727ce4a 100644 --- a/core/src/test/java/art/arcane/iris/generation/mantle/MatterGeneratorConcurrencyTest.java +++ b/core/src/test/java/art/arcane/iris/generation/mantle/MatterGeneratorConcurrencyTest.java @@ -7,6 +7,7 @@ import art.arcane.iris.testsupport.Await; import art.arcane.iris.testsupport.PlatformBinding; import art.arcane.iris.generation.context.ChunkContext; import art.arcane.iris.generation.context.IrisContext; +import art.arcane.iris.spi.LogLevel; import art.arcane.volmlib.util.mantle.flag.MantleFlag; import art.arcane.volmlib.util.mantle.flag.ReservedFlag; import art.arcane.volmlib.util.mantle.runtime.Mantle; @@ -46,6 +47,8 @@ import static org.junit.Assert.assertThrows; import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.ArgumentMatchers.startsWith; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; @@ -420,12 +423,21 @@ public class MatterGeneratorConcurrencyTest { } @Test - public void timedOutRunningComponentRetainsWriterAndSharedClaimUntilItExits() throws Exception { + public void slowRunningComponentRetainsWriterAndSharedClaimAndCompletesSuccessfully() throws Exception { GeneratorFixture fixture = new GeneratorFixture(true); CountDownLatch started = new CountDownLatch(1); CountDownLatch release = new CountDownLatch(1); CountDownLatch writersAcquired = new CountDownLatch(2); - CountDownLatch cancelled = new CountDownLatch(1); + CountDownLatch warned = new CountDownLatch(1); + CountDownLatch polls = new CountDownLatch(6); + AtomicInteger warnings = new AtomicInteger(); + AtomicReference diagnostic = new AtomicReference<>(); + doAnswer(invocation -> { + warnings.incrementAndGet(); + diagnostic.set(invocation.getArgument(1)); + warned.countDown(); + return null; + }).when(PLATFORM.platform()).log(eq(LogLevel.WARN), startsWith("Mantle component OBJECT at 0,0 is still pending")); AtomicInteger runs = new AtomicInteger(); MantleChunk chunk = fixture.mantle.getChunk(0, 0); when(chunk.use()).thenAnswer(invocation -> { @@ -448,9 +460,9 @@ public class MatterGeneratorConcurrencyTest { MatterGenerator.MatterTaskKey key = new MatterGenerator.MatterTaskKey(fixture.mantle, 0, 0, ReservedFlag.OBJECT); MatterGenerator.MatterComponentTask task = MatterGenerator.IN_FLIGHT_COMPONENTS.get(key); Future submission = mock(Future.class); - when(submission.cancel(true)).thenAnswer(invocation -> { - cancelled.countDown(); - return true; + when(submission.isDone()).thenAnswer(invocation -> { + polls.countDown(); + return false; }); task.setSubmission(submission); CompletableFuture second = MultiBurst.burst.completeValueAsync(() -> { @@ -458,45 +470,86 @@ public class MatterGeneratorConcurrencyTest { return null; }); CompletableFuture timeout = CompletableFuture.runAsync(() -> task.await(0L)); + CompletableFuture otherTimeout = CompletableFuture.runAsync(() -> task.await(0L)); try { await(writersAcquired); - await(cancelled); + await(warned); + await(polls); assertSame(task, MatterGenerator.IN_FLIGHT_COMPONENTS.get(key)); assertFalse(first.isDone()); assertFalse(second.isDone()); assertFalse(timeout.isDone()); + assertFalse(otherTimeout.isDone()); + assertEquals(1, warnings.get()); + assertTrue(diagnostic.get().contains("Worker=")); + assertTrue(diagnostic.get().contains("runningForMs=")); + assertTrue(diagnostic.get().contains("\n\tat ")); + verify(submission, never()).cancel(anyBoolean()); verify(chunk, never()).release(); assertEquals(1, runs.get()); } finally { release.countDown(); } - assertThrows(ExecutionException.class, () -> timeout.get(5L, TimeUnit.SECONDS)); - assertThrows(ExecutionException.class, () -> first.get(5L, TimeUnit.SECONDS)); - assertThrows(ExecutionException.class, () -> second.get(5L, TimeUnit.SECONDS)); + timeout.get(5L, TimeUnit.SECONDS); + otherTimeout.get(5L, TimeUnit.SECONDS); + first.get(5L, TimeUnit.SECONDS); + second.get(5L, TimeUnit.SECONDS); verify(chunk, times(2)).release(); assertFalse(MatterGenerator.IN_FLIGHT_COMPONENTS.containsKey(key)); } @Test - public void queuedTimeoutPreventsLateExecutionAndAllowsANewClaim() { + public void slowQueuedComponentCanStartAndCompleteSuccessfully() throws Exception { GeneratorFixture fixture = new GeneratorFixture(); - MatterGenerator.MatterTaskKey key = new MatterGenerator.MatterTaskKey(fixture.mantle, 0, 0, ReservedFlag.OBJECT); + MatterGenerator.MatterTaskKey key = new MatterGenerator.MatterTaskKey(fixture.mantle, 1, 0, ReservedFlag.OBJECT); MatterGenerator.MatterComponentTask task = new MatterGenerator.MatterComponentTask(key); MatterGenerator.IN_FLIGHT_COMPONENTS.put(key, task); Future submission = mock(Future.class); task.setSubmission(submission); - - CompletionException failure = assertThrows(CompletionException.class, () -> task.await(0L)); - - assertTrue(failure.getCause().getMessage().contains("did not complete")); - assertFalse(task.start()); + CountDownLatch warned = new CountDownLatch(1); + AtomicReference diagnostic = new AtomicReference<>(); + doAnswer(invocation -> { + diagnostic.set(invocation.getArgument(1)); + warned.countDown(); + return null; + }).when(PLATFORM.platform()).log(eq(LogLevel.WARN), startsWith("Mantle component OBJECT at 1,0 is still pending")); + CompletableFuture completion = CompletableFuture.runAsync(() -> task.await(0L)); + try { + await(warned); + assertTrue(diagnostic.get().contains("Task is queued")); + assertFalse(completion.isDone()); + assertSame(task, MatterGenerator.IN_FLIGHT_COMPONENTS.get(key)); + verify(submission, never()).cancel(anyBoolean()); + assertTrue(task.start()); + } finally { + task.finish(null); + } + completion.get(5L, TimeUnit.SECONDS); assertFalse(MatterGenerator.IN_FLIGHT_COMPONENTS.containsKey(key)); - verify(submission).cancel(true); - MatterGenerator.MatterComponentTask replacement = new MatterGenerator.MatterComponentTask(key); - assertNull(MatterGenerator.IN_FLIGHT_COMPONENTS.putIfAbsent(key, replacement)); - task.finish(null); - assertSame(replacement, MatterGenerator.IN_FLIGHT_COMPONENTS.get(key)); - replacement.finish(null); + } + + @Test + public void slowComponentPreservesItsActualFailure() throws Exception { + GeneratorFixture fixture = new GeneratorFixture(); + MatterGenerator.MatterTaskKey key = new MatterGenerator.MatterTaskKey(fixture.mantle, 2, 0, ReservedFlag.OBJECT); + MatterGenerator.MatterComponentTask task = new MatterGenerator.MatterComponentTask(key); + CountDownLatch warned = new CountDownLatch(1); + doAnswer(invocation -> { + warned.countDown(); + return null; + }).when(PLATFORM.platform()).log(eq(LogLevel.WARN), startsWith("Mantle component OBJECT at 2,0 is still pending")); + IllegalStateException failure = new IllegalStateException("Object generation failed"); + assertTrue(task.start()); + CompletableFuture completion = CompletableFuture.runAsync(() -> task.await(0L)); + try { + await(warned); + assertFalse(completion.isDone()); + } finally { + task.finish(failure); + } + ExecutionException observed = assertThrows(ExecutionException.class, + () -> completion.get(5L, TimeUnit.SECONDS)); + assertSame(failure, observed.getCause()); } @Test diff --git a/core/src/test/java/art/arcane/iris/generation/mantle/ObjectDestinationTransactionTest.java b/core/src/test/java/art/arcane/iris/generation/mantle/ObjectDestinationTransactionTest.java index 2a905b8f1..a42259f1f 100644 --- a/core/src/test/java/art/arcane/iris/generation/mantle/ObjectDestinationTransactionTest.java +++ b/core/src/test/java/art/arcane/iris/generation/mantle/ObjectDestinationTransactionTest.java @@ -82,7 +82,7 @@ public class ObjectDestinationTransactionTest { } ObjectSourcePlan plan = source.sourcePlanSince(0); assertEquals(16, source.mutationCheckpoint()); - assertEquals(35, plan.mutationWeight()); + assertEquals(Integer.MAX_VALUE, plan.estimatedRetainedBytes()); for (int[] position : positions) { int x = position[0]; int z = position[1]; diff --git a/core/src/test/java/art/arcane/iris/generation/mantle/ObjectSourcePlanCacheTest.java b/core/src/test/java/art/arcane/iris/generation/mantle/ObjectSourcePlanCacheTest.java index ec516e2bf..a25d56646 100644 --- a/core/src/test/java/art/arcane/iris/generation/mantle/ObjectSourcePlanCacheTest.java +++ b/core/src/test/java/art/arcane/iris/generation/mantle/ObjectSourcePlanCacheTest.java @@ -22,14 +22,14 @@ import static org.junit.Assert.assertTrue; public class ObjectSourcePlanCacheTest { @Test - public void defaultCapacityRetainsNeighboringSourceWorkingSetWithoutRebuilding() { - ObjectSourcePlanCache cache = new ObjectSourcePlanCache(); + public void budgetRetainsNeighboringSourceWorkingSetWithoutRebuilding() { + ObjectSourcePlanCache cache = new ObjectSourcePlanCache(1024L * 1024L); AtomicInteger builds = new AtomicInteger(); ObjectDestinationTransaction.DataKey key = new ObjectDestinationTransaction.DataKey( 0, 4, 0, String.class); ObjectDestinationTransaction.Mutation mutation = new ObjectDestinationTransaction.SetMutation(key, "marker"); - ObjectSourcePlan expected = new ObjectSourcePlan(Collections.nCopies(2048, mutation)); - assertTrue(expected.mutationWeight() * 1024L > 1_048_576L); + ObjectSourcePlan expected = new ObjectSourcePlan(Collections.nCopies(2, mutation)); + assertTrue(expected.estimatedRetainedBytes() * 1024L < 1_048_576L); for (int sweep = 0; sweep < 3; sweep++) { for (int x = -16; x < 16; x++) { for (int z = -16; z < 16; z++) { @@ -53,7 +53,7 @@ public class ObjectSourcePlanCacheTest { @Test public void unrelatedCoordinatesBuildWhileAnotherPlanIsPending() throws Exception { - ObjectSourcePlanCache cache = new ObjectSourcePlanCache(64L); + ObjectSourcePlanCache cache = new ObjectSourcePlanCache(4096L); ObjectSourcePlan first = planAt(1); ObjectSourcePlan second = planAt(2); CountDownLatch firstEntered = new CountDownLatch(1); @@ -88,7 +88,7 @@ public class ObjectSourcePlanCacheTest { @Test public void concurrentRequestsBuildOneSourcePlan() throws Exception { - ObjectSourcePlanCache cache = new ObjectSourcePlanCache(64L); + ObjectSourcePlanCache cache = new ObjectSourcePlanCache(4096L); ObjectSourcePlan expected = new ObjectSourcePlan(List.of()); AtomicInteger builds = new AtomicInteger(); CountDownLatch ready = new CountDownLatch(8); @@ -119,10 +119,10 @@ public class ObjectSourcePlanCacheTest { } @Test - public void cacheEvictsByMutationWeightAndCanBeCleared() { + public void cacheEvictsByRetainedBytesAndCanBeCleared() { ObjectSourcePlan first = planAt(0); ObjectSourcePlan second = planAt(1); - ObjectSourcePlanCache cache = new ObjectSourcePlanCache(first.mutationWeight()); + ObjectSourcePlanCache cache = new ObjectSourcePlanCache(first.estimatedRetainedBytes()); cache.get(0, 0, () -> first); cache.get(1, 0, () -> second); @@ -133,13 +133,13 @@ public class ObjectSourcePlanCacheTest { } @Test - public void cacheRejectsNonPositiveMutationCapacity() { + public void cacheRejectsNonPositiveByteCapacity() { assertThrows(IllegalArgumentException.class, () -> new ObjectSourcePlanCache(0L)); } @Test public void clearingPendingWorkKeepsNewGenerationIsolated() throws Exception { - ObjectSourcePlanCache cache = new ObjectSourcePlanCache(64L); + ObjectSourcePlanCache cache = new ObjectSourcePlanCache(4096L); ObjectSourcePlan old = planAt(1); ObjectSourcePlan replacement = planAt(2); CountDownLatch entered = new CountDownLatch(1); @@ -164,7 +164,7 @@ public class ObjectSourcePlanCacheTest { @Test public void waitingForkJoinWorkerAllowsQueuedDependencyToRun() throws Exception { - ObjectSourcePlanCache cache = new ObjectSourcePlanCache(64L); + ObjectSourcePlanCache cache = new ObjectSourcePlanCache(4096L); ObjectSourcePlan expected = planAt(1); CountDownLatch entered = new CountDownLatch(1); CountDownLatch waiting = new CountDownLatch(1); @@ -194,7 +194,7 @@ public class ObjectSourcePlanCacheTest { @Test public void failedAndNullBuildsAreRetryableAndRecursionFailsImmediately() { - ObjectSourcePlanCache cache = new ObjectSourcePlanCache(64L); + ObjectSourcePlanCache cache = new ObjectSourcePlanCache(4096L); IllegalArgumentException failure = new IllegalArgumentException("Build failed"); assertSame(failure, assertThrows(IllegalArgumentException.class, () -> cache.get(7, 8, () -> { throw failure; }))); @@ -205,6 +205,43 @@ public class ObjectSourcePlanCacheTest { assertSame(expected, cache.get(7, 8, () -> expected)); } + @Test + public void byteBudgetScalesWithHeapAndHasAnUpperBound() { + assertEquals(64L * 1024L * 1024L, ObjectSourcePlanCache.retainedByteBudget(4L * 1024L * 1024L * 1024L)); + assertEquals(16L * 1024L * 1024L, ObjectSourcePlanCache.retainedByteBudget(1024L * 1024L * 1024L)); + assertEquals(256L * 1024L * 1024L, ObjectSourcePlanCache.retainedByteBudget(Long.MAX_VALUE)); + } + + @Test + public void emptyPlansConsumeTheByteBudget() { + ObjectSourcePlan empty = new ObjectSourcePlan(List.of()); + ObjectSourcePlanCache cache = new ObjectSourcePlanCache(4L * empty.estimatedRetainedBytes()); + for (int chunk = 0; chunk < 1000; chunk++) { + cache.get(chunk, 0, () -> empty); + } + assertTrue(cache.estimatedSize() <= 4L); + } + + @Test + public void oversizedAndUnmeasuredPlansAreReturnedWithoutRetention() { + ObjectSourcePlan large = planAt(1); + ObjectSourcePlan unmeasured = new ObjectSourcePlan(List.of(new ObjectDestinationTransaction.SetMutation( + new ObjectDestinationTransaction.DataKey(0, 4, 0, Object.class), new Object()))); + ObjectSourcePlanCache cache = new ObjectSourcePlanCache(large.estimatedRetainedBytes() - 1L); + assertSame(large, cache.get(1, 0, () -> large)); + assertSame(unmeasured, cache.get(2, 0, () -> unmeasured)); + assertEquals(0L, cache.estimatedSize()); + AtomicInteger builds = new AtomicInteger(); + cache.get(1, 0, () -> { + builds.incrementAndGet(); + return large; + }); + assertEquals(1, builds.get()); + ObjectSourcePlanCache roomy = new ObjectSourcePlanCache(Long.MAX_VALUE); + assertSame(unmeasured, roomy.get(2, 0, () -> unmeasured)); + assertEquals(0L, roomy.estimatedSize()); + } + private static void await(CountDownLatch latch) { try { assertTrue(latch.await(10L, TimeUnit.SECONDS)); diff --git a/core/src/test/java/art/arcane/iris/generation/mantle/ObjectSourcePlanTest.java b/core/src/test/java/art/arcane/iris/generation/mantle/ObjectSourcePlanTest.java index 353ff7daf..3ebad7320 100644 --- a/core/src/test/java/art/arcane/iris/generation/mantle/ObjectSourcePlanTest.java +++ b/core/src/test/java/art/arcane/iris/generation/mantle/ObjectSourcePlanTest.java @@ -45,11 +45,19 @@ public class ObjectSourcePlanTest { ObjectSourcePlan compact = new ObjectSourcePlan(List.of(first, nearby)); ObjectSourcePlan scattered = new ObjectSourcePlan(List.of(first, remote)); - assertTrue(compact.mutationWeight() > 1 + first.weight() + nearby.weight()); - assertTrue(scattered.mutationWeight() > compact.mutationWeight()); + assertTrue(compact.estimatedRetainedBytes() > new ObjectSourcePlan(List.of()).estimatedRetainedBytes()); + assertTrue(scattered.estimatedRetainedBytes() > compact.estimatedRetainedBytes()); assertThrows(NullPointerException.class, () -> new ObjectSourcePlan(Arrays.asList(first, null))); } + @Test + public void retainedBytesIncludeVariableLengthMarkers() { + ObjectDestinationTransaction.DataKey key = new ObjectDestinationTransaction.DataKey(0, 4, 0, String.class); + ObjectSourcePlan shortMarker = new ObjectSourcePlan(List.of(new ObjectDestinationTransaction.SetMutation(key, "a"))); + ObjectSourcePlan longMarker = new ObjectSourcePlan(List.of(new ObjectDestinationTransaction.SetMutation(key, "a".repeat(4096)))); + assertTrue(longMarker.estimatedRetainedBytes() - shortMarker.estimatedRetainedBytes() >= 8190); + } + private static ObjectDestinationTransaction.Mutation mutationAt(int x, int z) { return new ObjectDestinationTransaction.SetMutation( new ObjectDestinationTransaction.DataKey(x, 4, z, String.class), "marker"); diff --git a/core/src/test/java/art/arcane/iris/world/pregen/AsyncPregenMethodConcurrencyCapTest.java b/core/src/test/java/art/arcane/iris/world/pregen/AsyncPregenMethodConcurrencyCapTest.java index 66d8e4322..00fbfdb07 100644 --- a/core/src/test/java/art/arcane/iris/world/pregen/AsyncPregenMethodConcurrencyCapTest.java +++ b/core/src/test/java/art/arcane/iris/world/pregen/AsyncPregenMethodConcurrencyCapTest.java @@ -16,6 +16,16 @@ import static org.junit.Assert.assertSame; import static org.junit.Assert.assertTrue; public class AsyncPregenMethodConcurrencyCapTest { + @Test + public void outstandingChunksReserveHeapForGenerationDependencies() { + assertEquals(16, AsyncPregenMethod.heapLimitedConcurrency(256, 4L << 30)); + assertEquals(32, AsyncPregenMethod.heapLimitedConcurrency(256, 8L << 30)); + assertEquals(64, AsyncPregenMethod.heapLimitedConcurrency(256, 16L << 30)); + assertEquals(8, AsyncPregenMethod.heapLimitedConcurrency(8, 16L << 30)); + assertEquals(1, AsyncPregenMethod.heapLimitedConcurrency(256, 0L)); + assertEquals(256, AsyncPregenMethod.heapLimitedConcurrency(256, Long.MAX_VALUE)); + } + @Test public void pregenWorkerTargetExpandsOnlyOptedInHighHeapJobs() { assertEquals(16, AsyncPregenMethod.pregenWorkerThreadTarget(16, (16L << 30) - 1L, true)); diff --git a/core/src/test/java/art/arcane/iris/world/pregen/AsyncPregenServerSaveOwnershipTest.java b/core/src/test/java/art/arcane/iris/world/pregen/AsyncPregenServerSaveOwnershipTest.java index bbab8e99b..74ee55644 100644 --- a/core/src/test/java/art/arcane/iris/world/pregen/AsyncPregenServerSaveOwnershipTest.java +++ b/core/src/test/java/art/arcane/iris/world/pregen/AsyncPregenServerSaveOwnershipTest.java @@ -6,7 +6,6 @@ import art.arcane.iris.world.IrisToolbelt; import art.arcane.iris.platform.bukkit.BukkitPlatform; import art.arcane.iris.spi.IrisLogging; import art.arcane.iris.world.task.J; -import art.arcane.volmlib.util.collection.KSet; import org.bukkit.Chunk; import org.bukkit.World; import org.bukkit.plugin.Plugin; @@ -31,7 +30,9 @@ import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; +import java.util.function.BooleanSupplier; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertEquals; @@ -240,7 +241,7 @@ public class AsyncPregenServerSaveOwnershipTest { try (SchedulingContext ignored = new SchedulingContext(fixture)) { Future flushed = executor.submit(() -> { try (SchedulingContext context = new SchedulingContext(fixture)) { - invoke(fixture.method, "flushAllRemainingChunks"); + fixture.method.reclaimMemory(); } return null; }); @@ -260,6 +261,57 @@ public class AsyncPregenServerSaveOwnershipTest { } } + @Test + public void emptyPublicReclaimDoesNotScheduleWorldIo() throws Exception { + Fixture fixture = new Fixture(); + fixture.chunks.clear(); + fixture.set("pendingEvictions", new ConcurrentLinkedQueue>()); + try (SchedulingContext context = new SchedulingContext(fixture)) { + fixture.method.reclaimMemory(); + context.scheduling.verifyNoInteractions(); + verifyNoInteractions(fixture.binding); + } + } + + @Test + public void publicReclaimFlushesCompletedChunksWithoutPriorEvictions() throws Exception { + Fixture fixture = new Fixture(); + fixture.set("pendingEvictions", new ConcurrentLinkedQueue>()); + try (SchedulingContext context = new SchedulingContext(fixture)) { + fixture.method.reclaimMemory(); + assertTrue(fixture.chunks.isEmpty()); + verify(fixture.binding).saveAndUnloadChunk(fixture.world, 2, 3); + verify(fixture.binding, times(2)).flushChunkIO(fixture.world); + } + } + + @Test + public void publicReclaimWaitsForEvictionsWithoutNewChunks() throws Exception { + Fixture fixture = new Fixture(); + fixture.chunks.clear(); + CountDownLatch started = new CountDownLatch(1); + ExecutorService executor = Executors.newSingleThreadExecutor(); + try (SchedulingContext owner = new SchedulingContext(fixture)) { + Future reclaimed = executor.submit(() -> { + try (SchedulingContext context = new SchedulingContext(fixture)) { + started.countDown(); + fixture.method.reclaimMemory(); + } + }); + assertTrue(started.await(5L, TimeUnit.SECONDS)); + assertThrows(TimeoutException.class, () -> reclaimed.get(30L, TimeUnit.MILLISECONDS)); + verifyNoInteractions(fixture.binding); + fixture.pending.complete(null); + reclaimed.get(5L, TimeUnit.SECONDS); + owner.scheduling.verifyNoInteractions(); + verify(fixture.binding).flushChunkIO(fixture.world); + } finally { + fixture.pending.complete(null); + executor.shutdownNow(); + assertTrue(executor.awaitTermination(5L, TimeUnit.SECONDS)); + } + } + @Test public void nativeFlushRunsOnThePregenFlushWorkerNotTheSharedIoPool() throws Exception { Fixture fixture = new Fixture(); @@ -280,10 +332,110 @@ public class AsyncPregenServerSaveOwnershipTest { } } + @Test + public void completedRegionsUnloadWithoutWaitingForTheirNeighbors() throws Exception { + Fixture fixture = new Fixture(); + fixture.chunks.clear(); + try (SchedulingContext context = new SchedulingContext(fixture)) { + for (int region = 0; region < 128; region++) { + long key = (long) region << 32; + fixture.pendingRegions.put(key, new AtomicInteger(2)); + invoke(fixture.method, "onChunkCompleted", region << 5, 0, fixture.chunk); + assertEquals(1, fixture.pendingRegions.get(key).get()); + fixture.method.onRegionSubmitted(region, 0); + assertTrue(fixture.pendingRegions.isEmpty()); + assertTrue(fixture.chunks.isEmpty()); + } + verify(fixture.binding, times(128)).saveAndUnloadChunk(fixture.world, 2, 3); + } + } + + @Test + public void partialRegionEvictionKeepsPendingSubmissionsAndAcceptsLaterCompletions() throws Exception { + Fixture fixture = new Fixture(); + AtomicInteger pendingChunks = new AtomicInteger(2); + fixture.pendingRegions.put(0L, pendingChunks); + try (SchedulingContext context = new SchedulingContext(fixture)) { + CompletableFuture eviction = (CompletableFuture) invoke(fixture.method, "evictRegion", 0L); + eviction.get(5L, TimeUnit.SECONDS); + assertSame(pendingChunks, fixture.pendingRegions.get(0L)); + assertEquals(2, pendingChunks.get()); + assertTrue(fixture.chunks.isEmpty()); + + invoke(fixture.method, "onChunkCompleted", 2, 3, fixture.chunk); + assertEquals(1, pendingChunks.get()); + assertSame(fixture.chunk, fixture.chunks.get(0L).peek()); + fixture.method.onRegionSubmitted(0, 0); + assertTrue(fixture.pendingRegions.isEmpty()); + assertTrue(fixture.chunks.isEmpty()); + verify(fixture.binding, times(2)).saveAndUnloadChunk(fixture.world, 2, 3); + } + } + + @Test + public void completionDuringHeapWaitEvictsWithoutWaitingForRegionSubmission() throws Exception { + Fixture fixture = new Fixture(); + fixture.chunks.clear(); + AtomicInteger pendingChunks = new AtomicInteger(2); + fixture.pendingRegions.put(0L, pendingChunks); + try (SchedulingContext context = new SchedulingContext(fixture); + MockedStatic pressure = mockStatic(MantleHeapPressure.class)) { + pressure.when(MantleHeapPressure::overHighWater).thenReturn(true); + invoke(fixture.method, "onChunkCompleted", 2, 3, fixture.chunk); + assertEquals(1, pendingChunks.get()); + assertSame(pendingChunks, fixture.pendingRegions.get(0L)); + assertTrue(fixture.chunks.isEmpty()); + verify(fixture.binding).saveAndUnloadChunk(fixture.world, 2, 3); + fixture.method.onRegionSubmitted(0, 0); + assertTrue(fixture.pendingRegions.isEmpty()); + } + } + + @Test + public void heapPressureStartingAfterInitialCheckStillReclaimsCompletedChunks() throws Exception { + Fixture fixture = new Fixture(); + fixture.pending.complete(null); + fixture.set("closing", new AtomicBoolean(true)); + PregenMantleBackpressure backpressure = mock(PregenMantleBackpressure.class); + fixture.set("backpressure", backpressure); + doAnswer(invocation -> { + BooleanSupplier pressure = invocation.getArgument(0); + Runnable reclaim = invocation.getArgument(1); + assertTrue(pressure.getAsBoolean()); + reclaim.run(); + return null; + }).when(backpressure).awaitHeapHeadroom(any(BooleanSupplier.class), any(Runnable.class)); + try (SchedulingContext context = new SchedulingContext(fixture); + MockedStatic pressure = mockStatic(MantleHeapPressure.class)) { + pressure.when(MantleHeapPressure::overHighWater).thenReturn(false, true); + fixture.method.generateChunk(2, 3, mock(PregenListener.class)); + assertTrue(fixture.chunks.isEmpty()); + verify(fixture.binding).saveAndUnloadChunk(fixture.world, 2, 3); + pressure.verify(MantleHeapPressure::requestPanicReclaim); + } + } + + @Test + public void emptyPressurePollDoesNotScheduleWorldIo() throws Exception { + Fixture fixture = new Fixture(); + fixture.chunks.clear(); + fixture.set("pendingEvictions", new ConcurrentLinkedQueue>()); + try (SchedulingContext context = new SchedulingContext(fixture); + MockedStatic pressure = mockStatic(MantleHeapPressure.class)) { + invoke(fixture.method, "reclaimHeapPressure"); + verifyNoInteractions(fixture.binding); + context.scheduling.verifyNoInteractions(); + pressure.verify(MantleHeapPressure::requestPanicReclaim); + } + } + private static Object invoke(AsyncPregenMethod target, String name, Object... arguments) throws Exception { - Method method = arguments.length == 0 - ? AsyncPregenMethod.class.getDeclaredMethod(name) - : AsyncPregenMethod.class.getDeclaredMethod(name, long.class); + Class[] parameterTypes = new Class[arguments.length]; + for (int index = 0; index < arguments.length; index++) { + parameterTypes[index] = arguments[index] instanceof Integer ? int.class + : arguments[index] instanceof Long ? long.class : Chunk.class; + } + Method method = AsyncPregenMethod.class.getDeclaredMethod(name, parameterTypes); method.setAccessible(true); try { return method.invoke(target, arguments); @@ -299,6 +451,7 @@ public class AsyncPregenServerSaveOwnershipTest { private final INMSBinding binding = mock(INMSBinding.class); private final ConcurrentHashMap> chunks = new ConcurrentHashMap<>(); private final CompletableFuture pending = new CompletableFuture<>(); + private final ConcurrentHashMap pendingRegions = new ConcurrentHashMap<>(); private Fixture() throws Exception { Queue region = new ConcurrentLinkedQueue<>(); @@ -308,9 +461,8 @@ public class AsyncPregenServerSaveOwnershipTest { evictions.add(pending); set("world", world); set("regionChunks", chunks); - set("regionPending", new ConcurrentHashMap<>()); + set("regionPending", pendingRegions); set("pendingEvictions", evictions); - set("evictedRegions", new KSet<>()); set("chunkIoExecutor", (Executor) Runnable::run); when(chunk.getX()).thenReturn(2); when(chunk.getZ()).thenReturn(3); diff --git a/core/src/test/java/art/arcane/iris/world/pregen/CachedPregenMethodCompletionTest.java b/core/src/test/java/art/arcane/iris/world/pregen/CachedPregenMethodCompletionTest.java index 80042e0c0..2bd85cbfe 100644 --- a/core/src/test/java/art/arcane/iris/world/pregen/CachedPregenMethodCompletionTest.java +++ b/core/src/test/java/art/arcane/iris/world/pregen/CachedPregenMethodCompletionTest.java @@ -106,7 +106,7 @@ public class CachedPregenMethodCompletionTest { } @Test - public void pregenStartReachesTheSelectedMethodThroughEveryWrapper() throws Exception { + public void pregenLifecycleHooksReachTheSelectedMethodThroughEveryWrapper() throws Exception { Constructor selectorConstructor = AsyncOrMedievalPregenMethod.class .getDeclaredConstructor(PregeneratorMethod.class); selectorConstructor.setAccessible(true); @@ -119,6 +119,9 @@ public class CachedPregenMethodCompletionTest { new PregenSavedChunkStatus((x, z) -> false))); wrapped.onPregenStart(123, -456); + wrapped.reclaimMemory(); + + assertEquals(1, underlying.reclaimCalls.get()); assertEquals(1, underlying.pregenStartCalls.get()); assertEquals(123, underlying.centerBlockX.get()); @@ -179,6 +182,7 @@ public class CachedPregenMethodCompletionTest { private final AtomicInteger generateChunkCalls = new AtomicInteger(); private final AtomicInteger generateRegionCalls = new AtomicInteger(); private final AtomicInteger pregenStartCalls = new AtomicInteger(); + private final AtomicInteger reclaimCalls = new AtomicInteger(); private final AtomicInteger centerBlockX = new AtomicInteger(); private final AtomicInteger centerBlockZ = new AtomicInteger(); private final AtomicReference capturedListener = new AtomicReference<>(); @@ -224,6 +228,11 @@ public class CachedPregenMethodCompletionTest { this.centerBlockZ.set(centerBlockZ); } + @Override + public void reclaimMemory() { + reclaimCalls.incrementAndGet(); + } + @Override public Mantle getMantle() { return null; diff --git a/core/src/test/java/art/arcane/iris/world/pregen/IrisPregeneratorInitTest.java b/core/src/test/java/art/arcane/iris/world/pregen/IrisPregeneratorInitTest.java index b35510657..7b51cf1dc 100644 --- a/core/src/test/java/art/arcane/iris/world/pregen/IrisPregeneratorInitTest.java +++ b/core/src/test/java/art/arcane/iris/world/pregen/IrisPregeneratorInitTest.java @@ -1,13 +1,18 @@ package art.arcane.iris.world.pregen; import art.arcane.iris.configuration.IrisSettings; +import art.arcane.iris.spi.IrisLogging; +import art.arcane.volmlib.util.scheduling.ChronoLatch; import art.arcane.volmlib.util.mantle.runtime.Mantle; import art.arcane.volmlib.util.math.Position2; import org.junit.Test; +import org.mockito.InOrder; +import org.mockito.MockedStatic; import java.io.ByteArrayOutputStream; import java.io.PrintStream; import java.lang.reflect.Method; +import java.lang.reflect.Field; import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.List; @@ -18,6 +23,12 @@ import java.util.concurrent.atomic.AtomicReference; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.CALLS_REAL_METHODS; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.inOrder; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.mockStatic; +import static org.mockito.Mockito.when; public class IrisPregeneratorInitTest { @Test @@ -164,6 +175,43 @@ public class IrisPregeneratorInitTest { } } + @Test + public void outerHeapReclaimReleasesServerChunksBeforeMantleAndSurvivesChunkFailure() throws Exception { + for (boolean failReclaim : new boolean[]{false, true}) { + PregeneratorMethod generator = mock(PregeneratorMethod.class); + Mantle mantle = mock(Mantle.class); + when(generator.getMantle()).thenReturn(mantle); + IllegalStateException failure = new IllegalStateException("chunk reclaim failed"); + if (failReclaim) { + doThrow(failure).when(generator).reclaimMemory(); + } + ChronoLatch latch = mock(ChronoLatch.class); + when(latch.flip()).thenReturn(true); + IrisPregenerator pregenerator = mock(IrisPregenerator.class, CALLS_REAL_METHODS); + Field generatorField = IrisPregenerator.class.getDeclaredField("generator"); + generatorField.setAccessible(true); + generatorField.set(pregenerator, generator); + Field latchField = IrisPregenerator.class.getDeclaredField("heapReclaimLatch"); + latchField.setAccessible(true); + latchField.set(pregenerator, latch); + Method reclaim = IrisPregenerator.class.getDeclaredMethod("reclaimHeapPressure"); + reclaim.setAccessible(true); + try (MockedStatic pressure = mockStatic(MantleHeapPressure.class); + MockedStatic logging = mockStatic(IrisLogging.class)) { + reclaim.invoke(pregenerator); + InOrder order = inOrder(generator, mantle); + order.verify(generator).reclaimMemory(); + order.verify(generator).getMantle(); + order.verify(mantle).saveOldestIdleTectonicPlate(); + pressure.verify(MantleHeapPressure::requestPanicReclaim); + if (failReclaim) { + logging.verify(() -> IrisLogging.reportError( + "Pregen could not reclaim completed server chunks under heap pressure.", failure)); + } + } + } + } + private static final class TrackingPregeneratorMethod implements PregeneratorMethod { private final AtomicInteger initCalls = new AtomicInteger(); private final AtomicInteger saveCalls = new AtomicInteger(); diff --git a/core/src/test/java/art/arcane/iris/world/pregen/PregenMantleBackpressureTest.java b/core/src/test/java/art/arcane/iris/world/pregen/PregenMantleBackpressureTest.java index 49954628b..b013e55f8 100644 --- a/core/src/test/java/art/arcane/iris/world/pregen/PregenMantleBackpressureTest.java +++ b/core/src/test/java/art/arcane/iris/world/pregen/PregenMantleBackpressureTest.java @@ -7,12 +7,15 @@ import org.junit.Test; import java.time.Duration; import java.util.ArrayList; import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Supplier; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; @@ -47,27 +50,47 @@ public class PregenMantleBackpressureTest { } @Test - public void residencyAtTwiceThePlateCapIsNotTrimmed() { + public void residencyAtTheConfiguredPlateCapIsNotTrimmed() { Mantle mantle = mock(Mantle.class); - when(mantle.getLoadedRegionCount()).thenReturn(8); + when(mantle.getLoadedRegionCount()).thenReturn(4); backpressure(() -> mantle, 4, 1_000L, () -> { }).enforceMantleBudget(); verify(mantle, never()).trim(0L); verify(mantle, never()).unloadTectonicPlate(0); + verify(mantle, never()).saveOldestIdleTectonicPlate(); } @Test - public void residencyOverTheHardCapIsTrimmedUntilItFits() { + public void residencyOverTheConfiguredPlateCapIsTrimmedUntilItFits() { Mantle mantle = mock(Mantle.class); - when(mantle.getLoadedRegionCount()).thenReturn(20, 20, 3); + when(mantle.getLoadedRegionCount()).thenReturn(5, 5, 4); backpressure(() -> mantle, 4, 1_000L, () -> { }).enforceMantleBudget(); verify(mantle, times(1)).trim(0L); verify(mantle, times(1)).unloadTectonicPlate(0); + verify(mantle, never()).saveOldestIdleTectonicPlate(); + } + + @Test + public void idleDirtyPlatesArePersistedUntilTheConfiguredCapIsMetEvenPastTheDeadline() { + Mantle mantle = mock(Mantle.class); + AtomicInteger resident = new AtomicInteger(7); + AtomicInteger timeouts = new AtomicInteger(); + when(mantle.getLoadedRegionCount()).thenAnswer(invocation -> resident.get()); + when(mantle.saveOldestIdleTectonicPlate()).thenAnswer(invocation -> { + resident.decrementAndGet(); + return true; + }); + + backpressure(() -> mantle, 4, 0L, timeouts::incrementAndGet).enforceMantleBudget(); + + assertEquals(4, resident.get()); + assertEquals(0, timeouts.get()); + verify(mantle, times(3)).saveOldestIdleTectonicPlate(); } @Test @@ -84,7 +107,7 @@ public class PregenMantleBackpressureTest { } @Test - public void anExhaustedBudgetReportsTheTimeoutOnceAndProceeds() { + public void pinnedResidencyReportsTheTimeoutOnceAndAllowsTheDependencyWorkingSet() { Mantle mantle = mock(Mantle.class); when(mantle.getLoadedRegionCount()).thenReturn(20); AtomicInteger timeouts = new AtomicInteger(); @@ -92,6 +115,7 @@ public class PregenMantleBackpressureTest { backpressure(() -> mantle, 4, 0L, timeouts::incrementAndGet).enforceMantleBudget(); assertEquals(1, timeouts.get()); + verify(mantle).saveOldestIdleTectonicPlate(); } @Test @@ -185,34 +209,91 @@ public class PregenMantleBackpressureTest { } @Test - public void pinnedPlatesStillRespectThePressureWaitDeadline() { + public void pinnedPlatesKeepWaitingPastTheDeadlineUntilHeapPressureClears() { Mantle mantle = mock(Mantle.class); when(mantle.getLoadedRegionCount()).thenReturn(9); when(mantle.saveOldestIdleTectonicPlate()).thenReturn(false); AtomicInteger collections = new AtomicInteger(); AtomicInteger timeouts = new AtomicInteger(); + AtomicBoolean pressured = new AtomicBoolean(true); backpressure(() -> mantle, 16, 0L, timeouts::incrementAndGet) - .awaitHeapHeadroom(() -> true, collections::incrementAndGet); + .awaitHeapHeadroom(pressured::get, () -> { + if (collections.incrementAndGet() == 3) { + pressured.set(false); + } + }); - verify(mantle, times(1)).saveOldestIdleTectonicPlate(); + verify(mantle, times(3)).saveOldestIdleTectonicPlate(); verify(mantle, never()).unloadTectonicPlate(0); + assertEquals(3, collections.get()); + assertEquals(1, timeouts.get()); + } + + @Test + public void failedPressureEvictionKeepsWaitingForHeapHeadroom() { + Mantle mantle = mock(Mantle.class); + doThrow(new IllegalStateException("plate write failed")).when(mantle).saveOldestIdleTectonicPlate(); + AtomicInteger collections = new AtomicInteger(); + AtomicInteger timeouts = new AtomicInteger(); + AtomicBoolean pressured = new AtomicBoolean(true); + + backpressure(() -> mantle, 16, 0L, timeouts::incrementAndGet) + .awaitHeapHeadroom(pressured::get, () -> { + if (collections.incrementAndGet() == 3) { + pressured.set(false); + } + }); + + assertEquals(3, collections.get()); + assertEquals(1, timeouts.get()); + } + + @Test + public void persistentHeapPressureCanBeCancelledAfterTheDeadline() { + AtomicBoolean cancelled = new AtomicBoolean(); + AtomicInteger timeouts = new AtomicInteger(); + AtomicInteger collections = new AtomicInteger(); + PregenMantleBackpressure backpressure = new PregenMantleBackpressure( + () -> null, 16, 0, 0L, () -> { + timeouts.incrementAndGet(); + cancelled.set(true); + }, () -> "cancelled", cancelled::get); + + backpressure.awaitHeapHeadroom(() -> true, collections::incrementAndGet); + assertEquals(1, collections.get()); assertEquals(1, timeouts.get()); } @Test - public void failedPressureEvictionStillAttemptsGcAndRespectsTheDeadline() { - Mantle mantle = mock(Mantle.class); - doThrow(new IllegalStateException("plate write failed")).when(mantle).saveOldestIdleTectonicPlate(); - AtomicInteger collections = new AtomicInteger(); + public void persistentHeapPressureWaitsAndRetainsInterruptionAfterTheDeadline() throws Exception { + CountDownLatch warned = new CountDownLatch(1); AtomicInteger timeouts = new AtomicInteger(); - - backpressure(() -> mantle, 16, 0L, timeouts::incrementAndGet) - .awaitHeapHeadroom(() -> true, collections::incrementAndGet); - - assertEquals(1, collections.get()); - assertEquals(1, timeouts.get()); + AtomicInteger collections = new AtomicInteger(); + AtomicBoolean interrupted = new AtomicBoolean(); + PregenMantleBackpressure backpressure = new PregenMantleBackpressure( + () -> null, 16, 600_000, 0L, () -> { + timeouts.incrementAndGet(); + warned.countDown(); + }, () -> "interrupted"); + Thread waiter = new Thread(() -> { + backpressure.awaitHeapHeadroom(() -> true, collections::incrementAndGet); + interrupted.set(Thread.currentThread().isInterrupted()); + }, "pregen-heap-pressure-waiter"); + waiter.start(); + try { + assertTrue(warned.await(5L, TimeUnit.SECONDS)); + Await.until("heap pressure waiter to park", () -> waiter.getState() == Thread.State.TIMED_WAITING); + assertTrue(waiter.isAlive()); + assertEquals(1, collections.get()); + assertEquals(1, timeouts.get()); + } finally { + waiter.interrupt(); + waiter.join(5_000L); + } + assertFalse(waiter.isAlive()); + assertTrue(interrupted.get()); } @Test diff --git a/core/src/test/java/art/arcane/iris/world/pregen/PregenSilentFailureContractTest.java b/core/src/test/java/art/arcane/iris/world/pregen/PregenSilentFailureContractTest.java index 9c8b3a985..c069c6308 100644 --- a/core/src/test/java/art/arcane/iris/world/pregen/PregenSilentFailureContractTest.java +++ b/core/src/test/java/art/arcane/iris/world/pregen/PregenSilentFailureContractTest.java @@ -53,7 +53,6 @@ public class PregenSilentFailureContractTest { String async = read("art/arcane/iris/world/pregen/AsyncPregenMethod.java"); assertTrue(method(async, "private int resolveWorkerPoolThreads()").contains("PregenDiagnostics.probeFailed(")); - assertTrue(method(async, "private int evictionWindow()").contains("PregenDiagnostics.probeFailed(")); assertTrue(method(async, "private Engine resolveMetricsEngine()").contains("PregenDiagnostics.probeFailed(")); }