Stabilized hydrology pregeneration

- Preserve exact river owner plans across cache eviction and persistence.
- Retry required hydrology plans without generating riverless terrain.
- Bound object plans and pregeneration residency by available heap.
- Release completed native chunks and avoid empty reclamation flushes.
- Cover persistence, admission, cancellation, and ownership regressions.
This commit is contained in:
Brian Neumann-Fopiano committed 2026-09-28 02:13:17 -04:00
1 parent 9eb05fd987
commit 21f8440299
40 files changed
+2218 -620

No files matched your search

@@ -27,7 +27,6 @@ import art.arcane.iris.spi.IrisPlatforms;
import art.arcane.volmlib.util.io.IO;
import art.arcane.volmlib.util.json.JSONException;
import art.arcane.volmlib.util.json.JSONObject;
import art.arcane.iris.platform.bootstrap.getHardware;
import art.arcane.iris.platform.bukkit.plugin.VolmitSender;
import lombok.Data;
@@ -226,7 +225,7 @@ public class IrisSettings {
public static class IrisSettingsPregen {
private static final int REFERENCE_WORLD_HEIGHT = 384;
private static final int MIN_RESIDENT_TECTONIC_PLATES = 16;
private static final double MANTLE_HEAP_FRACTION = 0.6D;
private static final double MANTLE_HEAP_FRACTION = 0.25D;
private static final int REFERENCE_PLATE_MEGABYTES = 48;
public IrisRuntimeSchedulerMode runtimeSchedulerMode = IrisRuntimeSchedulerMode.AUTO;
public IrisPaperLikeBackendMode paperLikeBackendMode = IrisPaperLikeBackendMode.AUTO;
@@ -256,10 +255,14 @@ public class IrisSettings {
}
public int getEffectiveResidentTectonicPlates(int worldHeight) {
return effectiveResidentTectonicPlates(worldHeight, Runtime.getRuntime().maxMemory());
}
int effectiveResidentTectonicPlates(int worldHeight, long maximumHeapBytes) {
int baseCap = getMaxResidentTectonicPlates();
int normalizedHeight = Math.max(1, worldHeight);
int heightScaledCap = (int) Math.round((double) baseCap * REFERENCE_WORLD_HEIGHT / (double) normalizedHeight);
long maxHeapMegabytes = getHardware.getProcessMemory();
long maxHeapMegabytes = maximumHeapBytes / (1024L * 1024L);
double plateMegabytes = (double) REFERENCE_PLATE_MEGABYTES * (double) normalizedHeight / (double) REFERENCE_WORLD_HEIGHT;
int byteBudgetCap = (int) Math.floor(MANTLE_HEAP_FRACTION * (double) maxHeapMegabytes / plateMegabytes);
int effective = Math.min(heightScaledCap, byteBudgetCap);
@@ -39,7 +39,8 @@ final class CrossTileResolutionContext {
CrossTileResolvedOwner existing = resolved.putIfAbsent(key, owner);
if (existing != null
&& !existing.withoutFootprintCompiler().equals(owner.withoutFootprintCompiler())) {
throw new IllegalStateException("Cross-tile owner resolution produced inconsistent cached results.");
throw new IllegalStateException("Cross-tile owner resolution produced inconsistent cached results for "
+ key.tileX() + "," + key.tileZ() + " while planning " + root.tileX() + "," + root.tileZ() + ".");
}
if (resolved.size() > maximumOwners) {
throw new IllegalStateException("Cross-tile owner dependencies exceeded their color-ranked count bound.");
@@ -24,6 +24,9 @@ final class HydrologyCacheWeights {
long bytes = network(tile.nodes(), tile.edges(), tile.outlets(), tile.courses(), tile.cavePlans());
bytes += diagnostics(tile.localDiagnosticCandidates());
bytes += (long) tile.regionalCourseIds().size() * 64L;
if (tile.resolvedOwner() != null) {
bytes += owner(tile.resolvedOwner());
}
RiverFootprint footprint = tile.footprint();
if (footprint != null) {
bytes += (long) footprint.size() * 64L;
@@ -1,6 +1,7 @@
package art.arcane.iris.generation.hydrology;
import java.util.List;
import java.util.Objects;
record HydrologyOwnerDraft(
HydrologyTileKey key,
@@ -8,6 +9,14 @@ record HydrologyOwnerDraft(
List<HydrologyDiagnosticCandidate> diagnostics,
HydrologyFootprintCompiler footprintCompiler
) {
HydrologyOwnerDraft {
Objects.requireNonNull(key, "key");
Objects.requireNonNull(result, "result");
result = new HydrologyCaveCourseFilter.Result(List.copyOf(result.nodes()), List.copyOf(result.edges()),
List.copyOf(result.outlets()), List.copyOf(result.courses()), List.copyOf(result.cavePlans()));
diagnostics = List.copyOf(diagnostics);
}
HydrologyOwnerDraft withoutFootprintCompiler() {
if (footprintCompiler == null) {
return this;
@@ -214,33 +214,12 @@ public final class HydrologyPlanner {
return settings;
}
/**
* A tile with the same identity as a planned tile but no rivers, used when planning the tile
* failed and terrain there must still generate.
*/
public HydrologyTile emptyTile(HydrologyTileKey key) {
Objects.requireNonNull(key, "key");
return new HydrologyTile(
key,
worldSeed,
settings.fingerprint(),
settings.routing().tileSize(),
List.of(),
List.of(),
List.of(),
List.of(),
Set.of(),
List.of(),
List.of(),
RiverFootprint.empty()
);
}
public HydrologyTile plan(HydrologyTileKey key) {
long started = System.nanoTime();
CrossTileResolution resolution = resolveCrossTileOwner(key);
long materializeStarted = System.nanoTime();
HydrologyTile tile = crossTile.materializeAcceptedTile(resolution);
HydrologyTile tile = crossTile.materializeAcceptedTile(resolution)
.withResolvedOwner(new CrossTileResolvedOwner(resolution.draft(), resolution.observedRejections()));
long finished = System.nanoTime();
IrisLogging.debug(
"Hydrology tile %d,%d planned in %dms: owners=%d resolve=%dms materialize=%dms courses=%d on %s",
@@ -277,32 +256,10 @@ public final class HydrologyPlanner {
|| tile.tileSize() != settings.routing().tileSize()) {
throw new IllegalArgumentException("Hydrology tile does not match this planner.");
}
ArrayList<RiverCourse> localCourses = new ArrayList<>(tile.courses().size());
for (RiverCourse course : tile.courses()) {
if (!tile.regionalCourseIds().contains(course.id())) {
localCourses.add(course);
}
CrossTileResolvedOwner owner = tile.resolvedOwner();
if (owner != null) {
resolvedOwners.asMap().putIfAbsent(tile.key(), owner);
}
ArrayList<HydrologyCavePlan> localPlans = new ArrayList<>(tile.cavePlans().size());
for (HydrologyCavePlan plan : tile.cavePlans()) {
if (!tile.regionalCourseIds().contains(plan.source().sourceId())) {
localPlans.add(plan);
}
}
HydrologyCaveCourseFilter.Result result = new HydrologyCaveCourseFilter.Result(
tile.nodes(),
tile.edges(),
tile.outlets(),
localCourses,
localPlans
);
HydrologyOwnerDraft draft = new HydrologyOwnerDraft(
tile.key(),
result,
tile.localDiagnosticCandidates(),
null
);
resolvedOwners.put(tile.key(), new CrossTileResolvedOwner(draft, List.of()));
}
CrossTileResolution resolveCrossTileOwner(HydrologyTileKey key) {
@@ -386,6 +343,11 @@ public final class HydrologyPlanner {
if (candidateKeys.size() < 2 || !IrisPlatforms.isBound()) {
ArrayList<CrossTileResolvedOwner> resolved = new ArrayList<>(candidateKeys.size());
for (HydrologyTileKey candidateKey : candidateKeys) {
CrossTileResolvedOwner local = context.resolved(candidateKey);
if (local != null) {
resolved.add(local);
continue;
}
HydrologyForkJoin.Task<CrossTileResolvedOwner> prepared = preparedOwners.get(candidateKey);
CrossTileResolvedOwner owner = prepared == null
? resolveCrossTileOwner(candidateKey, context, false) : prepared.await();
@@ -397,16 +359,25 @@ public final class HydrologyPlanner {
// Neighbour drafts use the current pool (or the burst pool from outside any pool).
// Waiting workers claim their own tasks without helping unrelated owner drafts.
ArrayList<Callable<CrossTileResolvedOwner>> tasks = new ArrayList<>(candidateKeys.size());
ArrayList<CrossTileResolvedOwner> resolved = new ArrayList<>(candidateKeys.size());
for (HydrologyTileKey candidateKey : candidateKeys) {
CrossTileResolvedOwner local = context.resolved(candidateKey);
resolved.add(local);
if (local != null) {
continue;
}
HydrologyForkJoin.Task<CrossTileResolvedOwner> prepared = preparedOwners.get(candidateKey);
tasks.add(prepared == null ? () -> resolveIndependentOwner(candidateKey) : prepared::await);
}
List<CrossTileResolvedOwner> owners = HydrologyForkJoin.invokeAll(tasks, MultiBurst.burst);
ArrayList<CrossTileResolvedOwner> resolved = new ArrayList<>(candidateKeys.size());
int completed = 0;
for (int index = 0; index < candidateKeys.size(); index++) {
CrossTileResolvedOwner owner = owners.get(index);
if (resolved.get(index) != null) {
continue;
}
CrossTileResolvedOwner owner = owners.get(completed++);
context.remember(candidateKeys.get(index), owner);
resolved.add(owner);
resolved.set(index, owner);
}
return List.copyOf(resolved);
}
@@ -34,6 +34,7 @@ public final class HydrologyTile {
private final Map<Long, RiverOutlet> outletsById;
private final Map<Long, HydrologyCavePlan> cavePlansByCourseId;
private final List<HydrologyFeatureRef> features;
private final CrossTileResolvedOwner resolvedOwner;
public HydrologyTile(
HydrologyTileKey key,
@@ -76,6 +77,39 @@ public final class HydrologyTile {
validateGraph();
validateCourses();
this.features = collectFeatures(footprint);
this.resolvedOwner = null;
}
private HydrologyTile(HydrologyTile tile, CrossTileResolvedOwner resolvedOwner) {
this.key = tile.key;
this.worldSeed = tile.worldSeed;
this.settingsFingerprint = tile.settingsFingerprint;
this.tileSize = tile.tileSize;
this.nodes = tile.nodes;
this.edges = tile.edges;
this.outlets = tile.outlets;
this.courses = tile.courses;
this.regionalCourseIds = tile.regionalCourseIds;
this.cavePlans = tile.cavePlans;
this.localDiagnosticCandidates = tile.localDiagnosticCandidates;
this.footprint = tile.footprint;
this.nodesById = tile.nodesById;
this.outletsById = tile.outletsById;
this.cavePlansByCourseId = tile.cavePlansByCourseId;
this.features = tile.features;
this.resolvedOwner = resolvedOwner;
}
HydrologyTile withResolvedOwner(CrossTileResolvedOwner owner) {
Objects.requireNonNull(owner, "owner");
if (!key.equals(owner.draft().key())) {
throw new IllegalArgumentException("Resolved hydrology owner does not match its tile.");
}
return new HydrologyTile(this, owner.withoutFootprintCompiler());
}
CrossTileResolvedOwner resolvedOwner() {
return resolvedOwner;
}
public HydrologyTileKey key() {
@@ -316,7 +316,7 @@ public final class HydrologyTileCache implements AutoCloseable {
}
try {
present = tiles.getIfPresent(key);
HydrologyTile tile = present == null ? planOrEmpty(key, epoch) : present;
HydrologyTile tile = present == null ? planRequired(key, epoch) : present;
synchronized (publicationLock) {
if (!closed.get() && cacheEpoch.get() == epoch) {
tiles.put(key, tile);
@@ -332,69 +332,107 @@ public final class HydrologyTileCache implements AutoCloseable {
}
}
/**
* A tile whose planning throws is published without rivers instead of failing the chunks that
* asked for it: one bad column must not take the chunk system down. The failure is logged in
* full so the cause can be traced, and the empty tile is cached like any other so the session
* stays consistent. A plan that was interrupted is not a bad tile: it is rethrown without being
* cached so the next request plans it again.
*/
private HydrologyTile planOrEmpty(HydrologyTileKey key, long planningEpoch) {
private HydrologyTile planRequired(HydrologyTileKey key, long planningEpoch) {
SharedTileKey sharedKey = sharedCacheScope == null ? null : new SharedTileKey(sharedCacheScope, key);
HydrologyTile shared = sharedKey == null ? null : SHARED_TILES.getIfPresent(sharedKey);
if (shared != null && validSharedTile(shared, sharedKey)) {
PreparedHydrologyTileStore store = persistentStore;
if (store != null && store.load(key).isEmpty()) {
persistTile(store, shared);
}
planner.reuseResolvedTile(shared);
IrisLogging.debug("Reused shared prepared hydrology tile %d,%d", key.tileX(), key.tileZ());
return shared;
}
if (shared != null) {
SHARED_TILES.invalidate(sharedKey);
}
PreparedHydrologyTileStore store = persistentStore;
try (HydrologyPlanningAdmission.Permit ignored = HydrologyPlanningAdmission.acquireRoot(closed::get)) {
return planAdmitted(key, planningEpoch, sharedKey, store);
long attempt = 0L;
long retryDelayMillis = 250L;
long nextFailureLog = 0L;
while (true) {
requirePlanningActive();
try {
if (attempt == 0L && sharedKey != null) {
HydrologyTile shared = SHARED_TILES.getIfPresent(sharedKey);
if (shared != null && validSharedTile(shared, sharedKey)) {
planner.reuseResolvedTile(shared);
if (store != null && !store.contains(key)) {
persistTile(store, shared);
}
IrisLogging.debug("Reused shared prepared hydrology tile %d,%d", key.tileX(), key.tileZ());
return shared;
}
if (shared != null) {
SHARED_TILES.invalidate(sharedKey);
}
}
try (HydrologyPlanningAdmission.Permit ignored = HydrologyPlanningAdmission.acquireRoot(
closed::get)) {
return planAdmitted(key, planningEpoch, sharedKey, store, attempt == 0L);
}
} catch (RuntimeException failure) {
if (failure instanceof CancellationException || interrupted(failure)) {
throw failure;
}
requirePlanningActive();
attempt++;
long now = System.nanoTime();
if (nextFailureLog == 0L || now - nextFailureLog >= 0L) {
IrisLogging.error("Hydrology tile " + key.tileX() + "," + key.tileZ()
+ " failed planning attempt " + attempt + "; retrying in " + retryDelayMillis
+ " ms. Generation is waiting for its river plan: " + failure.getMessage());
IrisLogging.reportError(failure);
nextFailureLog = now + TimeUnit.SECONDS.toNanos(30L);
}
if (sharedKey != null) {
SHARED_TILES.invalidate(sharedKey);
}
planner.clearOwnerDrafts();
awaitPlanningRetry(retryDelayMillis);
retryDelayMillis = Math.min(5000L, retryDelayMillis * 2L);
}
}
}
private void requirePlanningActive() {
requireOpen();
if (Thread.currentThread().isInterrupted()) {
throw new CancellationException("Hydrology plan retry interrupted.");
}
}
private void awaitPlanningRetry(long delayMillis) {
long deadline = System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(delayMillis);
while (true) {
requirePlanningActive();
long remaining = deadline - System.nanoTime();
if (remaining <= 0L) {
return;
}
try {
TimeUnit.NANOSECONDS.sleep(Math.min(remaining, TimeUnit.MILLISECONDS.toNanos(100L)));
} catch (InterruptedException interruption) {
Thread.currentThread().interrupt();
throw new CancellationException("Hydrology plan retry interrupted.");
}
}
}
private HydrologyTile planAdmitted(HydrologyTileKey key, long planningEpoch,
SharedTileKey sharedKey, PreparedHydrologyTileStore store) {
if (store != null) {
SharedTileKey sharedKey, PreparedHydrologyTileStore store,
boolean reusePrepared) {
if (store != null && reusePrepared) {
HydrologyTile persisted = store.load(key).orElse(null);
if (persisted != null) {
SHARED_TILES.put(sharedKey, persisted);
planner.reuseResolvedTile(persisted);
SHARED_TILES.put(sharedKey, persisted);
IrisLogging.debug("Loaded persisted prepared hydrology tile %d,%d", key.tileX(), key.tileZ());
return persisted;
}
}
prepareTerrain();
try {
HydrologyTile planned = planner.plan(key);
synchronized (publicationLock) {
if (sharedKey != null && !closed.get() && planningEpoch == cacheEpoch.get()) {
HydrologyTile existing = SHARED_TILES.asMap().putIfAbsent(sharedKey, planned);
if (existing == null) {
IrisLogging.debug("Published shared prepared hydrology tile %d,%d", key.tileX(), key.tileZ());
}
HydrologyTile planned = planner.plan(key);
synchronized (publicationLock) {
if (sharedKey != null && !closed.get() && planningEpoch == cacheEpoch.get()) {
HydrologyTile existing = SHARED_TILES.asMap().putIfAbsent(sharedKey, planned);
if (existing == null) {
IrisLogging.debug("Published shared prepared hydrology tile %d,%d", key.tileX(), key.tileZ());
}
}
if (store != null && !closed.get() && planningEpoch == cacheEpoch.get()) {
persistTile(store, planned);
}
return planned;
} catch (RuntimeException failure) {
if (failure instanceof CancellationException || interrupted(failure)) {
throw failure;
}
IrisLogging.error("Hydrology tile " + key.tileX() + "," + key.tileZ()
+ " failed to plan and generates without rivers: " + failure.getMessage());
IrisLogging.reportError(failure);
return planner.emptyTile(key);
}
if (store != null && !closed.get() && planningEpoch == cacheEpoch.get()) {
persistTile(store, planned);
}
return planned;
}
private void persistTile(PreparedHydrologyTileStore store, HydrologyTile tile) {
@@ -0,0 +1,706 @@
package art.arcane.iris.generation.hydrology;
import art.arcane.iris.generation.hydrology.cave.CavePosition;
import art.arcane.iris.generation.hydrology.cave.HydrologyCaveMode;
import art.arcane.iris.generation.hydrology.cave.HydrologyCavePlan;
import art.arcane.iris.generation.hydrology.cave.HydrologyCaveRejection;
import art.arcane.iris.generation.hydrology.cave.HydrologyCaveSource;
import art.arcane.iris.generation.hydrology.policy.SurfaceRiverPolicy;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.IdentityHashMap;
import java.util.List;
import java.util.Map;
import java.util.OptionalLong;
import java.util.Set;
final class HydrologyTileCodec {
private static final int MAGIC = 0x49524854;
private static final int FORMAT = 2;
private static final long MAXIMUM_ALLOCATION_BYTES = Math.max(16L * 1024L * 1024L,
Math.min(512L * 1024L * 1024L, Runtime.getRuntime().maxMemory() / 8L));
private static final int MAXIMUM_COLLECTION_SIZE = 2_000_000;
private static final HydrologyFeatureType[] HYDROLOGY_FEATURE_TYPE_VALUES = HydrologyFeatureType.values();
private static final RiverCourseType[] RIVER_COURSE_TYPE_VALUES = RiverCourseType.values();
private static final HydrologyCandidateKind[] HYDROLOGY_CANDIDATE_KIND_VALUES = HydrologyCandidateKind.values();
private static final HydrologyCandidateRejection[] HYDROLOGY_CANDIDATE_REJECTION_VALUES = HydrologyCandidateRejection.values();
private static final HydrologyCaveMode[] HYDROLOGY_CAVE_MODE_VALUES = HydrologyCaveMode.values();
private static final HydrologyCaveRejection[] HYDROLOGY_CAVE_REJECTION_VALUES = HydrologyCaveRejection.values();
private final Map<String, String> strings = new HashMap<>();
private long remainingAllocation = MAXIMUM_ALLOCATION_BYTES;
void write(DataOutputStream output, HydrologyTile tile, String scopeIdentity) throws IOException {
output.writeInt(MAGIC);
output.writeInt(FORMAT);
output.writeUTF(scopeIdentity);
output.writeInt(tile.key().tileX());
output.writeInt(tile.key().tileZ());
output.writeLong(tile.worldSeed());
output.writeLong(tile.settingsFingerprint());
output.writeInt(tile.tileSize());
writeList(output, tile.nodes(), this::writeDrainageNode);
writeList(output, tile.edges(), this::writeDrainageEdge);
writeList(output, tile.outlets(), this::writeRiverOutlet);
writeList(output, tile.courses(), this::writeRiverCourse);
writeCount(output, tile.regionalCourseIds().size(), 64);
for (long id : tile.regionalCourseIds()) {
output.writeLong(id);
}
writeList(output, tile.cavePlans(), this::writeCavePlan);
writeList(output, tile.localDiagnosticCandidates(), this::writeHydrologyDiagnosticCandidate);
writeCount(output, tile.footprint().columns().size(), 256);
for (HydrologyColumnSample column : tile.footprint().columns().values()) {
writeHydrologyColumnSample(output, column);
}
output.writeBoolean(tile.resolvedOwner() != null);
if (tile.resolvedOwner() != null) {
writeResolvedOwner(output, tile);
}
}
HydrologyTile read(DataInputStream input, String scopeIdentity, HydrologyTileKey key,
HydrologyTileCache.SharedCacheScope scope, int expectedTileSize) throws IOException {
if (input.readInt() != MAGIC || input.readInt() != FORMAT || !scopeIdentity.equals(input.readUTF())
|| input.readInt() != key.tileX() || input.readInt() != key.tileZ()
|| input.readLong() != scope.worldSeed() || input.readLong() != scope.settingsFingerprint()
|| input.readInt() != expectedTileSize) {
throw new IOException("Prepared hydrology tile header does not match its scope.");
}
List<DrainageNode> nodes = readList(input, this::readDrainageNode);
List<DrainageEdge> edges = readList(input, this::readDrainageEdge);
List<RiverOutlet> outlets = readList(input, this::readRiverOutlet);
List<RiverCourse> courses = readList(input, this::readRiverCourse);
int regionalCount = readCount(input, 64);
Set<Long> regionalCourseIds = HashSet.newHashSet(regionalCount);
for (int index = 0; index < regionalCount; index++) {
if (!regionalCourseIds.add(input.readLong())) {
throw new IOException("Duplicate regional hydrology course.");
}
}
List<HydrologyCavePlan> cavePlans = readList(input, this::readCavePlan);
List<HydrologyDiagnosticCandidate> candidates = readList(input, this::readHydrologyDiagnosticCandidate);
int columnCount = readCount(input, 256);
Map<Long, HydrologyColumnSample> columns = HashMap.newHashMap(columnCount);
for (int index = 0; index < columnCount; index++) {
HydrologyColumnSample column = readHydrologyColumnSample(input);
if (columns.put(RiverFootprint.pack(column.x(), column.z()), column) != null) {
throw new IOException("Duplicate hydrology footprint column.");
}
}
HydrologyTile tile = new HydrologyTile(key, scope.worldSeed(), scope.settingsFingerprint(), expectedTileSize,
nodes, edges, outlets, courses, regionalCourseIds, cavePlans, candidates,
new RiverFootprint(columns));
if (input.readBoolean()) {
tile = tile.withResolvedOwner(readResolvedOwner(input, tile));
}
if (input.read() != -1) {
throw new IOException("Unexpected data after prepared hydrology tile.");
}
return tile;
}
private void writeResolvedOwner(DataOutputStream output, HydrologyTile tile) throws IOException {
allocate(256L);
CrossTileResolvedOwner owner = tile.resolvedOwner();
HydrologyCaveCourseFilter.Result result = owner.draft().result();
writeOwnerList(output, result.nodes(), tile.nodes(), this::writeDrainageNode);
writeOwnerList(output, result.edges(), tile.edges(), this::writeDrainageEdge);
writeOwnerList(output, result.outlets(), tile.outlets(), this::writeRiverOutlet);
writeOwnerList(output, result.courses(), tile.courses(), this::writeRiverCourse);
writeOwnerList(output, result.cavePlans(), tile.cavePlans(), this::writeCavePlan);
writeOwnerList(output, owner.draft().diagnostics(), tile.localDiagnosticCandidates(),
this::writeHydrologyDiagnosticCandidate);
writeList(output, owner.observedRejections(), this::writeRejectedCourse);
}
private CrossTileResolvedOwner readResolvedOwner(DataInputStream input, HydrologyTile tile) throws IOException {
allocate(256L);
HydrologyCaveCourseFilter.Result result = new HydrologyCaveCourseFilter.Result(
readOwnerList(input, tile.nodes(), this::readDrainageNode),
readOwnerList(input, tile.edges(), this::readDrainageEdge),
readOwnerList(input, tile.outlets(), this::readRiverOutlet),
readOwnerList(input, tile.courses(), this::readRiverCourse),
readOwnerList(input, tile.cavePlans(), this::readCavePlan));
List<HydrologyDiagnosticCandidate> diagnostics = readOwnerList(input, tile.localDiagnosticCandidates(),
this::readHydrologyDiagnosticCandidate);
return new CrossTileResolvedOwner(new HydrologyOwnerDraft(tile.key(), result, diagnostics, null),
readList(input, this::readRejectedCourse));
}
private void writeRejectedCourse(DataOutputStream output, CrossTileRejectedCourse rejection) throws IOException {
writeRiverCourse(output, rejection.course());
output.writeLong(rejection.winnerSourceId());
}
private CrossTileRejectedCourse readRejectedCourse(DataInputStream input) throws IOException {
return new CrossTileRejectedCourse(readRiverCourse(input), input.readLong());
}
private <T> void writeOwnerList(DataOutputStream output, List<T> values, List<T> published,
ElementWriter<T> writer) throws IOException {
writeCount(output, values.size(), 16);
Map<T, Integer> indexes = new IdentityHashMap<>(published.size());
for (int index = 0; index < published.size(); index++) {
indexes.put(published.get(index), index);
}
for (T value : values) {
Integer index = indexes.get(value);
output.writeInt(index == null ? -1 : index);
if (index == null) {
allocate(256L);
writer.write(output, value);
}
}
}
private <T> List<T> readOwnerList(DataInputStream input, List<T> published, ElementReader<T> reader) throws IOException {
int count = readCount(input, 16);
List<T> values = new ArrayList<>(Math.min(count, 4096));
for (int index = 0; index < count; index++) {
int reference = input.readInt();
if (reference == -1) {
allocate(256L);
values.add(reader.read(input));
} else if (reference >= 0 && reference < published.size()) {
values.add(published.get(reference));
} else {
throw new IOException("Invalid prepared hydrology owner reference.");
}
}
return List.copyOf(values);
}
private void writeCavePlan(DataOutputStream output, HydrologyCavePlan plan) throws IOException {
writeHydrologyCaveSource(output, plan.source());
output.writeByte(plan.rejection().ordinal());
writeOptionalLong(output, plan.arbitrationWinnerSourceId());
validateCount(plan.baselinePreconditions().size(), 96);
plan.writeCells(output);
}
private HydrologyCavePlan readCavePlan(DataInputStream input) throws IOException {
HydrologyCaveSource source = readHydrologyCaveSource(input);
HydrologyCaveRejection rejection = readEnum(input, HYDROLOGY_CAVE_REJECTION_VALUES);
OptionalLong winner = readOptionalLong(input);
int cells = readCount(input, 96);
return HydrologyCavePlan.readCells(source, rejection, winner, input, cells);
}
private void writeHydraulicChannelProfile(DataOutputStream output, HydraulicChannelProfile profile) throws IOException {
writeCount(output, profile.size(), 32);
for (int station = 0; station < profile.size(); station++) {
output.writeDouble(profile.widthAt(station));
output.writeDouble(profile.depthAt(station));
}
}
private HydraulicChannelProfile readHydraulicChannelProfile(DataInputStream input) throws IOException {
int count = readCount(input, 32);
double[] widths = new double[count];
double[] depths = new double[count];
for (int station = 0; station < count; station++) {
widths[station] = input.readDouble();
depths[station] = input.readDouble();
}
return new HydraulicChannelProfile(widths, depths);
}
private void writeDrainageNode(DataOutputStream output, DrainageNode value) throws IOException {
output.writeLong(value.id());
output.writeInt(value.x());
output.writeInt(value.z());
writeHydrologyTerrainSample(output, value.terrain());
output.writeDouble(value.potential());
output.writeLong(value.outletId());
}
private DrainageNode readDrainageNode(DataInputStream input) throws IOException {
return new DrainageNode(
input.readLong(),
input.readInt(),
input.readInt(),
readHydrologyTerrainSample(input),
input.readDouble(),
input.readLong()
);
}
private void writeDrainageEdge(DataOutputStream output, DrainageEdge value) throws IOException {
output.writeLong(value.id());
output.writeLong(value.upstreamNodeId());
output.writeLong(value.downstreamNodeId());
output.writeLong(value.outletId());
output.writeDouble(value.cost());
output.writeInt(value.contributingSurfaceSources());
output.writeInt(value.contributingUndergroundSources());
writeList(output, value.centerline(), this::writeHydrologyPoint);
}
private DrainageEdge readDrainageEdge(DataInputStream input) throws IOException {
return new DrainageEdge(
input.readLong(),
input.readLong(),
input.readLong(),
input.readLong(),
input.readDouble(),
input.readInt(),
input.readInt(),
readList(input, this::readHydrologyPoint)
);
}
private void writeHydrologyTerrainSample(DataOutputStream output, HydrologyTerrainSample value) throws IOException {
output.writeInt(value.naturalHeight());
output.writeDouble(value.slope());
output.writeBoolean(value.ocean());
output.writeBoolean(value.caveAvailable());
output.writeInt(value.caveFloorY());
output.writeInt(value.caveFluidY());
output.writeBoolean(value.transitAllowed());
output.writeBoolean(value.outletAllowed());
output.writeBoolean(value.surfaceSourceAllowed());
output.writeBoolean(value.surfaceSourceRequired());
output.writeBoolean(value.undergroundSourceAllowed());
output.writeBoolean(value.undergroundSourceRequired());
output.writeDouble(value.routingCost());
output.writeDouble(value.surfaceSourceWeight());
output.writeDouble(value.undergroundSourceWeight());
output.writeDouble(value.widthMultiplier());
output.writeDouble(value.depthMultiplier());
output.writeDouble(value.incisionMultiplier());
output.writeDouble(value.routingMultiplier());
output.writeDouble(value.bankMultiplier());
writeString(output, value.parentBiomeKey());
writeString(output, value.surfaceBiomeKey());
writeString(output, value.mouthBiomeKey());
writeString(output, value.shoreBiomeKey());
writeString(output, value.bankBiomeKey());
writeString(output, value.floodedCaveBiomeKey());
writeList(output, value.preferredProfileKeys(), this::writeString);
writeList(output, value.surfacePoolKeys(), this::writeString);
output.writeDouble(value.shoreBiomeWidth());
writeString(output, value.confinesKey());
output.writeDouble(value.shoreWidth());
output.writeBoolean(value.erosion());
writeSurfaceRiverPolicy(output, value.surfacePolicy());
}
private HydrologyTerrainSample readHydrologyTerrainSample(DataInputStream input) throws IOException {
return new HydrologyTerrainSample(
input.readInt(),
input.readDouble(),
input.readBoolean(),
input.readBoolean(),
input.readInt(),
input.readInt(),
input.readBoolean(),
input.readBoolean(),
input.readBoolean(),
input.readBoolean(),
input.readBoolean(),
input.readBoolean(),
input.readDouble(),
input.readDouble(),
input.readDouble(),
input.readDouble(),
input.readDouble(),
input.readDouble(),
input.readDouble(),
input.readDouble(),
readString(input),
readString(input),
readString(input),
readString(input),
readString(input),
readString(input),
readList(input, this::readString),
readList(input, this::readString),
input.readDouble(),
readString(input),
input.readDouble(),
input.readBoolean(),
readSurfaceRiverPolicy(input)
);
}
private void writeRiverOutlet(DataOutputStream output, RiverOutlet value) throws IOException {
output.writeLong(value.id());
output.writeByte(value.type().ordinal());
output.writeLong(value.drainageNodeId());
writeHydrologyPoint(output, value.landwardPoint());
writeHydrologyPoint(output, value.connectionPoint());
output.writeInt(value.seaLevel());
output.writeBoolean(value.directOcean());
}
private RiverOutlet readRiverOutlet(DataInputStream input) throws IOException {
return new RiverOutlet(
input.readLong(),
readEnum(input, HYDROLOGY_FEATURE_TYPE_VALUES),
input.readLong(),
readHydrologyPoint(input),
readHydrologyPoint(input),
input.readInt(),
input.readBoolean()
);
}
private void writeRiverCourse(DataOutputStream output, RiverCourse value) throws IOException {
output.writeLong(value.id());
output.writeByte(value.type().ordinal());
writeOptionalLong(output, value.sourceNodeId());
writeOptionalLong(output, value.outletId());
writeString(output, value.profileKey());
output.writeInt(value.discharge());
writeList(output, value.drainageEdges(), this::writeDrainageEdge);
writeList(output, value.segments(), this::writeHydraulicSegment);
}
private RiverCourse readRiverCourse(DataInputStream input) throws IOException {
return new RiverCourse(
input.readLong(),
readEnum(input, RIVER_COURSE_TYPE_VALUES),
readOptionalLong(input),
readOptionalLong(input),
readString(input),
input.readInt(),
readList(input, this::readDrainageEdge),
readList(input, this::readHydraulicSegment)
);
}
private void writeHydraulicSegment(DataOutputStream output, HydraulicSegment value) throws IOException {
output.writeLong(value.id());
output.writeLong(value.courseId());
output.writeByte(value.type().ordinal());
output.writeInt(value.upstreamHeadY());
output.writeInt(value.downstreamHeadY());
output.writeInt(value.width());
output.writeInt(value.depth());
output.writeBoolean(value.fallingFluid());
output.writeBoolean(value.receivingPool());
writeList(output, value.centerline(), this::writeHydrologyPoint);
writeHydraulicChannelProfile(output, value.channelProfile());
}
private HydraulicSegment readHydraulicSegment(DataInputStream input) throws IOException {
return new HydraulicSegment(
input.readLong(),
input.readLong(),
readEnum(input, HYDROLOGY_FEATURE_TYPE_VALUES),
input.readInt(),
input.readInt(),
input.readInt(),
input.readInt(),
input.readBoolean(),
input.readBoolean(),
readList(input, this::readHydrologyPoint),
readHydraulicChannelProfile(input)
);
}
private void writeHydrologyDiagnosticCandidate(DataOutputStream output, HydrologyDiagnosticCandidate value) throws IOException {
output.writeLong(value.id());
output.writeByte(value.kind().ordinal());
output.writeByte(value.projectedType().ordinal());
writeHydrologyPoint(output, value.point());
output.writeByte(value.rejection().ordinal());
output.writeInt(value.detail());
}
private HydrologyDiagnosticCandidate readHydrologyDiagnosticCandidate(DataInputStream input) throws IOException {
return new HydrologyDiagnosticCandidate(
input.readLong(),
readEnum(input, HYDROLOGY_CANDIDATE_KIND_VALUES),
readEnum(input, HYDROLOGY_FEATURE_TYPE_VALUES),
readHydrologyPoint(input),
readEnum(input, HYDROLOGY_CANDIDATE_REJECTION_VALUES),
input.readInt()
);
}
private void writeHydrologyColumnSample(DataOutputStream output, HydrologyColumnSample value) throws IOException {
output.writeInt(value.x());
output.writeInt(value.z());
output.writeInt(value.naturalHeight());
output.writeInt(value.seaLevel());
output.writeBoolean(value.ocean());
writeString(output, value.parentBiomeKey());
writeList(output, value.layers(), this::writeHydrologyColumnLayer);
}
private HydrologyColumnSample readHydrologyColumnSample(DataInputStream input) throws IOException {
return new HydrologyColumnSample(
input.readInt(),
input.readInt(),
input.readInt(),
input.readInt(),
input.readBoolean(),
readString(input),
readList(input, this::readHydrologyColumnLayer)
);
}
private void writeHydrologyColumnLayer(DataOutputStream output, HydrologyColumnLayer value) throws IOException {
writeHydrologyFeatureRef(output, value.feature());
output.writeInt(value.bedY());
output.writeInt(value.fluidHeadY());
output.writeInt(value.ceilingY());
output.writeBoolean(value.channel());
output.writeBoolean(value.shore());
output.writeBoolean(value.grading());
output.writeBoolean(value.connectedFluid());
output.writeBoolean(value.fallingFluid());
output.writeBoolean(value.receivingPool());
output.writeBoolean(value.terrainOwned());
output.writeBoolean(value.fluidOwned());
output.writeBoolean(value.oceanApron());
writeString(output, value.profileKey());
writeString(output, value.surfaceBiomeKey());
writeString(output, value.mouthBiomeKey());
writeString(output, value.shoreBiomeKey());
writeString(output, value.bankBiomeKey());
writeString(output, value.floodedCaveBiomeKey());
}
private HydrologyColumnLayer readHydrologyColumnLayer(DataInputStream input) throws IOException {
return new HydrologyColumnLayer(
readHydrologyFeatureRef(input),
input.readInt(),
input.readInt(),
input.readInt(),
input.readBoolean(),
input.readBoolean(),
input.readBoolean(),
input.readBoolean(),
input.readBoolean(),
input.readBoolean(),
input.readBoolean(),
input.readBoolean(),
input.readBoolean(),
readString(input),
readString(input),
readString(input),
readString(input),
readString(input),
readString(input)
);
}
private void writeHydrologyFeatureRef(DataOutputStream output, HydrologyFeatureRef value) throws IOException {
output.writeLong(value.id());
output.writeByte(value.type().ordinal());
output.writeLong(value.courseId());
output.writeLong(value.segmentId());
output.writeInt(value.x());
output.writeInt(value.y());
output.writeInt(value.z());
output.writeInt(value.flowDeltaX());
output.writeInt(value.flowDeltaZ());
output.writeBoolean(value.source());
}
private HydrologyFeatureRef readHydrologyFeatureRef(DataInputStream input) throws IOException {
return new HydrologyFeatureRef(
input.readLong(),
readEnum(input, HYDROLOGY_FEATURE_TYPE_VALUES),
input.readLong(),
input.readLong(),
input.readInt(),
input.readInt(),
input.readInt(),
input.readInt(),
input.readInt(),
input.readBoolean()
);
}
private void writeHydrologyPoint(DataOutputStream output, HydrologyPoint value) throws IOException {
output.writeInt(value.x());
output.writeInt(value.y());
output.writeInt(value.z());
}
private HydrologyPoint readHydrologyPoint(DataInputStream input) throws IOException {
return new HydrologyPoint(
input.readInt(),
input.readInt(),
input.readInt()
);
}
private void writeHydrologyCaveSource(DataOutputStream output, HydrologyCaveSource value) throws IOException {
output.writeLong(value.sourceId());
writeCavePosition(output, value.entry());
writeCavePosition(output, value.target());
output.writeInt(value.waterHeadY());
output.writeByte(value.mode().ordinal());
}
private HydrologyCaveSource readHydrologyCaveSource(DataInputStream input) throws IOException {
return new HydrologyCaveSource(
input.readLong(),
readCavePosition(input),
readCavePosition(input),
input.readInt(),
readEnum(input, HYDROLOGY_CAVE_MODE_VALUES)
);
}
private void writeCavePosition(DataOutputStream output, CavePosition value) throws IOException {
output.writeInt(value.x());
output.writeInt(value.y());
output.writeInt(value.z());
}
private CavePosition readCavePosition(DataInputStream input) throws IOException {
return new CavePosition(
input.readInt(),
input.readInt(),
input.readInt()
);
}
private void writeSurfaceRiverPolicy(DataOutputStream output, SurfaceRiverPolicy value) throws IOException {
writeString(output, value.areaKey());
writeDouble(output, value.sourceDensity());
writeInteger(output, value.sourceSpacing());
writeInteger(output, value.tributaries());
writeInteger(output, value.inlandOutlets());
writeInteger(output, value.coastalOutlets());
writeInteger(output, value.minimumCourseLength());
writeInteger(output, value.maximumIncision());
}
private SurfaceRiverPolicy readSurfaceRiverPolicy(DataInputStream input) throws IOException {
return new SurfaceRiverPolicy(
readString(input),
readDouble(input),
readInteger(input),
readInteger(input),
readInteger(input),
readInteger(input),
readInteger(input),
readInteger(input)
);
}
private void writeString(DataOutputStream output, String value) throws IOException {
output.writeBoolean(value != null);
if (value != null) {
allocate(40L + (long) value.length() * Character.BYTES);
output.writeUTF(value);
}
}
private String readString(DataInputStream input) throws IOException {
if (!input.readBoolean()) {
return null;
}
String value = input.readUTF();
allocate(40L + (long) value.length() * Character.BYTES);
if (strings.size() >= 4096) {
return value;
}
String existing = strings.putIfAbsent(value, value);
return existing == null ? value : existing;
}
private void writeInteger(DataOutputStream output, Integer value) throws IOException {
output.writeBoolean(value != null);
if (value != null) {
output.writeInt(value);
}
}
private Integer readInteger(DataInputStream input) throws IOException {
return input.readBoolean() ? input.readInt() : null;
}
private void writeDouble(DataOutputStream output, Double value) throws IOException {
output.writeBoolean(value != null);
if (value != null) {
output.writeDouble(value);
}
}
private Double readDouble(DataInputStream input) throws IOException {
return input.readBoolean() ? input.readDouble() : null;
}
private void writeOptionalLong(DataOutputStream output, OptionalLong value) throws IOException {
output.writeBoolean(value.isPresent());
if (value.isPresent()) {
output.writeLong(value.getAsLong());
}
}
private OptionalLong readOptionalLong(DataInputStream input) throws IOException {
return input.readBoolean() ? OptionalLong.of(input.readLong()) : OptionalLong.empty();
}
private <T> void writeList(DataOutputStream output, List<T> values, ElementWriter<T> writer) throws IOException {
writeCount(output, values.size(), 256);
for (T value : values) {
writer.write(output, value);
}
}
private <T> List<T> readList(DataInputStream input, ElementReader<T> reader) throws IOException {
int count = readCount(input, 256);
List<T> values = new ArrayList<>(Math.min(count, 4096));
for (int index = 0; index < count; index++) {
values.add(reader.read(input));
}
return values;
}
private int readCount(DataInputStream input, int bytesPerElement) throws IOException {
int count = input.readInt();
validateCount(count, bytesPerElement);
return count;
}
private void writeCount(DataOutputStream output, int count, int bytesPerElement) throws IOException {
validateCount(count, bytesPerElement);
output.writeInt(count);
}
private void validateCount(int count, int bytesPerElement) throws IOException {
if (count < 0 || count > MAXIMUM_COLLECTION_SIZE) {
throw new IOException("Invalid hydrology collection length: " + count);
}
allocate((long) count * bytesPerElement);
}
private void allocate(long bytes) throws IOException {
if (bytes > remainingAllocation) {
throw new IOException("Prepared hydrology tile exceeds its allocation budget.");
}
remainingAllocation -= bytes;
}
private <T extends Enum<T>> T readEnum(DataInputStream input, T[] values) throws IOException {
int ordinal = input.readUnsignedByte();
if (ordinal >= values.length) {
throw new IOException("Invalid hydrology enum value: " + ordinal);
}
return values[ordinal];
}
@FunctionalInterface
private interface ElementWriter<T> {
void write(DataOutputStream output, T value) throws IOException;
}
@FunctionalInterface
private interface ElementReader<T> {
T read(DataInputStream input) throws IOException;
}
}
@@ -1,24 +1,13 @@
package art.arcane.iris.generation.hydrology;
import art.arcane.iris.generation.hydrology.cave.CavePosition;
import art.arcane.iris.generation.hydrology.cave.CaveVoxelPrecondition;
import art.arcane.iris.generation.hydrology.cave.HydrologyCaveAction;
import art.arcane.iris.generation.hydrology.cave.HydrologyCavePlan;
import art.arcane.iris.generation.hydrology.cave.HydrologyCaveRejection;
import art.arcane.iris.generation.hydrology.cave.HydrologyCaveSource;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.FilterInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.io.Reader;
import java.io.Writer;
import java.nio.charset.StandardCharsets;
import java.nio.file.AtomicMoveNotSupportedException;
import java.nio.file.Files;
@@ -27,29 +16,15 @@ import java.nio.file.Path;
import java.nio.file.StandardCopyOption;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HexFormat;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.OptionalLong;
import java.util.Set;
import java.util.zip.GZIPInputStream;
import java.util.zip.GZIPOutputStream;
final class PreparedHydrologyTileStore {
private static final int SCHEMA_VERSION = 6;
private static final long MAXIMUM_COMPRESSED_BYTES = 128L * 1024L * 1024L;
private static final long MAXIMUM_DECOMPRESSED_BYTES = 512L * 1024L * 1024L;
private static final Gson GSON = new GsonBuilder()
.disableHtmlEscaping()
.serializeSpecialFloatingPointValues()
.create();
private final Path directory;
private final String scopeIdentity;
private final HydrologyTileCache.SharedCacheScope scope;
@@ -68,32 +43,27 @@ final class PreparedHydrologyTileStore {
.resolve(scopeIdentity);
}
Optional<HydrologyTile> load(HydrologyTileKey key) {
boolean contains(HydrologyTileKey key) {
Path file = file(key);
try {
if (!Files.isRegularFile(file, LinkOption.NOFOLLOW_LINKS)
|| Files.isSymbolicLink(file)
|| Files.size(file) > MAXIMUM_COMPRESSED_BYTES) {
return Optional.empty();
}
return Files.isRegularFile(file, LinkOption.NOFOLLOW_LINKS)
&& !Files.isSymbolicLink(file)
&& Files.size(file) <= MAXIMUM_COMPRESSED_BYTES;
} catch (IOException failure) {
return false;
}
}
Optional<HydrologyTile> load(HydrologyTileKey key) {
if (!contains(key)) {
return Optional.empty();
}
try (InputStream raw = Files.newInputStream(file);
try (InputStream raw = Files.newInputStream(file(key));
InputStream buffered = new BufferedInputStream(raw);
InputStream compressed = new GZIPInputStream(buffered);
InputStream compressed = new GZIPInputStream(buffered, 32768);
InputStream bounded = new LimitedInputStream(compressed, MAXIMUM_DECOMPRESSED_BYTES);
Reader reader = new InputStreamReader(bounded, StandardCharsets.UTF_8)) {
PersistedTile persisted = GSON.fromJson(reader, PersistedTile.class);
if (persisted == null
|| persisted.schemaVersion() != SCHEMA_VERSION
|| !scopeIdentity.equals(persisted.scopeIdentity())
|| !key.equals(persisted.key())
|| persisted.worldSeed() != scope.worldSeed()
|| persisted.settingsFingerprint() != scope.settingsFingerprint()) {
return Optional.empty();
}
HydrologyTile tile = persisted.toTile();
DataInputStream input = new DataInputStream(new BufferedInputStream(bounded, 32768))) {
HydrologyTile tile = new HydrologyTileCodec().read(input, scopeIdentity, key, scope, expectedTileSize);
return valid(tile, key) ? Optional.of(tile) : Optional.empty();
} catch (IOException | RuntimeException failure) {
return Optional.empty();
@@ -110,9 +80,9 @@ final class PreparedHydrologyTileStore {
try {
try (OutputStream raw = Files.newOutputStream(staged);
OutputStream buffered = new BufferedOutputStream(raw);
OutputStream compressed = new GZIPOutputStream(buffered);
Writer writer = new OutputStreamWriter(compressed, StandardCharsets.UTF_8)) {
GSON.toJson(PersistedTile.from(tile, scopeIdentity), writer);
OutputStream compressed = new GZIPOutputStream(buffered, 32768);
DataOutputStream output = new DataOutputStream(new BufferedOutputStream(compressed, 32768))) {
new HydrologyTileCodec().write(output, tile, scopeIdentity);
}
if (Files.size(staged) > MAXIMUM_COMPRESSED_BYTES) {
throw new IOException("Prepared hydrology tile cache entry exceeds the size limit.");
@@ -128,7 +98,7 @@ final class PreparedHydrologyTileStore {
}
Path file(HydrologyTileKey key) {
return directory.resolve("tile-" + key.tileX() + "-" + key.tileZ() + ".json.gz");
return directory.resolve("tile-" + key.tileX() + "-" + key.tileZ() + ".bin.gz");
}
private boolean valid(HydrologyTile tile, HydrologyTileKey key) {
@@ -162,194 +132,6 @@ final class PreparedHydrologyTileStore {
digest.update(bytes);
}
private record PersistedTile(
int schemaVersion,
String scopeIdentity,
HydrologyTileKey key,
long worldSeed,
long settingsFingerprint,
int tileSize,
List<DrainageNode> nodes,
List<DrainageEdge> edges,
List<RiverOutlet> outlets,
List<PersistedCourse> courses,
List<Long> regionalCourseIds,
List<PersistedCavePlan> cavePlans,
List<HydrologyDiagnosticCandidate> localDiagnosticCandidates,
List<HydrologyColumnSample> columns
) {
private static PersistedTile from(HydrologyTile tile, String scopeIdentity) {
ArrayList<PersistedCourse> courses = new ArrayList<>(tile.courses().size());
for (RiverCourse course : tile.courses()) {
courses.add(PersistedCourse.from(course));
}
ArrayList<Long> regionalCourseIds = new ArrayList<>(tile.regionalCourseIds());
regionalCourseIds.sort(Long::compareTo);
ArrayList<PersistedCavePlan> cavePlans = new ArrayList<>(tile.cavePlans().size());
for (HydrologyCavePlan cavePlan : tile.cavePlans()) {
cavePlans.add(PersistedCavePlan.from(cavePlan));
}
return new PersistedTile(
SCHEMA_VERSION,
scopeIdentity,
tile.key(),
tile.worldSeed(),
tile.settingsFingerprint(),
tile.tileSize(),
tile.nodes(),
tile.edges(),
tile.outlets(),
courses,
regionalCourseIds,
cavePlans,
tile.localDiagnosticCandidates(),
new ArrayList<>(tile.footprint().columns().values())
);
}
private HydrologyTile toTile() {
requireCollection(nodes, "nodes");
requireCollection(edges, "edges");
requireCollection(outlets, "outlets");
requireCollection(courses, "courses");
requireCollection(regionalCourseIds, "regionalCourseIds");
requireCollection(cavePlans, "cavePlans");
requireCollection(localDiagnosticCandidates, "localDiagnosticCandidates");
requireCollection(columns, "columns");
ArrayList<RiverCourse> restoredCourses = new ArrayList<>(courses.size());
for (PersistedCourse course : courses) {
restoredCourses.add(Objects.requireNonNull(course, "course").toCourse());
}
ArrayList<HydrologyCavePlan> restoredCavePlans = new ArrayList<>(cavePlans.size());
for (PersistedCavePlan cavePlan : cavePlans) {
restoredCavePlans.add(Objects.requireNonNull(cavePlan, "cavePlan").toPlan());
}
HashMap<Long, HydrologyColumnSample> restoredColumns = HashMap.newHashMap(columns.size());
for (HydrologyColumnSample column : columns) {
HydrologyColumnSample required = Objects.requireNonNull(column, "column");
HydrologyColumnSample existing = restoredColumns.put(
RiverFootprint.pack(required.x(), required.z()), required);
if (existing != null) {
throw new IllegalArgumentException("Duplicate hydrology footprint column.");
}
}
return new HydrologyTile(
key,
worldSeed,
settingsFingerprint,
tileSize,
nodes,
edges,
outlets,
restoredCourses,
Set.copyOf(regionalCourseIds),
restoredCavePlans,
localDiagnosticCandidates,
new RiverFootprint(restoredColumns)
);
}
private static void requireCollection(List<?> values, String name) {
Objects.requireNonNull(values, name);
}
}
private record PersistedCourse(
long id,
RiverCourseType type,
Long sourceNodeId,
Long outletId,
String profileKey,
int discharge,
List<DrainageEdge> drainageEdges,
List<HydraulicSegment> segments
) {
private static PersistedCourse from(RiverCourse course) {
return new PersistedCourse(
course.id(),
course.type(),
course.sourceNodeId().isPresent() ? course.sourceNodeId().getAsLong() : null,
course.outletId().isPresent() ? course.outletId().getAsLong() : null,
course.profileKey(),
course.discharge(),
course.drainageEdges(),
course.segments()
);
}
private RiverCourse toCourse() {
return new RiverCourse(
id,
type,
sourceNodeId == null ? OptionalLong.empty() : OptionalLong.of(sourceNodeId),
outletId == null ? OptionalLong.empty() : OptionalLong.of(outletId),
profileKey,
discharge,
drainageEdges,
segments
);
}
}
private record PersistedCavePlan(
HydrologyCaveSource source,
HydrologyCaveRejection rejection,
List<PersistedCaveCell> cells,
Long arbitrationWinnerSourceId
) {
private static PersistedCavePlan from(HydrologyCavePlan plan) {
ArrayList<PersistedCaveCell> cells = new ArrayList<>(plan.baselinePreconditions().size());
plan.forEachPrecondition((CavePosition position, CaveVoxelPrecondition precondition) -> cells.add(
new PersistedCaveCell(position, plan.actions().get(position), precondition)));
cells.sort(Comparator.comparingInt((PersistedCaveCell cell) -> cell.position().x())
.thenComparingInt(cell -> cell.position().z())
.thenComparingInt(cell -> cell.position().y()));
return new PersistedCavePlan(
plan.source(),
plan.rejection(),
cells,
plan.arbitrationWinnerSourceId().isPresent()
? plan.arbitrationWinnerSourceId().getAsLong()
: null
);
}
private HydrologyCavePlan toPlan() {
LinkedHashMap<CavePosition, HydrologyCaveAction> actions = new LinkedHashMap<>();
LinkedHashMap<CavePosition, CaveVoxelPrecondition> preconditions = new LinkedHashMap<>();
for (PersistedCaveCell cell : Objects.requireNonNull(cells, "cells")) {
PersistedCaveCell required = Objects.requireNonNull(cell, "cell");
CaveVoxelPrecondition existing = preconditions.put(required.position(), required.precondition());
if (existing != null) {
throw new IllegalArgumentException("Duplicate hydrology cave cell.");
}
if (required.action() != null) {
actions.put(required.position(), required.action());
}
}
return new HydrologyCavePlan(
source,
rejection,
actions,
preconditions,
arbitrationWinnerSourceId == null
? OptionalLong.empty()
: OptionalLong.of(arbitrationWinnerSourceId)
);
}
}
private record PersistedCaveCell(
CavePosition position,
HydrologyCaveAction action,
CaveVoxelPrecondition precondition
) {
private PersistedCaveCell {
Objects.requireNonNull(position, "position");
Objects.requireNonNull(precondition, "precondition");
}
}
private static final class LimitedInputStream extends FilterInputStream {
private final long limit;
private long count;
@@ -5,6 +5,9 @@ import it.unimi.dsi.fastutil.HashCommon;
import it.unimi.dsi.fastutil.longs.Long2ObjectMap;
import it.unimi.dsi.fastutil.longs.Long2ObjectOpenHashMap;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import java.util.AbstractCollection;
import java.util.AbstractList;
import java.util.AbstractMap;
@@ -49,7 +52,11 @@ public record HydrologyCavePlan(
&& (!requestedActions.isEmpty() || !requestedPreconditions.isEmpty())) {
throw new IllegalArgumentException("Rejected cave plans cannot contain mutations or preconditions");
}
if (!requestedPreconditions.keySet().containsAll(requestedActions.keySet())) {
boolean packedInput = requestedActions instanceof ActionMap actionMap
&& requestedPreconditions instanceof PreconditionMap preconditionMap
&& actionMap.packed == preconditionMap.packed
&& actionMap.spatialIndex == preconditionMap.spatialIndex;
if (!packedInput && !requestedPreconditions.keySet().containsAll(requestedActions.keySet())) {
throw new IllegalArgumentException("Every cave action requires a baseline precondition");
}
if (rejection != HydrologyCaveRejection.OVERLAPPING_SOURCE && arbitrationWinnerSourceId.isPresent()) {
@@ -58,7 +65,7 @@ public record HydrologyCavePlan(
if (requestedPreconditions.isEmpty()) {
actions = Map.of();
baselinePreconditions = Map.of();
} else {
} else if (!packedInput) {
PackedPositions packed = new PackedPositions(requestedActions, requestedPreconditions);
PositionSpatialIndex spatialIndex = new PositionSpatialIndex(packed);
actions = new ActionMap(packed, requestedActions.size(), spatialIndex);
@@ -66,6 +73,53 @@ public record HydrologyCavePlan(
}
}
public static HydrologyCavePlan readCells(HydrologyCaveSource source, HydrologyCaveRejection rejection,
OptionalLong winner, DataInput input, int count) throws IOException {
if (count < 0 || count > 2_000_000) {
throw new IOException("Invalid hydrology cave cell count: " + count);
}
int actionCount = input.readInt();
if (actionCount < 0 || actionCount > count) {
throw new IOException("Invalid hydrology cave action count: " + actionCount);
}
if (count == 0) {
return new HydrologyCavePlan(source, rejection, Map.of(), Map.of(), winner);
}
PackedPositions packed = new PackedPositions(input, count, actionCount);
int[] order = new int[count];
boolean[] seen = new boolean[count];
for (int index = 0; index < count; index++) {
int row = input.readInt();
if (row < 0 || row >= count || seen[row]) {
throw new IOException("Invalid hydrology cave precondition order.");
}
seen[row] = true;
order[index] = row;
}
PositionSpatialIndex spatialIndex = new PositionSpatialIndex(packed);
return new HydrologyCavePlan(source, rejection,
new ActionMap(packed, actionCount, spatialIndex),
new PreconditionMap(packed, new PositionOrder(packed, order), spatialIndex), winner);
}
public void writeCells(DataOutput output) throws IOException {
output.writeInt(baselinePreconditions.size());
output.writeInt(actions.size());
if (!(baselinePreconditions instanceof PreconditionMap preconditions)) {
return;
}
PackedPositions packed = preconditions.packed;
for (int row = 0; row < packed.size(); row++) {
output.writeInt(packed.x[row]);
output.writeInt(packed.y[row]);
output.writeInt(packed.z[row]);
output.writeByte(packed.flags[row]);
}
for (int row : preconditions.order.rows) {
output.writeInt(row);
}
}
public boolean accepted() {
return rejection == HydrologyCaveRejection.NONE;
}
@@ -204,6 +258,32 @@ public record HydrologyCavePlan(
private final int mask;
private int count;
private PackedPositions(DataInput input, int size, int actionCount) throws IOException {
x = new int[size];
y = new int[size];
z = new int[size];
flags = new byte[size];
int required = (int) Math.max(2L, ((long) size * 4L + 2L) / 3L);
int capacity = Integer.highestOneBit(required - 1) << 1;
slots = new int[capacity];
mask = capacity - 1;
for (int row = 0; row < size; row++) {
CavePosition position = new CavePosition(input.readInt(), input.readInt(), input.readInt());
byte value = input.readByte();
int unsigned = Byte.toUnsignedInt(value);
if ((unsigned & 0x80) != 0 || (unsigned & VOXEL_MASK) >>> VOXEL_SHIFT >= VOXELS.length
|| hasAction(value) != (row < actionCount)
|| (hasAction(value) && (unsigned & ACTION_MASK) >= ACTIONS.length)
|| (!hasAction(value) && (unsigned & ACTION_MASK) != 0)) {
throw new IOException("Invalid hydrology cave cell flags.");
}
if (row(position) >= 0) {
throw new IOException("Duplicate hydrology cave cell.");
}
append(position, value);
}
}
private PackedPositions(Map<CavePosition, HydrologyCaveAction> actions,
Map<CavePosition, CaveVoxelPrecondition> preconditions) {
int size = preconditions.size();
@@ -577,6 +577,9 @@ public interface MatterGenerator {
private volatile Future<?> submission;
private volatile Throwable failure;
private boolean started;
private boolean slowWarningIssued;
private Thread runningThread;
private long startedNanos;
MatterComponentTask(MatterTaskKey key) {
this.key = key;
@@ -587,6 +590,8 @@ public interface MatterGenerator {
return false;
}
started = true;
runningThread = Thread.currentThread();
startedNanos = System.nanoTime();
return true;
}
@@ -612,6 +617,7 @@ public interface MatterGenerator {
}
synchronized void finish(Throwable cause) {
runningThread = null;
if (failure == null) {
failure = cause;
} else if (cause != null && cause != failure) {
@@ -644,13 +650,14 @@ public interface MatterGenerator {
Future<?> submitted = submission;
boolean dropped = submitted != null && submitted.isDone();
long waited = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start);
if (!dropped && waited < timeoutMillis) {
continue;
}
IllegalStateException timeoutFailure = new IllegalStateException("Mantle component " + key
+ (dropped ? " was dropped by the dispatcher" : " did not complete in " + waited + "ms"));
if (cancel(timeoutFailure)) {
IrisLogging.error(timeoutFailure.getMessage());
if (dropped) {
IllegalStateException dispatchFailure = new IllegalStateException(
"Mantle component " + key + " was dropped by the dispatcher");
if (cancel(dispatchFailure)) {
IrisLogging.reportError(dispatchFailure.getMessage(), dispatchFailure);
}
} else if (waited >= timeoutMillis) {
warnSlowComponent(waited);
}
} catch (ExecutionException executionFailure) {
Throwable cause = executionFailure.getCause();
@@ -663,6 +670,33 @@ public interface MatterGenerator {
}
}
}
private void warnSlowComponent(long waited) {
Thread worker;
long runningMillis;
synchronized (this) {
if (slowWarningIssued || future.isDone() || failure != null) {
return;
}
slowWarningIssued = true;
worker = runningThread;
runningMillis = started ? TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startedNanos) : 0L;
}
StringBuilder diagnostic = new StringBuilder("Mantle component ")
.append(key).append(" is still pending after ").append(waited)
.append("ms; waiting for completion. ");
if (worker == null) {
diagnostic.append("Task is queued for the dispatcher.");
} else {
diagnostic.append("Worker=").append(worker.getName())
.append(" state=").append(worker.getState())
.append(" runningForMs=").append(runningMillis);
for (StackTraceElement frame : worker.getStackTrace()) {
diagnostic.append("\n\tat ").append(frame);
}
}
IrisLogging.warn(diagnostic.toString());
}
}
record MatterPassPlan(MantlePass pass, int passChunkRadius, int downstreamBlockRadius) {
@@ -424,10 +424,6 @@ final class ObjectDestinationTransaction implements ObjectPassPlacer {
DataKey key();
void apply(IObjectPlacer placer);
default int weight() {
return 1;
}
}
record SetMutation(DataKey key, Object value) implements Mutation {
@@ -462,10 +458,5 @@ final class ObjectDestinationTransaction implements ObjectPassPlacer {
public void apply(IObjectPlacer placer) {
placer.set(key.x(), key.y(), key.z(), state);
}
@Override
public int weight() {
return 2;
}
}
}
@@ -1,5 +1,10 @@
package art.arcane.iris.generation.mantle;
import art.arcane.iris.generation.decoration.tree.TreeBlockMaterial;
import art.arcane.iris.generation.hydrology.cave.HydrologyCaveCell;
import art.arcane.iris.integration.Identifier;
import art.arcane.volmlib.nativelib.terrain.NativeBlockState;
import art.arcane.volmlib.util.matter.MatterCavern;
import it.unimi.dsi.fastutil.longs.Long2ObjectMap;
import it.unimi.dsi.fastutil.longs.Long2ObjectOpenHashMap;
@@ -7,29 +12,35 @@ import java.util.ArrayList;
import java.util.List;
final class ObjectSourcePlan {
private static final int DESTINATION_INDEX_WEIGHT = 8;
private static final int PLAN_BYTES = 256;
private static final int DESTINATION_INDEX_BYTES = 96;
private static final int MUTATION_BYTES = 96;
private final Long2ObjectOpenHashMap<List<ObjectDestinationTransaction.Mutation>> destinations;
private final int mutationWeight;
private final int estimatedRetainedBytes;
ObjectSourcePlan(List<ObjectDestinationTransaction.Mutation> mutations) {
Long2ObjectOpenHashMap<ArrayList<ObjectDestinationTransaction.Mutation>> grouped = new Long2ObjectOpenHashMap<>(2);
long weight = 1L;
long weight = PLAN_BYTES;
for (ObjectDestinationTransaction.Mutation mutation : mutations) {
long key = destinationKey(mutation.x() >> 4, mutation.z() >> 4);
ArrayList<ObjectDestinationTransaction.Mutation> local = grouped.get(key);
if (local == null) {
local = new ArrayList<>();
grouped.put(key, local);
weight += DESTINATION_INDEX_WEIGHT;
weight += DESTINATION_INDEX_BYTES;
}
local.add(mutation);
weight += mutation.weight();
Object payload = switch (mutation) {
case ObjectDestinationTransaction.SetMutation set -> set.value();
case ObjectDestinationTransaction.CustomBlockMutation custom -> custom.state();
};
weight = Math.min(Integer.MAX_VALUE, weight + MUTATION_BYTES + payloadBytes(payload));
}
destinations = new Long2ObjectOpenHashMap<>(grouped.size());
for (Long2ObjectMap.Entry<ArrayList<ObjectDestinationTransaction.Mutation>> entry : grouped.long2ObjectEntrySet()) {
destinations.put(entry.getLongKey(), List.copyOf(entry.getValue()));
}
this.mutationWeight = weight >= Integer.MAX_VALUE ? Integer.MAX_VALUE : (int) weight;
this.estimatedRetainedBytes = weight >= Integer.MAX_VALUE ? Integer.MAX_VALUE : (int) weight;
}
List<ObjectDestinationTransaction.Mutation> mutationsFor(int chunkX, int chunkZ) {
@@ -37,8 +48,34 @@ final class ObjectSourcePlan {
return mutations == null ? List.of() : mutations;
}
int mutationWeight() {
return mutationWeight;
int estimatedRetainedBytes() {
return estimatedRetainedBytes;
}
private static long payloadBytes(Object value) {
return switch (value) {
case null -> 0L;
case String text -> stringBytes(text);
case TreeBlockMaterial material -> 24L + stringBytes(material.materialKey());
case Identifier identifier -> 32L + stringBytes(identifier.namespace()) + stringBytes(identifier.key());
case NativeBlockState ignored -> 64L;
case MatterCavern cavern -> 32L + stringBytes(cavern.getCustomBiome());
case HydrologyCaveCell cell -> 64L + stringBytes(cell.fluidProfileKey()) + stringBytes(cell.floodedBiomeKey());
case Byte ignored -> 24L;
case Short ignored -> 24L;
case Integer ignored -> 24L;
case Long ignored -> 24L;
case Float ignored -> 24L;
case Double ignored -> 24L;
case Boolean ignored -> 24L;
case Character ignored -> 24L;
case Enum<?> ignored -> 0L;
default -> Integer.MAX_VALUE;
};
}
private static long stringBytes(String value) {
return value == null ? 0L : 48L + 2L * value.length();
}
private static long destinationKey(int chunkX, int chunkZ) {
@@ -13,21 +13,25 @@ import java.util.function.Supplier;
import static art.arcane.iris.generation.cache.Cache.key;
final class ObjectSourcePlanCache {
static final long MAXIMUM_MUTATION_WEIGHT = 4_194_304L;
private static final long MAXIMUM_RETAINED_BYTES = 256L * 1024L * 1024L;
private final long maximumMutationWeight;
private final long maximumRetainedBytes;
private volatile State state;
ObjectSourcePlanCache() {
this(MAXIMUM_MUTATION_WEIGHT);
this(retainedByteBudget(Runtime.getRuntime().maxMemory()));
}
ObjectSourcePlanCache(long maximumMutationWeight) {
if (maximumMutationWeight <= 0L) {
throw new IllegalArgumentException("Maximum mutation weight must be positive");
ObjectSourcePlanCache(long maximumRetainedBytes) {
if (maximumRetainedBytes <= 0L) {
throw new IllegalArgumentException("Maximum retained bytes must be positive");
}
this.maximumMutationWeight = maximumMutationWeight;
state = new State(maximumMutationWeight);
this.maximumRetainedBytes = maximumRetainedBytes;
state = new State(maximumRetainedBytes);
}
static long retainedByteBudget(long maximumHeapBytes) {
return Math.max(1L, Math.min(MAXIMUM_RETAINED_BYTES, maximumHeapBytes / 64L));
}
ObjectSourcePlan get(int sourceChunkX, int sourceChunkZ, Supplier<ObjectSourcePlan> builder) {
@@ -47,7 +51,8 @@ final class ObjectSourcePlanCache {
ObjectSourcePlan plan = current.plans.getIfPresent(source);
if (plan == null) {
plan = builder.get();
if (plan != null) {
if (plan != null && plan.estimatedRetainedBytes() < Integer.MAX_VALUE
&& plan.estimatedRetainedBytes() <= maximumRetainedBytes) {
current.plans.put(source, plan);
}
}
@@ -63,7 +68,7 @@ final class ObjectSourcePlanCache {
}
void clear() {
state = new State(maximumMutationWeight);
state = new State(maximumRetainedBytes);
}
long estimatedSize() {
@@ -76,10 +81,10 @@ final class ObjectSourcePlanCache {
private final Cache<Long, ObjectSourcePlan> plans;
private final ConcurrentHashMap<Long, Pending> pending = new ConcurrentHashMap<>();
private State(long maximumMutationWeight) {
private State(long maximumRetainedBytes) {
plans = Caffeine.newBuilder()
.maximumWeight(maximumMutationWeight)
.weigher((Long key, ObjectSourcePlan plan) -> plan.mutationWeight())
.maximumWeight(maximumRetainedBytes)
.weigher((Long key, ObjectSourcePlan plan) -> plan.estimatedRetainedBytes())
.build();
}
}
@@ -95,6 +95,11 @@ public class AsyncOrMedievalPregenMethod implements PregeneratorMethod {
method.onRegionSubmitted(regionX, regionZ);
}
@Override
public void reclaimMemory() {
method.reclaimMemory();
}
@Override
public Mantle getMantle() {
return method.getMantle();
@@ -31,7 +31,6 @@ import art.arcane.iris.platform.bukkit.nms.INMSBinding;
import art.arcane.iris.generation.runtime.Engine;
import art.arcane.iris.generation.hydrology.HydrologyTileCache;
import art.arcane.iris.platform.bukkit.BukkitPlatform;
import art.arcane.volmlib.util.collection.KSet;
import art.arcane.volmlib.util.mantle.runtime.Mantle;
import art.arcane.volmlib.util.math.M;
import art.arcane.iris.generation.concurrent.MultiBurst;
@@ -97,9 +96,6 @@ public class AsyncPregenMethod implements PregeneratorMethod {
private final ConcurrentHashMap<Long, AtomicInteger> regionPending;
private final ConcurrentHashMap<Long, Queue<Chunk>> regionChunks;
private final Queue<CompletableFuture<Void>> pendingEvictions;
private final KSet<Long> drainedRegions;
private final KSet<Long> evictedRegions;
private volatile int evictionWindowRegions;
private volatile int boundsMinRegionX;
private volatile int boundsMinRegionZ;
private volatile int boundsMaxRegionX;
@@ -164,6 +160,7 @@ public class AsyncPregenMethod implements PregeneratorMethod {
int configuredThreads = foliaRuntime
? computeFoliaRecommendedCap(workerThreadsForCap)
: computePaperLikeRecommendedCap(workerThreadsForCap);
configuredThreads = heapLimitedConcurrency(configuredThreads, Runtime.getRuntime().maxMemory());
this.threads = selectConcurrencyCap(configuredThreads, strictSerial);
this.workerPoolThreads = detectedWorkerPoolThreads;
this.runtimeCpuThreads = detectedCpuThreads;
@@ -183,9 +180,6 @@ public class AsyncPregenMethod implements PregeneratorMethod {
this.regionPending = new ConcurrentHashMap<>();
this.regionChunks = new ConcurrentHashMap<>();
this.pendingEvictions = new ConcurrentLinkedQueue<>();
this.drainedRegions = new KSet<>();
this.evictedRegions = new KSet<>();
this.evictionWindowRegions = -1;
this.boundsMinRegionX = Integer.MIN_VALUE;
this.boundsMinRegionZ = Integer.MIN_VALUE;
this.boundsMaxRegionX = Integer.MAX_VALUE;
@@ -232,28 +226,6 @@ public class AsyncPregenMethod implements PregeneratorMethod {
return (((long) rx) << 32) | (rz & 0xFFFFFFFFL);
}
private int evictionWindow() {
int cached = evictionWindowRegions;
if (cached > 0) {
return cached;
}
Engine engine = resolveMetricsEngine();
if (engine == null) {
return 2;
}
try {
int radius = engine.getMantle().getRadius();
int resolved = radius > 0 ? Math.max(1, (int) Math.ceil(radius / 32.0)) : 2;
evictionWindowRegions = resolved;
return resolved;
} catch (Throwable e) {
PregenDiagnostics.probeFailed("mantle radius for eviction window", e);
return 2;
}
}
private void onChunkCompleted(int x, int z, Chunk chunk) {
if (chunk == null) {
return;
@@ -261,10 +233,16 @@ public class AsyncPregenMethod implements PregeneratorMethod {
try {
long rk = rkey(x >> 5, z >> 5);
regionChunks.computeIfAbsent(rk, k -> new ConcurrentLinkedQueue<>()).add(chunk);
regionChunks.compute(rk, (key, existing) -> {
Queue<Chunk> chunks = existing == null ? new ConcurrentLinkedQueue<>() : existing;
chunks.add(chunk);
return chunks;
});
AtomicInteger pending = regionPending.get(rk);
if (pending != null && pending.decrementAndGet() == 0) {
onRegionDrained(rk);
} else if (MantleHeapPressure.overHighWater()) {
evictRegion(rk);
}
} catch (Throwable e) {
IrisLogging.reportError(e);
@@ -317,10 +295,6 @@ public class AsyncPregenMethod implements PregeneratorMethod {
}
}
private boolean inBounds(int rx, int rz) {
return rx >= boundsMinRegionX && rx <= boundsMaxRegionX && rz >= boundsMinRegionZ && rz <= boundsMaxRegionZ;
}
@Override
public void onRegionSubmitted(int regionX, int regionZ) {
try {
@@ -335,58 +309,14 @@ public class AsyncPregenMethod implements PregeneratorMethod {
}
private void onRegionDrained(long rk) {
if (!drainedRegions.add(rk)) {
return;
}
int w = evictionWindow();
int rx = (int) (rk >> 32);
int rz = (int) rk;
for (int dx = -w; dx <= w; dx++) {
for (int dz = -w; dz <= w; dz++) {
int cx = rx + dx;
int cz = rz + dz;
long candidate = rkey(cx, cz);
if (evictedRegions.contains(candidate)) {
continue;
}
if (!drainedRegions.contains(candidate)) {
continue;
}
if (allNeighborsDrained(cx, cz, w)) {
evictRegion(candidate);
}
}
}
}
private boolean allNeighborsDrained(int rx, int rz, int w) {
for (int dx = -w; dx <= w; dx++) {
for (int dz = -w; dz <= w; dz++) {
int nx = rx + dx;
int nz = rz + dz;
if (!inBounds(nx, nz)) {
continue;
}
if (!drainedRegions.contains(rkey(nx, nz))) {
return false;
}
}
}
return true;
regionPending.remove(rk);
evictRegion(rk);
}
private CompletableFuture<Void> evictRegion(long c) {
if (IrisToolbelt.isServerStopping()) {
return CompletableFuture.completedFuture(null);
}
if (!evictedRegions.add(c)) {
return CompletableFuture.completedFuture(null);
}
regionPending.remove(c);
Queue<Chunk> chunks = regionChunks.remove(c);
if (chunks == null || chunks.isEmpty()) {
return CompletableFuture.completedFuture(null);
@@ -651,6 +581,11 @@ public class AsyncPregenMethod implements PregeneratorMethod {
return strictSerial ? 1 : Math.max(1, recommendedCap);
}
static int heapLimitedConcurrency(int recommendedCap, long maximumHeapBytes) {
long heapCap = Math.max(1L, maximumHeapBytes / (256L * 1024L * 1024L));
return (int) Math.min(Math.max(1, recommendedCap), heapCap);
}
static int computeInitialInFlightLimit(int concurrencyCap, int workerThreads) {
long initial = Math.max(1L, workerThreads) * 8L;
return (int) Math.min(Math.max(1, concurrencyCap), initial);
@@ -880,6 +815,14 @@ public class AsyncPregenMethod implements PregeneratorMethod {
public void save() {
}
@Override
public void reclaimMemory() {
if (regionChunks.isEmpty() && pendingEvictions.isEmpty()) {
return;
}
flushAllRemainingChunks();
}
@Override
public boolean supportsRegions(int x, int z, PregenListener listener) {
return false;
@@ -893,8 +836,11 @@ public class AsyncPregenMethod implements PregeneratorMethod {
@Override
public void generateChunk(int x, int z, PregenListener listener) {
listener.onChunkGenerating(x, z);
if (MantleHeapPressure.overHighWater()) {
reclaimMemory();
}
backpressure.enforceMantleBudget();
backpressure.awaitHeapHeadroom();
backpressure.awaitHeapHeadroom(MantleHeapPressure::overHighWater, this::reclaimHeapPressure);
if (isCancelled()) {
return;
}
@@ -924,6 +870,13 @@ public class AsyncPregenMethod implements PregeneratorMethod {
executor.generate(x, z, listener);
}
private void reclaimHeapPressure() {
if (!regionChunks.isEmpty() || !pendingEvictions.isEmpty()) {
reclaimMemory();
}
MantleHeapPressure.requestPanicReclaim();
}
private CompletableFuture<Chunk> requestChunkAsync(int x, int z) {
Throwable failure = null;
@@ -203,6 +203,11 @@ public class CachedPregenMethod implements PregeneratorMethod {
PregenSavedChunkStatus savedStatus) {
}
@Override
public void reclaimMemory() {
method.reclaimMemory();
}
@Override
public Mantle getMantle() {
return method.getMantle();
@@ -88,6 +88,11 @@ public class HybridPregenMethod implements PregeneratorMethod {
inWorld.onRegionSubmitted(regionX, regionZ);
}
@Override
public void reclaimMemory() {
inWorld.reclaimMemory();
}
@Override
public Mantle getMantle() {
return inWorld.getMantle();
@@ -438,6 +438,12 @@ public class IrisPregenerator {
return;
}
try {
generator.reclaimMemory();
} catch (Throwable e) {
IrisLogging.reportError("Pregen could not reclaim completed server chunks under heap pressure.", e);
}
try {
Mantle mantle = getMantle();
if (mantle != null) {
@@ -29,6 +29,8 @@ import java.util.function.BooleanSupplier;
import java.util.function.Supplier;
public final class PregenMantleBackpressure {
private static final long LOG_INTERVAL_MS = 5_000L;
private final Supplier<Mantle> mantleSupplier;
private final int maxResidentTectonicPlates;
private final int waitMs;
@@ -46,7 +48,7 @@ public final class PregenMantleBackpressure {
public PregenMantleBackpressure(Supplier<Mantle> mantleSupplier, int maxResidentTectonicPlates, int waitMs, long timeoutMs, Runnable onBudgetTimeout, Supplier<String> diagnostics, BooleanSupplier cancelled) {
this.mantleSupplier = mantleSupplier;
this.maxResidentTectonicPlates = maxResidentTectonicPlates;
this.waitMs = waitMs;
this.waitMs = Math.max(1, waitMs);
this.timeoutMs = timeoutMs;
this.onBudgetTimeout = onBudgetTimeout;
this.diagnostics = diagnostics;
@@ -69,14 +71,13 @@ public final class PregenMantleBackpressure {
return;
}
int hardCap = cap * 2;
if (mantle.getLoadedRegionCount() <= hardCap) {
if (mantle.getLoadedRegionCount() <= cap) {
return;
}
long waitStart = M.ms();
long lastLog = 0L;
while (mantle.getLoadedRegionCount() > hardCap) {
while (mantle.getLoadedRegionCount() > cap) {
if (isCancelled()) {
return;
}
@@ -87,29 +88,34 @@ public final class PregenMantleBackpressure {
mantle.trim(0L);
freed = mantle.unloadTectonicPlate(0);
resident = mantle.getLoadedRegionCount();
if (resident > cap && mantle.saveOldestIdleTectonicPlate()) {
freed++;
resident = mantle.getLoadedRegionCount();
}
} catch (Throwable e) {
IrisLogging.reportError(e);
break;
}
if (resident <= hardCap) {
if (resident <= cap) {
break;
}
long elapsed = M.ms() - waitStart;
if (elapsed >= timeoutMs) {
if (elapsed >= timeoutMs && freed == 0) {
IrisLogging.warn("Pregen mantle backpressure exceeded " + timeoutMs + "ms with " + resident
+ " tectonic plates resident (hard cap " + hardCap + "); proceeding to avoid deadlock. "
+ "Raise pregen.maxResidentTectonicPlates if this persists. " + diagnostics.get());
+ " tectonic plates resident (residency target " + cap + "); no idle plate could be evicted. "
+ "Allowing the active dependency working set to exceed the residency target; "
+ "heap pressure still blocks new chunk submissions. " + diagnostics.get());
onBudgetTimeout.run();
return;
}
long logNow = M.ms();
if (logNow - lastLog >= 5_000L) {
if (logNow - lastLog >= LOG_INTERVAL_MS) {
lastLog = logNow;
// Pausing to stay under the plate cap is the design working, and it is reported every five
// seconds for the whole duration of a large pregen. Only exceeding the budget is a warning.
IrisLogging.info("Pregen mantle backpressure: " + resident + " tectonic plates resident (hard cap " + hardCap
IrisLogging.info("Pregen mantle backpressure: " + resident + " tectonic plates resident (residency target " + cap
+ "), freed " + freed + " last pass, waited " + elapsed + "ms.");
}
@@ -127,6 +133,9 @@ public final class PregenMantleBackpressure {
Mantle mantle = resolveMantle();
long waitStart = M.ms();
long lastLog = 0L;
long nextWarningElapsed = Math.max(0L, timeoutMs);
long nextEvictionErrorLog = 0L;
long warningInterval = Math.max(LOG_INTERVAL_MS, timeoutMs);
while (heapPressure.getAsBoolean()) {
if (isCancelled()) {
return;
@@ -137,22 +146,30 @@ public final class PregenMantleBackpressure {
mantle.saveOldestIdleTectonicPlate();
}
} catch (Throwable e) {
IrisLogging.reportError(e);
long now = M.ms();
if (now >= nextEvictionErrorLog) {
nextEvictionErrorLog = now + warningInterval;
IrisLogging.reportError(e);
}
}
panicReclaim.run();
long elapsed = M.ms() - waitStart;
if (elapsed >= timeoutMs) {
IrisLogging.warn("Pregen heap pressure wait exceeded " + timeoutMs + "ms at "
+ Math.round(MantleHeapPressure.usedFraction() * 100.0D) + "% heap; proceeding to avoid deadlock. "
+ diagnostics.get());
onBudgetTimeout.run();
if (isCancelled() || !heapPressure.getAsBoolean()) {
return;
}
long elapsed = M.ms() - waitStart;
long logNow = M.ms();
if (logNow - lastLog >= 5_000L) {
if (elapsed >= nextWarningElapsed) {
nextWarningElapsed = elapsed + warningInterval;
lastLog = logNow;
IrisLogging.warn("Pregen heap pressure is still blocking new chunk submissions after " + elapsed + "ms at "
+ Math.round(MantleHeapPressure.usedFraction() * 100.0D) + "% heap; waiting for headroom. "
+ diagnostics.get());
onBudgetTimeout.run();
}
if (logNow - lastLog >= LOG_INTERVAL_MS) {
lastLog = logNow;
IrisLogging.info("Pregen heap pressure: pausing generation at "
+ Math.round(MantleHeapPressure.usedFraction() * 100.0D) + "% heap; evicting tectonic plates and waiting for headroom"
@@ -89,5 +89,8 @@ public interface PregeneratorMethod {
default void onRegionSubmitted(int regionX, int regionZ) {
}
default void reclaimMemory() {
}
Mantle getMantle();
}