Merge W2F-hydroplanner

core/src/main/java/art/arcane/iris/generation/hydrology/HydrologyPlanner.java

@@ -28,6 +28,8 @@ import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.CompletionException;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ForkJoinWorkerThread;
+import java.util.function.Function;
+import java.util.function.Predicate;
 
 public final class HydrologyPlanner {
     static final int ROUTING_CONTEXT_CACHE_SIZE = 64;
@@ -64,6 +66,8 @@ public final class HydrologyPlanner {
     final HydrologySegmentBuilder segments;
     final HydrologyFeatureSitePlanner featureSites;
     final HydrologyRegionalPlanner regional;
+    private volatile Predicate<HydrologyTileKey> earlyOwnerScope;
+    private volatile Function<HydrologyTileKey, HydrologyTile> plannedTiles;
 
     public HydrologyPlanner(long worldSeed, HydrologyPlannerSettings settings, HydrologyTerrainSampler sampler) {
         this(
@@ -244,6 +248,23 @@ public final class HydrologyPlanner {
         return List.copyOf(unique);
     }
 
+    /**
+     * Limits the lower-rank neighbour drafts an owner starts before its admission demands them to the
+     * tiles the scope accepts; null starts all of them. A neighbour that no admission demands and
+     * nothing plans is a draft computed for nothing, and so are the early neighbours it starts itself.
+     */
+    void limitEarlyOwners(Predicate<HydrologyTileKey> scope) {
+        earlyOwnerScope = scope;
+    }
+
+    /**
+     * Where the plans of tiles already planned are found. An owner draft that left the owner cache is
+     * restored from its tile's plan when a neighbour demands it again, instead of being drafted twice.
+     */
+    void usePlannedTiles(Function<HydrologyTileKey, HydrologyTile> source) {
+        plannedTiles = source;
+    }
+
     void clearOwnerDrafts() {
         resolvedOwners.invalidateAll();
         regional.clear();
@@ -289,6 +310,9 @@ public final class HydrologyPlanner {
             return local;
         }
         CrossTileResolvedOwner cached = resolvedOwners.getIfPresent(key);
+        if (cached == null) {
+            cached = plannedOwner(key);
+        }
         if (cached != null) {
             context.remember(key, cached);
             return cached;
@@ -335,6 +359,17 @@ public final class HydrologyPlanner {
         }
     }
 
+    private CrossTileResolvedOwner plannedOwner(HydrologyTileKey key) {
+        Function<HydrologyTileKey, HydrologyTile> source = plannedTiles;
+        HydrologyTile tile = source == null ? null : source.apply(key);
+        if (tile == null || tile.resolvedOwner() == null) {
+            return null;
+        }
+        reuseResolvedTile(tile);
+        CrossTileResolvedOwner restored = resolvedOwners.getIfPresent(key);
+        return restored == null ? tile.resolvedOwner() : restored;
+    }
+
     List<CrossTileResolvedOwner> resolveLowerRankOwners(
             List<HydrologyTileKey> candidateKeys,
             CrossTileResolutionContext context,
@@ -751,6 +786,7 @@ public final class HydrologyPlanner {
                 return;
             }
             int radius = settings.crossTileColorPeriod() - 1;
+            Predicate<HydrologyTileKey> scope = earlyOwnerScope;
             ArrayList<HydrologyTileKey> candidates = new ArrayList<>();
             for (long tileZ = (long) ownerKey.tileZ() - radius; tileZ <= (long) ownerKey.tileZ() + radius; tileZ++) {
                 for (long tileX = (long) ownerKey.tileX() - radius; tileX <= (long) ownerKey.tileX() + radius; tileX++) {
@@ -759,7 +795,8 @@ public final class HydrologyPlanner {
                         continue;
                     }
                     HydrologyTileKey key = new HydrologyTileKey((int) tileX, (int) tileZ);
-                    if (crossTile.ownerColorRank(key) >= ownerRank || preparedOwners.containsKey(key)
+                    if (crossTile.ownerColorRank(key) >= ownerRank || scope != null && !scope.test(key)
+                            || preparedOwners.containsKey(key)
                             || resolvedOwners.getIfPresent(key) != null || resolvingOwners.containsKey(key)) {
                         continue;
                     }

core/src/main/java/art/arcane/iris/generation/hydrology/HydrologyTileCache.java

@@ -161,6 +161,17 @@ public final class HydrologyTileCache implements AutoCloseable {
         this.closed = new AtomicBoolean();
         this.localChunks = ThreadLocal.withInitial(LocalChunks::new);
         this.neighbourPrefetchEnabled = true;
+        planner.usePlannedTiles(this::plannedTile);
+    }
+
+    /** A tile already planned, resident or in the prepared-plan store, loaded without planning it. */
+    private HydrologyTile plannedTile(HydrologyTileKey key) {
+        HydrologyTile resident = tiles.getIfPresent(key);
+        if (resident != null) {
+            return resident;
+        }
+        PreparedHydrologyTileStore store = persistentStore;
+        return store == null ? null : store.load(key).orElse(null);
     }
 
     /**
@@ -690,7 +701,7 @@ public final class HydrologyTileCache implements AutoCloseable {
                 tileCoordinate(Math.addExact(area.maximumBlockZ(), publicationRadius), tileSize));
         List<HydrologyTileKey> keys = nearestFirst(required, tileCoordinate(area.centerBlockX(), tileSize),
                 tileCoordinate(area.centerBlockZ(), tileSize), MAXIMUM_PREGENERATION_TILES);
-        PregenerationScope scope = new PregenerationScope(bounds);
+        PregenerationScope scope = new PregenerationScope(bounds, required);
         try {
             enqueuePrefetchArea(keys, scope);
             return scope;
@@ -1082,6 +1093,7 @@ public final class HydrologyTileCache implements AutoCloseable {
                 scope.previousNeighbourPrefetch = pregenerationScope == null
                         ? neighbourPrefetchEnabled : pregenerationScope.previousNeighbourPrefetch;
                 pregenerationScope = scope;
+                planner.limitEarlyOwners(scope.required::contains);
                 prefetchQueue.removeIf(key -> {
                     if (!area.contains(key)) {
                         queuedPrefetches.remove(key);
@@ -1619,12 +1631,19 @@ public final class HydrologyTileCache implements AutoCloseable {
         }
     }
 
+    /**
+     * Speculative planning during a pregeneration: tiles inside {@code bounds} may be planned ahead of
+     * generation, and owner drafts start their lower-rank neighbours early only inside {@code required},
+     * the tiles the area composes from.
+     */
     public final class PregenerationScope implements AutoCloseable {
         private final TileBounds bounds;
+        private final TileBounds required;
         private boolean previousNeighbourPrefetch;
 
-        private PregenerationScope(TileBounds bounds) {
+        private PregenerationScope(TileBounds bounds, TileBounds required) {
             this.bounds = bounds;
+            this.required = required;
         }
 
         @Override
@@ -1634,6 +1653,7 @@ public final class HydrologyTileCache implements AutoCloseable {
                     return;
                 }
                 pregenerationScope = null;
+                planner.limitEarlyOwners(null);
                 neighbourPrefetchEnabled = previousNeighbourPrefetch;
                 prefetchQueue.clear();
                 queuedPrefetches.clear();

core/src/test/java/art/arcane/iris/generation/hydrology/HydrologyPlannerTest.java

@@ -133,6 +133,33 @@ public class HydrologyPlannerTest {
         assertTileContentsEqual(tile, planner.plan(EARLY_OWNER_TILE));
     }
 
+    @Test
+    public void evictedOwnerIsRestoredFromItsPlannedTileWithoutDrafting() {
+        AtomicInteger samples = new AtomicInteger();
+        HydrologyTerrainSampler counted = (int x, int z) -> {
+            samples.incrementAndGet();
+            return EARLY_OWNER_TERRAIN.sample(x, z);
+        };
+        HydrologyPlanner planner = new HydrologyPlanner(77L, EARLY_OWNER_SETTINGS, counted);
+        HydrologyTile tile = planner.plan(EARLY_OWNER_TILE);
+        CrossTileResolvedOwner original = tile.resolvedOwner();
+        planner.clearOwnerDrafts();
+        AtomicInteger lookups = new AtomicInteger();
+        planner.usePlannedTiles(key -> {
+            lookups.incrementAndGet();
+            return key.equals(EARLY_OWNER_TILE) ? tile : null;
+        });
+        samples.set(0);
+
+        CrossTileResolvedOwner restored = planner.resolveIndependentOwner(EARLY_OWNER_TILE);
+
+        assertEquals(original, restored);
+        assertEquals(0, samples.get());
+        assertEquals(1, lookups.get());
+        assertSame(restored, planner.resolveIndependentOwner(EARLY_OWNER_TILE));
+        assertEquals(1, lookups.get());
+    }
+
     @Test
     public void restoringPreparedTileKeepsExistingCanonicalOwner() {
         HydrologyPlanner planner = new HydrologyPlanner(77L, EARLY_OWNER_SETTINGS, EARLY_OWNER_TERRAIN);
@@ -381,6 +408,48 @@ public class HydrologyPlannerTest {
         }
     }
 
+    @Test
+    public void earlyOwnersStayInsideTheirScope() throws Exception {
+        org.junit.Assume.assumeTrue(Runtime.getRuntime().availableProcessors() > 2);
+        HydrologyPlannerSettings settings = withPeriodTwo(standardSettings(4D, 2D, true, false, List.of()));
+        HydrologyPlanner planner = new HydrologyPlanner(77L, settings, (x, z) -> blockedTerrain());
+        planner.limitEarlyOwners(key -> key.tileX() >= -1);
+        AutoCloseable admission = earlyAdmission(planner, new HydrologyTileKey(-1, -1));
+        Map<HydrologyTileKey, HydrologyForkJoin.Task<?>> tasks = earlyTasks(admission);
+        Method prepare = admission.getClass().getDeclaredMethod("prepare");
+        prepare.setAccessible(true);
+        CapturingHydrologyPool pool = new CapturingHydrologyPool();
+        try {
+            pool.submit(() -> {
+                prepare.invoke(admission);
+                return null;
+            }).get(5, TimeUnit.SECONDS);
+            assertEquals(List.of(new HydrologyTileKey(0, -1),
+                    new HydrologyTileKey(-1, -2), new HydrologyTileKey(-1, 0),
+                    new HydrologyTileKey(0, -2), new HydrologyTileKey(0, 0)), new ArrayList<>(tasks.keySet()));
+            assertEquals(new ArrayList<>(tasks.values()), pool.submitted);
+        } finally {
+            tasks.clear();
+            admission.close();
+            pool.shutdownNow();
+        }
+        planner.limitEarlyOwners(null);
+        AutoCloseable unlimited = earlyAdmission(planner, new HydrologyTileKey(-1, -1));
+        Map<HydrologyTileKey, HydrologyForkJoin.Task<?>> unlimitedTasks = earlyTasks(unlimited);
+        CapturingHydrologyPool unlimitedPool = new CapturingHydrologyPool();
+        try {
+            unlimitedPool.submit(() -> {
+                prepare.invoke(unlimited);
+                return null;
+            }).get(5, TimeUnit.SECONDS);
+            assertEquals(8, unlimitedTasks.size());
+        } finally {
+            unlimitedTasks.clear();
+            unlimited.close();
+            unlimitedPool.shutdownNow();
+        }
+    }
+
     private static final class CapturingHydrologyPool extends ForkJoinPool {
         private final ArrayList<Runnable> submitted = new ArrayList<>();
 

core/src/test/java/art/arcane/iris/generation/hydrology/HydrologyTileCacheTest.java

@@ -38,11 +38,15 @@ import java.util.concurrent.TimeoutException;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.Function;
+import java.util.function.Predicate;
+import org.mockito.ArgumentCaptor;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertSame;
 import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertThrows;
 import static org.junit.Assert.assertTrue;
 import static org.mockito.ArgumentMatchers.any;
@@ -590,6 +594,48 @@ public class HydrologyTileCacheTest {
         assertEquals(16, planned.size());
     }
 
+    @Test
+    @SuppressWarnings("unchecked")
+    public void pregenScopeLimitsEarlyOwnersToTheTilesTheAreaComposesFrom() {
+        HydrologyPlanner planner = mock(HydrologyPlanner.class);
+        HydrologyPlannerSettings settings = stalePrefetchSettings();
+        when(planner.settings()).thenReturn(settings);
+        when(planner.plan(any(HydrologyTileKey.class))).thenReturn(mock(HydrologyTile.class));
+        HydrologyTileCache cache = new HydrologyTileCache(planner, 128, new LinkedBlockingQueue<Runnable>()::add);
+        ArgumentCaptor<Predicate<HydrologyTileKey>> scope = ArgumentCaptor.forClass(Predicate.class);
+
+        HydrologyTileCache.PregenerationScope pregen = cache.preparePregeneration(
+                new HydrologyTileCache.PregenerationArea(0, 0, -1536, -1536, 2047, 2047));
+        verify(planner).limitEarlyOwners(scope.capture());
+        for (HydrologyTileKey key : prefetchRectangle(-2, 2, -2, 2)) {
+            assertTrue(key.toString(), scope.getValue().test(key));
+        }
+        assertFalse(scope.getValue().test(new HydrologyTileKey(-3, 0)));
+        assertFalse(scope.getValue().test(new HydrologyTileKey(0, 3)));
+
+        pregen.close();
+        verify(planner).limitEarlyOwners(null);
+    }
+
+    @Test
+    @SuppressWarnings("unchecked")
+    public void plannerFindsTilesThisCachePlanned() {
+        HydrologyPlanner planner = mock(HydrologyPlanner.class);
+        HydrologyTile tile = mock(HydrologyTile.class);
+        HydrologyPlannerSettings settings = stalePrefetchSettings();
+        when(planner.settings()).thenReturn(settings);
+        when(planner.plan(any(HydrologyTileKey.class))).thenReturn(tile);
+        HydrologyTileCache cache = new HydrologyTileCache(planner, 128);
+        ArgumentCaptor<Function<HydrologyTileKey, HydrologyTile>> planned = ArgumentCaptor.forClass(Function.class);
+        verify(planner).usePlannedTiles(planned.capture());
+        HydrologyTileKey key = new HydrologyTileKey(3, -2);
+
+        assertNull(planned.getValue().apply(key));
+        assertSame(tile, cache.get(key));
+        assertSame(tile, planned.getValue().apply(key));
+        verify(planner, times(1)).plan(key);
+    }
+
     @Test
     public void invalidPregenBoundsLeaveQueuedPrefetchIntact() {
         HydrologyPlanner planner = mock(HydrologyPlanner.class);

probe/src/main/java/art/arcane/iris/probe/HydrologyPlanDigestProbe.java

@@ -0,0 +1,159 @@
+package art.arcane.iris.probe;
+
+import art.arcane.iris.generation.hydrology.HydrologyTile;
+import art.arcane.iris.generation.hydrology.HydrologyTileCache;
+import art.arcane.iris.generation.hydrology.HydrologyTileKey;
+import art.arcane.iris.generation.hydrology.runtime.IrisHydrologyRuntime;
+import art.arcane.iris.generation.runtime.Engine;
+import com.sun.management.OperatingSystemMXBean;
+
+import java.io.ByteArrayOutputStream;
+import java.io.DataOutputStream;
+import java.io.File;
+import java.lang.management.ManagementFactory;
+import java.lang.reflect.Constructor;
+import java.lang.reflect.Method;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.security.MessageDigest;
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.HexFormat;
+import java.util.List;
+import java.util.Locale;
+import java.util.stream.Stream;
+
+/**
+ * Plans a rectangle of hydrology tiles of a real pack and prints one digest per tile (the prepared-plan
+ * encoding, owner resolution included) with the planning CPU. Two builds plan identically when every
+ * digest matches; an expected-digest file makes the probe fail on the first difference. With a pregen
+ * radius the tiles are planned the way a pregeneration of that radius plans them: prepared-plan store,
+ * pregeneration scope and speculative roots.
+ */
+public final class HydrologyPlanDigestProbe {
+    private static final String LOG_PREFIX = "[hydroplan]";
+
+    private HydrologyPlanDigestProbe() {
+    }
+
+    public static void main(String[] arguments) throws Exception {
+        if (arguments.length < 7) {
+            throw new IllegalArgumentException("Expected: <pack> <dimension> <seed> <minimumTileX> <maximumTileX> "
+                    + "<minimumTileZ> <maximumTileZ> [digestOut|-] [expectedDigests|-] [pregenRadius]");
+        }
+        File pack = new File(arguments[0]);
+        String dimension = arguments[1];
+        long seed = Long.parseLong(arguments[2]);
+        int minimumTileX = Integer.parseInt(arguments[3]);
+        int maximumTileX = Integer.parseInt(arguments[4]);
+        int minimumTileZ = Integer.parseInt(arguments[5]);
+        int maximumTileZ = Integer.parseInt(arguments[6]);
+        Path output = arguments.length > 7 && !arguments[7].equals("-") ? Path.of(arguments[7]) : null;
+        List<String> expected = arguments.length > 8 && !arguments[8].equals("-") ? Files.readAllLines(Path.of(arguments[8])) : null;
+        Integer pregenRadius = arguments.length > 9 ? Integer.valueOf(arguments[9]) : null;
+        List<HydrologyTileKey> keys = nearestFirst(minimumTileX, maximumTileX, minimumTileZ, maximumTileZ);
+        OperatingSystemMXBean system = (OperatingSystemMXBean) ManagementFactory.getOperatingSystemMXBean();
+        ArrayList<String> lines = new ArrayList<>(keys.size());
+        int mismatches;
+        try (RealPackProbeSupport.Workspace workspace = RealPackProbeSupport.openWorkspace(pack, dimension, LOG_PREFIX)) {
+            try (RealPackProbeSupport.EngineSession session = workspace.openEngine(seed, false, "plan-" + seed)) {
+                Engine engine = session.engine();
+                IrisHydrologyRuntime runtime = engine.getComplex().getHydrologyRuntime();
+                if (runtime == null) {
+                    throw new IllegalStateException("Dimension '" + dimension + "' has no active hydrology runtime.");
+                }
+                Path store = pregenRadius == null ? null : Files.createTempDirectory("iris-hydroplan-store-");
+                try {
+                    if (store != null) {
+                        runtime.enableSharedCache("probe", store);
+                    }
+                    mismatches = plan(runtime, keys, pregenRadius, expected, lines, system);
+                } finally {
+                    if (store != null) {
+                        delete(store);
+                    }
+                }
+            }
+        }
+        if (output != null) {
+            Files.write(output, lines, StandardCharsets.UTF_8);
+        }
+        System.exit(mismatches == 0 ? 0 : 1);
+    }
+
+    private static int plan(IrisHydrologyRuntime runtime, List<HydrologyTileKey> keys, Integer pregenRadius,
+                            List<String> expected, List<String> lines,
+                            OperatingSystemMXBean system) throws Exception {
+        int mismatches = 0;
+        long cpuStart = system.getProcessCpuTime();
+        long wallStart = System.nanoTime();
+        try (HydrologyTileCache.PregenerationScope ignored = pregenRadius == null ? null
+                : runtime.preparePregeneration(pregenerationArea(pregenRadius))) {
+            for (HydrologyTileKey key : keys) {
+                long tileCpu = system.getProcessCpuTime();
+                long tileWall = System.nanoTime();
+                HydrologyTile tile = runtime.tile(key);
+                String line = key.tileX() + "," + key.tileZ() + " " + digest(tile);
+                System.out.printf(Locale.ROOT, "%s tile %s courses=%d columns=%d wall_ms=%d cpu_ms=%d%n", LOG_PREFIX,
+                        line, tile.courses().size(), tile.footprint().size(),
+                        (System.nanoTime() - tileWall) / 1_000_000L, (system.getProcessCpuTime() - tileCpu) / 1_000_000L);
+                if (expected != null && !expected.contains(line)) {
+                    mismatches++;
+                    System.out.println(LOG_PREFIX + " MISMATCH " + line);
+                }
+                lines.add(line);
+            }
+        }
+        System.out.printf(Locale.ROOT, "%s RESULT tiles=%d wall_ms=%d cpu_ms=%d mismatches=%d%n", LOG_PREFIX,
+                keys.size(), (System.nanoTime() - wallStart) / 1_000_000L,
+                (system.getProcessCpuTime() - cpuStart) / 1_000_000L, mismatches);
+        return mismatches;
+    }
+
+    private static void delete(Path root) throws Exception {
+        try (Stream<Path> paths = Files.walk(root)) {
+            for (Path path : paths.sorted(Comparator.reverseOrder()).toList()) {
+                Files.delete(path);
+            }
+        }
+    }
+
+    /** The region-aligned area a square pregeneration of the radius covers, as the pregenerator reports it. */
+    static HydrologyTileCache.PregenerationArea pregenerationArea(int radius) {
+        long minimum = Math.floorDiv(-radius, 512) * 512L;
+        long maximum = (Math.floorDiv(radius, 512) + 1L) * 512L - 1L;
+        return new HydrologyTileCache.PregenerationArea(0, 0, minimum, minimum, maximum, maximum);
+    }
+
+    private static List<HydrologyTileKey> nearestFirst(int minimumTileX, int maximumTileX, int minimumTileZ, int maximumTileZ) {
+        ArrayList<HydrologyTileKey> keys = new ArrayList<>();
+        for (int tileZ = minimumTileZ; tileZ <= maximumTileZ; tileZ++) {
+            for (int tileX = minimumTileX; tileX <= maximumTileX; tileX++) {
+                keys.add(new HydrologyTileKey(tileX, tileZ));
+            }
+        }
+        keys.sort(Comparator.comparingLong((HydrologyTileKey key) -> distanceSquared(key))
+                .thenComparingInt(HydrologyTileKey::tileZ).thenComparingInt(HydrologyTileKey::tileX));
+        return keys;
+    }
+
+    private static long distanceSquared(HydrologyTileKey key) {
+        long x = key.tileX() * 2L + 1L;
+        long z = key.tileZ() * 2L + 1L;
+        return x * x + z * z;
+    }
+
+    private static String digest(HydrologyTile tile) throws Exception {
+        Class<?> codecType = Class.forName("art.arcane.iris.generation.hydrology.HydrologyTileCodec");
+        Constructor<?> constructor = codecType.getDeclaredConstructor();
+        constructor.setAccessible(true);
+        Method write = codecType.getDeclaredMethod("write", DataOutputStream.class, HydrologyTile.class, String.class);
+        write.setAccessible(true);
+        ByteArrayOutputStream bytes = new ByteArrayOutputStream();
+        try (DataOutputStream output = new DataOutputStream(bytes)) {
+            write.invoke(constructor.newInstance(), output, tile, "probe");
+        }
+        return HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256").digest(bytes.toByteArray()));
+    }
+}

probe/src/test/java/art/arcane/iris/probe/HydrologyPlanDigestProbeTest.java

@@ -0,0 +1,22 @@
+package art.arcane.iris.probe;
+
+import art.arcane.iris.generation.hydrology.HydrologyTileCache;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+
+public final class HydrologyPlanDigestProbeTest {
+    @Test
+    public void pregenerationAreaCoversTheRegionsOfTheRadius() {
+        assertArea(HydrologyPlanDigestProbe.pregenerationArea(1000), -1024L, 1023L);
+        assertArea(HydrologyPlanDigestProbe.pregenerationArea(1500), -1536L, 1535L);
+        assertArea(HydrologyPlanDigestProbe.pregenerationArea(2000), -2048L, 2047L);
+    }
+
+    private static void assertArea(HydrologyTileCache.PregenerationArea area, long minimum, long maximum) {
+        assertEquals(minimum, area.minimumBlockX());
+        assertEquals(minimum, area.minimumBlockZ());
+        assertEquals(maximum, area.maximumBlockX());
+        assertEquals(maximum, area.maximumBlockZ());
+    }
+}