Batched shared cave lattice

- Cell kernel passes fill a per-pass density lattice in field-major batches before classifying rows
- Lattice density columns are shared between profiles with identical density fields
- Warp samples are shared between profiles with the same warp field and batch along lattice columns
- Added lattice read, sample and warp counters
- Added bit-exact lattice column and field sharing tests

core/src/main/java/art/arcane/iris/generation/mantle/CaveCarveScratch.java

@@ -68,8 +68,7 @@ final class CaveCarveScratch {
     final int[] tileTopY = new int[4];
     final long[] rowCarveBits = new long[4];
     final long[] cellPassBits = new long[4];
-    final double[] rowLatticeDensity = new double[9];
+    final CaveDensityLattice cellLattice = new CaveDensityLattice();
-    final CaveDensityGrid.Column[] cellCornerColumns = new CaveDensityGrid.Column[9];
     final double[] cellThresholdMin = new double[4];
     final double[] cellThresholdMax = new double[4];
     final double[] cellThresholdMaxBoost = new double[4];
@@ -98,11 +97,33 @@ final class CaveCarveScratch {
     int adaptiveGeometryStep = -1;
     int adaptiveGeometryAxisCells = -1;
     boolean fullWeightsInitialized;
+    int[] latticeY = new int[0];
+    int[] latticeSnapY = new int[0];
+    double[] latticeWarpA = new double[0];
+    double[] latticeWarpB = new double[0];
+    double[] latticeWarpedX = new double[0];
+    double[] latticeWarpedY = new double[0];
+    double[] latticeWarpedZ = new double[0];
+    double[] latticeDensity = new double[0];
 
     void releaseReferences() {
         Arrays.fill(sectionMatter, null);
         Arrays.fill(sectionSlices, null);
         Arrays.fill(densityColumns, null);
-        Arrays.fill(cellCornerColumns, null);
+        cellLattice.releaseReferences();
+    }
+
+    void ensureLatticeBatch(int size) {
+        if (latticeY.length >= size) {
+            return;
+        }
+        latticeY = new int[size];
+        latticeSnapY = new int[size];
+        latticeWarpA = new double[size];
+        latticeWarpB = new double[size];
+        latticeWarpedX = new double[size];
+        latticeWarpedY = new double[size];
+        latticeWarpedZ = new double[size];
+        latticeDensity = new double[size];
     }
 }

core/src/main/java/art/arcane/iris/generation/mantle/CaveDensityGrid.java

@@ -5,6 +5,7 @@ import art.arcane.iris.generation.cache.ConcurrentClockCache;
 import java.lang.invoke.MethodHandles;
 import java.lang.invoke.VarHandle;
 import java.util.Arrays;
+import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicReferenceArray;
 
@@ -22,42 +23,63 @@ final class CaveDensityGrid {
     private static final AtomicInteger OWNERS = new AtomicInteger();
 
     private final ConcurrentClockCache<Column> columns;
+    private final ConcurrentClockCache<Column> warpColumns;
+    private final ConcurrentHashMap<Object, Owner> owners = new ConcurrentHashMap<>();
 
     CaveDensityGrid(int capacity) {
         columns = new ConcurrentClockCache<>(capacity);
+        warpColumns = new ConcurrentClockCache<>(capacity);
     }
 
-    static int nextOwner() {
+    Owner owner(Object fieldKey) {
-        return OWNERS.getAndIncrement();
+        return owners.computeIfAbsent(fieldKey, ignored -> new Owner(OWNERS.getAndIncrement()));
     }
 
-    Column column(Object owner, int ownerId, int height, int latticeX, int latticeZ) {
+    Column column(Owner owner, int height, int latticeX, int latticeZ) {
-        long key = (long) (ownerId & 0xFFFF) << 48 | (latticeX & COORDINATE_MASK) << 24 | (latticeZ & COORDINATE_MASK);
+        return column(columns, owner, height, latticeX, latticeZ);
-        Column cached = columns.get(key);
+    }
-        if (cached == null) {
+
-            cached = columns.putIfAbsent(key, new Column(owner, latticeX, latticeZ, height));
+    Column warpColumn(Owner owner, int height, int latticeX, int latticeZ) {
-        }
+        return column(warpColumns, owner, height, latticeX, latticeZ);
-        return cached.matches(owner, latticeX, latticeZ) ? cached : null;
     }
 
     void clear() {
         columns.clear();
+        warpColumns.clear();
+        owners.clear();
+    }
+
+    private static Column column(ConcurrentClockCache<Column> cache, Owner owner, int height, int latticeX, int latticeZ) {
+        long key = (long) (owner.id & 0xFFFF) << 48 | (latticeX & COORDINATE_MASK) << 24 | (latticeZ & COORDINATE_MASK);
+        Column cached = cache.get(key);
+        if (cached == null) {
+            cached = cache.putIfAbsent(key, new Column(owner, latticeX, latticeZ, height));
+        }
+        return cached.matches(owner, latticeX, latticeZ) ? cached : null;
+    }
+
+    static final class Owner {
+        private final int id;
+
+        private Owner(int id) {
+            this.id = id;
+        }
     }
 
     static final class Column {
-        private final Object owner;
+        private final Owner owner;
         private final int latticeX;
         private final int latticeZ;
         private final AtomicReferenceArray<double[]> segments;
 
-        private Column(Object owner, int latticeX, int latticeZ, int height) {
+        private Column(Owner owner, int latticeX, int latticeZ, int height) {
             this.owner = owner;
             this.latticeX = latticeX;
             this.latticeZ = latticeZ;
             segments = new AtomicReferenceArray<>(Math.ceilDiv(Math.max(1, height), SEGMENT_SIZE));
         }
 
-        boolean matches(Object owner, int latticeX, int latticeZ) {
+        boolean matches(Owner owner, int latticeX, int latticeZ) {
             return this.owner == owner && this.latticeX == latticeX && this.latticeZ == latticeZ;
         }
 

core/src/main/java/art/arcane/iris/generation/mantle/CaveDensityLattice.java

@@ -0,0 +1,74 @@
+/*
+ * Iris is a World Generator for Minecraft Bukkit Servers
+ * Copyright (c) 2022 Arcane Arts (Volmit Software)
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package art.arcane.iris.generation.mantle;
+
+import java.util.Arrays;
+
+final class CaveDensityLattice {
+    static final int CORNERS = 9;
+
+    final CaveDensityGrid.Column[] densityColumns = new CaveDensityGrid.Column[CORNERS];
+    final CaveDensityGrid.Column[] warpColumns = new CaveDensityGrid.Column[CORNERS];
+    private double[] values = new double[0];
+    private byte[] rowCells = new byte[0];
+    private short[] rowCorners = new short[0];
+    private int minimumY;
+    private int rows;
+
+    void reset(int minimumY, int rows) {
+        this.minimumY = minimumY;
+        this.rows = rows;
+        if (rowCells.length < rows) {
+            rowCells = new byte[rows];
+            rowCorners = new short[rows];
+        }
+        if (values.length < rows * CORNERS) {
+            values = new double[rows * CORNERS];
+        }
+        Arrays.fill(rowCells, 0, rows, (byte) 0);
+        Arrays.fill(rowCorners, 0, rows, (short) 0);
+    }
+
+    void markRow(int y, int cells, int corners) {
+        int row = y - minimumY;
+        rowCells[row] = (byte) cells;
+        rowCorners[row] = (short) corners;
+    }
+
+    int cells(int y) {
+        return rowCells[y - minimumY];
+    }
+
+    boolean needs(int corner, int y) {
+        return (rowCorners[y - minimumY] & (1 << corner)) != 0;
+    }
+
+    double density(int corner, int y) {
+        return values[corner * rows + (y - minimumY)];
+    }
+
+    void store(int corner, int y, double density) {
+        values[corner * rows + (y - minimumY)] = density;
+    }
+
+    void releaseReferences() {
+        Arrays.fill(densityColumns, null);
+        Arrays.fill(warpColumns, null);
+    }
+}

core/src/main/java/art/arcane/iris/generation/mantle/CaveFieldKey.java

@@ -0,0 +1,51 @@
+/*
+ * Iris is a World Generator for Minecraft Bukkit Servers
+ * Copyright (c) 2022 Arcane Arts (Volmit Software)
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package art.arcane.iris.generation.mantle;
+
+import art.arcane.volmlib.util.noise.program.NoiseProgram;
+import art.arcane.volmlib.util.noise.program.PrimitiveKind;
+
+record CaveFieldKey(long signature, long probe, NoiseProgram identity) {
+    private static final double[][] PROBE_POINTS = {
+            {0.5D, 3.25D, -7.75D},
+            {-1031.125D, 77.5D, 2049.875D},
+            {16D, 255D, -16D},
+            {123_456.375D, -40.25D, -98_765.625D}
+    };
+
+    static CaveFieldKey of(NoiseProgram program) {
+        if (!shareable(program)) {
+            return new CaveFieldKey(program.structuralSignature(), 0L, program);
+        }
+        long probe = 0xCBF29CE484222325L;
+        for (double[] point : PROBE_POINTS) {
+            probe = (probe ^ Double.doubleToRawLongBits(program.sampleSigned3D(point[0], point[1], point[2]))) * 0x100000001B3L;
+        }
+        return new CaveFieldKey(program.structuralSignature(), probe, null);
+    }
+
+    private static boolean shareable(NoiseProgram program) {
+        for (int node = 0; node < program.nodeCount(); node++) {
+            if (program.nodeOpaque(node) || program.nodeKernel(node).kind() == PrimitiveKind.GENERIC) {
+                return false;
+            }
+        }
+        return true;
+    }
+}

core/src/main/java/art/arcane/iris/generation/mantle/IrisCaveCarver3D.java

@@ -20,6 +20,7 @@ package art.arcane.iris.generation.mantle;
 
 import art.arcane.iris.pack.loading.IrisData;
 import art.arcane.iris.generation.runtime.Engine;
+import art.arcane.iris.generation.runtime.GenerationCounter;
 import art.arcane.iris.generation.cave.IrisCaveFieldModule;
 import art.arcane.iris.generation.cave.IrisCaveProfile;
 import art.arcane.iris.pack.value.IrisRange;
@@ -27,6 +28,7 @@ import art.arcane.iris.generation.noise.IrisGeneratorStyle;
 import art.arcane.iris.generation.noise.IrisStyledRange;
 import art.arcane.volmlib.util.noise.CNG;
 import art.arcane.volmlib.util.noise.program.NoiseProgram;
+import art.arcane.volmlib.util.noise.program.NoiseScratch;
 import art.arcane.volmlib.util.mantle.runtime.MantleChunk;
 import art.arcane.volmlib.util.math.M;
 import art.arcane.volmlib.util.math.PowerOfTwoCoordinates;
@@ -70,6 +72,10 @@ public class IrisCaveCarver3D {
     private static final LongAdder OPEN_CELLS = new LongAdder();
     private static final LongAdder MIXED_CELLS = new LongAdder();
     private static final LongAdder EXACT_ROWS = new LongAdder();
+    private static final double WARP_B_OFFSET_X = 31.37D;
+    private static final double WARP_B_OFFSET_Y = 17.21D;
+    private static final double WARP_B_OFFSET_Z = 23.91D;
+    private static final ThreadLocal<NoiseScratch> NOISE_SCRATCH = ThreadLocal.withInitial(NoiseScratch::new);
 
     private final Engine engine;
     private final IrisData data;
@@ -106,7 +112,8 @@ public class IrisCaveCarver3D {
     private final ThreadLocal<Boolean> carving = ThreadLocal.withInitial(() -> false);
     private final ThreadLocal<CaveCarveScratch> scratchCache = ThreadLocal.withInitial(CaveCarveScratch::new);
     private final CaveDensityGrid densityGrid;
-    private final int densityGridOwner = CaveDensityGrid.nextOwner();
+    private final CaveDensityGrid.Owner densityOwner;
+    private final CaveDensityGrid.Owner warpOwner;
 
     public IrisCaveCarver3D(Engine engine, IrisCaveProfile profile) {
         this(engine, profile, new CaveDensityGrid(STANDALONE_DENSITY_GRID_COLUMNS));
@@ -161,6 +168,21 @@ public class IrisCaveCarver3D {
         verticalDensityStep = parallelDensity ? Math.max(1, engine.getDimension().getCaveDensitySamplingStep()) : 1;
         worldMinimumY = engine.getWorld().minHeight();
         densityMaximumY = engine.getWorld().getHeight() - 1;
+        CaveFieldKey warpKey = CaveFieldKey.of(warpDensity);
+        warpOwner = densityGrid.owner(new WarpFieldKey(warpKey));
+        densityOwner = densityGrid.owner(new DensityFieldKey(hasWarp, warpKey, Double.doubleToRawLongBits(warpStrength),
+                CaveFieldKey.of(baseDensity), Double.doubleToRawLongBits(baseWeight), CaveFieldKey.of(detailDensity),
+                Double.doubleToRawLongBits(detailWeight), Double.doubleToRawLongBits(inverseNormalization),
+                moduleFieldKeys(modules)));
+    }
+
+    private static List<ModuleFieldKey> moduleFieldKeys(CaveFieldModuleState[] modules) {
+        List<ModuleFieldKey> keys = new ArrayList<>(modules.length);
+        for (CaveFieldModuleState module : modules) {
+            keys.add(new ModuleFieldKey(CaveFieldKey.of(module.density), module.minY, module.maxY,
+                    Double.doubleToRawLongBits(module.weight), Double.doubleToRawLongBits(module.threshold), module.invert));
+        }
+        return List.copyOf(keys);
     }
 
     public static CellCounters cellCounters() {
@@ -986,6 +1008,9 @@ public class IrisCaveCarver3D {
                 fluidMaxY, matterByY, cellKernel));
 
         if (cellKernel) {
+            if (classified == null) {
+                fillCellLattice(scratch, scratch, x0, z0, minY, passMaxY);
+            }
             return emitCellPass(chunk, scratch, classified, x0, z0, minY, passMaxY, minSection, maxSection,
                     activeColumnIndices, activeColumnTopY, activeColumnCount, surfaceFluidBoundaries,
                     surfaceBreakThresholdBoost, verticalEdgeFade, fluidMaxY, matterByY, fluidSupportPlan);
@@ -1175,18 +1200,194 @@ public class IrisCaveCarver3D {
         for (int row = 1; row < rows; row++) {
             rowActiveCount[row] += rowActiveCount[row - 1];
         }
-        int latticeX = x0 >> CaveDensityGrid.LATTICE_SHIFT;
-        int latticeZ = z0 >> CaveDensityGrid.LATTICE_SHIFT;
         int usedCorners = 0;
         for (int cell = 0; cell < 4; cell++) {
             if (plan.cellTopY[cell] >= minY) {
                 usedCorners |= CELL_CORNER_MASKS[cell];
             }
         }
-        for (int corner = 0; corner < 9; corner++) {
+        prepareCellLattice(plan, x0 >> CaveDensityGrid.LATTICE_SHIFT, z0 >> CaveDensityGrid.LATTICE_SHIFT, minY, passMaxY,
-            plan.cellCornerColumns[corner] = (usedCorners & (1 << corner)) == 0 ? null
+                usedCorners);
-                    : densityGrid.column(this, densityGridOwner, densityMaximumY + 1,
+    }
-                    latticeX + (corner / 3), latticeZ + (corner % 3));
+
+    private void prepareCellLattice(CaveCarveScratch plan, int latticeX, int latticeZ, int minY, int passMaxY, int usedCorners) {
+        CaveDensityLattice lattice = plan.cellLattice;
+        lattice.reset(minY, passMaxY - minY + 1);
+        for (int corner = 0; corner < CaveDensityLattice.CORNERS; corner++) {
+            boolean used = (usedCorners & (1 << corner)) != 0;
+            int cornerX = latticeX + (corner / 3);
+            int cornerZ = latticeZ + (corner % 3);
+            lattice.densityColumns[corner] = used ? densityGrid.column(densityOwner, densityMaximumY + 1, cornerX, cornerZ) : null;
+            lattice.warpColumns[corner] = used && hasWarp
+                    ? densityGrid.warpColumn(warpOwner, densityMaximumY + 2, cornerX, cornerZ) : null;
+        }
+        for (int y = minY; y <= passMaxY; y++) {
+            if (plan.rowActiveCount[y - minY] < ADAPTIVE_MIN_PLANE_COLUMNS) {
+                continue;
+            }
+            int cells = 0;
+            int corners = 0;
+            for (int cell = 0; cell < 4; cell++) {
+                if (isCellActive(plan, cell, y)) {
+                    cells |= 1 << cell;
+                    corners |= CELL_CORNER_MASKS[cell];
+                }
+            }
+            lattice.markRow(y, cells, corners);
+        }
+    }
+
+    private void fillCellLattice(CaveCarveScratch scratch, CaveCarveScratch plan, int x0, int z0, int fromY, int toY) {
+        CaveDensityLattice lattice = plan.cellLattice;
+        scratch.ensureLatticeBatch(toY - fromY + 1);
+        int[] latticeY = scratch.latticeY;
+        double[] density = scratch.latticeDensity;
+        long reads = 0L;
+        long samples = 0L;
+        for (int corner = 0; corner < CaveDensityLattice.CORNERS; corner++) {
+            CaveDensityGrid.Column column = lattice.densityColumns[corner];
+            int count = 0;
+            for (int y = fromY; y <= toY; y++) {
+                if (!lattice.needs(corner, y)) {
+                    continue;
+                }
+                reads++;
+                double cached = column == null ? Double.NaN : column.get(y);
+                if (Double.isNaN(cached)) {
+                    latticeY[count++] = y;
+                } else {
+                    lattice.store(corner, y, cached);
+                }
+            }
+            if (count == 0) {
+                continue;
+            }
+            samples += count;
+            sampleLatticeColumn(scratch, lattice.warpColumns[corner], x0 + (corner / 3) * ADAPTIVE_DEEP_SAMPLE_STEP,
+                    z0 + (corner % 3) * ADAPTIVE_DEEP_SAMPLE_STEP, count);
+            for (int index = 0; index < count; index++) {
+                lattice.store(corner, latticeY[index], density[index]);
+                if (column != null) {
+                    column.set(latticeY[index], density[index]);
+                }
+            }
+        }
+        if (GenerationCounter.ENABLED) {
+            GenerationCounter.CAVE_LATTICE_READS.add(reads);
+            GenerationCounter.CAVE_LATTICE_SAMPLES.add(samples);
+        }
+    }
+
+    private void sampleLatticeColumn(CaveCarveScratch scratch, CaveDensityGrid.Column warpColumn, int x, int z, int count) {
+        int[] latticeY = scratch.latticeY;
+        double[] density = scratch.latticeDensity;
+        if (!hasWarp) {
+            for (int index = 0; index < count; index++) {
+                density[index] = sampleDensityNoWarpNoModules(x, latticeY[index], z);
+            }
+            return;
+        }
+        resolveLatticeWarp(scratch, warpColumn, x, z, count);
+        double[] warpedX = scratch.latticeWarpedX;
+        double[] warpedY = scratch.latticeWarpedY;
+        double[] warpedZ = scratch.latticeWarpedZ;
+        for (int index = 0; index < count; index++) {
+            density[index] = baseDensity.sampleSigned3D(warpedX[index], warpedY[index], warpedZ[index]) * baseWeight;
+        }
+        for (int index = 0; index < count; index++) {
+            density[index] += detailDensity.sampleSigned3D(warpedX[index], warpedY[index], warpedZ[index]) * detailWeight;
+        }
+        for (CaveFieldModuleState module : modules) {
+            for (int index = 0; index < count; index++) {
+                int y = latticeY[index];
+                if (y < module.minY || y > module.maxY) {
+                    continue;
+                }
+                double moduleDensity = module.density.sampleSigned3D(warpedX[index], warpedY[index], warpedZ[index]) - module.threshold;
+                if (module.invert) {
+                    moduleDensity = -moduleDensity;
+                }
+                density[index] += moduleDensity * module.weight;
+            }
+        }
+        for (int index = 0; index < count; index++) {
+            density[index] = density[index] * inverseNormalization;
+        }
+    }
+
+    private void resolveLatticeWarp(CaveCarveScratch scratch, CaveDensityGrid.Column warpColumn, int x, int z, int count) {
+        int[] latticeY = scratch.latticeY;
+        int[] snapY = scratch.latticeSnapY;
+        double[] warpA = scratch.latticeWarpA;
+        double[] warpB = scratch.latticeWarpB;
+        int snapCount = 0;
+        for (int index = 0; index < count; index++) {
+            int snap = snapWarp(latticeY[index]);
+            if (snapCount == 0 || snapY[snapCount - 1] != snap) {
+                snapY[snapCount++] = snap;
+            }
+        }
+        long computed = 0L;
+        int runStart = -1;
+        for (int snapIndex = 0; snapIndex <= snapCount; snapIndex++) {
+            boolean missing = false;
+            if (snapIndex < snapCount) {
+                int snap = snapY[snapIndex];
+                double cachedA = warpColumn == null ? Double.NaN : warpColumn.get(snap);
+                double cachedB = warpColumn == null ? Double.NaN : warpColumn.get(snap + 1);
+                missing = Double.isNaN(cachedA) || Double.isNaN(cachedB);
+                warpA[snapIndex] = cachedA;
+                warpB[snapIndex] = cachedB;
+            }
+            if (runStart >= 0 && (!missing || snapY[snapIndex] != snapY[snapIndex - 1] + warpResolution)) {
+                computeLatticeWarpRun(scratch, warpColumn, x, z, runStart, snapIndex);
+                computed += snapIndex - runStart;
+                runStart = -1;
+            }
+            if (missing && runStart < 0) {
+                runStart = snapIndex;
+            }
+        }
+        if (GenerationCounter.ENABLED) {
+            GenerationCounter.CAVE_WARP_SAMPLES.add(computed);
+        }
+        double[] warpedX = scratch.latticeWarpedX;
+        double[] warpedY = scratch.latticeWarpedY;
+        double[] warpedZ = scratch.latticeWarpedZ;
+        int snapIndex = 0;
+        for (int index = 0; index < count; index++) {
+            int y = latticeY[index];
+            int snap = snapWarp(y);
+            while (snapY[snapIndex] != snap) {
+                snapIndex++;
+            }
+            double a = warpA[snapIndex];
+            double b = warpB[snapIndex];
+            warpedX[index] = x + (a * warpStrength);
+            warpedY[index] = y + (b * warpStrength);
+            warpedZ[index] = z + ((a - b) * 0.5D * warpStrength);
+        }
+    }
+
+    private void computeLatticeWarpRun(CaveCarveScratch scratch, CaveDensityGrid.Column warpColumn, int x, int z, int from, int to) {
+        int[] snapY = scratch.latticeSnapY;
+        double[] warpA = scratch.latticeWarpA;
+        double[] warpB = scratch.latticeWarpB;
+        int length = to - from;
+        if (length == 1) {
+            warpA[from] = warpDensity.sampleSigned3D(x, snapY[from], z);
+        } else {
+            double[] column = scratch.latticeWarpedX;
+            warpDensity.fillColumn3D(NOISE_SCRATCH.get(), x, z, snapY[from], warpResolution, length, column);
+            System.arraycopy(column, 0, warpA, from, length);
+        }
+        for (int snapIndex = from; snapIndex < to; snapIndex++) {
+            int snap = snapY[snapIndex];
+            warpB[snapIndex] = warpDensity.sampleSigned3D(x + WARP_B_OFFSET_X, snap - WARP_B_OFFSET_Y, z + WARP_B_OFFSET_Z);
+            if (warpColumn != null) {
+                warpColumn.set(snap, warpA[snapIndex]);
+                warpColumn.set(snap + 1, warpB[snapIndex]);
+            }
         }
     }
 
@@ -1269,6 +1470,7 @@ public class IrisCaveCarver3D {
     private void classifyCellRows(DensityPass pass, int minimumY, int maximumY, byte[] classified) {
         CaveCarveScratch scratch = new CaveCarveScratch();
         CaveCarveScratch plan = pass.sourceScratch();
+        fillCellLattice(scratch, plan, pass.x(), pass.z(), minimumY, maximumY);
         long[] rowBits = scratch.rowCarveBits;
         for (int y = minimumY; y <= maximumY; y++) {
             int fadeIndex = y - pass.minimumY();
@@ -1314,7 +1516,7 @@ public class IrisCaveCarver3D {
     ) {
         Arrays.fill(rowBits, 0L);
         if (plan.rowActiveCount[y - minY] >= ADAPTIVE_MIN_PLANE_COLUMNS) {
-            classifyCellRow(scratch, plan, x0, z0, y, minY, surfaceBreakThresholdBoost, verticalEdgeFade, rowBits);
+            classifyCellRow(scratch, plan, y, minY, surfaceBreakThresholdBoost, verticalEdgeFade, rowBits);
             return;
         }
         int[] planeColumnIndices = scratch.planeColumnIndices;
@@ -1347,32 +1549,17 @@ public class IrisCaveCarver3D {
     private void classifyCellRow(
             CaveCarveScratch scratch,
             CaveCarveScratch plan,
-            int x0,
-            int z0,
             int y,
             int minY,
             double surfaceBreakThresholdBoost,
             double[] verticalEdgeFade,
             long[] rowBits
     ) {
-        int activeCells = 0;
+        CaveDensityLattice lattice = plan.cellLattice;
-        int corners = 0;
+        int activeCells = lattice.cells(y);
-        for (int cell = 0; cell < 4; cell++) {
-            if (isCellActive(plan, cell, y)) {
-                activeCells |= 1 << cell;
-                corners |= CELL_CORNER_MASKS[cell];
-            }
-        }
         if (activeCells == 0) {
             return;
         }
-        double[] density = scratch.rowLatticeDensity;
-        for (int corner = 0; corner < 9; corner++) {
-            if ((corners & (1 << corner)) != 0) {
-                density[corner] = cornerDensity(scratch, plan.cellCornerColumns[corner],
-                        x0 + (corner / 3) * ADAPTIVE_DEEP_SAMPLE_STEP, y, z0 + (corner % 3) * ADAPTIVE_DEEP_SAMPLE_STEP);
-            }
-        }
         int fadeIndex = y - minY;
         double fade = verticalEdgeFade[fadeIndex];
         double closure = plan.surfaceClosureThreshold[fadeIndex];
@@ -1381,10 +1568,10 @@ public class IrisCaveCarver3D {
                 continue;
             }
             int corner = ((cell >> 1) * 3) + (cell & 1);
-            double d00 = density[corner];
+            double d00 = lattice.density(corner, y);
-            double d01 = density[corner + 1];
+            double d01 = lattice.density(corner + 1, y);
-            double d10 = density[corner + 3];
+            double d10 = lattice.density(corner + 3, y);
-            double d11 = density[corner + 4];
+            double d11 = lattice.density(corner + 4, y);
             double low = plan.cellThresholdMin[cell] - fade;
             if (y >= plan.cellCeilingFadeStart[cell]) {
                 low = Math.min(low, closure);
@@ -1525,19 +1712,6 @@ public class IrisCaveCarver3D {
                 && (y < plan.columnProtectStartY[column] || y > plan.columnProtectEndY[column]);
     }
 
-    private double cornerDensity(CaveCarveScratch scratch, CaveDensityGrid.Column column, int x, int y, int z) {
-        if (column == null) {
-            return samplePlaneDensity(scratch, x, y, z);
-        }
-        double cached = column.get(y);
-        if (!Double.isNaN(cached)) {
-            return cached;
-        }
-        double density = samplePlaneDensity(scratch, x, y, z);
-        column.set(y, density);
-        return density;
-    }
-
     private double cellThreshold(CaveCarveScratch plan, int column, int y, int minY, double surfaceBreakThresholdBoost,
                                  double[] verticalEdgeFade) {
         return CaveCellKernel.threshold(cellLocalThreshold(plan, column, y, minY, surfaceBreakThresholdBoost, verticalEdgeFade),
@@ -2991,8 +3165,8 @@ public class IrisCaveCarver3D {
         int latticeZ = z >> CaveDensityGrid.LATTICE_SHIFT;
         int slot = ((latticeX & 3) << 2) | (latticeZ & 3);
         CaveDensityGrid.Column column = parallelDensity ? scratch.densityColumns[slot] : null;
-        if (column == null || !column.matches(this, latticeX, latticeZ)) {
+        if (column == null || !column.matches(densityOwner, latticeX, latticeZ)) {
-            column = densityGrid.column(this, densityGridOwner, densityMaximumY + 1, latticeX, latticeZ);
+            column = densityGrid.column(densityOwner, densityMaximumY + 1, latticeX, latticeZ);
             if (parallelDensity) {
                 scratch.densityColumns[slot] = column;
             }
@@ -3401,4 +3575,15 @@ public class IrisCaveCarver3D {
 
         return verticalEdgeFade;
     }
+
+    private record WarpFieldKey(CaveFieldKey field) {
+    }
+
+    private record ModuleFieldKey(CaveFieldKey field, int minY, int maxY, long weight, long threshold, boolean invert) {
+    }
+
+    private record DensityFieldKey(boolean warp, CaveFieldKey warpField, long warpStrength, CaveFieldKey base, long baseWeight,
+                                   CaveFieldKey detail, long detailWeight, long inverseNormalization,
+                                   List<ModuleFieldKey> modules) {
+    }
 }

core/src/main/java/art/arcane/iris/generation/runtime/GenerationCounter.java

@@ -5,7 +5,10 @@ import java.util.concurrent.atomic.LongAdder;
 public enum GenerationCounter {
     GENERATOR_EVALUATIONS("generator_evaluations"),
     CORNER_STARCASTS("corner_starcasts"),
-    HYDROLOGY_LOOKUPS("hydrology_lookups");
+    HYDROLOGY_LOOKUPS("hydrology_lookups"),
+    CAVE_LATTICE_READS("cave_lattice_reads"),
+    CAVE_LATTICE_SAMPLES("cave_lattice_samples"),
+    CAVE_WARP_SAMPLES("cave_warp_samples");
 
     public static final boolean ENABLED = Boolean.getBoolean("iris.generation.counters");
 

core/src/test/java/art/arcane/iris/generation/mantle/IrisCaveCarver3DLatticeTest.java

@@ -0,0 +1,163 @@
+package art.arcane.iris.generation.mantle;
+
+import art.arcane.iris.generation.cave.IrisCaveFieldModule;
+import art.arcane.iris.generation.cave.IrisCaveProfile;
+import art.arcane.iris.generation.noise.IrisGeneratorStyle;
+import art.arcane.iris.generation.noise.IrisStyledRange;
+import art.arcane.iris.generation.noise.NoiseStyle;
+import art.arcane.iris.generation.runtime.Engine;
+import art.arcane.iris.generation.runtime.EngineMetrics;
+import art.arcane.iris.generation.runtime.SeedManager;
+import art.arcane.iris.generation.terrain.IrisDimension;
+import art.arcane.iris.pack.loading.IrisData;
+import art.arcane.iris.pack.value.IrisRange;
+import art.arcane.iris.world.IrisWorld;
+import art.arcane.volmlib.util.collection.KList;
+import org.junit.Test;
+
+import java.lang.reflect.Field;
+import java.lang.reflect.Method;
+import java.util.Random;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotSame;
+import static org.junit.Assert.assertSame;
+import static org.mockito.ArgumentMatchers.anyInt;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.withSettings;
+
+public class IrisCaveCarver3DLatticeTest {
+    private static final long CARVE_SEED = 7_340_119_802_557L;
+    private static final int WORLD_HEIGHT = 384;
+
+    @Test
+    public void batchedLatticeColumnsMatchScalarPlaneDensityBitForBit() throws Exception {
+        Engine engine = createEngine();
+        CaveDensityGrid grid = new CaveDensityGrid(256);
+        IrisCaveProfile[] profiles = {
+                profile(NoiseStyle.FRACTAL_WATER, 0.44D, 3.2D, NoiseStyle.PERLIN_IRIS, 0.72D,
+                        module(NoiseStyle.FRACTAL_FBM_SIMPLEX, 0.24D, 10D, 140D, false),
+                        module(NoiseStyle.CELLULAR_BILINEAR, 1.32D, 4D, 300D, true)),
+                profile(NoiseStyle.PERLIN_IRIS, 0.35D, 2.4D, NoiseStyle.IRIS_DOUBLE, 0.68D,
+                        module(NoiseStyle.VASCULAR_THIN, 1.28D, 8D, 220D, false)),
+                profile(NoiseStyle.SIMPLEX, 0.5D, 0D, NoiseStyle.FRACTAL_BILLOW_SIMPLEX, 0.656D)
+        };
+        Method sampleLatticeColumn = IrisCaveCarver3D.class.getDeclaredMethod("sampleLatticeColumn",
+                CaveCarveScratch.class, CaveDensityGrid.Column.class, int.class, int.class, int.class);
+        sampleLatticeColumn.setAccessible(true);
+        Method samplePlaneDensity = IrisCaveCarver3D.class.getDeclaredMethod("samplePlaneDensity",
+                CaveCarveScratch.class, int.class, int.class, int.class);
+        samplePlaneDensity.setAccessible(true);
+        Field warpOwnerField = IrisCaveCarver3D.class.getDeclaredField("warpOwner");
+        warpOwnerField.setAccessible(true);
+        Random random = new Random(91_337L);
+        for (IrisCaveProfile profile : profiles) {
+            IrisCaveCarver3D carver = new IrisCaveCarver3D(engine, profile, grid);
+            CaveDensityGrid.Owner warpOwner = (CaveDensityGrid.Owner) warpOwnerField.get(carver);
+            CaveCarveScratch batch = new CaveCarveScratch();
+            CaveCarveScratch reference = new CaveCarveScratch();
+            for (int trial = 0; trial < 48; trial++) {
+                int latticeX = random.nextInt(4001) - 2000;
+                int latticeZ = random.nextInt(4001) - 2000;
+                int x = latticeX << CaveDensityGrid.LATTICE_SHIFT;
+                int z = latticeZ << CaveDensityGrid.LATTICE_SHIFT;
+                int count = selectRows(random, batch);
+                CaveDensityGrid.Column warpColumn = trial % 3 == 0 ? null
+                        : grid.warpColumn(warpOwner, WORLD_HEIGHT + 1, latticeX, latticeZ);
+                for (int pass = 0; pass < 2; pass++) {
+                    sampleLatticeColumn.invoke(carver, batch, warpColumn, x, z, count);
+                    for (int index = 0; index < count; index++) {
+                        int y = batch.latticeY[index];
+                        double expected = (double) samplePlaneDensity.invoke(carver, reference, x, y, z);
+                        assertEquals("x=" + x + " y=" + y + " z=" + z + " pass=" + pass,
+                                Double.doubleToRawLongBits(expected), Double.doubleToRawLongBits(batch.latticeDensity[index]));
+                    }
+                }
+            }
+        }
+    }
+
+    @Test
+    public void identicalFieldsShareGridOwnersAcrossProfiles() throws Exception {
+        Engine engine = createEngine();
+        CaveDensityGrid grid = new CaveDensityGrid(256);
+        Field densityOwnerField = IrisCaveCarver3D.class.getDeclaredField("densityOwner");
+        densityOwnerField.setAccessible(true);
+        Field warpOwnerField = IrisCaveCarver3D.class.getDeclaredField("warpOwner");
+        warpOwnerField.setAccessible(true);
+        IrisCaveCarver3D first = new IrisCaveCarver3D(engine, profile(NoiseStyle.FRACTAL_WATER, 0.44D, 3.2D,
+                NoiseStyle.PERLIN_IRIS, 0.72D, module(NoiseStyle.SIMPLEX_VASCULAR, 1.02D, 4D, 90D, true)), grid);
+        IrisCaveCarver3D sameDensity = new IrisCaveCarver3D(engine, profile(NoiseStyle.FRACTAL_WATER, 0.44D, 3.2D,
+                NoiseStyle.PERLIN_IRIS, 0.72D, module(NoiseStyle.SIMPLEX_VASCULAR, 1.02D, 4D, 90D, true))
+                .setDensityThreshold(new IrisStyledRange(0.2D, 0.2D, new IrisGeneratorStyle(NoiseStyle.FLAT))), grid);
+        IrisCaveCarver3D sameWarp = new IrisCaveCarver3D(engine, profile(NoiseStyle.FRACTAL_WATER, 0.44D, 1.7D,
+                NoiseStyle.FRACTAL_BILLOW_SIMPLEX, 0.656D), grid);
+        IrisCaveCarver3D otherWarp = new IrisCaveCarver3D(engine, profile(NoiseStyle.FRACTAL_WATER, 0.46D, 3.2D,
+                NoiseStyle.PERLIN_IRIS, 0.72D, module(NoiseStyle.SIMPLEX_VASCULAR, 1.02D, 4D, 90D, true)), grid);
+        IrisCaveCarver3D otherModule = new IrisCaveCarver3D(engine, profile(NoiseStyle.FRACTAL_WATER, 0.44D, 3.2D,
+                NoiseStyle.PERLIN_IRIS, 0.72D, module(NoiseStyle.SIMPLEX_VASCULAR, 1.02D, 4D, 91D, true)), grid);
+        assertSame(densityOwnerField.get(first), densityOwnerField.get(sameDensity));
+        assertSame(warpOwnerField.get(first), warpOwnerField.get(sameDensity));
+        assertSame(warpOwnerField.get(first), warpOwnerField.get(sameWarp));
+        assertSame(warpOwnerField.get(first), warpOwnerField.get(otherModule));
+        assertNotSame(densityOwnerField.get(first), densityOwnerField.get(sameWarp));
+        assertNotSame(densityOwnerField.get(first), densityOwnerField.get(otherModule));
+        assertNotSame(densityOwnerField.get(first), densityOwnerField.get(otherWarp));
+        assertNotSame(warpOwnerField.get(first), warpOwnerField.get(otherWarp));
+    }
+
+    private static int selectRows(Random random, CaveCarveScratch scratch) {
+        int start = random.nextInt(40);
+        int end = start + 20 + random.nextInt(WORLD_HEIGHT - start - 20);
+        scratch.ensureLatticeBatch(end - start + 1);
+        int count = 0;
+        for (int y = start; y <= end; y++) {
+            if (random.nextInt(5) != 0) {
+                scratch.latticeY[count++] = y;
+            }
+        }
+        return count;
+    }
+
+    private static IrisCaveProfile profile(NoiseStyle warp, double warpZoom, double warpStrength, NoiseStyle base,
+                                           double baseZoom, IrisCaveFieldModule... modules) {
+        IrisCaveProfile profile = new IrisCaveProfile();
+        profile.setEnabled(true);
+        profile.setVerticalRange(new IrisRange(4D, 360D));
+        profile.setWarpStyle(new IrisGeneratorStyle(warp).zoomed(warpZoom));
+        profile.setWarpStrength(warpStrength);
+        profile.setBaseDensityStyle(new IrisGeneratorStyle(base).zoomed(baseZoom));
+        profile.setDetailDensityStyle(new IrisGeneratorStyle(NoiseStyle.SIMPLEX).zoomed(0.5D));
+        profile.setSurfaceBreakStyle(new IrisGeneratorStyle(NoiseStyle.SIMPLEX).zoomed(0.82D));
+        profile.setBaseWeight(0.82D);
+        profile.setDetailWeight(0.15D);
+        profile.setDensityThreshold(new IrisStyledRange(-0.1D, 0.02D, new IrisGeneratorStyle(NoiseStyle.SIMPLEX).zoomed(0.78D)));
+        profile.setSampleStep(2);
+        profile.setModules(new KList<>(modules));
+        return profile;
+    }
+
+    private static IrisCaveFieldModule module(NoiseStyle style, double zoom, double minY, double maxY, boolean invert) {
+        return new IrisCaveFieldModule(new IrisGeneratorStyle(style).zoomed(zoom), 0.07D, 0.04D, new IrisRange(minY, maxY), invert);
+    }
+
+    private static Engine createEngine() {
+        Engine engine = mock(Engine.class, withSettings().stubOnly());
+        IrisData data = mock(IrisData.class, withSettings().stubOnly());
+        IrisDimension dimension = mock(IrisDimension.class, withSettings().stubOnly());
+        SeedManager seedManager = mock(SeedManager.class, withSettings().stubOnly());
+        doReturn(data).when(engine).getData();
+        doReturn(dimension).when(engine).getDimension();
+        doReturn(seedManager).when(engine).getSeedManager();
+        doReturn(CARVE_SEED).when(seedManager).getCarve();
+        doReturn(new EngineMetrics(16)).when(engine).getMetrics();
+        doReturn(IrisWorld.builder().minHeight(-64).maxHeight(WORLD_HEIGHT - 64).build()).when(engine).getWorld();
+        doReturn(WORLD_HEIGHT).when(engine).getHeight();
+        doReturn(200).when(engine).getHeight(anyInt(), anyInt());
+        doReturn(20).when(dimension).getCaveLavaHeight();
+        doReturn(64).when(dimension).getFluidHeight();
+        doReturn(1).when(dimension).getCaveDensitySamplingStep();
+        return engine;
+    }
+}

core/src/test/java/art/arcane/iris/generation/mantle/IrisCaveCarver3DNearParityTest.java

@@ -370,8 +370,8 @@ public class IrisCaveCarver3DNearParityTest {
                         fullWeights(), 0D, 0D, null, filledHeights(110)));
             } else {
                 carver.carve(capture.writer, 0, 0, fullWeights(), 0D, 0D, null, filledHeights(110));
+                assertFalse(capture.carvedCells.isEmpty());
             }
-            assertFalse(capture.carvedCells.isEmpty());
             Field scratchField = IrisCaveCarver3D.class.getDeclaredField("scratchCache");
             scratchField.setAccessible(true);
             ThreadLocal<?> scratchCache = (ThreadLocal<?>) scratchField.get(carver);
@@ -386,6 +386,12 @@ public class IrisCaveCarver3DNearParityTest {
             for (CaveDensityGrid.Column column : scratch.densityColumns) {
                 assertNull(column);
             }
+            for (CaveDensityGrid.Column column : scratch.cellLattice.densityColumns) {
+                assertNull(column);
+            }
+            for (CaveDensityGrid.Column column : scratch.cellLattice.warpColumns) {
+                assertNull(column);
+            }
         }
     }
 
@@ -1269,7 +1275,7 @@ public class IrisCaveCarver3DNearParityTest {
         doAnswer(invocation -> {
             lookups.incrementAndGet();
             return invocation.callRealMethod();
-        }).when(grid).column(any(), anyInt(), anyInt(), anyInt(), anyInt());
+        }).when(grid).column(any(), anyInt(), anyInt(), anyInt());
         return grid;
     }