Lock-free Terrain3D caches

- Replace synchronized move-to-last LRU stripes with chunk-block CLOCK cache
- Copy-on-write compiled profile map, lock only on miss
- Raise column cache to 2048 chunks

core/src/main/java/art/arcane/iris/generation/cache/ConcurrentClockCache.java

@@ -0,0 +1,128 @@
+package art.arcane.iris.generation.cache;
+
+import art.arcane.volmlib.util.cache.CacheKey;
+
+import java.util.Objects;
+import java.util.concurrent.atomic.AtomicReferenceArray;
+import java.util.function.Consumer;
+
+/**
+ * A bounded, lock-free, lossy long-keyed cache for memoized values. Keys hash into 4-way sets and a full set
+ * evicts with CLOCK (second chance), so reads never write shared state except to set a reference bit once.
+ * Concurrent inserts of the same key may both compute; the first published value wins.
+ */
+public final class ConcurrentClockCache<V> {
+    private static final int WAYS = 4;
+
+    private final AtomicReferenceArray<Entry<V>> slots;
+    private final long sets;
+
+    public ConcurrentClockCache(int capacity) {
+        if (capacity <= 0) {
+            throw new IllegalArgumentException("Cache capacity must be positive");
+        }
+        int setCount = Math.ceilDiv(capacity, WAYS);
+        sets = setCount;
+        slots = new AtomicReferenceArray<>(Math.multiplyExact(setCount, WAYS));
+    }
+
+    public int capacity() {
+        return slots.length();
+    }
+
+    public V get(long key) {
+        long hash = CacheKey.mix(key);
+        int base = base(hash);
+        for (int way = 0; way < WAYS; way++) {
+            Entry<V> entry = slots.getAcquire(base + way);
+            if (entry != null && entry.key == key) {
+                entry.touch();
+                return entry.value;
+            }
+        }
+        return null;
+    }
+
+    public V putIfAbsent(long key, V value) {
+        Objects.requireNonNull(value, "Cached value");
+        long hash = CacheKey.mix(key);
+        int base = base(hash);
+        int hand = (int) (hash >>> 60) & (WAYS - 1);
+        while (true) {
+            int empty = -1;
+            for (int way = 0; way < WAYS; way++) {
+                Entry<V> entry = slots.getAcquire(base + way);
+                if (entry == null) {
+                    if (empty < 0) {
+                        empty = way;
+                    }
+                } else if (entry.key == key) {
+                    entry.touch();
+                    return entry.value;
+                }
+            }
+            Entry<V> created = new Entry<>(key, value);
+            if (empty >= 0) {
+                if (slots.compareAndSet(base + empty, null, created)) {
+                    return value;
+                }
+                continue;
+            }
+            int victim = -1;
+            Entry<V> victimEntry = null;
+            for (int step = 0; step < WAYS * 2 && victim < 0; step++) {
+                int way = (hand + step) & (WAYS - 1);
+                Entry<V> entry = slots.getAcquire(base + way);
+                if (entry == null || !entry.referenced) {
+                    victim = way;
+                    victimEntry = entry;
+                } else {
+                    entry.referenced = false;
+                }
+            }
+            if (victim < 0) {
+                victim = hand;
+                victimEntry = slots.getAcquire(base + victim);
+            }
+            if (slots.compareAndSet(base + victim, victimEntry, created)) {
+                return value;
+            }
+        }
+    }
+
+    public void forEach(Consumer<? super V> action) {
+        for (int index = 0; index < slots.length(); index++) {
+            Entry<V> entry = slots.getAcquire(index);
+            if (entry != null) {
+                action.accept(entry.value);
+            }
+        }
+    }
+
+    public void clear() {
+        for (int index = 0; index < slots.length(); index++) {
+            slots.setRelease(index, null);
+        }
+    }
+
+    private int base(long hash) {
+        return (int) (((hash & 0xffffffffL) * sets) >>> 32) * WAYS;
+    }
+
+    private static final class Entry<V> {
+        private final long key;
+        private final V value;
+        private boolean referenced;
+
+        private Entry(long key, V value) {
+            this.key = key;
+            this.value = value;
+        }
+
+        private void touch() {
+            if (!referenced) {
+                referenced = true;
+            }
+        }
+    }
+}

core/src/main/java/art/arcane/iris/generation/runtime/IrisComplex.java

@@ -115,7 +115,7 @@ public class IrisComplex implements DataProvider {
     private static final AtomicLong lastBoundsFailureLog = new AtomicLong(0L);
     private static final int GRID_BOUNDS_CACHE_SIZE = 8192;
     private static final int STUDIO_NOISE_CACHE_SIZE = 32_768;
-    private static final int TERRAIN_COLUMN_CACHE_SIZE = 65_536;
+    private static final int TERRAIN_COLUMN_CACHE_SIZE = 524_288;
     /** One million corners: about 16 MB, roughly a 4000 by 4000 block area at the 4-block grid. */
     private static final int SHARED_CORNER_BOUNDS_CAPACITY = 1 << 20;
     /** The slope streams measure the rise across a run of this many blocks, so gradient is slope over run. */

core/src/main/java/art/arcane/iris/generation/terrain/Terrain3DRuntime.java

@@ -1,12 +1,11 @@
 package art.arcane.iris.generation.terrain;
 
+import art.arcane.iris.generation.cache.ConcurrentClockCache;
 import art.arcane.iris.pack.loading.IrisData;
 import art.arcane.iris.generation.biome.IrisBiome;
 import art.arcane.iris.generation.noise.IrisGeneratorStyle;
 import art.arcane.volmlib.util.noise.CNG;
-import art.arcane.volmlib.util.cache.CacheKey;
 import art.arcane.volmlib.util.math.RNG;
-import it.unimi.dsi.fastutil.longs.Long2ObjectLinkedOpenHashMap;
 
 import java.util.Arrays;
 import java.util.IdentityHashMap;
@@ -16,9 +15,10 @@ import java.util.concurrent.atomic.AtomicReferenceArray;
 
 public final class Terrain3DRuntime {
     private static final int STEP = 4;
-    private static final int CACHE_STRIPES = 16;
+    private static final int MINIMUM_COLUMNS = 16;
+    private static final int ANCHOR_BLOCK_SIZE = 16;
     private static final int MAXIMUM_ANCHORS = 32_768;
-    private static final long ANCHOR_CACHE_BYTES = 64L * 1024 * 1024;
+    private static final long ANCHOR_CACHE_BYTES = 128L * 1024 * 1024;
     private static final long DENSITY_SALT = 0x536E4A11C924B3D7L;
     private static final long CRACK_SALT = 0x7839A16DC4052EFBL;
     private static final NodeSample EMPTY_SAMPLE = new NodeSample(0D, 0D, 0D);
@@ -28,10 +28,11 @@ public final class Terrain3DRuntime {
     private final Sources sources;
     private final Options options;
     private final NoiseFactory noiseFactory;
-    private final BoundedCache<Terrain3DFragmentFilter.DensityColumn> columns;
+    private final BlockCache<Terrain3DFragmentFilter.DensityColumn> columns;
-    private final BoundedCache<Anchor> anchors;
+    private final BlockCache<Anchor> anchors;
     private final Terrain3DFragmentFilter fragments;
-    private final Map<IrisTerrain3D, CompiledProfile> profiles = new IdentityHashMap<>();
+    private final Object profileLock = new Object();
+    private volatile Map<IrisTerrain3D, CompiledProfile> profiles = new IdentityHashMap<>();
     private final ThreadLocal<ColumnMemo> localColumn = ThreadLocal.withInitial(ColumnMemo::new);
     private volatile Object cacheGeneration = new Object();
 
@@ -46,8 +47,8 @@ public final class Terrain3DRuntime {
         this.sources = Objects.requireNonNull(sources, "Terrain sources");
         this.options = Objects.requireNonNull(options, "Terrain options");
         this.noiseFactory = Objects.requireNonNull(noiseFactory, "Terrain noise factory");
-        columns = new BoundedCache<>(options.maximumColumns());
+        columns = new BlockCache<>(options.maximumColumns(), 0);
-        anchors = new BoundedCache<>(anchorCacheCapacity(options.height(), options.maximumColumns()));
+        anchors = new BlockCache<>(anchorCacheCapacity(options.height(), options.maximumColumns()), 2);
         fragments = new Terrain3DFragmentFilter(this::rawColumn);
     }
 
@@ -59,7 +60,7 @@ public final class Terrain3DRuntime {
         long bytesPerAnchor = 160L + 48L * (Math.ceilDiv((long) height, STEP) + 1);
         long maximumEntries = Math.min(MAXIMUM_ANCHORS,
                 Math.min((long) maximumColumns * 8, ANCHOR_CACHE_BYTES / bytesPerAnchor));
-        return Math.max(CACHE_STRIPES, (int) (maximumEntries / CACHE_STRIPES) * CACHE_STRIPES);
+        return Math.max(ANCHOR_BLOCK_SIZE, (int) (maximumEntries / ANCHOR_BLOCK_SIZE) * ANCHOR_BLOCK_SIZE);
     }
 
     public double height(int x, int z) {
@@ -94,8 +95,8 @@ public final class Terrain3DRuntime {
         columns.clear();
         anchors.clear();
         fragments.clear();
-        synchronized (profiles) {
+        synchronized (profileLock) {
-            profiles.clear();
+            profiles = new IdentityHashMap<>();
         }
         cacheGeneration = new Object();
         localColumn.remove();
@@ -106,9 +107,8 @@ public final class Terrain3DRuntime {
     }
 
     private Terrain3DFragmentFilter.DensityColumn rawColumn(int x, int z) {
-        long key = pack(x, z);
+        Terrain3DFragmentFilter.DensityColumn cached = columns.get(x, z);
-        Terrain3DFragmentFilter.DensityColumn cached = columns.get(key);
+        return cached == null ? columns.putIfAbsent(x, z,
-        return cached == null ? columns.putIfAbsent(key,
                 new Terrain3DFragmentFilter.DensityColumn(createColumn(x, z))) : cached;
     }
 
@@ -196,9 +196,8 @@ public final class Terrain3DRuntime {
     }
 
     private Anchor anchor(int x, int z) {
-        long key = pack(x, z);
+        Anchor cached = anchors.get(x, z);
-        Anchor cached = anchors.get(key);
+        return cached == null ? anchors.putIfAbsent(x, z, createAnchor(x, z)) : cached;
-        return cached == null ? anchors.putIfAbsent(key, createAnchor(x, z)) : cached;
     }
 
     private Anchor createAnchor(int x, int z) {
@@ -227,19 +226,19 @@ public final class Terrain3DRuntime {
         if (config == null || !config.isEnabled()) {
             return DISABLED;
         }
-        synchronized (profiles) {
+        CompiledProfile cached = profiles.get(config);
-            CompiledProfile cached = profiles.get(config);
+        if (cached != null) {
-            if (cached != null) {
+            return cached;
-                return cached;
-            }
         }
         CompiledProfile compiled = compile(config);
-        synchronized (profiles) {
+        synchronized (profileLock) {
             CompiledProfile existing = profiles.get(config);
             if (existing != null) {
                 return existing;
             }
-            profiles.put(config, compiled);
+            Map<IrisTerrain3D, CompiledProfile> updated = new IdentityHashMap<>(profiles);
+            updated.put(config, compiled);
+            profiles = updated;
             return compiled;
         }
     }
@@ -314,7 +313,7 @@ public final class Terrain3DRuntime {
                           boolean enabled, int maximumColumns) {
         public Options {
             if (height < 2 || height > 4096 || !Double.isFinite(fluidHeight)
-                    || maximumColumns < CACHE_STRIPES) {
+                    || maximumColumns < MINIMUM_COLUMNS) {
                 throw new IllegalArgumentException("Invalid volumetric terrain runtime options");
             }
         }
@@ -408,74 +407,61 @@ public final class Terrain3DRuntime {
                                    double fluidFade, NoiseSource density, NoiseSource cracks) {
     }
 
-    private static final class BoundedCache<T> {
+    /**
-        private final CacheStripe<T>[] stripes;
+     * Groups the 16x16 block area of one chunk into a single cache entry, so neighbouring lookups share one
+     * lock-free set probe and a working set of whole chunks is evicted together.
+     */
+    private static final class BlockCache<T> {
+        private final ConcurrentClockCache<AtomicReferenceArray<T>> blocks;
+        private final int cellShift;
+        private final int axisMask;
+        private final int axisBits;
 
-        @SuppressWarnings("unchecked")
+        private BlockCache(int maximumEntries, int cellShift) {
-        private BoundedCache(int maximumSize) {
+            this.cellShift = cellShift;
-            stripes = (CacheStripe<T>[]) new CacheStripe<?>[CACHE_STRIPES];
+            axisBits = 4 - cellShift;
-            for (int index = 0; index < stripes.length; index++) {
+            axisMask = (1 << axisBits) - 1;
-                stripes[index] = new CacheStripe<>(Math.ceilDiv(maximumSize, CACHE_STRIPES));
+            int blockSize = 1 << (axisBits * 2);
+            blocks = new ConcurrentClockCache<>(Math.ceilDiv(maximumEntries, blockSize));
+        }
+
+        private T get(int x, int z) {
+            AtomicReferenceArray<T> block = blocks.get(blockKey(x, z));
+            return block == null ? null : block.getAcquire(slot(x, z));
+        }
+
+        private T putIfAbsent(int x, int z, T value) {
+            long key = blockKey(x, z);
+            AtomicReferenceArray<T> block = blocks.get(key);
+            if (block == null) {
+                block = blocks.putIfAbsent(key, new AtomicReferenceArray<>(1 << (axisBits * 2)));
             }
-        }
+            T existing = block.compareAndExchange(slot(x, z), null, value);
-
+            return existing == null ? value : existing;
-        private T get(long key) {
-            return stripe(key).get(key);
-        }
-
-        private T putIfAbsent(long key, T value) {
-            return stripe(key).putIfAbsent(key, value);
         }
 
         private void clear() {
-            for (CacheStripe<T> stripe : stripes) {
+            blocks.clear();
-                stripe.clear();
-            }
         }
 
         private int size() {
-            int size = 0;
+            int[] size = new int[1];
-            for (CacheStripe<T> stripe : stripes) {
+            blocks.forEach(block -> {
-                size += stripe.size();
+                for (int index = 0; index < block.length(); index++) {
-            }
+                    if (block.getAcquire(index) != null) {
-            return size;
+                        size[0]++;
+                    }
+                }
+            });
+            return size[0];
         }
 
-        private CacheStripe<T> stripe(long key) {
+        private static long blockKey(int x, int z) {
-            return stripes[(int) CacheKey.mix(key) & (CACHE_STRIPES - 1)];
+            return pack(x >> 4, z >> 4);
-        }
-    }
-
-    private static final class CacheStripe<T> {
-        private final int maximumSize;
-        private final Long2ObjectLinkedOpenHashMap<T> entries = new Long2ObjectLinkedOpenHashMap<>(16);
-
-        private CacheStripe(int maximumSize) {
-            this.maximumSize = maximumSize;
         }
 
-        private synchronized T get(long key) {
+        private int slot(int x, int z) {
-            return entries.getAndMoveToLast(key);
+            return ((x >> cellShift) & axisMask) << axisBits | ((z >> cellShift) & axisMask);
-        }
-
-        private synchronized T putIfAbsent(long key, T value) {
-            T existing = entries.getAndMoveToLast(key);
-            if (existing != null) {
-                return existing;
-            }
-            entries.putAndMoveToLast(key, value);
-            if (entries.size() > maximumSize) {
-                entries.removeFirst();
-            }
-            return value;
-        }
-
-        private synchronized void clear() {
-            entries.clear();
-        }
-
-        private synchronized int size() {
-            return entries.size();
         }
     }
 }

core/src/test/java/art/arcane/iris/generation/cache/ConcurrentClockCacheTest.java

@@ -0,0 +1,84 @@
+package art.arcane.iris.generation.cache;
+
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertSame;
+import static org.junit.Assert.assertTrue;
+
+public class ConcurrentClockCacheTest {
+    @Test
+    public void firstPublishedValueWins() {
+        ConcurrentClockCache<Object> cache = new ConcurrentClockCache<>(16);
+        Object first = new Object();
+        assertNull(cache.get(7L));
+        assertSame(first, cache.putIfAbsent(7L, first));
+        assertSame(first, cache.putIfAbsent(7L, new Object()));
+        assertSame(first, cache.get(7L));
+        cache.clear();
+        assertNull(cache.get(7L));
+    }
+
+    @Test
+    public void capacityBoundsRetainedEntries() {
+        ConcurrentClockCache<Long> cache = new ConcurrentClockCache<>(64);
+        for (long key = 0; key < 10_000; key++) {
+            assertEquals(Long.valueOf(key), cache.putIfAbsent(key, key));
+        }
+        AtomicInteger retained = new AtomicInteger();
+        cache.forEach(value -> {
+            retained.incrementAndGet();
+            assertEquals(value, cache.get(value));
+        });
+        assertEquals(cache.capacity(), retained.get());
+        assertTrue(cache.capacity() >= 64);
+    }
+
+    @Test
+    public void referencedEntriesSurviveAStreamOfOneTimeKeys() {
+        ConcurrentClockCache<Long> cache = new ConcurrentClockCache<>(4);
+        cache.putIfAbsent(-1L, -1L);
+        for (long key = 0; key < 1_000; key++) {
+            assertEquals(Long.valueOf(-1L), cache.get(-1L));
+            cache.putIfAbsent(key, key);
+        }
+        assertEquals(Long.valueOf(-1L), cache.get(-1L));
+    }
+
+    @Test
+    public void concurrentWritersAgreeOnOneValuePerKey() throws Exception {
+        ConcurrentClockCache<Object> cache = new ConcurrentClockCache<>(1 << 12);
+        List<Future<Object[]>> results = new ArrayList<>();
+        try (ExecutorService executor = Executors.newFixedThreadPool(8)) {
+            for (int thread = 0; thread < 8; thread++) {
+                results.add(executor.submit(() -> {
+                    Object[] seen = new Object[256];
+                    for (int key = 0; key < seen.length; key++) {
+                        seen[key] = cache.putIfAbsent(key, new Object());
+                    }
+                    return seen;
+                }));
+            }
+            Object[] reference = results.getFirst().get(10, TimeUnit.SECONDS);
+            for (Future<Object[]> result : results) {
+                Object[] seen = result.get(10, TimeUnit.SECONDS);
+                for (int key = 0; key < seen.length; key++) {
+                    Object cached = cache.get(key);
+                    if (cached != null) {
+                        assertSame(cached, seen[key]);
+                        assertSame(reference[key], seen[key]);
+                    }
+                }
+            }
+        }
+    }
+}

core/src/test/java/art/arcane/iris/generation/terrain/Terrain3DRuntimeTest.java

@@ -302,7 +302,7 @@ public class Terrain3DRuntimeTest {
                 assertEquals(0, capacity % 16);
                 assertTrue(capacity >= 16);
                 assertTrue(capacity <= 32_768);
-                assertTrue(capacity * fullHeightStorage <= 64L * 1024 * 1024);
+                assertTrue(capacity * fullHeightStorage <= 128L * 1024 * 1024);
             }
         }
     }
@@ -354,9 +354,10 @@ public class Terrain3DRuntimeTest {
         for (int x = -65_536; x < 65_536; x++) {
             assertEquals(96, runtime.column(x, -1).topY());
         }
-        assertEquals(65_536, runtime.cachedColumnCount());
+        assertTrue(runtime.cachedColumnCount() > 0);
+        assertTrue(runtime.cachedColumnCount() <= 65_536);
         assertEquals(96, runtime.column(-65_536, -1).topY());
-        assertEquals(65_536, runtime.cachedColumnCount());
+        assertTrue(runtime.cachedColumnCount() <= 65_536);
         runtime.clear();
         assertEquals(0, runtime.cachedColumnCount());
     }