Fix hotloading bricking the engine

src/main/java/com/volmit/iris/Iris.java

@@ -50,10 +50,7 @@ import com.volmit.iris.util.math.RNG;
 import com.volmit.iris.util.parallel.MultiBurst;
 import com.volmit.iris.util.plugin.*;
 import com.volmit.iris.util.reflect.ShadeFix;
-import com.volmit.iris.util.scheduling.GroupedExecutor;
+import com.volmit.iris.util.scheduling.*;
-import com.volmit.iris.util.scheduling.J;
-import com.volmit.iris.util.scheduling.Queue;
-import com.volmit.iris.util.scheduling.ShurikenQueue;
 import io.papermc.lib.PaperLib;
 import net.kyori.adventure.platform.bukkit.BukkitAudiences;
 import net.kyori.adventure.text.serializer.ComponentSerializer;
@@ -71,6 +68,7 @@ import java.io.*;
 import java.lang.annotation.Annotation;
 import java.net.URL;
 import java.util.Date;
+import java.util.Map;
 
 @SuppressWarnings("CanBeFinal")
 public class Iris extends VolmitPlugin implements Listener {
@@ -707,4 +705,37 @@ public class Iris extends VolmitPlugin implements Listener {
 
         }
     }
+
+    public static void dump()
+    {
+        try
+        {
+            File fi = Iris.instance.getDataFile("dump", "td-" + new java.sql.Date(M.ms()) + ".txt");
+            FileOutputStream fos = new FileOutputStream(fi );
+            Map<Thread, StackTraceElement[]> f = Thread.getAllStackTraces();
+            PrintWriter pw = new PrintWriter(fos);
+            for(Thread i : f.keySet())
+            {
+                pw.println("========================================");
+                pw.println("Thread: '" + i.getName() + "' ID: " + i.getId() + " STATUS: " + i.getState().name());
+
+                for(StackTraceElement j : f.get(i))
+                {
+                    pw.println("    @ " + j.toString());
+                }
+
+                pw.println("========================================");
+                pw.println();
+                pw.println();
+            }
+
+            pw.close();
+            System.out.println("DUMPED! See " + fi.getAbsolutePath());
+        }
+
+        catch(Throwable e)
+        {
+            e.printStackTrace();
+        }
+    }
 }

src/main/java/com/volmit/iris/core/decrees/DecIris.java

@@ -28,8 +28,13 @@ import com.volmit.iris.util.decree.DecreeOrigin;
 import com.volmit.iris.util.decree.annotations.Decree;
 import com.volmit.iris.util.decree.annotations.Param;
 import com.volmit.iris.util.format.C;
+import com.volmit.iris.util.math.M;
 
 import java.io.File;
+import java.io.FileOutputStream;
+import java.io.PrintWriter;
+import java.sql.Date;
+import java.util.Map;
 
 @Decree(name = "irisd", aliases = {"ird"}, description = "Basic Command")
 public class DecIris implements DecreeExecutor {

src/main/java/com/volmit/iris/engine/IrisEngine.java

@@ -65,6 +65,8 @@ import org.bukkit.generator.BlockPopulator;
 import java.io.File;
 import java.io.IOException;
 import java.util.Random;
+import java.util.concurrent.Semaphore;
+import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicLong;
@@ -287,12 +289,11 @@ public class IrisEngine extends BlockPopulator implements Engine {
 
     @Override
     public void close() {
-        J.car(art);
         closed = true;
+        J.car(art);
         getWorldManager().close();
         getTarget().close();
         saveEngineData();
-        getMantle().close();
         getTerrainActuator().close();
         getDecorantActuator().close();
         getBiomeActuator().close();
@@ -300,6 +301,8 @@ public class IrisEngine extends BlockPopulator implements Engine {
         getRavineModifier().close();
         getCaveModifier().close();
         getPostModifier().close();
+        getMantle().close();
+        Iris.debug("Engine Fully Shutdown!");
     }
 
     @Override
@@ -355,7 +358,12 @@ public class IrisEngine extends BlockPopulator implements Engine {
 
     @BlockCoordinates
     @Override
-    public void generate(int x, int z, Hunk<BlockData> vblocks, Hunk<Biome> vbiomes, boolean multicore) {
+    public void generate(int x, int z, Hunk<BlockData> vblocks, Hunk<Biome> vbiomes, boolean multicore) throws WrongEngineBroException {
+        if(closed)
+        {
+            throw new WrongEngineBroException();
+        }
+
         context.touch();
         getEngineData().getStatistics().generatedChunk();
         try {

src/main/java/com/volmit/iris/engine/framework/Engine.java

@@ -139,7 +139,7 @@ public interface Engine extends DataProvider, Fallible, LootProvider, BlockUpdat
     double modifyZ(double z);
 
     @BlockCoordinates
-    void generate(int x, int z, Hunk<BlockData> blocks, Hunk<Biome> biomes, boolean multicore);
+    void generate(int x, int z, Hunk<BlockData> blocks, Hunk<Biome> biomes, boolean multicore) throws WrongEngineBroException;
 
     EngineMetrics getMetrics();
 

src/main/java/com/volmit/iris/engine/framework/WrongEngineBroException.java

@@ -0,0 +1,22 @@
+/*
+ * Iris is a World Generator for Minecraft Bukkit Servers
+ * Copyright (c) 2021 Arcane Arts (Volmit Software)
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.volmit.iris.engine.framework;
+
+public class WrongEngineBroException extends Exception {
+}

src/main/java/com/volmit/iris/engine/platform/BukkitChunkGenerator.java

@@ -21,11 +21,13 @@ package com.volmit.iris.engine.platform;
 import com.volmit.iris.Iris;
 import com.volmit.iris.engine.data.chunk.TerrainChunk;
 import com.volmit.iris.engine.framework.Engine;
+import com.volmit.iris.engine.framework.WrongEngineBroException;
 import com.volmit.iris.engine.object.common.IrisWorld;
 import com.volmit.iris.util.collection.KList;
 import com.volmit.iris.util.hunk.Hunk;
 import com.volmit.iris.util.io.ReactiveFolder;
 import com.volmit.iris.util.scheduling.ChronoLatch;
+import com.volmit.iris.util.scheduling.J;
 import com.volmit.iris.util.scheduling.Looper;
 import com.volmit.iris.util.scheduling.PrecisionStopwatch;
 import org.bukkit.Bukkit;
@@ -40,8 +42,10 @@ import org.jetbrains.annotations.NotNull;
 import java.io.File;
 import java.util.List;
 import java.util.Random;
+import java.util.concurrent.Semaphore;
 
 public class BukkitChunkGenerator extends ChunkGenerator implements PlatformChunkGenerator {
+    private static final int HOTLOAD_LOCKS = 1000000;
     private final EngineProvider provider;
     private final IrisWorld world;
     private final File dataLocation;
@@ -50,11 +54,13 @@ public class BukkitChunkGenerator extends ChunkGenerator implements PlatformChun
     private final KList<BlockPopulator> populators;
     private final ChronoLatch hotloadChecker;
     private final Looper hotloader;
+    private final Semaphore hotloadLock;
     private final boolean studio;
 
     public BukkitChunkGenerator(IrisWorld world, boolean studio, File dataLocation, String dimensionKey) {
         populators = new KList<>();
         this.world = world;
+        this.hotloadLock = new Semaphore(HOTLOAD_LOCKS);
         this.hotloadChecker = new ChronoLatch(1000, false);
         this.studio = studio;
         this.dataLocation = dataLocation;
@@ -78,9 +84,12 @@ public class BukkitChunkGenerator extends ChunkGenerator implements PlatformChun
         hotloader.setName(getTarget().getWorld().name() + " Hotloader");
     }
 
-    public Engine getEngine() {
+    public synchronized Engine getEngine() {
+        synchronized (provider)
+        {
             return provider.getEngine();
         }
+    }
 
     @Override
     public boolean isHeadless() {
@@ -89,9 +98,12 @@ public class BukkitChunkGenerator extends ChunkGenerator implements PlatformChun
 
     @Override
     public void close() {
+        synchronized (provider)
+        {
             hotloader.interrupt();
             provider.close();
         }
+    }
 
     @Override
     public boolean isStudio() {
@@ -100,19 +112,36 @@ public class BukkitChunkGenerator extends ChunkGenerator implements PlatformChun
 
     @Override
     public void hotload() {
+        J.aBukkit(() -> {
+            try {
+                hotloadLock.acquire(HOTLOAD_LOCKS);
                 initialize();
+                hotloadLock.release(HOTLOAD_LOCKS);
+            } catch (InterruptedException e) {
+                e.printStackTrace();
+            }
+        });
     }
 
     private void initialize() {
+        synchronized (provider)
+        {
             provider.provideEngine(world, dimensionKey, dataLocation, isStudio(), (e) -> {
                 populators.clear();
                 populators.add((BlockPopulator) e);
                 folder.checkIgnore();
             });
         }
+    }
 
     @Override
     public @NotNull ChunkData generateChunkData(@NotNull World world, @NotNull Random ignored, int x, int z, @NotNull BiomeGrid biome) {
+        try {
+            hotloadLock.acquire();
+        } catch (InterruptedException e) {
+            e.printStackTrace();
+        }
+
         try {
             Iris.debug("Generated " + x + " " + z);
             PrecisionStopwatch ps = PrecisionStopwatch.start();
@@ -121,8 +150,11 @@ public class BukkitChunkGenerator extends ChunkGenerator implements PlatformChun
             Hunk<Biome> biomes = Hunk.view((BiomeGrid) tc);
             this.world.bind(world);
             getEngine().generate(x * 16, z * 16, blocks, biomes, true);
+            hotloadLock.release();
             return tc.getRaw();
-        } catch (Throwable e) {
+        }
+
+        catch (Throwable e) {
             Iris.error("======================================");
             e.printStackTrace();
             Iris.reportErrorChunk(x, z, e, "CHUNK");
@@ -136,8 +168,10 @@ public class BukkitChunkGenerator extends ChunkGenerator implements PlatformChun
                 }
             }
 
+            hotloadLock.release();
             return d;
         }
+
     }
 
     @NotNull

src/main/java/com/volmit/iris/util/mantle/Mantle.java

@@ -179,6 +179,11 @@ public class Mantle {
                 .get(x & 15, y & 15, z & 15);
     }
 
+    public boolean isClosed()
+    {
+        return closed.get();
+    }
+
     /**
      * Closes the Mantle. By closing the mantle, you can no longer read or write
      * any data to the mantle or it's Tectonic Plates. Closing will also flush any