From e926a08def15d764de6fbb80af717668af150890 Mon Sep 17 00:00:00 2001 From: Daniel Mills Date: Wed, 30 Dec 2020 07:09:39 -0500 Subject: [PATCH] Fix mca gens --- src/main/java/com/volmit/iris/Iris.java | 4 + .../manager/command/CommandIrisPregen.java | 68 +++-- .../volmit/iris/pregen/DirectWorldWriter.java | 40 +-- .../com/volmit/iris/pregen/Pregenerator.java | 252 +++++++++++++----- .../iris/scaffold/parallel/BurstExecutor.java | 9 +- .../com/volmit/iris/util/ChunkPosition.java | 5 + 6 files changed, 254 insertions(+), 124 deletions(-) diff --git a/src/main/java/com/volmit/iris/Iris.java b/src/main/java/com/volmit/iris/Iris.java index 6dfe6b6b6..e437f2955 100644 --- a/src/main/java/com/volmit/iris/Iris.java +++ b/src/main/java/com/volmit/iris/Iris.java @@ -516,4 +516,8 @@ public class Iris extends VolmitPlugin syncJobs.clear(); } } + + public boolean isMCA() { + return IrisSettings.get().useExperimentalGleamMCADirectWriteMode; + } } diff --git a/src/main/java/com/volmit/iris/manager/command/CommandIrisPregen.java b/src/main/java/com/volmit/iris/manager/command/CommandIrisPregen.java index 222e1b17a..d00014de6 100644 --- a/src/main/java/com/volmit/iris/manager/command/CommandIrisPregen.java +++ b/src/main/java/com/volmit/iris/manager/command/CommandIrisPregen.java @@ -1,13 +1,14 @@ package com.volmit.iris.manager.command; -import com.volmit.iris.util.KList; -import org.bukkit.World; -import org.bukkit.entity.Player; - import com.volmit.iris.Iris; +import com.volmit.iris.pregen.Pregenerator; +import com.volmit.iris.scaffold.IrisWorlds; +import com.volmit.iris.util.KList; import com.volmit.iris.util.MortarCommand; import com.volmit.iris.util.MortarSender; import com.volmit.iris.util.PregenJob; +import org.bukkit.World; +import org.bukkit.entity.Player; public class CommandIrisPregen extends MortarCommand { @@ -45,22 +46,44 @@ public class CommandIrisPregen extends MortarCommand if(args[0].equalsIgnoreCase("stop") || args[0].equalsIgnoreCase("x")) { - if(PregenJob.task == -1) - { - sender.sendMessage("No Active Pregens"); + if(PregenJob.task == -1) { + if (Pregenerator.shutdownInstance()) { + sender.sendMessage("Stopped Pregen."); + } else + { + sender.sendMessage("No Active Pregens."); + } } else { - sender.sendMessage("Stopped All Pregens."); + sender.sendMessage("Stopped Pregen."); PregenJob.stop(); } return true; } - else if(args[0].equalsIgnoreCase("pause")) + else if(args[0].equalsIgnoreCase("pause") || args[0].equalsIgnoreCase("resume")) { if(PregenJob.task == -1) { - sender.sendMessage("No Active Pregens"); + if(Pregenerator.getInstance() != null) + { + Pregenerator.pauseResume(); + + if(Pregenerator.isPaused()) + { + sender.sendMessage("Pregen Paused"); + } + + else + { + sender.sendMessage("Pregen Resumed"); + } + } + + else + { + sender.sendMessage("No Active Pregens"); + } } else @@ -79,27 +102,24 @@ public class CommandIrisPregen extends MortarCommand } return true; - } - else if(args[0].equalsIgnoreCase("resume")) - { - if(PregenJob.isPaused()){ - sender.sendMessage("Pregen Resumed"); - } - else - { - sender.sendMessage("No paused pregens. Use /ir pregen pause to pause."); - } } - else - if(sender.isPlayer()) + else if(sender.isPlayer()) { Player p = sender.player(); World world = p.getWorld(); try { - new PregenJob(world, getVal(args[0]), sender, () -> + if(Iris.instance.isMCA() && IrisWorlds.access(world) != null) { - }); + new Pregenerator(world, getVal(args[0]), 0, 0); + } + + else + { + new PregenJob(world, getVal(args[0]), sender, () -> + { + }); + } } catch (NumberFormatException e){ sender.sendMessage("Invalid argument in command"); return true; diff --git a/src/main/java/com/volmit/iris/pregen/DirectWorldWriter.java b/src/main/java/com/volmit/iris/pregen/DirectWorldWriter.java index f1da8e247..d49bdc9ed 100644 --- a/src/main/java/com/volmit/iris/pregen/DirectWorldWriter.java +++ b/src/main/java/com/volmit/iris/pregen/DirectWorldWriter.java @@ -1,6 +1,5 @@ package com.volmit.iris.pregen; -import com.volmit.iris.Iris; import com.volmit.iris.nms.INMS; import com.volmit.iris.scaffold.cache.Cache; import com.volmit.iris.scaffold.parallel.BurstExecutor; @@ -8,9 +7,10 @@ import com.volmit.iris.scaffold.parallel.MultiBurst; import com.volmit.iris.util.B; import com.volmit.iris.util.KList; import com.volmit.iris.util.KMap; -import com.volmit.iris.util.M; -import io.timeandspace.smoothie.SmoothieMap; -import net.querz.mca.*; +import net.querz.mca.Chunk; +import net.querz.mca.MCAFile; +import net.querz.mca.MCAUtil; +import net.querz.mca.Section; import net.querz.nbt.tag.CompoundTag; import net.querz.nbt.tag.StringTag; import org.bukkit.NamespacedKey; @@ -18,20 +18,17 @@ import org.bukkit.block.Biome; import org.bukkit.block.data.BlockData; import java.io.File; -import java.io.IOException; import java.util.Map; public class DirectWorldWriter { private final File worldFolder; private final Map writeBuffer; - private final Map lastUse; private static final Map blockDataCache = new KMap<>(); private static final Map biomeIds = computeBiomeIDs(); public DirectWorldWriter(File worldFolder) { this.worldFolder = worldFolder; - lastUse = SmoothieMap.newBuilder().build(); writeBuffer = new KMap<>(); new File(worldFolder, "iris/mca-region").mkdirs(); } @@ -42,11 +39,6 @@ public class DirectWorldWriter { for(Long i : new KList<>(writeBuffer.keySet())) { - if(M.ms() - lastUse.get(i) < 10000) - { - continue; - } - ex2.queue(() -> { int x = Cache.keyX(i); int z = Cache.keyZ(i); @@ -69,7 +61,6 @@ public class DirectWorldWriter { } - lastUse.remove(i); MCAUtil.write(writeBuffer.get(i), f, true); writeBuffer.remove(i); } catch (Throwable e) { @@ -77,6 +68,8 @@ public class DirectWorldWriter { } }); } + + ex2.complete(); } public void optimizeChunk(int x, int z) @@ -217,7 +210,6 @@ public class DirectWorldWriter { if(c == null) { c = Chunk.newChunk(); - lastUse.put(Cache.key(x >> 5, z >> 5), M.ms()); mca.setChunk(x&31, z&31, c); } @@ -236,27 +228,7 @@ public class DirectWorldWriter { File f = getMCAFile(x, z); mca = new MCAFile(x, z); - if(f.exists()) - { - try { - Iris.warn("HAD TO LOAD MCA REGION FILE " + x + " " + z + " which is EXPENSIVE!"); - mca = MCAUtil.read(f); - try - { - mca.cleanupPalettesAndBlockStates(); - } - catch(Throwable ee) - { - - } - } catch (IOException e) { - e.printStackTrace(); - Iris.warn("Failed to read RandomAccessFile " + f.getAbsolutePath() + ", assuming empty region!"); - } - } - - lastUse.put(key, M.ms()); writeBuffer.put(key, mca); return mca; } diff --git a/src/main/java/com/volmit/iris/pregen/Pregenerator.java b/src/main/java/com/volmit/iris/pregen/Pregenerator.java index 36af82f03..044fc3e85 100644 --- a/src/main/java/com/volmit/iris/pregen/Pregenerator.java +++ b/src/main/java/com/volmit/iris/pregen/Pregenerator.java @@ -2,10 +2,12 @@ package com.volmit.iris.pregen; import com.volmit.iris.Iris; import com.volmit.iris.IrisSettings; +import com.volmit.iris.scaffold.IrisWorlds; import com.volmit.iris.scaffold.engine.IrisAccess; import com.volmit.iris.scaffold.parallel.BurstExecutor; import com.volmit.iris.scaffold.parallel.MultiBurst; import com.volmit.iris.util.*; +import io.papermc.lib.PaperLib; import lombok.Data; import org.bukkit.Bukkit; import org.bukkit.Chunk; @@ -24,18 +26,22 @@ import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.Comparator; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.locks.ReentrantLock; @Data public class Pregenerator implements Listener { + private static Pregenerator instance; private static final Color COLOR_MCA_PREPARE = Color.DARK_GRAY; private static final Color COLOR_MCA_GENERATE = Color.MAGENTA; - private static final Color COLOR_MCA_GENERATE_SLOW = Color.MAGENTA.darker().darker(); - private static final Color COLOR_MCA_GENERATED = Color.CYAN; + private static final Color COLOR_MCA_GENERATE_SLOW = Color.YELLOW; + private static final Color COLOR_MCA_GENERATED = Color.GREEN; private static final Color COLOR_MCA_SEALED = Color.GREEN; private static final Color COLOR_MCA_SEALING = Color.GREEN.darker().darker(); + private static final Color COLOR_MCA_DEFERRED = Color.YELLOW.darker().darker(); private final World world; private final DirectWorldWriter directWriter; private final AtomicBoolean active; @@ -44,78 +50,50 @@ public class Pregenerator implements Listener private final ChunkPosition max; private final ChunkPosition min; private final MCAPregenGui gui; + private final KList mcaDefer; + private final AtomicInteger generated; + private final AtomicInteger generatedLast; + private final AtomicInteger perSecond; - public Pregenerator(World world, IrisAccess access, int blockSize, int xoffset, int zoffset) + public Pregenerator(World world, int blockSize, int xoffset, int zoffset) { + instance(); this.world = world; + perSecond = new AtomicInteger(0); + generatedLast = new AtomicInteger(0); + generated = new AtomicInteger(0); + mcaDefer = new KList<>(); + IrisAccess access = IrisWorlds.access(world); this.directWriter = new DirectWorldWriter(world.getWorldFolder()); this.running = new AtomicBoolean(true); this.active = new AtomicBoolean(true); MultiBurst burst = new MultiBurst(Runtime.getRuntime().availableProcessors()); - int mcaSize = (((blockSize >> 4) + 2) >> 5) + 2; - int xaoff = (xoffset >> 4) >> 5; - int zaoff = (zoffset >> 4) >> 5; + int mcaSize = (((blockSize >> 4) + 2) >> 5) + 1; onComplete = new KList<>(); - max = new ChunkPosition(xoffset + (blockSize/2), zoffset + (blockSize/2)); - min = new ChunkPosition(xoffset - (blockSize/2), zoffset - (blockSize/2)); + max = new ChunkPosition(0,0); + min = new ChunkPosition(0,0); + new Spiraler(mcaSize, mcaSize, (xx,zz) -> { + min.setX(Math.min(xx << 5, min.getX())); + min.setZ(Math.min(zz << 5, min.getZ())); + max.setX(Math.max((xx << 5) + 31, max.getX())); + max.setZ(Math.max((zz << 5) + 31, max.getZ())); + }).drain(); gui = IrisSettings.get().isLocalPregenGui() && IrisSettings.get().isUseServerLaunchedGuis() ? MCAPregenGui.createAndShowGUI(this) : null; + KList order = computeChunkOrder(); + Consumer3> mcaIteration = + (ox, oz, r) -> order.forEach((i) + -> r.accept(i.getX() + ox, i.getZ() + oz)); Spiraler spiraler = new Spiraler(mcaSize, mcaSize, (xx,zz) -> { - int x = xaoff + xx; - int z = zaoff + zz; - try { + flushWorld(); + drawMCA(xx, zz, COLOR_MCA_PREPARE); + if(generateMCARegion(xx, zz, burst, access, mcaIteration)) + { flushWorld(); - drawMCA(x, z, COLOR_MCA_PREPARE); - File mca = new File(world.getWorldFolder(), "region/r." + x + "." + z + ".mca"); - File mcg = directWriter.getMCAFile(x, z); - Path fileToMovePath = Paths.get(mcg.toURI()); - Path targetPath = Paths.get(mca.toURI()); - BurstExecutor e = burst.burst(1024); - int mcaox = x << 5; - int mcaoz = z << 5; + } - if(isMCAWritable(x,z) && !mcg.exists()) - { - for(int i = 0; i < 32; i++) - { - int ii = i; - for(int j = 0; j < 32; j++) - { - int jj = j; - e.queue(() -> { - draw(ii + mcaox, jj + mcaoz, COLOR_MCA_GENERATE); - access.directWriteChunk(world, ii + mcaox, jj + mcaoz, directWriter); - draw(ii + mcaox, jj + mcaoz, COLOR_MCA_GENERATED); - }); - } - } - - directWriter.flush(); - Files.move(fileToMovePath, targetPath); - } - - else - { - for(int i = 0; i < 32; i++) - { - int ii = i; - for(int j = 0; j < 32; j++) - { - int jj = j; - e.queue(() -> { - draw(ii + mcaox, jj + mcaoz, COLOR_MCA_GENERATE_SLOW); - access.generatePaper(world, ii+mcaox, jj + mcaoz); - draw(ii + mcaox, jj + mcaoz, COLOR_MCA_GENERATED); - }); - } - } - } - - e.complete(); - drawMCA(x, z, COLOR_MCA_SEALED); - flushWorld(); - drawMCA(x, z, COLOR_MCA_SEALED); - } catch (IOException e) { - e.printStackTrace(); + else + { + drawMCA(xx, zz, COLOR_MCA_DEFERRED); } }); new Thread(() -> { @@ -127,6 +105,17 @@ public class Pregenerator implements Listener } } + mcaDefer.removeDuplicates(); + + while(running.get() && mcaDefer.isNotEmpty()) + { + ChunkPosition p = mcaDefer.popLast(); + generateDeferedMCARegion(p.getX(), p.getZ(), burst, access, mcaIteration); + drawMCA(p.getX(), p.getZ(), COLOR_MCA_SEALING); + flushWorld(); + drawMCA(p.getX(), p.getZ(), COLOR_MCA_SEALED); + } + burst.shutdownNow(); directWriter.flush(); flushWorld(); @@ -136,7 +125,142 @@ public class Pregenerator implements Listener { gui.close(); } - }); + }).start(); + new Thread(() -> { + PrecisionStopwatch p = PrecisionStopwatch.start(); + + while(running.get() && active.get()) { + int m = generated.get(); + int w = generatedLast.get(); + int up = m - w; + double dur = p.getMilliseconds(); + perSecond.set((int) (up / (dur / 1000D))); + p.reset(); + p.begin(); + generatedLast.set(m); + } + }).start(); + } + + private boolean generateMCARegion(int x, int z, MultiBurst burst, IrisAccess access, Consumer3> mcaIteration) { + File mca = new File(world.getWorldFolder(), "region/r." + x + "." + z + ".mca"); + File mcg = directWriter.getMCAFile(x, z); + Path fileToMovePath = Paths.get(mcg.toURI()); + Path targetPath = Paths.get(mca.toURI()); + BurstExecutor e = burst.burst(1024); + int mcaox = x << 5; + int mcaoz = z << 5; + if(isMCAWritable(x,z) && !mca.exists()) + { + mcaIteration.accept(mcaox, mcaoz, (ii, jj) -> e.queue(() -> { + draw(ii, jj, COLOR_MCA_GENERATE); + access.directWriteChunk(world, ii, jj, directWriter); + draw(ii, jj, COLOR_MCA_GENERATED); + generated.getAndIncrement(); + })); + e.complete(); + directWriter.flush(); + try { + Files.move(fileToMovePath, targetPath); + } catch (IOException ef) { + ef.printStackTrace(); + } + } + + else + { + mcaDefer.add(new ChunkPosition(x, z)); + e.complete(); + return false; + } + + return true; + } + + private void generateDeferedMCARegion(int x, int z, MultiBurst burst, IrisAccess access, Consumer3> mcaIteration) { + BurstExecutor e = burst.burst(1024); + int mcaox = x << 5; + int mcaoz = z << 5; + + if(PaperLib.isPaper()) + { + mcaIteration.accept(mcaox, mcaoz, (ii, jj) -> e.queue(() -> { + draw(ii, jj, COLOR_MCA_GENERATE_SLOW); + access.generatePaper(world, ii, jj); + draw(ii, jj, COLOR_MCA_GENERATED); + generated.getAndIncrement(); + })); + e.complete(); + } + + else + { + AtomicInteger m = new AtomicInteger(); + mcaIteration.accept(mcaox, mcaoz, (ii, jj) -> { + draw(ii, jj, COLOR_MCA_GENERATE_SLOW); + J.s(() -> { + world.getChunkAt(ii, jj).load(true); + draw(ii, jj, COLOR_MCA_GENERATED); + m.getAndIncrement(); + generated.getAndIncrement(); + }); + }); + + while(m.get() < 1024) + { + J.sleep(25); + } + } + } + + private KList computeChunkOrder() { + ChunkPosition center = new ChunkPosition(15, 15); + KList p = new KList<>(); + new Spiraler(33, 33, (x, z) -> { + int xx = x + 15; + int zz = z + 15; + if(xx < 0 || xx > 31 || zz < 0 || zz > 31) + { + return; + } + + p.add(new ChunkPosition(xx, zz)); + }).drain(); + p.sort(Comparator.comparing((i) -> i.distance(center))); + return p; + } + + public static Pregenerator getInstance() + { + return instance; + } + + public static boolean shutdownInstance(){ + if(instance != null) + { + instance.shutdown(); + instance = null; + return true; + } + + return false; + } + + public static void pauseResume() { + instance.active.set(!instance.active.get()); + } + + public static boolean isPaused() { + return instance.paused(); + } + + private void instance() { + if(instance != null) + { + instance.shutdown(); + } + + instance = this; } public void shutdown() @@ -214,7 +338,7 @@ public class Pregenerator implements Listener } public String[] getProgress() { - return new String[]{"Derp"}; + return new String[]{"Chunks/s: " + perSecond.get(), "Generated: " + Form.f(generated.get())}; } public boolean paused() { diff --git a/src/main/java/com/volmit/iris/scaffold/parallel/BurstExecutor.java b/src/main/java/com/volmit/iris/scaffold/parallel/BurstExecutor.java index 768190ced..da5b05126 100644 --- a/src/main/java/com/volmit/iris/scaffold/parallel/BurstExecutor.java +++ b/src/main/java/com/volmit/iris/scaffold/parallel/BurstExecutor.java @@ -1,11 +1,11 @@ package com.volmit.iris.scaffold.parallel; +import com.volmit.iris.util.KList; + import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; -import com.volmit.iris.util.KList; - public class BurstExecutor { private ExecutorService executor; @@ -45,6 +45,11 @@ public class BurstExecutor { synchronized(futures) { + if(futures.isEmpty()) + { + return; + } + try { CompletableFuture.allOf(futures.toArray(new CompletableFuture[futures.size()])).get(); diff --git a/src/main/java/com/volmit/iris/util/ChunkPosition.java b/src/main/java/com/volmit/iris/util/ChunkPosition.java index a11deb628..45aed4ca7 100644 --- a/src/main/java/com/volmit/iris/util/ChunkPosition.java +++ b/src/main/java/com/volmit/iris/util/ChunkPosition.java @@ -55,4 +55,9 @@ public class ChunkPosition ChunkPosition other = (ChunkPosition) obj; return x == other.x && z == other.z; } + + public double distance(ChunkPosition center) + { + return Math.pow(center.getX() - x, 2) + Math.pow(center.getZ() - z, 2); + } }