diff --git a/src/main/java/com/volmit/iris/core/ConversionManager.java b/src/main/java/com/volmit/iris/core/ConversionManager.java index e2b2c291e..2d7801aa7 100644 --- a/src/main/java/com/volmit/iris/core/ConversionManager.java +++ b/src/main/java/com/volmit/iris/core/ConversionManager.java @@ -20,7 +20,7 @@ package com.volmit.iris.core; import com.google.gson.Gson; import com.volmit.iris.Iris; -import com.volmit.iris.engine.data.DirectWorldWriter; +import com.volmit.iris.engine.data.mca.NBTWorld; import com.volmit.iris.engine.data.nbt.io.NBTUtil; import com.volmit.iris.engine.data.nbt.io.NamedTag; import com.volmit.iris.engine.data.nbt.tag.CompoundTag; @@ -151,7 +151,7 @@ public class ConversionManager { @SuppressWarnings("unchecked") ListTag paletteList = (ListTag) compound.getListTag("palette"); for (int i = 0; i < paletteList.size(); i++) { CompoundTag cp = paletteList.get(i); - palette.add(DirectWorldWriter.getBlockData(cp)); + palette.add(NBTWorld.getBlockData(cp)); } IrisJigsawPiece piece = new IrisJigsawPiece(); IrisObject object = new IrisObject(w, h, d); @@ -171,7 +171,7 @@ public class ConversionManager { CompoundTag finalState = new CompoundTag(); finalState.putString("Name", nbt.getString("final_state")); BlockData jd = bd.clone(); - bd = DirectWorldWriter.getBlockData(finalState); + bd = NBTWorld.getBlockData(finalState); String joint = nbt.getString("joint"); String pool = nbt.getString("pool"); String poolId = toPoolName(pool); diff --git a/src/main/java/com/volmit/iris/core/gui/Pregenerator.java b/src/main/java/com/volmit/iris/core/gui/Pregenerator.java index 73424e082..4aa0b77d8 100644 --- a/src/main/java/com/volmit/iris/core/gui/Pregenerator.java +++ b/src/main/java/com/volmit/iris/core/gui/Pregenerator.java @@ -21,8 +21,8 @@ package com.volmit.iris.core.gui; import com.volmit.iris.Iris; import com.volmit.iris.core.IrisSettings; import com.volmit.iris.engine.IrisWorlds; -import com.volmit.iris.engine.data.DirectWorldWriter; import com.volmit.iris.engine.data.mca.MCAFile; +import com.volmit.iris.engine.data.mca.NBTWorld; import com.volmit.iris.engine.framework.IrisAccess; import com.volmit.iris.engine.parallel.BurstExecutor; import com.volmit.iris.engine.parallel.MultiBurst; @@ -77,7 +77,7 @@ public class Pregenerator implements Listener { private static final Color COLOR_MCA_DEFERRED = Color.decode("#3CB57A"); private final World world; private int lowestBedrock; - private final DirectWorldWriter directWriter; + private final NBTWorld directWriter; private final AtomicBoolean active; private final AtomicBoolean running; private final KList errors; @@ -89,6 +89,7 @@ public class Pregenerator implements Listener { private final AtomicInteger generated; private final AtomicInteger generatedLast; private final RollingSequence perSecond; + private final RollingSequence perMinute; private final AtomicInteger totalChunks; private final AtomicLong memory; private final AtomicReference memoryMetric; @@ -125,16 +126,17 @@ public class Pregenerator implements Listener { vmcaz = new AtomicInteger(); vcax = new AtomicInteger(); vcaz = new AtomicInteger(); + perMinute = new RollingSequence(200); perSecond = new RollingSequence(20); generatedLast = new AtomicInteger(0); totalChunks = new AtomicInteger(0); generated = new AtomicInteger(0); mcaDefer = new KList<>(); access = IrisWorlds.access(world); - this.directWriter = new DirectWorldWriter(world.getWorldFolder()); + this.directWriter = new NBTWorld(world.getWorldFolder()); this.running = new AtomicBoolean(true); this.active = new AtomicBoolean(true); - MultiBurst burst = new MultiBurst("Iris Pregenerator", 9, Runtime.getRuntime().availableProcessors() + 4); + MultiBurst burst = new MultiBurst("Iris Pregenerator", 9, Runtime.getRuntime().availableProcessors()); int mcaSize = (((blockSize >> 4) + 2) >> 5) + 1; onComplete = new KList<>(); max = new ChunkPosition(0, 0); @@ -196,7 +198,7 @@ public class Pregenerator implements Listener { } burst.shutdownNow(); - directWriter.flush(); + directWriter.close(); flushWorld(); onComplete.forEach(Runnable::run); running.set(false); @@ -214,6 +216,7 @@ public class Pregenerator implements Listener { int up = m - w; double dur = p.getMilliseconds(); perSecond.put((int) (up / (dur / 1000D))); + perSecond.put((int) (up / (dur / 60000D))); p.reset(); p.begin(); updateProgress(); @@ -239,8 +242,7 @@ public class Pregenerator implements Listener { return false; } - File mca = new File(world.getWorldFolder(), "region/r." + x + "." + z + ".mca"); - File mcg = directWriter.getMCAFile(x, z); + File mca = directWriter.getRegionFile(x, z); BurstExecutor e = burst.burst(1024); int mcaox = x << 5; int mcaoz = z << 5; @@ -255,9 +257,8 @@ public class Pregenerator implements Listener { vcaz.set(jj); })); e.complete(); - verifyMCA(x, z, burst); - directWriter.flush(); - install(mcg, mca); + //verifyMCA(x, z, burst); + directWriter.save(); } else { totalChunks.getAndAdd(1024); mcaDefer.add(new ChunkPosition(x, z)); @@ -299,27 +300,6 @@ public class Pregenerator implements Listener { } } - private boolean install(File from, File to) { - try { - Files.move(from.toPath(), to.toPath()); - return true; - } catch (Throwable e) { - Iris.reportError(e); - - } - - try { - IO.copyFile(from, to); - from.delete(); - return true; - } catch (IOException e) { - Iris.reportError(e); - - } - - return false; - } - public void updateProgress() { if (!latch.flip()) { return; @@ -335,7 +315,7 @@ public class Pregenerator implements Listener { if (PaperLib.isPaper()) { method.set("PaperAsync (Slow)"); - while (wait.size() > 32) { + while (wait.size() > 16) { J.sleep(5); } @@ -509,7 +489,8 @@ public class Pregenerator implements Listener { return new String[]{ "Progress: " + Form.f(generated.get()) + " of " + Form.f(totalChunks.get()) + " (" + Form.pc((double) generated.get() / (double) totalChunks.get(), 0) + ")", "ETA: " + Form.duration(eta, 0), - "Chunks/s: " + Form.f((int) perSecond.getAverage()), + "Chunks/s: " + Form.f((int) perSecond.getAverage()) + " (" + Form.f((int)perSecond.getMax()) + " Peak)", + "Chunks/min: " + Form.f((int) perMinute.getAverage())+ " (" + Form.f((int)perMinute.getMax()) + " Peak)", "Memory: " + memoryMetric.get(), "Cursor: " + "MCA(" + vmcax.get() + ", " + vmcaz.get() + ") @ (" + vcax.get() + ", " + vcaz.get() + ")", "Gen Mode: " + method.get(), diff --git a/src/main/java/com/volmit/iris/engine/data/mca/MCAFile.java b/src/main/java/com/volmit/iris/engine/data/mca/MCAFile.java index be77a3bce..86b04fe1b 100644 --- a/src/main/java/com/volmit/iris/engine/data/mca/MCAFile.java +++ b/src/main/java/com/volmit/iris/engine/data/mca/MCAFile.java @@ -22,6 +22,7 @@ import com.volmit.iris.engine.data.nbt.tag.CompoundTag; import java.io.IOException; import java.io.RandomAccessFile; +import java.util.concurrent.atomic.AtomicReferenceArray; @SuppressWarnings("ALL") public class MCAFile { @@ -33,7 +34,7 @@ public class MCAFile { private final int regionX; private final int regionZ; - private Chunk[] chunks; + private AtomicReferenceArray chunks; /** * MCAFile represents a world save file used by Minecraft to store world @@ -69,7 +70,7 @@ public class MCAFile { * @throws IOException If something went wrong during deserialization. */ public void deserialize(RandomAccessFile raf, long loadFlags) throws IOException { - chunks = new Chunk[1024]; + chunks = new AtomicReferenceArray<>(1024); for (int i = 0; i < 1024; i++) { raf.seek(i * 4); int offset = raf.read() << 16; @@ -83,11 +84,11 @@ public class MCAFile { Chunk chunk = new Chunk(timestamp); raf.seek(4096L * offset + 4); //+4: skip data size chunk.deserialize(raf, loadFlags); - chunks[i] = chunk; + chunks.set(i, chunk); } } - public Chunk[] getChunks() { + public AtomicReferenceArray getChunks() { return chunks; } @@ -128,7 +129,7 @@ public class MCAFile { for (int cx = 0; cx < 32; cx++) { for (int cz = 0; cz < 32; cz++) { int index = getChunkIndex(cx, cz); - Chunk chunk = chunks[index]; + Chunk chunk = chunks.get(index); if (chunk == null) { continue; } @@ -175,9 +176,9 @@ public class MCAFile { public void setChunk(int index, Chunk chunk) { checkIndex(index); if (chunks == null) { - chunks = new Chunk[1024]; + chunks = new AtomicReferenceArray<>(1024); } - chunks[index] = chunk; + chunks.set(index, chunk); } /** @@ -203,7 +204,7 @@ public class MCAFile { if (chunks == null) { return null; } - return chunks[index]; + return chunks.get(index); } /** @@ -217,6 +218,11 @@ public class MCAFile { return getChunk(getChunkIndex(chunkX, chunkZ)); } + public boolean hasChunk(int chunkX, int chunkZ) + { + return getChunk(chunkX, chunkZ) != null; + } + /** * Calculates the index of a chunk from its x- and z-coordinates in this region. * This works with absolute and relative coordinates. @@ -319,15 +325,4 @@ public class MCAFile { } return chunk.getBlockStateAt(blockX, blockY, blockZ); } - - /** - * Recalculates the Palette and the BlockStates of all chunks and sections of this region. - */ - public void cleanupPalettesAndBlockStates() { - for (Chunk chunk : chunks) { - if (chunk != null) { - chunk.cleanupPalettesAndBlockStates(); - } - } - } } diff --git a/src/main/java/com/volmit/iris/engine/data/DirectWorldWriter.java b/src/main/java/com/volmit/iris/engine/data/mca/NBTWorld.java similarity index 53% rename from src/main/java/com/volmit/iris/engine/data/DirectWorldWriter.java rename to src/main/java/com/volmit/iris/engine/data/mca/NBTWorld.java index 9458ecf16..2f910e469 100644 --- a/src/main/java/com/volmit/iris/engine/data/DirectWorldWriter.java +++ b/src/main/java/com/volmit/iris/engine/data/mca/NBTWorld.java @@ -16,75 +16,160 @@ * along with this program. If not, see . */ -package com.volmit.iris.engine.data; +package com.volmit.iris.engine.data.mca; import com.volmit.iris.Iris; import com.volmit.iris.core.nms.INMS; import com.volmit.iris.engine.cache.Cache; -import com.volmit.iris.engine.data.mca.Chunk; -import com.volmit.iris.engine.data.mca.MCAFile; -import com.volmit.iris.engine.data.mca.MCAUtil; -import com.volmit.iris.engine.data.mca.Section; +import com.volmit.iris.engine.data.B; import com.volmit.iris.engine.data.nbt.tag.CompoundTag; import com.volmit.iris.engine.data.nbt.tag.StringTag; -import com.volmit.iris.engine.parallel.BurstExecutor; -import com.volmit.iris.engine.parallel.MultiBurst; import com.volmit.iris.util.collection.KList; import com.volmit.iris.util.collection.KMap; +import com.volmit.iris.util.collection.KSet; +import com.volmit.iris.util.format.C; +import com.volmit.iris.util.math.M; +import com.volmit.iris.util.scheduling.IrisLock; import org.bukkit.NamespacedKey; import org.bukkit.block.Biome; import org.bukkit.block.data.BlockData; +import org.jetbrains.annotations.NotNull; import java.io.File; import java.io.IOException; import java.util.Map; +import java.util.concurrent.*; -@SuppressWarnings("EmptyMethod") -public class DirectWorldWriter { - private final File worldFolder; - private final Map writeBuffer; +public class NBTWorld { + private static final BlockData AIR = B.get("AIR"); private static final Map blockDataCache = new KMap<>(); private static final Map biomeIds = computeBiomeIDs(); + private final IrisLock regionLock = new IrisLock("Region"); + private final KMap loadedRegions; + private final KMap lastUse; + private final File worldFolder; + private final ExecutorService saveQueue; - public DirectWorldWriter(File worldFolder) { + public NBTWorld(File worldFolder) + { this.worldFolder = worldFolder; - writeBuffer = new KMap<>(); - new File(worldFolder, "iris/mca-region").mkdirs(); + this.loadedRegions = new KMap<>(); + this.lastUse = new KMap<>(); + saveQueue = Executors.newSingleThreadExecutor(r -> { + Thread t = new Thread(r); + t.setName("Iris MCA Writer"); + t.setPriority(Thread.MIN_PRIORITY); + return t; + }); } - public void flush() { - BurstExecutor ex2 = MultiBurst.burst.burst(writeBuffer.size()); + public void close() + { + regionLock.lock(); - for (Long i : new KList<>(writeBuffer.keySet())) { - ex2.queue(() -> { - int x = Cache.keyX(i); - int z = Cache.keyZ(i); - try { - File f = getMCAFile(x, z); - - if (!f.exists()) { - f.getParentFile().mkdirs(); - f.createNewFile(); - } - - MCAUtil.write(writeBuffer.get(i), f, true); - writeBuffer.remove(i); - } catch (Throwable e) { - Iris.reportError(e); - e.printStackTrace(); - } - }); + for(Long i : loadedRegions.k()) + { + queueSaveUnload(Cache.keyX(i), Cache.keyZ(i)); } - ex2.complete(); + regionLock.unlock(); + saveQueue.shutdown(); + try { + while(!saveQueue.awaitTermination(3, TimeUnit.SECONDS)) + { + Iris.info("Still Waiting to save MCA Files..."); + } + } catch (InterruptedException e) { + e.printStackTrace(); + } } - public void optimizeChunk(int x, int z) { - getChunk(x, z).cleanupPalettesAndBlockStates(); + public void queueSaveUnload(int x, int z) + { + saveQueue.submit(() -> { + MCAFile f = getMCAOrNull(x, z); + if(f != null) + { + unloadRegion(x, z); + } + + saveRegion(x, z, f); + }); } - public File getMCAFile(int x, int z) { - return new File(worldFolder, "iris/mca-region/r." + x + "." + z + ".mca"); + public void save() + { + regionLock.lock(); + + boolean saving = true; + + for(Long i : loadedRegions.k()) + { + int x = Cache.keyX(i); + int z = Cache.keyZ(i); + + if(!lastUse.containsKey(i)) + { + lastUse.put(i, M.ms()); + } + + if(shouldUnload(x, z)) + { + queueSaveUnload(x, z); + } + } + + Iris.debug("Regions: " + C.GOLD + loadedRegions.size() + C.LIGHT_PURPLE); + + regionLock.unlock(); + } + + public void queueSave() + { + + } + + public synchronized void unloadRegion(int x, int z) + { + long key = Cache.key(x, z); + regionLock.lock(); + loadedRegions.remove(key); + lastUse.remove(key); + regionLock.unlock(); + Iris.debug("Unloaded Region " + C.GOLD + x + " " + z); + } + + public void saveRegion(int x, int z) + { + long k = Cache.key(x, z); + MCAFile mca = getMCAOrNull(x, z); + try { + MCAUtil.write(mca, getRegionFile(x, z), true); + Iris.debug("Saved Region " + C.GOLD + x + " " + z); + } catch (IOException e) { + Iris.error("Failed to save region " + getRegionFile(x, z).getPath()); + e.printStackTrace(); + } + } + + public void saveRegion(int x, int z, MCAFile mca) + { + try { + MCAUtil.write(mca, getRegionFile(x, z), true); + Iris.debug("Saved Region " + C.GOLD + x + " " + z); + } catch (IOException e) { + Iris.error("Failed to save region " + getRegionFile(x, z).getPath()); + e.printStackTrace(); + } + } + + public boolean shouldUnload(int x, int z) + { + return getIdleDuration(x, z) > 60000; + } + + public File getRegionFile(int x, int z) { + return new File(worldFolder, "region/r." + x + "." + z + ".mca"); } public static BlockData getBlockData(CompoundTag tag) { @@ -125,7 +210,6 @@ public class DirectWorldWriter { NamespacedKey key = blockData.getMaterial().getKey(); s.putString("Name", key.getNamespace() + ":" + key.getKey()); - if (data.contains("[")) { String raw = data.split("\\Q[\\E")[1].replaceAll("\\Q]\\E", ""); CompoundTag props = new CompoundTag(); @@ -154,7 +238,7 @@ public class DirectWorldWriter { CompoundTag tag = getChunkSection(x >> 4, y >> 4, z >> 4).getBlockStateAt(x & 15, y & 15, z & 15); if (tag == null) { - return B.get("AIR"); + return AIR; } return getBlockData(tag); @@ -162,7 +246,7 @@ public class DirectWorldWriter { Iris.reportError(e); } - return B.get("AIR"); + return AIR; } public void setBlockData(int x, int y, int z, BlockData data) { @@ -185,10 +269,6 @@ public class DirectWorldWriter { return s; } - public void deleteChunk(int x, int z) { - - } - public Chunk getChunk(int x, int z) { MCAFile mca = getMCA(x >> 5, z >> 5); Chunk c = mca.getChunk(x & 31, z & 31); @@ -201,28 +281,57 @@ public class DirectWorldWriter { return c; } + public long getIdleDuration(int x, int z) + { + Long l = lastUse.get(Cache.key(x, z)); + + return l == null ? 0 : (M.ms() - l); + } + public MCAFile getMCA(int x, int z) { long key = Cache.key(x, z); - MCAFile mca = writeBuffer.get(key); - if (mca != null) { - return mca; + regionLock.lock(); + lastUse.put(key, M.ms()); + MCAFile mcaf = loadedRegions.get(key); + regionLock.unlock(); + + if(mcaf == null) + { + File f = getRegionFile(x, z); + try { + mcaf = f.exists() ? MCAUtil.read(f) : new MCAFile(x, z); + } catch (IOException e) { + Iris.error("Failed to properly read MCA File " + f.getPath() + " Using a blank one."); + e.printStackTrace(); + mcaf = new MCAFile(x, z); + } + + regionLock.lock(); + loadedRegions.put(key, mcaf); + regionLock.unlock(); } - File f = getMCAFile(x, z); - try { - mca = f.exists() ? MCAUtil.read(f) : new MCAFile(x, z); - } catch (IOException e) { - e.printStackTrace(); - mca = new MCAFile(x, z); + return mcaf; + } + + public MCAFile getMCAOrNull(int x, int z) { + long key = Cache.key(x, z); + MCAFile ff = null; + regionLock.lock(); + + if(loadedRegions.containsKey(key)) + { + lastUse.put(key, M.ms()); + ff = loadedRegions.get(key); } - writeBuffer.put(key, mca); - return mca; + regionLock.unlock(); + return ff; } public int size() { - return writeBuffer.size(); + return loadedRegions.size(); } private static Map computeBiomeIDs() { @@ -236,18 +345,4 @@ public class DirectWorldWriter { return biomeIds; } - - public void verify(int mcaox, int mcaoz) { - MCAFile file = getMCA(mcaox, mcaoz); - - for (int i = 0; i < 32; i++) { - for (int j = 0; j < 32; j++) { - Chunk c = file.getChunk(i, j); - - if (c == null) { - Iris.warn("Chunk " + ((mcaox << 5) + i) + ", " + ((mcaoz << 5) + j) + " is null in MCA File " + mcaox + ", " + mcaoz); - } - } - } - } }