NBTWorlds!

This commit is contained in:
Daniel Mills 2021-07-18 04:46:33 -04:00
parent c2e1ecf5b1
commit c420a8a686
4 changed files with 199 additions and 128 deletions

View File

@ -20,7 +20,7 @@ package com.volmit.iris.core;
import com.google.gson.Gson;
import com.volmit.iris.Iris;
import com.volmit.iris.engine.data.DirectWorldWriter;
import com.volmit.iris.engine.data.mca.NBTWorld;
import com.volmit.iris.engine.data.nbt.io.NBTUtil;
import com.volmit.iris.engine.data.nbt.io.NamedTag;
import com.volmit.iris.engine.data.nbt.tag.CompoundTag;
@ -151,7 +151,7 @@ public class ConversionManager {
@SuppressWarnings("unchecked") ListTag<CompoundTag> paletteList = (ListTag<CompoundTag>) compound.getListTag("palette");
for (int i = 0; i < paletteList.size(); i++) {
CompoundTag cp = paletteList.get(i);
palette.add(DirectWorldWriter.getBlockData(cp));
palette.add(NBTWorld.getBlockData(cp));
}
IrisJigsawPiece piece = new IrisJigsawPiece();
IrisObject object = new IrisObject(w, h, d);
@ -171,7 +171,7 @@ public class ConversionManager {
CompoundTag finalState = new CompoundTag();
finalState.putString("Name", nbt.getString("final_state"));
BlockData jd = bd.clone();
bd = DirectWorldWriter.getBlockData(finalState);
bd = NBTWorld.getBlockData(finalState);
String joint = nbt.getString("joint");
String pool = nbt.getString("pool");
String poolId = toPoolName(pool);

View File

@ -21,8 +21,8 @@ package com.volmit.iris.core.gui;
import com.volmit.iris.Iris;
import com.volmit.iris.core.IrisSettings;
import com.volmit.iris.engine.IrisWorlds;
import com.volmit.iris.engine.data.DirectWorldWriter;
import com.volmit.iris.engine.data.mca.MCAFile;
import com.volmit.iris.engine.data.mca.NBTWorld;
import com.volmit.iris.engine.framework.IrisAccess;
import com.volmit.iris.engine.parallel.BurstExecutor;
import com.volmit.iris.engine.parallel.MultiBurst;
@ -77,7 +77,7 @@ public class Pregenerator implements Listener {
private static final Color COLOR_MCA_DEFERRED = Color.decode("#3CB57A");
private final World world;
private int lowestBedrock;
private final DirectWorldWriter directWriter;
private final NBTWorld directWriter;
private final AtomicBoolean active;
private final AtomicBoolean running;
private final KList<ChunkPosition> errors;
@ -89,6 +89,7 @@ public class Pregenerator implements Listener {
private final AtomicInteger generated;
private final AtomicInteger generatedLast;
private final RollingSequence perSecond;
private final RollingSequence perMinute;
private final AtomicInteger totalChunks;
private final AtomicLong memory;
private final AtomicReference<String> memoryMetric;
@ -125,16 +126,17 @@ public class Pregenerator implements Listener {
vmcaz = new AtomicInteger();
vcax = new AtomicInteger();
vcaz = new AtomicInteger();
perMinute = new RollingSequence(200);
perSecond = new RollingSequence(20);
generatedLast = new AtomicInteger(0);
totalChunks = new AtomicInteger(0);
generated = new AtomicInteger(0);
mcaDefer = new KList<>();
access = IrisWorlds.access(world);
this.directWriter = new DirectWorldWriter(world.getWorldFolder());
this.directWriter = new NBTWorld(world.getWorldFolder());
this.running = new AtomicBoolean(true);
this.active = new AtomicBoolean(true);
MultiBurst burst = new MultiBurst("Iris Pregenerator", 9, Runtime.getRuntime().availableProcessors() + 4);
MultiBurst burst = new MultiBurst("Iris Pregenerator", 9, Runtime.getRuntime().availableProcessors());
int mcaSize = (((blockSize >> 4) + 2) >> 5) + 1;
onComplete = new KList<>();
max = new ChunkPosition(0, 0);
@ -196,7 +198,7 @@ public class Pregenerator implements Listener {
}
burst.shutdownNow();
directWriter.flush();
directWriter.close();
flushWorld();
onComplete.forEach(Runnable::run);
running.set(false);
@ -214,6 +216,7 @@ public class Pregenerator implements Listener {
int up = m - w;
double dur = p.getMilliseconds();
perSecond.put((int) (up / (dur / 1000D)));
perSecond.put((int) (up / (dur / 60000D)));
p.reset();
p.begin();
updateProgress();
@ -239,8 +242,7 @@ public class Pregenerator implements Listener {
return false;
}
File mca = new File(world.getWorldFolder(), "region/r." + x + "." + z + ".mca");
File mcg = directWriter.getMCAFile(x, z);
File mca = directWriter.getRegionFile(x, z);
BurstExecutor e = burst.burst(1024);
int mcaox = x << 5;
int mcaoz = z << 5;
@ -255,9 +257,8 @@ public class Pregenerator implements Listener {
vcaz.set(jj);
}));
e.complete();
verifyMCA(x, z, burst);
directWriter.flush();
install(mcg, mca);
//verifyMCA(x, z, burst);
directWriter.save();
} else {
totalChunks.getAndAdd(1024);
mcaDefer.add(new ChunkPosition(x, z));
@ -299,27 +300,6 @@ public class Pregenerator implements Listener {
}
}
private boolean install(File from, File to) {
try {
Files.move(from.toPath(), to.toPath());
return true;
} catch (Throwable e) {
Iris.reportError(e);
}
try {
IO.copyFile(from, to);
from.delete();
return true;
} catch (IOException e) {
Iris.reportError(e);
}
return false;
}
public void updateProgress() {
if (!latch.flip()) {
return;
@ -335,7 +315,7 @@ public class Pregenerator implements Listener {
if (PaperLib.isPaper()) {
method.set("PaperAsync (Slow)");
while (wait.size() > 32) {
while (wait.size() > 16) {
J.sleep(5);
}
@ -509,7 +489,8 @@ public class Pregenerator implements Listener {
return new String[]{
"Progress: " + Form.f(generated.get()) + " of " + Form.f(totalChunks.get()) + " (" + Form.pc((double) generated.get() / (double) totalChunks.get(), 0) + ")",
"ETA: " + Form.duration(eta, 0),
"Chunks/s: " + Form.f((int) perSecond.getAverage()),
"Chunks/s: " + Form.f((int) perSecond.getAverage()) + " (" + Form.f((int)perSecond.getMax()) + " Peak)",
"Chunks/min: " + Form.f((int) perMinute.getAverage())+ " (" + Form.f((int)perMinute.getMax()) + " Peak)",
"Memory: " + memoryMetric.get(),
"Cursor: " + "MCA(" + vmcax.get() + ", " + vmcaz.get() + ") @ (" + vcax.get() + ", " + vcaz.get() + ")",
"Gen Mode: " + method.get(),

View File

@ -22,6 +22,7 @@ import com.volmit.iris.engine.data.nbt.tag.CompoundTag;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.util.concurrent.atomic.AtomicReferenceArray;
@SuppressWarnings("ALL")
public class MCAFile {
@ -33,7 +34,7 @@ public class MCAFile {
private final int regionX;
private final int regionZ;
private Chunk[] chunks;
private AtomicReferenceArray<Chunk> chunks;
/**
* MCAFile represents a world save file used by Minecraft to store world
@ -69,7 +70,7 @@ public class MCAFile {
* @throws IOException If something went wrong during deserialization.
*/
public void deserialize(RandomAccessFile raf, long loadFlags) throws IOException {
chunks = new Chunk[1024];
chunks = new AtomicReferenceArray<>(1024);
for (int i = 0; i < 1024; i++) {
raf.seek(i * 4);
int offset = raf.read() << 16;
@ -83,11 +84,11 @@ public class MCAFile {
Chunk chunk = new Chunk(timestamp);
raf.seek(4096L * offset + 4); //+4: skip data size
chunk.deserialize(raf, loadFlags);
chunks[i] = chunk;
chunks.set(i, chunk);
}
}
public Chunk[] getChunks() {
public AtomicReferenceArray<Chunk> getChunks() {
return chunks;
}
@ -128,7 +129,7 @@ public class MCAFile {
for (int cx = 0; cx < 32; cx++) {
for (int cz = 0; cz < 32; cz++) {
int index = getChunkIndex(cx, cz);
Chunk chunk = chunks[index];
Chunk chunk = chunks.get(index);
if (chunk == null) {
continue;
}
@ -175,9 +176,9 @@ public class MCAFile {
public void setChunk(int index, Chunk chunk) {
checkIndex(index);
if (chunks == null) {
chunks = new Chunk[1024];
chunks = new AtomicReferenceArray<>(1024);
}
chunks[index] = chunk;
chunks.set(index, chunk);
}
/**
@ -203,7 +204,7 @@ public class MCAFile {
if (chunks == null) {
return null;
}
return chunks[index];
return chunks.get(index);
}
/**
@ -217,6 +218,11 @@ public class MCAFile {
return getChunk(getChunkIndex(chunkX, chunkZ));
}
public boolean hasChunk(int chunkX, int chunkZ)
{
return getChunk(chunkX, chunkZ) != null;
}
/**
* Calculates the index of a chunk from its x- and z-coordinates in this region.
* This works with absolute and relative coordinates.
@ -319,15 +325,4 @@ public class MCAFile {
}
return chunk.getBlockStateAt(blockX, blockY, blockZ);
}
/**
* Recalculates the Palette and the BlockStates of all chunks and sections of this region.
*/
public void cleanupPalettesAndBlockStates() {
for (Chunk chunk : chunks) {
if (chunk != null) {
chunk.cleanupPalettesAndBlockStates();
}
}
}
}

View File

@ -16,75 +16,160 @@
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.volmit.iris.engine.data;
package com.volmit.iris.engine.data.mca;
import com.volmit.iris.Iris;
import com.volmit.iris.core.nms.INMS;
import com.volmit.iris.engine.cache.Cache;
import com.volmit.iris.engine.data.mca.Chunk;
import com.volmit.iris.engine.data.mca.MCAFile;
import com.volmit.iris.engine.data.mca.MCAUtil;
import com.volmit.iris.engine.data.mca.Section;
import com.volmit.iris.engine.data.B;
import com.volmit.iris.engine.data.nbt.tag.CompoundTag;
import com.volmit.iris.engine.data.nbt.tag.StringTag;
import com.volmit.iris.engine.parallel.BurstExecutor;
import com.volmit.iris.engine.parallel.MultiBurst;
import com.volmit.iris.util.collection.KList;
import com.volmit.iris.util.collection.KMap;
import com.volmit.iris.util.collection.KSet;
import com.volmit.iris.util.format.C;
import com.volmit.iris.util.math.M;
import com.volmit.iris.util.scheduling.IrisLock;
import org.bukkit.NamespacedKey;
import org.bukkit.block.Biome;
import org.bukkit.block.data.BlockData;
import org.jetbrains.annotations.NotNull;
import java.io.File;
import java.io.IOException;
import java.util.Map;
import java.util.concurrent.*;
@SuppressWarnings("EmptyMethod")
public class DirectWorldWriter {
private final File worldFolder;
private final Map<Long, MCAFile> writeBuffer;
public class NBTWorld {
private static final BlockData AIR = B.get("AIR");
private static final Map<String, CompoundTag> blockDataCache = new KMap<>();
private static final Map<Biome, Integer> biomeIds = computeBiomeIDs();
private final IrisLock regionLock = new IrisLock("Region");
private final KMap<Long, MCAFile> loadedRegions;
private final KMap<Long, Long> lastUse;
private final File worldFolder;
private final ExecutorService saveQueue;
public DirectWorldWriter(File worldFolder) {
public NBTWorld(File worldFolder)
{
this.worldFolder = worldFolder;
writeBuffer = new KMap<>();
new File(worldFolder, "iris/mca-region").mkdirs();
this.loadedRegions = new KMap<>();
this.lastUse = new KMap<>();
saveQueue = Executors.newSingleThreadExecutor(r -> {
Thread t = new Thread(r);
t.setName("Iris MCA Writer");
t.setPriority(Thread.MIN_PRIORITY);
return t;
});
}
public void flush() {
BurstExecutor ex2 = MultiBurst.burst.burst(writeBuffer.size());
public void close()
{
regionLock.lock();
for (Long i : new KList<>(writeBuffer.keySet())) {
ex2.queue(() -> {
int x = Cache.keyX(i);
int z = Cache.keyZ(i);
try {
File f = getMCAFile(x, z);
if (!f.exists()) {
f.getParentFile().mkdirs();
f.createNewFile();
}
MCAUtil.write(writeBuffer.get(i), f, true);
writeBuffer.remove(i);
} catch (Throwable e) {
Iris.reportError(e);
e.printStackTrace();
}
});
for(Long i : loadedRegions.k())
{
queueSaveUnload(Cache.keyX(i), Cache.keyZ(i));
}
ex2.complete();
regionLock.unlock();
saveQueue.shutdown();
try {
while(!saveQueue.awaitTermination(3, TimeUnit.SECONDS))
{
Iris.info("Still Waiting to save MCA Files...");
}
} catch (InterruptedException e) {
e.printStackTrace();
}
}
public void optimizeChunk(int x, int z) {
getChunk(x, z).cleanupPalettesAndBlockStates();
public void queueSaveUnload(int x, int z)
{
saveQueue.submit(() -> {
MCAFile f = getMCAOrNull(x, z);
if(f != null)
{
unloadRegion(x, z);
}
saveRegion(x, z, f);
});
}
public File getMCAFile(int x, int z) {
return new File(worldFolder, "iris/mca-region/r." + x + "." + z + ".mca");
public void save()
{
regionLock.lock();
boolean saving = true;
for(Long i : loadedRegions.k())
{
int x = Cache.keyX(i);
int z = Cache.keyZ(i);
if(!lastUse.containsKey(i))
{
lastUse.put(i, M.ms());
}
if(shouldUnload(x, z))
{
queueSaveUnload(x, z);
}
}
Iris.debug("Regions: " + C.GOLD + loadedRegions.size() + C.LIGHT_PURPLE);
regionLock.unlock();
}
public void queueSave()
{
}
public synchronized void unloadRegion(int x, int z)
{
long key = Cache.key(x, z);
regionLock.lock();
loadedRegions.remove(key);
lastUse.remove(key);
regionLock.unlock();
Iris.debug("Unloaded Region " + C.GOLD + x + " " + z);
}
public void saveRegion(int x, int z)
{
long k = Cache.key(x, z);
MCAFile mca = getMCAOrNull(x, z);
try {
MCAUtil.write(mca, getRegionFile(x, z), true);
Iris.debug("Saved Region " + C.GOLD + x + " " + z);
} catch (IOException e) {
Iris.error("Failed to save region " + getRegionFile(x, z).getPath());
e.printStackTrace();
}
}
public void saveRegion(int x, int z, MCAFile mca)
{
try {
MCAUtil.write(mca, getRegionFile(x, z), true);
Iris.debug("Saved Region " + C.GOLD + x + " " + z);
} catch (IOException e) {
Iris.error("Failed to save region " + getRegionFile(x, z).getPath());
e.printStackTrace();
}
}
public boolean shouldUnload(int x, int z)
{
return getIdleDuration(x, z) > 60000;
}
public File getRegionFile(int x, int z) {
return new File(worldFolder, "region/r." + x + "." + z + ".mca");
}
public static BlockData getBlockData(CompoundTag tag) {
@ -125,7 +210,6 @@ public class DirectWorldWriter {
NamespacedKey key = blockData.getMaterial().getKey();
s.putString("Name", key.getNamespace() + ":" + key.getKey());
if (data.contains("[")) {
String raw = data.split("\\Q[\\E")[1].replaceAll("\\Q]\\E", "");
CompoundTag props = new CompoundTag();
@ -154,7 +238,7 @@ public class DirectWorldWriter {
CompoundTag tag = getChunkSection(x >> 4, y >> 4, z >> 4).getBlockStateAt(x & 15, y & 15, z & 15);
if (tag == null) {
return B.get("AIR");
return AIR;
}
return getBlockData(tag);
@ -162,7 +246,7 @@ public class DirectWorldWriter {
Iris.reportError(e);
}
return B.get("AIR");
return AIR;
}
public void setBlockData(int x, int y, int z, BlockData data) {
@ -185,10 +269,6 @@ public class DirectWorldWriter {
return s;
}
public void deleteChunk(int x, int z) {
}
public Chunk getChunk(int x, int z) {
MCAFile mca = getMCA(x >> 5, z >> 5);
Chunk c = mca.getChunk(x & 31, z & 31);
@ -201,28 +281,57 @@ public class DirectWorldWriter {
return c;
}
public long getIdleDuration(int x, int z)
{
Long l = lastUse.get(Cache.key(x, z));
return l == null ? 0 : (M.ms() - l);
}
public MCAFile getMCA(int x, int z) {
long key = Cache.key(x, z);
MCAFile mca = writeBuffer.get(key);
if (mca != null) {
return mca;
regionLock.lock();
lastUse.put(key, M.ms());
MCAFile mcaf = loadedRegions.get(key);
regionLock.unlock();
if(mcaf == null)
{
File f = getRegionFile(x, z);
try {
mcaf = f.exists() ? MCAUtil.read(f) : new MCAFile(x, z);
} catch (IOException e) {
Iris.error("Failed to properly read MCA File " + f.getPath() + " Using a blank one.");
e.printStackTrace();
mcaf = new MCAFile(x, z);
}
regionLock.lock();
loadedRegions.put(key, mcaf);
regionLock.unlock();
}
File f = getMCAFile(x, z);
try {
mca = f.exists() ? MCAUtil.read(f) : new MCAFile(x, z);
} catch (IOException e) {
e.printStackTrace();
mca = new MCAFile(x, z);
return mcaf;
}
public MCAFile getMCAOrNull(int x, int z) {
long key = Cache.key(x, z);
MCAFile ff = null;
regionLock.lock();
if(loadedRegions.containsKey(key))
{
lastUse.put(key, M.ms());
ff = loadedRegions.get(key);
}
writeBuffer.put(key, mca);
return mca;
regionLock.unlock();
return ff;
}
public int size() {
return writeBuffer.size();
return loadedRegions.size();
}
private static Map<Biome, Integer> computeBiomeIDs() {
@ -236,18 +345,4 @@ public class DirectWorldWriter {
return biomeIds;
}
public void verify(int mcaox, int mcaoz) {
MCAFile file = getMCA(mcaox, mcaoz);
for (int i = 0; i < 32; i++) {
for (int j = 0; j < 32; j++) {
Chunk c = file.getChunk(i, j);
if (c == null) {
Iris.warn("Chunk " + ((mcaox << 5) + i) + ", " + ((mcaoz << 5) + j) + " is null in MCA File " + mcaox + ", " + mcaoz);
}
}
}
}
}