The Mantle API

This commit is contained in:
Daniel Mills 2021-08-07 08:15:28 -04:00
parent f258d5f932
commit 6683eee49a
6 changed files with 344 additions and 239 deletions

View File

@ -21,50 +21,241 @@ package com.volmit.iris.util.mantle;
import com.volmit.iris.Iris;
import com.volmit.iris.engine.data.cache.Cache;
import com.volmit.iris.util.collection.KMap;
import com.volmit.iris.util.collection.KSet;
import com.volmit.iris.util.documentation.BlockCoordinates;
import com.volmit.iris.util.documentation.RegionCoordinates;
import com.volmit.iris.util.format.C;
import com.volmit.iris.util.format.Form;
import com.volmit.iris.util.math.M;
import com.volmit.iris.util.parallel.BurstExecutor;
import com.volmit.iris.util.parallel.HyperLock;
import com.volmit.iris.util.parallel.MultiBurst;
import java.io.DataInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.*;
import java.nio.charset.StandardCharsets;
import java.util.Map;
import java.util.UUID;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.atomic.AtomicBoolean;
/**
* The mantle can store any type of data slice anywhere and manage regions & IO on it's own.
* This class is fully thread safe read & write
*/
public class Mantle
{
private final File dataFolder;
private final int worldHeight;
private final Map<Long, MantleRegion> loadedRegions;
private final Map<Long, Long> lastUse;
private final Map<Long, TectonicPlate> loadedRegions;
private final HyperLock hyperLock;
private final KSet<Long> unload;
private final AtomicBoolean closed;
private final MultiBurst ioBurst;
/**
* Create a new mantle
* @param dataFolder the data folder
* @param worldHeight the world's height (in blocks)
*/
@BlockCoordinates
public Mantle(File dataFolder, int worldHeight)
{
this.hyperLock = new HyperLock();
this.closed = new AtomicBoolean(false);
this.dataFolder = dataFolder;
this.worldHeight = worldHeight;
dataFolder.mkdirs();
unload = new KSet<>();
loadedRegions = new KMap<>();
lastUse = new KMap<>();
ioBurst = new MultiBurst("Iris Mantle[" + dataFolder.hashCode() + "]", Thread.MIN_PRIORITY, Runtime.getRuntime().availableProcessors() / 2);
Iris.debug("Opened The Mantle " + C.DARK_AQUA + dataFolder.getAbsolutePath());
}
@RegionCoordinates
public MantleRegion get(int x, int z)
/**
* Set data T at the given block position. This method will attempt to find a
* Tectonic Plate either by loading it or creating a new one. This method uses
* the hyper lock packaged with each Mantle. The hyperlock allows locking of multiple
* threads at a single region while still allowing other threads to continue
* reading & writing other regions. Hyperlocks are slow sync, but in multicore
* environments, they drastically speed up loading & saving large counts of plates
*
* @param x the block's x coordinate
* @param y the block's y coordinate
* @param z the block's z coordinate
* @param t the data to set at the block
* @param <T> the type of data (generic method)
*/
@BlockCoordinates
public <T> void set(int x, int y, int z, T t)
{
Long k = key(x, z);
MantleRegion region = loadedRegions.get(k);
if(region != null)
if(closed.get())
{
return region;
throw new RuntimeException("The Mantle is closed");
}
synchronized (loadedRegions)
MantleMatter matter = null;
try {
matter = get((x >> 4) >> 5, (z >> 4) >> 5).get()
.getOrCreate((x >> 4) & 31, (z >> 4) & 31)
.getOrCreate(y >> 4);
} catch (InterruptedException e) {
Iris.error("Failed to get Tectonic Plate " + ((x >> 4) >> 5) + " " + ((z >> 4) >> 5) + " Due to a thread intterruption");
Iris.reportError(e);
e.printStackTrace();
} catch (ExecutionException e) {
Iris.error("Failed to get Tectonic Plate " + ((x >> 4) >> 5) + " " + ((z >> 4) >> 5) + " Due to a thread execution exception");
Iris.reportError(e);
e.printStackTrace();
}
if(matter == null)
{
// Ensure we are the first loading thread
region = loadedRegions.get(k);
return;
}
matter.slice(matter.getClass(t))
.set(x & 15, y & 15, z & 15, t);
}
/**
* Gets the data tat the current block position This method will attempt to find a
* Tectonic Plate either by loading it or creating a new one. This method uses
* the hyper lock packaged with each Mantle. The hyperlock allows locking of multiple
* threads at a single region while still allowing other threads to continue
* reading & writing other regions. Hyperlocks are slow sync, but in multicore
* environments, they drastically speed up loading & saving large counts of plates
*
* @param x the block's x coordinate
* @param y the block's y coordinate
* @param z the block's z coordinate
* @param t the class representing the type of data being requested
* @param <T> the type assumed from the provided class
* @return the returned result (or null) if it doesnt exist
*/
@SuppressWarnings("unchecked")
@BlockCoordinates
public <T> T get(int x, int y, int z, Class<T> t)
{
if(closed.get())
{
throw new RuntimeException("The Mantle is closed");
}
try {
return (T) get((x >> 4) >> 5, (z >> 4) >> 5).get()
.getOrCreate((x >> 4) & 31, (z >> 4) & 31)
.getOrCreate(y >> 4).slice(t)
.get(x & 15, y & 15, z & 15);
} catch (InterruptedException e) {
Iris.error("Failed to get Tectonic Plate " + ((x >> 4) >> 5) + " " + ((z >> 4) >> 5) + " Due to a thread intterruption");
Iris.reportError(e);
e.printStackTrace();
} catch (ExecutionException e) {
Iris.error("Failed to get Tectonic Plate " + ((x >> 4) >> 5) + " " + ((z >> 4) >> 5) + " Due to a thread execution exception");
Iris.reportError(e);
e.printStackTrace();
}
return null;
}
/**
* Closes the Mantle. By closing the mantle, you can no longer read or write
* any data to the mantle or it's Tectonic Plates. Closing will also flush any
* loaded regions to the disk in parallel.
*/
public synchronized void close()
{
Iris.debug("Closing The Mantle " + C.DARK_AQUA + dataFolder.getAbsolutePath());
if(closed.get())
{
throw new RuntimeException("The Mantle is closed");
}
closed.set(true);
BurstExecutor b = ioBurst.burst(loadedRegions.size());
for(Long i : loadedRegions.keySet())
{
b.queue(() -> {
try {
loadedRegions.get(i).write(fileForRegion(dataFolder, i));
} catch (IOException e) {
e.printStackTrace();
}
});
}
b.complete();
ioBurst.shutdownNow();
Iris.debug("The Mantle has Closed " + C.DARK_AQUA + dataFolder.getAbsolutePath());
}
/**
* Save & unload regions that have not been used for more than the
* specified amount of milliseconds
* @param idleDuration the duration
*/
public synchronized void trim(long idleDuration)
{
if(closed.get())
{
throw new RuntimeException("The Mantle is closed");
}
Iris.debug("Trimming Tectonic Plates older than " + Form.duration((double)idleDuration, 0));
unload.clear();
for(Long i : lastUse.keySet())
{
if(M.ms() - lastUse.get(i) >= idleDuration)
{
unload.add(i);
}
}
for(Long i : unload)
{
TectonicPlate m = loadedRegions.remove(i);
lastUse.remove(i);
Iris.debug("Unloaded Tectonic Plate " + C.DARK_GREEN + i);
if(m != null)
{
ioBurst.lazy(() -> {
try {
m.write(fileForRegion(dataFolder, i));
} catch (IOException e) {
e.printStackTrace();
}
});
}
}
}
/**
* This retreives a future of the Tectonic Plate at the given coordinates.
* All methods accessing tectonic plates should go through this method
* @param x the region x
* @param z the region z
* @return the future of a tectonic plate.
*/
@RegionCoordinates
private CompletableFuture<TectonicPlate> get(int x, int z)
{
return ioBurst.completeValue(() -> hyperLock.withResult(x, z, () -> {
Long k = key(x, z);
lastUse.put(k, M.ms());
TectonicPlate region = loadedRegions.get(k);
if(region != null)
{
return region;
}
File file = fileForRegion(x, z);
File file = fileForRegion(dataFolder, x, z);
if(file.exists())
{
@ -72,35 +263,44 @@ public class Mantle
{
FileInputStream fin = new FileInputStream(file);
DataInputStream din = new DataInputStream(fin);
region = new MantleRegion(worldHeight, din);
region = new TectonicPlate(worldHeight, din);
din.close();
Iris.debug("Loaded Mantle Region " + C.RED + x + " " + z + C.DARK_AQUA + " " + file.getName());
loadedRegions.put(k, region);
Iris.debug("Loaded Tectonic Plate " + C.DARK_GREEN + x + " " + z + C.DARK_AQUA + " " + file.getName());
}
catch(Throwable e)
{
Iris.error("Failed to read Mantle Region " + file.getAbsolutePath() + " creating a new chunk instead.");
Iris.error("Failed to read Tectonic Plate " + file.getAbsolutePath() + " creating a new chunk instead.");
Iris.reportError(e);
e.printStackTrace();
region = null;
region = new TectonicPlate(worldHeight);
loadedRegions.put(k, region);
Iris.debug("Created new Tectonic Plate (Due to Load Failure) " + C.DARK_GREEN + x + " " + z);
}
}
if(region != null)
{
return region;
}
Iris.debug("Created new Mantle Region " + C.RED + x + " " + z);
return new MantleRegion(worldHeight);
}
region = new TectonicPlate(worldHeight);
loadedRegions.put(k, region);
Iris.debug("Created new Tectonic Plate (Due to Load Failure) " + C.DARK_GREEN + x + " " + z);
return region;
}));
}
private File fileForRegion(int x, int z) {
return new File("m." + x + "." + z + ".mtl");
public static File fileForRegion(File folder, int x, int z) {
return fileForRegion(folder, key(x, z));
}
public Long key(int x, int z)
public static File fileForRegion(File folder, Long key) {
String id = UUID.nameUUIDFromBytes(("TectonicPlate:" + key).getBytes(StandardCharsets.UTF_8)).toString();
File f = new File(folder, id.substring(0, 2) + "/" + id.split("\\Q-\\E")[3] + "/" + id + ".ttp");
f.getParentFile().mkdirs();
return f;
}
public static Long key(int x, int z)
{
return Cache.key(x, z);
}

View File

@ -21,6 +21,8 @@ package com.volmit.iris.util.mantle;
import com.volmit.iris.engine.data.chunk.MCATerrainChunk;
import com.volmit.iris.util.data.Varint;
import com.volmit.iris.util.documentation.ChunkCoordinates;
import com.volmit.iris.util.matter.IrisMatter;
import com.volmit.iris.util.matter.Matter;
import com.volmit.iris.util.nbt.mca.Section;
import lombok.Data;
@ -29,15 +31,30 @@ import java.io.DataOutputStream;
import java.io.IOException;
import java.util.concurrent.atomic.AtomicReferenceArray;
/**
* Represents a mantle chunk. Mantle chunks contain sections of matter (see matter api)
* Mantle Chunks are fully atomic & thread safe
*/
public class MantleChunk {
private final AtomicReferenceArray<MantleMatter> sections;
private final AtomicReferenceArray<Matter> sections;
/**
* Create a mantle chunk
* @param sectionHeight the height of the world in sections (blocks >> 4)
*/
@ChunkCoordinates
public MantleChunk(int sectionHeight)
{
sections = new AtomicReferenceArray<>(sectionHeight);
}
/**
* Load a mantle chunk from a data stream
* @param sectionHeight the height of the world in sections (blocks >> 4)
* @param din the data input
* @throws IOException shit happens
* @throws ClassNotFoundException shit happens
*/
public MantleChunk(int sectionHeight, DataInputStream din) throws IOException, ClassNotFoundException {
this(sectionHeight);
int s = Varint.readUnsignedVarInt(din);
@ -46,23 +63,36 @@ public class MantleChunk {
{
if(din.readBoolean())
{
sections.set(i, MantleMatter.read(din));
sections.set(i, Matter.read(din));
}
}
}
/**
* Check if a section exists (same as get(section) != null)
* @param section the section (0 - (worldHeight >> 4))
* @return true if it exists
*/
@ChunkCoordinates
public boolean exists(int section)
{
return get(section) != null;
}
/**
* Get thje matter at the given section or null if it doesnt exist
* @param section the section (0 - (worldHeight >> 4))
* @return the matter or null if it doesnt exist
*/
@ChunkCoordinates
public MantleMatter get(int section)
public Matter get(int section)
{
return sections.get(section);
}
/**
* Clear all matter from this chunk
*/
public void clear()
{
for(int i = 0; i < sections.length(); i++)
@ -71,26 +101,40 @@ public class MantleChunk {
}
}
/**
* Delete the matter from the given section
* @param section the section (0 - (worldHeight >> 4))
*/
@ChunkCoordinates
public void delete(int section)
{
sections.set(section, null);
}
/**
* Get or create a new matter section at the given section
* @param section the section (0 - (worldHeight >> 4))
* @return the matter
*/
@ChunkCoordinates
public MantleMatter getOrCreate(int section)
public Matter getOrCreate(int section)
{
MantleMatter matter = get(section);
Matter matter = get(section);
if(matter == null)
{
matter = new MantleMatter(16, 16, 16);
matter = new IrisMatter(16, 16, 16);
sections.set(section, matter);
}
return matter;
}
/**
* Write this chunk to a data stream
* @param dos the stream
* @throws IOException shit happens
*/
public void write(DataOutputStream dos) throws IOException {
Varint.writeUnsignedVarInt(sections.length(), dos);
@ -99,7 +143,7 @@ public class MantleChunk {
if(exists(i))
{
dos.writeBoolean(true);
MantleMatter matter = get(i);
Matter matter = get(i);
matter.writeDos(dos);
}

View File

@ -1,69 +0,0 @@
/*
* Iris is a World Generator for Minecraft Bukkit Servers
* Copyright (c) 2021 Arcane Arts (Volmit Software)
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.volmit.iris.util.mantle;
import com.volmit.iris.Iris;
import com.volmit.iris.util.collection.KMap;
import com.volmit.iris.util.matter.IrisMatter;
import com.volmit.iris.util.matter.Matter;
import com.volmit.iris.util.matter.MatterSlice;
import com.volmit.iris.util.matter.Sliced;
import java.io.DataInputStream;
import java.io.IOException;
public class MantleMatter extends IrisMatter
{
protected static final KMap<Class<?>, MatterSlice<?>> slicers = buildSlicers();
public MantleMatter(int width, int height, int depth) {
super(width, height, depth);
}
public static MantleMatter read(DataInputStream din) throws IOException, ClassNotFoundException {
return (MantleMatter) Matter.read(din, (b) -> new MantleMatter(b.getX(), b.getY(), b.getZ()));
}
@Override
public <T> MatterSlice<T> createSlice(Class<T> type, Matter m) {
MatterSlice<?> slice = slicers.get(type);
if (slice == null) {
return null;
}
try {
return slice.getClass().getConstructor(int.class, int.class, int.class).newInstance(getWidth(), getHeight(), getDepth());
} catch (Throwable e) {
e.printStackTrace();
}
return null;
}
private static KMap<Class<?>, MatterSlice<?>> buildSlicers() {
KMap<Class<?>, MatterSlice<?>> c = new KMap<>();
for (Object i : Iris.initialize("com.volmit.iris.util.mantle.slices", Sliced.class)) {
MatterSlice<?> s = (MatterSlice<?>) i;
c.put(s.getType(), s);
}
return c;
}
}

View File

@ -18,24 +18,39 @@
package com.volmit.iris.util.mantle;
import com.volmit.iris.Iris;
import com.volmit.iris.util.documentation.ChunkCoordinates;
import com.volmit.iris.util.format.C;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.IOException;
import java.io.*;
import java.util.concurrent.atomic.AtomicReferenceArray;
public class MantleRegion {
/**
* Tectonic Plates are essentially representations of regions in minecraft.
* Tectonic Plates are fully atomic & thread safe
*/
public class TectonicPlate {
private final int sectionHeight;
private final AtomicReferenceArray<MantleChunk> chunks;
public MantleRegion(int worldHeight)
/**
* Create a new tectonic plate
* @param worldHeight the height of the world
*/
public TectonicPlate(int worldHeight)
{
this.sectionHeight = worldHeight >> 4;
this.chunks = new AtomicReferenceArray<>(1024);
}
public MantleRegion(int worldHeight, DataInputStream din) throws IOException, ClassNotFoundException {
/**
* Load a tectonic plate from a data stream
* @param worldHeight the height of the world
* @param din the data input
* @throws IOException shit happens yo
* @throws ClassNotFoundException real shit bro
*/
public TectonicPlate(int worldHeight, DataInputStream din) throws IOException, ClassNotFoundException {
this(worldHeight);
for(int i = 0; i < chunks.length(); i++)
@ -47,18 +62,33 @@ public class MantleRegion {
}
}
/**
* Check if a chunk exists in this plate or not (same as get(x, z) != null)
* @param x the chunk relative x (0-31)
* @param z the chunk relative z (0-31)
* @return true if the chunk exists
*/
@ChunkCoordinates
public boolean exists(int x, int z)
{
return get(x, z) != null;
}
/**
* Get a chunk at the given coordinates or null if it doesnt exist
* @param x the chunk relative x (0-31)
* @param z the chunk relative z (0-31)
* @return the chunk or null if it doesnt exist
*/
@ChunkCoordinates
public MantleChunk get(int x, int z)
{
return chunks.get(index(x, z));
}
/**
* Clear all chunks from this tectonic plate
*/
public void clear()
{
for(int i = 0; i < chunks.length(); i++)
@ -67,12 +97,23 @@ public class MantleRegion {
}
}
/**
* Delete a chunk from this tectonic plate
* @param x the chunk relative x (0-31)
* @param z the chunk relative z (0-31)
*/
@ChunkCoordinates
public void delete(int x, int z)
{
chunks.set(index(x, z), null);
}
/**
* Get a tectonic plate, or create one and insert it & return it if it diddnt exist
* @param x the chunk relative x (0-31)
* @param z the chunk relative z (0-31)
* @return the chunk (read or created & inserted)
*/
@ChunkCoordinates
public MantleChunk getOrCreate(int x, int z)
{
@ -92,6 +133,24 @@ public class MantleRegion {
return (x & 0x1F) + (z & 0x1F) * 32;
}
/**
* Write this tectonic plate to file
* @param file the file to write it to
* @throws IOException shit happens
*/
public void write(File file) throws IOException {
FileOutputStream fos = new FileOutputStream(file);
DataOutputStream dos = new DataOutputStream(fos);
write(dos);
dos.close();
Iris.debug("Saved Tectonic Plate " + C.DARK_GREEN + file.getName().split("\\Q.\\E")[0]);
}
/**
* Write this tectonic plate to a data stream
* @param dos the data output
* @throws IOException shit happens
*/
public void write(DataOutputStream dos) throws IOException {
for(int i = 0; i < chunks.length(); i++)
{

View File

@ -1,59 +0,0 @@
/*
* Iris is a World Generator for Minecraft Bukkit Servers
* Copyright (c) 2021 Arcane Arts (Volmit Software)
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.volmit.iris.util.mantle.slices;
import com.volmit.iris.engine.parallax.ParallaxAccess;
import com.volmit.iris.engine.parallax.ParallaxWorld;
import com.volmit.iris.util.data.B;
import com.volmit.iris.util.matter.Sliced;
import com.volmit.iris.util.matter.slices.RawMatter;
import org.bukkit.World;
import org.bukkit.block.data.BlockData;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.IOException;
@Sliced
public class MantleBlockMatter extends RawMatter<BlockData> {
public MantleBlockMatter() {
this(1, 1, 1);
}
public MantleBlockMatter(int width, int height, int depth) {
super(width, height, depth, BlockData.class);
registerWriter(World.class, ((w, d, x, y, z) -> w.getBlockAt(x, y, z).setBlockData(d)));
registerWriter(ParallaxWorld.class, (w, d, x, y, z) -> w.setBlock(x, y, z, d));
registerReader(World.class, (w, x, y, z) -> {
BlockData d = w.getBlockAt(x, y, z).getBlockData();
return d.getMaterial().isAir() ? null : d;
});
registerReader(ParallaxWorld.class, ParallaxAccess::getBlock);
}
@Override
public void writeNode(BlockData b, DataOutputStream dos) throws IOException {
dos.writeUTF(b.getAsString(true));
}
@Override
public BlockData readNode(DataInputStream din) throws IOException {
return B.get(din.readUTF());
}
}

View File

@ -1,70 +0,0 @@
/*
* Iris is a World Generator for Minecraft Bukkit Servers
* Copyright (c) 2021 Arcane Arts (Volmit Software)
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.volmit.iris.util.mantle.slices;
import com.volmit.iris.util.collection.KMap;
import com.volmit.iris.util.hunk.storage.ArrayHunk;
import com.volmit.iris.util.hunk.storage.AtomicHunk;
import com.volmit.iris.util.hunk.storage.MappedHunk;
import com.volmit.iris.util.matter.MatterReader;
import com.volmit.iris.util.matter.MatterSlice;
import com.volmit.iris.util.matter.MatterWriter;
import lombok.Getter;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.IOException;
public abstract class RawMantleMatter<T> extends AtomicHunk<T> implements MatterSlice<T> {
@Getter
private final Class<T> type;
protected final KMap<Class<?>, MatterWriter<?, T>> writers;
protected final KMap<Class<?>, MatterReader<?, T>> readers;
public RawMantleMatter(int width, int height, int depth, Class<T> type) {
super(width, height, depth);
writers = new KMap<>();
readers = new KMap<>();
this.type = type;
}
protected <W> void registerWriter(Class<W> mediumType, MatterWriter<W, T> injector) {
writers.put(mediumType, injector);
}
protected <W> void registerReader(Class<W> mediumType, MatterReader<W, T> injector) {
readers.put(mediumType, injector);
}
@Override
public <W> MatterWriter<W, T> writeInto(Class<W> mediumType) {
return (MatterWriter<W, T>) writers.get(mediumType);
}
@Override
public <W> MatterReader<W, T> readFrom(Class<W> mediumType) {
return (MatterReader<W, T>) readers.get(mediumType);
}
@Override
public abstract void writeNode(T b, DataOutputStream dos) throws IOException;
@Override
public abstract T readNode(DataInputStream din) throws IOException;
}