mirrors/Iris
1
0
Fork 0
mirror of https://github.com/VolmitSoftware/Iris.git synced 2025-07-02 16:07:06 +00:00
Code Issues Packages Projects Releases Wiki Activity

add more safety to mantle

Browse Source
This commit is contained in:
Julian Krings 2024-11-27 19:45:08 +01:00
parent cacef8c8fc
commit 1dca502a90
7 changed files with 228 additions and 59 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
core/src/main/java/com/volmit/iris/core/commands/CommandDeveloper.java
View File

@ -42,6 +42,7 @@ import com.volmit.iris.util.decree.annotations.Decree;
import com.volmit.iris.util.decree.annotations.Param;
import com.volmit.iris.util.format.C;
import com.volmit.iris.util.format.Form;
import com.volmit.iris.util.io.CountingDataInputStream;
import com.volmit.iris.util.io.IO;
import com.volmit.iris.util.mantle.TectonicPlate;
import com.volmit.iris.util.math.Spiraler;
@ -262,7 +263,7 @@ public class CommandDeveloper implements DecreeExecutor {
VolmitSender sender = sender();
service.submit(() -> {
try {
DataInputStream raw = new DataInputStream(new FileInputStream(file));
CountingDataInputStream raw = CountingDataInputStream.wrap(new FileInputStream(file));
TectonicPlate plate = new TectonicPlate(height, raw);
raw.close();
@ -281,7 +282,7 @@ public class CommandDeveloper implements DecreeExecutor {
if (size == 0)
size = tmp.length();
start = System.currentTimeMillis();
DataInputStream din = createInput(tmp, algorithm);
CountingDataInputStream din = createInput(tmp, algorithm);
new TectonicPlate(height, din);
din.close();
d2 += System.currentTimeMillis() - start;
@ -301,10 +302,10 @@ public class CommandDeveloper implements DecreeExecutor {
}
}
private DataInputStream createInput(File file, String algorithm) throws Throwable {
private CountingDataInputStream createInput(File file, String algorithm) throws Throwable {
FileInputStream in = new FileInputStream(file);
return new DataInputStream(switch (algorithm) {
return CountingDataInputStream.wrap(switch (algorithm) {
case "gzip" -> new GZIPInputStream(in);
case "lz4f" -> new LZ4FrameInputStream(in);
case "lz4b" -> new LZ4BlockInputStream(in);

2
core/src/main/java/com/volmit/iris/engine/IrisWorldManager.java
View File

@ -459,7 +459,7 @@ public class IrisWorldManager extends EngineAssignedWorldManager {
IrisEngineData ed = getEngine().getEngineData();
IrisEngineSpawnerCooldown cd = null;
for (IrisEngineSpawnerCooldown j : ed.getSpawnerCooldowns()) {
for (IrisEngineSpawnerCooldown j : ed.getSpawnerCooldowns().copy()) {
if (j.getSpawner().equals(i.getLoadKey())) {
cd = j;
}

87
core/src/main/java/com/volmit/iris/util/io/CountingDataInputStream.java Normal file
View File

@ -0,0 +1,87 @@
package com.volmit.iris.util.io;
import lombok.RequiredArgsConstructor;
import org.jetbrains.annotations.NotNull;
import java.io.DataInputStream;
import java.io.IOException;
import java.io.InputStream;
public class CountingDataInputStream extends DataInputStream {
private final Counter counter;
private CountingDataInputStream(@NotNull InputStream in) {
super(in);
if (!(in instanceof Counter c))
throw new IllegalArgumentException("Underlying stream must be a Counter");
this.counter = c;
}
public static CountingDataInputStream wrap(@NotNull InputStream in) {
return new CountingDataInputStream(new Counter(in));
}
public long count() {
return counter.count;
}
public void skipTo(long target) throws IOException {
skipNBytes(Math.max(target - counter.count, 0));
}
@RequiredArgsConstructor
private static class Counter extends InputStream {
private final InputStream in;
private long count;
private long mark = -1;
private int markLimit = 0;
@Override
public int read() throws IOException {
int i = in.read();
if (i != -1) count(1);
return i;
}
@Override
public int read(@NotNull byte[] b, int off, int len) throws IOException {
int i = in.read(b, off, len);
count(i);
return i;
}
private void count(int i) {
count += i;
if (mark == -1)
return;
markLimit -= i;
if (markLimit <= 0)
mark = -1;
}
@Override
public boolean markSupported() {
return in.markSupported();
}
@Override
public synchronized void mark(int readlimit) {
if (!in.markSupported()) return;
in.mark(readlimit);
mark = count;
markLimit = readlimit;
}
@Override
public synchronized void reset() throws IOException {
in.reset();
count = mark;
}
@Override
public void close() throws IOException {
in.close();
}
}
}

3
core/src/main/java/com/volmit/iris/util/mantle/Mantle.java
View File

@ -567,9 +567,6 @@ public class Mantle {
}
File file = fileForRegion(dataFolder, x, z);
if (!file.exists())
file = new File(dataFolder, file.getName().substring(".lz4b".length()));
if (file.exists()) {
try {
Iris.addPanic("reading.tectonic-plate", file.getAbsolutePath());

37
core/src/main/java/com/volmit/iris/util/mantle/MantleChunk.java
View File

@ -21,12 +21,13 @@ package com.volmit.iris.util.mantle;
import com.volmit.iris.Iris;
import com.volmit.iris.util.documentation.ChunkCoordinates;
import com.volmit.iris.util.function.Consumer4;
import com.volmit.iris.util.io.CountingDataInputStream;
import com.volmit.iris.util.matter.IrisMatter;
import com.volmit.iris.util.matter.Matter;
import com.volmit.iris.util.matter.MatterSlice;
import lombok.Getter;
import java.io.DataInputStream;
import java.io.ByteArrayOutputStream;
import java.io.DataOutputStream;
import java.io.IOException;
import java.util.concurrent.atomic.AtomicIntegerArray;
@ -69,7 +70,7 @@ public class MantleChunk {
* @throws IOException shit happens
* @throws ClassNotFoundException shit happens
*/
public MantleChunk(int sectionHeight, DataInputStream din) throws IOException, ClassNotFoundException {
public MantleChunk(int sectionHeight, CountingDataInputStream din) throws IOException {
this(sectionHeight, din.readByte(), din.readByte());
int s = din.readByte();
@ -79,8 +80,23 @@ public class MantleChunk {
for (int i = 0; i < s; i++) {
Iris.addPanic("read.section", "Section[" + i + "]");
if (din.readBoolean()) {
long size = din.readInt();
if (size == 0) continue;
long start = din.count();
try {
sections.set(i, Matter.readDin(din));
} catch (IOException e) {
long end = start + size;
Iris.error("Failed to read chunk section, skipping it.");
Iris.addPanic("read.byte.range", start + " " + end);
Iris.addPanic("read.byte.current", din.count() + "");
Iris.reportError(e);
e.printStackTrace();
Iris.panic();
din.skipTo(end);
TectonicPlate.addError();
}
}
}
@ -174,15 +190,22 @@ public class MantleChunk {
dos.writeBoolean(flags.get(i) == 1);
}
var bytes = new ByteArrayOutputStream(8192);
var sub = new DataOutputStream(bytes);
for (int i = 0; i < sections.length(); i++) {
trimSlice(i);
if (exists(i)) {
dos.writeBoolean(true);
Matter matter = get(i);
matter.writeDos(dos);
try {
Matter matter = get(i);
matter.writeDos(sub);
dos.writeInt(bytes.size());
bytes.writeTo(dos);
} finally {
bytes.reset();
}
} else {
dos.writeBoolean(false);
dos.writeInt(0);
}
}
}

106
core/src/main/java/com/volmit/iris/util/mantle/TectonicPlate.java
View File

@ -21,26 +21,31 @@ package com.volmit.iris.util.mantle;
import com.volmit.iris.Iris;
import com.volmit.iris.engine.EnginePanic;
import com.volmit.iris.engine.data.cache.Cache;
import com.volmit.iris.util.collection.KSet;
import com.volmit.iris.util.documentation.ChunkCoordinates;
import com.volmit.iris.util.format.C;
import com.volmit.iris.util.format.Form;
import com.volmit.iris.util.io.CountingDataInputStream;
import com.volmit.iris.util.scheduling.PrecisionStopwatch;
import lombok.Getter;
import net.jpountz.lz4.LZ4BlockInputStream;
import net.jpountz.lz4.LZ4BlockOutputStream;
import net.jpountz.lz4.LZ4FrameInputStream;
import net.jpountz.lz4.LZ4FrameOutputStream;
import java.io.*;
import java.nio.channels.Channels;
import java.nio.channels.FileChannel;
import java.nio.file.Files;
import java.nio.file.StandardCopyOption;
import java.nio.file.StandardOpenOption;
import java.util.concurrent.atomic.AtomicReferenceArray;
import java.util.zip.GZIPInputStream;
import java.util.zip.GZIPOutputStream;
/**
* Tectonic Plates are essentially representations of regions in minecraft.
* Tectonic Plates are fully atomic & thread safe
*/
public class TectonicPlate {
private static final KSet<Thread> errors = new KSet<>();
private final int sectionHeight;
private final AtomicReferenceArray<MantleChunk> chunks;
@ -68,33 +73,58 @@ public class TectonicPlate {
* @param worldHeight the height of the world
* @param din the data input
* @throws IOException shit happens yo
* @throws ClassNotFoundException real shit bro
*/
public TectonicPlate(int worldHeight, DataInputStream din) throws IOException, ClassNotFoundException {
public TectonicPlate(int worldHeight, CountingDataInputStream din) throws IOException {
this(worldHeight, din.readInt(), din.readInt());
if (!din.markSupported())
throw new IOException("Mark not supported!");
for (int i = 0; i < chunks.length(); i++) {
if (din.readBoolean()) {
long size = din.readInt();
if (size == 0) continue;
long start = din.count();
try {
Iris.addPanic("read-chunk", "Chunk[" + i + "]");
chunks.set(i, new MantleChunk(sectionHeight, din));
EnginePanic.saveLast();
} catch (Throwable e) {
long end = start + size;
Iris.error("Failed to read chunk, creating a new chunk instead.");
Iris.addPanic("read.byte.range", start + " " + end);
Iris.addPanic("read.byte.current", din.count() + "");
Iris.reportError(e);
e.printStackTrace();
Iris.panic();
din.skipTo(end);
TectonicPlate.addError();
}
}
}
public static TectonicPlate read(int worldHeight, File file) throws IOException, ClassNotFoundException {
FileInputStream fin = new FileInputStream(file);
DataInputStream din;
if (file.getName().endsWith("ttp.lz4b")) {
LZ4BlockInputStream lz4 = new LZ4BlockInputStream(fin);
din = new DataInputStream(new BufferedInputStream(lz4));
} else {
GZIPInputStream gzi = new GZIPInputStream(fin);
din = new DataInputStream(new BufferedInputStream(gzi));
}
TectonicPlate p = new TectonicPlate(worldHeight, din);
din.close();
public static TectonicPlate read(int worldHeight, File file) throws IOException {
try (FileChannel fc = FileChannel.open(file.toPath(), StandardOpenOption.READ, StandardOpenOption.WRITE, StandardOpenOption.SYNC)) {
fc.lock();
return p;
InputStream fin = Channels.newInputStream(fc);
LZ4BlockInputStream lz4 = new LZ4BlockInputStream(fin);
BufferedInputStream bis = new BufferedInputStream(lz4);
try (CountingDataInputStream din = CountingDataInputStream.wrap(bis)) {
return new TectonicPlate(worldHeight, din);
}
} finally {
if (errors.remove(Thread.currentThread())) {
File dump = Iris.instance.getDataFolder("dump", file.getName() + ".bin");
try (FileChannel fc = FileChannel.open(file.toPath(), StandardOpenOption.READ, StandardOpenOption.WRITE, StandardOpenOption.SYNC)) {
fc.lock();
InputStream fin = Channels.newInputStream(fc);
LZ4BlockInputStream lz4 = new LZ4BlockInputStream(fin);
Files.copy(lz4, dump.toPath(), StandardCopyOption.REPLACE_EXISTING);
}
}
}
}
/**
@ -173,18 +203,15 @@ public class TectonicPlate {
*/
public void write(File file) throws IOException {
PrecisionStopwatch p = PrecisionStopwatch.start();
FileOutputStream fos = new FileOutputStream(file);
DataOutputStream dos;
if (file.getName().endsWith("ttp.lz4b")) {
LZ4BlockOutputStream lz4 = new LZ4BlockOutputStream(fos);
dos = new DataOutputStream(lz4);
} else {
GZIPOutputStream gzo = new GZIPOutputStream(fos);
dos = new DataOutputStream(gzo);
try (FileChannel fc = FileChannel.open(file.toPath(), StandardOpenOption.READ, StandardOpenOption.WRITE, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.SYNC)) {
fc.lock();
OutputStream fos = Channels.newOutputStream(fc);
try (DataOutputStream dos = new DataOutputStream(new LZ4BlockOutputStream(fos))) {
write(dos);
Iris.debug("Saved Tectonic Plate " + C.DARK_GREEN + file.getName().split("\\Q.\\E")[0] + C.RED + " in " + Form.duration(p.getMilliseconds(), 2));
}
}
write(dos);
dos.close();
Iris.debug("Saved Tectonic Plate " + C.DARK_GREEN + file.getName().split("\\Q.\\E")[0] + C.RED + " in " + Form.duration(p.getMilliseconds(), 2));
}
/**
@ -197,15 +224,26 @@ public class TectonicPlate {
dos.writeInt(x);
dos.writeInt(z);
var bytes = new ByteArrayOutputStream(8192);
var sub = new DataOutputStream(bytes);
for (int i = 0; i < chunks.length(); i++) {
MantleChunk chunk = chunks.get(i);
if (chunk != null) {
dos.writeBoolean(true);
chunk.write(dos);
try {
chunk.write(sub);
dos.writeInt(bytes.size());
bytes.writeTo(dos);
} finally {
bytes.reset();
}
} else {
dos.writeBoolean(false);
dos.writeInt(0);
}
}
}
public static void addError() {
errors.add(Thread.currentThread());
}
}

43
core/src/main/java/com/volmit/iris/util/matter/Matter.java
View File

@ -23,6 +23,8 @@ import com.volmit.iris.engine.object.IrisObject;
import com.volmit.iris.engine.object.IrisPosition;
import com.volmit.iris.util.collection.KSet;
import com.volmit.iris.util.hunk.Hunk;
import com.volmit.iris.util.io.CountingDataInputStream;
import com.volmit.iris.util.mantle.TectonicPlate;
import com.volmit.iris.util.math.BlockPosition;
import org.bukkit.World;
import org.bukkit.block.data.BlockData;
@ -96,18 +98,18 @@ public interface Matter {
return m;
}
static Matter read(File f) throws IOException, ClassNotFoundException {
static Matter read(File f) throws IOException {
FileInputStream in = new FileInputStream(f);
Matter m = read(in);
in.close();
return m;
}
static Matter read(InputStream in) throws IOException, ClassNotFoundException {
static Matter read(InputStream in) throws IOException {
return read(in, (b) -> new IrisMatter(b.getX(), b.getY(), b.getZ()));
}
static Matter readDin(DataInputStream in) throws IOException, ClassNotFoundException {
static Matter readDin(CountingDataInputStream in) throws IOException {
return readDin(in, (b) -> new IrisMatter(b.getX(), b.getY(), b.getZ()));
}
@ -120,11 +122,11 @@ public interface Matter {
* @return the matter object
* @throws IOException shit happens yo
*/
static Matter read(InputStream in, Function<BlockPosition, Matter> matterFactory) throws IOException, ClassNotFoundException {
return readDin(new DataInputStream(in), matterFactory);
static Matter read(InputStream in, Function<BlockPosition, Matter> matterFactory) throws IOException {
return readDin(CountingDataInputStream.wrap(in), matterFactory);
}
static Matter readDin(DataInputStream din, Function<BlockPosition, Matter> matterFactory) throws IOException, ClassNotFoundException {
static Matter readDin(CountingDataInputStream din, Function<BlockPosition, Matter> matterFactory) throws IOException {
Matter matter = matterFactory.apply(new BlockPosition(
din.readInt(),
din.readInt(),
@ -137,17 +139,30 @@ public interface Matter {
Iris.addPanic("read.matter.header", matter.getHeader().toString());
for (int i = 0; i < sliceCount; i++) {
long size = din.readInt();
if (size == 0) continue;
long start = din.count();
Iris.addPanic("read.matter.slice", i + "");
String cn = din.readUTF();
Iris.addPanic("read.matter.slice.class", cn);
try {
String cn = din.readUTF();
Iris.addPanic("read.matter.slice.class", cn);
Class<?> type = Class.forName(cn);
MatterSlice<?> slice = matter.createSlice(type, matter);
slice.read(din);
matter.putSlice(type, slice);
} catch (Throwable e) {
long end = start + size;
Iris.error("Failed to read matter slice, skipping it.");
Iris.addPanic("read.byte.range", start + " " + end);
Iris.addPanic("read.byte.current", din.count() + "");
Iris.reportError(e);
e.printStackTrace();
throw new IOException("Can't read class '" + cn + "' (slice count reverse at " + sliceCount + ")");
Iris.panic();
din.skipTo(end);
TectonicPlate.addError();
}
}
@ -414,8 +429,16 @@ public interface Matter {
dos.writeByte(getSliceTypes().size());
getHeader().write(dos);
var bytes = new ByteArrayOutputStream(1024);
var sub = new DataOutputStream(bytes);
for (Class<?> i : getSliceTypes()) {
getSlice(i).write(dos);
try {
getSlice(i).write(sub);
dos.writeInt(bytes.size());
bytes.writeTo(dos);
} finally {
bytes.reset();
}
}
}