mirror of
https://github.com/VolmitSoftware/Iris.git
synced 2026-10-27 06:20:02 +00:00
Improved saved world startup
- Startup holds generation admission through recovery and runtime attachment - Prepared startup state removes the second recovery scan before generator readiness - Recovery batches allocated chunks by region with reusable snapshot buffers - Region decoding uses up to four workers within processor and heap limits - Oversized regions and external chunks use bounded direct reads - Status parsing validates chunk coordinates without constructing the full NBT tree - Region snapshots reject changed files, allocation headers and truncated payloads - Native zlib decompression uses jlibdeflate with a Java inflater fallback - Chunk readers reuse region handles and decompression buffers during recovery and boundary capture - Semantic shards reuse validated summaries with invalidation on mutation - Semantic key validation avoids regular expression allocation - Saved biome indexes use buffered sequential reads with reusable CRC state - Pack inspection reads validated manifest and pack state without opening terrain indexes - Datapack compilation reuses pack inspection within each operation - Kernel staging avoids duplicate pack hashing when no activation is pending - Pack copying and directory synchronization use up to four available processors - Boundary publication batches directory synchronization around durable catalog publication - Configured world reconciliation reuses ready runtimes after identity and saved-state checks - Explicit world loads retain the full validation path - Regression tests cover startup admission, recovery, malformed data, resource limits, publication and reconciliation
This commit is contained in:
1 parent
30c533feaf
commit
98ddb6644d
35 files changed
+3301
-197
No files matched your search
@@ -198,12 +198,13 @@ public final class IrisDatapackCompiler {
|
||||
Objects.requireNonNull(fixer, "fixer");
|
||||
LinkedHashMap<String, String> requirements = new LinkedHashMap<>();
|
||||
Map<String, Set<String>> biomeTags = new TreeMap<>();
|
||||
Map<Path, GenerationHistory.PackInspection> histories = new LinkedHashMap<>();
|
||||
for (File packRoot : packRoots) {
|
||||
PackDirectoryResolver.requireSafePackTree(packRoot);
|
||||
if (!hasDimensions(packRoot.toPath())) {
|
||||
continue;
|
||||
}
|
||||
BiomeAliasContext aliasContext = biomeAliasContext(packRoot.toPath());
|
||||
BiomeAliasContext aliasContext = biomeAliasContext(packRoot.toPath(), histories);
|
||||
IrisData data = IrisData.openDatapackCompiler(packRoot);
|
||||
try {
|
||||
if (aliasContext.retainedContract() != null) {
|
||||
@@ -230,7 +231,7 @@ public final class IrisDatapackCompiler {
|
||||
data.close();
|
||||
}
|
||||
}
|
||||
for (GenerationRegistryContract contract : retainedRegistryContracts(packRoots)) {
|
||||
for (GenerationRegistryContract contract : retainedRegistryContracts(packRoots, histories)) {
|
||||
for (PhysicalResourceKey key : contract.generatedSources().keySet()) {
|
||||
putRegistryRequirement(requirements, registryPath(key) + "/" + key.resourceKey(),
|
||||
fingerprintContent(GenerationRegistryContractFactory.renderGeneratedSource(contract, key, fixer)));
|
||||
@@ -453,6 +454,7 @@ public final class IrisDatapackCompiler {
|
||||
DimensionHeight height = new DimensionHeight(fixer);
|
||||
Map<String, KSet<String>> biomes = new LinkedHashMap<>();
|
||||
Map<String, List<DimensionCandidate>> dimensions = new LinkedHashMap<>();
|
||||
Map<Path, GenerationHistory.PackInspection> histories = new LinkedHashMap<>();
|
||||
int packCount = 0;
|
||||
int dimensionCount = 0;
|
||||
for (File packRoot : packRoots) {
|
||||
@@ -460,7 +462,7 @@ public final class IrisDatapackCompiler {
|
||||
if (!hasDimensions(packRoot.toPath())) {
|
||||
continue;
|
||||
}
|
||||
BiomeAliasContext aliasContext = biomeAliasContext(packRoot.toPath());
|
||||
BiomeAliasContext aliasContext = biomeAliasContext(packRoot.toPath(), histories);
|
||||
IrisData data = IrisData.openDatapackCompiler(packRoot);
|
||||
try {
|
||||
if (aliasContext.retainedContract() != null) {
|
||||
@@ -512,7 +514,7 @@ public final class IrisDatapackCompiler {
|
||||
}
|
||||
|
||||
IrisDimension.writeShared(datapackRoots, height, adjustVanillaHeight);
|
||||
installRetainedRegistrySources(packRoots, datapackRoots, fixer);
|
||||
installRetainedRegistrySources(packRoots, datapackRoots, fixer, histories);
|
||||
installLevelStemBindings(normalizedBindings, dimensions, datapackRoots);
|
||||
validateOutputs(datapackRoots, dimensionCount, normalizedBindings.size());
|
||||
return new CompilationResult(packCount, dimensionCount, countBiomes(biomes));
|
||||
@@ -520,9 +522,15 @@ public final class IrisDatapackCompiler {
|
||||
|
||||
public static void installRetainedRegistrySources(List<File> packRoots, Collection<File> datapackRoots,
|
||||
IDataFixer fixer) throws IOException {
|
||||
installRetainedRegistrySources(packRoots, datapackRoots, fixer, new LinkedHashMap<>());
|
||||
}
|
||||
|
||||
private static void installRetainedRegistrySources(List<File> packRoots, Collection<File> datapackRoots,
|
||||
IDataFixer fixer,
|
||||
Map<Path, GenerationHistory.PackInspection> histories) throws IOException {
|
||||
Map<PhysicalResourceKey, String> sources = new TreeMap<>();
|
||||
Map<String, Set<String>> tags = new TreeMap<>();
|
||||
for (GenerationRegistryContract contract : retainedRegistryContracts(packRoots)) {
|
||||
for (GenerationRegistryContract contract : retainedRegistryContracts(packRoots, histories)) {
|
||||
for (PhysicalResourceKey key : contract.generatedSources().keySet()) {
|
||||
String source = GenerationRegistryContractFactory.renderGeneratedSource(contract, key, fixer);
|
||||
String previous = sources.putIfAbsent(key, source);
|
||||
@@ -588,6 +596,11 @@ public final class IrisDatapackCompiler {
|
||||
}
|
||||
|
||||
private static List<GenerationRegistryContract> retainedRegistryContracts(List<File> packRoots) throws IOException {
|
||||
return retainedRegistryContracts(packRoots, new LinkedHashMap<>());
|
||||
}
|
||||
|
||||
private static List<GenerationRegistryContract> retainedRegistryContracts(
|
||||
List<File> packRoots, Map<Path, GenerationHistory.PackInspection> histories) throws IOException {
|
||||
Map<String, GenerationRegistryContract> contracts = new TreeMap<>();
|
||||
Set<Path> loadedWorlds = new LinkedHashSet<>();
|
||||
for (File pack : packRoots) {
|
||||
@@ -595,7 +608,7 @@ public final class IrisDatapackCompiler {
|
||||
if (dimensionRoot == null || !loadedWorlds.add(dimensionRoot)) {
|
||||
continue;
|
||||
}
|
||||
for (GenerationEpoch epoch : GenerationHistory.open(dimensionRoot).manifest().epochs()) {
|
||||
for (GenerationEpoch epoch : inspectHistory(dimensionRoot, histories).manifest().epochs()) {
|
||||
contracts.putIfAbsent(epoch.registryContract().fingerprint(), epoch.registryContract());
|
||||
}
|
||||
}
|
||||
@@ -765,11 +778,9 @@ public final class IrisDatapackCompiler {
|
||||
if (!Files.isSymbolicLink(irisRoot)
|
||||
&& (Files.exists(generationRoot, LinkOption.NOFOLLOW_LINKS)
|
||||
|| Files.isSymbolicLink(generationRoot))) {
|
||||
GenerationHistory history = GenerationHistory.open(worldRoot);
|
||||
GenerationHistory.PackInspection history = GenerationHistory.inspectPacks(worldRoot);
|
||||
candidates.add(history.activePackRoot());
|
||||
if (history.pendingActivation().isPresent()) {
|
||||
candidates.add(history.packRoot(history.pendingActivation().orElseThrow().activationId()));
|
||||
}
|
||||
history.pendingPackRoot().ifPresent(candidates::add);
|
||||
return true;
|
||||
}
|
||||
Path legacyPack = irisRoot.resolve("pack");
|
||||
@@ -816,7 +827,8 @@ public final class IrisDatapackCompiler {
|
||||
roots.putIfAbsent(identity, normalized.toFile());
|
||||
}
|
||||
|
||||
private static BiomeAliasContext biomeAliasContext(Path packRoot) throws IOException {
|
||||
private static BiomeAliasContext biomeAliasContext(
|
||||
Path packRoot, Map<Path, GenerationHistory.PackInspection> histories) throws IOException {
|
||||
Path normalizedRoot = packRoot.toAbsolutePath().normalize();
|
||||
Path rootParent = normalizedRoot.getParent();
|
||||
if (rootParent == null || !"pack".equals(normalizedRoot.getFileName().toString())) {
|
||||
@@ -841,7 +853,7 @@ public final class IrisDatapackCompiler {
|
||||
}
|
||||
|
||||
String epochId = rootParent.getFileName().toString();
|
||||
GenerationHistory history = GenerationHistory.open(dimensionRoot);
|
||||
GenerationHistory.PackInspection history = inspectHistory(dimensionRoot, histories);
|
||||
GenerationEpoch epoch = history.manifest().epoch(epochId).orElseThrow(
|
||||
() -> new IOException("Retained Iris pack is not referenced by its generation history: "
|
||||
+ normalizedRoot)
|
||||
@@ -853,6 +865,17 @@ public final class IrisDatapackCompiler {
|
||||
return BiomeAliasContext.retained(epoch.registryContract());
|
||||
}
|
||||
|
||||
private static GenerationHistory.PackInspection inspectHistory(
|
||||
Path dimensionRoot, Map<Path, GenerationHistory.PackInspection> histories) throws IOException {
|
||||
Path root = dimensionRoot.toAbsolutePath().normalize();
|
||||
GenerationHistory.PackInspection history = histories.get(root);
|
||||
if (history == null) {
|
||||
history = GenerationHistory.inspectPacks(root);
|
||||
histories.put(root, history);
|
||||
}
|
||||
return history;
|
||||
}
|
||||
|
||||
private static boolean hasDimensions(Path root) {
|
||||
Path dimensions = root.resolve("dimensions");
|
||||
if (Files.isSymbolicLink(dimensions)
|
||||
|
||||
@@ -581,16 +581,11 @@ public class BukkitChunkGenerator extends ChunkGenerator implements PlatformChun
|
||||
}
|
||||
|
||||
private void setupEngine() {
|
||||
try (GenerationAdmission.RuntimeLease startupAdmission = generationHistory == null ? null : generationHistory.retainRuntime()) {
|
||||
try (GenerationAdmission.RuntimeLease startupAdmission = generationHistory == null ? null : generationHistory.retainRuntime();
|
||||
GenerationHistory.StartupPreparation preparation = generationHistory == null ? null
|
||||
: prepareGenerationHistoryTarget(IrisSettings.get().getGenerator().getGenerationTransitionWidthBlocks())) {
|
||||
lastMode = StudioMode.NORMAL;
|
||||
lastJigsawStudioRequestId = null;
|
||||
if (generationHistory != null) {
|
||||
try {
|
||||
prepareGenerationHistoryTarget(IrisSettings.get().getGenerator().getGenerationTransitionWidthBlocks());
|
||||
} catch (IOException failure) {
|
||||
throw new IllegalStateException("Unable to prepare saved Iris terrain for the current generator.", failure);
|
||||
}
|
||||
}
|
||||
EngineTarget engineTarget = getTarget();
|
||||
String packKey = engineTarget.getDimension().getLoadKey();
|
||||
JigsawStudioActivation.Request request = studio
|
||||
@@ -605,24 +600,9 @@ public class BukkitChunkGenerator extends ChunkGenerator implements PlatformChun
|
||||
objectStudioActive = studio && ObjectStudioActivation.isActive(packKey);
|
||||
IrisEngine createdEngine = createEngine(engineTarget);
|
||||
if (generationHistory != null) {
|
||||
long initialActivationId = generationHistory.activeActivation().activationId();
|
||||
int transitionWidthBlocks = IrisSettings.get()
|
||||
.getGenerator()
|
||||
.getGenerationTransitionWidthBlocks();
|
||||
try {
|
||||
createdEngine.attachGenerationHistory(
|
||||
generationHistory,
|
||||
IrisBoundarySignatureSampler.INSTANCE,
|
||||
transitionWidthBlocks);
|
||||
if (generationHistory.activeActivation().activationId() != initialActivationId) {
|
||||
createdEngine.close();
|
||||
targetCache.reset();
|
||||
createdEngine = createEngine(loadActiveGenerationHistoryTarget());
|
||||
createdEngine.attachGenerationHistory(
|
||||
generationHistory,
|
||||
IrisBoundarySignatureSampler.INSTANCE,
|
||||
transitionWidthBlocks);
|
||||
}
|
||||
GenerationHistoryRuntimeRouter.attachPrepared(
|
||||
createdEngine, preparation, IrisBoundarySignatureSampler.INSTANCE);
|
||||
} catch (Throwable failure) {
|
||||
try {
|
||||
createdEngine.close();
|
||||
@@ -641,6 +621,8 @@ public class BukkitChunkGenerator extends ChunkGenerator implements PlatformChun
|
||||
publishInitializedEngine(createdEngine);
|
||||
populators.clear();
|
||||
targetCache.reset();
|
||||
} catch (IOException failure) {
|
||||
throw new IllegalStateException("Unable to prepare saved Iris terrain for the current generator.", failure);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -651,25 +633,31 @@ public class BukkitChunkGenerator extends ChunkGenerator implements PlatformChun
|
||||
engine = initializedEngine;
|
||||
}
|
||||
|
||||
void prepareGenerationHistoryTarget(int transitionWidthBlocks) throws IOException {
|
||||
generationHistory.prepareCurrentGenerator(transitionWidthBlocks);
|
||||
EngineTarget cachedTarget = targetCache.getIfPresent();
|
||||
if (cachedTarget == null) {
|
||||
return;
|
||||
}
|
||||
Path activePackRoot = generationHistory.activePackRoot();
|
||||
IrisData cachedData = cachedTarget.getData();
|
||||
if (!cachedData.isClosed() && cachedData.getDataFolder().toPath().equals(activePackRoot)) {
|
||||
return;
|
||||
}
|
||||
targetCache.reset();
|
||||
EngineTarget pendingTarget = startupTarget;
|
||||
Engine activeEngine = engine;
|
||||
if (!cachedData.isClosed()
|
||||
&& cachedData.getEngines().isEmpty()
|
||||
&& (pendingTarget == null || pendingTarget.getData() != cachedData)
|
||||
&& (activeEngine == null || activeEngine.getData() != cachedData)) {
|
||||
cachedData.close();
|
||||
GenerationHistory.StartupPreparation prepareGenerationHistoryTarget(int transitionWidthBlocks) throws IOException {
|
||||
GenerationHistory.StartupPreparation preparation = generationHistory.prepareStartup(transitionWidthBlocks);
|
||||
try {
|
||||
EngineTarget cachedTarget = targetCache.getIfPresent();
|
||||
if (cachedTarget == null) {
|
||||
return preparation;
|
||||
}
|
||||
Path activePackRoot = generationHistory.activePackRoot();
|
||||
IrisData cachedData = cachedTarget.getData();
|
||||
if (!cachedData.isClosed() && cachedData.getDataFolder().toPath().equals(activePackRoot)) {
|
||||
return preparation;
|
||||
}
|
||||
targetCache.reset();
|
||||
EngineTarget pendingTarget = startupTarget;
|
||||
Engine activeEngine = engine;
|
||||
if (!cachedData.isClosed()
|
||||
&& cachedData.getEngines().isEmpty()
|
||||
&& (pendingTarget == null || pendingTarget.getData() != cachedData)
|
||||
&& (activeEngine == null || activeEngine.getData() != cachedData)) {
|
||||
cachedData.close();
|
||||
}
|
||||
return preparation;
|
||||
} catch (IOException | RuntimeException | Error failure) {
|
||||
preparation.close();
|
||||
throw failure;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -249,23 +249,23 @@ public final class ChunkGenerationSemantics {
|
||||
throw new IllegalArgumentException("Generation semantic resource key contains an invalid character");
|
||||
}
|
||||
}
|
||||
if (requiredKey.getBytes(StandardCharsets.UTF_8).length > MAX_KEY_BYTES) {
|
||||
if (requiredKey.length() > MAX_KEY_BYTES / 3
|
||||
&& requiredKey.getBytes(StandardCharsets.UTF_8).length > MAX_KEY_BYTES) {
|
||||
throw new IllegalArgumentException("Generation semantic resource key is too long");
|
||||
}
|
||||
return requiredKey;
|
||||
}
|
||||
|
||||
private static Set<String> immutableKeys(Collection<String> keys, String kind) {
|
||||
if (keys.isEmpty()) {
|
||||
return Set.of();
|
||||
}
|
||||
if (keys.size() > MAX_KEYS_PER_KIND) {
|
||||
throw new IllegalArgumentException(
|
||||
"A chunk cannot contain more than " + MAX_KEYS_PER_KIND + " " + kind + " keys"
|
||||
);
|
||||
}
|
||||
TreeSet<String> validated = new TreeSet<>();
|
||||
for (String key : keys) {
|
||||
validated.add(requireResourceKey(key));
|
||||
}
|
||||
return Collections.unmodifiableSet(validated);
|
||||
return Collections.unmodifiableSet(new TreeSet<>(keys));
|
||||
}
|
||||
|
||||
private static Set<StructureOccurrence> immutableStructures(Collection<StructureOccurrence> structures) {
|
||||
|
||||
@@ -9,13 +9,13 @@ import java.util.Objects;
|
||||
public final class DiskBoundaryCapture implements TerrainBoundarySignatureStore.SignatureSampler {
|
||||
private static final int MAXIMUM_CACHED_CHUNKS = 32;
|
||||
|
||||
private final Path dimensionRoot;
|
||||
private final SavedTerrainChunkReader.StatusReader reader;
|
||||
private final int minimumY;
|
||||
private final int height;
|
||||
private final LinkedHashMap<Long, SavedTerrainChunk> chunks = new LinkedHashMap<>(32, 0.75F, true);
|
||||
|
||||
public DiskBoundaryCapture(Path dimensionRoot, int minimumY, int height) {
|
||||
this.dimensionRoot = Objects.requireNonNull(dimensionRoot, "dimension root");
|
||||
this.reader = new SavedTerrainChunkReader.StatusReader(Objects.requireNonNull(dimensionRoot, "dimension root"));
|
||||
this.minimumY = minimumY;
|
||||
this.height = height;
|
||||
}
|
||||
@@ -27,7 +27,7 @@ public final class DiskBoundaryCapture implements TerrainBoundarySignatureStore.
|
||||
long key = ChunkGenerationOwnership.packChunk(chunkX, chunkZ);
|
||||
SavedTerrainChunk chunk = chunks.get(key);
|
||||
if (chunk == null) {
|
||||
chunk = SavedTerrainChunk.read(dimensionRoot, chunkX, chunkZ, minimumY, height);
|
||||
chunk = reader.readChunk(chunkX, chunkZ, minimumY, height);
|
||||
chunks.put(key, chunk);
|
||||
if (chunks.size() > MAXIMUM_CACHED_CHUNKS) {
|
||||
Map.Entry<Long, SavedTerrainChunk> oldest = chunks.firstEntry();
|
||||
@@ -38,7 +38,8 @@ public final class DiskBoundaryCapture implements TerrainBoundarySignatureStore.
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() {
|
||||
public void close() throws IOException {
|
||||
chunks.clear();
|
||||
reader.close();
|
||||
}
|
||||
}
|
||||
@@ -67,7 +67,7 @@ public final class GenerationHistory {
|
||||
this.admission = new GenerationAdmission(paths.dimensionRoot());
|
||||
this.boundaryCache = boundedCache(MAXIMUM_CACHED_BOUNDARIES);
|
||||
this.terrainSignatureCache = boundedCache(MAXIMUM_CACHED_TERRAIN_SIGNATURES);
|
||||
validateReferencedState(publication);
|
||||
validateReferencedState(publication, true);
|
||||
}
|
||||
|
||||
public GenerationAdmission.RuntimeLease retainRuntime() {
|
||||
@@ -148,6 +148,26 @@ public final class GenerationHistory {
|
||||
return open(dimensionRoot, GenerationKernelRegistry.standard());
|
||||
}
|
||||
|
||||
public static PackInspection inspectPacks(Path dimensionRoot) throws IOException {
|
||||
GenerationHistoryPaths paths = GenerationHistoryPaths.forDimension(dimensionRoot);
|
||||
requireSafeStatePaths(paths);
|
||||
GenerationManifest manifest = GenerationHistoryStore.open(paths.generationRoot()).manifest();
|
||||
for (GenerationEpoch epoch : manifest.epochs()) {
|
||||
GenerationPackFingerprint.requireSupported(epoch.packFingerprintVersion());
|
||||
}
|
||||
GenerationPackRepository packs = new GenerationPackRepository(paths.dimensionRoot());
|
||||
GenerationEpoch active = manifest.activeEpoch();
|
||||
Path activePack = packs.requireExactPack(
|
||||
active.epochId(), active.packFingerprint(), active.packFingerprintVersion());
|
||||
Optional<Path> pendingPack = Optional.empty();
|
||||
if (manifest.pendingEpoch().isPresent()) {
|
||||
GenerationEpoch pending = manifest.pendingEpoch().orElseThrow();
|
||||
pendingPack = Optional.of(pending.epochId().equals(active.epochId()) ? activePack
|
||||
: packs.requireExactPack(pending.epochId(), pending.packFingerprint(), pending.packFingerprintVersion()));
|
||||
}
|
||||
return new PackInspection(paths, manifest, activePack, pendingPack);
|
||||
}
|
||||
|
||||
public static GenerationHistory open(Path dimensionRoot, long expectedWorldSeed) throws IOException {
|
||||
GenerationHistory history = open(dimensionRoot, GenerationKernelRegistry.standard());
|
||||
history.requireWorldSeed(expectedWorldSeed);
|
||||
@@ -393,7 +413,7 @@ public final class GenerationHistory {
|
||||
);
|
||||
try (GenerationAdmission.CutoverLease ignored = admission.beginStartupCutover()) {
|
||||
synchronized (this) {
|
||||
return promotePendingLocked(requiredCapture);
|
||||
return promotePendingLocked(requiredCapture, true, true);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -414,7 +434,7 @@ public final class GenerationHistory {
|
||||
GenerationActivation promote(BoundarySignatureCapture signatureCapture) throws IOException {
|
||||
requireOpen();
|
||||
synchronized (GenerationHistory.this) {
|
||||
return promotePendingLocked(Objects.requireNonNull(signatureCapture, "signature capture"));
|
||||
return promotePendingLocked(Objects.requireNonNull(signatureCapture, "signature capture"), false, true);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -433,10 +453,10 @@ public final class GenerationHistory {
|
||||
}
|
||||
|
||||
private GenerationActivation promotePendingLocked(
|
||||
BoundarySignatureCapture signatureCapture
|
||||
BoundarySignatureCapture signatureCapture, boolean startupRecovery, boolean validateSemantics
|
||||
) throws IOException {
|
||||
WorldChunkInventory inventory = recoverUnstoredClaims();
|
||||
validateReferencedState(Optional.empty());
|
||||
WorldChunkInventory inventory = recoverUnstoredClaims(startupRecovery);
|
||||
validateReferencedState(Optional.empty(), validateSemantics);
|
||||
Optional<GenerationActivation> pending = store.pendingActivation();
|
||||
if (pending.isEmpty()) {
|
||||
return store.activeActivation();
|
||||
@@ -498,7 +518,7 @@ public final class GenerationHistory {
|
||||
) throws IOException {
|
||||
GenerationEpoch active = store.activeEpoch();
|
||||
return stageUpdate(
|
||||
activePackRoot(),
|
||||
store.pendingActivation().isPresent() ? activePackRoot() : packs.requireSafePackRoot(active.epochId()),
|
||||
active.packFingerprint(),
|
||||
active.packFingerprintVersion(),
|
||||
active.dimensionContract(),
|
||||
@@ -509,11 +529,17 @@ public final class GenerationHistory {
|
||||
}
|
||||
|
||||
public void prepareCurrentGenerator(int transitionWidthBlocks) throws IOException {
|
||||
try (GenerationAdmission.CutoverLease ignored = admission.beginStartupCutover()) {
|
||||
try (StartupPreparation ignored = prepareStartup(transitionWidthBlocks)) {
|
||||
}
|
||||
}
|
||||
|
||||
public StartupPreparation prepareStartup(int transitionWidthBlocks) throws IOException {
|
||||
GenerationAdmission.CutoverLease lease = admission.beginStartupCutover();
|
||||
try {
|
||||
synchronized (this) {
|
||||
if (store.pendingActivation().isEmpty() && usesCurrentGenerator()) {
|
||||
recoverUnstoredClaims();
|
||||
validateReferencedState(Optional.empty());
|
||||
recoverUnstoredClaims(true);
|
||||
validateReferencedState(Optional.empty(), false);
|
||||
} else {
|
||||
if (store.pendingActivation().isEmpty()) {
|
||||
stageCurrentKernel(transitionWidthBlocks);
|
||||
@@ -524,27 +550,80 @@ public final class GenerationHistory {
|
||||
promoteSavedBoundary();
|
||||
}
|
||||
}
|
||||
return new StartupPreparation(lease, store.manifest());
|
||||
}
|
||||
} catch (IOException | RuntimeException | Error failure) {
|
||||
lease.close();
|
||||
throw failure;
|
||||
}
|
||||
}
|
||||
|
||||
public final class StartupPreparation implements AutoCloseable {
|
||||
private final GenerationAdmission.CutoverLease lease;
|
||||
private final GenerationManifest preparedManifest;
|
||||
private final Thread owner = Thread.currentThread();
|
||||
private boolean consumed;
|
||||
private boolean closed;
|
||||
|
||||
private StartupPreparation(GenerationAdmission.CutoverLease lease, GenerationManifest preparedManifest) {
|
||||
this.lease = lease;
|
||||
this.preparedManifest = preparedManifest;
|
||||
}
|
||||
|
||||
GenerationHistory history() {
|
||||
return GenerationHistory.this;
|
||||
}
|
||||
|
||||
void consume() {
|
||||
requireOwner();
|
||||
if (closed || consumed) {
|
||||
throw new IllegalStateException("Generation startup preparation is closed or already attached.");
|
||||
}
|
||||
if (store.manifest() != preparedManifest) {
|
||||
throw new IllegalStateException("Generation history changed after startup preparation.");
|
||||
}
|
||||
consumed = true;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() {
|
||||
requireOwner();
|
||||
if (!closed) {
|
||||
closed = true;
|
||||
lease.close();
|
||||
}
|
||||
}
|
||||
|
||||
private void requireOwner() {
|
||||
if (Thread.currentThread() != owner) {
|
||||
throw new IllegalStateException("Generation startup preparation belongs to another thread.");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void promoteSavedBoundary() throws IOException {
|
||||
GenerationEpoch.DimensionContract contract = activeEpoch().dimensionContract();
|
||||
promotePendingLocked(boundary -> new DiskBoundaryCapture(paths.dimensionRoot(), contract.minHeight(), contract.height()));
|
||||
promotePendingLocked(boundary -> new DiskBoundaryCapture(paths.dimensionRoot(), contract.minHeight(), contract.height()), true, false);
|
||||
}
|
||||
|
||||
private WorldChunkInventory recoverUnstoredClaims() throws IOException {
|
||||
private WorldChunkInventory recoverUnstoredClaims(boolean startupRecovery) throws IOException {
|
||||
long outgoing = store.activeActivation().activationId();
|
||||
Set<Long> selected = store.pendingActivation()
|
||||
.map(pending -> Set.of(outgoing, pending.activationId()))
|
||||
.orElseGet(() -> Set.of(outgoing));
|
||||
WorldChunkInventory inventory = WorldChunkInventory.scan(paths.dimensionRoot()).filter((chunkX, chunkZ) -> {
|
||||
if (ownership.isExplicitlyAssigned(chunkX, chunkZ)
|
||||
&& !selected.contains(ownership.resolve(chunkX, chunkZ, outgoing))) {
|
||||
return true;
|
||||
WorldChunkInventory allocated = WorldChunkInventory.scan(paths.dimensionRoot());
|
||||
WorldChunkInventory.ChunkPredicate retained = (chunkX, chunkZ) ->
|
||||
ownership.isExplicitlyAssigned(chunkX, chunkZ)
|
||||
&& !selected.contains(ownership.resolve(chunkX, chunkZ, outgoing));
|
||||
WorldChunkInventory inventory;
|
||||
if (startupRecovery) {
|
||||
inventory = SavedTerrainRecoveryScan.scan(paths.dimensionRoot(), allocated, retained);
|
||||
} else {
|
||||
try (SavedTerrainChunkReader.StatusReader reader = new SavedTerrainChunkReader.StatusReader(paths.dimensionRoot())) {
|
||||
inventory = allocated.filter((chunkX, chunkZ) -> retained.test(chunkX, chunkZ)
|
||||
|| SavedTerrainChunk.hasTerrain(reader.readStatus(chunkX, chunkZ)));
|
||||
}
|
||||
return SavedTerrainChunk.hasTerrain(SavedTerrainChunk.readStatus(paths.dimensionRoot(), chunkX, chunkZ));
|
||||
});
|
||||
}
|
||||
semantics.discardUnstoredClaims(inventory, selected);
|
||||
savedBiomes.discardUnstoredClaims(inventory, selected);
|
||||
ownership.discardUnstoredClaims(inventory, selected);
|
||||
@@ -808,7 +887,7 @@ public final class GenerationHistory {
|
||||
return ownership.explicitChunkCount();
|
||||
}
|
||||
|
||||
private void validateReferencedState(Optional<FreshPublication> publication) throws IOException {
|
||||
private void validateReferencedState(Optional<FreshPublication> publication, boolean validateSemantics) throws IOException {
|
||||
validateRuntimeVersions();
|
||||
GenerationEpoch active = store.activeEpoch();
|
||||
if (publication.isEmpty()
|
||||
@@ -833,7 +912,9 @@ public final class GenerationHistory {
|
||||
}
|
||||
}
|
||||
validateOwnershipReferences();
|
||||
validateSemanticReferences();
|
||||
if (validateSemantics) {
|
||||
validateSemanticReferences();
|
||||
}
|
||||
}
|
||||
|
||||
private static void requireTransitionSnapshotIdentities(
|
||||
@@ -1175,6 +1256,10 @@ public final class GenerationHistory {
|
||||
};
|
||||
}
|
||||
|
||||
public record PackInspection(GenerationHistoryPaths paths, GenerationManifest manifest,
|
||||
Path activePackRoot, Optional<Path> pendingPackRoot) {
|
||||
}
|
||||
|
||||
private record FreshPublication(Path pack, GenerationEpoch epoch) {
|
||||
}
|
||||
|
||||
|
||||
@@ -117,6 +117,27 @@ public final class GenerationHistoryRuntimeRouter implements AutoCloseable {
|
||||
);
|
||||
}
|
||||
|
||||
public static GenerationHistoryRuntimeRouter attachPrepared(
|
||||
IrisEngine engine,
|
||||
GenerationHistory.StartupPreparation preparation,
|
||||
GenerationBoundarySignatureSampler signatureSampler
|
||||
) throws IOException {
|
||||
return attachPrepared(engine, preparation, signatureSampler, new DefaultActivationRuntimeFactory());
|
||||
}
|
||||
|
||||
static GenerationHistoryRuntimeRouter attachPrepared(
|
||||
IrisEngine engine,
|
||||
GenerationHistory.StartupPreparation preparation,
|
||||
GenerationBoundarySignatureSampler signatureSampler,
|
||||
ActivationRuntimeFactory runtimeFactory
|
||||
) throws IOException {
|
||||
GenerationHistory history = Objects.requireNonNull(preparation, "startup preparation").history();
|
||||
synchronized (history) {
|
||||
preparation.consume();
|
||||
return attach(engine, history, signatureSampler, runtimeFactory);
|
||||
}
|
||||
}
|
||||
|
||||
public static GenerationHistoryRuntimeRouter attachAndPromotePending(
|
||||
IrisEngine engine,
|
||||
GenerationHistory history,
|
||||
|
||||
@@ -118,17 +118,22 @@ public final class GenerationPackRepository {
|
||||
String packFingerprint,
|
||||
int packFingerprintVersion
|
||||
) throws IOException {
|
||||
Path pack = requireSafePackRoot(epochId);
|
||||
requireFingerprint(
|
||||
pack,
|
||||
requireDigest(packFingerprint, "packFingerprint"),
|
||||
packFingerprintVersion
|
||||
);
|
||||
return pack;
|
||||
}
|
||||
|
||||
Path requireSafePackRoot(String epochId) throws IOException {
|
||||
String requiredEpochId = requireDigest(epochId, "epochId");
|
||||
validateExistingAncestors(requiredEpochId);
|
||||
Path epoch = epochRoot(requiredEpochId);
|
||||
requireSafeDirectory(epoch, "Generation epoch path is not a safe directory");
|
||||
Path pack = epoch.resolve("pack");
|
||||
requireSafeDirectory(pack, "Generation epoch pack is missing or unsafe");
|
||||
requireFingerprint(
|
||||
pack,
|
||||
requireDigest(packFingerprint, "packFingerprint"),
|
||||
packFingerprintVersion
|
||||
);
|
||||
return pack;
|
||||
}
|
||||
|
||||
@@ -198,7 +203,7 @@ public final class GenerationPackRepository {
|
||||
|
||||
public static void copyPackTree(Path source, Path target) throws IOException {
|
||||
Executor executor = IrisPlatforms.isBound() ? MultiBurst.ioBurst : ForkJoinPool.commonPool();
|
||||
int workers = Math.max(1, Math.min(MAXIMUM_COPY_WORKERS, Runtime.getRuntime().availableProcessors() / 2));
|
||||
int workers = Math.max(1, Math.min(MAXIMUM_COPY_WORKERS, Runtime.getRuntime().availableProcessors()));
|
||||
copyPackTree(source, target, executor, workers);
|
||||
}
|
||||
|
||||
@@ -272,7 +277,7 @@ public final class GenerationPackRepository {
|
||||
|
||||
static void forceDirectoryTree(Path root, FileAction force) throws IOException {
|
||||
Executor executor = IrisPlatforms.isBound() ? MultiBurst.ioBurst : ForkJoinPool.commonPool();
|
||||
int workers = Math.max(1, Math.min(MAXIMUM_COPY_WORKERS, Runtime.getRuntime().availableProcessors() / 2));
|
||||
int workers = Math.max(1, Math.min(MAXIMUM_COPY_WORKERS, Runtime.getRuntime().availableProcessors()));
|
||||
forceDirectoryTree(root, executor, workers, force);
|
||||
}
|
||||
|
||||
|
||||
@@ -1013,13 +1013,13 @@ public final class GenerationSemanticIndex {
|
||||
}
|
||||
RegionShard cachedRegion = regions.get(regionKey);
|
||||
if (cachedRegion != null) {
|
||||
RegionSummary summary = RegionSummary.from(cachedRegion);
|
||||
RegionSummary summary = cachedRegion.summary();
|
||||
cacheSummary(regionKey, summary);
|
||||
return summary;
|
||||
}
|
||||
if (journalEntries.containsKey(regionKey)) {
|
||||
RegionShard region = loadRegionLocked(regionKey, regionX(regionKey), regionZ(regionKey));
|
||||
return RegionSummary.from(region);
|
||||
return region.summary();
|
||||
}
|
||||
String shardHash = shardHashes.get(regionKey);
|
||||
if (shardHash == null) {
|
||||
@@ -1051,7 +1051,7 @@ public final class GenerationSemanticIndex {
|
||||
|
||||
private void cacheRegion(long regionKey, RegionShard region) {
|
||||
regions.put(regionKey, region);
|
||||
cacheSummary(regionKey, RegionSummary.from(region));
|
||||
cacheSummary(regionKey, region.summary());
|
||||
while (regions.size() > MAXIMUM_CACHED_REGIONS) {
|
||||
Iterator<Map.Entry<Long, RegionShard>> entries = regions.entrySet().iterator();
|
||||
entries.next();
|
||||
@@ -1663,7 +1663,10 @@ public final class GenerationSemanticIndex {
|
||||
EnumSet<HydrologyFeatureType> riverTypes = EnumSet.noneOf(HydrologyFeatureType.class);
|
||||
LongOpenHashSet activations = new LongOpenHashSet();
|
||||
LongOpenHashSet sealedActivations = new LongOpenHashSet();
|
||||
for (ChunkGenerationSemantics semantics : region.records()) {
|
||||
for (ChunkGenerationSemantics semantics : region.records) {
|
||||
if (semantics == null) {
|
||||
continue;
|
||||
}
|
||||
activations.add(semantics.activationId());
|
||||
mutableKeys(keys, SemanticKind.SURFACE_BIOME).addAll(semantics.surfaceBiomeKeys());
|
||||
mutableKeys(keys, SemanticKind.CAVE_BIOME).addAll(semantics.caveBiomeKeys());
|
||||
@@ -2298,6 +2301,7 @@ public final class GenerationSemanticIndex {
|
||||
private final int regionZ;
|
||||
private final ChunkGenerationSemantics[] records;
|
||||
private int recordCount;
|
||||
private RegionSummary summary;
|
||||
|
||||
private RegionShard(int regionX, int regionZ) {
|
||||
this.regionX = regionX;
|
||||
@@ -2496,7 +2500,7 @@ public final class GenerationSemanticIndex {
|
||||
if (input.available() != 0) {
|
||||
throw invalid(source, "unexpected trailing data");
|
||||
}
|
||||
if (!RegionSummary.from(region).equals(storedSummary)) {
|
||||
if (!region.summary().equals(storedSummary)) {
|
||||
throw invalid(source, "summary does not match semantic records");
|
||||
}
|
||||
return region;
|
||||
@@ -2519,6 +2523,13 @@ public final class GenerationSemanticIndex {
|
||||
return recordCount;
|
||||
}
|
||||
|
||||
private RegionSummary summary() {
|
||||
if (summary == null) {
|
||||
summary = RegionSummary.from(this);
|
||||
}
|
||||
return summary;
|
||||
}
|
||||
|
||||
private ChunkGenerationSemantics get(int chunkX, int chunkZ) {
|
||||
if ((chunkX >> 5) != regionX || (chunkZ >> 5) != regionZ) {
|
||||
return null;
|
||||
@@ -2560,6 +2571,7 @@ public final class GenerationSemanticIndex {
|
||||
}
|
||||
records[index] = semantics;
|
||||
recordCount++;
|
||||
summary = null;
|
||||
}
|
||||
|
||||
private List<ChunkGenerationSemantics> records() {
|
||||
@@ -2632,7 +2644,7 @@ public final class GenerationSemanticIndex {
|
||||
}
|
||||
|
||||
ByteArrayOutputStream bodyBytes = new ByteArrayOutputStream(8_192);
|
||||
byte[] summary = RegionSummary.from(this).encode();
|
||||
byte[] summary = summary().encode();
|
||||
try (DataOutputStream output = new DataOutputStream(bodyBytes)) {
|
||||
output.writeInt(MAGIC);
|
||||
output.writeShort(FORMAT_VERSION);
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
package art.arcane.iris.world.history;
|
||||
|
||||
import java.io.BufferedInputStream;
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.DataInputStream;
|
||||
@@ -8,6 +9,7 @@ import java.io.IOException;
|
||||
import java.io.File;
|
||||
import java.io.RandomAccessFile;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.nio.channels.Channels;
|
||||
import java.nio.channels.FileChannel;
|
||||
import java.nio.file.DirectoryStream;
|
||||
import java.nio.file.Files;
|
||||
@@ -356,8 +358,10 @@ public final class SavedBiomeStore {
|
||||
if (!Files.isRegularFile(region.path, LinkOption.NOFOLLOW_LINKS)) {
|
||||
throw new IOException("Saved biome region is not a regular file: " + region.path);
|
||||
}
|
||||
try (RandomAccessFile input = new RandomAccessFile(region.path.toFile(), "rw")) {
|
||||
long fileLength = input.length();
|
||||
try (RandomAccessFile file = new RandomAccessFile(region.path.toFile(), "rw");
|
||||
DataInputStream input = new DataInputStream(new BufferedInputStream(
|
||||
Channels.newInputStream(file.getChannel()), 65536))) {
|
||||
long fileLength = file.length();
|
||||
if (fileLength < HEADER_BYTES || fileLength > MAXIMUM_REGION_BYTES) {
|
||||
throw new IOException("Invalid saved biome region size: " + region.path);
|
||||
}
|
||||
@@ -371,21 +375,22 @@ public final class SavedBiomeStore {
|
||||
}
|
||||
byte[] buffer = new byte[8192];
|
||||
int records = 0;
|
||||
while (input.getFilePointer() < fileLength) {
|
||||
long offset = input.getFilePointer();
|
||||
long offset = HEADER_BYTES;
|
||||
CRC32 crc = new CRC32();
|
||||
while (offset < fileLength) {
|
||||
if (fileLength - offset < Integer.BYTES) {
|
||||
truncateTail(input, offset);
|
||||
truncateTail(file, offset);
|
||||
break;
|
||||
}
|
||||
int length = input.readInt();
|
||||
if (length < 12 || length > MAXIMUM_RECORD_BYTES) {
|
||||
throw new IOException("Invalid saved biome record size: " + region.path);
|
||||
}
|
||||
if (fileLength - input.getFilePointer() < length + 4L) {
|
||||
truncateTail(input, offset);
|
||||
if (fileLength - offset - Integer.BYTES < length + 4L) {
|
||||
truncateTail(file, offset);
|
||||
break;
|
||||
}
|
||||
CRC32 crc = new CRC32();
|
||||
crc.reset();
|
||||
input.readFully(buffer, 0, 8);
|
||||
crc.update(buffer, 0, 8);
|
||||
ByteBuffer coordinates = ByteBuffer.wrap(buffer, 0, 8);
|
||||
@@ -406,6 +411,7 @@ public final class SavedBiomeStore {
|
||||
}
|
||||
region.offsets[slot] = offset;
|
||||
region.lengths[slot] = length;
|
||||
offset += length + 2L * Integer.BYTES;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -15,6 +15,8 @@ import art.arcane.volmlib.util.nbt.tag.NumberTag;
|
||||
import art.arcane.volmlib.util.nbt.tag.StringTag;
|
||||
import art.arcane.volmlib.util.nbt.tag.Tag;
|
||||
import net.jpountz.lz4.LZ4BlockInputStream;
|
||||
import com.fulcrumgenomics.jlibdeflate.LibdeflateDecompressor;
|
||||
import com.fulcrumgenomics.jlibdeflate.LibdeflateException;
|
||||
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.IOException;
|
||||
@@ -23,6 +25,7 @@ import java.io.RandomAccessFile;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.LinkOption;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.attribute.BasicFileAttributes;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.HashMap;
|
||||
@@ -30,7 +33,8 @@ import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.TreeMap;
|
||||
import java.util.zip.GZIPInputStream;
|
||||
import java.util.zip.InflaterInputStream;
|
||||
import java.util.zip.Inflater;
|
||||
import java.util.zip.DataFormatException;
|
||||
|
||||
final class SavedTerrainChunkReader {
|
||||
private static final int SECTOR_BYTES = 4096;
|
||||
@@ -153,7 +157,9 @@ final class SavedTerrainChunkReader {
|
||||
}
|
||||
|
||||
static String readStatus(Path dimensionRoot, int chunkX, int chunkZ) throws IOException {
|
||||
return string(readRoot(dimensionRoot, chunkX, chunkZ), "Status");
|
||||
try (StatusReader reader = new StatusReader(dimensionRoot)) {
|
||||
return reader.readStatus(chunkX, chunkZ);
|
||||
}
|
||||
}
|
||||
|
||||
private static Section requireSection(Map<Integer, Section> sections, int y) throws IOException {
|
||||
@@ -245,77 +251,9 @@ final class SavedTerrainChunkReader {
|
||||
}
|
||||
|
||||
private static CompoundTag readRoot(Path dimensionRoot, int chunkX, int chunkZ) throws IOException {
|
||||
Path regionDirectory = dimensionRoot.toAbsolutePath().normalize().resolve("region");
|
||||
Path region = regionDirectory.resolve("r." + Math.floorDiv(chunkX, 32) + "."
|
||||
+ Math.floorDiv(chunkZ, 32) + ".mca");
|
||||
if (!Files.isRegularFile(region, LinkOption.NOFOLLOW_LINKS)) {
|
||||
throw new IOException("Saved chunk region is unavailable: " + region);
|
||||
try (StatusReader reader = new StatusReader(dimensionRoot)) {
|
||||
return reader.readRoot(chunkX, chunkZ);
|
||||
}
|
||||
byte[] compressed;
|
||||
int compression;
|
||||
try (RandomAccessFile input = new RandomAccessFile(region.toFile(), "r")) {
|
||||
int slot = Math.floorMod(chunkZ, 32) * 32 + Math.floorMod(chunkX, 32);
|
||||
input.seek((long) slot * 4);
|
||||
int allocation = input.readInt();
|
||||
long offset = (long) (allocation >>> 8) * SECTOR_BYTES;
|
||||
int sectors = allocation & 255;
|
||||
if (offset < 2L * SECTOR_BYTES || sectors == 0 || offset + 5 > input.length()) {
|
||||
throw new IOException("Saved chunk allocation is unavailable: " + chunkX + "," + chunkZ);
|
||||
}
|
||||
input.seek(offset);
|
||||
int length = input.readInt();
|
||||
compression = input.readUnsignedByte();
|
||||
if ((compression & 128) != 0) {
|
||||
if (length != 1) {
|
||||
throw new IOException("External saved chunk has an invalid allocation length");
|
||||
}
|
||||
Path external = regionDirectory.resolve("c." + chunkX + "." + chunkZ + ".mcc");
|
||||
if (!Files.isRegularFile(external, LinkOption.NOFOLLOW_LINKS)
|
||||
|| Files.size(external) > MAX_CHUNK_BYTES) {
|
||||
throw new IOException("External saved chunk is unavailable or too large: " + external);
|
||||
}
|
||||
compressed = Files.readAllBytes(external);
|
||||
compression &= 127;
|
||||
} else {
|
||||
if (length < 1 || length > MAX_CHUNK_BYTES || (long) length + 4 > (long) sectors * SECTOR_BYTES
|
||||
|| offset + 4L + length > input.length()) {
|
||||
throw new IOException("Saved chunk payload is truncated or too large: " + chunkX + "," + chunkZ);
|
||||
}
|
||||
compressed = new byte[length - 1];
|
||||
input.readFully(compressed);
|
||||
}
|
||||
input.seek((long) slot * 4);
|
||||
if (input.readInt() != allocation) {
|
||||
throw new IOException("Saved chunk allocation changed during boundary capture: " + chunkX + "," + chunkZ);
|
||||
}
|
||||
}
|
||||
try (InputStream decoded = decompress(compressed, compression)) {
|
||||
byte[] bytes = decoded.readNBytes(MAX_CHUNK_BYTES + 1);
|
||||
if (bytes.length > MAX_CHUNK_BYTES) {
|
||||
throw new IOException("Saved chunk decompressed payload exceeds the size limit");
|
||||
}
|
||||
Tag<?> tag = NBTUtil.read(new ByteArrayInputStream(bytes), false).getTag();
|
||||
if (!(tag instanceof CompoundTag root)) {
|
||||
throw new IOException("Saved chunk root is not a compound");
|
||||
}
|
||||
if (number(root, "xPos") != chunkX || number(root, "zPos") != chunkZ) {
|
||||
throw new IOException("Saved chunk coordinates do not match its region allocation");
|
||||
}
|
||||
return root;
|
||||
} catch (IllegalArgumentException | ClassCastException exception) {
|
||||
throw new IOException("Invalid saved chunk data at " + chunkX + "," + chunkZ, exception);
|
||||
}
|
||||
}
|
||||
|
||||
private static InputStream decompress(byte[] encoded, int compression) throws IOException {
|
||||
InputStream input = new ByteArrayInputStream(encoded);
|
||||
return switch (compression) {
|
||||
case 1 -> new GZIPInputStream(input);
|
||||
case 2 -> new InflaterInputStream(input);
|
||||
case 3 -> input;
|
||||
case 4 -> new LZ4BlockInputStream(input);
|
||||
default -> throw new IOException("Unsupported saved chunk compression " + compression);
|
||||
};
|
||||
}
|
||||
|
||||
private static long[] packedData(CompoundTag tag) throws IOException {
|
||||
@@ -357,6 +295,268 @@ final class SavedTerrainChunkReader {
|
||||
return value.asInt();
|
||||
}
|
||||
|
||||
static final class StatusReader implements AutoCloseable {
|
||||
private static final boolean NATIVE_AVAILABLE = nativeAvailable();
|
||||
|
||||
private final Path regionDirectory;
|
||||
private final Inflater inflater = new Inflater();
|
||||
private final LibdeflateDecompressor nativeInflater = NATIVE_AVAILABLE ? new LibdeflateDecompressor() : null;
|
||||
private byte[] compressed = new byte[65536];
|
||||
private byte[] decoded = new byte[65536];
|
||||
private Path region;
|
||||
private RandomAccessFile input;
|
||||
private Object regionFileKey;
|
||||
private int regionX;
|
||||
private int regionZ;
|
||||
private boolean closed;
|
||||
|
||||
StatusReader(Path dimensionRoot) {
|
||||
regionDirectory = dimensionRoot.toAbsolutePath().normalize().resolve("region");
|
||||
}
|
||||
|
||||
SavedTerrainChunk readChunk(int chunkX, int chunkZ, int minimumY, int height) throws IOException {
|
||||
return SavedTerrainChunkReader.readRoot(readRoot(chunkX, chunkZ), chunkX, chunkZ, minimumY, height);
|
||||
}
|
||||
|
||||
String readStatus(int chunkX, int chunkZ) throws IOException {
|
||||
try {
|
||||
Payload payload = readPayload(chunkX, chunkZ);
|
||||
return SavedTerrainStatusNbt.read(payload.bytes(), payload.offset(), payload.length(), chunkX, chunkZ);
|
||||
} catch (IOException exception) {
|
||||
throw new IOException("Cannot read saved chunk status at " + chunkX + "," + chunkZ
|
||||
+ " in " + regionDirectory, exception);
|
||||
}
|
||||
}
|
||||
|
||||
String readStatus(byte[] bytes, SavedTerrainRegionSnapshot.Payload stored, int chunkX, int chunkZ)
|
||||
throws IOException {
|
||||
if (closed) {
|
||||
throw new IOException("Saved chunk status reader is closed");
|
||||
}
|
||||
if (stored.external()) {
|
||||
return readStatus(chunkX, chunkZ);
|
||||
}
|
||||
try {
|
||||
Payload payload = decode(stored.compression(), bytes, stored.offset(), stored.length());
|
||||
return SavedTerrainStatusNbt.read(payload.bytes(), payload.offset(), payload.length(), chunkX, chunkZ);
|
||||
} catch (IOException exception) {
|
||||
throw new IOException("Cannot read saved chunk status at " + chunkX + "," + chunkZ
|
||||
+ " in " + regionDirectory, exception);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() throws IOException {
|
||||
if (closed) {
|
||||
return;
|
||||
}
|
||||
closed = true;
|
||||
try {
|
||||
if (input != null) {
|
||||
input.close();
|
||||
}
|
||||
} finally {
|
||||
inflater.end();
|
||||
if (nativeInflater != null) {
|
||||
nativeInflater.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private CompoundTag readRoot(int chunkX, int chunkZ) throws IOException {
|
||||
try {
|
||||
Payload payload = readPayload(chunkX, chunkZ);
|
||||
Tag<?> tag = NBTUtil.read(
|
||||
new ByteArrayInputStream(payload.bytes(), payload.offset(), payload.length()), false).getTag();
|
||||
if (!(tag instanceof CompoundTag root)) {
|
||||
throw new IOException("Saved chunk root is not a compound");
|
||||
}
|
||||
if (number(root, "xPos") != chunkX || number(root, "zPos") != chunkZ) {
|
||||
throw new IOException("Saved chunk coordinates do not match its region allocation");
|
||||
}
|
||||
return root;
|
||||
} catch (IllegalArgumentException | ClassCastException exception) {
|
||||
throw new IOException("Invalid saved chunk data at " + chunkX + "," + chunkZ, exception);
|
||||
}
|
||||
}
|
||||
|
||||
private Payload readPayload(int chunkX, int chunkZ) throws IOException {
|
||||
openRegion(chunkX, chunkZ);
|
||||
int slot = Math.floorMod(chunkZ, 32) * 32 + Math.floorMod(chunkX, 32);
|
||||
input.seek((long) slot * 4);
|
||||
int allocation = input.readInt();
|
||||
long offset = (long) (allocation >>> 8) * SECTOR_BYTES;
|
||||
int sectors = allocation & 255;
|
||||
if (offset < 2L * SECTOR_BYTES || sectors == 0 || offset + 5 > input.length()) {
|
||||
throw new IOException("Saved chunk allocation is unavailable: " + chunkX + "," + chunkZ);
|
||||
}
|
||||
input.seek(offset);
|
||||
int length = input.readInt();
|
||||
int compression = input.readUnsignedByte();
|
||||
int compressedLength;
|
||||
if ((compression & 128) != 0) {
|
||||
if (length != 1) {
|
||||
throw new IOException("External saved chunk has an invalid allocation length");
|
||||
}
|
||||
compressedLength = readExternal(chunkX, chunkZ);
|
||||
compression &= 127;
|
||||
} else {
|
||||
if (length < 1 || length > MAX_CHUNK_BYTES || (long) length + 4 > (long) sectors * SECTOR_BYTES
|
||||
|| offset + 4L + length > input.length()) {
|
||||
throw new IOException("Saved chunk payload is truncated or too large: " + chunkX + "," + chunkZ);
|
||||
}
|
||||
compressedLength = length - 1;
|
||||
ensureCompressedCapacity(compressedLength);
|
||||
input.readFully(compressed, 0, compressedLength);
|
||||
}
|
||||
input.seek((long) slot * 4);
|
||||
if (input.readInt() != allocation) {
|
||||
throw new IOException("Saved chunk allocation changed during boundary capture: " + chunkX + "," + chunkZ);
|
||||
}
|
||||
return decode(compression, compressed, 0, compressedLength);
|
||||
}
|
||||
|
||||
private void openRegion(int chunkX, int chunkZ) throws IOException {
|
||||
if (closed) {
|
||||
throw new IOException("Saved chunk status reader is closed");
|
||||
}
|
||||
int nextRegionX = Math.floorDiv(chunkX, 32);
|
||||
int nextRegionZ = Math.floorDiv(chunkZ, 32);
|
||||
boolean sameRegion = input != null && regionX == nextRegionX && regionZ == nextRegionZ;
|
||||
Path nextRegion = sameRegion ? region
|
||||
: regionDirectory.resolve("r." + nextRegionX + "." + nextRegionZ + ".mca");
|
||||
BasicFileAttributes attributes = Files.readAttributes(nextRegion, BasicFileAttributes.class,
|
||||
LinkOption.NOFOLLOW_LINKS);
|
||||
if (!attributes.isRegularFile()) {
|
||||
throw new IOException("Saved chunk region is unavailable: " + nextRegion);
|
||||
}
|
||||
Object nextFileKey = attributes.fileKey();
|
||||
if (sameRegion && nextFileKey != null && nextFileKey.equals(regionFileKey)) {
|
||||
return;
|
||||
}
|
||||
if (input != null) {
|
||||
input.close();
|
||||
input = null;
|
||||
}
|
||||
input = new RandomAccessFile(nextRegion.toFile(), "r");
|
||||
region = nextRegion;
|
||||
regionFileKey = nextFileKey;
|
||||
regionX = nextRegionX;
|
||||
regionZ = nextRegionZ;
|
||||
}
|
||||
|
||||
private int readExternal(int chunkX, int chunkZ) throws IOException {
|
||||
Path external = regionDirectory.resolve("c." + chunkX + "." + chunkZ + ".mcc");
|
||||
if (!Files.isRegularFile(external, LinkOption.NOFOLLOW_LINKS)) {
|
||||
throw new IOException("External saved chunk is unavailable: " + external);
|
||||
}
|
||||
long length = Files.size(external);
|
||||
if (length > MAX_CHUNK_BYTES) {
|
||||
throw new IOException("External saved chunk is too large: " + external);
|
||||
}
|
||||
int size = (int) length;
|
||||
ensureCompressedCapacity(size);
|
||||
try (InputStream externalInput = Files.newInputStream(external)) {
|
||||
if (externalInput.readNBytes(compressed, 0, size) != size || externalInput.read() != -1) {
|
||||
throw new IOException("External saved chunk length changed during capture: " + external);
|
||||
}
|
||||
}
|
||||
return size;
|
||||
}
|
||||
|
||||
private void ensureCompressedCapacity(int size) {
|
||||
if (size > compressed.length) {
|
||||
compressed = new byte[Math.max(size, Math.min(MAX_CHUNK_BYTES, compressed.length * 2))];
|
||||
}
|
||||
}
|
||||
|
||||
private Payload decode(int compression, byte[] bytes, int offset, int length) throws IOException {
|
||||
if (compression == 2) {
|
||||
int size = inflate(bytes, offset, length);
|
||||
return new Payload(decoded, 0, size);
|
||||
}
|
||||
if (compression == 3) {
|
||||
return new Payload(bytes, offset, length);
|
||||
}
|
||||
InputStream encoded = new ByteArrayInputStream(bytes, offset, length);
|
||||
try (InputStream stream = switch (compression) {
|
||||
case 1 -> new GZIPInputStream(encoded);
|
||||
case 4 -> new LZ4BlockInputStream(encoded);
|
||||
default -> throw new IOException("Unsupported saved chunk compression " + compression);
|
||||
}) {
|
||||
int size = 0;
|
||||
int count;
|
||||
do {
|
||||
ensureDecodedCapacity(size);
|
||||
count = stream.read(decoded, size, decoded.length - size);
|
||||
if (count > 0) {
|
||||
size += count;
|
||||
checkDecodedSize(size);
|
||||
}
|
||||
} while (count != -1);
|
||||
return new Payload(decoded, 0, size);
|
||||
}
|
||||
}
|
||||
|
||||
private int inflate(byte[] bytes, int offset, int length) throws IOException {
|
||||
if (nativeInflater != null) {
|
||||
try {
|
||||
int size = nativeInflater.zlibDecompressEx(bytes, offset, length, decoded, 0, decoded.length)
|
||||
.outputBytesProduced();
|
||||
checkDecodedSize(size);
|
||||
return size;
|
||||
} catch (LibdeflateException exception) {
|
||||
return inflateJava(bytes, offset, length);
|
||||
}
|
||||
}
|
||||
return inflateJava(bytes, offset, length);
|
||||
}
|
||||
|
||||
private static boolean nativeAvailable() {
|
||||
try (LibdeflateDecompressor ignored = new LibdeflateDecompressor()) {
|
||||
return true;
|
||||
} catch (LinkageError | SecurityException exception) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
private int inflateJava(byte[] bytes, int offset, int length) throws IOException {
|
||||
inflater.reset();
|
||||
inflater.setInput(bytes, offset, length);
|
||||
int size = 0;
|
||||
try {
|
||||
while (!inflater.finished()) {
|
||||
ensureDecodedCapacity(size);
|
||||
int count = inflater.inflate(decoded, size, decoded.length - size);
|
||||
size += count;
|
||||
checkDecodedSize(size);
|
||||
if (count == 0 && !inflater.finished()) {
|
||||
throw new IOException("Saved chunk zlib payload is truncated or requires a dictionary");
|
||||
}
|
||||
}
|
||||
} catch (DataFormatException exception) {
|
||||
throw new IOException("Invalid saved chunk zlib payload", exception);
|
||||
}
|
||||
return size;
|
||||
}
|
||||
|
||||
private void ensureDecodedCapacity(int size) throws IOException {
|
||||
checkDecodedSize(size);
|
||||
if (size == decoded.length) {
|
||||
decoded = Arrays.copyOf(decoded, Math.min(MAX_CHUNK_BYTES + 1, decoded.length * 2));
|
||||
}
|
||||
}
|
||||
|
||||
private void checkDecodedSize(int size) throws IOException {
|
||||
if (size > MAX_CHUNK_BYTES) {
|
||||
throw new IOException("Saved chunk decompressed payload exceeds the size limit");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private record Payload(byte[] bytes, int offset, int length) {
|
||||
}
|
||||
|
||||
private record Section(Palette<BoundaryColumnGeometry.Voxel> blocks, Palette<String> biomes) {
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,219 @@
|
||||
package art.arcane.iris.world.history;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Path;
|
||||
import java.util.ArrayList;
|
||||
import java.util.BitSet;
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
import java.util.concurrent.ExecutionException;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.Future;
|
||||
|
||||
final class SavedTerrainRecoveryScan implements AutoCloseable {
|
||||
private static final long WORKER_HEAP_BYTES = 768L * 1024 * 1024;
|
||||
private static final int MAX_REGION_BYTES = 64 * 1024 * 1024;
|
||||
private static final int MIN_PARALLEL_CHUNKS = 128;
|
||||
|
||||
private final Path world;
|
||||
private final int regionByteLimit;
|
||||
private final SavedTerrainChunkReader.StatusReader[] readers;
|
||||
private final ExecutorService executor;
|
||||
private byte[] regionBuffer = new byte[0];
|
||||
|
||||
SavedTerrainRecoveryScan(Path world, Limits limits) {
|
||||
this.world = world;
|
||||
regionByteLimit = limits.regionBytes();
|
||||
readers = new SavedTerrainChunkReader.StatusReader[limits.workers()];
|
||||
for (int index = 0; index < readers.length; index++) {
|
||||
readers[index] = new SavedTerrainChunkReader.StatusReader(world);
|
||||
}
|
||||
executor = readers.length == 1 ? null : Executors.newFixedThreadPool(readers.length,
|
||||
Thread.ofPlatform().daemon().name("Iris terrain recovery ", 1).factory());
|
||||
}
|
||||
|
||||
static WorldChunkInventory scan(Path world, WorldChunkInventory inventory,
|
||||
WorldChunkInventory.ChunkPredicate retained) throws IOException {
|
||||
Runtime runtime = Runtime.getRuntime();
|
||||
try (SavedTerrainRecoveryScan scan = new SavedTerrainRecoveryScan(world,
|
||||
Limits.forRuntime(runtime.availableProcessors(), runtime.maxMemory()))) {
|
||||
return scan.filter(inventory, retained);
|
||||
}
|
||||
}
|
||||
|
||||
WorldChunkInventory filter(WorldChunkInventory inventory, WorldChunkInventory.ChunkPredicate retained)
|
||||
throws IOException {
|
||||
return inventory.filterRegions((regionX, regionZ, allocated) ->
|
||||
filterRegion(new Region(regionX, regionZ, allocated), retained));
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() throws IOException {
|
||||
if (executor != null) {
|
||||
executor.close();
|
||||
}
|
||||
IOException failure = null;
|
||||
for (SavedTerrainChunkReader.StatusReader reader : readers) {
|
||||
try {
|
||||
reader.close();
|
||||
} catch (IOException exception) {
|
||||
if (failure == null) {
|
||||
failure = exception;
|
||||
} else {
|
||||
failure.addSuppressed(exception);
|
||||
}
|
||||
}
|
||||
}
|
||||
if (failure != null) {
|
||||
throw failure;
|
||||
}
|
||||
}
|
||||
|
||||
private BitSet filterRegion(Region region, WorldChunkInventory.ChunkPredicate retained) throws IOException {
|
||||
requireNotInterrupted();
|
||||
BitSet accepted = new BitSet(1024);
|
||||
BitSet pending = (BitSet) region.allocated().clone();
|
||||
for (int slot = pending.nextSetBit(0); slot >= 0; slot = pending.nextSetBit(slot + 1)) {
|
||||
if (retained.test(region.chunkX(slot), region.chunkZ(slot))) {
|
||||
accepted.set(slot);
|
||||
pending.clear(slot);
|
||||
}
|
||||
}
|
||||
if (pending.isEmpty()) {
|
||||
return accepted;
|
||||
}
|
||||
Path file = world.resolve("region/r." + region.x() + "." + region.z() + ".mca");
|
||||
int snapshotLimit = Math.min(regionByteLimit, pending.cardinality() * 64 * 1024);
|
||||
Optional<SavedTerrainRegionSnapshot> captured = SavedTerrainRegionSnapshot.read(file, snapshotLimit, regionBuffer);
|
||||
if (captured.isEmpty()) {
|
||||
for (int slot = pending.nextSetBit(0); slot >= 0; slot = pending.nextSetBit(slot + 1)) {
|
||||
requireNotInterrupted();
|
||||
if (SavedTerrainChunk.hasTerrain(readers[0].readStatus(region.chunkX(slot), region.chunkZ(slot)))) {
|
||||
accepted.set(slot);
|
||||
}
|
||||
}
|
||||
return accepted;
|
||||
}
|
||||
SavedTerrainRegionSnapshot snapshot = captured.get();
|
||||
regionBuffer = snapshot.bytes();
|
||||
for (int slot = pending.nextSetBit(0); slot >= 0; slot = pending.nextSetBit(slot + 1)) {
|
||||
requireNotInterrupted();
|
||||
if (snapshot.payload(slot).external()) {
|
||||
if (SavedTerrainChunk.hasTerrain(readers[0].readStatus(region.chunkX(slot), region.chunkZ(slot)))) {
|
||||
accepted.set(slot);
|
||||
}
|
||||
pending.clear(slot);
|
||||
}
|
||||
}
|
||||
if (executor == null || pending.cardinality() < MIN_PARALLEL_CHUNKS) {
|
||||
accepted.or(decode(new DecodeRegion(region, pending, snapshot), 0, 1));
|
||||
} else {
|
||||
List<Future<BitSet>> tasks = new ArrayList<>(readers.length);
|
||||
DecodeRegion input = new DecodeRegion(region, pending, snapshot);
|
||||
for (int index = 0; index < readers.length; index++) {
|
||||
int worker = index;
|
||||
tasks.add(executor.submit(() -> decode(input, worker, readers.length)));
|
||||
}
|
||||
accepted.or(await(tasks));
|
||||
}
|
||||
snapshot.validateUnchanged();
|
||||
return accepted;
|
||||
}
|
||||
|
||||
private BitSet decode(DecodeRegion input, int worker, int stride) throws IOException {
|
||||
BitSet accepted = new BitSet(1024);
|
||||
int ordinal = 0;
|
||||
for (int slot = input.pending().nextSetBit(0); slot >= 0; slot = input.pending().nextSetBit(slot + 1)) {
|
||||
if (ordinal++ % stride != worker) {
|
||||
continue;
|
||||
}
|
||||
requireNotInterrupted();
|
||||
String status = readers[worker].readStatus(input.snapshot().bytes(), input.snapshot().payload(slot),
|
||||
input.region().chunkX(slot), input.region().chunkZ(slot));
|
||||
if (SavedTerrainChunk.hasTerrain(status)) {
|
||||
accepted.set(slot);
|
||||
}
|
||||
}
|
||||
return accepted;
|
||||
}
|
||||
|
||||
private BitSet await(List<Future<BitSet>> tasks) throws IOException {
|
||||
BitSet accepted = new BitSet(1024);
|
||||
Throwable failure = null;
|
||||
boolean interrupted = false;
|
||||
try {
|
||||
for (Future<BitSet> task : tasks) {
|
||||
boolean completed = false;
|
||||
while (!completed) {
|
||||
try {
|
||||
accepted.or(task.get());
|
||||
completed = true;
|
||||
} catch (InterruptedException exception) {
|
||||
interrupted = true;
|
||||
if (failure == null) {
|
||||
failure = new IOException("Interrupted while reading saved terrain", exception);
|
||||
}
|
||||
} catch (ExecutionException exception) {
|
||||
if (failure == null) {
|
||||
failure = exception.getCause();
|
||||
} else if (failure != exception.getCause()) {
|
||||
failure.addSuppressed(exception.getCause());
|
||||
}
|
||||
completed = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
if (interrupted) {
|
||||
Thread.currentThread().interrupt();
|
||||
}
|
||||
}
|
||||
if (failure instanceof IOException exception) {
|
||||
throw exception;
|
||||
}
|
||||
if (failure instanceof RuntimeException exception) {
|
||||
throw exception;
|
||||
}
|
||||
if (failure instanceof Error error) {
|
||||
throw error;
|
||||
}
|
||||
if (failure != null) {
|
||||
throw new IOException("Saved terrain recovery failed", failure);
|
||||
}
|
||||
return accepted;
|
||||
}
|
||||
|
||||
private void requireNotInterrupted() throws IOException {
|
||||
if (Thread.currentThread().isInterrupted()) {
|
||||
throw new IOException("Saved terrain recovery interrupted");
|
||||
}
|
||||
}
|
||||
|
||||
record Limits(int workers, int regionBytes) {
|
||||
Limits {
|
||||
if (workers < 1 || workers > 4 || regionBytes < 8192 || regionBytes > MAX_REGION_BYTES) {
|
||||
throw new IllegalArgumentException("Invalid saved terrain recovery limits");
|
||||
}
|
||||
}
|
||||
|
||||
static Limits forRuntime(int processors, long heapBytes) {
|
||||
int workers = (int) Math.max(1, Math.min(Math.min(processors, 4), heapBytes / WORKER_HEAP_BYTES));
|
||||
int regionBytes = (int) Math.max(8192, Math.min(MAX_REGION_BYTES, heapBytes / 16));
|
||||
return new Limits(workers, regionBytes);
|
||||
}
|
||||
}
|
||||
|
||||
private record Region(int x, int z, BitSet allocated) {
|
||||
int chunkX(int slot) {
|
||||
return x * 32 + (slot & 31);
|
||||
}
|
||||
|
||||
int chunkZ(int slot) {
|
||||
return z * 32 + (slot >> 5);
|
||||
}
|
||||
}
|
||||
|
||||
private record DecodeRegion(Region region, BitSet pending, SavedTerrainRegionSnapshot snapshot) {
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,138 @@
|
||||
package art.arcane.iris.world.history;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.nio.channels.FileChannel;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.LinkOption;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.StandardOpenOption;
|
||||
import java.nio.file.attribute.BasicFileAttributes;
|
||||
import java.util.Arrays;
|
||||
import java.util.Objects;
|
||||
import java.util.Optional;
|
||||
|
||||
final class SavedTerrainRegionSnapshot {
|
||||
private static final int SECTOR_BYTES = 4_096;
|
||||
private static final int HEADER_BYTES = 2 * SECTOR_BYTES;
|
||||
private static final int MAX_CHUNK_BYTES = 64 * 1024 * 1024;
|
||||
|
||||
private final Path file;
|
||||
private final BasicFileAttributes attributes;
|
||||
private final byte[] bytes;
|
||||
private final int byteLength;
|
||||
|
||||
private SavedTerrainRegionSnapshot(SnapshotData data) {
|
||||
file = data.file();
|
||||
attributes = data.attributes();
|
||||
bytes = data.bytes();
|
||||
byteLength = Math.toIntExact(attributes.size());
|
||||
}
|
||||
|
||||
static Optional<SavedTerrainRegionSnapshot> read(Path file, int maxBytes, byte[] reusable) throws IOException {
|
||||
Objects.requireNonNull(file, "file");
|
||||
Objects.requireNonNull(reusable, "reusable");
|
||||
if (maxBytes < 0) {
|
||||
throw new IllegalArgumentException("Snapshot byte limit must not be negative");
|
||||
}
|
||||
Path normalized = file.toAbsolutePath().normalize();
|
||||
BasicFileAttributes attributes = readAttributes(normalized);
|
||||
if (attributes.size() < HEADER_BYTES) {
|
||||
throw new IOException("Truncated saved chunk region: " + normalized);
|
||||
}
|
||||
if (attributes.size() > maxBytes || attributes.fileKey() == null) {
|
||||
return Optional.empty();
|
||||
}
|
||||
int byteLength = (int) attributes.size();
|
||||
byte[] bytes = reusable.length >= byteLength ? reusable : new byte[byteLength];
|
||||
try (FileChannel input = FileChannel.open(normalized, StandardOpenOption.READ, LinkOption.NOFOLLOW_LINKS)) {
|
||||
readFully(input, ByteBuffer.wrap(bytes, 0, byteLength), normalized);
|
||||
if (input.size() != byteLength) {
|
||||
throw new IOException("Saved chunk region length changed during capture: " + normalized);
|
||||
}
|
||||
}
|
||||
SavedTerrainRegionSnapshot snapshot = new SavedTerrainRegionSnapshot(
|
||||
new SnapshotData(normalized, attributes, bytes));
|
||||
snapshot.validateAttributes();
|
||||
return Optional.of(snapshot);
|
||||
}
|
||||
|
||||
byte[] bytes() {
|
||||
return bytes;
|
||||
}
|
||||
|
||||
Payload payload(int index) throws IOException {
|
||||
if (index < 0 || index >= 1024) {
|
||||
throw new IOException("Saved chunk allocation index is invalid: " + index);
|
||||
}
|
||||
int allocation = readInt(bytes, index * Integer.BYTES);
|
||||
long offset = (long) (allocation >>> Byte.SIZE) * SECTOR_BYTES;
|
||||
int sectors = allocation & 255;
|
||||
if (offset < HEADER_BYTES || sectors == 0 || offset + 5 > byteLength) {
|
||||
throw new IOException("Saved chunk allocation is unavailable in " + file + " at " + index);
|
||||
}
|
||||
int start = (int) offset;
|
||||
int length = readInt(bytes, start);
|
||||
int compression = bytes[start + Integer.BYTES] & 255;
|
||||
if (length < 1 || length > MAX_CHUNK_BYTES || (long) length + Integer.BYTES > (long) sectors * SECTOR_BYTES
|
||||
|| offset + Integer.BYTES + length > byteLength) {
|
||||
throw new IOException("Saved chunk payload is truncated or too large in " + file + " at " + index);
|
||||
}
|
||||
boolean external = (compression & 128) != 0;
|
||||
if (external && length != 1) {
|
||||
throw new IOException("External saved chunk has an invalid allocation length in " + file);
|
||||
}
|
||||
return new Payload(start + 5, length - 1, compression & 127, external);
|
||||
}
|
||||
|
||||
void validateUnchanged() throws IOException {
|
||||
validateAttributes();
|
||||
byte[] header = new byte[HEADER_BYTES];
|
||||
try (FileChannel input = FileChannel.open(file, StandardOpenOption.READ, LinkOption.NOFOLLOW_LINKS)) {
|
||||
readFully(input, ByteBuffer.wrap(header), file);
|
||||
if (input.size() != byteLength) {
|
||||
throw new IOException("Saved chunk region length changed during capture: " + file);
|
||||
}
|
||||
}
|
||||
if (!Arrays.equals(bytes, 0, HEADER_BYTES, header, 0, HEADER_BYTES)) {
|
||||
throw new IOException("Saved chunk region header changed during capture: " + file);
|
||||
}
|
||||
validateAttributes();
|
||||
}
|
||||
|
||||
private static BasicFileAttributes readAttributes(Path file) throws IOException {
|
||||
BasicFileAttributes attributes = Files.readAttributes(file, BasicFileAttributes.class,
|
||||
LinkOption.NOFOLLOW_LINKS);
|
||||
if (!attributes.isRegularFile()) {
|
||||
throw new IOException("Saved chunk region is not a regular file: " + file);
|
||||
}
|
||||
return attributes;
|
||||
}
|
||||
|
||||
private static void readFully(FileChannel input, ByteBuffer buffer, Path file) throws IOException {
|
||||
while (buffer.hasRemaining()) {
|
||||
if (input.read(buffer) < 0) {
|
||||
throw new IOException("Truncated saved chunk region: " + file);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static int readInt(byte[] bytes, int offset) {
|
||||
return (bytes[offset] & 255) << 24 | (bytes[offset + 1] & 255) << 16
|
||||
| (bytes[offset + 2] & 255) << 8 | bytes[offset + 3] & 255;
|
||||
}
|
||||
|
||||
private void validateAttributes() throws IOException {
|
||||
BasicFileAttributes current = readAttributes(file);
|
||||
if (!Objects.equals(attributes.fileKey(), current.fileKey()) || attributes.size() != current.size()
|
||||
|| !attributes.lastModifiedTime().equals(current.lastModifiedTime())) {
|
||||
throw new IOException("Saved chunk region changed during capture: " + file);
|
||||
}
|
||||
}
|
||||
|
||||
record Payload(int offset, int length, int compression, boolean external) {
|
||||
}
|
||||
|
||||
private record SnapshotData(Path file, BasicFileAttributes attributes, byte[] bytes) {
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,225 @@
|
||||
package art.arcane.iris.world.history;
|
||||
|
||||
import art.arcane.volmlib.util.nbt.tag.Tag;
|
||||
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.DataInputStream;
|
||||
import java.io.EOFException;
|
||||
import java.io.IOException;
|
||||
|
||||
final class SavedTerrainStatusNbt {
|
||||
private final byte[] bytes;
|
||||
private final int limit;
|
||||
private int position;
|
||||
|
||||
private SavedTerrainStatusNbt(byte[] bytes, int length) {
|
||||
this.bytes = bytes;
|
||||
this.limit = length;
|
||||
}
|
||||
|
||||
static String read(byte[] bytes, int offset, int length, int chunkX, int chunkZ) throws IOException {
|
||||
if (offset < 0 || length < 0 || offset > bytes.length - length) {
|
||||
throw new IOException("Invalid saved chunk NBT payload bounds");
|
||||
}
|
||||
SavedTerrainStatusNbt input = new SavedTerrainStatusNbt(bytes, offset + length);
|
||||
input.position = offset;
|
||||
if (input.readUnsignedByte() != 10) {
|
||||
throw new IOException("Saved chunk root is not a compound");
|
||||
}
|
||||
input.readUtf();
|
||||
Integer savedX = null;
|
||||
Integer savedZ = null;
|
||||
String status = null;
|
||||
for (int type = input.readUnsignedByte(); type != 0; type = input.readUnsignedByte()) {
|
||||
int start = input.position + 2;
|
||||
int size = input.readUtf();
|
||||
boolean xName = input.matches(start, size, "xPos");
|
||||
boolean zName = input.matches(start, size, "zPos");
|
||||
boolean statusName = input.matches(start, size, "Status");
|
||||
if (xName || zName) {
|
||||
Integer value = null;
|
||||
if (type >= 1 && type <= 6) {
|
||||
value = input.number(type);
|
||||
} else {
|
||||
input.skip(type, Tag.DEFAULT_MAX_DEPTH - 1);
|
||||
}
|
||||
if (xName) {
|
||||
savedX = value;
|
||||
} else {
|
||||
savedZ = value;
|
||||
}
|
||||
} else if (statusName) {
|
||||
if (type == 8) {
|
||||
int valueStart = input.position;
|
||||
int valueLength = input.readUtf();
|
||||
status = new DataInputStream(new ByteArrayInputStream(bytes, valueStart, valueLength + 2)).readUTF();
|
||||
} else {
|
||||
input.skip(type, Tag.DEFAULT_MAX_DEPTH - 1);
|
||||
status = null;
|
||||
}
|
||||
} else {
|
||||
input.skip(type, Tag.DEFAULT_MAX_DEPTH - 1);
|
||||
}
|
||||
}
|
||||
if (savedX == null || savedZ == null || savedX != chunkX || savedZ != chunkZ) {
|
||||
throw new IOException("Saved chunk coordinates do not match its region allocation");
|
||||
}
|
||||
if (status == null || status.isBlank()) {
|
||||
throw new IOException("Saved chunk field Status is not a nonempty string");
|
||||
}
|
||||
return status;
|
||||
}
|
||||
|
||||
private boolean matches(int start, int size, String name) {
|
||||
if (size < name.length() || size > name.length() * 3) {
|
||||
return false;
|
||||
}
|
||||
if (size == name.length()) {
|
||||
for (int index = 0; index < name.length(); index++) {
|
||||
if (bytes[start + index] != name.charAt(index)) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
int end = start + size;
|
||||
for (int index = 0; index < name.length(); index++) {
|
||||
if (start == end) {
|
||||
return false;
|
||||
}
|
||||
int value = bytes[start++] & 255;
|
||||
if ((value & 224) == 192) {
|
||||
value = (value & 31) << 6 | bytes[start++] & 63;
|
||||
} else if ((value & 240) == 224) {
|
||||
value = (value & 15) << 12 | (bytes[start++] & 63) << 6 | bytes[start++] & 63;
|
||||
}
|
||||
if (value != name.charAt(index)) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return start == end;
|
||||
}
|
||||
|
||||
private int number(int type) throws IOException {
|
||||
return switch (type) {
|
||||
case 1 -> (byte) readUnsignedByte();
|
||||
case 2 -> (short) readUnsignedShort();
|
||||
case 3 -> readInt();
|
||||
case 4 -> (int) readLong();
|
||||
case 5 -> (int) Float.intBitsToFloat(readInt());
|
||||
case 6 -> (int) Double.longBitsToDouble(readLong());
|
||||
default -> throw new IOException("Not numeric");
|
||||
};
|
||||
}
|
||||
|
||||
private void require(long count) throws EOFException {
|
||||
if (count < 0 || count > limit - position) {
|
||||
throw new EOFException("Truncated saved chunk NBT payload");
|
||||
}
|
||||
}
|
||||
|
||||
private void advance(long count) throws IOException {
|
||||
require(count);
|
||||
position += (int) count;
|
||||
}
|
||||
|
||||
private int readUnsignedByte() throws IOException {
|
||||
require(1);
|
||||
return bytes[position++] & 255;
|
||||
}
|
||||
|
||||
private int readUnsignedShort() throws IOException {
|
||||
require(2);
|
||||
int result = (bytes[position] & 255) << 8 | bytes[position + 1] & 255;
|
||||
position += 2;
|
||||
return result;
|
||||
}
|
||||
|
||||
private int readInt() throws IOException {
|
||||
require(4);
|
||||
int result = (bytes[position] & 255) << 24 | (bytes[position + 1] & 255) << 16
|
||||
| (bytes[position + 2] & 255) << 8 | bytes[position + 3] & 255;
|
||||
position += 4;
|
||||
return result;
|
||||
}
|
||||
|
||||
private long readLong() throws IOException {
|
||||
return (long) readInt() << 32 | Integer.toUnsignedLong(readInt());
|
||||
}
|
||||
|
||||
private int readUtf() throws IOException {
|
||||
int length = readUnsignedShort();
|
||||
require(length);
|
||||
int end = position + length;
|
||||
while (position < end) {
|
||||
int first = bytes[position++] & 255;
|
||||
if (first < 128) {
|
||||
continue;
|
||||
}
|
||||
if ((first & 224) == 192) {
|
||||
if (position >= end || (bytes[position++] & 192) != 128) {
|
||||
throw new IOException("Invalid saved chunk modified UTF-8");
|
||||
}
|
||||
} else if ((first & 240) == 224) {
|
||||
if (end - position < 2 || (bytes[position++] & 192) != 128 || (bytes[position++] & 192) != 128) {
|
||||
throw new IOException("Invalid saved chunk modified UTF-8");
|
||||
}
|
||||
} else {
|
||||
throw new IOException("Invalid modified UTF-8");
|
||||
}
|
||||
}
|
||||
return length;
|
||||
}
|
||||
|
||||
private void skip(int type, int depth) throws IOException {
|
||||
if (depth < 0) {
|
||||
throw new IOException("Saved chunk NBT nesting limit exceeded");
|
||||
}
|
||||
switch (type) {
|
||||
case 0 -> {
|
||||
}
|
||||
case 1 -> advance(1);
|
||||
case 2 -> advance(2);
|
||||
case 3, 5 -> advance(4);
|
||||
case 4, 6 -> advance(8);
|
||||
case 7, 11, 12 -> {
|
||||
int length = readInt();
|
||||
if (length < 0) {
|
||||
throw new IOException("Negative saved chunk NBT array length");
|
||||
}
|
||||
advance((long) length * (type == 7 ? 1 : type == 11 ? 4 : 8));
|
||||
}
|
||||
case 8 -> readUtf();
|
||||
case 9 -> {
|
||||
int child = readUnsignedByte();
|
||||
int length = readInt();
|
||||
if (length > 0 && child > 12) {
|
||||
throw new IOException("Invalid saved chunk NBT list type");
|
||||
}
|
||||
if (length > 0 && depth == 0) {
|
||||
throw new IOException("Saved chunk NBT nesting limit exceeded");
|
||||
}
|
||||
if (child >= 1 && child <= 6) {
|
||||
int width = switch (child) {
|
||||
case 1 -> 1;
|
||||
case 2 -> 2;
|
||||
case 3, 5 -> 4;
|
||||
default -> 8;
|
||||
};
|
||||
advance((long) Math.max(0, length) * width);
|
||||
} else if (child != 0) {
|
||||
for (int index = 0; index < length; index++) {
|
||||
skip(child, depth - 1);
|
||||
}
|
||||
}
|
||||
}
|
||||
case 10 -> {
|
||||
for (int child = readUnsignedByte(); child != 0; child = readUnsignedByte()) {
|
||||
readUtf();
|
||||
skip(child, depth - 1);
|
||||
}
|
||||
}
|
||||
default -> throw new IOException("Invalid NBT tag type");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -174,6 +174,7 @@ public final class TerrainBoundarySignatureStore {
|
||||
}
|
||||
|
||||
private Snapshot publishCatalog(long activationId, PublishedCatalog catalog) throws IOException {
|
||||
forceDirectory(directory);
|
||||
Path target = snapshotPath(activationId);
|
||||
if (Files.exists(target, LinkOption.NOFOLLOW_LINKS)) {
|
||||
Snapshot existing = load(activationId);
|
||||
@@ -186,6 +187,7 @@ public final class TerrainBoundarySignatureStore {
|
||||
return existing;
|
||||
}
|
||||
publishAtomic(target, catalog.encoded(), "terrain signature catalog");
|
||||
forceDirectory(directory);
|
||||
return load(activationId);
|
||||
}
|
||||
|
||||
@@ -533,7 +535,6 @@ public final class TerrainBoundarySignatureStore {
|
||||
} catch (AtomicMoveNotSupportedException error) {
|
||||
throw new IOException("Terrain boundary " + kind + " publication requires an atomic move", error);
|
||||
}
|
||||
forceDirectory(directory);
|
||||
} finally {
|
||||
Files.deleteIfExists(temporary);
|
||||
}
|
||||
|
||||
@@ -87,6 +87,30 @@ public final class WorldChunkInventory {
|
||||
boolean test(int chunkX, int chunkZ) throws IOException;
|
||||
}
|
||||
|
||||
WorldChunkInventory filterRegions(RegionPredicate predicate) throws IOException {
|
||||
Objects.requireNonNull(predicate, "region predicate");
|
||||
Long2ObjectOpenHashMap<BitSet> selected = new Long2ObjectOpenHashMap<>(regionMasks.size());
|
||||
for (long regionKey : regionKeys) {
|
||||
BitSet source = regionMasks.get(regionKey);
|
||||
BitSet accepted = predicate.test(ChunkGenerationOwnership.chunkX(regionKey),
|
||||
ChunkGenerationOwnership.chunkZ(regionKey), (BitSet) source.clone());
|
||||
BitSet unexpected = (BitSet) accepted.clone();
|
||||
unexpected.andNot(source);
|
||||
if (!unexpected.isEmpty()) {
|
||||
throw new IOException("Filtered world region contains unallocated chunks");
|
||||
}
|
||||
if (!accepted.isEmpty()) {
|
||||
selected.put(regionKey, accepted);
|
||||
}
|
||||
}
|
||||
return new WorldChunkInventory(selected);
|
||||
}
|
||||
|
||||
@FunctionalInterface
|
||||
interface RegionPredicate {
|
||||
BitSet test(int regionX, int regionZ, BitSet allocated) throws IOException;
|
||||
}
|
||||
|
||||
public static boolean isDurablyAllocated(Path worldDirectory, int chunkX, int chunkZ) throws IOException {
|
||||
Path regionDirectory = Objects.requireNonNull(worldDirectory, "worldDirectory")
|
||||
.toAbsolutePath()
|
||||
|
||||
Reference in new issue
Block a user