diff --git a/buildSrc/build.gradle b/buildSrc/build.gradle index 2046a409d..1f878dab1 100644 --- a/buildSrc/build.gradle +++ b/buildSrc/build.gradle @@ -21,6 +21,7 @@ repositories { dependencies { implementation(gradleApi()) implementation('org.ow2.asm:asm:9.10.1') + implementation('org.apache.commons:commons-compress:1.28.0') implementation('com.github.VolmitSoftware:NMSTools:c88961416f') implementation('io.papermc.paperweight:paperweight-userdev:2.0.0-beta.21') testImplementation('junit:junit:4.13.2') diff --git a/buildSrc/src/main/java/JarCompactor.java b/buildSrc/src/main/java/JarCompactor.java index 5ee5388b1..1906ea8fe 100644 --- a/buildSrc/src/main/java/JarCompactor.java +++ b/buildSrc/src/main/java/JarCompactor.java @@ -1,18 +1,22 @@ +import org.apache.commons.compress.archivers.zip.ExtraFieldUtils; +import org.apache.commons.compress.archivers.zip.Zip64Mode; +import org.apache.commons.compress.archivers.zip.ZipArchiveEntry; +import org.apache.commons.compress.archivers.zip.ZipArchiveOutputStream; import org.gradle.api.GradleException; -import java.io.BufferedInputStream; -import java.io.BufferedOutputStream; import java.io.File; import java.io.IOException; import java.io.InputStream; -import java.io.OutputStream; import java.nio.file.AtomicMoveNotSupportedException; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.StandardCopyOption; +import java.util.Enumeration; +import java.util.HashSet; +import java.util.Set; +import java.util.zip.CRC32; import java.util.zip.ZipEntry; -import java.util.zip.ZipInputStream; -import java.util.zip.ZipOutputStream; +import java.util.zip.ZipFile; public final class JarCompactor { private static final int BUFFER_BYTES = 64 * 1024; @@ -45,31 +49,50 @@ public final class JarCompactor { private static void rewrite(Path source, Path destination) throws IOException { byte[] buffer = new byte[BUFFER_BYTES]; - try (InputStream rawInput = new BufferedInputStream(Files.newInputStream(source)); - ZipInputStream input = new ZipInputStream(rawInput); - OutputStream rawOutput = new BufferedOutputStream(Files.newOutputStream(destination)); - ZipOutputStream output = new ZipOutputStream(rawOutput)) { + try (ZipFile input = new ZipFile(source.toFile()); + ZipArchiveOutputStream output = new ZipArchiveOutputStream(destination)) { + if (!output.isSeekable()) { + throw new IOException("Jar compaction requires seekable output."); + } output.setLevel(9); - ZipEntry entry; - while ((entry = input.getNextEntry()) != null) { + output.setUseZip64(Zip64Mode.Never); + output.setCreateUnicodeExtraFields(ZipArchiveOutputStream.UnicodeExtraFieldPolicy.NEVER); + Set names = new HashSet<>(); + Enumeration entries = input.entries(); + while (entries.hasMoreElements()) { + ZipEntry entry = entries.nextElement(); if (entry.isDirectory()) { continue; } - ZipEntry compacted = copyMetadata(entry); - output.putNextEntry(compacted); - int read; - while ((read = input.read(buffer)) >= 0) { - if (read > 0) { - output.write(buffer, 0, read); - } + if (!names.add(entry.getName())) { + throw new IOException("Duplicate jar entry: " + entry.getName()); } - output.closeEntry(); + output.putArchiveEntry(copyMetadata(entry)); + copyEntry(input, output, entry, buffer); + output.closeArchiveEntry(); } } } - private static ZipEntry copyMetadata(ZipEntry source) { - ZipEntry target = new ZipEntry(source.getName()); + private static void copyEntry(ZipFile input, ZipArchiveOutputStream output, ZipEntry entry, byte[] buffer) + throws IOException { + CRC32 crc = new CRC32(); + long size = 0L; + try (InputStream content = input.getInputStream(entry)) { + int read; + while ((read = content.read(buffer)) != -1) { + output.write(buffer, 0, read); + crc.update(buffer, 0, read); + size += read; + } + } + if (size != entry.getSize() || crc.getValue() != entry.getCrc()) { + throw new IOException("Jar entry failed size or CRC verification: " + entry.getName()); + } + } + + private static ZipArchiveEntry copyMetadata(ZipEntry source) throws IOException { + ZipArchiveEntry target = new ZipArchiveEntry(source.getName()); target.setMethod(ZipEntry.DEFLATED); if (source.getTime() >= 0L) { target.setTime(source.getTime()); @@ -78,7 +101,8 @@ public final class JarCompactor { target.setComment(source.getComment()); } if (source.getExtra() != null) { - target.setExtra(source.getExtra()); + target.setExtraFields(ExtraFieldUtils.parse(source.getExtra(), true, + ZipArchiveEntry.ExtraFieldParsingMode.BEST_EFFORT)); } return target; } diff --git a/buildSrc/src/test/java/JarCompactorTest.java b/buildSrc/src/test/java/JarCompactorTest.java index ffd5c2704..ac1763ded 100644 --- a/buildSrc/src/test/java/JarCompactorTest.java +++ b/buildSrc/src/test/java/JarCompactorTest.java @@ -1,16 +1,33 @@ +import org.apache.commons.compress.archivers.zip.ZipArchiveEntry; +import org.apache.commons.compress.archivers.zip.ZipArchiveOutputStream; +import org.apache.commons.compress.archivers.zip.ZipFile; +import org.gradle.api.GradleException; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; import java.io.File; import java.io.FileOutputStream; +import java.io.InputStream; import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.security.MessageDigest; +import java.util.Enumeration; +import java.util.HexFormat; +import java.util.Map; +import java.util.TreeMap; import java.util.jar.JarEntry; import java.util.jar.JarFile; import java.util.jar.JarOutputStream; +import java.util.zip.CRC32; +import java.util.zip.ZipEntry; import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; public class JarCompactorTest { @Rule @@ -36,4 +53,128 @@ public class JarCompactorTest { jar.getJarEntry("example/value.txt")).readAllBytes()); } } + + @Test + public void preservesJarIdentityAndProducesDeterministicSeekableOutput() throws Exception { + File artifact = temporaryFolder.newFile("identity.jar"); + try (JarOutputStream output = new JarOutputStream(new FileOutputStream(artifact)); + InputStream classResource = getClass().getResourceAsStream("/JarCompactorTest.class")) { + output.setLevel(9); + writeEntry(output, "META-INF/MANIFEST.MF", "Manifest-Version: 1.0\r\nMulti-Release: true\r\n\r\n" + .getBytes(StandardCharsets.UTF_8)); + writeEntry(output, "JarCompactorTest.class", classResource.readAllBytes()); + writeEntry(output, "META-INF/iris/generation-kernels/abi-1.seal", "registry fixture".getBytes(StandardCharsets.UTF_8)); + writeEntry(output, "example/caf\u00e9.txt", "unicode name".getBytes(StandardCharsets.UTF_8)); + for (int index = 0; index < 100; index++) { + writeEntry(output, "example/item-" + index + ".txt", ("value-" + index).getBytes(StandardCharsets.UTF_8)); + } + } + Map original = identity(artifact); + long originalBytes = artifact.length(); + + JarCompactor.compact(artifact); + + assertEquals(original, identity(artifact)); + assertTrue(artifact.length() < originalBytes); + try (JarFile jar = new JarFile(artifact)) { + assertEquals("true", jar.getManifest().getMainAttributes().getValue("Multi-Release")); + assertEquals(0xCA, jar.getInputStream(jar.getJarEntry("JarCompactorTest.class")).read()); + } + try (ZipFile zip = ZipFile.builder().setFile(artifact).get()) { + Enumeration entries = zip.getEntries(); + while (entries.hasMoreElements()) { + assertFalse(entries.nextElement().getGeneralPurposeBit().usesDataDescriptor()); + } + } + byte[] first = Files.readAllBytes(artifact.toPath()); + JarCompactor.compact(artifact); + assertArrayEquals(first, Files.readAllBytes(artifact.toPath())); + } + + @Test + public void rejectsMalformedArchivesWithoutReplacingTheSource() throws Exception { + File artifact = temporaryFolder.newFile("invalid.jar"); + byte[] original = "not a zip archive".getBytes(StandardCharsets.UTF_8); + Files.write(artifact.toPath(), original); + + assertThrows(GradleException.class, () -> JarCompactor.compact(artifact)); + + assertArrayEquals(original, Files.readAllBytes(artifact.toPath())); + assertEquals(1, temporaryFolder.getRoot().listFiles().length); + } + + @Test + public void rejectsCorruptPayloadWithoutReplacingTheSource() throws Exception { + File artifact = temporaryFolder.newFile("corrupt.jar"); + byte[] payload = "checked-payload".getBytes(StandardCharsets.UTF_8); + CRC32 crc = new CRC32(); + crc.update(payload); + try (JarOutputStream output = new JarOutputStream(new FileOutputStream(artifact))) { + JarEntry entry = new JarEntry("value.txt"); + entry.setMethod(ZipEntry.STORED); + entry.setSize(payload.length); + entry.setCrc(crc.getValue()); + output.putNextEntry(entry); + output.write(payload); + output.closeEntry(); + } + byte[] corrupt = Files.readAllBytes(artifact.toPath()); + int payloadOffset = new String(corrupt, StandardCharsets.ISO_8859_1).indexOf("checked-payload"); + assertTrue(payloadOffset > 0); + corrupt[payloadOffset] ^= 1; + Files.write(artifact.toPath(), corrupt); + + assertThrows(GradleException.class, () -> JarCompactor.compact(artifact)); + + assertArrayEquals(corrupt, Files.readAllBytes(artifact.toPath())); + } + + @Test + public void rejectsDuplicateEntriesWithoutReplacingTheSource() throws Exception { + File artifact = temporaryFolder.newFile("duplicate.jar"); + try (ZipArchiveOutputStream output = new ZipArchiveOutputStream(artifact)) { + for (int index = 0; index < 2; index++) { + output.putArchiveEntry(new ZipArchiveEntry("duplicate.txt")); + output.write(index); + output.closeArchiveEntry(); + } + } + byte[] original = Files.readAllBytes(artifact.toPath()); + + assertThrows(GradleException.class, () -> JarCompactor.compact(artifact)); + + assertArrayEquals(original, Files.readAllBytes(artifact.toPath())); + } + + private static void writeEntry(JarOutputStream output, String name, byte[] content) throws Exception { + JarEntry entry = new JarEntry(name); + entry.setTime(315_532_800_000L); + entry.setComment("preserved entry comment"); + entry.setExtra(new byte[]{(byte) 0xFE, (byte) 0xCA, 1, 0, 7}); + output.putNextEntry(entry); + output.write(content); + output.closeEntry(); + } + + private static Map identity(File artifact) throws Exception { + Map entries = new TreeMap<>(); + try (JarFile jar = new JarFile(artifact)) { + Enumeration contents = jar.entries(); + while (contents.hasMoreElements()) { + JarEntry entry = contents.nextElement(); + byte[] bytes; + try (InputStream input = jar.getInputStream(entry)) { + bytes = input.readAllBytes(); + } + entries.put(entry.getName(), new EntryIdentity( + HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256").digest(bytes)), + entry.getSize(), entry.getCrc(), entry.getTime(), entry.getComment(), + entry.getExtra() == null ? "" : HexFormat.of().formatHex(entry.getExtra()))); + } + } + return entries; + } + + private record EntryIdentity(String sha256, long size, long crc, long time, String comment, String extra) { + } }