Testing Jar Packing for minimization

core/src/main/java/art/arcane/iris/platform/bootstrap/NativeRuntimeLibraries.java

@@ -26,6 +26,7 @@ import java.net.URL;
 import java.net.URLClassLoader;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.nio.file.StandardCopyOption;
 import java.security.MessageDigest;
 import java.security.NoSuchAlgorithmException;
 import java.util.ArrayList;
@@ -60,6 +61,11 @@ public final class NativeRuntimeLibraries {
     }
 
     static NativeRuntimeLibraries from(Properties manifest, String minecraftVersion) throws IOException {
+        String storage = manifest.getProperty("storage", "repository");
+        if (!storage.equals("repository") && !storage.equals("embedded")) {
+            throw new IllegalStateException("Invalid native runtime storage: " + storage);
+        }
+        boolean embedded = storage.equals("embedded");
         NativeVersion version = NativeVersion.resolve(minecraftVersion).orElseThrow(
                 () -> new UnsupportedOperationException("No native provider for Minecraft " + minecraftVersion));
         List<String> modules = List.of(manifest.getProperty("modules", "").split(","));
@@ -71,13 +77,14 @@ public final class NativeRuntimeLibraries {
             String[] coordinate = manifest.getProperty(module + ".coordinate", "").split(":");
             String digest = manifest.getProperty(module + ".sha256", "");
             if (coordinate.length != 3 || !coordinate[0].equals("com.github.VolmitSoftware.VolmLib")
-                    || !coordinate[1].equals(module) || coordinate[2].isBlank() || !digest.matches("[0-9a-f]{64}")) {
+                    || !coordinate[1].equals(module) || coordinate[2].isBlank() || !digest.matches("[0-9a-f]{64}")
+                    || embedded && !coordinate[2].equals("embedded-" + digest)) {
                 throw new IllegalStateException("Invalid native dependency manifest entry: " + module);
             }
             selected.put(new Dependency(coordinate[0], coordinate[1], coordinate[2], null, List.of()), digest);
         }
-        return new NativeRuntimeLibraries(Map.copyOf(selected),
-                new Repository(URI.create(manifest.getProperty("repository")).toURL()));
+        Repository repository = embedded ? null : new Repository(URI.create(manifest.getProperty("repository")).toURL());
+        return new NativeRuntimeLibraries(Map.copyOf(selected), repository);
     }
 
     public <T extends ApplicationBuilder<?>> T configure(T builder, Path downloads) {
@@ -111,15 +118,7 @@ public final class NativeRuntimeLibraries {
 
     public URLClassLoader openProviderLoader(Path downloads) {
         try {
-            List<URL> urls = new ArrayList<>(dependencies.size());
-            for (Map.Entry<Dependency, String> entry : dependencies.entrySet()) {
-                Dependency dependency = entry.getKey();
-                Path artifact = downloads.resolve(dependency.groupId().replace('.', '/'))
-                        .resolve(dependency.artifactId()).resolve(dependency.version())
-                        .resolve(dependency.artifactId() + "-" + dependency.version() + ".jar");
-                verify(artifact.toFile(), entry.getValue());
-                urls.add(artifact.toUri().toURL());
-            }
+            List<URL> urls = providerUrls(downloads, NativeRuntimeLibraries.class.getClassLoader());
             URLClassLoader loader = new URLClassLoader("Iris native providers", urls.toArray(URL[]::new),
                     NativeAdapters.class.getClassLoader());
             NativeAdapters.registerProviderLoader(loader);
@@ -130,6 +129,9 @@ public final class NativeRuntimeLibraries {
     }
 
     DependencyData merge(DependencyData original) {
+        if (repository == null) {
+            return original;
+        }
         List<Dependency> combined = new ArrayList<>(original.dependencies());
         combined.addAll(dependencies.keySet());
         List<Repository> repositories = new ArrayList<>(original.repositories());
@@ -139,14 +141,61 @@ public final class NativeRuntimeLibraries {
         return new DependencyData(original.mirrors(), repositories, combined, original.relocations());
     }
 
+    List<URL> providerUrls(Path downloads, ClassLoader resources) throws IOException {
+        List<URL> urls = new ArrayList<>(dependencies.size());
+        for (Map.Entry<Dependency, String> entry : dependencies.entrySet()) {
+            Dependency dependency = entry.getKey();
+            Path artifact = downloads.resolve(dependency.groupId().replace('.', '/'))
+                    .resolve(dependency.artifactId()).resolve(dependency.version())
+                    .resolve(dependency.artifactId() + "-" + dependency.version() + ".jar");
+            if (repository == null) {
+                extractEmbedded(resources, dependency.artifactId(), artifact, entry.getValue());
+            } else {
+                verify(artifact.toFile(), entry.getValue());
+            }
+            urls.add(artifact.toUri().toURL());
+        }
+        return urls;
+    }
+
+    private static void extractEmbedded(ClassLoader resources, String module, Path artifact, String expected) throws IOException {
+        if (Files.isRegularFile(artifact) && digest(artifact).equals(expected)) {
+            return;
+        }
+        String resource = "META-INF/iris/native/" + module + ".jar";
+        try (InputStream input = resources.getResourceAsStream(resource)) {
+            if (input == null) {
+                throw new IOException("Missing embedded native provider: " + resource);
+            }
+            Files.createDirectories(artifact.getParent());
+            Path temporary = Files.createTempFile(artifact.getParent(), module + "-", ".tmp");
+            try {
+                Files.copy(input, temporary, StandardCopyOption.REPLACE_EXISTING);
+                verify(temporary.toFile(), expected);
+                Files.move(temporary, artifact, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING);
+            } finally {
+                Files.deleteIfExists(temporary);
+            }
+        }
+    }
+
+    private static String digest(Path file) throws IOException {
+        try {
+            return HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256").digest(Files.readAllBytes(file)));
+        } catch (NoSuchAlgorithmException exception) {
+            throw new IllegalStateException(exception);
+        }
+    }
+
     static void verify(File file, String expected) {
         try {
-            String actual = HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256").digest(Files.readAllBytes(file.toPath())));
+            String actual = digest(file.toPath());
             if (!actual.equals(expected)) {
                 throw new DownloaderException("Native dependency does not match this Iris build: " + file.getName());
             }
-        } catch (IOException | NoSuchAlgorithmException exception) {
+        } catch (IOException exception) {
             throw new DownloaderException("Cannot verify native dependency: " + file, exception);
         }
     }
+
 }

core/src/test/java/art/arcane/iris/platform/bootstrap/NativeRuntimeLibrariesTest.java

@@ -3,17 +3,39 @@ package art.arcane.iris.platform.bootstrap;
 import io.github.slimjar.exceptions.DownloaderException;
 import io.github.slimjar.resolver.data.Dependency;
 import io.github.slimjar.resolver.data.DependencyData;
+import org.junit.Rule;
 import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
 
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.net.URI;
+import java.net.URL;
+import java.net.URLClassLoader;
+import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.nio.file.attribute.FileTime;
+import java.security.MessageDigest;
+import java.util.Arrays;
+import java.util.HexFormat;
 import java.util.List;
 import java.util.Properties;
+import java.util.jar.JarEntry;
+import java.util.jar.JarOutputStream;
+import java.util.stream.Stream;
 
+import static org.junit.Assert.assertArrayEquals;
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertSame;
 import static org.junit.Assert.assertThrows;
+import static org.junit.Assert.assertTrue;
 
 public class NativeRuntimeLibrariesTest {
+    @Rule
+    public TemporaryFolder temporary = new TemporaryFolder();
+
     @Test
     public void selectsOnlyCommonAndMatchingProvider() throws Exception {
         Properties manifest = manifest();
@@ -39,6 +61,115 @@ public class NativeRuntimeLibrariesTest {
         }
     }
 
+    @Test
+    public void embeddedProvidersLoadWithoutRepositoryAndSelectOnlyCurrentVersion() throws Exception {
+        EmbeddedFixture fixture = embeddedFixture();
+        NativeRuntimeLibraries libraries = NativeRuntimeLibraries.from(fixture.manifest(), "26.2.0");
+        Dependency original = new Dependency("example", "library", "1", null, List.of());
+        DependencyData data = new DependencyData(List.of(), List.of(), List.of(original), List.of());
+        assertSame(data, libraries.merge(data));
+        Path downloads = temporary.newFolder("downloads").toPath();
+        try (URLClassLoader resources = new URLClassLoader(new URL[]{fixture.resources().toUri().toURL()}, null)) {
+            List<URL> providers = libraries.providerUrls(downloads, resources);
+            assertEquals(2, providers.size());
+            assertEquals(List.of("native-common", "native-v26_2_R1"), providers.stream()
+                    .map(url -> Path.of(URI.create(url.toString())).getParent().getParent().getFileName().toString())
+                    .sorted().toList());
+            for (URL provider : providers) {
+                Path artifact = Path.of(provider.toURI());
+                String module = artifact.getParent().getParent().getFileName().toString();
+                assertArrayEquals(Files.readAllBytes(fixture.resources().resolve("META-INF/iris/native/" + module + ".jar")),
+                        Files.readAllBytes(artifact));
+                FileTime timestamp = FileTime.fromMillis(1000);
+                Files.setLastModifiedTime(artifact, timestamp);
+                libraries.providerUrls(downloads, resources);
+                assertEquals(timestamp, Files.getLastModifiedTime(artifact));
+            }
+        }
+    }
+
+    @Test
+    public void embeddedProvidersRepairCorruptCachedBytes() throws Exception {
+        EmbeddedFixture fixture = embeddedFixture();
+        NativeRuntimeLibraries libraries = NativeRuntimeLibraries.from(fixture.manifest(), "26.2.0");
+        Path downloads = temporary.newFolder("downloads").toPath();
+        try (URLClassLoader resources = new URLClassLoader(new URL[]{fixture.resources().toUri().toURL()}, null)) {
+            List<URL> providers = libraries.providerUrls(downloads, resources);
+            Path corrupted = Path.of(providers.getFirst().toURI());
+            byte[] expected = Files.readAllBytes(corrupted);
+            Files.writeString(corrupted, "truncated");
+            libraries.providerUrls(downloads, resources);
+            assertArrayEquals(expected, Files.readAllBytes(corrupted));
+        }
+    }
+
+    @Test
+    public void embeddedProvidersRejectMissingResourceWithoutCachingIt() throws Exception {
+        EmbeddedFixture fixture = embeddedFixture();
+        Files.delete(fixture.resources().resolve("META-INF/iris/native/native-v26_2_R1.jar"));
+        NativeRuntimeLibraries libraries = NativeRuntimeLibraries.from(fixture.manifest(), "26.2.0");
+        Path downloads = temporary.newFolder("downloads").toPath();
+        try (URLClassLoader resources = new URLClassLoader(new URL[]{fixture.resources().toUri().toURL()}, null)) {
+            IOException failure = assertThrows(IOException.class, () -> libraries.providerUrls(downloads, resources));
+            assertTrue(failure.getMessage().contains("native-v26_2_R1"));
+            assertNoFailedExtract(downloads);
+        }
+    }
+
+    @Test
+    public void embeddedProvidersRejectTruncatedResourceWithoutCachingIt() throws Exception {
+        EmbeddedFixture fixture = embeddedFixture();
+        Path resource = fixture.resources().resolve("META-INF/iris/native/native-v26_2_R1.jar");
+        byte[] bytes = Files.readAllBytes(resource);
+        Files.write(resource, Arrays.copyOf(bytes, bytes.length / 2));
+        NativeRuntimeLibraries libraries = NativeRuntimeLibraries.from(fixture.manifest(), "26.2.0");
+        Path downloads = temporary.newFolder("downloads").toPath();
+        try (URLClassLoader resources = new URLClassLoader(new URL[]{fixture.resources().toUri().toURL()}, null)) {
+            DownloaderException failure = assertThrows(DownloaderException.class, () -> libraries.providerUrls(downloads, resources));
+            assertTrue(failure.getMessage().contains("native-v26_2_R1"));
+            assertNoFailedExtract(downloads);
+        }
+    }
+
+    @Test
+    public void embeddedCoordinatesMustMatchTheirContentHash() throws Exception {
+        EmbeddedFixture fixture = embeddedFixture();
+        fixture.manifest().setProperty("native-common.coordinate", "com.github.VolmitSoftware.VolmLib:native-common:revision");
+        assertThrows(IllegalStateException.class, () -> NativeRuntimeLibraries.from(fixture.manifest(), "26.2.0"));
+    }
+
+    private EmbeddedFixture embeddedFixture() throws Exception {
+        Properties properties = manifest();
+        properties.setProperty("storage", "embedded");
+        properties.remove("repository");
+        Path resources = temporary.newFolder("resources").toPath();
+        Path providers = Files.createDirectories(resources.resolve("META-INF/iris/native"));
+        for (String module : properties.getProperty("modules").split(",")) {
+            ByteArrayOutputStream output = new ByteArrayOutputStream();
+            try (JarOutputStream jar = new JarOutputStream(output)) {
+                jar.putNextEntry(new JarEntry("provider.txt"));
+                jar.write(module.getBytes(StandardCharsets.UTF_8));
+                jar.closeEntry();
+            }
+            byte[] bytes = output.toByteArray();
+            String hash = HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256").digest(bytes));
+            properties.setProperty(module + ".coordinate", "com.github.VolmitSoftware.VolmLib:" + module + ":embedded-" + hash);
+            properties.setProperty(module + ".sha256", hash);
+            Files.write(providers.resolve(module + ".jar"), bytes);
+        }
+        return new EmbeddedFixture(properties, resources);
+    }
+
+    private static void assertNoFailedExtract(Path downloads) throws IOException {
+        try (Stream<Path> files = Files.walk(downloads)) {
+            assertFalse(files.anyMatch(path -> Files.isRegularFile(path)
+                    && (path.getFileName().toString().endsWith(".tmp") || path.getFileName().toString().startsWith("native-v26_2_R1-"))));
+        }
+    }
+
+    private record EmbeddedFixture(Properties manifest, Path resources) {
+    }
+
     private static Properties manifest() {
         Properties properties = new Properties();
         List<String> modules = List.of("native-common", "native-v26_2_R1", "native-v26_3_R1");