diff --git a/adapters/forge/build.gradle b/adapters/forge/build.gradle index e7171ebd1..b8a3c8f35 100644 --- a/adapters/forge/build.gradle +++ b/adapters/forge/build.gradle @@ -17,6 +17,7 @@ */ import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar +import net.minecraftforge.gradle.shadow.net.minecraftforge.gradleutils.shared.EnhancedPlugin import org.gradle.api.file.DeleteSpec import org.gradle.api.file.FileCopyDetails import org.gradle.api.file.FileSystemOperations @@ -62,6 +63,14 @@ file('../../gradle.properties').withInputStream { InputStream stream -> rootProp String irisVersion = providers.gradleProperty('irisVersion').getOrElse(rootProperties.getProperty('irisVersion', '4.3.5')) String minecraftVersion = providers.gradleProperty('minecraftVersion').getOrElse(rootProperties.getProperty('minecraftVersion', '26.3')) String forgeVersion = providers.gradleProperty('forgeVersion').getOrElse(irisModTarget.forge.toString()) +String buildLane = gradle.extensions.extraProperties.has('irisBuildLane') + ? gradle.extensions.extraProperties.get('irisBuildLane').toString() : null +if (buildLane != null) { + plugins.withType(EnhancedPlugin).configureEach { EnhancedPlugin forgePlugin -> + forgePlugin.globalCaches().set(layout.projectDirectory.dir(".gradle/iris-lanes/${buildLane}/forgegradle")) + } + minecraft.mavenizerOutput.set(layout.projectDirectory.dir(".gradle/iris-lanes/${buildLane}/mavenizer/repo")) +} Closure loaderDisplayVersion = { String loaderVersion -> String coordinatePrefix = "${minecraftVersion}-" if (loaderVersion.startsWith(coordinatePrefix)) { @@ -161,7 +170,9 @@ configurations.testRuntimeClasspath.extendsFrom(configurations.runtimeClasspath, } dependencies { - implementation minecraft.dependency("net.minecraftforge:forge:${forgeVersion}") + implementation(buildLane == null + ? minecraft.dependency("net.minecraftforge:forge:${forgeVersion}") + : minecraft.dependency("iris-${buildLane}", "net.minecraftforge:forge:${forgeVersion}")) testImplementation('junit:junit:4.13.2') testImplementation('org.mockito:mockito-core:5.23.0') testImplementation(testFixtures('art.arcane:core')) diff --git a/adapters/modded-common/src/main/java/art/arcane/iris/modded/command/ModdedPregenMethod.java b/adapters/modded-common/src/main/java/art/arcane/iris/modded/command/ModdedPregenMethod.java index df9b737dd..9d1df42f9 100644 --- a/adapters/modded-common/src/main/java/art/arcane/iris/modded/command/ModdedPregenMethod.java +++ b/adapters/modded-common/src/main/java/art/arcane/iris/modded/command/ModdedPregenMethod.java @@ -30,6 +30,7 @@ import art.arcane.iris.generation.runtime.Engine; import art.arcane.iris.modded.ModdedGenPool; import art.arcane.volmlib.util.mantle.runtime.Mantle; +import java.util.Objects; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; import java.util.concurrent.Semaphore; @@ -39,6 +40,7 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; +import java.util.function.LongSupplier; public final class ModdedPregenMethod implements PregeneratorMethod { private static final int ADAPTIVE_TIMEOUT_STEP = 3; @@ -68,8 +70,14 @@ public final class ModdedPregenMethod implements PregeneratorMethod { private final int timeoutSeconds; private final PregenMantleBackpressure backpressure; private final PauseWhenEmptyGuard pauseGuard; + private final LongSupplier clock; public ModdedPregenMethod(Configuration configuration) { + this(configuration, System::nanoTime); + } + + ModdedPregenMethod(Configuration configuration, LongSupplier clock) { + this.clock = Objects.requireNonNull(clock); runtime = configuration.runtime(); engine = configuration.engine(); sync = configuration.sync(); @@ -178,7 +186,7 @@ public final class ModdedPregenMethod implements PregeneratorMethod { if (sync || runtime.isServerThread()) { return; } - long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(timeoutSeconds) + long deadline = clock.getAsLong() + TimeUnit.SECONDS.toNanos(timeoutSeconds) + TimeUnit.MILLISECONDS.toNanos(FINAL_SAVE_TIMEOUT_MILLIS); boolean interrupted = false; try { @@ -186,7 +194,7 @@ public final class ModdedPregenMethod implements PregeneratorMethod { if (finalSaveOnServerThread.get() || !runtime.isRunning()) { return; } - if (System.nanoTime() >= deadline) { + if (clock.getAsLong() >= deadline) { throw new IllegalStateException("Iris pregen chunk callbacks did not finish for " + runtime.worldIdentity() + ": " + inFlight.get() + " pending"); } @@ -263,14 +271,14 @@ public final class ModdedPregenMethod implements PregeneratorMethod { } private void awaitFinalSave(FinalSaveRequest request) { - long deadline = System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(FINAL_SAVE_TIMEOUT_MILLIS); + long deadline = clock.getAsLong() + TimeUnit.MILLISECONDS.toNanos(FINAL_SAVE_TIMEOUT_MILLIS); boolean interrupted = false; try { while (true) { if (deferFinalSaveIfRequested()) { return; } - long remainingNanos = deadline - System.nanoTime(); + long remainingNanos = deadline - clock.getAsLong(); if (remainingNanos <= 0L) { ModdedIrisLog.warn("Iris pregen level save did not complete in time for {}", runtime.worldIdentity()); return; diff --git a/adapters/modded-common/src/test/java/art/arcane/iris/modded/command/ModdedPregenMethodLifecycleTest.java b/adapters/modded-common/src/test/java/art/arcane/iris/modded/command/ModdedPregenMethodLifecycleTest.java index 137c7718b..5e6447dac 100644 --- a/adapters/modded-common/src/test/java/art/arcane/iris/modded/command/ModdedPregenMethodLifecycleTest.java +++ b/adapters/modded-common/src/test/java/art/arcane/iris/modded/command/ModdedPregenMethodLifecycleTest.java @@ -21,6 +21,7 @@ import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; import static org.junit.Assert.assertEquals; @@ -97,6 +98,9 @@ public class ModdedPregenMethodLifecycleTest { @Test public void finalSaveWaitsForEveryCallbackBeyondFiveSeconds() throws Exception { + AtomicLong clock = new AtomicLong(); + CountDownLatch started = new CountDownLatch(1); + CountDownLatch advanced = new CountDownLatch(2); CompletableFuture load = new CompletableFuture<>(); CompletableFuture failedLoad = new CompletableFuture<>(); PregenListener listener = mock(PregenListener.class); @@ -110,16 +114,22 @@ public class ModdedPregenMethodLifecycleTest { }).when(runtime).execute(any()); try (MockedStatic pressure = mockStatic(MantleHeapPressure.class)) { pressure.when(MantleHeapPressure::memoryBudgetBytes).thenReturn(512L << 20); - method = new ModdedPregenMethod(new ModdedPregenMethod.Configuration(runtime, engine, false)); + method = new ModdedPregenMethod(new ModdedPregenMethod.Configuration(runtime, engine, false), () -> { + long now = clock.get(); + if (now >= TimeUnit.SECONDS.toNanos(6)) { + advanced.countDown(); + } else { + started.countDown(); + } + return now; + }); method.generateChunk(2, 3, listener); method.generateChunk(4, 5, listener); - CountDownLatch started = new CountDownLatch(1); - Future closing = executor.submit(() -> { - started.countDown(); - method.close(); - }); + Future closing = executor.submit(method::close); assertTrue(started.await(5, TimeUnit.SECONDS)); - assertThrows(TimeoutException.class, () -> closing.get(6, TimeUnit.SECONDS)); + clock.set(TimeUnit.SECONDS.toNanos(6)); + assertTrue(advanced.await(5, TimeUnit.SECONDS)); + assertFalse(closing.isDone()); verify(runtime, never()).save(); verify(engine, never()).syncGenerationHistory(); load.complete(new ChunkLoad(true, null)); diff --git a/build.gradle b/build.gradle index e2799e5cd..421011946 100644 --- a/build.gradle +++ b/build.gradle @@ -329,7 +329,12 @@ List moddedSharedInputs = [] 'gradle/mod-target.gradle', 'gradle/mod-targets.json', 'gradle/runtime-cache.settings.gradle', - 'gradle/modded-version-sources.gradle' + 'gradle/nested-build.init.gradle', + 'gradle/modded-version-sources.gradle', + 'gradle/wrapper/gradle-wrapper.properties', + 'gradle/wrapper/gradle-wrapper.jar', + 'gradlew', + 'gradlew.bat' ].each { String path -> moddedSharedInputs.add(layout.projectDirectory.file(path).asFile) } if (localVolmLibCheckout != null) { moddedSharedInputs.add(fileTree(localVolmLibCheckout) { @@ -343,11 +348,11 @@ if (localVolmLibCheckout != null) { // The published master-SNAPSHOT is resolved inside the nested build, so no outer input changes when it // moves. Without a local checkout the Exec tasks always run and the nested build decides what is stale. -Closure declareVolmLibInputs = { Exec task -> +Closure declareVolmLibInputs = { IrisModdedBuild task -> task.inputs.property('volmLibStamp', volmLibStamp) String nativeBillow = providers.gradleProperty('includeNativeBillow').getOrElse('false') task.inputs.property('includeNativeBillow', nativeBillow) - task.environment('ORG_GRADLE_PROJECT_includeNativeBillow', nativeBillow) + task.environment.put('ORG_GRADLE_PROJECT_includeNativeBillow', nativeBillow) if (localVolmLibCheckout == null) { task.outputs.upToDateWhen { false } } @@ -413,9 +418,29 @@ TaskProvider verifyModdedMatrix = tasks.register('verifyModdedMatrix') { description = 'Builds, tests and verifies the Fabric, Forge and NeoForge jars of every supported Minecraft mod target.' } List> moddedExports = [] -Map> selectedModdedBuilds = [:] +Map> selectedModdedBuilds = [:] Map> selectedModdedExports = [:] -TaskProvider previousMatrixBuild = null +int matrixParallelism = Math.min(gradle.startParameter.maxWorkerCount, Math.max(1, + providers.gradleProperty('moddedBuildParallelism').getOrElse( + Math.min(12, Math.max(1, Runtime.runtime.availableProcessors() - 4)).toString()).toInteger())) +Provider matrixWorkers = gradle.sharedServices.registerIfAbsent( + 'irisModdedBuilds', IrisModdedBuildService) { + maxParallelUsages.set(matrixParallelism) +} +List nestedBuildArguments = [gradle.startParameter.buildCacheEnabled ? '--build-cache' : '--no-build-cache'] +String nestedJvmArguments = providers.gradleProperty('org.gradle.jvmargs').getOrElse('') + + ' -XX:ActiveProcessorCount=2' +if (gradle.startParameter.rerunTasks) { + nestedBuildArguments.add('--rerun-tasks') +} +if (gradle.startParameter.refreshDependencies) { + nestedBuildArguments.add('--refresh-dependencies') +} +for (File initScript : gradle.startParameter.initScripts) { + nestedBuildArguments.addAll(['--init-script', initScript.absolutePath]) +} +Map> daemonSlots = [:] +int nextDaemonSlot = 0 irisModTargets.each { String targetMinecraft, Map target -> String suffix = targetMinecraft.replace('.', '_') boolean selected = targetMinecraft == minecraftVersion @@ -426,46 +451,66 @@ irisModTargets.each { String targetMinecraft, Map target -> String artifactName = irisArtifactName(label, "${targetMinecraft}+${displayedLoader}") File artifact = file("adapters/${loader}/build/${targetMinecraft}/libs/${artifactName}") boolean unitTests = ((List) (target.testLoaders ?: moddedLoaderLabels.keySet().toList())).contains(loader) - TaskProvider buildTarget = tasks.register("verify${label}${suffix}", Exec) { + int daemonSlot = nextDaemonSlot++ % matrixParallelism + TaskProvider buildTarget = project(':adapters').tasks.register("verify${label}${suffix}", IrisModdedBuild) { group = 'verification' description = "Builds, tests and verifies the ${label} ${targetMinecraft} jar in its own Gradle build." - workingDir = file("adapters/${loader}") - List command = [file(moddedWrapperScript).absolutePath, 'assemble', 'compileTestJava', + usesService(matrixWorkers) + dependsOn(project(':core').tasks.named('jar'), project(':core').tasks.named('testFixturesJar')) + workingDirectory.set(rootProject.file("adapters/${loader}")) + getLabel().set("${label} ${targetMinecraft}".toString()) + artifactFile.set(artifact) + logFile.set(rootProject.layout.buildDirectory.file("reports/modded/${loader}-${targetMinecraft}.log")) + requiredEntries.set(commonModdedArtifactEntries + moddedLoaderEntries[loader]) + forbiddenCoreClasses.set(bukkitCoupledCoreClasses) + javaRuntime.set((int) target.java) + List command = [rootProject.file(moddedWrapperScript).absolutePath, 'assemble', 'compileTestJava', 'verifyLoggingPolicy', 'verifyNativeClassBoundary', '--console=plain', - "-PirisVersion=${project.version}", "-PminecraftVersion=${targetMinecraft}", + "--max-workers=${Math.min(2, gradle.startParameter.maxWorkerCount)}", + "-Dorg.gradle.jvmargs=${nestedJvmArguments.trim()}", + "-Dorg.gradle.daemon.registry.base=${rootProject.file(".gradle/iris-daemons/slot-${daemonSlot}").absolutePath}", + '-Dorg.gradle.daemon.idletimeout=3600000', + "-PirisBuildLane=${loader}-${targetMinecraft}", + '--project-cache-dir', rootProject.file("adapters/${loader}/.gradle/iris-lanes/${loader}-${targetMinecraft}").absolutePath, + '--init-script', rootProject.file('gradle/nested-build.init.gradle').absolutePath, + "-PirisVersion=${rootProject.version}", "-PminecraftVersion=${targetMinecraft}", "-P${moddedLoaderProperties[loader]}=${loaderPin}", "-PuseLocalVolmLib=${localVolmLibCheckout != null}"] if (unitTests) { command.add('test') } - commandLine(command) + command.add("-PtestForks=${providers.gradleProperty('moddedTestForks').getOrElse('1')}") + command.addAll(nestedBuildArguments) + commandLine.set(command.collect { Object argument -> argument.toString() }) if (localVolmLibCheckout != null) { - environment('ORG_GRADLE_PROJECT_localVolmLibDirectory', localVolmLibCheckout.absolutePath) + environment.put('ORG_GRADLE_PROJECT_localVolmLibDirectory', localVolmLibCheckout.absolutePath) } inputs.files(moddedSharedInputs).withPropertyName('moddedSharedSources') .withPathSensitivity(PathSensitivity.RELATIVE) - inputs.files(file("adapters/${loader}/src"), file("adapters/${loader}/build.gradle"), - file("adapters/${loader}/settings.gradle")).withPropertyName('adapterSources') + inputs.files(rootProject.file("adapters/${loader}/src"), rootProject.file("adapters/${loader}/build.gradle"), + rootProject.file("adapters/${loader}/settings.gradle"), rootProject.file("adapters/${loader}/gradle.properties")) + .withPropertyName('adapterSources') + inputs.files(gradle.startParameter.allInitScripts).withPropertyName('initScripts') + .withPathSensitivity(PathSensitivity.NONE) + outputs.upToDateWhen { !gradle.startParameter.refreshDependencies } inputs.properties(target) inputs.property('loaderVersion', loaderPin) - inputs.property('irisVersion', project.version.toString()) + inputs.property('irisVersion', rootProject.version.toString()) declareVolmLibInputs(it) - outputs.file(artifact) if (unitTests) { - outputs.dir(file("adapters/${loader}/build/${targetMinecraft}/test-results/test")) - } - doLast { - ModdedArtifactVerifier.verify(artifact, commonModdedArtifactEntries + moddedLoaderEntries[loader], - bukkitCoupledCoreClasses) - ModdedArtifactVerifier.verifyJavaRuntime(artifact, (int) target.java) - logger.lifecycle("Verified ${artifact.name}") + outputs.dir(rootProject.file("adapters/${loader}/build/${targetMinecraft}/test-results/test")) } } - if (previousMatrixBuild != null) { - TaskProvider preceding = previousMatrixBuild - buildTarget.configure { mustRunAfter(preceding) } + TaskProvider previousSlotBuild = daemonSlots[daemonSlot] + if (previousSlotBuild != null) { + buildTarget.configure { mustRunAfter(previousSlotBuild) } + } + daemonSlots[daemonSlot] = buildTarget + tasks.register("verify${label}${suffix}") { + group = 'verification' + description = "Builds, tests and verifies the ${label} ${targetMinecraft} jar." + dependsOn(buildTarget) } - previousMatrixBuild = buildTarget verifyModdedMatrix.configure { dependsOn(buildTarget) } File legacyOutJar = new File(pluginOutDirectory, "Iris-${project.version}+mc${targetMinecraft}-${loader}.jar") FileTree staleTargetJars = fileTree(pluginOutDirectory) { @@ -512,18 +557,6 @@ tasks.register('buildNeoforge') { dependsOn(selectedModdedExports.neoforge) } -gradle.projectsEvaluated { - List sharedOutputTasks = allprojects.collectMany { Project candidate -> - candidate.tasks.findAll { Task task -> - task instanceof JavaCompile || task instanceof Jar || task instanceof Test - } - } - tasks.withType(Exec).matching { Task task -> task.name.startsWith('verify') }.configureEach { - mustRunAfter(sharedOutputTasks) - mustRunAfter(tasks.named('buildBukkit')) - } -} - tasks.named('check').configure { dependsOn(selectedModdedBuilds.fabric) } diff --git a/buildSrc/build.gradle b/buildSrc/build.gradle index 6aa6169ff..6049e11e9 100644 --- a/buildSrc/build.gradle +++ b/buildSrc/build.gradle @@ -21,11 +21,14 @@ repositories { // Nothing else asks buildSrc for `test`: Gradle only requests the jar that carries the build logic. // Gating the jar on the tests is what makes them run at all, and it costs nothing once they are green. tasks.named('jar').configure { - dependsOn(tasks.named('test')) + if (!gradle.extensions.extraProperties.has('irisBuildLane')) { + dependsOn(tasks.named('test')) + } } tasks.named('test').configure { jvmArgs('--add-opens=java.base/java.lang=ALL-UNNAMED') + jvmArgs('-XX:ParallelGCThreads=2', '-XX:ConcGCThreads=1', '-XX:CICompilerCount=2') } dependencies { diff --git a/buildSrc/src/main/java/IrisModdedBuild.java b/buildSrc/src/main/java/IrisModdedBuild.java new file mode 100644 index 000000000..6c3aa73c3 --- /dev/null +++ b/buildSrc/src/main/java/IrisModdedBuild.java @@ -0,0 +1,67 @@ +import org.gradle.api.DefaultTask; +import org.gradle.api.file.DirectoryProperty; +import org.gradle.api.file.RegularFileProperty; +import org.gradle.api.provider.ListProperty; +import org.gradle.api.provider.MapProperty; +import org.gradle.api.provider.Property; +import org.gradle.api.provider.SetProperty; +import org.gradle.api.tasks.Input; +import org.gradle.api.tasks.Internal; +import org.gradle.api.tasks.LocalState; +import org.gradle.api.tasks.OutputFile; +import org.gradle.api.tasks.TaskAction; +import org.gradle.workers.WorkerExecutor; + +import javax.inject.Inject; +import java.util.Map; + +public abstract class IrisModdedBuild extends DefaultTask { + public IrisModdedBuild() { + getEnvironment().convention(Map.of()); + } + + @Input + public abstract Property getLabel(); + + @Internal + public abstract DirectoryProperty getWorkingDirectory(); + + @Input + public abstract ListProperty getCommandLine(); + + @Input + public abstract MapProperty getEnvironment(); + + @Input + public abstract ListProperty getRequiredEntries(); + + @Input + public abstract SetProperty getForbiddenCoreClasses(); + + @Input + public abstract Property getJavaRuntime(); + + @OutputFile + public abstract RegularFileProperty getArtifactFile(); + + @LocalState + public abstract RegularFileProperty getLogFile(); + + @Inject + protected abstract WorkerExecutor getWorkerExecutor(); + + @TaskAction + public void build() { + getWorkerExecutor().noIsolation().submit(IrisModdedBuildAction.class, parameters -> { + parameters.getLabel().set(getLabel()); + parameters.getWorkingDirectory().set(getWorkingDirectory()); + parameters.getCommandLine().set(getCommandLine()); + parameters.getEnvironment().set(getEnvironment()); + parameters.getRequiredEntries().set(getRequiredEntries()); + parameters.getForbiddenCoreClasses().set(getForbiddenCoreClasses()); + parameters.getJavaRuntime().set(getJavaRuntime()); + parameters.getArtifactFile().set(getArtifactFile()); + parameters.getLogFile().set(getLogFile()); + }); + } +} diff --git a/buildSrc/src/main/java/IrisModdedBuildAction.java b/buildSrc/src/main/java/IrisModdedBuildAction.java new file mode 100644 index 000000000..7c3f82302 --- /dev/null +++ b/buildSrc/src/main/java/IrisModdedBuildAction.java @@ -0,0 +1,85 @@ +import org.gradle.api.GradleException; +import org.gradle.api.logging.Logger; +import org.gradle.api.logging.Logging; +import org.gradle.process.ExecOperations; +import org.gradle.workers.WorkAction; + +import javax.inject.Inject; +import java.io.IOException; +import java.io.OutputStream; +import java.io.PrintWriter; +import java.io.RandomAccessFile; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.Arrays; +import java.util.concurrent.TimeUnit; + +public abstract class IrisModdedBuildAction implements WorkAction { + private static final Logger LOGGER = Logging.getLogger(IrisModdedBuildAction.class); + private static final int LOG_TAIL_BYTES = 16 * 1024; + private static final int LOG_TAIL_LINES = 40; + + @Inject + protected abstract ExecOperations getExecOperations(); + + @Override + public void execute() { + IrisModdedBuildParameters parameters = getParameters(); + String label = parameters.getLabel().get(); + Path log = parameters.getLogFile().get().getAsFile().toPath(); + long started = System.nanoTime(); + LOGGER.lifecycle("{} started; log: {}", label, log); + try { + Files.createDirectories(log.getParent()); + try (OutputStream output = Files.newOutputStream(log)) { + getExecOperations().exec(spec -> { + spec.setWorkingDir(parameters.getWorkingDirectory().get().getAsFile()); + spec.commandLine(parameters.getCommandLine().get()); + spec.environment(parameters.getEnvironment().get()); + spec.setStandardOutput(output); + spec.setErrorOutput(output); + }).assertNormalExitValue(); + } + ModdedArtifactVerifier.verify(parameters.getArtifactFile().get().getAsFile(), + parameters.getRequiredEntries().get(), parameters.getForbiddenCoreClasses().get()); + ModdedArtifactVerifier.verifyJavaRuntime(parameters.getArtifactFile().get().getAsFile(), + parameters.getJavaRuntime().get()); + LOGGER.lifecycle("{} succeeded in {} ms", label, + TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - started)); + } catch (Exception failure) { + appendFailure(log, failure); + String details; + try { + details = logTail(log); + } catch (IOException unreadable) { + failure.addSuppressed(unreadable); + details = "Log could not be read."; + } + LOGGER.error(label + " failed; log: " + log + System.lineSeparator() + details, failure); + throw new GradleException(label + " failed; log: " + log, failure); + } + } + + static String logTail(Path log) throws IOException { + try (RandomAccessFile input = new RandomAccessFile(log.toFile(), "r")) { + long offset = Math.max(0L, input.length() - LOG_TAIL_BYTES); + input.seek(offset); + byte[] bytes = new byte[(int) (input.length() - offset)]; + input.readFully(bytes); + String[] lines = new String(bytes, StandardCharsets.UTF_8).split("\\R"); + return String.join(System.lineSeparator(), Arrays.copyOfRange(lines, + Math.max(0, lines.length - LOG_TAIL_LINES), lines.length)); + } + } + + private static void appendFailure(Path log, Exception failure) { + try (PrintWriter output = new PrintWriter(Files.newBufferedWriter(log, StandardCharsets.UTF_8, + StandardOpenOption.CREATE, StandardOpenOption.APPEND))) { + failure.printStackTrace(output); + } catch (IOException unwritable) { + failure.addSuppressed(unwritable); + } + } +} diff --git a/buildSrc/src/main/java/IrisModdedBuildParameters.java b/buildSrc/src/main/java/IrisModdedBuildParameters.java new file mode 100644 index 000000000..e67ae1a5e --- /dev/null +++ b/buildSrc/src/main/java/IrisModdedBuildParameters.java @@ -0,0 +1,27 @@ +import org.gradle.api.file.DirectoryProperty; +import org.gradle.api.file.RegularFileProperty; +import org.gradle.api.provider.ListProperty; +import org.gradle.api.provider.MapProperty; +import org.gradle.api.provider.Property; +import org.gradle.api.provider.SetProperty; +import org.gradle.workers.WorkParameters; + +public interface IrisModdedBuildParameters extends WorkParameters { + Property getLabel(); + + DirectoryProperty getWorkingDirectory(); + + ListProperty getCommandLine(); + + MapProperty getEnvironment(); + + ListProperty getRequiredEntries(); + + SetProperty getForbiddenCoreClasses(); + + Property getJavaRuntime(); + + RegularFileProperty getArtifactFile(); + + RegularFileProperty getLogFile(); +} diff --git a/buildSrc/src/main/java/IrisModdedBuildService.java b/buildSrc/src/main/java/IrisModdedBuildService.java new file mode 100644 index 000000000..2556c7bcb --- /dev/null +++ b/buildSrc/src/main/java/IrisModdedBuildService.java @@ -0,0 +1,5 @@ +import org.gradle.api.services.BuildService; +import org.gradle.api.services.BuildServiceParameters; + +public abstract class IrisModdedBuildService implements BuildService { +} diff --git a/buildSrc/src/test/java/IrisModdedBuildTest.java b/buildSrc/src/test/java/IrisModdedBuildTest.java new file mode 100644 index 000000000..a2a256cb2 --- /dev/null +++ b/buildSrc/src/test/java/IrisModdedBuildTest.java @@ -0,0 +1,79 @@ +import org.gradle.api.Project; +import org.gradle.testfixtures.ProjectBuilder; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +public class IrisModdedBuildTest { + @Rule + public TemporaryFolder temporary = new TemporaryFolder(); + + @Test + public void capturesTheCompleteTargetConfiguration() throws Exception { + Project project = ProjectBuilder.builder().withProjectDir(temporary.newFolder()).build(); + IrisModdedBuild task = project.getTasks().create("verifyFabric", IrisModdedBuild.class); + task.getLabel().set("Fabric 26.3"); + task.getWorkingDirectory().set(project.getLayout().getProjectDirectory()); + task.getCommandLine().set(List.of("gradlew", "assemble", "test")); + task.getRequiredEntries().set(List.of("fabric.mod.json")); + task.getForbiddenCoreClasses().set(Set.of("org/bukkit/Test")); + task.getJavaRuntime().set(25); + task.getArtifactFile().set(project.getLayout().getProjectDirectory().file("build/mod.jar")); + task.getLogFile().set(project.getLayout().getProjectDirectory().file("build/reports/matrix/fabric.log")); + + assertEquals(Map.of(), task.getEnvironment().get()); + assertEquals(List.of("gradlew", "assemble", "test"), task.getCommandLine().get()); + assertEquals(List.of("fabric.mod.json"), task.getRequiredEntries().get()); + assertEquals(Set.of("org/bukkit/Test"), task.getForbiddenCoreClasses().get()); + assertEquals(Integer.valueOf(25), task.getJavaRuntime().get()); + assertEquals(Set.of(task.getArtifactFile().get().getAsFile()), task.getOutputs().getFiles().getFiles()); + assertFalse(task.getOutputs().getFiles().contains(task.getLogFile().get().getAsFile())); + } + + @Test + public void failureSummaryRetainsTheFinalLinesWithoutReadingAnEntireLog() throws Exception { + Path log = temporary.newFile().toPath(); + StringBuilder content = new StringBuilder("discarded first line\n"); + for (int index = 0; index < 2000; index++) { + content.append("task output ").append(index).append('\n'); + } + content.append("final failure\n"); + Files.writeString(log, content); + + String tail = IrisModdedBuildAction.logTail(log); + + assertEquals(40, tail.lines().count()); + assertTrue(tail.endsWith("final failure")); + assertFalse(tail.contains("discarded first line")); + assertTrue(tail.length() <= 16 * 1024); + } + + @Test + public void failureSummaryAcceptsEmptyAndUnterminatedLogs() throws Exception { + Path log = temporary.newFile().toPath(); + assertEquals("", IrisModdedBuildAction.logTail(log)); + Files.writeString(log, "first\r\nlast"); + assertEquals("first" + System.lineSeparator() + "last", IrisModdedBuildAction.logTail(log)); + } + + @Test + public void failureSummaryBoundsAnUnbrokenFailureLine() throws Exception { + Path log = temporary.newFile().toPath(); + Files.writeString(log, "x".repeat(100_000) + "final failure"); + + String tail = IrisModdedBuildAction.logTail(log); + + assertEquals(16 * 1024, tail.length()); + assertTrue(tail.endsWith("final failure")); + } +} diff --git a/core/src/main/java/art/arcane/iris/generation/runtime/EngineBackgroundTasks.java b/core/src/main/java/art/arcane/iris/generation/runtime/EngineBackgroundTasks.java index 820366469..62b7c4ee2 100644 --- a/core/src/main/java/art/arcane/iris/generation/runtime/EngineBackgroundTasks.java +++ b/core/src/main/java/art/arcane/iris/generation/runtime/EngineBackgroundTasks.java @@ -26,6 +26,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.IdentityHashMap; import java.util.List; +import java.util.Objects; import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; @@ -33,6 +34,7 @@ import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.LongSupplier; import static art.arcane.iris.generation.runtime.EngineShutdownSequence.appendFailure; import static art.arcane.iris.generation.runtime.EngineShutdownSequence.propagate; @@ -47,9 +49,18 @@ final class EngineBackgroundTasks { private final Object backgroundTaskLock = new Object(); private final List backgroundTasks = new ArrayList<>(); + private final LongSupplier clock; private boolean backgroundTaskAdmission; private int activeDrains; + EngineBackgroundTasks() { + this(System::nanoTime); + } + + EngineBackgroundTasks(LongSupplier clock) { + this.clock = Objects.requireNonNull(clock); + } + boolean scheduleTrackedTask(Runnable task) { return scheduleTrackedTask(task, false); } @@ -106,7 +117,7 @@ final class EngineBackgroundTasks { } BackgroundTaskDrain drainBackgroundTasks(String reason) { - long deadline = System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(BACKGROUND_TASK_TIMEOUT_MILLIS); + long deadline = clock.getAsLong() + TimeUnit.MILLISECONDS.toNanos(BACKGROUND_TASK_TIMEOUT_MILLIS); Set observed = Collections.newSetFromMap(new IdentityHashMap<>()); Set settledAtEntry = Collections.newSetFromMap(new IdentityHashMap<>()); List tasks; @@ -135,7 +146,7 @@ final class EngineBackgroundTasks { return new BackgroundTaskDrain(failure, allTasksComplete()); } } - if (System.nanoTime() >= deadline) { + if (clock.getAsLong() >= deadline) { for (TrackedBackgroundTask task : tasks) { cancelBackgroundTask(task, reason); } @@ -170,7 +181,7 @@ final class EngineBackgroundTasks { long deadline, String reason) { Throwable failure = null; for (TrackedBackgroundTask task : tasks) { - long remaining = deadline - System.nanoTime(); + long remaining = deadline - clock.getAsLong(); if (remaining <= 0L) { cancelBackgroundTask(task, reason); failure = appendFailure(failure, new TimeoutException("Timed out waiting for Iris background tasks during " + reason + ".")); diff --git a/core/src/test/java/art/arcane/iris/generation/runtime/EngineBackgroundTasksAdmissionTest.java b/core/src/test/java/art/arcane/iris/generation/runtime/EngineBackgroundTasksAdmissionTest.java index 0fb2811dd..eb7e01ef1 100644 --- a/core/src/test/java/art/arcane/iris/generation/runtime/EngineBackgroundTasksAdmissionTest.java +++ b/core/src/test/java/art/arcane/iris/generation/runtime/EngineBackgroundTasksAdmissionTest.java @@ -14,6 +14,7 @@ import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; import java.util.concurrent.locks.LockSupport; @@ -115,7 +116,8 @@ public class EngineBackgroundTasksAdmissionTest { @Test public void continuationsShareTheOriginalDrainDeadline() throws Exception { - Fixture fixture = fixture(); + AtomicLong clock = new AtomicLong(); + Fixture fixture = fixture(new EngineBackgroundTasks(clock::get)); CountDownLatch firstEntered = new CountDownLatch(1); CountDownLatch firstRelease = new CountDownLatch(1); CountDownLatch saveEntered = new CountDownLatch(1); @@ -132,9 +134,9 @@ public class EngineBackgroundTasksAdmissionTest { fixture.tasks().scheduleAdmittedTask(fixture.engine(), () -> waitForRelease(saveEntered, saveRelease)))); assertTrue(saveEntered.await(5, TimeUnit.SECONDS)); - TimeUnit.SECONDS.sleep(7); + clock.set(TimeUnit.SECONDS.toNanos(15) - TimeUnit.MILLISECONDS.toNanos(1)); firstRelease.countDown(); - EngineBackgroundTasks.BackgroundTaskDrain result = drain.get(10, TimeUnit.SECONDS); + EngineBackgroundTasks.BackgroundTaskDrain result = drain.get(5, TimeUnit.SECONDS); assertTrue(result.failure() instanceof TimeoutException); assertThrows(IllegalStateException.class, () -> result.requireComplete("Studio cutover")); } finally { @@ -287,7 +289,10 @@ public class EngineBackgroundTasksAdmissionTest { } private static Fixture fixture() { - EngineBackgroundTasks tasks = new EngineBackgroundTasks(); + return fixture(new EngineBackgroundTasks()); + } + + private static Fixture fixture(EngineBackgroundTasks tasks) { tasks.openBackgroundTaskAdmission(); GenerationSessionManager sessions = new GenerationSessionManager(true); Engine engine = mock(Engine.class); diff --git a/gradle.properties b/gradle.properties index 8ddbf1b9a..363ffb006 100644 --- a/gradle.properties +++ b/gradle.properties @@ -17,6 +17,7 @@ # org.gradle.daemon=true org.gradle.parallel=true +org.gradle.workers.max=32 org.gradle.jvmargs=-Xmx3072m -XX:+HeapDumpOnOutOfMemoryError -Dfile.encoding=UTF-8 org.gradle.caching=true org.gradle.configureondemand=false diff --git a/gradle/modded-generation-build-revision.gradle b/gradle/modded-generation-build-revision.gradle index df258591b..899a85654 100644 --- a/gradle/modded-generation-build-revision.gradle +++ b/gradle/modded-generation-build-revision.gradle @@ -11,11 +11,13 @@ if (coreBuild == null) { throw new GradleException('The modded artifact requires the included Iris core build and its generation build revision.') } TaskReference buildRevisionVerification = coreBuild.task(':core:verifyGenerationBuildRevision') +String buildLogicVariant = gradle.extensions.extraProperties.has('irisBuildLane') + ? "iris-lanes/${gradle.extensions.extraProperties.get('irisBuildLane')}/" : '' TaskProvider resolvedRevisionVerification = tasks.register('verifyResolvedGenerationKernelDependencies', JavaExec) { group = 'verification' description = 'Checks the actual bundled generation library contents against the current build revision.' dependsOn(buildRevisionVerification) - classpath = files('../../buildSrc/build/libs/buildSrc.jar') + classpath = files("../../buildSrc/build/${buildLogicVariant}libs/buildSrc.jar") mainClass.set('art.arcane.iris.buildtools.GenerationBuildRevision') doFirst { String runtimeVariant = gradle.extensions.extraProperties.get('irisRuntimeOutputVariant').toString() diff --git a/gradle/modded-version-sources.gradle b/gradle/modded-version-sources.gradle index b0556f577..2d1458186 100644 --- a/gradle/modded-version-sources.gradle +++ b/gradle/modded-version-sources.gradle @@ -2,11 +2,17 @@ import groovy.json.JsonSlurper List sourceOverlays = (List) irisModTarget.overlays List adapterSourceRoots = [projectDir, file('../modded-common'), file('../client-common')] +int testForks = Math.max(1, Integer.parseInt(providers.gradleProperty('testForks').getOrElse('2'))) tasks.named('compileTestJava', JavaCompile).configure { options.release.set((int) irisModTarget.java) } tasks.withType(Test).configureEach { + maxParallelForks = Math.min(2, Math.min(Runtime.runtime.availableProcessors(), + Math.min(gradle.startParameter.maxWorkerCount, testForks))) + if (gradle.extensions.extraProperties.has('irisBuildLane')) { + jvmArgs('-XX:ParallelGCThreads=2', '-XX:ConcGCThreads=1', '-XX:CICompilerCount=2') + } javaLauncher.set(javaToolchains.launcherFor { languageVersion = JavaLanguageVersion.of((int) irisModTarget.java) }) diff --git a/gradle/nested-build.init.gradle b/gradle/nested-build.init.gradle new file mode 100644 index 000000000..bea4ae61e --- /dev/null +++ b/gradle/nested-build.init.gradle @@ -0,0 +1,31 @@ +Gradle rootBuild = gradle +while (rootBuild.parent != null) { + rootBuild = rootBuild.parent +} +String buildLane = rootBuild.startParameter.projectProperties['irisBuildLane'] +if (buildLane == null) { + throw new GradleException('The nested build requires -PirisBuildLane.') +} +if (!(buildLane ==~ /[a-zA-Z0-9][a-zA-Z0-9._-]*/)) { + throw new GradleException('irisBuildLane must contain only letters, numbers, dots, underscores and hyphens.') +} + +gradle.extensions.extraProperties.set('irisBuildLane', buildLane) +gradle.startParameter.projectCacheDir = new File(gradle.startParameter.currentDir, ".gradle/iris-lanes/${buildLane}") +gradle.beforeSettings { Settings settings -> + settings.gradle.startParameter.projectCacheDir = new File(settings.settingsDir, ".gradle/iris-lanes/${buildLane}") +} +gradle.beforeProject { Project candidate -> + boolean sharedRuntime = candidate.rootProject.name == 'Iris' + && candidate.path in [':core', ':core:agent', ':spi'] + if (!candidate.rootProject.name.startsWith('iris-') && !sharedRuntime) { + candidate.layout.buildDirectory.set(candidate.layout.projectDirectory.dir("build/iris-lanes/${buildLane}")) + } + candidate.plugins.withId('io.papermc.paperweight.userdev') { + candidate.tasks.named('paperweightUserdevSetup').configure { Task setup -> + Directory taskCache = candidate.layout.projectDirectory.dir(".gradle/iris-lanes/${buildLane}/caches/paperweight/taskCache") + setup.mappedServerJar.set(taskCache.file('mappedServerJar.jar')) + setup.reobfMappings.set(taskCache.file('reobfMappings.tiny')) + } + } +} diff --git a/gradle/runtime-cache.settings.gradle b/gradle/runtime-cache.settings.gradle index 989beba22..5995746d4 100644 --- a/gradle/runtime-cache.settings.gradle +++ b/gradle/runtime-cache.settings.gradle @@ -11,8 +11,10 @@ if (modTarget == null) { } String runtimeJava = providers.gradleProperty('irisRuntimeJava').getOrElse(modTarget.java.toString()) String variant = adapter ? "minecraft${minecraftVersion}-java${runtimeJava}" : "java${runtimeJava}" -File projectCacheRoot = gradle.startParameter.projectCacheDir ?: new File(settingsDir, '.gradle') -gradle.startParameter.projectCacheDir = new File(projectCacheRoot, variant) +if (!gradle.extensions.extraProperties.has('irisBuildLane')) { + File projectCacheRoot = gradle.startParameter.projectCacheDir ?: new File(settingsDir, '.gradle') + gradle.startParameter.projectCacheDir = new File(projectCacheRoot, variant) +} String runtimeOutputVariant = runtimeJava == '21' ? 'java21/' : '' if (adapter) {