Split pregen queue metrics

- Publish in-flight pregen chunk requests as iris.pregen-queue
- Publish chunks left in the job as iris.pregen-remaining
- Mark generation timings unavailable after 10 seconds without generation
This commit is contained in:
Brian Neumann-Fopiano committed 2026-09-29 18:09:55 -04:00
1 parent 593c3f732a
commit b8433340c9
16 files changed
+183 -21

No files matched your search

@@ -0,0 +1,35 @@
package art.arcane.iris.generation.runtime;
final class GenerationRateWindow {
static final long TIMING_IDLE_MILLIS = 10_000L;
private long lastSampleAtMs;
private int lastGeneratedCount;
private long lastGenerationAtMs;
double sampleChunksPerSecond(int generatedCount, long sampledAtMs) {
int safeGeneratedCount = Math.max(0, generatedCount);
long previousAt = lastSampleAtMs;
int previousCount = lastGeneratedCount;
lastSampleAtMs = sampledAtMs;
lastGeneratedCount = safeGeneratedCount;
if (previousAt <= 0L) {
if (safeGeneratedCount > 0) {
lastGenerationAtMs = sampledAtMs;
}
return 0D;
}
if (sampledAtMs <= previousAt) {
return 0D;
}
int generatedDelta = Math.max(0, safeGeneratedCount - previousCount);
if (generatedDelta > 0) {
lastGenerationAtMs = sampledAtMs;
}
return generatedDelta * 1000D / (sampledAtMs - previousAt);
}
boolean generationTimingsFresh(long sampledAtMs) {
return lastGenerationAtMs > 0L && sampledAtMs - lastGenerationAtMs <= TIMING_IDLE_MILLIS;
}
}
@@ -460,8 +460,12 @@ public final class IrisEngineSVC implements IrisService {
continue;
}
double chunksPerSecond = registered.sampleChunksPerSecond(engine.getGenerated(), now);
worldSnapshots.add(EngineTelemetrySnapshot.capture(engine, chunksPerSecond, now));
double chunksPerSecond = registered.generationRate.sampleChunksPerSecond(engine.getGenerated(), now);
worldSnapshots.add(EngineTelemetrySnapshot.capture(
engine,
chunksPerSecond,
registered.generationRate.generationTimingsFresh(now),
now));
registered.clearMetricsFailure();
} catch (Throwable exception) {
if (EngineMaintenance.isMantleClosed(exception)) {
@@ -552,6 +556,7 @@ public final class IrisEngineSVC implements IrisService {
progress.generated(),
progress.totalChunks(),
progress.chunksRemaining(),
progress.chunksInFlight(),
progress.chunksPerSecond(),
progress.eta(),
progress.elapsed(),
@@ -612,8 +617,7 @@ public final class IrisEngineSVC implements IrisService {
private final AtomicReference<CompletableFuture<Void>> activeMaintenance = new AtomicReference<>();
private volatile ScheduledFuture<?> maintenance;
private volatile boolean closed;
private long lastGeneratedSampleAtMs;
private int lastGeneratedCount;
private final GenerationRateWindow generationRate = new GenerationRateWindow();
private Registered(Registration registration, ScheduledThreadPoolExecutor executor) {
name = registration.name();
@@ -696,19 +700,6 @@ public final class IrisEngineSVC implements IrisService {
return activeMaintenance.get() != null;
}
private double sampleChunksPerSecond(int generatedCount, long sampledAtMs) {
int safeGeneratedCount = Math.max(0, generatedCount);
long previousAt = lastGeneratedSampleAtMs;
int previousCount = lastGeneratedCount;
lastGeneratedSampleAtMs = sampledAtMs;
lastGeneratedCount = safeGeneratedCount;
if (previousAt <= 0L || sampledAtMs <= previousAt) {
return 0D;
}
int generatedDelta = Math.max(0, safeGeneratedCount - previousCount);
return generatedDelta * 1000D / (sampledAtMs - previousAt);
}
private void close() {
if (closed) {
return;
@@ -101,6 +101,7 @@ public record IrisTelemetrySnapshot(
long generated,
long total,
long remaining,
long inFlight,
double chunksPerSecond,
long etaMs,
long elapsedMs,
@@ -115,6 +116,7 @@ public record IrisTelemetrySnapshot(
0L,
0L,
0L,
0L,
0D,
0L,
0L,
@@ -130,6 +132,7 @@ public record IrisTelemetrySnapshot(
generated = Math.max(0L, generated);
total = Math.max(0L, total);
remaining = Math.max(0L, remaining);
inFlight = Math.max(0L, inFlight);
chunksPerSecond = Double.isFinite(chunksPerSecond) ? Math.max(0D, chunksPerSecond) : 0D;
etaMs = Math.max(0L, etaMs);
elapsedMs = Math.max(0L, elapsedMs);
@@ -319,6 +319,7 @@ public class IrisIntegrationService implements IrisService, IntegrationServiceCo
IntegrationMetricSchema.IRIS_PREGEN_GENERATED,
IntegrationMetricSchema.IRIS_PREGEN_TOTAL,
IntegrationMetricSchema.IRIS_PREGEN_QUEUE,
IntegrationMetricSchema.IRIS_PREGEN_REMAINING,
IntegrationMetricSchema.IRIS_PREGEN_THROUGHPUT,
IntegrationMetricSchema.IRIS_PREGEN_ETA_MS,
IntegrationMetricSchema.IRIS_PREGEN_ELAPSED_MS,
@@ -332,7 +333,8 @@ public class IrisIntegrationService implements IrisService, IntegrationServiceCo
put(samples, IntegrationMetricSchema.IRIS_PREGEN_PROGRESS, pregenerator.progressPercent(), sampledAtMs);
put(samples, IntegrationMetricSchema.IRIS_PREGEN_GENERATED, pregenerator.generated(), sampledAtMs);
put(samples, IntegrationMetricSchema.IRIS_PREGEN_TOTAL, pregenerator.total(), sampledAtMs);
put(samples, IntegrationMetricSchema.IRIS_PREGEN_QUEUE, pregenerator.remaining(), sampledAtMs);
put(samples, IntegrationMetricSchema.IRIS_PREGEN_QUEUE, pregenerator.inFlight(), sampledAtMs);
put(samples, IntegrationMetricSchema.IRIS_PREGEN_REMAINING, pregenerator.remaining(), sampledAtMs);
put(samples, IntegrationMetricSchema.IRIS_PREGEN_THROUGHPUT, pregenerator.chunksPerSecond(), sampledAtMs);
put(samples, IntegrationMetricSchema.IRIS_PREGEN_ETA_MS, pregenerator.etaMs(), sampledAtMs);
put(samples, IntegrationMetricSchema.IRIS_PREGEN_ELAPSED_MS, pregenerator.elapsedMs(), sampledAtMs);