Skip to content

Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.

fix(forge): sweep forge-finalize cron rows concurrently instead of one at a time

nirholas376f160be97b
diff --git a/api/cron/forge-finalize.js b/api/cron/forge-finalize.js
index 81f72ebea..928605598 100644
--- a/api/cron/forge-finalize.js
+++ b/api/cron/forge-finalize.js
@@ -57,10 +57,21 @@ import { resolveTier } from '../_lib/forge-tiers.js';
import { acquireLock, cacheGet, cacheSet } from '../_lib/cache.js';
import { getRedis } from '../_lib/redis.js';
import { NIM_TRELLIS_COOLDOWN_KEY, nimCooldownSeconds } from '../forge.js';
+import { mapPoolSettled } from '../_lib/pool.js';
export const maxDuration = 60;
const BATCH_LIMIT = Number(process.env.FORGE_FINALIZE_BATCH) || 25;
+// Each row's status poll is an independent network round trip
+// (api/_providers/gcp.js status() allows up to 15s x 2 attempts on its own), so
+// sweeping BATCH_LIMIT rows one at a time can take minutes once a lane like
+// trellis_selfhost is unhealthy and every poll on it runs its full timeout.
+// That is exactly what blew the 60s cron budget on 2026-10-24 (see
+// docs/ops/production-log-triage.md "HTTP timeout with no cron context"): rows
+// are independent (keyed by their own replicate_job_id, no shared mutable
+// state besides the stateless provider clients), so sweeping them concurrently
+// bounds the tick's wall-clock time to the slowest row instead of the sum.
+export const ROW_CONCURRENCY = Number(process.env.FORGE_FINALIZE_CONCURRENCY) || 8;
// Leave fresh rows to the attended browser poll path.
const GRACE_MINUTES = 2;
// Past this age a non-terminal generation is declared dead. The slowest real
@@ -291,15 +302,151 @@ export default wrapCron(async (req, res) => {
// NIM unconfigured: forge-token rows can't be polled here; hard TTL reaps.
}
+ const results = await mapPoolSettled(rows, ROW_CONCURRENCY, (row) => sweepRow(row, { gcp, nvidia }));
const out = { done: 0, failed: 0, failed_over: 0, resubmitted: 0, timed_out: 0, still_running: 0, unpollable: 0 };
+ for (const result of results) {
+ if (result.ok) {
+ for (const key of Object.keys(out)) out[key] += result.value[key] || 0;
+ } else {
+ console.warn(`[forge-finalize] row sweep threw: ${result.error?.message || result.error}`);
+ out.unpollable++;
+ }
+ }
- for (const row of rows) {
- const ageMinutes = (Date.now() - Date.parse(row.created_at)) / 60_000;
- const envelope = decodeWorkerEnvelope(row.replicate_job_id);
+ return json(res, 200, {
+ enabled: true,
+ swept: rows.length,
+ ...out,
+ elapsed_ms: Date.now() - t0,
+ });
+}, { maxDurationMs: maxDuration * 1000 });
- if (envelope && gcp) {
- const status = await gcp.status(row.replicate_job_id).catch(() => null);
+// One row's worth of the sweep above, factored out so it can run concurrently
+// across the batch (mapPoolSettled in the caller) instead of blocking the next
+// row on this row's network round trips. Returns the per-row delta to merge
+// into the tick's aggregate counters; never throws (mapPoolSettled would catch
+// it anyway, but a thrown row loses its counters, so every awaited call here
+// already carries its own `.catch`).
+export async function sweepRow(row, { gcp, nvidia }) {
+ const out = { done: 0, failed: 0, failed_over: 0, resubmitted: 0, timed_out: 0, still_running: 0, unpollable: 0 };
+ const ageMinutes = (Date.now() - Date.parse(row.created_at)) / 60_000;
+ const envelope = decodeWorkerEnvelope(row.replicate_job_id);
+
+ if (envelope && gcp) {
+ const status = await gcp.status(row.replicate_job_id).catch(() => null);
+
+ if (status?.status === 'done' && status.resultGlbUrl) {
+ const durable = await materializeCreation({
+ replicateJobId: row.replicate_job_id,
+ clientKey: row.client_key,
+ glbUrl: status.resultGlbUrl,
+ });
+ if (durable) {
+ out.done++;
+ if (row.user_id) {
+ notifyForgeComplete({
+ userId: row.user_id,
+ creationId: durable.id,
+ prompt: row.prompt,
+ previewImageUrl: durable.previewImageUrl ?? row.preview_image_url ?? null,
+ });
+ }
+ return out;
+ }
+ // Copy-to-durable failed (transient storage/network): leave the row
+ // for the next tick rather than failing a finished model.
+ out.still_running++;
+ return out;
+ }
+ if (status?.status === 'failed') {
+ // Inside the attended window a live browser may poll this failure
+ // seconds from now and run its own failover; acting here would race
+ // it into duplicate GPU work. Leave the row: an attended client
+ // resolves it, and an abandoned one ages into the branch below.
+ if (decideFailedSweep({ ageMinutes, hasReferenceImage: true }) === 'defer') {
+ out.still_running++;
+ return out;
+ }
+ if (row.backend) await markLaneUnhealthy(row.backend).catch(() => {});
+ const redispatched = await tryCronRedispatch(row, ageMinutes);
+ if (redispatched) {
+ await markFailed({
+ replicateJobId: row.replicate_job_id,
+ clientKey: row.client_key,
+ error: status.error || 'generation failed',
+ });
+ // Link the attempt to its successor so the ledger reads as a
+ // recovery: without it the health sensor and the error report both
+ // count this row as a user-visible loss, when the request is in
+ // fact still running on another lane.
+ if (redispatched.successorId) {
+ await markSupersededBy({
+ replicateJobId: row.replicate_job_id,
+ clientKey: row.client_key,
+ successorId: redispatched.successorId,
+ });
+ }
+ out.failed_over++;
+ // No failure notification: the successor row carries the job on,
+ // and its completion (or terminal failure) notifies instead.
+ return out;
+ }
+ await markFailed({
+ replicateJobId: row.replicate_job_id,
+ clientKey: row.client_key,
+ error: status.error || 'generation failed',
+ });
+ out.failed++;
+ if (row.user_id) {
+ notifyForgeFailed({ userId: row.user_id, creationId: row.id, prompt: row.prompt, error: status.error, backend: row.backend, tier: row.tier });
+ }
+ return out;
+ }
+
+ // queued / running / poll blip: fall through to the TTL check.
+ } else if (!envelope) {
+ // The paid x402 lane stores the FULL signed forge token (provider +
+ // upstream task id) so unattended agent purchases can be completed
+ // server-side here, so an agent that got its poll_url answer and moved on
+ // still ends with a finished gallery row, not a TTL orphan.
+ const forgeToken = decodeJobToken(row.replicate_job_id);
+ if (forgeToken?.provider === 'nvidia' && forgeToken.taskId && nvidia) {
+ const status = await nvidia
+ .status({ kind: forgeToken.kind, taskId: forgeToken.taskId })
+ .catch(() => null);
+ if (status?.status === 'done' && status.resultGlbUrl) {
+ const durable = await materializeCreation({
+ replicateJobId: row.replicate_job_id,
+ clientKey: row.client_key,
+ glbUrl: status.resultGlbUrl,
+ });
+ if (durable) {
+ out.done++;
+ return out;
+ }
+ out.still_running++;
+ return out;
+ }
+ if (status?.status === 'failed') {
+ await markFailed({
+ replicateJobId: row.replicate_job_id,
+ clientKey: row.client_key,
+ error: status.error || 'generation failed',
+ });
+ out.failed++;
+ return out;
+ }
+ // queued / running / poll blip: fall through to the TTL check.
+ } else if (row.backend === 'nvidia' && nvidia) {
+ // Browser/free-lane NIM rows store the BARE NVCF request id (only the
+ // x402 lane stores the signed token), so they used to be unpollable
+ // here and died at the hard TTL as "generation timed out" even when
+ // the GPU had finished. Poll the bare id directly, chasing the alias
+ // an attended expiry recovery may have written first.
+ const alias = await cacheGet(`nvcf:alias:${row.replicate_job_id}`).catch(() => null);
+ const taskId = typeof alias === 'string' && alias ? alias : row.replicate_job_id;
+ const status = await nvidia.status({ taskId }).catch(() => null);
if (status?.status === 'done' && status.resultGlbUrl) {
const durable = await materializeCreation({
replicateJobId: row.replicate_job_id,
@@ -316,47 +463,90 @@ export default wrapCron(async (req, res) => {
previewImageUrl: durable.previewImageUrl ?? row.preview_image_url ?? null,
});
}
- continue;
+ return out;
}
- // Copy-to-durable failed (transient storage/network): leave the row
- // for the next tick rather than failing a finished model.
out.still_running++;
- continue;
+ return out;
}
-
- if (status?.status === 'failed') {
- // Inside the attended window a live browser may poll this failure
- // seconds from now and run its own failover; acting here would race
- // it into duplicate GPU work. Leave the row: an attended client
- // resolves it, and an abandoned one ages into the branch below.
- if (decideFailedSweep({ ageMinutes, hasReferenceImage: true }) === 'defer') {
- out.still_running++;
- continue;
+ if ((!status || status.status === 'running' || status.status === 'queued') && row.prompt) {
+ // Still running/queued, or the poll itself errored (status null): no
+ // alternate lane exists for text-only nvidia, so give the job one real
+ // resubmit before the hard TTL reaps it as dead. A distinct lock from
+ // nvcf:resub: below means this can never double-fire with the
+ // nvcf_expired recovery.
+ const stallDecision = decideNvidiaStall({
+ ageMinutes,
+ alreadyResubmitted: Boolean(
+ await cacheGet(`nvcf:stall-resub:${row.replicate_job_id}`).catch(() => null),
+ ),
+ });
+ if (stallDecision === 'resubmit' && (await acquireLock(`nvcf:stall-lock:${row.replicate_job_id}`, NVCF_ALIAS_TTL_S))) {
+ try {
+ const resub = await nvidia.textTo3d({ prompt: row.prompt, tier: resolveTier(row.tier) });
+ await cacheSet(`nvcf:stall-resub:${row.replicate_job_id}`, '1', NVCF_ALIAS_TTL_S);
+ if (resub.taskId) {
+ await cacheSet(`nvcf:alias:${row.replicate_job_id}`, resub.taskId, NVCF_ALIAS_TTL_S);
+ console.warn(
+ `[forge-finalize] NVCF request stalled; resubmitted unattended job (creation ${row.id})`,
+ );
+ out.resubmitted++;
+ return out;
+ }
+ if (resub.resultGlbUrl) {
+ const durable = await materializeCreation({
+ replicateJobId: row.replicate_job_id,
+ clientKey: row.client_key,
+ glbUrl: resub.resultGlbUrl,
+ });
+ if (durable) {
+ out.done++;
+ return out;
+ }
+ }
+ } catch (err) {
+ console.warn(`[forge-finalize] NVCF stall resubmit failed: ${err?.message || err}`);
+ }
}
- if (row.backend) await markLaneUnhealthy(row.backend).catch(() => {});
- const redispatched = await tryCronRedispatch(row, ageMinutes);
- if (redispatched) {
- await markFailed({
- replicateJobId: row.replicate_job_id,
- clientKey: row.client_key,
- error: status.error || 'generation failed',
- });
- // Link the attempt to its successor so the ledger reads as a
- // recovery: without it the health sensor and the error report both
- // count this row as a user-visible loss, when the request is in
- // fact still running on another lane.
- if (redispatched.successorId) {
- await markSupersededBy({
- replicateJobId: row.replicate_job_id,
- clientKey: row.client_key,
- successorId: redispatched.successorId,
- });
+ }
+ if (status?.code === 'nvcf_expired' && row.prompt) {
+ // The request id is dead but the inputs live on the row: the same
+ // never-dead-end recovery the attended poll runs, sharing its
+ // once-per-job resub lock so the two paths can't double-submit.
+ // The alias keeps the row's handle stable; the next tick polls the
+ // resubmitted task and materializes onto this same row.
+ if (await acquireLock(`nvcf:resub:${row.replicate_job_id}`, NVCF_ALIAS_TTL_S)) {
+ try {
+ const resub = await nvidia.textTo3d({ prompt: row.prompt, tier: resolveTier(row.tier) });
+ if (resub.taskId) {
+ await cacheSet(`nvcf:alias:${row.replicate_job_id}`, resub.taskId, NVCF_ALIAS_TTL_S);
+ console.warn(
+ `[forge-finalize] NVCF request expired; resubmitted unattended job (creation ${row.id})`,
+ );
+ out.resubmitted++;
+ return out;
+ }
+ if (resub.resultGlbUrl) {
+ const durable = await materializeCreation({
+ replicateJobId: row.replicate_job_id,
+ clientKey: row.client_key,
+ glbUrl: resub.resultGlbUrl,
+ });
+ if (durable) {
+ out.done++;
+ return out;
+ }
+ }
+ } catch (err) {
+ console.warn(`[forge-finalize] NVCF resubmit failed: ${err?.message || err}`);
}
- out.failed_over++;
- // No failure notification: the successor row carries the job on,
- // and its completion (or terminal failure) notifies instead.
- continue;
}
+ // Lock held by a prior recovery or the resubmit failed: the lane is
+ // cron-observed broken with no alternate text-to-3D lane to send new
+ // submissions to, so cool the same breaker forge.js's submit path
+ // reads before every new free-text request. Fall through to the TTL
+ // check so a second expiry fails honestly, not instantly.
+ await markProviderCooldown(NIM_TRELLIS_COOLDOWN_KEY, nimCooldownSeconds()).catch(() => {});
+ } else if (status?.status === 'failed') {
await markFailed({
replicateJobId: row.replicate_job_id,
clientKey: row.client_key,
@@ -366,195 +556,34 @@ export default wrapCron(async (req, res) => {
if (row.user_id) {
notifyForgeFailed({ userId: row.user_id, creationId: row.id, prompt: row.prompt, error: status.error, backend: row.backend, tier: row.tier });
}
- continue;
+ return out;
}
-
// queued / running / poll blip: fall through to the TTL check.
- } else if (!envelope) {
- // The paid x402 lane stores the FULL signed forge token (provider +
- // upstream task id) so unattended agent purchases can be completed
- // server-side here, so an agent that got its poll_url answer and moved on
- // still ends with a finished gallery row, not a TTL orphan.
- const forgeToken = decodeJobToken(row.replicate_job_id);
- if (forgeToken?.provider === 'nvidia' && forgeToken.taskId && nvidia) {
- const status = await nvidia
- .status({ kind: forgeToken.kind, taskId: forgeToken.taskId })
- .catch(() => null);
- if (status?.status === 'done' && status.resultGlbUrl) {
- const durable = await materializeCreation({
- replicateJobId: row.replicate_job_id,
- clientKey: row.client_key,
- glbUrl: status.resultGlbUrl,
- });
- if (durable) {
- out.done++;
- continue;
- }
- out.still_running++;
- continue;
- }
- if (status?.status === 'failed') {
- await markFailed({
- replicateJobId: row.replicate_job_id,
- clientKey: row.client_key,
- error: status.error || 'generation failed',
- });
- out.failed++;
- continue;
- }
- // queued / running / poll blip: fall through to the TTL check.
- } else if (row.backend === 'nvidia' && nvidia) {
- // Browser/free-lane NIM rows store the BARE NVCF request id (only the
- // x402 lane stores the signed token), so they used to be unpollable
- // here and died at the hard TTL as "generation timed out" even when
- // the GPU had finished. Poll the bare id directly, chasing the alias
- // an attended expiry recovery may have written first.
- const alias = await cacheGet(`nvcf:alias:${row.replicate_job_id}`).catch(() => null);
- const taskId = typeof alias === 'string' && alias ? alias : row.replicate_job_id;
- const status = await nvidia.status({ taskId }).catch(() => null);
- if (status?.status === 'done' && status.resultGlbUrl) {
- const durable = await materializeCreation({
- replicateJobId: row.replicate_job_id,
- clientKey: row.client_key,
- glbUrl: status.resultGlbUrl,
- });
- if (durable) {
- out.done++;
- if (row.user_id) {
- notifyForgeComplete({
- userId: row.user_id,
- creationId: durable.id,
- prompt: row.prompt,
- previewImageUrl: durable.previewImageUrl ?? row.preview_image_url ?? null,
- });
- }
- continue;
- }
- out.still_running++;
- continue;
- }
- if ((!status || status.status === 'running' || status.status === 'queued') && row.prompt) {
- // Still running/queued, or the poll itself errored (status null): no
- // alternate lane exists for text-only nvidia, so give the job one real
- // resubmit before the hard TTL reaps it as dead. A distinct lock from
- // nvcf:resub: below means this can never double-fire with the
- // nvcf_expired recovery.
- const stallDecision = decideNvidiaStall({
- ageMinutes,
- alreadyResubmitted: Boolean(
- await cacheGet(`nvcf:stall-resub:${row.replicate_job_id}`).catch(() => null),
- ),
- });
- if (stallDecision === 'resubmit' && (await acquireLock(`nvcf:stall-lock:${row.replicate_job_id}`, NVCF_ALIAS_TTL_S))) {
- try {
- const resub = await nvidia.textTo3d({ prompt: row.prompt, tier: resolveTier(row.tier) });
- await cacheSet(`nvcf:stall-resub:${row.replicate_job_id}`, '1', NVCF_ALIAS_TTL_S);
- if (resub.taskId) {
- await cacheSet(`nvcf:alias:${row.replicate_job_id}`, resub.taskId, NVCF_ALIAS_TTL_S);
- console.warn(
- `[forge-finalize] NVCF request stalled; resubmitted unattended job (creation ${row.id})`,
- );
- out.resubmitted++;
- continue;
- }
- if (resub.resultGlbUrl) {
- const durable = await materializeCreation({
- replicateJobId: row.replicate_job_id,
- clientKey: row.client_key,
- glbUrl: resub.resultGlbUrl,
- });
- if (durable) {
- out.done++;
- continue;
- }
- }
- } catch (err) {
- console.warn(`[forge-finalize] NVCF stall resubmit failed: ${err?.message || err}`);
- }
- }
- }
- if (status?.code === 'nvcf_expired' && row.prompt) {
- // The request id is dead but the inputs live on the row — the same
- // never-dead-end recovery the attended poll runs, sharing its
- // once-per-job resub lock so the two paths can't double-submit.
- // The alias keeps the row's handle stable; the next tick polls the
- // resubmitted task and materializes onto this same row.
- if (await acquireLock(`nvcf:resub:${row.replicate_job_id}`, NVCF_ALIAS_TTL_S)) {
- try {
- const resub = await nvidia.textTo3d({ prompt: row.prompt, tier: resolveTier(row.tier) });
- if (resub.taskId) {
- await cacheSet(`nvcf:alias:${row.replicate_job_id}`, resub.taskId, NVCF_ALIAS_TTL_S);
- console.warn(
- `[forge-finalize] NVCF request expired; resubmitted unattended job (creation ${row.id})`,
- );
- out.resubmitted++;
- continue;
- }
- if (resub.resultGlbUrl) {
- const durable = await materializeCreation({
- replicateJobId: row.replicate_job_id,
- clientKey: row.client_key,
- glbUrl: resub.resultGlbUrl,
- });
- if (durable) {
- out.done++;
- continue;
- }
- }
- } catch (err) {
- console.warn(`[forge-finalize] NVCF resubmit failed: ${err?.message || err}`);
- }
- }
- // Lock held by a prior recovery or the resubmit failed: the lane is
- // cron-observed broken with no alternate text-to-3D lane to send new
- // submissions to, so cool the same breaker forge.js's submit path
- // reads before every new free-text request. Fall through to the TTL
- // check so a second expiry fails honestly, not instantly.
- await markProviderCooldown(NIM_TRELLIS_COOLDOWN_KEY, nimCooldownSeconds()).catch(() => {});
- } else if (status?.status === 'failed') {
- await markFailed({
- replicateJobId: row.replicate_job_id,
- clientKey: row.client_key,
- error: status.error || 'generation failed',
- });
- out.failed++;
- if (row.user_id) {
- notifyForgeFailed({ userId: row.user_id, creationId: row.id, prompt: row.prompt, error: status.error, backend: row.backend, tier: row.tier });
- }
- continue;
- }
- // queued / running / poll blip: fall through to the TTL check.
- } else {
- out.unpollable++;
- }
+ } else {
+ out.unpollable++;
}
+ }
- if (ageMinutes > HARD_TTL_MINUTES) {
- if (row.backend === 'nvidia') {
- // Reaping the bare TTL on nvidia is itself a cron-observed failure of
- // the lane: cool the submit-time breaker so new free-text requests
- // skip straight to the paid/image-intermediate path instead of landing
- // on the same dead end for another HARD_TTL_MINUTES.
- await markProviderCooldown(NIM_TRELLIS_COOLDOWN_KEY, nimCooldownSeconds()).catch(() => {});
- }
- await markFailed({
- replicateJobId: row.replicate_job_id,
- clientKey: row.client_key,
- error: `generation timed out after ${Math.round(ageMinutes)} minutes`,
- });
- out.timed_out++;
- if (row.user_id) {
- notifyForgeFailed({ userId: row.user_id, creationId: row.id, prompt: row.prompt, error: 'generation timed out', backend: row.backend, tier: row.tier });
- }
- } else if (envelope) {
- out.still_running++;
+ if (ageMinutes > HARD_TTL_MINUTES) {
+ if (row.backend === 'nvidia') {
+ // Reaping the bare TTL on nvidia is itself a cron-observed failure of
+ // the lane: cool the submit-time breaker so new free-text requests
+ // skip straight to the paid/image-intermediate path instead of landing
+ // on the same dead end for another HARD_TTL_MINUTES.
+ await markProviderCooldown(NIM_TRELLIS_COOLDOWN_KEY, nimCooldownSeconds()).catch(() => {});
+ }
+ await markFailed({
+ replicateJobId: row.replicate_job_id,
+ clientKey: row.client_key,
+ error: `generation timed out after ${Math.round(ageMinutes)} minutes`,
+ });
+ out.timed_out++;
+ if (row.user_id) {
+ notifyForgeFailed({ userId: row.user_id, creationId: row.id, prompt: row.prompt, error: 'generation timed out', backend: row.backend, tier: row.tier });
}
+ } else if (envelope) {
+ out.still_running++;
}
- return json(res, 200, {
- enabled: true,
- swept: rows.length,
- ...out,
- elapsed_ms: Date.now() - t0,
- });
-}, { maxDurationMs: maxDuration * 1000 });
+ return out;
+}
diff --git a/tests/forge-finalize.test.js b/tests/forge-finalize.test.js
index aa2c66327..1adcd8521 100644
--- a/tests/forge-finalize.test.js
+++ b/tests/forge-finalize.test.js
@@ -4,7 +4,8 @@
// Importing the cron module is safe: db/neon clients are created lazily.
import { describe, it, expect } from 'vitest';
-import { decodeWorkerEnvelope, decideFailedSweep, decideNvidiaStall } from '../api/cron/forge-finalize.js';
+import { decodeWorkerEnvelope, decideFailedSweep, decideNvidiaStall, sweepRow, ROW_CONCURRENCY } from '../api/cron/forge-finalize.js';
+import { mapPoolSettled } from '../api/_lib/pool.js';
function pack(obj) {
return Buffer.from(JSON.stringify(obj), 'utf8').toString('base64url');
@@ -112,3 +113,75 @@ describe('decideNvidiaStall', () => {
expect(decideNvidiaStall({ ageMinutes: undefined })).toBe('defer');
});
});
+
+// 2026-10-24 production incident: the cron's own `[cron] forge-finalize
+// exceeded its 60000ms budget` warning fired dozens of times in a few hours
+// (docs/ops/production-log-triage.md "HTTP timeout with no cron context").
+// Root cause: every row's gcp.status() poll is an independent network round
+// trip (up to 15s x 2 attempts, api/_providers/gcp.js), and the sweep ran them
+// one row at a time in a plain `for` loop, so a handful of slow rows against an
+// unhealthy lane (trellis_selfhost that day) summed past the 60s budget. Rows
+// are independent (keyed by their own replicate_job_id, no shared mutable
+// state besides the stateless provider clients), so they can run concurrently
+// instead of blocking on each other.
+describe('sweepRow concurrency', () => {
+ function envelopeRow(n, { ageMinutes = 5 } = {}) {
+ return {
+ id: `row-${n}`,
+ replicate_job_id: pack({ mode: 'trellis', taskId: `task-${n}`, baseUrl: 'https://worker.example.test' }),
+ client_key: `client-${n}`,
+ user_id: null,
+ prompt: null,
+ preview_image_url: null,
+ backend: 'trellis_selfhost',
+ tier: 'draft',
+ path: null,
+ created_at: new Date(Date.now() - ageMinutes * 60_000).toISOString(),
+ destination: null,
+ internal: false,
+ };
+ }
+
+ // A fake worker that answers after POLL_DELAY_MS, like a real HTTP round
+ // trip to a loaded or degraded GCP lane.
+ const POLL_DELAY_MS = 60;
+ const ROW_COUNT = 10;
+
+ function slowGcp() {
+ return {
+ status: async () => {
+ await new Promise((resolve) => setTimeout(resolve, POLL_DELAY_MS));
+ return { status: 'running' };
+ },
+ };
+ }
+
+ it('sweeps a batch in roughly one row\'s latency, not the sum of every row\'s latency', async () => {
+ const rows = Array.from({ length: ROW_COUNT }, (_, i) => envelopeRow(i));
+ const gcp = slowGcp();
+
+ const t0 = Date.now();
+ const results = await mapPoolSettled(rows, ROW_CONCURRENCY, (row) => sweepRow(row, { gcp, nvidia: null }));
+ const elapsedMs = Date.now() - t0;
+
+ expect(results.every((r) => r.ok)).toBe(true);
+ // Sequential (the pre-fix `for` loop) would take ~ROW_COUNT * POLL_DELAY_MS
+ // (600ms here). Concurrent sweeping bounds it to a couple of waves.
+ expect(elapsedMs).toBeLessThan(ROW_COUNT * POLL_DELAY_MS * 0.6);
+ });
+
+ it('defaults to sweeping more than one row at a time', () => {
+ // A regression to ROW_CONCURRENCY = 1 silently reintroduces the serial
+ // sweep this test exists to prevent.
+ expect(ROW_CONCURRENCY).toBeGreaterThan(1);
+ });
+
+ it('still_running still counts every polled-but-not-done row when swept concurrently', async () => {
+ const rows = Array.from({ length: 4 }, (_, i) => envelopeRow(i));
+ const gcp = slowGcp();
+
+ const results = await mapPoolSettled(rows, ROW_CONCURRENCY, (row) => sweepRow(row, { gcp, nvidia: null }));
+ const totalStillRunning = results.reduce((sum, r) => sum + (r.ok ? r.value.still_running : 0), 0);
+ expect(totalStillRunning).toBe(4);
+ });
+});