Skip to content

Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.

feat(forge): measure self-host lane occupancy and report honest job timing

nirholas1a13e9d622ca
diff --git a/api/_lib/forge-scale.js b/api/_lib/forge-scale.js
index 0dcffb420..72fac0998 100644
--- a/api/_lib/forge-scale.js
+++ b/api/_lib/forge-scale.js
@@ -101,6 +101,77 @@ export async function clearInFlight(hash) {
}
}
+// ── Self-host lane occupancy ─────────────────────────────────────────────────
+// How many generations WE have in flight on each of our own GPU lanes right now.
+//
+// This exists because liveness is not capacity. Our GPU workers accept a job
+// with a 202 and run it on a background task, so an overloaded worker still
+// answers /health instantly and still reads `ok` to the health-aware router,
+// which then hands it the next job, and the next. Cloud Run cannot see the
+// backlog either: request concurrency never rises above the poll traffic, so the
+// autoscaler leaves the service at one instance no matter how deep the queue is.
+// A 10-job load test on 2026-09-04 landed all ten on the single warm TRELLIS
+// instance and 7 were still queued 12 minutes later, while the Hunyuan3D worker
+// (a different accelerator, its own quota, min-scale 1, completely idle) and both
+// free external lanes sat unused.
+//
+// The count is of OUR OWN submissions, not a guess at the worker's internals: one
+// sorted-set member per live job keyed by its handle, scored with an expiry, so
+// an instance that dies without ever seeing the job finish cannot pin the lane
+// (the same self-healing eviction acquireBlockingSlot uses below). Best-effort
+// throughout: without Redis every lane reports 0 occupancy and routing behaves
+// exactly as it did before this existed.
+
+const LANE_BUSY_PREFIX = 'fc:lane:';
+// A generation that has not reached a terminal poll by now is not still running:
+// it is a client that closed the tab, or a job whose successor took over. Matches
+// INFLIGHT_TTL_S, the same "longest a generation stays pollable" horizon.
+const LANE_BUSY_TTL_S = INFLIGHT_TTL_S;
+
+// Record that `jobHandle` now occupies `backendId`. Called at submit.
+export async function markLaneJobStarted(backendId, jobHandle) {
+ if (!redis || !backendId || !jobHandle) return;
+ const key = `${LANE_BUSY_PREFIX}${backendId}`;
+ try {
+ await redis.zadd(key, { score: Date.now() + LANE_BUSY_TTL_S * 1000, member: String(jobHandle) });
+ await redis.expire(key, LANE_BUSY_TTL_S + 10);
+ } catch {
+ /* best-effort: a missed mark just means the lane looks emptier */
+ }
+}
+
+// Release a lane slot. Called when a poll reaches a terminal status, and when a
+// submit throws before the job ever ran.
+export async function markLaneJobFinished(backendId, jobHandle) {
+ if (!redis || !backendId || !jobHandle) return;
+ try {
+ await redis.zrem(`${LANE_BUSY_PREFIX}${backendId}`, String(jobHandle));
+ } catch {
+ /* the score-based eviction reclaims it */
+ }
+}
+
+// Live occupancy for a set of lane ids, as { [id]: count }. Expired members are
+// evicted first so a stale entry can never make a lane look permanently full.
+// Returns an empty object (every lane unconstrained) without Redis or on error.
+export async function laneOccupancy(backendIds = []) {
+ const ids = [...new Set(backendIds)].filter(Boolean);
+ if (!redis || !ids.length) return {};
+ const now = Date.now();
+ try {
+ const counts = await Promise.all(
+ ids.map(async (id) => {
+ const key = `${LANE_BUSY_PREFIX}${id}`;
+ await redis.zremrangebyscore(key, 0, now);
+ return [id, Number(await redis.zcard(key)) || 0];
+ }),
+ );
+ return Object.fromEntries(counts);
+ } catch {
+ return {};
+ }
+}
+
// ── Bounded-concurrency lease (for the blocking free lane) ────────────────────
// The HuggingFace Spaces lane BLOCKS a serverless worker for up to ~280s per call
// (see huggingface.js HF_INFERENCE_TIMEOUT_MS). Past a few dozen concurrent holds,
diff --git a/api/forge.js b/api/forge.js
index 11ba25a9f..526f4f183 100644
--- a/api/forge.js
+++ b/api/forge.js
@@ -164,6 +164,28 @@ const NIM_TRELLIS_COOLDOWN_KEY = 'forge-nim-trellis';
const NIM_FORGE_COOLDOWN_SECONDS = 120;
const NIM_FORGE_GATEWAY_COOLDOWN_SECONDS = 30;
+// Honest cold-start signal for a self-host lane: true only when the liveness
+// probe reached the worker but it answered slowly (a scale-to-zero container
+// spinning up). Reuses the cached lane-health snapshot, so the common path pays
+// no extra probe. Used to widen the ETA + flag `cold_start`, never to fabricate
+// progress; the client's real polling still drives actual status.
+//
+// Module-level because BOTH ends of a job need it. It used to be a closure inside
+// the submit handler, so only the submit response could say a worker was booting;
+// a client that never saw that response (a coalesced job, a page resumed after a
+// reload, a poll-time failover successor) was told "queued" with no explanation
+// for a wait that is a container boot, not a stall.
+async function laneColdStart(backendId) {
+ if (!backendId || !isSelfHostBackend(backendId) || !coldStartSecondsFor(backendId)) return false;
+ try {
+ const snap = await laneHealthSnapshot([backendId]);
+ const rec = snap.byId[backendId];
+ return Boolean(rec && rec.status === 'ok' && rec.warm === false);
+ } catch {
+ return false;
+ }
+}
+
// The platform-keyed paid reconstruct lane (Replicate TRELLIS) recorded as down.
// Set only on an out-of-credit/billing failure: which won't self-heal until ops
// tops the account up, so the window is long (reason 'auth'). The NIM-cooldown
@@ -1148,21 +1170,7 @@ async function startJob(req, res) {
}
}
- // Honest cold-start signal for a chosen self-host lane: true only when the
- // liveness probe reached the worker but it answered slowly (a scale-to-zero
- // container spinning up). Reuses the cached snapshot, so the common path pays no
- // extra probe. Used to widen the ETA + flag `cold_start` in the response, never
- // to fabricate progress; the client's real polling still drives actual status.
- const coldStartFor = async (id) => {
- if (!isSelfHostBackend(id) || !coldStartSecondsFor(id)) return false;
- try {
- const snap = await laneHealthSnapshot([id]);
- const rec = snap.byId[id];
- return Boolean(rec && rec.status === 'ok' && rec.warm === false);
- } catch {
- return false;
- }
- };
+ const coldStartFor = laneColdStart;
// Content-addressed result cache (see _lib/forge-cache.js): a text→3D request
// on a platform-keyed lane (never BYOK: that spends the caller's own account,
@@ -2792,6 +2800,34 @@ async function pollJob(req, res, jobId) {
}
: {};
+ // Honest timing on every non-terminal poll: how long this job has actually
+ // been alive, the lane's typical total, and what is left of it. /api/gpt-forge
+ // has returned these since the ChatGPT surface needed them; /api/forge is the
+ // endpoint every browser and MCP client polls and it returned neither, so each
+ // caller either invented its own copy of the estimate or fell back to a static
+ // "usually under a minute" line that nothing measured.
+ //
+ // eta_remaining_seconds is floored at 5 for the reason gpt-forge floors it: the
+ // estimate is a typical duration, not a deadline, and a countdown that goes
+ // negative reads as "stuck" for a job that is merely slower than average.
+ const elapsedSeconds = meta?.created_at
+ ? Math.max(0, Math.round((Date.now() - Date.parse(meta.created_at)) / 1000))
+ : null;
+ const etaTotalSeconds = meta?.backend
+ ? estimateEtaSeconds({ backendId: meta.backend, tier: meta.tier })
+ : null;
+ const timingFields = {
+ ...(Number.isFinite(elapsedSeconds) ? { elapsed_seconds: elapsedSeconds } : {}),
+ ...(Number.isFinite(etaTotalSeconds) && etaTotalSeconds > 0
+ ? {
+ eta_seconds: etaTotalSeconds,
+ eta_remaining_seconds: Number.isFinite(elapsedSeconds)
+ ? Math.max(5, etaTotalSeconds - elapsedSeconds)
+ : etaTotalSeconds,
+ }
+ : {}),
+ };
+
// Poll the provider that owns this job. BYOK providers re-resolve the key per
// poll (the client resends it, or it loads from the session store).
let result;
@@ -2864,7 +2900,7 @@ async function pollJob(req, res, jobId) {
} catch {
// A transient poll error shouldn't fail the job: report running so the
// client's loop retries.
- return json(res, 200, { job_id: jobId, status: 'running', ...metaFields });
+ return json(res, 200, { job_id: jobId, status: 'running', ...metaFields, ...timingFields });
}
if (result.status === 'done' && result.resultGlbUrl) {
@@ -3004,6 +3040,7 @@ async function pollJob(req, res, jobId) {
job_id: jobId,
status: 'running',
...metaFields,
+ ...timingFields,
backend: nextLane,
failover_from: failedBackend,
});
@@ -3030,7 +3067,23 @@ async function pollJob(req, res, jobId) {
...(suggestions.length ? { retryable: true, retry_backends: suggestions } : {}),
});
}
- return json(res, 200, { job_id: jobId, status: result.status || 'running', ...metaFields });
+ // A job still QUEUED on a scale-to-zero self-host worker is waiting on a
+ // container boot, and that is the one wait a client can explain instead of
+ // showing a stalled bar. Resolved from the cached lane-health snapshot (one
+ // probe per 20s per instance, shared across every concurrent poll), and only
+ // while queued: a worker that answered "running" is up by definition, so the
+ // flag clears on a real signal rather than on a timer.
+ const pendingStatus = result.status || 'running';
+ const cold = pendingStatus === 'queued' ? await laneColdStart(metaFields.backend) : false;
+ return json(res, 200, {
+ job_id: jobId,
+ status: pendingStatus,
+ ...metaFields,
+ ...timingFields,
+ ...(cold
+ ? { cold_start: true, cold_start_seconds: coldStartSecondsFor(metaFields.backend) }
+ : {}),
+ });
}
export default wrap(async (req, res) => {
diff --git a/src/shared/forge-frames.js b/src/shared/forge-frames.js
index a471d5d1f..91f28d9f8 100644
--- a/src/shared/forge-frames.js
+++ b/src/shared/forge-frames.js
@@ -77,10 +77,22 @@ export function laneLabel(backendId) {
// failed → generation failed
// `mode` ('image' before the mesh stage) and `eta_seconds` refine the line when
// the pipeline reports them. Unknown states fall back to a neutral "working".
+//
+// Two optional numbers make a wait measurable instead of merely named, and both
+// come off the poll payload (/api/forge returns `elapsed_seconds` and
+// `eta_remaining_seconds` on every non-terminal poll): `elapsed_seconds` is how
+// long the job has really been alive, `eta_remaining_seconds` is the lane's
+// typical total minus that. Neither is ever synthesized here from a local clock,
+// so a page reopened mid-generation reports the job's age, not the tab's.
export function forgeStageNarration(state = {}) {
const status = String(state.status || state.stage || '').toLowerCase();
- const eta = Number(state.eta_seconds) > 0 ? Math.round(Number(state.eta_seconds)) : null;
- const etaSuffix = eta ? ` — ~${eta}s` : '';
+ const num = (v) => (Number(v) > 0 ? Math.round(Number(v)) : null);
+ const eta = num(state.eta_seconds);
+ const elapsed = Number(state.elapsed_seconds) >= 0 ? Math.round(Number(state.elapsed_seconds)) : null;
+ // Prefer the live remaining estimate: repeating the lane's static total on
+ // every frame is what made a poll loop read as "stuck at ~60s" for a minute.
+ const remaining = num(state.eta_remaining_seconds) ?? eta;
+ const etaSuffix = remaining ? ` (~${remaining}s left)` : '';
// The lane that actually ran, when the caller knows it. Older callers pass no
// `backend`, so every line degrades to a lane-agnostic phrasing rather than
// naming the wrong engine.
@@ -90,8 +102,11 @@ export function forgeStageNarration(state = {}) {
// `cold_seconds` is the lane's spin-up budget from the API, never a timer we
// invent; without it we still name the state but promise no number.
const cold = Boolean(state.cold_start);
- const coldSeconds = Number(state.cold_seconds) > 0 ? Math.round(Number(state.cold_seconds)) : null;
+ const coldSeconds = num(state.cold_seconds);
const coldWho = lane ? ` ${lane}` : '';
+ // Elapsed is only worth showing once it means something; under 5s it just
+ // flickers. A boot past its budget says so instead of counting into negatives.
+ const elapsedSuffix = elapsed != null && elapsed >= 5 ? ` (${elapsed}s in)` : '';
switch (status) {
case 'submitting':
@@ -100,18 +115,23 @@ export function forgeStageNarration(state = {}) {
case 'queued':
case 'queue':
if (cold) {
- return coldSeconds
- ? `Waking up the${coldWho} GPU worker (about ${coldSeconds}s), then sculpting starts`
- : `Waking up the${coldWho} GPU worker, then sculpting starts`;
+ if (!coldSeconds) return `Waking up the${coldWho} GPU worker, then sculpting starts`;
+ const bootLeft = elapsed != null ? coldSeconds - elapsed : null;
+ if (bootLeft != null && bootLeft <= 0) {
+ return `Still waking the${coldWho} GPU worker${elapsedSuffix}. The job is accepted and starts the moment it answers`;
+ }
+ return bootLeft != null
+ ? `Waking up the${coldWho} GPU worker${elapsedSuffix}, about ${bootLeft}s of boot left, then sculpting starts`
+ : `Waking up the${coldWho} GPU worker (about ${coldSeconds}s), then sculpting starts`;
}
- return `Queued${laneSuffix}${etaSuffix}`;
+ return `Queued${laneSuffix}${elapsedSuffix}${etaSuffix}`;
case 'image':
case 'texturing':
return 'Drafting the look…';
case 'running':
case 'reconstruct':
case 'mesh':
- return `Building geometry & texturing${etaSuffix}`;
+ return `Building geometry & texturing${elapsedSuffix}${etaSuffix}`;
case 'done':
case 'ready':
return 'Model ready — loading into the cam';