Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.
JavaScript 3d-generationai-agentsanimationavatardigital-humangenerative-aiglbgltfimage-to-3dllmmcpmcp-servermodel-context-protocolsolanatext-to-3dthreejsweb-componentswebglwebxrx402
feat(forge): measure self-host lane occupancy and report honest job timing
nirholas1a13e9d622ca
diff --git a/api/_lib/forge-scale.js b/api/_lib/forge-scale.jsindex 0dcffb420..72fac0998 100644--- a/api/_lib/forge-scale.js+++ b/api/_lib/forge-scale.js@@ -101,6 +101,77 @@ export async function clearInFlight(hash) {}}+// ── Self-host lane occupancy ─────────────────────────────────────────────────+// How many generations WE have in flight on each of our own GPU lanes right now.+//+// This exists because liveness is not capacity. Our GPU workers accept a job+// with a 202 and run it on a background task, so an overloaded worker still+// answers /health instantly and still reads `ok` to the health-aware router,+// which then hands it the next job, and the next. Cloud Run cannot see the+// backlog either: request concurrency never rises above the poll traffic, so the+// autoscaler leaves the service at one instance no matter how deep the queue is.+// A 10-job load test on 2026-09-04 landed all ten on the single warm TRELLIS+// instance and 7 were still queued 12 minutes later, while the Hunyuan3D worker+// (a different accelerator, its own quota, min-scale 1, completely idle) and both+// free external lanes sat unused.+//+// The count is of OUR OWN submissions, not a guess at the worker's internals: one+// sorted-set member per live job keyed by its handle, scored with an expiry, so+// an instance that dies without ever seeing the job finish cannot pin the lane+// (the same self-healing eviction acquireBlockingSlot uses below). Best-effort+// throughout: without Redis every lane reports 0 occupancy and routing behaves+// exactly as it did before this existed.++const LANE_BUSY_PREFIX = 'fc:lane:';+// A generation that has not reached a terminal poll by now is not still running:+// it is a client that closed the tab, or a job whose successor took over. Matches+// INFLIGHT_TTL_S, the same "longest a generation stays pollable" horizon.+const LANE_BUSY_TTL_S = INFLIGHT_TTL_S;++// Record that `jobHandle` now occupies `backendId`. Called at submit.+export async function markLaneJobStarted(backendId, jobHandle) {+ if (!redis || !backendId || !jobHandle) return;+ const key = `${LANE_BUSY_PREFIX}${backendId}`;+ try {+ await redis.zadd(key, { score: Date.now() + LANE_BUSY_TTL_S * 1000, member: String(jobHandle) });+ await redis.expire(key, LANE_BUSY_TTL_S + 10);+ } catch {+ /* best-effort: a missed mark just means the lane looks emptier */+ }+}++// Release a lane slot. Called when a poll reaches a terminal status, and when a+// submit throws before the job ever ran.+export async function markLaneJobFinished(backendId, jobHandle) {+ if (!redis || !backendId || !jobHandle) return;+ try {+ await redis.zrem(`${LANE_BUSY_PREFIX}${backendId}`, String(jobHandle));+ } catch {+ /* the score-based eviction reclaims it */+ }+}++// Live occupancy for a set of lane ids, as { [id]: count }. Expired members are+// evicted first so a stale entry can never make a lane look permanently full.+// Returns an empty object (every lane unconstrained) without Redis or on error.+export async function laneOccupancy(backendIds = []) {+ const ids = [...new Set(backendIds)].filter(Boolean);+ if (!redis || !ids.length) return {};+ const now = Date.now();+ try {+ const counts = await Promise.all(+ ids.map(async (id) => {+ const key = `${LANE_BUSY_PREFIX}${id}`;+ await redis.zremrangebyscore(key, 0, now);+ return [id, Number(await redis.zcard(key)) || 0];+ }),+ );+ return Object.fromEntries(counts);+ } catch {+ return {};+ }+}+// ── Bounded-concurrency lease (for the blocking free lane) ────────────────────// The HuggingFace Spaces lane BLOCKS a serverless worker for up to ~280s per call// (see huggingface.js HF_INFERENCE_TIMEOUT_MS). Past a few dozen concurrent holds,diff --git a/api/forge.js b/api/forge.jsindex 11ba25a9f..526f4f183 100644--- a/api/forge.js+++ b/api/forge.js@@ -164,6 +164,28 @@ const NIM_TRELLIS_COOLDOWN_KEY = 'forge-nim-trellis';const NIM_FORGE_COOLDOWN_SECONDS = 120;const NIM_FORGE_GATEWAY_COOLDOWN_SECONDS = 30;+// Honest cold-start signal for a self-host lane: true only when the liveness+// probe reached the worker but it answered slowly (a scale-to-zero container+// spinning up). Reuses the cached lane-health snapshot, so the common path pays+// no extra probe. Used to widen the ETA + flag `cold_start`, never to fabricate+// progress; the client's real polling still drives actual status.+//+// Module-level because BOTH ends of a job need it. It used to be a closure inside+// the submit handler, so only the submit response could say a worker was booting;+// a client that never saw that response (a coalesced job, a page resumed after a+// reload, a poll-time failover successor) was told "queued" with no explanation+// for a wait that is a container boot, not a stall.+async function laneColdStart(backendId) {+ if (!backendId || !isSelfHostBackend(backendId) || !coldStartSecondsFor(backendId)) return false;+ try {+ const snap = await laneHealthSnapshot([backendId]);+ const rec = snap.byId[backendId];+ return Boolean(rec && rec.status === 'ok' && rec.warm === false);+ } catch {+ return false;+ }+}+// The platform-keyed paid reconstruct lane (Replicate TRELLIS) recorded as down.// Set only on an out-of-credit/billing failure: which won't self-heal until ops// tops the account up, so the window is long (reason 'auth'). The NIM-cooldown@@ -1148,21 +1170,7 @@ async function startJob(req, res) {}}- // Honest cold-start signal for a chosen self-host lane: true only when the- // liveness probe reached the worker but it answered slowly (a scale-to-zero- // container spinning up). Reuses the cached snapshot, so the common path pays no- // extra probe. Used to widen the ETA + flag `cold_start` in the response, never- // to fabricate progress; the client's real polling still drives actual status.- const coldStartFor = async (id) => {- if (!isSelfHostBackend(id) || !coldStartSecondsFor(id)) return false;- try {- const snap = await laneHealthSnapshot([id]);- const rec = snap.byId[id];- return Boolean(rec && rec.status === 'ok' && rec.warm === false);- } catch {- return false;- }- };+ const coldStartFor = laneColdStart;// Content-addressed result cache (see _lib/forge-cache.js): a text→3D request// on a platform-keyed lane (never BYOK: that spends the caller's own account,@@ -2792,6 +2800,34 @@ async function pollJob(req, res, jobId) {}: {};+ // Honest timing on every non-terminal poll: how long this job has actually+ // been alive, the lane's typical total, and what is left of it. /api/gpt-forge+ // has returned these since the ChatGPT surface needed them; /api/forge is the+ // endpoint every browser and MCP client polls and it returned neither, so each+ // caller either invented its own copy of the estimate or fell back to a static+ // "usually under a minute" line that nothing measured.+ //+ // eta_remaining_seconds is floored at 5 for the reason gpt-forge floors it: the+ // estimate is a typical duration, not a deadline, and a countdown that goes+ // negative reads as "stuck" for a job that is merely slower than average.+ const elapsedSeconds = meta?.created_at+ ? Math.max(0, Math.round((Date.now() - Date.parse(meta.created_at)) / 1000))+ : null;+ const etaTotalSeconds = meta?.backend+ ? estimateEtaSeconds({ backendId: meta.backend, tier: meta.tier })+ : null;+ const timingFields = {+ ...(Number.isFinite(elapsedSeconds) ? { elapsed_seconds: elapsedSeconds } : {}),+ ...(Number.isFinite(etaTotalSeconds) && etaTotalSeconds > 0+ ? {+ eta_seconds: etaTotalSeconds,+ eta_remaining_seconds: Number.isFinite(elapsedSeconds)+ ? Math.max(5, etaTotalSeconds - elapsedSeconds)+ : etaTotalSeconds,+ }+ : {}),+ };+// Poll the provider that owns this job. BYOK providers re-resolve the key per// poll (the client resends it, or it loads from the session store).let result;@@ -2864,7 +2900,7 @@ async function pollJob(req, res, jobId) {} catch {// A transient poll error shouldn't fail the job: report running so the// client's loop retries.- return json(res, 200, { job_id: jobId, status: 'running', ...metaFields });+ return json(res, 200, { job_id: jobId, status: 'running', ...metaFields, ...timingFields });}if (result.status === 'done' && result.resultGlbUrl) {@@ -3004,6 +3040,7 @@ async function pollJob(req, res, jobId) {job_id: jobId,status: 'running',...metaFields,+ ...timingFields,backend: nextLane,failover_from: failedBackend,});@@ -3030,7 +3067,23 @@ async function pollJob(req, res, jobId) {...(suggestions.length ? { retryable: true, retry_backends: suggestions } : {}),});}- return json(res, 200, { job_id: jobId, status: result.status || 'running', ...metaFields });+ // A job still QUEUED on a scale-to-zero self-host worker is waiting on a+ // container boot, and that is the one wait a client can explain instead of+ // showing a stalled bar. Resolved from the cached lane-health snapshot (one+ // probe per 20s per instance, shared across every concurrent poll), and only+ // while queued: a worker that answered "running" is up by definition, so the+ // flag clears on a real signal rather than on a timer.+ const pendingStatus = result.status || 'running';+ const cold = pendingStatus === 'queued' ? await laneColdStart(metaFields.backend) : false;+ return json(res, 200, {+ job_id: jobId,+ status: pendingStatus,+ ...metaFields,+ ...timingFields,+ ...(cold+ ? { cold_start: true, cold_start_seconds: coldStartSecondsFor(metaFields.backend) }+ : {}),+ });}export default wrap(async (req, res) => {diff --git a/src/shared/forge-frames.js b/src/shared/forge-frames.jsindex a471d5d1f..91f28d9f8 100644--- a/src/shared/forge-frames.js+++ b/src/shared/forge-frames.js@@ -77,10 +77,22 @@ export function laneLabel(backendId) {// failed → generation failed// `mode` ('image' before the mesh stage) and `eta_seconds` refine the line when// the pipeline reports them. Unknown states fall back to a neutral "working".+//+// Two optional numbers make a wait measurable instead of merely named, and both+// come off the poll payload (/api/forge returns `elapsed_seconds` and+// `eta_remaining_seconds` on every non-terminal poll): `elapsed_seconds` is how+// long the job has really been alive, `eta_remaining_seconds` is the lane's+// typical total minus that. Neither is ever synthesized here from a local clock,+// so a page reopened mid-generation reports the job's age, not the tab's.export function forgeStageNarration(state = {}) {const status = String(state.status || state.stage || '').toLowerCase();- const eta = Number(state.eta_seconds) > 0 ? Math.round(Number(state.eta_seconds)) : null;- const etaSuffix = eta ? ` — ~${eta}s` : '';+ const num = (v) => (Number(v) > 0 ? Math.round(Number(v)) : null);+ const eta = num(state.eta_seconds);+ const elapsed = Number(state.elapsed_seconds) >= 0 ? Math.round(Number(state.elapsed_seconds)) : null;+ // Prefer the live remaining estimate: repeating the lane's static total on+ // every frame is what made a poll loop read as "stuck at ~60s" for a minute.+ const remaining = num(state.eta_remaining_seconds) ?? eta;+ const etaSuffix = remaining ? ` (~${remaining}s left)` : '';// The lane that actually ran, when the caller knows it. Older callers pass no// `backend`, so every line degrades to a lane-agnostic phrasing rather than// naming the wrong engine.@@ -90,8 +102,11 @@ export function forgeStageNarration(state = {}) {// `cold_seconds` is the lane's spin-up budget from the API, never a timer we// invent; without it we still name the state but promise no number.const cold = Boolean(state.cold_start);- const coldSeconds = Number(state.cold_seconds) > 0 ? Math.round(Number(state.cold_seconds)) : null;+ const coldSeconds = num(state.cold_seconds);const coldWho = lane ? ` ${lane}` : '';+ // Elapsed is only worth showing once it means something; under 5s it just+ // flickers. A boot past its budget says so instead of counting into negatives.+ const elapsedSuffix = elapsed != null && elapsed >= 5 ? ` (${elapsed}s in)` : '';switch (status) {case 'submitting':@@ -100,18 +115,23 @@ export function forgeStageNarration(state = {}) {case 'queued':case 'queue':if (cold) {- return coldSeconds- ? `Waking up the${coldWho} GPU worker (about ${coldSeconds}s), then sculpting starts`- : `Waking up the${coldWho} GPU worker, then sculpting starts`;+ if (!coldSeconds) return `Waking up the${coldWho} GPU worker, then sculpting starts`;+ const bootLeft = elapsed != null ? coldSeconds - elapsed : null;+ if (bootLeft != null && bootLeft <= 0) {+ return `Still waking the${coldWho} GPU worker${elapsedSuffix}. The job is accepted and starts the moment it answers`;+ }+ return bootLeft != null+ ? `Waking up the${coldWho} GPU worker${elapsedSuffix}, about ${bootLeft}s of boot left, then sculpting starts`+ : `Waking up the${coldWho} GPU worker (about ${coldSeconds}s), then sculpting starts`;}- return `Queued${laneSuffix}${etaSuffix}`;+ return `Queued${laneSuffix}${elapsedSuffix}${etaSuffix}`;case 'image':case 'texturing':return 'Drafting the look…';case 'running':case 'reconstruct':case 'mesh':- return `Building geometry & texturing${etaSuffix}`;+ return `Building geometry & texturing${elapsedSuffix}${etaSuffix}`;case 'done':case 'ready':return 'Model ready — loading into the cam';