Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.
JavaScript 3d-generationai-agentsanimationavatardigital-humangenerative-aiglbgltfimage-to-3dllmmcpmcp-servermodel-context-protocolsolanatext-to-3dthreejsweb-componentswebglwebxrx402
fix(gcp-triage): stop truncating the exception out of long app-log tracebacks
nirholas426f89c5a1ac
diff --git a/docs/ops/production-log-triage.md b/docs/ops/production-log-triage.mdindex 3632425a1..089aa0199 100644--- a/docs/ops/production-log-triage.md+++ b/docs/ops/production-log-triage.md@@ -1291,6 +1291,24 @@ ERROR 502 POST /api/pump/launch-agent[tests/gcp-triage.test.js](../../tests/gcp-triage.test.js) ("classifies ananimated-render R2 fetch failure..." / "...a clip-detail R2 fetchfailure...").+- **Worker-side app-log variant (same outage, different shape):** a GPU/mesh+ worker that routes a caller-supplied model URL through+ `worker_security.py`'s `fetch_remote_bytes` (`remesh-service` confirmed+ 2026-10-15; any worker vendoring that module can hit it) logs a Python+ traceback ending in `httpx.HTTPStatusError: Client error '403 Forbidden'+ for url 'https://pub-<account>.r2.dev/...'` when the fetched model lives on+ R2's public `.r2.dev` domain during the flap. The worker already catches+ this cleanly (`safe_error` marks the task `failed` with an opaque message;+ the traceback is logged for operators only, not an unhandled crash). This+ also exposed a real bug in the monitor itself: `buildFindings` was cutting+ its app-log classification sample to the first 400 bytes of the message, so+ the one line that names the fault in a multi-frame traceback (always last)+ never reached `classify()`, and every such hit fell through to+ `investigate` instead of this already-tracked `owner` signature. Fixed+ 2026-10-15 with a `headTail()` helper that keeps both ends of an+ over-budget message. Regression-covered in+ [tests/gcp-triage.test.js](../../tests/gcp-triage.test.js) ("classifies a+ GPU worker R2 .r2.dev 403 traceback as the r2-downstream family").---diff --git a/scripts/gcp-triage.mjs b/scripts/gcp-triage.mjsindex fa9030997..1da80b2a9 100644--- a/scripts/gcp-triage.mjs+++ b/scripts/gcp-triage.mjs@@ -67,6 +67,21 @@ const KNOWN_SIGNATURES = [class: 'owner',action: `Replicate out of credit; forge degrades to free NVIDIA NIM lane meanwhile. Owner adds credit at replicate.com/account/billing. ${RUNBOOK} §replicate.`,},+ {+ // 2026-10-15 deep sweep: a GPU/mesh worker (remesh-service first, any+ // worker that routes a caller-supplied URL through worker_security.py's+ // fetch_remote_bytes can hit it) logs this exact Python traceback when+ // the fetched model lives on R2's public .r2.dev domain during the same+ // Cloudflare outage the HTTP-level r2-downstream-5xx family already+ // tracks (row 36). Same root cause, different log shape: the worker+ // catches the exception cleanly (safe_error marks the task "failed" and+ // returns an opaque message) and only logs the traceback for operators,+ // so this is not a crash to fix in code.+ id: 'r2-downstream-5xx',+ match: /HTTPStatusError: Client error '403 Forbidden' for url 'https:\/\/pub-[^']*\.r2\.dev\//,+ class: 'owner',+ action: `Worker-side symptom of the same Cloudflare R2 outage the HTTP r2-downstream-5xx family tracks (production-100-OWNER-ACTIONS.md row 36): a caller-supplied model URL lives on R2's public .r2.dev domain, which is edge-blocked during the flap. The worker already handles this cleanly (task marked failed, opaque error to the caller); nothing to fix in code. Confirm with: curl -s https://three.ws/api/healthz | jq '.subsystems.subsystems[] | select(.name=="object_storage")'. ${RUNBOOK} §r2-downstream.`,+ },{id: 'db-storage-cap',match: /db at storage cap/i,@@ -573,6 +588,17 @@ export function fingerprint(message) {.slice(0, 160);}+// A classification sample cut to a flat head slice loses the one line that+// actually names the fault in a multi-frame stack trace: Python's traceback+// format always puts the exception type and message last, well past 400+// bytes of call-stack frames. Keep both ends so KNOWN_SIGNATURES regexes can+// match on either the opening context or the trailing exception line.+function headTail(message, budget) {+ if (message.length <= budget) return message;+ const half = Math.floor(budget / 2);+ return `${message.slice(0, half)}\n...\n${message.slice(-half)}`;+}+export function classify(message, services) {for (const sig of KNOWN_SIGNATURES) {if (!sig.match.test(message)) continue;@@ -631,7 +657,7 @@ export function buildFindings(entries) {const fp = fingerprint(message);const g = appGroups.get(fp) || {kind: 'app', title: fp, count: 0, services: new Set(),- severity: entry.severity || 'WARNING', firstSeen: ts, lastSeen: ts, sample: message.slice(0, 400),+ severity: entry.severity || 'WARNING', firstSeen: ts, lastSeen: ts, sample: headTail(message, 400),};g.count++;g.services.add(service);diff --git a/tests/gcp-triage.test.js b/tests/gcp-triage.test.jsindex 9c502d5a0..1bab53a27 100644--- a/tests/gcp-triage.test.js+++ b/tests/gcp-triage.test.js@@ -142,6 +142,46 @@ describe('gcp-triage HTTP classification', () => {expect(finding.signature).toBe('r2-downstream-5xx');expect(finding.class).toBe('owner');});++ // 2026-10-15 deep sweep: remesh-service logged this exact traceback for a+ // mesh fetch hitting the R2 .r2.dev public-domain 403 (the same row-36+ // outage the HTTP-level r2-downstream-5xx family already tracks), but the+ // Python traceback's call-stack frames push the one line that actually+ // names the fault (httpx.HTTPStatusError ... 403 Forbidden ... r2.dev) past+ // byte 400, which is where buildFindings used to cut the classification+ // sample. The finding fell through to `investigate` every sweep even+ // though nothing in the worker is broken.+ const remeshR2Traceback = () => ({+ resource: { labels: { service_name: 'remesh-service' } },+ timestamp: '2026-10-15T00:00:05Z',+ severity: 'ERROR',+ textPayload: [+ 'Traceback (most recent call last):',+ ' File "/app/main.py", line 741, in _process',+ ' artifacts, meta = await loop.run_in_executor(',+ ' ^^^^^^^^^^^^^^^^^^^^^^^^^^^',+ ' File "/usr/local/lib/python3.11/concurrent/futures/thread.py", line 58, in run',+ ' result = self.fn(*self.args, **self.kwargs)',+ ' ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^',+ ' File "/app/main.py", line 701, in _run_processing',+ ' data, suffix = _fetch_mesh(mesh_url)',+ ' ^^^^^^^^^^^^^^^^^^^^^',+ ' File "/app/main.py", line 142, in _fetch_mesh',+ ' data = fetch_remote_bytes(url, timeout=60, max_bytes=MAX_MESH_BYTES)',+ ' ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^',+ ' File "/app/worker_security.py", line 154, in fetch_remote_bytes',+ ' resp.raise_for_status()',+ ' File "/usr/local/lib/python3.11/site-packages/httpx/_models.py", line 763, in raise_for_status',+ ' raise HTTPStatusError(message, request=request, response=self)',+ "httpx.HTTPStatusError: Client error '403 Forbidden' for url 'https://pub-2534e921bf9c4314addcd4d8a6e98b7b.r2.dev/forge/2152b20cad96/fd0a25bf-ecfe-491a-9541-e805a7dc0cb3.glb'",+ ].join('\n'),+ });++ it('classifies a GPU worker R2 .r2.dev 403 traceback as the r2-downstream family', () => {+ const [finding] = buildFindings([remeshR2Traceback()]);+ expect(finding.signature).toBe('r2-downstream-5xx');+ expect(finding.class).toBe('owner');+ });});describe('gcp-triage page failure summary', () => {