Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.
JavaScript 3d-generationai-agentsanimationavatardigital-humangenerative-aiglbgltfimage-to-3dllmmcpmcp-servermodel-context-protocolsolanatext-to-3dthreejsweb-componentswebglwebxrx402
fix(http): make wrapCron answer a clean 503 at its own time budget instead of hanging until Cloud Run's 900s kill
nirholas1ef950e605ec
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 70729d887..c01202019 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -6,6 +6,7 @@ Public history for [three.ws](https://three.ws), newest first. New pages come fr## 2026-10-11+- **A stuck background job now fails fast instead of hanging for 15 minutes**: Every scheduled background job (cron) on the platform now answers cleanly and quickly if something inside it gets stuck, instead of silently hanging until the server's own 15-minute connection limit cuts it off with no explanation. Each job now gets a real time budget matching what it already declared, the status board records the failed run immediately, and an alert fires with the job's name attached so it can be diagnosed in seconds instead of traced by hand from a blank timeout. Image rendering for 3D creations got the same treatment: a stuck browser launch, page load, or screenshot now times out on its own instead of tying up the render queue. `[infra]`- **An agent set to Grok now answers as Grok everywhere, with a badge when it can't**: Profile chat, the agent editor's copilot, scheduled strategy runs, and the gateway all now read an agent's chosen brain through the same resolver, so picking Grok for an agent actually means Grok on every surface, not just the one where it was set. A Grok pick first tries the agent owner's own saved xAI key, then the platform's, before falling back to the free default chain. When Grok genuinely can't answer (no key reachable anywhere), profile chat now shows a small badge naming which model answered instead, so it's never a silent swap. `[improvement]`- **Choose which @-mentions your agent answers, right from its editor**: An agent's Social tab now has an "Answer mentions" section: turn it on, keep it in dry run while you watch what it would say, cap it at a daily reply count, and pick exactly which kinds of mentions it may act on. A new decisions list on the same page shows the agent's own recent calls, what it decided and why, so the owner never has to ask an admin to check a console on their behalf. (`/agents`) `[feature]`- **Sentiment Scout's X reads gained a second, independent path through xAI**: The Sentiment Scout reads X for posts quoting a coin's exact contract address, as evidence a holder can open and check. When our own X search credentials are unavailable or rate limited, it now fails over to xAI's x_search tool, a second route to the same posts, metered by its own daily cap so an outage on one path never silently costs more than planned on the other. No change for anyone reading the Scout: the evidence looks the same either way. (`/docs/sentiment-scout`) `[improvement, infra]`diff --git a/api/_lib/http.js b/api/_lib/http.jsindex 6b0b6a9d7..811f0d38e 100644--- a/api/_lib/http.js+++ b/api/_lib/http.js@@ -766,7 +766,19 @@ export function wrap(handler) {// then reclaims space and the next tick resumes (see isStoragePressured). This is the// proactive complement to the reactive catch below — it stops crons that swallow// their own write errors (the pump-intel firehose) from storming the logs at the cap.-export function wrapCron(handler, { requireWriteCapacity = false } = {}) {+//+// `maxDurationMs` gives the cron a real time budget. Before the 2026-07-07 Cloud Run+// migration, the `maxDuration` a handler exported was Vercel-enforced; nothing ever+// replaced that enforcement, so a hang anywhere inside a handler ran unbounded until+// Cloud Run's own 900s ingress timeout killed the connection with no cron name, no+// elapsed time, and no heartbeat write. Defaults to 120000 (the most common declared+// `maxDuration` value) when the caller does not pass one.+// The most common declared `maxDuration` value across api/cron/*.js today, and+// what a cron gets for free when it declares none of its own.+export const DEFAULT_CRON_MAX_DURATION_MS = 120_000;++export function wrapCron(handler, { requireWriteCapacity = false, maxDurationMs } = {}) {+ const budgetMs = maxDurationMs ?? DEFAULT_CRON_MAX_DURATION_MS;return wrap(async (req, res, ...rest) => {// Derive cron name from the request URL for heartbeat tracking.const cronName = (req.url || '').replace(/^\/api\/cron\//, '').split('?')[0] || 'unknown';@@ -786,31 +798,69 @@ export function wrapCron(handler, { requireWriteCapacity = false } = {}) {return;}}- try {- await handler(req, res, ...rest);- // Write heartbeat after the handler returns, fire-and-forget, never blocks.- // A handler that reports its own failure as a 5xx (instead of throwing)- // failed too; recording it as healthy hid a broken cron on the ops board.- const failed = res.statusCode >= 500;- const beat = failed- ? { ok: false, t: t0, ms: Date.now() - t0, err: `answered HTTP ${res.statusCode}` }- : { ok: true, t: t0, ms: Date.now() - t0 };++ const writeHeartbeat = (beat) => {import('./cache.js').then(({ cacheSet }) => {cacheSet(`cron:heartbeat:${cronName}`, beat, 7 * 24 * 60 * 60).catch(() => {});}).catch(() => {});- } catch (err) {- import('./cache.js').then(({ cacheSet }) => {- cacheSet(`cron:heartbeat:${cronName}`, { ok: false, t: t0, ms: Date.now() - t0, err: err?.message?.slice(0, 200) }, 7 * 24 * 60 * 60).catch(() => {});- }).catch(() => {});- if (isDbUnavailableError(err) || isDbCapacityError(err)) {- const full = isDbCapacityError(err);- console.warn(`[cron] db ${full ? 'at storage cap' : 'unavailable'} — skipping tick:`, err.message);- if (!res.headersSent && !res.writableEnded) {- json(res, 200, { ok: false, reason: full ? 'db_full' : 'db_unavailable' });+ };++ // True once the budget timer below has already answered the request and+ // recorded the tick as timed out. Guards the handler's own (late) heartbeat+ // write from clobbering that record if it eventually finishes on its own.+ let timedOut = false;++ const runHandler = (async () => {+ try {+ await handler(req, res, ...rest);+ // A handler that reports its own failure as a 5xx (instead of throwing)+ // failed too; recording it as healthy hid a broken cron on the ops board.+ if (timedOut) return;+ const failed = res.statusCode >= 500;+ writeHeartbeat(failed+ ? { ok: false, t: t0, ms: Date.now() - t0, err: `answered HTTP ${res.statusCode}` }+ : { ok: true, t: t0, ms: Date.now() - t0 });+ } catch (err) {+ if (!timedOut) {+ writeHeartbeat({ ok: false, t: t0, ms: Date.now() - t0, err: err?.message?.slice(0, 200) });}- return;+ if (isDbUnavailableError(err) || isDbCapacityError(err)) {+ const full = isDbCapacityError(err);+ console.warn(`[cron] db ${full ? 'at storage cap' : 'unavailable'}, skipping tick:`, err.message);+ if (!res.headersSent && !res.writableEnded) {+ json(res, 200, { ok: false, reason: full ? 'db_full' : 'db_unavailable' });+ }+ return;+ }+ throw err;}- throw err;+ })();++ let timer;+ const budget = new Promise((resolve) => {+ timer = setTimeout(() => {+ timedOut = true;+ if (!res.headersSent && !res.writableEnded) {+ writeHeartbeat({ ok: false, t: t0, ms: Date.now() - t0, timed_out: true });+ console.warn(`[cron] ${cronName} exceeded its ${budgetMs}ms budget, answering 503 before Cloud Run's own timeout kills the connection`);+ sendOpsAlert(`cron timeout: ${cronName}`, `exceeded its ${budgetMs}ms budget`, { signature: `cron:timeout:${cronName}` });+ json(res, 503, { ok: false, reason: 'cron_timeout', cron: cronName, budget_ms: budgetMs });+ }+ resolve();+ }, budgetMs);+ if (typeof timer.unref === 'function') timer.unref();+ });++ try {+ await Promise.race([runHandler, budget]);+ } finally {+ clearTimeout(timer);+ // Node cannot preempt arbitrary async work without an AbortSignal threaded+ // through every I/O call inside the handler, so the handler itself is never+ // cancelled when the budget wins the race. Swallow whatever it eventually+ // does (resolve or reject) so it can never try to write to the response this+ // request already ended, or crash the process with an unhandled rejection.+ runHandler.catch(() => {});}});}diff --git a/api/_lib/render-glb.js b/api/_lib/render-glb.jsindex b478c9a19..f29300f88 100644--- a/api/_lib/render-glb.js+++ b/api/_lib/render-glb.js@@ -418,10 +418,28 @@ export async function renderGlbToPngBrowser({ glbUrl, width = 1200, height = 630}}+// Bounds how long a single render stage may wait before failing with a named+// error instead of hanging until Cloud Run's own request timeout kills the+// connection. The underlying puppeteer call is never cancelled (chromium keeps+// running in the background), so this only bounds how long THIS caller waits+// for it; a future "maximum request timeout" investigation gets a stage name+// and an elapsed budget instead of a blank multi-minute gap.+function withStageTimeout(promise, ms, stage) {+ let timer;+ const timeout = new Promise((_, reject) => {+ timer = setTimeout(() => reject(new Error(`render timed out at ${stage}`)), ms);+ if (typeof timer.unref === 'function') timer.unref();+ });+ return Promise.race([promise, timeout]).finally(() => clearTimeout(timer));+}+// One page lifecycle on the shared browser: boot page, load the inline viewer,// wait for the first clean frame, screenshot.async function renderOnce({ glbBase64, width, height, background, backdrop, poseClip, onPose }) {- const browser = await getBrowser();+ // getBrowser()'s cold-launch path (chromium binary download on a fresh+ // container) had no cap before this; a stalled download hung the whole+ // request until Cloud Run's own timeout.+ const browser = await withStageTimeout(getBrowser(), 30_000, 'getBrowser');const page = await browser.newPage();try {await page.setViewport({ width, height, deviceScaleFactor: 1 });@@ -432,7 +450,11 @@ async function renderOnce({ glbBase64, width, height, background, backdrop, poseconst html = viewerHtml({ glbBase64, width, height, background, backdrop, poseClip, threeBase });// data: URL avoids needing a network fetch for the bootstrap page itself.// importmap dependencies (three, GLTFLoader) still come from the CDN.- await page.setContent(html, { waitUntil: 'domcontentloaded' });+ await withStageTimeout(+ page.setContent(html, { waitUntil: 'domcontentloaded', timeout: 20_000 }),+ 20_000,+ 'setContent',+ );await page.waitForFunction('window.__renderDone === true || window.__renderError !== null',{ timeout: 15_000 },@@ -444,11 +466,18 @@ async function renderOnce({ glbBase64, width, height, background, backdrop, poseconst poseResult = (await page.evaluate(() => window.__poseResult)) || { posed: false, mode: 'bind' };if (poseResult.mode === 'pose-error') console.warn('[render] chromium lane could not pose the model:', poseResult.error);onPose?.({ posed: Boolean(poseResult.posed), mode: poseResult.mode });- return await page.screenshot({- type: 'png',- omitBackground: background === 'transparent',- clip: { x: 0, y: 0, width, height },- });+ // page.screenshot() has no native timeout option (not part of puppeteer's+ // ScreenshotOptions), so the budget here is enforced entirely by the race.+ return await withStageTimeout(+ page.screenshot({+ type: 'png',+ omitBackground: background === 'transparent',+ clip: { x: 0, y: 0, width, height },+ timeout: 15_000,+ }),+ 15_000,+ 'screenshot',+ );} finally {await page.close().catch(() => {});}diff --git a/api/cron/agent-avatar-backfill.js b/api/cron/agent-avatar-backfill.jsindex ece34ac24..77cdc90f9 100644--- a/api/cron/agent-avatar-backfill.js+++ b/api/cron/agent-avatar-backfill.js@@ -51,4 +51,4 @@ export default wrapCron(async (req, res) => {...result,coverage: cov,});-});+}, { maxDurationMs: maxDuration * 1000 });diff --git a/api/cron/avatar-thumbnail-backfill.js b/api/cron/avatar-thumbnail-backfill.jsindex 45d841f03..b40fb187f 100644--- a/api/cron/avatar-thumbnail-backfill.js+++ b/api/cron/avatar-thumbnail-backfill.js@@ -124,4 +124,4 @@ export default wrapCron(async (req, res) => {...(r.status === 'done' ? { url: r.url, bytes: r.bytes, ms: r.ms } : { error: r.error }),})),});-});+}, { maxDurationMs: maxDuration * 1000 });diff --git a/api/cron/avatar-thumbnail-render.js b/api/cron/avatar-thumbnail-render.jsindex 2eac8b6ad..97701bba3 100644--- a/api/cron/avatar-thumbnail-render.js+++ b/api/cron/avatar-thumbnail-render.js@@ -124,4 +124,4 @@ export default wrapCron(async (req, res) => {failed,results,});-});+}, { maxDurationMs: maxDuration * 1000 });diff --git a/api/cron/companion-poll.js b/api/cron/companion-poll.jsindex d41b760e0..28a4f244d 100644--- a/api/cron/companion-poll.js+++ b/api/cron/companion-poll.js@@ -22,4 +22,4 @@ export default wrapCron(async (req, res) => {const started = Date.now();const summary = await pollDue({ limit: BATCH_LIMIT, budgetMs: TIME_BUDGET_MS });return json(res, 200, { ok: true, ...summary, ms: Date.now() - started });-});+}, { maxDurationMs: maxDuration * 1000 });diff --git a/api/cron/feedback-triage.js b/api/cron/feedback-triage.jsindex 0e713b472..41222814a 100644--- a/api/cron/feedback-triage.js+++ b/api/cron/feedback-triage.js@@ -51,4 +51,4 @@ export default wrapCron(async (req, res) => {clusters: clusters.size,ms: Date.now() - started,});-});+}, { maxDurationMs: maxDuration * 1000 });diff --git a/api/cron/forge-finalize.js b/api/cron/forge-finalize.jsindex 9b958743e..9e353b9cc 100644--- a/api/cron/forge-finalize.js+++ b/api/cron/forge-finalize.js@@ -478,4 +478,4 @@ export default wrapCron(async (req, res) => {...out,elapsed_ms: Date.now() - t0,});-});+}, { maxDurationMs: maxDuration * 1000 });diff --git a/api/cron/forge-thumbnail-backfill.js b/api/cron/forge-thumbnail-backfill.jsindex 79d86bd24..32b708ea3 100644--- a/api/cron/forge-thumbnail-backfill.js+++ b/api/cron/forge-thumbnail-backfill.js@@ -83,4 +83,4 @@ export default wrapCron(async (req, res) => {failed: result.failed,aborted: result.aborted,});-});+}, { maxDurationMs: maxDuration * 1000 });diff --git a/api/cron/forge-web-variants.js b/api/cron/forge-web-variants.jsindex cf1034502..19eb93f38 100644--- a/api/cron/forge-web-variants.js+++ b/api/cron/forge-web-variants.js@@ -107,4 +107,4 @@ export default wrapCron(async (req, res) => {}return json(res, 200, { ok: true, processed: rows.length, built, skipped, failed, remaining });-});+}, { maxDurationMs: maxDuration * 1000 });diff --git a/api/cron/likeness-eval.js b/api/cron/likeness-eval.jsindex 17bf3ad55..386217e03 100644--- a/api/cron/likeness-eval.js+++ b/api/cron/likeness-eval.js@@ -126,4 +126,4 @@ export default wrapCron(async (req, res) => {// Always 200: a low score is a reported finding, not a failed invocation. A// non-2xx would make Cloud Scheduler retry an expensive render sweep.json(res, 200, { ok: true, ...summary, distribution });-});+}, { maxDurationMs: maxDuration * 1000 });diff --git a/api/cron/marketplace-escrow-sweep.js b/api/cron/marketplace-escrow-sweep.jsindex ebaab6212..f9c4aa0d2 100644--- a/api/cron/marketplace-escrow-sweep.js+++ b/api/cron/marketplace-escrow-sweep.js@@ -24,4 +24,4 @@ export default wrapCron(async (req, res) => {const summary = await runMarketplaceSweep();if (summary.errors.length) console.warn('[marketplace-escrow-sweep] errors', JSON.stringify(summary.errors).slice(0, 2000));return json(res, 200, { ok: true, ms: Date.now() - t0, ...summary });-});+}, { maxDurationMs: maxDuration * 1000 });diff --git a/api/cron/payment-session-sweep.js b/api/cron/payment-session-sweep.jsindex 995aa822c..39ee033d1 100644--- a/api/cron/payment-session-sweep.js+++ b/api/cron/payment-session-sweep.js@@ -96,4 +96,4 @@ export default wrapCron(async (req, res) => {? `Batch limit (${BATCH_LIMIT}) reached. More sessions may remain; the next tick continues.`: null,});-});+}, { maxDurationMs: maxDuration * 1000 });diff --git a/api/cron/quality-bench.js b/api/cron/quality-bench.jsindex 8df620f30..3ffa4ebb8 100644--- a/api/cron/quality-bench.js+++ b/api/cron/quality-bench.js@@ -153,4 +153,4 @@ export default wrapCron(async (req, res) => {// Always 200: a regression is a reported finding, not a failed invocation.// A non-2xx here would make Cloud Scheduler retry the (expensive) bench run.json(res, 200, { ok: true, regressed, ...summary });-});+}, { maxDurationMs: maxDuration * 1000 });diff --git a/api/cron/reflect-sweep.js b/api/cron/reflect-sweep.jsindex 6958e338c..d316a8a7f 100644--- a/api/cron/reflect-sweep.js+++ b/api/cron/reflect-sweep.js@@ -87,4 +87,4 @@ export default wrapCron(async (req, res) => {summary.elapsedMs = Date.now() - startedAt;return json(res, 200, summary);-});+}, { maxDurationMs: maxDuration * 1000 });diff --git a/api/cron/settlement-verify.js b/api/cron/settlement-verify.jsindex 15e893dd4..066aa70fc 100644--- a/api/cron/settlement-verify.js+++ b/api/cron/settlement-verify.js@@ -177,4 +177,4 @@ export default wrapCron(async (req, res) => {}return json(res, 200, { ok: broken.length === 0, tips, pays, took_ms: Date.now() - t0 });-});+}, { maxDurationMs: maxDuration * 1000 });diff --git a/data/changelog.json b/data/changelog.jsonindex b05777785..209382eb3 100644--- a/data/changelog.json+++ b/data/changelog.json@@ -1,6 +1,12 @@{"$comment": "Curated changelog entries for three.ws, the editorial layer of the public changelog. New PAGE launches are tracked automatically via the `added` field in data/pages.json; this file is for everything else holders care about: improvements, fixes, SDK releases, trading/payments capabilities, security work. scripts/build-page-index.mjs merges both sources into CHANGELOG.md, public/changelog.json, and public/changelog.xml. Newest first. Allowed tags: feature, improvement, fix, sdk, infra, docs, security, desktop (desktop entries become the desktop app's release notes). `link` is optional and must be a live three.ws path.","entries": [+ {+ "date": "2026-10-11",+ "title": "A stuck background job now fails fast instead of hanging for 15 minutes",+ "summary": "Every scheduled background job (cron) on the platform now answers cleanly and quickly if something inside it gets stuck, instead of silently hanging until the server's own 15-minute connection limit cuts it off with no explanation. Each job now gets a real time budget matching what it already declared, the status board records the failed run immediately, and an alert fires with the job's name attached so it can be diagnosed in seconds instead of traced by hand from a blank timeout. Image rendering for 3D creations got the same treatment: a stuck browser launch, page load, or screenshot now times out on its own instead of tying up the render queue.",+ "tags": ["infra"]+ },{"date": "2026-10-11","title": "The on-chain leak alarm now scans smarter instead of scanning everything",diff --git a/docs/ops/production-log-triage.md b/docs/ops/production-log-triage.mdindex 20d45372a..94e46752a 100644--- a/docs/ops/production-log-triage.md+++ b/docs/ops/production-log-triage.md@@ -941,6 +941,54 @@ ws error: Unexpected server response: 301[api/_lib/crypto-token-holders.js](../../api/_lib/crypto-token-holders.js)is the composition logic if it does.+## 🟡 HTTP timeout with no cron context++```+The request has been terminated because it has reached the maximum request+timeout+```++- **Source (pre-2026-10-11):** nothing. `maxDuration` exports on+ `api/cron/*.js` handlers were a real Vercel-enforced per-function budget;+ nothing replaced that enforcement after the 2026-07-07 Cloud Run migration,+ so a hang anywhere inside a cron handler ran unbounded until Cloud Run's own+ 900s ingress timeout (`--timeout 900` in+ [server/cloudbuild.yaml](../../server/cloudbuild.yaml)) killed the+ connection. That log line carries no cron name, no elapsed time, and no+ heartbeat write, which is why `/api/cron/x402-ring-leak-scan` (see the+ `rpc_lanes` entry above) looked like a blank wall until it was traced by+ hand.+- **Fixed 2026-10-11:** `wrapCron()` in+ [api/_lib/http.js](../../api/_lib/http.js) now races every cron handler+ against its own time budget (`maxDurationMs`, wired from each file's+ `export const maxDuration * 1000`, falling back to+ `DEFAULT_CRON_MAX_DURATION_MS` (120000ms) when a handler declares none). On+ timeout it answers a clean `503 { ok: false, reason: 'cron_timeout', cron:+ '<name>', budget_ms: <n> }`, writes the heartbeat+ (`cron:heartbeat:<name>`) as `{ ok: false, timed_out: true }` so uptime+ tooling sees the tick as failed rather than stalled, and fires a single+ `sendOpsAlert` deduplicated by signature `cron:timeout:<name>`. Node has no+ generic way to preempt arbitrary in-flight async work, so the handler itself+ keeps running in the background; a late settlement after the timeout has+ already answered is a no-op (it can never overwrite the `timed_out`+ heartbeat or touch the already-sent response).+- **Read it:** `gcloud logging read 'resource.type="cloud_run_revision"+ resource.labels.service_name="three-ws-api"+ textPayload:"exceeded its"' --freshness=24h --project+ aerial-vehicle-466722-p5` surfaces the new `[cron] <name> exceeded its+ <n>ms budget, answering 503 before Cloud Run's own timeout kills the+ connection` warn line with the cron name attached. A dashboard or alert+ firing on the ops-alert signature `cron:timeout:<name>` names the exact+ handler; a bare "maximum request timeout" with no matching `cron:timeout:*`+ alert is now a sign the hang is outside `wrapCron` entirely (a non-cron API+ route, or `server/index.mjs` itself), not inside a cron tick.+- **Resolve:** treat a recurring `cron:timeout:<name>` the same as any other+ cron bug: find why that handler's own work (a slow RPC lane, an unbounded+ DB scan, a Puppeteer render) is exceeding its declared `maxDuration`, not+ by raising the budget first. Raising `maxDuration` only widens the window+ before Cloud Run's 900s kill reappears if the underlying work keeps+ growing.+---## 🟡 `[solana-rpc] https://<host> 403`, cooling 30m, failing overdiff --git a/prompts/finish/_context/production-100-OWNER-ACTIONS.md b/prompts/finish/_context/production-100-OWNER-ACTIONS.mdindex f59360db9..c094a8b72 100644--- a/prompts/finish/_context/production-100-OWNER-ACTIONS.md+++ b/prompts/finish/_context/production-100-OWNER-ACTIONS.md@@ -48,6 +48,7 @@ on it; some resolve themselves as other work lands.| 34 | **Decide how to reconcile the `evolve` worktree branch with `main`: it is 358 commits behind** (added 2026-10-11 by the evolve loop, while working x-grok order 050). `git log --oneline HEAD..main \| wc -l` from `/home/user/projects/.evolve-wt` on branch `evolve` returns 358: `main` shipped (among much else) x-grok orders 044, 046 to 052, and 055 to 058 (the `x-mention-poll.js` bot, its safety rails, budget controller, and the make/image3d/avatar handlers) while `evolve` has no record of any of it, because `evolve` never merged or rebased from `main` after it branched. `evolve` also carries 20+ of its own commits not on `main` yet (x-grok orders 038 to 040, 043, 045, 049, an accessibility fix, an a11y wire-up), so this is a real two-way divergence, not a simple fast-forward. The practical cost: this session spent real effort re-discovering and re-implementing order 050's mention-poll cron from scratch before noticing `main` already shipped a more complete version, and the same evidence shows orders 051, 052, 055, 056, 057 and 058's order files are stale duplicates sitting in `prompts/finish/` on `evolve` for the same reason (noted in [x-grok-PROGRESS.md](x-grok-PROGRESS.md), 2026-10-11 entry); every future evolve session that picks one up will repeat this exact waste until the branch is caught up. Merging or rebasing 358 commits safely is a judgment call about which side wins on any file both branches touched (this evolve worktree's own x-search/agent-brain/agent-gateway/gateway-adapter work overlaps the same `api/cron/[name].js` and `vercel.json` main also edited); that is not a mechanical fix an agent should attempt unattended. Decide the reconciliation approach (likely: rebase `evolve`'s unique commits onto current `main`, resolving the handful of shared-file conflicts by hand) and either do it or direct an agent to. **Re-measured 2026-10-11 while verifying order 051 against `main` from a throwaway detached worktree (removed after, nothing committed to `main`): `main`'s shipped `x-mention-reply.js` has a live bug that this reconciliation should fix in the same pass.** `composePublicReply`'s `chainFor()` builds its model chain from `agent-brain.js`/`agent-model.js` (`{name,url,key,model}` records meant for `copilot-engine.js`'s own round-trip reader), then hands that chain to `llmComplete` in `llm.js`, whose attempt loop calls `p.buildBody(...)`, a method only `llm.js`'s own, differently-shaped `providerChain()` objects define. Every attempt on every rung throws the same `TypeError` and the whole chain "exhausts" in milliseconds, always landing on the safe fixed fallback reply. Verified live with production's own `GROQ_API_KEY`/`OPENROUTER_API_KEY`/`NVIDIA_API_KEY` (`RUN_LLM_TESTS=1 npx vitest run tests/x-mention-reply.llm.test.js --reporter=verbose`): 16/16 "passed" (the test accepts a safe fallback as a pass) but all 16 fell back with `reason: 'llm_error', detail: '... p.buildBody is not a function'`, and the real `x_mention_events` table already carries 129 rows with `reason: 'fallback:llm_error'` from the live mention-poll cron, recorded before this was diagnosed. The public reply bot has likely never sent a single real model-composed reply to a stranger; it is safe (every post-check still holds) but has silently answered every public mention with the fixed help text only. Fix, once someone is on `main`: change `chainFor()` to build from `llm.js`'s own `providerChain({ grokKey, grokModel })` (resolving the agent's Grok key/model the way `agent-grok-key.js` already does) instead of `modelChain`/`resolveAgentBrain`, so the chain shape matches what `llmComplete` reads. Full writeup: [x-grok-PROGRESS.md](x-grok-PROGRESS.md), 2026-10-11 entry for order 051. **Re-measured again 2026-10-11 while working order 060 (agent mention replies): the gap is now 362 commits behind / 117 ahead, and order 060 cannot be built at all until this reconciles, because its entire prerequisite subsystem (`x-mention-poll.js` and its ~14-file import closure, orders 045-058) exists only on `main` and has no file to extend on `evolve`; left open with a dated status note rather than hand-copied or duplicated. Full writeup: [x-grok-PROGRESS.md](x-grok-PROGRESS.md), 2026-10-11 entry for order 060.** | decision, then a rebase (the `chainFor()` fix itself is a small, mechanical change once on `main`) | Stops repeat duplicate work across the whole x-grok campaign and any other area both branches touched, and fixes a currently-silent production defect in the public X reply bot | [x-grok-PROGRESS.md](x-grok-PROGRESS.md), 2026-10-11 entries || 35 | **Apply the agent mention-policy migration, then ship it with the next deploy** (added 2026-10-11 by the evolve loop, x-grok order 061, which is otherwise complete and retired). `api/_lib/migrations/20261011220000_agent_mention_policy.sql` adds `mention_replies_enabled`, `mention_replies_dry_run`, `mention_replies_daily_cap` and `mention_replies_allowed_intents` to `agent_x_policies`, the four fields the new "Answer mentions" panel on `pages/agent-edit.html` (Social tab) reads and writes. `npm run db:status` on 2026-10-11 lists it as the only pending migration. The read path (`GET /api/x/agents`) already works without it, since the query is `select *` and `rowToPolicy()` defaults the missing columns; only the write path breaks: `saveAgentPolicy`'s `INSERT` names the four columns explicitly, so a save attempt against the unmigrated table answers 500 with Postgres `42703 undefined_column`, verified live in a local run of the full UI against the real dev database on 2026-10-11 (every other state, no-connection/missing-scope/connected/populated-decisions, was verified working by temporarily granting `X_OAUTH_CLIENT_ID`/`SECRET` in `.env.local` only, never in production). Apply it with `node scripts/apply-migrations.mjs --apply --file 20261011220000_agent_mention_policy.sql`, or `npm run db:status` then `npm run db:migrate` (nothing else is pending as of this writing). `npm run db:check` blocks the next deploy on it until then. Separately, and not blocking this row: production has never set `X_OAUTH_CLIENT_ID`/`X_OAUTH_CLIENT_SECRET` at all (`node scripts/read-service-env.mjs '^X_OAUTH_CLIENT'` finds nothing), so the whole per-agent "connect this agent's own X account" flow (distinct from the company bot account's `X_API_KEY`/`X_ACCESS_TOKEN` posting credentials, which are set) answers "X connections are not enabled on this deployment" for every agent today; that is a pre-existing gap this order did not create and its settings UI already degrades to that state correctly. | one command | The "Answer mentions" save button in the agent settings UI | [x-grok-PROGRESS.md](x-grok-PROGRESS.md), 2026-10-11 entry for order 061 || 36 | **Check Cloudflare R2's own status/dashboard for the bucket behind `S3_BUCKET`: it flaps between healthy and unreachable without any corresponding change on our side** (added 2026-10-11 by the evolve scout lane). Queried `ops_alerts` directly (columns: `signature, title, detail, severity, count, environment, first_seen, last_seen`): since 2026-09-08 the `object_storage` sensor (`api/_lib/ops/object-storage-health.js`) has logged roughly 167 "DEGRADED: Object storage (Cloudflare R2)" events (`bucket reachable but slow, signed list ~1095ms` against a 1000ms threshold) and about 18 "DOWN" events, interleaved with many "RESOLVED" events minutes apart showing a healthy 150 to 300ms. Reproduced live with production's own `S3_ACCESS_KEY_ID`/`S3_SECRET_ACCESS_KEY`/`S3_BUCKET`/`S3_ENDPOINT` (pulled via `node scripts/read-service-env.mjs` one variable at a time, used in-process, then deleted from disk immediately): `gatherObjectStorageHealth()` returned `{"status":"down","detail":"bucket unreachable: Please enable R2 through the Cloudflare Dashboard.","metrics":{"latencyMs":135,"credentialFault":false}}`, a 135ms response, i.e. the bucket answered fast but Cloudflare's own API told us R2 was not enabled, which cannot be true most of the time this account is clearly serving reads. Ruled out as a code cause: the credential is not rejected (`credentialFault:false`, no `SignatureDoesNotMatch`), and the AWS SDK's Node HTTP handler already defaults to keep-alive on (`keepAlive:true, maxSockets:50` in `@smithy/node-http-handler`), so this is not the classic missing-keep-alive latency bug. The probe itself is a single signed `ListObjectsV2Command` with no retry (`api/_lib/ops/object-storage-health.js:76-92`), so a genuinely transient one-off Cloudflare hiccup is indistinguishable from a sustained outage in this sensor today, which may be inflating the DOWN/DEGRADED counts above a one-retry version would show. Two independent things to do, neither owned by this reconciliation order: (a) check Cloudflare's own R2 dashboard/status page for the account behind `S3_BUCKET` for incidents correlated with the alert timestamps above (owner/dashboard access this agent does not have); (b) if the pattern looks like a true transient flap rather than sustained Cloudflare-side degradation, a future order could add one immediate retry before `gatherObjectStorageHealth()` classifies `down`/`degraded`, with a test proving a single synthetic failure followed by a synthetic success no longer pages. Not attempted here because the current evidence does not confidently distinguish "add a retry" from "mute a real recurring problem", and CLAUDE.md is explicit that a check does not get weakened without a dated, measured reason; this row exists so neither path gets silently skipped. | Cloudflare dashboard check (owner), then optionally a small, test-covered retry (agent) | Every avatar, thumbnail and generated GLB reads through this bucket; `/cdn/*` and the forge both degrade when it flaps | `ops_alerts` table, queried 2026-10-11 |+| 37 | **Ship the cron time-budget fix with the next deploy** (added 2026-10-11 by the evolve loop, order 403, otherwise complete and retired). `wrapCron()` in `api/_lib/http.js` now races every cron handler against a real time budget (`maxDurationMs`, wired from each handler's own `export const maxDuration * 1000`, defaulting to 120000ms) and answers a clean `503 { reason: 'cron_timeout' }` instead of hanging until Cloud Run's blind 900s ingress kill, which is what `/api/cron/x402-ring-leak-scan` was hitting repeatedly before order 400 landed its own fix. Verified locally: a real `http.createServer` hosting `wrapCron` with an 800ms budget and a never-resolving handler answered `503 cron_timeout` at 0.809s (not 900s, not instantly); the new `tests/http-cron-timeout.test.js` (4 tests) and the added `wrapCron` case in `tests/http-db-unavailable-boundary.test.js` pass 10/10, and the timeout test was proven red (indefinite hang) against the pre-fix code via a path-scoped `git stash` before the fix landed. No migration, no env var, nothing else needed; this is a pure code fix sitting on `evolve` waiting for row 1's routine deploy cadence to carry it to `main` and production. Ship with `npm run deploy:gcp:full` (row 1 already tracks the general "approve the next deploy" gate; this row exists only so the new behavior is named for verification). Post-deploy verification: re-run the step 0 query from `prompts/finish/403-auto-cron-timeout-enforcement.md` (`gcloud logging read 'resource.type="cloud_run_revision" resource.labels.service_name="three-ws-api" textPayload:"maximum request timeout"' --freshness=24h --project aerial-vehicle-466722-p5`) after the deploy window; a healthy rollout shows that query going quiet (or near-quiet) while `textPayload:"exceeded its"` picks up any genuine stuck cron with its name attached, cross-referenced against order 400's `x402-ring-leak-scan` fix so a recurrence is distinguished from a different handler entirely. | one deploy (already covered by row 1's cadence) | Clean, diagnosable cron timeouts platform-wide instead of a blank "maximum request timeout" | `docs/ops/production-log-triage.md`, "HTTP timeout with no cron context" section |Deleted 2026-09-01: the former row 5 (`WALLET_ENCRYPTION_KEY_PREVIOUS` policy), whichdeclared itself "delete on reading" and had outlived that by three weeks.diff --git a/public/changelog-recent.json b/public/changelog-recent.jsonindex 95362390f..babc542d2 100644--- a/public/changelog-recent.json+++ b/public/changelog-recent.json@@ -1,14 +1,24 @@{- "generated_at": "2026-10-11T11:56:18.006Z",+ "generated_at": "2026-10-11T13:17:18.385Z","generated_by": "scripts/build-page-index.mjs from data/pages.json + data/changelog.json","site": {"name": "three.ws","url": "https://three.ws"},- "total": 3858,+ "total": 3859,"complete": false,"full_feed": "/changelog.json","entries": [+ {+ "date": "2026-10-11",+ "type": "update",+ "title": "A stuck background job now fails fast instead of hanging for 15 minutes",+ "summary": "Every scheduled background job (cron) on the platform now answers cleanly and quickly if something inside it gets stuck, instead of silently hanging until the server's own 15-minute connection limit cuts it off with no explanation. Each job now gets a real time budget matching what it already declared, the status board records the failed run immediately, and an alert fires with the job's name attached so it can be diagnosed in seconds instead of traced by hand from a blank timeout. Image rendering for 3D creations got the same treatment: a stuck browser launch, page load, or screenshot now times out on its own instead of tying up the render queue.",+ "link": null,+ "tags": [+ "infra"+ ]+ },{"date": "2026-10-11","type": "update",@@ -1546,17 +1556,6 @@"feature","improvement"]- },- {- "date": "2026-09-18",- "type": "update",- "title": "Granite Guardian now answers on three.ws, not just in the docs",- "summary": "The governance check that screens what an agent is asked to do, and vetoes it when the request is a jailbreak, harmful, or fraudulent, runs on IBM Granite Guardian. It was built to call IBM's hosted watsonx.ai service, so on our live site it answered with a not-configured error, and the new agent identity check quietly skipped itself for the same reason. IBM publishes Granite Guardian's weights openly, so we now run the model ourselves on a dedicated GPU as a second lane: watsonx still leads wherever it is set up, and our own copy answers whenever it is not. Every verdict now says which model and which lane produced it. The identity check also compares a new agent against existing ones using the site's regular embedding service when Granite embeddings are unavailable; it lists the look-alikes it finds but only scores them on Granite, because that is what its thresholds were tuned on.",- "link": "/docs/ibm",- "tags": [- "fix",- "infra"- ]}]}diff --git a/public/changelog.json b/public/changelog.jsonindex 15cac269b..985a3192a 100644--- a/public/changelog.json+++ b/public/changelog.json@@ -1,11 +1,21 @@{- "generated_at": "2026-10-11T11:56:17.985Z",+ "generated_at": "2026-10-11T13:17:18.358Z","generated_by": "scripts/build-page-index.mjs from data/pages.json + data/changelog.json","site": {"name": "three.ws","url": "https://three.ws"},"entries": [+ {+ "date": "2026-10-11",+ "type": "update",+ "title": "A stuck background job now fails fast instead of hanging for 15 minutes",+ "summary": "Every scheduled background job (cron) on the platform now answers cleanly and quickly if something inside it gets stuck, instead of silently hanging until the server's own 15-minute connection limit cuts it off with no explanation. Each job now gets a real time budget matching what it already declared, the status board records the failed run immediately, and an alert fires with the job's name attached so it can be diagnosed in seconds instead of traced by hand from a blank timeout. Image rendering for 3D creations got the same treatment: a stuck browser launch, page load, or screenshot now times out on its own instead of tying up the render queue.",+ "link": null,+ "tags": [+ "infra"+ ]+ },{"date": "2026-10-11","type": "update",diff --git a/public/changelog.xml b/public/changelog.xmlindex 0ffc1e9a5..7231338da 100644--- a/public/changelog.xml+++ b/public/changelog.xml@@ -6,6 +6,14 @@<atom:link href="https://three.ws/changelog.xml" rel="self" type="application/rss+xml"/><description>What's new on three.ws: features, improvements, and releases, newest first.</description><language>en</language>+ <item>+ <title>A stuck background job now fails fast instead of hanging for 15 minutes</title>+ <link>https://three.ws/changelog</link>+ <guid isPermaLink="false">2026-10-11:A stuck background job now fails fast instead of hanging for 15 minutes</guid>+ <pubDate>Sun, 11 Oct 2026 12:00:00 GMT</pubDate>+ <category>infra</category>+ <description>Every scheduled background job (cron) on the platform now answers cleanly and quickly if something inside it gets stuck, instead of silently hanging until the server's own 15-minute connection limit cuts it off with no explanation. Each job now gets a real time budget matching what it already declared, the status board records the failed run immediately, and an alert fires with the job's name attached so it can be diagnosed in seconds instead of traced by hand from a blank timeout. Image rendering for 3D creations got the same treatment: a stuck browser launch, page load, or screenshot now times out on its own instead of tying up the render queue.</description>+ </item><item><title>An agent set to Grok now answers as Grok everywhere, with a badge when it can't</title><link>https://three.ws/changelog</link>diff --git a/public/features.json b/public/features.jsonindex 5df9dbe86..d3acb5a27 100644--- a/public/features.json+++ b/public/features.json@@ -1,5 +1,5 @@{- "generated_at": "2026-10-11T11:56:17.975Z",+ "generated_at": "2026-10-11T13:17:18.347Z","generated_by": "scripts/build-page-index.mjs from data/pages.json","site": {"name": "three.ws",diff --git a/tests/http-cron-timeout.test.js b/tests/http-cron-timeout.test.jsnew file mode 100644index 000000000..5b1e1d5b3--- /dev/null+++ b/tests/http-cron-timeout.test.js@@ -0,0 +1,159 @@+/**+ * wrapCron()'s time budget (api/_lib/http.js).+ *+ * Before this, `maxDuration` was a Vercel-enforced per-function budget. Nothing+ * replaced that enforcement after the 2026-07-07 Cloud Run migration, so every+ * one of the 100 cron handlers ran with no ceiling except Cloud Run's own 900s+ * ingress timeout: a hang anywhere inside one surfaced 15 minutes later as+ * "The request has been terminated because it has reached the maximum request+ * timeout" with no cron name, no elapsed time, and no heartbeat write.+ *+ * wrapCron now races the handler against its own budget and answers a clean+ * 503 `cron_timeout` before that happens, while never cancelling the handler+ * itself (Node has no generic preemption). Real (short) timers are used below+ * rather than fake ones, because the heartbeat write goes through a+ * fire-and-forget dynamic `import('./cache.js')` whose settling is not+ * observable through the fake-timer clock.+ */++import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';++const sent = [];+const cache = new Map();++vi.mock('../api/_lib/alerts.js', () => ({+ sendOpsAlert: async (title, detail, opts) => { sent.push({ title, detail, opts }); },+}));+vi.mock('../api/_lib/cache.js', () => ({+ cacheGet: async (k) => (cache.has(k) ? cache.get(k) : null),+ cacheSet: async (k, v) => { cache.set(k, v); },+}));++const { wrapCron, DEFAULT_CRON_MAX_DURATION_MS } = await import('../api/_lib/http.js');++function fakeRes() {+ const headers = {};+ return {+ statusCode: 0,+ body: undefined,+ ended: false,+ headersSent: false,+ writableEnded: false,+ setHeader(k, v) { headers[String(k).toLowerCase()] = v; },+ getHeader(k) { return headers[String(k).toLowerCase()]; },+ end(b) { this.body = b; this.ended = true; this.writableEnded = true; },+ _headers: headers,+ };+}++function fakeReq(url) {+ return { method: 'GET', url, headers: {} };+}++// Let fire-and-forget heartbeat writes (behind a dynamic import) settle.+function flush(ms = 20) {+ return new Promise((r) => setTimeout(r, ms));+}++let errSpy, warnSpy;+beforeEach(() => {+ cache.clear();+ sent.length = 0;+ errSpy = vi.spyOn(console, 'error').mockImplementation(() => {});+ warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {});+});+afterEach(() => {+ errSpy.mockRestore();+ warnSpy.mockRestore();+});++describe('wrapCron() time budget', () => {+ it('answers 503 cron_timeout before a never-resolving handler settles', async () => {+ const handler = wrapCron(() => new Promise(() => {}), { maxDurationMs: 30 });+ const res = fakeRes();++ await handler(fakeReq('/api/cron/stuck-forever'), res);+ await flush();++ expect(res.statusCode).toBe(503);+ const body = JSON.parse(res.body);+ expect(body).toMatchObject({ ok: false, reason: 'cron_timeout', cron: 'stuck-forever', budget_ms: 30 });++ const beat = cache.get('cron:heartbeat:stuck-forever');+ expect(beat).toMatchObject({ ok: false, timed_out: true });++ expect(sent).toHaveLength(1);+ expect(sent[0].opts.signature).toBe('cron:timeout:stuck-forever');+ });++ it('a handler that resolves well inside the budget is completely unaffected', async () => {+ const budgetHandler = wrapCron(async (req, res) => {+ res.setHeader('x-ok', '1');+ res.statusCode = 200;+ res.end(JSON.stringify({ ok: true, did: 'work' }));+ }, { maxDurationMs: 30_000 });+ const plainHandler = wrapCron(async (req, res) => {+ res.setHeader('x-ok', '1');+ res.statusCode = 200;+ res.end(JSON.stringify({ ok: true, did: 'work' }));+ });++ const resBudget = fakeRes();+ await budgetHandler(fakeReq('/api/cron/quick-one'), resBudget);+ await flush();+ const beatBudget = cache.get('cron:heartbeat:quick-one');++ cache.clear();+ const resPlain = fakeRes();+ await plainHandler(fakeReq('/api/cron/quick-one'), resPlain);+ await flush();+ const beatPlain = cache.get('cron:heartbeat:quick-one');++ expect(resBudget.statusCode).toBe(resPlain.statusCode);+ expect(resBudget.body).toBe(resPlain.body);+ expect(beatBudget).toMatchObject({ ok: true });+ expect(beatPlain).toMatchObject({ ok: true });+ expect(beatBudget.timed_out).toBeUndefined();+ expect(sent).toHaveLength(0);+ });++ it('falls back to a 120000ms budget when maxDurationMs is not passed', async () => {+ expect(DEFAULT_CRON_MAX_DURATION_MS).toBe(120_000);++ // Prove wrapCron actually arms its timer with that default (not waiting out+ // a real 120s budget in a unit test) by capturing the ms argument the+ // race's setTimeout is called with.+ const setTimeoutSpy = vi.spyOn(global, 'setTimeout');+ const handler = wrapCron(() => new Promise(() => {}));+ const res = fakeRes();+ const invocation = handler(fakeReq('/api/cron/no-budget-given'), res);+ await flush();++ const budgetCall = setTimeoutSpy.mock.calls.find((call) => call[1] === DEFAULT_CRON_MAX_DURATION_MS);+ expect(budgetCall).toBeDefined();+ expect(res.statusCode).toBe(0);++ setTimeoutSpy.mockRestore();+ // Unstick the still-pending invocation so it doesn't leak into later tests.+ void invocation;+ });++ it('never lets the handler\'s late settlement crash the process or double-write', async () => {+ let rejectLate;+ const handler = wrapCron(() => new Promise((_resolve, reject) => { rejectLate = reject; }), { maxDurationMs: 30 });+ const res = fakeRes();++ await handler(fakeReq('/api/cron/late-reject'), res);+ await flush();+ expect(res.statusCode).toBe(503);++ // The real handler finally rejects well after the budget already answered.+ // This must never throw an unhandled rejection or try to mutate `res` again.+ expect(() => rejectLate(new Error('late failure, nobody is listening anymore'))).not.toThrow();+ await flush();++ expect(res.statusCode).toBe(503);+ const beat = cache.get('cron:heartbeat:late-reject');+ expect(beat).toMatchObject({ ok: false, timed_out: true });+ });+});diff --git a/tests/http-db-unavailable-boundary.test.js b/tests/http-db-unavailable-boundary.test.jsindex 3de117670..35f55bfb6 100644--- a/tests/http-db-unavailable-boundary.test.js+++ b/tests/http-db-unavailable-boundary.test.js@@ -10,7 +10,15 @@*/import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';-import { wrap, serverError } from '../api/_lib/http.js';+import { wrap, serverError, wrapCron } from '../api/_lib/http.js';++vi.mock('../api/_lib/alerts.js', () => ({+ sendOpsAlert: async () => {},+}));+vi.mock('../api/_lib/cache.js', () => ({+ cacheGet: async () => null,+ cacheSet: async () => {},+}));// Minimal ServerResponse stand-in: case-insensitive header store + end capture.function fakeRes() {@@ -91,6 +99,20 @@ describe('wrap() under a DB outage', () => {const unhandled = errSpy.mock.calls.some((c) => String(c[0]).includes('[api] unhandled'));expect(unhandled).toBe(true);});++ it('wrapCron() keeps the same db_unavailable degradation now that a time budget races the handler', async () => {+ const handler = wrapCron(async () => { throw dbDownError(); }, { maxDurationMs: 30_000 });+ const res = fakeRes();+ await handler({ method: 'GET', url: '/api/cron/smart-money-rollup', headers: {} }, res);++ // Still the pre-existing db_unavailable degradation (200, not the budget's+ // 503 cron_timeout, and not wrap()'s generic 500): the race around the+ // handler must never reclassify this path.+ expect(res.statusCode).toBe(200);+ expect(JSON.parse(res.body)).toMatchObject({ ok: false, reason: 'db_unavailable' });+ const unhandled = errSpy.mock.calls.some((c) => String(c[0]).includes('[api] unhandled'));+ expect(unhandled).toBe(false);+ });});describe('serverError() under a DB outage', () => {