Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.
JavaScript 3d-generationai-agentsanimationavatardigital-humangenerative-aiglbgltfimage-to-3dllmmcpmcp-servermodel-context-protocolsolanatext-to-3dthreejsweb-componentswebglwebxrx402
fix(cron): say why a declared cron has no Cloud Scheduler job
nirholas1df41bef2a21
diff --git a/docs/ops/gcp-production.md b/docs/ops/gcp-production.mdindex 17bf24618..0dc40549d 100644--- a/docs/ops/gcp-production.md+++ b/docs/ops/gcp-production.md@@ -332,12 +332,19 @@ Declaring a cron in `vercel.json` does nothing on its own; only the sync scriptbelow creates the job. That gap has bitten repeatedly (on 2026-07-25 fivedeclared crons had no scheduler job at all and had therefore never run inproduction: `kol-tracker-refresh`, `forge-thumbnail-backfill`,-`forge-off-crown`, `quality-bench`, `confirm-pending-purchases`), so audit with-the two checks rather than by eye:+`forge-off-crown`, `quality-bench`, `confirm-pending-purchases`; again on+2026-08-14 for `retention-rollup` and `likeness-eval`), so audit with the two+checks rather than by eye:```bash# Which declared crons are missing, paused, or on a different schedule than# vercel.json says? Needs a live gcloud session; add --json for a report.+# Each MISSING row also says WHY, from production's own answer on that path:+# "deployed, never synced" -> the handler is live and only the job is absent,+# so the sync below fixes it right now.+# "handler not deployed" -> the path 404s, so its job belongs to the deploy+# that ships the handler; syncing now would only+# schedule a 404 every run.npm run check:cron-drift# Do the handlers behind them route, load, run, and refuse an anonymous caller?diff --git a/scripts/check-cron-drift.mjs b/scripts/check-cron-drift.mjsindex 66f882b33..28a63cc6e 100644--- a/scripts/check-cron-drift.mjs+++ b/scripts/check-cron-drift.mjs@@ -38,6 +38,43 @@ const argv = process.argv.slice(2);const OFFLINE = argv.includes('--offline');const AS_JSON = argv.includes('--json');+// Where a declared cron path is probed to tell the two causes of MISSING apart.+const SITE = 'https://three.ws';++/**+ * Why a declared cron has no Cloud Scheduler job, from the production response+ * to its own path. The two causes need opposite actions and look identical in a+ * plain MISSING list:+ *+ * 404 the handler is not in the running revision yet, so creating the job now+ * would only schedule a 404 every run. It belongs to the deploy that+ * ships the handler.+ * any the handler is live (its cron gate answers 401/503 unauthenticated) and+ * nothing but the scheduler sync is missing, so the job can be created+ * right now.+ *+ * A probe that never answered returns null rather than guessing, because an+ * unreachable site is not evidence about the handler.+ */+export function classifyMissing(status) {+ if (status == null) return null;+ return status === 404 ? 'handler not deployed' : 'deployed, never synced';+}++/** Probe a cron path on the live site. Returns its status, or null if unreachable. */+async function probeStatus(cronPath) {+ try {+ const res = await fetch(`${SITE}${cronPath}`, {+ method: 'GET',+ redirect: 'manual',+ signal: AbortSignal.timeout(8000),+ });+ return res.status;+ } catch {+ return null;+ }+}+/** Parse a cron expression, returning its next fire times or an error. */function validate(schedule) {try {@@ -59,7 +96,9 @@ async function liveJobs() {const { stdout } = await execFileP('gcloud',[- 'scheduler', 'jobs', 'list',+ 'scheduler',+ 'jobs',+ 'list',`--project=${PROJECT}`,`--location=${LOCATION}`,'--format=json',@@ -71,131 +110,176 @@ async function liveJobs() {const byId = new Map();for (const job of rows) {// name is fully qualified: projects/…/locations/…/jobs/<id>- const id = String(job.name || '').split('/').pop();+ const id = String(job.name || '')+ .split('/')+ .pop();byId.set(id, { schedule: job.schedule, state: job.state, timeZone: job.timeZone });}return byId;}-const { crons } = JSON.parse(readFileSync(path.join(ROOT, 'vercel.json'), 'utf8'));-if (!Array.isArray(crons) || !crons.length) {- console.error('No crons found in vercel.json.');- process.exit(1);-}+async function main() {+ const { crons } = JSON.parse(readFileSync(path.join(ROOT, 'vercel.json'), 'utf8'));+ if (!Array.isArray(crons) || !crons.length) {+ console.error('No crons found in vercel.json.');+ process.exit(1);+ }++ const invalid = [];+ const declared = [];+ for (const { path: cronPath, schedule } of crons) {+ const v = validate(schedule);+ if (!v.ok) {+ invalid.push({ path: cronPath, schedule, error: v.error });+ continue;+ }+ declared.push({+ path: cronPath,+ schedule,+ id: jobId(cronPath),+ everyMinutes: intervalMinutes(v.next),+ nextFire: v.next[0],+ });+ }-const invalid = [];-const declared = [];-for (const { path: cronPath, schedule } of crons) {- const v = validate(schedule);- if (!v.ok) {- invalid.push({ path: cronPath, schedule, error: v.error });- continue;+ // Duplicate paths would make one job silently overwrite the other on sync.+ const seen = new Map();+ const duplicates = [];+ for (const d of declared) {+ if (seen.has(d.id)) duplicates.push({ id: d.id, paths: [seen.get(d.id), d.path] });+ else seen.set(d.id, d.path);}- declared.push({- path: cronPath,- schedule,- id: jobId(cronPath),- everyMinutes: intervalMinutes(v.next),- nextFire: v.next[0],- });-}-// Duplicate paths would make one job silently overwrite the other on sync.-const seen = new Map();-const duplicates = [];-for (const d of declared) {- if (seen.has(d.id)) duplicates.push({ id: d.id, paths: [seen.get(d.id), d.path] });- else seen.set(d.id, d.path);-}+ const report = {+ declared: declared.length,+ invalid,+ duplicates,+ missing: [],+ mismatched: [],+ paused: [],+ orphaned: [],+ checkedLive: false,+ };-const report = {- declared: declared.length,- invalid,- duplicates,- missing: [],- mismatched: [],- paused: [],- orphaned: [],- checkedLive: false,-};--if (!OFFLINE) {- try {- const live = await liveJobs();- report.checkedLive = true;- for (const d of declared) {- const job = live.get(d.id);- if (!job) {- report.missing.push({ path: d.path, id: d.id, schedule: d.schedule });- continue;+ if (!OFFLINE) {+ try {+ const live = await liveJobs();+ report.checkedLive = true;+ for (const d of declared) {+ const job = live.get(d.id);+ if (!job) {+ report.missing.push({ path: d.path, id: d.id, schedule: d.schedule });+ continue;+ }+ if (String(job.schedule).trim() !== String(d.schedule).trim()) {+ report.mismatched.push({+ path: d.path,+ id: d.id,+ declared: d.schedule,+ live: job.schedule,+ });+ }+ if (job.state && job.state !== 'ENABLED') {+ report.paused.push({ path: d.path, id: d.id, state: job.state });+ }}- if (String(job.schedule).trim() !== String(d.schedule).trim()) {- report.mismatched.push({- path: d.path,- id: d.id,- declared: d.schedule,- live: job.schedule,- });- }- if (job.state && job.state !== 'ENABLED') {- report.paused.push({ path: d.path, id: d.id, state: job.state });- }- }- const declaredIds = new Set(declared.map((d) => d.id));- for (const [id, job] of live) {- if (id.startsWith('cron-') && !declaredIds.has(id)) {- report.orphaned.push({ id, schedule: job.schedule, state: job.state });+ const declaredIds = new Set(declared.map((d) => d.id));+ for (const [id, job] of live) {+ if (id.startsWith('cron-') && !declaredIds.has(id)) {+ report.orphaned.push({ id, schedule: job.schedule, state: job.state });+ }}+ } catch (err) {+ report.liveError = err.message.split('\n')[0];}- } catch (err) {- report.liveError = err.message.split('\n')[0];++ // A MISSING job has two causes with opposite fixes. Ask production which.+ await Promise.all(+ report.missing.map(async (m) => {+ m.reason = classifyMissing(await probeStatus(m.path));+ }),+ );}-}-const problems =- report.invalid.length +- report.duplicates.length +- report.missing.length +- report.mismatched.length +- report.paused.length;+ const problems =+ report.invalid.length ++ report.duplicates.length ++ report.missing.length ++ report.mismatched.length ++ report.paused.length;-if (AS_JSON) {- console.log(JSON.stringify(report, null, 2));- process.exit(problems ? 1 : 0);-}+ if (AS_JSON) {+ console.log(JSON.stringify(report, null, 2));+ process.exit(problems ? 1 : 0);+ }-console.log(`Declared crons in vercel.json: ${report.declared}`);-const busiest = [...declared]- .filter((d) => d.everyMinutes != null)- .sort((a, b) => a.everyMinutes - b.everyMinutes)- .slice(0, 5);-if (busiest.length) {- console.log('Most frequent:');- for (const d of busiest) console.log(` every ${d.everyMinutes}m ${d.path}`);-}+ console.log(`Declared crons in vercel.json: ${report.declared}`);+ const busiest = [...declared]+ .filter((d) => d.everyMinutes != null)+ .sort((a, b) => a.everyMinutes - b.everyMinutes)+ .slice(0, 5);+ if (busiest.length) {+ console.log('Most frequent:');+ for (const d of busiest) console.log(` every ${d.everyMinutes}m ${d.path}`);+ }-for (const [label, rows, fmt] of [- ['INVALID expression', report.invalid, (r) => `${r.path} "${r.schedule}" ${r.error}`],- ['DUPLICATE job id', report.duplicates, (r) => `${r.id} ${r.paths.join(' vs ')}`],- ['MISSING in Cloud Scheduler', report.missing, (r) => `${r.path} (${r.id})`],- ['SCHEDULE MISMATCH', report.mismatched, (r) => `${r.path} declared "${r.declared}" live "${r.live}"`],- ['NOT ENABLED', report.paused, (r) => `${r.path} state=${r.state}`],- ['ORPHANED job (no longer declared)', report.orphaned, (r) => `${r.id} "${r.schedule}"`],-]) {- if (!rows.length) continue;- console.log(`\n${label}: ${rows.length}`);- for (const r of rows) console.log(` ${fmt(r)}`);-}+ for (const [label, rows, fmt] of [+ ['INVALID expression', report.invalid, (r) => `${r.path} "${r.schedule}" ${r.error}`],+ ['DUPLICATE job id', report.duplicates, (r) => `${r.id} ${r.paths.join(' vs ')}`],+ [+ 'MISSING in Cloud Scheduler',+ report.missing,+ (r) => `${r.path} (${r.id})${r.reason ? ` ${r.reason}` : ''}`,+ ],+ [+ 'SCHEDULE MISMATCH',+ report.mismatched,+ (r) => `${r.path} declared "${r.declared}" live "${r.live}"`,+ ],+ ['NOT ENABLED', report.paused, (r) => `${r.path} state=${r.state}`],+ ['ORPHANED job (no longer declared)', report.orphaned, (r) => `${r.id} "${r.schedule}"`],+ ]) {+ if (!rows.length) continue;+ console.log(`\n${label}: ${rows.length}`);+ for (const r of rows) console.log(` ${fmt(r)}`);+ }++ const unsynced = report.missing.filter((m) => m.reason === 'deployed, never synced');+ const undeployed = report.missing.filter((m) => m.reason === 'handler not deployed');+ if (unsynced.length) {+ console.log(+ `\n${unsynced.length} declared cron(s) are live in production with no job, so they have never fired.`,+ );+ console.log('Create them now: node scripts/create-gcp-scheduler.mjs --env-file <prod env>');+ }+ if (undeployed.length) {+ console.log(+ `\n${undeployed.length} declared cron(s) answer 404 in production: their handler is not in the running revision.`,+ );+ console.log(+ 'Their jobs belong to the deploy that ships the handler, not to a sync run now.',+ );+ }-if (report.liveError) {- console.log(`\nCould not read Cloud Scheduler: ${report.liveError}`);- console.log('Expression validation still ran. Re-run after `gcloud auth login` to compare live jobs.');-} else if (!OFFLINE && !problems) {- console.log('\nNo drift: every declared cron exists, is enabled, and matches its live schedule.');-} else if (OFFLINE && !problems) {- console.log('\nAll cron expressions are valid (offline mode: live jobs not compared).');+ if (report.liveError) {+ console.log(`\nCould not read Cloud Scheduler: ${report.liveError}`);+ console.log(+ 'Expression validation still ran. Re-run after `gcloud auth login` to compare live jobs.',+ );+ } else if (!OFFLINE && !problems) {+ console.log(+ '\nNo drift: every declared cron exists, is enabled, and matches its live schedule.',+ );+ } else if (OFFLINE && !problems) {+ console.log('\nAll cron expressions are valid (offline mode: live jobs not compared).');+ }++ // Orphans alone do not fail the check: a job intentionally left behind during+ // a migration is not a broken deploy. Everything else is.+ process.exit(problems ? 1 : 0);}-// Orphans alone do not fail the check — a job intentionally left behind during-// a migration is not a broken deploy. Everything else is.-process.exit(problems ? 1 : 0);+// Importing this file (the tests do, for classifyMissing) must not run the check.+if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {+ await main();+}diff --git a/server/README.md b/server/README.mdindex 859f75985..c7285bb56 100644--- a/server/README.md+++ b/server/README.md@@ -116,6 +116,10 @@ restarts every job.Verify the mirror with `npm run check:cron-drift` (declared vs. live schedulesand run state) and the handlers behind it with `npm run audit:cron-liveness`.+A cron with no job at all is reported with the reason production gives for it:+`deployed, never synced` means the handler is live and a sync fixes it now,+while `handler not deployed` means the path still 404s, so its job belongs to+the deploy that ships the handler.## Envdiff --git a/tests/cron-drift-classify.test.js b/tests/cron-drift-classify.test.jsnew file mode 100644index 000000000..0e0d0908b--- /dev/null+++ b/tests/cron-drift-classify.test.js@@ -0,0 +1,34 @@+// Tests for how `scripts/check-cron-drift.mjs` explains a MISSING Cloud+// Scheduler job.+//+// "MISSING" has two causes that look identical in the report and need opposite+// actions, and picking the wrong one is expensive both ways: syncing a cron+// whose handler is not in the running revision schedules a 404 every run, while+// waiting for a deploy that a live handler does not need leaves a declared cron+// that has never fired once (that is exactly how /api/cron/retention-rollup and+// /api/cron/likeness-eval sat unscheduled on 2026-08-14). The classifier reads+// production's own answer on the cron path instead of guessing.++import { describe, it, expect } from 'vitest';+import { classifyMissing } from '../scripts/check-cron-drift.mjs';++describe('classifyMissing', () => {+ it('calls a 404 an undeployed handler, whose job belongs to the next deploy', () => {+ expect(classifyMissing(404)).toBe('handler not deployed');+ });++ it('treats every answering status as live, so the sync is the only thing missing', () => {+ // 401 is a real CRON_SECRET rejecting an unauthenticated probe and 503 is+ // the gate failing closed with the secret unset. Both prove the handler is+ // in the running revision, which is the only question being asked.+ expect(classifyMissing(401)).toBe('deployed, never synced');+ expect(classifyMissing(503)).toBe('deployed, never synced');+ expect(classifyMissing(200)).toBe('deployed, never synced');+ expect(classifyMissing(500)).toBe('deployed, never synced');+ });++ it('stays silent when the probe never answered rather than inventing a cause', () => {+ expect(classifyMissing(null)).toBe(null);+ expect(classifyMissing(undefined)).toBe(null);+ });+});