Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.
JavaScript 3d-generationai-agentsanimationavatardigital-humangenerative-aiglbgltfimage-to-3dllmmcpmcp-servermodel-context-protocolsolanatext-to-3dthreejsweb-componentswebglwebxrx402
security: harden post-Vercel credential handling (#233)
ni3a03ebb425ff72
diff --git a/.env.example b/.env.exampleindex 0f03f576b..2517e7999 100644--- a/.env.example+++ b/.env.example@@ -1,6 +1,7 @@# ────────────────────────────────────────────────────────────────────────────# three.ws — backend environment-# Copy to .env.local for local dev. In production, set these in Vercel.+# Copy to .env.local for local dev. Production runs on Cloud Run; store secrets+# in Google Secret Manager and attach them as Cloud Run secret references.# ────────────────────────────────────────────────────────────────────────────# Public site origin (no trailing slash)@@ -9,7 +10,8 @@ PUBLIC_APP_ORIGIN=https://three.ws# Postgres (Neon serverless driver — HTTPS connection string)DATABASE_URL=postgres://user:[email protected]/neondb?sslmode=require-# S3-compatible storage (AWS S3, Cloudflare R2, Backblaze B2, etc.)+# S3-compatible storage (AWS S3, Cloudflare R2, Backblaze B2, etc.). Treat both+# credential fields as secrets in production; see docs/ops/hosting-and-credentials.md.S3_ENDPOINT=S3_ACCESS_KEY_ID=S3_SECRET_ACCESS_KEY=@@ -32,8 +34,9 @@ UPSTASH_REDIS_REST_TOKEN=# agent lists, …) at a SECOND Upstash store stops large cache writes from contending# with — and burning the command quota of — the limiter. That contention is what# causes the redis SET timeouts on /api/galaxy/flows. Co-locate this store with your-# prod Vercel region for low write latency. Unset = caches share the store above (no-# behavior change). Vercel-KV second store prefixed `cache`: cache_KV_REST_API_*.+# production Cloud Run region for low write latency. Unset = caches share the store+# above (no behavior change). Legacy KV integrations may prefix the second store+# with `cache`: cache_KV_REST_API_*.UPSTASH_CACHE_REST_URL=UPSTASH_CACHE_REST_TOKEN=@@ -82,7 +85,8 @@ ANTHROPIC_API_KEY=CHAT_MODEL=claude-sonnet-5CHAT_MAX_TOKENS=1024-# Admin key for three.ws chat brand config. Set in Vercel, enter in chat settings to unlock branding panel.+# Admin key for three.ws chat brand config. Store in Secret Manager and attach to+# Cloud Run; enter it in chat settings to unlock the branding panel.# Generate with: openssl rand -base64 32CHAT_ADMIN_KEY=@@ -279,7 +283,8 @@ HF_RECONSTRUCT_API_NAME=generation_all## Generate a fresh keypair with:# node scripts/generate-persona-key.mjs-# then paste the output here / into Vercel env.+# then keep the private value local or add it to Google Secret Manager. Attach+# the production secret to Cloud Run; never pass it in a command argument.PERSONA_JWKS_KID=PERSONA_JWKS_PRIVATE_KEY_PEM=# Optional — auto-derived from the private key if omitted:@@ -292,7 +297,8 @@ PERMISSIONS_RELAYER_ENABLED=false# Hex private key of the relayer EOA that pays gas for redeemDelegations.# This is NOT the user's key — only the agent's server-side relayer.# Generate: node -e "const {Wallet} = require('ethers'); const w = Wallet.createRandom(); console.log(w.privateKey, w.address)"-# NEVER commit a real value. Rotate via Vercel env vars.+# NEVER commit a real value. Rotate through Google Secret Manager and update+# the Cloud Run secret reference.AGENT_RELAYER_KEY=0x...# EIP-55 checksummed address derived from AGENT_RELAYER_KEY — fund with testnet ETH.@@ -952,9 +958,9 @@ APP_ORIGIN=https://three.ws# ── Cron protection ──────────────────────────────────────────────────────────# Shared secret expected as `Authorization: Bearer $CRON_SECRET` on calls to-# /api/cron/*. Vercel cron requests bypass this check via the `x-vercel-cron`-# header, so the secret only matters for manual / external invocation. Without-# it, only the Vercel platform can fire cron jobs.+# /api/cron/*. Google Cloud Scheduler supplies it to production jobs. The+# `x-vercel-cron` header is compatibility behavior, not the production auth+# path. Without CRON_SECRET, production scheduler calls fail closed.# Generate: openssl rand -base64 32CRON_SECRET=@@ -1183,8 +1189,9 @@ AGENC_RPC_URL=# ── Treasury + relayer signing keys ──────────────────────────────────────────# DANGER: these wallets hold real funds. Never share, never commit. Rotate-# via Vercel env vars only. Each is loaded on first use; missing keys cause-# the relevant route to 500 with a clear "<KEY> not configured" message.+# through Google Secret Manager and Cloud Run secret references only. Each is+# loaded on first use; missing keys cause the relevant route to 500 with a clear+# "<KEY> not configured" message.# Solana platform treasury for cron buyback / distribution flows. Base58 or# JSON-array encoded Keypair secret (api/_lib/evm-transfer.js + cron/[name].js).diff --git a/api/_lib/scrub-secrets.js b/api/_lib/scrub-secrets.jsindex 1e202bd3d..abc1ea5ee 100644--- a/api/_lib/scrub-secrets.js+++ b/api/_lib/scrub-secrets.js@@ -66,6 +66,13 @@ const URL_QUERY_CREDENTIAL_RE = /([?&](?:api[-_]?key|access[-_]?token|token|secr// `//user:` then anything up to the `@`. The username is kept (it is useful for// debugging and is not the secret); only the password is masked.const URL_USERINFO_RE = /(\/\/[^/@\s:]+:)[^/@\s]+@/g;+// Provider tokens are sometimes pasted into a user-supplied message rather+// than carried under a secret-shaped object key. Keep this list narrow to+// self-identifying formats so ordinary hashes and transaction ids survive.+const INLINE_PROVIDER_CREDENTIAL_RE = /\b(?:gh[pousr]_[A-Za-z0-9]{36,}|github_pat_[A-Za-z0-9_]{60,}|cf(?:at|ut)_[A-Za-z0-9_-]{32,}|npm_[A-Za-z0-9]{36})\b/g;+// R2 S3 credentials do not have a unique prefix. Only redact their hex value+// when a credential label makes the meaning unambiguous.+const INLINE_R2_CREDENTIAL_RE = /((?:S3_ACCESS_KEY_ID|S3_SECRET_ACCESS_KEY|access\s+key\s+id|secret\s+access\s+key)\s*[:=]\s*)[a-f0-9]{32,64}\b/gi;/*** Mask credential VALUES embedded in a URL inside a free-form string (typically@@ -81,5 +88,7 @@ const URL_USERINFO_RE = /(\/\/[^/@\s:]+:)[^/@\s]+@/g;export function redactUrlSecrets(text) {return String(text ?? '').replace(URL_QUERY_CREDENTIAL_RE, '$1REDACTED')- .replace(URL_USERINFO_RE, '$1REDACTED@');+ .replace(URL_USERINFO_RE, '$1REDACTED@')+ .replace(INLINE_PROVIDER_CREDENTIAL_RE, 'REDACTED')+ .replace(INLINE_R2_CREDENTIAL_RE, '$1REDACTED');}diff --git a/docs/ops/README.md b/docs/ops/README.mdindex a6700049f..554497b85 100644--- a/docs/ops/README.md+++ b/docs/ops/README.md@@ -7,6 +7,7 @@ Start with [gcp-production.md](gcp-production.md); it is the complete operationa| Runbook | What it covers ||---|---|| [gcp-production.md](gcp-production.md) | The production platform end to end: Cloud Run services, LB/DNS/TLS, env, deploy, rollback, recovery. |+| [hosting-and-credentials.md](hosting-and-credentials.md) | Hosting source of truth, Vercel GitHub App removal, GCP authentication, Secret Manager boundaries, R2 rotation, and exposed-credential response. || [gcp-credits-plan.md](gcp-credits-plan.md) | Standing map of the ~$100k GCP credit spend: fleet, quota, pre-approved scaling, what to do next without asking. || [gcp-credits.md](gcp-credits.md) | The Vertex AI and GCP footprint runbook backing the credits plan. || [gcp-model-workers.md](gcp-model-workers.md) | Self-hosted GPU generation lanes on Cloud Run: workers, weights, deploys. |diff --git a/docs/ops/hosting-and-credentials.md b/docs/ops/hosting-and-credentials.mdnew file mode 100644index 000000000..062f24272--- /dev/null+++ b/docs/ops/hosting-and-credentials.md@@ -0,0 +1,164 @@+# Hosting and credential operations++This is the operator record for three.ws hosting, repository integrations, and production+credentials. It exists to prevent three recurring failures: treating `vercel.json` as proof that+Vercel still hosts production, pasting a replacement credential into an issue or terminal command,+and updating one Cloud Run variable in a way that erases the rest.++## Hosting source of truth++Production runs on Google Cloud Run in project `aerial-vehicle-466722-p5`, primarily in+`us-central1`. The public site is served through the Google Cloud load balancer and CDN. Check the+deployed revision at `GET https://three.ws/api/version` and the subsystem roll-up at+`GET https://three.ws/api/healthz`.++`vercel.json` remains a live compatibility manifest. The Cloud Run Express server reads its route,+function, header, and cron declarations, and the scheduler tooling reads the same cron list. The+file name is historical; deleting or casually renaming it breaks production routing and schedule+drift checks. Its `git.deploymentEnabled: false` setting prevents Vercel Git deployments.++The `Vercel - three.ws` check on GitHub pull requests is separate. It is emitted by the installed+Vercel GitHub App even when deployments are disabled. Remove three.ws from that app at+`https://github.com/settings/installations` under **Vercel > Configure**, or uninstall the app if it+serves no repository. This is a GitHub account-administration action; repository push permission is+not enough.++## Authentication checks++These commands print identity and configuration, never credential values:++```sh+gh auth status+gcloud auth list --filter=status:ACTIVE --format='table(account,status)'+gcloud config get-value project+gcloud projects describe aerial-vehicle-466722-p5 --format='value(projectId)'+```++Use `gh auth login` for an interactive GitHub login. Do not put a personal access token in a command+argument, issue, pull request, chat, shell history, or repository file. The repository-administration+tasks in this runbook require an account with `admin` permission on `nirholas/three.ws`.++## Credential boundary++Production secrets belong in Google Secret Manager and reach Cloud Run through `valueFrom`+references. Non-secret configuration may remain a literal Cloud Run environment variable. Inspect+names and storage modes without reading values:++```sh+node scripts/read-service-env.mjs --names+gcloud secrets list --project aerial-vehicle-466722-p5 --format='value(name)'+```++Never use `--set-env-vars` or `--set-secrets` for a single change; both can replace the complete+configuration. Use `--update-env-vars` and `--update-secrets`.++### R2 mapping++The object-storage adapter uses S3-compatible names:++| Variable | Secret? | Production source |+|---|---|---|+| `S3_ENDPOINT` | No | Cloud Run literal |+| `S3_BUCKET` | No | Cloud Run literal |+| `S3_PUBLIC_DOMAIN` | No | Cloud Run literal |+| `S3_ACCESS_KEY_ID` | Yes | Secret Manager |+| `S3_SECRET_ACCESS_KEY` | Yes | Secret Manager |++The long-lived R2 credential needs object read/write access only to the production bucket. Do not+grant account administration or cross-bucket access to the application credential.++### Safe R2 rotation++Create the replacement in the provider console, then transfer each value from a private local file+or standard input. Never paste it into the command itself. The examples below assume files outside+the repository with mode `0600`:++```sh+gcloud secrets describe s3-access-key-id --project aerial-vehicle-466722-p5 >/dev/null 2>&1 || \+ gcloud secrets create s3-access-key-id --project aerial-vehicle-466722-p5 --replication-policy=automatic+gcloud secrets describe s3-secret-access-key --project aerial-vehicle-466722-p5 >/dev/null 2>&1 || \+ gcloud secrets create s3-secret-access-key --project aerial-vehicle-466722-p5 --replication-policy=automatic++gcloud secrets versions add s3-access-key-id \+ --project aerial-vehicle-466722-p5 --data-file=/private/path/r2-access-key-id+gcloud secrets versions add s3-secret-access-key \+ --project aerial-vehicle-466722-p5 --data-file=/private/path/r2-secret-access-key++gcloud run services update three-ws-api \+ --project aerial-vehicle-466722-p5 \+ --region us-central1 \+ --update-secrets S3_ACCESS_KEY_ID=s3-access-key-id:latest,S3_SECRET_ACCESS_KEY=s3-secret-access-key:latest+```++Wait for the revision to become Ready, then verify the storage credential without uploading user+data:++```sh+curl -fsS -X POST https://three.ws/api/forge-upload \+ -H 'content-type: application/json' \+ -H 'x-forge-client: ops-storage-check' \+ --data '{"content_type":"image/png","size_bytes":1024}'+```++A healthy response is `200` with `method: "PUT"` and `expires_in`; do not print or share the signed+`upload_url`. `503 storage_unavailable` means the bucket rejected the credential. Revert Cloud Run+to the previous secret version before investigating scope, bucket, endpoint, or signature mismatch.+Only revoke the old provider credential after the production check passes.++## Exposed-credential incident++A credential pasted into chat, an issue, a pull request, a terminal command, or a committed file is+compromised even if the message is later deleted. The response is rotation, not concealment:++1. Revoke the exposed credential at the provider.+2. Create a least-privilege replacement through the provider's secure console.+3. Add a new Secret Manager version without printing the value.+4. update the Cloud Run secret reference and wait for a Ready revision;+5. verify the exact dependent feature;+6. revoke the prior production credential after verification;+7. review provider audit logs for use after the exposure time;+8. record names, timestamps, scope, and verification outcome, never values.++For GitHub, revoke both classic and fine-grained personal access tokens from account settings. For+Cloudflare, revoke the API token and the R2 S3 credential separately. For npm, revoke the automation+token, create a scoped replacement with the narrowest required package permissions, and review the+account's token activity.++The pre-push guard at `scripts/check-secrets.mjs` rejects provider token formats and R2 credential+pairs in added content. The runtime `redactUrlSecrets` guard also masks those formats when they+arrive inside free-form text destined for structured logs. Both are defense in depth, not a+substitute for rotation after disclosure.++## Production verification after any credential change++Run the narrow feature check first, then the platform sweep:++```sh+curl -fsS https://three.ws/api/version+curl -fsS https://three.ws/api/healthz+npm run triage:gcp -- --json --deep --since 1h+```++Interpret a mass page failure alongside reachability. If `/api/version`, `/api/healthz`, and the home+page all answer immediately while a single deep sweep reports every page failed, repeat the pages+probe before changing routes; a shared probe timeout is not hundreds of independent routing bugs.++## Current incident record: 2026-09-15++- GCP authentication succeeded for `aerial-vehicle-466722-p5`.+- All 47 Cloud Run services reported Ready.+- All 117 declared cron jobs matched Cloud Scheduler and loaded successfully.+- The public health, version, and home endpoints returned `200` on independent probes.+- The deep sweep's 822 page failures, version timeout, and TLS timeout shared one transient probe+ window and were not reproduced independently.+- During revision `three-ws-api-00430-bbn` rollout, `POST /api/forge-upload` returned+ `503 storage_unavailable`. After the revision became Ready, the same storage preflight returned+ `200` with a presigned `PUT` URL. No object or user data was uploaded. Treat the replacement R2+ credential as operational, and revoke every credential exposed during the incident.+- The x402 self-facilitator reported 2,011 `payer_fee_unfunded` verification rejects and 139+ `fee_wallet_below_floor` settlement failures. Funding or moving on-chain value remains an+ owner-confirmed action under the transaction confirmation policy.+- GitHub pull requests still receive the obsolete Vercel App status. It did not prevent the approved+ marketing pull request from merging, but the app should be removed to eliminate the false failure.++No credential value belongs in this record.diff --git a/scripts/check-secrets.mjs b/scripts/check-secrets.mjsindex 03685a33e..d8361eb25 100644--- a/scripts/check-secrets.mjs+++ b/scripts/check-secrets.mjs@@ -113,6 +113,7 @@ const SKIP_CONTENT = [// must hold synthetic examples of every shape they redact./^api\/_lib\/scrub-secrets\.js$/,/^tests\/scrub-secrets\.test\.js$/,+ /^tests\/check-secrets\.test\.js$/,];const skippedContent = (file) => SKIP_CONTENT.some((re) => re.test(file));@@ -132,9 +133,23 @@ const CONTENT_RULES = [// Each alternative is a format one vendor issues and nothing else// produces. Lengths are the vendor minimums, so a truncated docs sample// ("sk-ant-...") does not match.- find: /(sk-ant-api[0-9]{2}-[A-Za-z0-9_-]{40,}|sk-proj-[A-Za-z0-9_-]{40,}|sk-[A-Za-z0-9]{48,}|gh[pousr]_[A-Za-z0-9]{36,}|github_pat_[A-Za-z0-9_]{60,}|glpat-[A-Za-z0-9_-]{20,}|AKIA[0-9A-Z]{16}|AIza[0-9A-Za-z_-]{35}|xox[baprs]-[0-9A-Za-z-]{20,}|hf_[A-Za-z0-9]{34,}|nvapi-[A-Za-z0-9_-]{40,}|r8_[A-Za-z0-9]{37,}|gsk_[A-Za-z0-9]{50,}|sk-or-v1-[a-f0-9]{60,}|SG\.[A-Za-z0-9_-]{20,}\.[A-Za-z0-9_-]{30,}|npm_[A-Za-z0-9]{36}|dckr_pat_[A-Za-z0-9_-]{20,}|ya29\.[0-9A-Za-z_-]{60,})/,+ find: /(sk-ant-api[0-9]{2}-[A-Za-z0-9_-]{40,}|sk-proj-[A-Za-z0-9_-]{40,}|sk-[A-Za-z0-9]{48,}|gh[pousr]_[A-Za-z0-9]{36,}|github_pat_[A-Za-z0-9_]{60,}|glpat-[A-Za-z0-9_-]{20,}|AKIA[0-9A-Z]{16}|AIza[0-9A-Za-z_-]{35}|xox[baprs]-[0-9A-Za-z-]{20,}|hf_[A-Za-z0-9]{34,}|nvapi-[A-Za-z0-9_-]{40,}|r8_[A-Za-z0-9]{37,}|gsk_[A-Za-z0-9]{50,}|sk-or-v1-[a-f0-9]{60,}|SG\.[A-Za-z0-9_-]{20,}\.[A-Za-z0-9_-]{30,}|npm_[A-Za-z0-9]{36}|dckr_pat_[A-Za-z0-9_-]{20,}|ya29\.[0-9A-Za-z_-]{60,}|cf(?:at|ut)_[A-Za-z0-9_-]{32,})/,except: (value) => FAKE_VALUE.test(value),},+ {+ id: 'r2-access-key',+ what: 'an R2 S3-compatible access key identifier',+ find: /(?:S3_ACCESS_KEY_ID|access\s+key\s+id)\W{0,8}([a-f0-9]{32})\b/i,+ group: 1,+ except: (value) => FAKE_VALUE.test(value) || new Set(value.toLowerCase()).size < 8,+ },+ {+ id: 'r2-secret-key',+ what: 'an R2 S3-compatible secret access key',+ find: /(?:S3_SECRET_ACCESS_KEY|secret\s+access\s+key)\W{0,8}([a-f0-9]{64})\b/i,+ group: 1,+ except: (value) => FAKE_VALUE.test(value) || new Set(value.toLowerCase()).size < 8,+ },{id: 'private-key-block',what: 'a PEM private-key block',diff --git a/tests/check-secrets.test.js b/tests/check-secrets.test.jsnew file mode 100644index 000000000..003cbbf35--- /dev/null+++ b/tests/check-secrets.test.js@@ -0,0 +1,51 @@+import { execFileSync, spawnSync } from 'node:child_process';+import { copyFileSync, mkdirSync, mkdtempSync, writeFileSync } from 'node:fs';+import { tmpdir } from 'node:os';+import { dirname, join } from 'node:path';+import { fileURLToPath } from 'node:url';+import { describe, expect, it } from 'vitest';++const REPO = join(dirname(fileURLToPath(import.meta.url)), '..');++function scannerRepo() {+ const root = mkdtempSync(join(tmpdir(), 'threews-secret-scan-'));+ mkdirSync(join(root, 'scripts'));+ copyFileSync(join(REPO, 'scripts', 'check-secrets.mjs'), join(root, 'scripts', 'check-secrets.mjs'));+ writeFileSync(join(root, 'README.md'), '# scanner fixture\n');+ execFileSync('git', ['init', '-q'], { cwd: root });+ execFileSync('git', ['config', 'user.email', '[email protected]'], { cwd: root });+ execFileSync('git', ['config', 'user.name', 'Scanner Test'], { cwd: root });+ execFileSync('git', ['add', 'README.md', 'scripts/check-secrets.mjs'], { cwd: root });+ execFileSync('git', ['commit', '-qm', 'fixture'], { cwd: root });+ return root;+}++function scanAddedLine(line) {+ const root = scannerRepo();+ writeFileSync(join(root, 'leak.txt'), `${line}\n`);+ return spawnSync('node', ['scripts/check-secrets.mjs'], {+ cwd: root,+ encoding: 'utf8',+ });+}++describe('credential leak guard', () => {+ it.each([+ ['Cloudflare legacy token', `cfat_${'Ab7_'.repeat(12)}`, 'provider-api-key'],+ ['Cloudflare current token', `cfut_${'Xy9_'.repeat(12)}`, 'provider-api-key'],+ ['GitHub fine-grained token', `github_pat_${'Az7_'.repeat(18)}`, 'provider-api-key'],+ ['npm automation token', `npm_${'Q7wE'.repeat(9)}`, 'provider-api-key'],+ ['R2 access key id', `Access Key ID: ${'9f4e8d2c7b6a1053'.repeat(2)}`, 'r2-access-key'],+ ['R2 secret access key', `Secret Access Key: ${'9f4e8d2c7b6a1053'.repeat(4)}`, 'r2-secret-key'],+ ])('rejects a %s', (_label, line, finding) => {+ const result = scanAddedLine(line);+ expect(result.status).toBe(1);+ expect(result.stderr).toContain(`[${finding}]`);+ });++ it('allows documented placeholders', () => {+ const result = scanAddedLine('S3_SECRET_ACCESS_KEY=YOUR_SECRET_ACCESS_KEY');+ expect(result.status).toBe(0);+ expect(result.stdout).toContain('OK');+ });+});diff --git a/tests/scrub-secrets.test.js b/tests/scrub-secrets.test.jsindex d2e08fb1c..285b1dfc5 100644--- a/tests/scrub-secrets.test.js+++ b/tests/scrub-secrets.test.js@@ -100,6 +100,25 @@ describe('redactUrlSecrets', () => {expect(redis).toContain('redis://default:[email protected]:6379');});+ it('masks provider tokens pasted into free-form text', () => {+ const github = `github_pat_${'Az7_'.repeat(18)}`;+ const cloudflare = `cfat_${'Kp9_'.repeat(12)}`;+ const npm = `npm_${'Q7wE'.repeat(9)}`;+ const out = redactUrlSecrets(`credentials: ${github} ${cloudflare} ${npm}`);+ for (const leaked of [github, cloudflare, npm]) expect(out).not.toContain(leaked);+ expect(out.match(/REDACTED/g)).toHaveLength(3);+ });++ it('masks labeled R2 access and secret keys in free-form text', () => {+ const access = '9f4e8d2c7b6a1053'.repeat(2);+ const secret = '9f4e8d2c7b6a1053'.repeat(4);+ const out = redactUrlSecrets(`Access Key ID: ${access} Secret Access Key: ${secret}`);+ expect(out).not.toContain(access);+ expect(out).not.toContain(secret);+ expect(out).toContain('Access Key ID: REDACTED');+ expect(out).toContain('Secret Access Key: REDACTED');+ });+it('keeps the username and host readable, masking only the password', () => {// Redaction has to leave enough behind to debug with: which database, which// host, which role. Only the secret goes.