Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.
JavaScript 3d-generationai-agentsanimationavatardigital-humangenerative-aiglbgltfimage-to-3dllmmcpmcp-servermodel-context-protocolsolanatext-to-3dthreejsweb-componentswebglwebxrx402
feat(ops): let the forge sensor see a stall that never writes a row, and cover the credential trim
nirholas4b7b7ba8b429
diff --git a/api/_lib/ops/forge-health-sensor.js b/api/_lib/ops/forge-health-sensor.jsindex caebb9464..00f742ef8 100644--- a/api/_lib/ops/forge-health-sensor.js+++ b/api/_lib/ops/forge-health-sensor.js@@ -34,6 +34,15 @@ const WINDOW_INTERVAL = '6 hours';// without letting a quiet stretch read as an outage.const MIN_ATTEMPTS = 15;+// Stall detection. The rate above can only judge generations that got far enough+// to write a forge_creations row, so a failure UPSTREAM of that row (object+// storage rejecting the reference-image upload, on 2026-09-07) produces no rows+// at all, and no rows reads as a quiet hour. Silence is only innocent when the+// forge was already quiet: a lane that was starting a generation every couple of+// minutes and has started nothing since is down, whatever the surviving rows say.+const STALL_MIN_HOURLY = 3;+const STALL_AGE_MS = 45 * 60 * 1000;+// Rate bands. 3D generation legitimately fails sometimes (hard prompts/images),// so the healthy bar is not 100%: ok ≥ 85%, degraded 60–85%, down < 60%. The// 2026-07 image burst (~52% success) reads `down`; a 77% day reads `degraded`.@@ -114,6 +123,27 @@ export function classifyForgeBuckets(buckets, { minAttempts = MIN_ATTEMPTS } = {};}+/**+ * Judge whether the forge has stopped STARTING generations. Pure: takes the age+ * of the newest row and the recent hourly rate, so the "quiet night vs outage"+ * boundary is unit-testable without a clock or a DB.+ * @param {{ lastCreatedAgeMs: number|null, hourlyRate: number }} input+ * @returns {{ stalled: boolean, detail?: string, hint?: string, ageMinutes: number|null }}+ */+export function classifyForgeStall({ lastCreatedAgeMs, hourlyRate }) {+ const ageMinutes = lastCreatedAgeMs == null ? null : Math.round(lastCreatedAgeMs / 60000);+ if (lastCreatedAgeMs == null || hourlyRate < STALL_MIN_HOURLY || lastCreatedAgeMs < STALL_AGE_MS) {+ return { stalled: false, ageMinutes };+ }+ return {+ stalled: true,+ ageMinutes,+ detail: `no generation has STARTED in ${ageMinutes}m (the forge was averaging ${hourlyRate.toFixed(0)}/hour)`,+ hint:+ 'Zero new rows is not a quiet hour at this volume: the request is failing BEFORE forge_creations is written, so the success-rate sensor cannot see it. That is the shape of a shared dependency, not a lane fault. Check object_storage in this same payload first (a rejected bucket credential blocks the reference-image upload every lane needs), then the text→image lane, then POST /api/forge with a one-word prompt and read the error body.',+ };+}+/*** Read the last WINDOW_INTERVAL of forge outcomes and classify them. Fail-soft:* a missing DB, absent table, or query error resolves to `unknown` so a health@@ -139,7 +169,28 @@ export async function gatherForgeHealth() {GROUP BY status, backend, path, reason`);+ // One extra bounded read: the newest row and the recent volume, so a total+ // stop (no rows at all) is distinguishable from a quiet stretch.+ const [pulse] = /** @type {Array<{ last_at: string|null, n6: number }>} */ (+ await sql`+ SELECT max(created_at) AS last_at,+ count(*)::int AS n6+ FROM forge_creations+ WHERE created_at >= now() - '6 hours'::interval+ `+ );+ const lastCreatedAgeMs = pulse?.last_at ? Date.now() - new Date(pulse.last_at).getTime() : null;+ const stall = classifyForgeStall({ lastCreatedAgeMs, hourlyRate: (Number(pulse?.n6) || 0) / 6 });const v = classifyForgeBuckets(buckets);+ if (stall.stalled) {+ return {+ ...base,+ status: 'down',+ detail: stall.detail || 'no generation has started recently',+ hint: stall.hint,+ metrics: { windowHours: 6, stalled: true, lastStartMinutesAgo: stall.ageMinutes, recentStarts: Number(pulse?.n6) || 0 },+ };+ }return {...base,status: v.status,diff --git a/tests/env-s3-cred.test.js b/tests/env-s3-cred.test.jsnew file mode 100644index 000000000..238876143--- /dev/null+++ b/tests/env-s3-cred.test.js@@ -0,0 +1,71 @@+/**+ * env.js S3 credential hygiene, unit test.+ *+ * The sibling of env-upstash-cred.test.js, for the bucket. A trailing newline or+ * stray space on S3_SECRET_ACCESS_KEY (the classic dashboard / secret-payload+ * paste artifact) is invisible and fatal: the AWS SDK signs the request with the+ * padded secret and R2 answers `SignatureDoesNotMatch`, which reads as "wrong+ * key" rather than "wrong bytes". That failure is total: the forge cannot park a+ * reference image, uploads cannot land, and /cdn cannot read an object, so the+ * getters trim at the source and no signer ever sees the padding.+ */++import { describe, it, expect, beforeEach, afterEach } from 'vitest';+import { env } from '../api/_lib/env.js';+import { objectStorageConfigured } from '../api/_lib/r2.js';++const KEYS = ['S3_ACCESS_KEY_ID', 'S3_SECRET_ACCESS_KEY', 'S3_ENDPOINT', 'S3_BUCKET', 'S3_PUBLIC_DOMAIN'];+let saved;++beforeEach(() => {+ saved = Object.fromEntries(KEYS.map((k) => [k, process.env[k]]));+});++afterEach(() => {+ for (const k of KEYS) {+ if (saved[k] === undefined) delete process.env[k];+ else process.env[k] = saved[k];+ }+});++describe('S3 credential getters', () => {+ it('strips a trailing newline from the secret access key', () => {+ process.env.S3_SECRET_ACCESS_KEY = 'abc123secret\n';+ expect(env.S3_SECRET_ACCESS_KEY).toBe('abc123secret');+ });++ it('strips surrounding whitespace from the access key id', () => {+ process.env.S3_ACCESS_KEY_ID = ' AKIAEXAMPLE ';+ expect(env.S3_ACCESS_KEY_ID).toBe('AKIAEXAMPLE');+ });++ it('leaves a clean credential untouched', () => {+ process.env.S3_SECRET_ACCESS_KEY = 'abc123secret';+ expect(env.S3_SECRET_ACCESS_KEY).toBe('abc123secret');+ });++ it('treats a whitespace-only credential as missing, not as configured', () => {+ process.env.S3_SECRET_ACCESS_KEY = ' \n';+ expect(() => env.S3_SECRET_ACCESS_KEY).toThrow(/S3_SECRET_ACCESS_KEY/);+ });+});++describe('objectStorageConfigured', () => {+ it('is false when a credential is present but blank', () => {+ process.env.S3_ENDPOINT = 'https://acct.r2.cloudflarestorage.com';+ process.env.S3_BUCKET = 'three-ws';+ process.env.S3_PUBLIC_DOMAIN = 'https://pub-example.r2.dev';+ process.env.S3_ACCESS_KEY_ID = 'AKIAEXAMPLE';+ process.env.S3_SECRET_ACCESS_KEY = '\n';+ expect(objectStorageConfigured()).toBe(false);+ });++ it('is true when every var carries a real value', () => {+ process.env.S3_ENDPOINT = 'https://acct.r2.cloudflarestorage.com';+ process.env.S3_BUCKET = 'three-ws';+ process.env.S3_PUBLIC_DOMAIN = 'https://pub-example.r2.dev';+ process.env.S3_ACCESS_KEY_ID = 'AKIAEXAMPLE';+ process.env.S3_SECRET_ACCESS_KEY = 'abc123secret';+ expect(objectStorageConfigured()).toBe(true);+ });+});