Skip to content

Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.

feat(ops): let the forge sensor see a stall that never writes a row, and cover the credential trim

nirholas4b7b7ba8b429
diff --git a/api/_lib/ops/forge-health-sensor.js b/api/_lib/ops/forge-health-sensor.js
index caebb9464..00f742ef8 100644
--- a/api/_lib/ops/forge-health-sensor.js
+++ b/api/_lib/ops/forge-health-sensor.js
@@ -34,6 +34,15 @@ const WINDOW_INTERVAL = '6 hours';
// without letting a quiet stretch read as an outage.
const MIN_ATTEMPTS = 15;
+// Stall detection. The rate above can only judge generations that got far enough
+// to write a forge_creations row, so a failure UPSTREAM of that row (object
+// storage rejecting the reference-image upload, on 2026-09-07) produces no rows
+// at all, and no rows reads as a quiet hour. Silence is only innocent when the
+// forge was already quiet: a lane that was starting a generation every couple of
+// minutes and has started nothing since is down, whatever the surviving rows say.
+const STALL_MIN_HOURLY = 3;
+const STALL_AGE_MS = 45 * 60 * 1000;
+
// Rate bands. 3D generation legitimately fails sometimes (hard prompts/images),
// so the healthy bar is not 100%: ok ≥ 85%, degraded 60–85%, down < 60%. The
// 2026-07 image burst (~52% success) reads `down`; a 77% day reads `degraded`.
@@ -114,6 +123,27 @@ export function classifyForgeBuckets(buckets, { minAttempts = MIN_ATTEMPTS } = {
};
}
+/**
+ * Judge whether the forge has stopped STARTING generations. Pure: takes the age
+ * of the newest row and the recent hourly rate, so the "quiet night vs outage"
+ * boundary is unit-testable without a clock or a DB.
+ * @param {{ lastCreatedAgeMs: number|null, hourlyRate: number }} input
+ * @returns {{ stalled: boolean, detail?: string, hint?: string, ageMinutes: number|null }}
+ */
+export function classifyForgeStall({ lastCreatedAgeMs, hourlyRate }) {
+ const ageMinutes = lastCreatedAgeMs == null ? null : Math.round(lastCreatedAgeMs / 60000);
+ if (lastCreatedAgeMs == null || hourlyRate < STALL_MIN_HOURLY || lastCreatedAgeMs < STALL_AGE_MS) {
+ return { stalled: false, ageMinutes };
+ }
+ return {
+ stalled: true,
+ ageMinutes,
+ detail: `no generation has STARTED in ${ageMinutes}m (the forge was averaging ${hourlyRate.toFixed(0)}/hour)`,
+ hint:
+ 'Zero new rows is not a quiet hour at this volume: the request is failing BEFORE forge_creations is written, so the success-rate sensor cannot see it. That is the shape of a shared dependency, not a lane fault. Check object_storage in this same payload first (a rejected bucket credential blocks the reference-image upload every lane needs), then the text→image lane, then POST /api/forge with a one-word prompt and read the error body.',
+ };
+}
+
/**
* Read the last WINDOW_INTERVAL of forge outcomes and classify them. Fail-soft:
* a missing DB, absent table, or query error resolves to `unknown` so a health
@@ -139,7 +169,28 @@ export async function gatherForgeHealth() {
GROUP BY status, backend, path, reason
`
);
+ // One extra bounded read: the newest row and the recent volume, so a total
+ // stop (no rows at all) is distinguishable from a quiet stretch.
+ const [pulse] = /** @type {Array<{ last_at: string|null, n6: number }>} */ (
+ await sql`
+ SELECT max(created_at) AS last_at,
+ count(*)::int AS n6
+ FROM forge_creations
+ WHERE created_at >= now() - '6 hours'::interval
+ `
+ );
+ const lastCreatedAgeMs = pulse?.last_at ? Date.now() - new Date(pulse.last_at).getTime() : null;
+ const stall = classifyForgeStall({ lastCreatedAgeMs, hourlyRate: (Number(pulse?.n6) || 0) / 6 });
const v = classifyForgeBuckets(buckets);
+ if (stall.stalled) {
+ return {
+ ...base,
+ status: 'down',
+ detail: stall.detail || 'no generation has started recently',
+ hint: stall.hint,
+ metrics: { windowHours: 6, stalled: true, lastStartMinutesAgo: stall.ageMinutes, recentStarts: Number(pulse?.n6) || 0 },
+ };
+ }
return {
...base,
status: v.status,
diff --git a/tests/env-s3-cred.test.js b/tests/env-s3-cred.test.js
new file mode 100644
index 000000000..238876143
--- /dev/null
+++ b/tests/env-s3-cred.test.js
@@ -0,0 +1,71 @@
+/**
+ * env.js S3 credential hygiene, unit test.
+ *
+ * The sibling of env-upstash-cred.test.js, for the bucket. A trailing newline or
+ * stray space on S3_SECRET_ACCESS_KEY (the classic dashboard / secret-payload
+ * paste artifact) is invisible and fatal: the AWS SDK signs the request with the
+ * padded secret and R2 answers `SignatureDoesNotMatch`, which reads as "wrong
+ * key" rather than "wrong bytes". That failure is total: the forge cannot park a
+ * reference image, uploads cannot land, and /cdn cannot read an object, so the
+ * getters trim at the source and no signer ever sees the padding.
+ */
+
+import { describe, it, expect, beforeEach, afterEach } from 'vitest';
+import { env } from '../api/_lib/env.js';
+import { objectStorageConfigured } from '../api/_lib/r2.js';
+
+const KEYS = ['S3_ACCESS_KEY_ID', 'S3_SECRET_ACCESS_KEY', 'S3_ENDPOINT', 'S3_BUCKET', 'S3_PUBLIC_DOMAIN'];
+let saved;
+
+beforeEach(() => {
+ saved = Object.fromEntries(KEYS.map((k) => [k, process.env[k]]));
+});
+
+afterEach(() => {
+ for (const k of KEYS) {
+ if (saved[k] === undefined) delete process.env[k];
+ else process.env[k] = saved[k];
+ }
+});
+
+describe('S3 credential getters', () => {
+ it('strips a trailing newline from the secret access key', () => {
+ process.env.S3_SECRET_ACCESS_KEY = 'abc123secret\n';
+ expect(env.S3_SECRET_ACCESS_KEY).toBe('abc123secret');
+ });
+
+ it('strips surrounding whitespace from the access key id', () => {
+ process.env.S3_ACCESS_KEY_ID = ' AKIAEXAMPLE ';
+ expect(env.S3_ACCESS_KEY_ID).toBe('AKIAEXAMPLE');
+ });
+
+ it('leaves a clean credential untouched', () => {
+ process.env.S3_SECRET_ACCESS_KEY = 'abc123secret';
+ expect(env.S3_SECRET_ACCESS_KEY).toBe('abc123secret');
+ });
+
+ it('treats a whitespace-only credential as missing, not as configured', () => {
+ process.env.S3_SECRET_ACCESS_KEY = ' \n';
+ expect(() => env.S3_SECRET_ACCESS_KEY).toThrow(/S3_SECRET_ACCESS_KEY/);
+ });
+});
+
+describe('objectStorageConfigured', () => {
+ it('is false when a credential is present but blank', () => {
+ process.env.S3_ENDPOINT = 'https://acct.r2.cloudflarestorage.com';
+ process.env.S3_BUCKET = 'three-ws';
+ process.env.S3_PUBLIC_DOMAIN = 'https://pub-example.r2.dev';
+ process.env.S3_ACCESS_KEY_ID = 'AKIAEXAMPLE';
+ process.env.S3_SECRET_ACCESS_KEY = '\n';
+ expect(objectStorageConfigured()).toBe(false);
+ });
+
+ it('is true when every var carries a real value', () => {
+ process.env.S3_ENDPOINT = 'https://acct.r2.cloudflarestorage.com';
+ process.env.S3_BUCKET = 'three-ws';
+ process.env.S3_PUBLIC_DOMAIN = 'https://pub-example.r2.dev';
+ process.env.S3_ACCESS_KEY_ID = 'AKIAEXAMPLE';
+ process.env.S3_SECRET_ACCESS_KEY = 'abc123secret';
+ expect(objectStorageConfigured()).toBe(true);
+ });
+});