Skip to content

Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.

fix(okx-health): judge the last hour of real forge submits, not just the static catalog

nirholas2464b759f2ad
diff --git a/api/okx/3d/[service].js b/api/okx/3d/[service].js
index 230965e69..754ae4282 100644
--- a/api/okx/3d/[service].js
+++ b/api/okx/3d/[service].js
@@ -39,6 +39,8 @@ import {
import { priceBatch, isDiscoveryOnlyBatch } from '../../_lib/mcp-batch-price.js';
import { OKX_CATALOG, catalogIndex, catalogEntry, listingDescription } from '../../_lib/okx-catalog.js';
import { headObject, putObject } from '../../_lib/r2.js';
+import { sql } from '../../_lib/db.js';
+import { FORGE_TOOL } from '../../_lib/okx-catalog.js';
import {
PAYMENT_IDENTIFIER,
checkCache,
@@ -83,7 +85,8 @@ async function probe(name, fn) {
const detail = await fn();
return { name, ok: true, latency_ms: Date.now() - started, ...(detail || {}) };
} catch (err) {
- return { name, ok: false, latency_ms: Date.now() - started, error: String(err?.message || err) };
+ const { message, stack, ...detail } = err && typeof err === 'object' ? err : {};
+ return { name, ok: false, latency_ms: Date.now() - started, error: String(err?.message || err), ...detail };
}
}
@@ -107,6 +110,36 @@ async function healthReport() {
if (!tiers || !backends) throw new Error('forge catalog empty');
return { tiers, backends };
}),
+ // What a buyer actually experiences. The `generation` probe proves the
+ // front door is up; it read all-green on 2026-08-25 while every text
+ // submit hung for 95 s+, because a stalled lane looks fine from a static
+ // catalog. Every forge_3d call lands in usage_events with its latency and
+ // status (makeDispatcher records both), so the last hour of real submits
+ // is the honest reading: how long acceptance took, and how many failed.
+ probe('submit-latency', async () => {
+ const rows = await sql`
+ select
+ count(*)::int as samples,
+ count(*) filter (where status <> 'ok')::int as errors,
+ percentile_cont(0.5) within group (order by latency_ms)::int as p50_ms,
+ percentile_cont(0.9) within group (order by latency_ms)::int as p90_ms
+ from usage_events
+ where kind = 'tool_call' and tool = ${FORGE_TOOL}
+ and created_at > now() - interval '60 minutes'
+ `;
+ const r = rows[0] || {};
+ const samples = Number(r.samples) || 0;
+ const errors = Number(r.errors) || 0;
+ const p50 = r.p50_ms == null ? null : Number(r.p50_ms);
+ const p90 = r.p90_ms == null ? null : Number(r.p90_ms);
+ const detail = { window_minutes: 60, samples, errors, p50_ms: p50, p90_ms: p90 };
+ // Three or more real submits in the hour is enough to judge. A median
+ // past 45 s means a ChatGPT-class client never sees the accept; more
+ // failures than successes means the lane is not taking jobs.
+ if (samples >= 3 && p50 != null && p50 > 45_000) throw Object.assign(new Error(`median submit ${p50} ms over the last hour`), detail);
+ if (samples >= 3 && errors * 2 > samples) throw Object.assign(new Error(`${errors} of ${samples} submits failed in the last hour`), detail);
+ return detail;
+ }),
probe('render', async () => {
const res = await fetch(`${BASE}/api/render/avatar-clip`, {
headers: { accept: 'application/json' },
diff --git a/docs/okx-marketplace.md b/docs/okx-marketplace.md
index e59309608..d58a1a95f 100644
--- a/docs/okx-marketplace.md
+++ b/docs/okx-marketplace.md
@@ -41,7 +41,8 @@ No payment, no account, no key.
# Machine-readable index: `services` is the listed line-up, `unlisted` is the back burner
curl https://three.ws/api/okx/3d/catalog
-# Live health of the lanes behind the paid services (real probes, not a static ok)
+# Live health of the lanes behind the paid services (real probes, not a static ok),
+# including the median and p90 of the last hour's real forge_3d submits
curl https://three.ws/api/okx/3d/health
```
diff --git a/tests/api/okx-identity-studio.test.js b/tests/api/okx-identity-studio.test.js
index 843bcf223..7fbd439dd 100644
--- a/tests/api/okx-identity-studio.test.js
+++ b/tests/api/okx-identity-studio.test.js
@@ -41,6 +41,14 @@ vi.mock('../../api/_lib/rate-limit.js', () => ({
clientIp: vi.fn(() => '203.0.113.9'),
}));
+// The submit-latency probe reads usage_events; default to an empty window
+// (nothing to judge), and let a case install rows of its own.
+const usageRows = vi.hoisted(() => ({ rows: [] }));
+vi.mock('../../api/_lib/db.js', () => ({
+ sql: async () => usageRows.rows,
+ isDbUnavailableError: () => false,
+}));
+
vi.mock('../../api/_lib/usage.js', () => ({
recordEvent: vi.fn(),
logger: () => ({ info: () => {}, warn: () => {}, error: () => {} }),
@@ -287,6 +295,7 @@ describe('free lanes over HTTP', () => {
'render',
'retarget',
'storage',
+ 'submit-latency',
]);
const rail = body.subsystems.find((s) => s.name === 'payment-rail');
expect(rail.token).toBe('USD₮0');
@@ -301,6 +310,34 @@ describe('free lanes over HTTP', () => {
expect(JSON.parse(res.body).ok).toBe(false);
});
+ // The static generation probe read all-green on 2026-08-25 while every text
+ // submit hung for 95 s+. The submit-latency probe judges the last hour of
+ // REAL forge_3d calls instead, so a stalled lane shows up here.
+ it('GET /health goes 503 when recent real submits are slow, even with every static probe green', async () => {
+ mountHealthyProbes();
+ usageRows.rows = [{ samples: 5, errors: 0, p50_ms: 96_000, p90_ms: 150_000 }];
+ try {
+ const res = makeRes();
+ await handler(makeReq({ method: 'GET', service: 'health' }), res);
+ expect(res.statusCode).toBe(503);
+ const probe = JSON.parse(res.body).subsystems.find((s) => s.name === 'submit-latency');
+ expect(probe.ok).toBe(false);
+ expect(probe.p50_ms).toBe(96_000);
+ expect(probe.error).toContain('median submit');
+ } finally {
+ usageRows.rows = [];
+ }
+ });
+
+ it('GET /health reports an empty submit window as nothing to judge, not as a failure', async () => {
+ mountHealthyProbes();
+ const res = makeRes();
+ await handler(makeReq({ method: 'GET', service: 'health' }), res);
+ const probe = JSON.parse(res.body).subsystems.find((s) => s.name === 'submit-latency');
+ expect(probe.ok).toBe(true);
+ expect(probe.samples).toBe(0);
+ });
+
// The generation probe used to GET /api/forge bare, which answers 400
// missing_job whether the lane is healthy or not — it only failed on a 5xx.
// It now reads ?catalog and requires a real tier + backend matrix.