Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.
JavaScript 3d-generationai-agentsanimationavatardigital-humangenerative-aiglbgltfimage-to-3dllmmcpmcp-servermodel-context-protocolsolanatext-to-3dthreejsweb-componentswebglwebxrx402
/**
* api/_lib/ops/subsystem-health.js — unit tests.
*
* The gatherer reads each subsystem's in-process degradation state and rolls it
* into one verdict. These pin the status vocabulary (ok/degraded/down/paused/
* unknown), the roll-up (only degraded/down count against overall), and the two
* cases the 2026-07-03 log export surfaced that a reachability probe can't see:
* a half-armed x402 ring and an unprotected world.
*
* probeDb:false everywhere so the suite never needs a live Neon connection — the
* DB ping has its own behavior (down when unreachable) that isn't the point here.
*/
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';
import { gatherSubsystemHealth, classifySniperBeat } from '../api/_lib/ops/subsystem-health.js';
import { cacheSet, cacheDel } from '../api/_lib/cache.js';
// Durable ops_alerts rows the world fallback reads. Default empty so every other
// case behaves as before; the world-fallback cases set it per test.
const dbState = vi.hoisted(() => ({ rows: [], throws: false, lastValues: null }));
vi.mock('../api/_lib/db.js', () => ({
sql: async (strings, ...values) => {
if (dbState.throws) throw new Error('db down');
dbState.lastValues = values;
return dbState.rows;
},
}));
// Env the ring + x402-config checks read. Save/restore so cases don't leak.
const ENV_KEYS = [
'X402_AUTONOMOUS_ENABLED',
'X402_RING_PAUSED',
'X402_EXTERNAL_ENABLED',
'X402_CHARITY_AUDIT_BPS',
'X402_SELF_FACILITATOR_ENABLED',
'X402_FACILITATOR_URL_SOLANA',
'X402_PAY_TO_SOLANA',
'X402_PAY_TO',
'X402_PAY_TO_BASE',
'X402_FEE_PAYER_SOLANA',
'THREE_TREASURY_WALLET',
'THREE_REWARDS_WALLET',
'THREE_QUOTE_SECRET',
'NODE_ENV',
'UPSTASH_CACHE_REST_URL',
'UPSTASH_CACHE_REST_TOKEN',
'UPSTASH_REDIS_REST_URL',
'UPSTASH_REDIS_REST_TOKEN',
];
let saved;
function sub(health, name) {
return health.subsystems.find((s) => s.name === name);
}
beforeEach(() => {
saved = {};
for (const k of ENV_KEYS) {
saved[k] = process.env[k];
delete process.env[k];
}
});
afterEach(() => {
for (const k of ENV_KEYS) {
if (saved[k] === undefined) delete process.env[k];
else process.env[k] = saved[k];
}
});
describe('gatherSubsystemHealth', () => {
it('reports the x402 ring as PAUSED when the loop is explicitly disabled', async () => {
process.env.X402_AUTONOMOUS_ENABLED = 'false';
const health = await gatherSubsystemHealth({ probeDb: false });
const ring = sub(health, 'x402_ring');
expect(ring.status).toBe('paused');
// A paused ring is a chosen state, not a fault — must not drag overall down.
expect(health.status).not.toBe('down');
});
it('reports the x402 ring as PAUSED via the dedicated X402_RING_PAUSED switch', async () => {
delete process.env.X402_AUTONOMOUS_ENABLED;
process.env.X402_RING_PAUSED = 'true';
const health = await gatherSubsystemHealth({ probeDb: false });
const ring = sub(health, 'x402_ring');
expect(ring.status).toBe('paused');
expect(ring.detail).toMatch(/X402_RING_PAUSED/);
});
it('reports the x402 ring as DEGRADED when enabled but guards are unset (half-armed)', async () => {
// Enabled (not 'false') with the closed-loop guards unset — the exact
// half-armed config the production log export showed.
delete process.env.X402_AUTONOMOUS_ENABLED;
delete process.env.X402_CHARITY_AUDIT_BPS;
process.env.X402_SELF_FACILITATOR_ENABLED = 'false';
const health = await gatherSubsystemHealth({ probeDb: false });
const ring = sub(health, 'x402_ring');
expect(ring.status).toBe('degraded');
expect(ring.detail).toMatch(/half-armed/);
expect(ring.hint).toBeTruthy();
expect(health.status).toBe('degraded');
expect(health.degraded).toContain('x402_ring');
});
it('reports the x402 ring as OK when all closed-loop guards are satisfied', async () => {
delete process.env.X402_AUTONOMOUS_ENABLED;
process.env.X402_EXTERNAL_ENABLED = 'false';
process.env.X402_CHARITY_AUDIT_BPS = '0';
process.env.X402_SELF_FACILITATOR_ENABLED = 'true';
// Leave the facilitator URL unset so it defaults to self.
const health = await gatherSubsystemHealth({ probeDb: false });
expect(sub(health, 'x402_ring').status).toBe('ok');
});
it('flags x402 payment config as DEGRADED when Solana pay-to is set without a fee payer', async () => {
process.env.X402_PAY_TO_SOLANA = 'THREEsynthetic1111111111111111111111111111111';
delete process.env.X402_FEE_PAYER_SOLANA;
const health = await gatherSubsystemHealth({ probeDb: false });
const cfg = sub(health, 'x402_config');
expect(cfg.status).toBe('degraded');
expect(cfg.detail).toMatch(/fee.?payer/i);
});
it('flags the $THREE rail as DOWN in production when its receivers are unset', async () => {
// The exact production state on 2026-09-20: none of the three vars set, so
// every $THREE-priced purchase answered 503 while every reachability probe
// stayed green. This check is the signature that condition never had.
process.env.NODE_ENV = 'production';
const health = await gatherSubsystemHealth({ probeDb: false });
const rail = sub(health, 'three_token_rail');
expect(rail.status).toBe('down');
expect(rail.missing).toEqual([
'THREE_TREASURY_WALLET',
'THREE_REWARDS_WALLET',
'THREE_QUOTE_SECRET',
]);
expect(health.degraded).toContain('three_token_rail');
});
it('names only the $THREE vars that are actually missing', async () => {
process.env.NODE_ENV = 'production';
process.env.THREE_TREASURY_WALLET = 'THREEsynthetic1111111111111111111111Treasury';
process.env.THREE_QUOTE_SECRET = 'test-quote-signing-key';
const health = await gatherSubsystemHealth({ probeDb: false });
const rail = sub(health, 'three_token_rail');
expect(rail.status).toBe('down');
expect(rail.missing).toEqual(['THREE_REWARDS_WALLET']);
});
it('reports the $THREE rail as OK once all three are configured', async () => {
process.env.NODE_ENV = 'production';
process.env.THREE_TREASURY_WALLET = 'THREEsynthetic1111111111111111111111Treasury';
process.env.THREE_REWARDS_WALLET = 'THREEsynthetic11111111111111111111111Rewards';
process.env.THREE_QUOTE_SECRET = 'test-quote-signing-key';
const health = await gatherSubsystemHealth({ probeDb: false });
expect(sub(health, 'three_token_rail').status).toBe('ok');
});
it('stays neutral outside production, where the $THREE dev fallbacks apply', async () => {
process.env.NODE_ENV = 'test';
const health = await gatherSubsystemHealth({ probeDb: false });
const rail = sub(health, 'three_token_rail');
expect(rail.status).toBe('unknown');
expect(health.degraded).not.toContain('three_token_rail');
});
it('reports world as UNKNOWN when no world-health report has been parked', async () => {
const health = await gatherSubsystemHealth({ probeDb: false });
expect(sub(health, 'world').status).toBe('unknown');
});
it('reports world as DEGRADED (unprotected) from a parked world-health outcome', async () => {
await cacheSet(
'world:health',
{ status: 'degraded', protected: false, problems: ['unprotected'], missingCount: 0, checkedAt: Date.now() },
3600,
);
const health = await gatherSubsystemHealth({ probeDb: false });
const world = sub(health, 'world');
expect(world.status).toBe('degraded');
expect(world.detail).toMatch(/UNPROTECTED/);
expect(world.hint).toMatch(/ADMIN_CODE/);
});
describe('world durable fallback (order 453)', () => {
afterEach(() => {
dbState.rows = [];
dbState.throws = false;
dbState.lastValues = null;
});
it('reports DOWN from a fresh ops_alerts row when the cache entry is missing', async () => {
await cacheDel('world:health');
dbState.rows = [
{
detail: 'https://world.three.ws/status returned HTTP 404',
last_seen: new Date(Date.now() - 4 * 60 * 1000).toISOString(),
},
];
const health = await gatherSubsystemHealth({ probeDb: false });
const world = sub(health, 'world');
expect(world.status).toBe('down');
expect(world.detail).toMatch(/HTTP 404/);
expect(health.degraded).toContain('world');
});
it('reports DOWN from a fresh alert when the cache entry is stale', async () => {
await cacheSet(
'world:health',
{ status: 'ok', protected: true, problems: [], checkedAt: Date.now() - 3 * 60 * 60 * 1000 },
3600,
);
dbState.rows = [{ detail: 'unreachable - timeout', last_seen: new Date().toISOString() }];
const health = await gatherSubsystemHealth({ probeDb: false });
expect(sub(health, 'world').status).toBe('down');
await cacheDel('world:health');
});
it('stays UNKNOWN when the only alert row is older than the cron window', async () => {
await cacheDel('world:health');
dbState.rows = [
{ detail: 'old', last_seen: new Date(Date.now() - 3 * 60 * 60 * 1000).toISOString() },
];
const health = await gatherSubsystemHealth({ probeDb: false });
expect(sub(health, 'world').status).toBe('unknown');
});
it('stays UNKNOWN when there is no alert row at all', async () => {
await cacheDel('world:health');
dbState.rows = [];
const health = await gatherSubsystemHealth({ probeDb: false });
expect(sub(health, 'world').status).toBe('unknown');
});
it('degrades to cache-only UNKNOWN when the database read fails', async () => {
await cacheDel('world:health');
dbState.throws = true;
const health = await gatherSubsystemHealth({ probeDb: false });
expect(sub(health, 'world').status).toBe('unknown');
});
it('queries only the two down signatures the cron writes (sha256 prefix of the literal)', async () => {
await cacheDel('world:health');
await gatherSubsystemHealth({ probeDb: false });
const { createHash } = await import('node:crypto');
const h = (t) => createHash('sha256').update(t).digest('hex').slice(0, 16);
expect(dbState.lastValues).toEqual([h('world-health:status-http'), h('world-health:status-unreachable')]);
});
});
it('reports world as DOWN from a parked world-health outcome', async () => {
await cacheSet(
'world:health',
{ status: 'down', protected: null, problems: ['HTTP 503'], missingCount: 0, checkedAt: Date.now() },
3600,
);
const health = await gatherSubsystemHealth({ probeDb: false });
const world = sub(health, 'world');
expect(world.status).toBe('down');
expect(world.detail).toMatch(/503/);
expect(health.status).toBe('down');
expect(health.degraded).toContain('world');
await cacheDel('world:health');
});
it('reports cache as OK on the in-memory backend when no Redis is configured', async () => {
const health = await gatherSubsystemHealth({ probeDb: false });
const cache = sub(health, 'cache');
expect(cache.status).toBe('ok');
expect(cache.backend).toBe('memory');
});
it('reports Helius as OK (public RPC) when no Helius key is configured', async () => {
const health = await gatherSubsystemHealth({ probeDb: false });
expect(sub(health, 'helius').status).toBe('ok');
});
it('skips the DB ping when probeDb is false (database → unknown, not down)', async () => {
const health = await gatherSubsystemHealth({ probeDb: false });
expect(sub(health, 'database').status).toBe('unknown');
});
it('skips the sniper heartbeat probe when probeDb is false (sniper → unknown)', async () => {
const health = await gatherSubsystemHealth({ probeDb: false });
expect(sub(health, 'sniper').status).toBe('unknown');
});
it('rolls the overall verdict to the worst subsystem and lists degraded names', async () => {
process.env.X402_PAY_TO_SOLANA = 'THREEsynthetic1111111111111111111111111111111';
delete process.env.X402_FEE_PAYER_SOLANA; // → x402_config degraded
const health = await gatherSubsystemHealth({ probeDb: false });
expect(health.status).toBe('degraded');
expect(health.degraded).toContain('x402_config');
// Counts is a tally of every status bucket.
expect(Object.values(health.counts).reduce((a, b) => a + b, 0)).toBe(health.subsystems.length);
});
});
// The sniper-worker heartbeat classifier — pins the exact failure that went
// unpaged on 2026-07-03: heartbeat frozen for 36h while /api/oracle/stats
// flat-lined at scored_24h=0. A stale beat must classify as DOWN (which the
// uptime cron escalates to Telegram), not blend into "ok".
describe('classifySniperBeat', () => {
const NOW = 1_800_000_000_000;
const beat = (ageMs, meta = {}, mode = 'live') => ({
mode,
last_beat_at: new Date(NOW - ageMs).toISOString(),
meta,
});
const liveMeta = { feedConnected: true, lastEventAgeMs: 2_000, feedWatchdogMs: 180_000, strategies: 6 };
it('reports UNKNOWN when no heartbeat row exists (fresh deploy, never started)', () => {
expect(classifySniperBeat(null, NOW).status).toBe('unknown');
});
it('reports OK for a fresh beat with a live feed', () => {
const s = classifySniperBeat(beat(30_000, liveMeta), NOW);
expect(s.status).toBe('ok');
expect(s.detail).toContain('mode=live');
});
it('reports DOWN with a redeploy hint once the heartbeat is stale (the 2026-07-03 outage)', () => {
const s = classifySniperBeat(beat(36 * 3_600_000, liveMeta), NOW);
expect(s.status).toBe('down');
expect(s.detail).toContain('2160 min');
expect(s.hint).toContain('deploy:sniper');
});
it('reports DEGRADED between fresh and dead (mid-restart window)', () => {
expect(classifySniperBeat(beat(3 * 60_000, liveMeta), NOW).status).toBe('degraded');
});
it('reports DEGRADED when alive but the feed is disconnected', () => {
const s = classifySniperBeat(beat(30_000, { ...liveMeta, feedConnected: false }), NOW);
expect(s.status).toBe('degraded');
expect(s.detail).toContain('feed');
});
it('reports DEGRADED when alive but the feed has been silent past its watchdog window', () => {
const s = classifySniperBeat(beat(30_000, { ...liveMeta, lastEventAgeMs: 240_000 }), NOW);
expect(s.status).toBe('degraded');
});
it('treats a row with no last_beat_at as DOWN, not ok', () => {
expect(classifySniperBeat({ mode: 'live', last_beat_at: null, meta: {} }, NOW).status).toBe('down');
});
// Solvency: the 2026-07-29 outage shape. Fresh heartbeat, live feed, strategies
// armed, and every wallet too poor to place a buy. This block used to report OK
// for ten days while the fleet booked 1,000+ failed entries and closed nothing.
it('reports DOWN when the process is healthy but no wallet can fund a trade', () => {
const s = classifySniperBeat(
beat(30_000, { ...liveMeta, solvency: { state: 'starved', agents: 2, starved: 2, tradeable: 0, deficitSol: 0.13, masterSol: 0.0018, masterCanCover: false } }),
NOW,
);
expect(s.status).toBe('down');
expect(s.detail).toContain('out of trading capital');
// The master held dust, so the hint must name the human action.
expect(s.hint).toContain('move SOL');
});
it('reports DEGRADED when only part of the fleet can still trade', () => {
const s = classifySniperBeat(
beat(30_000, { ...liveMeta, solvency: { state: 'degraded', agents: 3, starved: 1, tradeable: 2, deficitSol: 0.05, masterSol: 5, masterCanCover: true } }),
NOW,
);
expect(s.status).toBe('degraded');
expect(s.hint).toContain('auto-funder');
});
it('ranks starvation above a silent feed: the more actionable diagnosis wins', () => {
const s = classifySniperBeat(
beat(30_000, { ...liveMeta, lastEventAgeMs: 240_000, solvency: { state: 'starved', agents: 1, starved: 1, tradeable: 0, deficitSol: 0.06, masterSol: 0, masterCanCover: false } }),
NOW,
);
expect(s.status).toBe('down');
expect(s.detail).toContain('out of trading capital');
});
it('still reports OK when the fleet is funded', () => {
const s = classifySniperBeat(
beat(30_000, { ...liveMeta, solvency: { state: 'funded', agents: 2, starved: 0, tradeable: 2, deficitSol: 0 } }),
NOW,
);
expect(s.status).toBe('ok');
});
it('does not downgrade a worker build that reports no solvency at all', () => {
// Older revisions publish no solvency key. Absence is not insolvency, or
// every pre-upgrade deploy would page as broken.
expect(classifySniperBeat(beat(30_000, liveMeta), NOW).status).toBe('ok');
expect(classifySniperBeat(beat(30_000, { ...liveMeta, solvency: null }), NOW).status).toBe('ok');
expect(classifySniperBeat(beat(30_000, { ...liveMeta, solvency: { state: 'unknown' } }), NOW).status).toBe('ok');
});
});