Skip to content

Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.

14.6 KB
Raw Blame
// Poll-time lane failover — api/_lib/forge-failover.js.
//
// Covers the successor-chain store (fake Redis), the terminal-failure lane
// suggestions (env-driven), and the redispatch dispatcher's guard rails. The
// live network paths (provider submits, health snapshot probes) are exercised
// end-to-end against production; these tests pin the pure contracts the poll
// handler depends on.

import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';
import {
	bindJobSuccessor,
	resolveLiveJob,
	retryBackendSuggestions,
	pickRedispatchLane,
	submitFailoverJob,
	MAX_FAILOVER_HOPS,
} from '../api/_lib/forge-failover.js';
import { decodeJobToken } from '../api/_lib/forge-job-token.js';
import { resolveTier } from '../api/_lib/forge-tiers.js';

// encodeJobToken signs handles with JWT_SECRET (lazily, at call time); give the
// suite a deterministic secret so handle-shape assertions can decode them.
process.env.JWT_SECRET = process.env.JWT_SECRET || 'test-jwt-secret';

// Mock the lane submission boundaries: submitFailoverJob dynamic-imports the
// provider modules, so the mocks intercept exactly the network edge and the
// handle-encoding logic under test stays real. laneHealthSnapshot is mocked so
// pickRedispatchLane's telemetry-failure fallback is drivable without probes.
const { gcpSubmit, replicateSubmit, laneHealth } = vi.hoisted(() => ({
	gcpSubmit: vi.fn(),
	replicateSubmit: vi.fn(),
	laneHealth: vi.fn(),
}));
vi.mock('../api/_providers/gcp.js', () => ({
	createRegenProvider: () => ({ submit: gcpSubmit }),
}));
vi.mock('../api/_providers/replicate.js', () => ({
	createRegenProvider: () => ({ submit: replicateSubmit }),
}));
vi.mock('../api/_lib/forge-lane-health.js', () => ({
	laneHealthSnapshot: (...args) => laneHealth(...args),
}));

function fakeRedis() {
	const store = new Map();
	return {
		store,
		async get(key) {
			return store.get(key) ?? null;
		},
		async set(key, value) {
			store.set(key, value);
		},
	};
}

const LANE_ENV = [
	'NVIDIA_API_KEY',
	'HF_TOKEN',
	'REPLICATE_API_TOKEN',
	'MODEL_TRELLIS_URL',
	'GCP_HUNYUAN3D_URL',
	'GCP_TRIPOSG_URL',
	'GCP_RECONSTRUCTION_KEY',
];
const saved = {};

beforeEach(() => {
	for (const k of LANE_ENV) {
		saved[k] = process.env[k];
		delete process.env[k];
	}
	gcpSubmit.mockReset();
	replicateSubmit.mockReset();
	laneHealth.mockReset();
	laneHealth.mockResolvedValue({ byId: {} });
});
afterEach(() => {
	for (const k of LANE_ENV) {
		if (saved[k] === undefined) delete process.env[k];
		else process.env[k] = saved[k];
	}
});

describe('successor chain', () => {
	it('resolves null when no failover happened', async () => {
		const redis = fakeRedis();
		expect(await resolveLiveJob('job-a', { redis })).toBeNull();
	});

	it('binds and resolves a successor, overwriting on the next hop', async () => {
		const redis = fakeRedis();
		expect(
			await bindJobSuccessor('job-a', { handle: 'job-b', backend: 'trellis_selfhost', hop: 1, attempted: ['nvidia'] }, { redis }),
		).toBe(true);
		let live = await resolveLiveJob('job-a', { redis });
		expect(live).toMatchObject({ handle: 'job-b', backend: 'trellis_selfhost', hop: 1 });

		// Second failover rebinds the ORIGINAL id — one record per job, no chains
		// to walk or corrupt.
		await bindJobSuccessor(
			'job-a',
			{ handle: 'job-c', backend: 'hunyuan3d', hop: 2, attempted: ['nvidia', 'trellis_selfhost'] },
			{ redis },
		);
		live = await resolveLiveJob('job-a', { redis });
		expect(live).toMatchObject({ handle: 'job-c', backend: 'hunyuan3d', hop: 2 });
		expect(live.attempted).toEqual(['nvidia', 'trellis_selfhost']);
	});

	it('fails open without Redis: bind reports false, resolve reports null', async () => {
		expect(await bindJobSuccessor('job-a', { handle: 'job-b', backend: 'x', hop: 1 }, { redis: null })).toBe(false);
		expect(await resolveLiveJob('job-a', { redis: null })).toBeNull();
	});

	it('never reports running for an unbindable successor (bind returns false on write error)', async () => {
		const redis = {
			async get() {
				return null;
			},
			async set() {
				throw new Error('redis down');
			},
		};
		expect(await bindJobSuccessor('job-a', { handle: 'job-b', backend: 'x', hop: 1 }, { redis })).toBe(false);
	});

	it('terminates on a cyclic successor chain (A -> B -> A) within the hop cap', async () => {
		// bindJobSuccessor always rebinds the ORIGINAL id, so a healthy flow never
		// writes a cycle; this pins the defensive loop bound against a corrupt or
		// hand-written record. Without it a poll would spin on redis.get forever.
		const backing = fakeRedis();
		await bindJobSuccessor('job-a', { handle: 'job-b', backend: 'trellis_selfhost', hop: 1 }, { redis: backing });
		await bindJobSuccessor('job-b', { handle: 'job-a', backend: 'hunyuan3d', hop: 2 }, { redis: backing });

		let reads = 0;
		const counting = {
			async get(key) {
				reads++;
				if (reads > 50) throw new Error('resolveLiveJob did not terminate on a cyclic chain');
				return backing.get(key);
			},
			async set(key, value) {
				return backing.set(key, value);
			},
		};
		const live = await resolveLiveJob('job-a', { redis: counting });
		// The chase is bounded at one read past the bind cap and still returns a
		// chaseable successor record rather than hanging or throwing.
		expect(reads).toBeLessThanOrEqual(MAX_FAILOVER_HOPS + 1);
		expect(live).not.toBeNull();
		expect(typeof live.handle).toBe('string');
		expect(['job-a', 'job-b']).toContain(live.handle);
	});
});

describe('retryBackendSuggestions', () => {
	it('returns only configured lanes, free-first, minus attempted', () => {
		process.env.MODEL_TRELLIS_URL = 'https://model-trellis.example';
		process.env.GCP_RECONSTRUCTION_KEY = 'k';
		process.env.HF_TOKEN = 'hf';
		process.env.REPLICATE_API_TOKEN = 'r8';

		const all = retryBackendSuggestions({ hasImage: true, country: 'US' });
		expect(all).toEqual(['trellis_selfhost', 'huggingface', 'trellis']);

		const minusFailed = retryBackendSuggestions({ hasImage: true, attempted: ['trellis_selfhost'], country: 'US' });
		expect(minusFailed).toEqual(['huggingface', 'trellis']);
	});

	it('offers the text-only NVIDIA lane for text jobs but never for photo input', () => {
		process.env.NVIDIA_API_KEY = 'nv';
		expect(retryBackendSuggestions({ hasImage: false, country: 'US' })).toEqual(['nvidia']);
		expect(retryBackendSuggestions({ hasImage: true, country: 'US' })).toEqual([]);
	});

	it('returns [] when nothing is configured — the caller omits the retry affordance', () => {
		expect(retryBackendSuggestions({ hasImage: true, country: 'US' })).toEqual([]);
	});
});

describe('pickRedispatchLane', () => {
	it('returns null with no configured async lane', async () => {
		expect(await pickRedispatchLane({ attempted: [], country: 'US' })).toBeNull();
	});

	it('skips attempted lanes even when configured', async () => {
		process.env.MODEL_TRELLIS_URL = 'https://model-trellis.example';
		process.env.GCP_RECONSTRUCTION_KEY = 'k';
		expect(await pickRedispatchLane({ attempted: ['trellis_selfhost'], country: 'US' })).toBeNull();
	});

	it('skips lanes the health snapshot marks down', async () => {
		process.env.MODEL_TRELLIS_URL = 'https://model-trellis.example';
		process.env.GCP_HUNYUAN3D_URL = 'https://hunyuan.example';
		process.env.GCP_RECONSTRUCTION_KEY = 'k';
		laneHealth.mockResolvedValue({ byId: { trellis_selfhost: { status: 'down' } } });
		expect(await pickRedispatchLane({ attempted: [], country: 'US' })).toBe('hunyuan3d');

		laneHealth.mockResolvedValue({
			byId: { trellis_selfhost: { status: 'down' }, hunyuan3d: { status: 'down' } },
		});
		expect(await pickRedispatchLane({ attempted: [], country: 'US' })).toBeNull();
	});

	it('falls through to the geometry-only TripoSG lane once every textured lane is down', async () => {
		// The 2026-09-18..28 outage: Hunyuan3D and TRELLIS self-host both down,
		// Replicate unconfigured, TripoSG healthy. Every photo job dead-ended.
		process.env.MODEL_TRELLIS_URL = 'https://model-trellis.example';
		process.env.GCP_HUNYUAN3D_URL = 'https://hunyuan.example';
		process.env.GCP_TRIPOSG_URL = 'https://triposg.example';
		process.env.GCP_RECONSTRUCTION_KEY = 'k';
		laneHealth.mockResolvedValue({
			byId: { trellis_selfhost: { status: 'down' }, hunyuan3d: { status: 'down' } },
		});
		expect(await pickRedispatchLane({ attempted: [], country: 'US' })).toBe('triposg');
		expect(await pickRedispatchLane({ attempted: ['hunyuan3d', 'trellis_selfhost'], country: 'US' })).toBe('triposg');
	});

	it('never prefers the untextured TripoSG lane over a live textured one', async () => {
		process.env.GCP_HUNYUAN3D_URL = 'https://hunyuan.example';
		process.env.GCP_TRIPOSG_URL = 'https://triposg.example';
		process.env.REPLICATE_API_TOKEN = 'r8_test';
		process.env.GCP_RECONSTRUCTION_KEY = 'k';
		expect(await pickRedispatchLane({ attempted: [], country: 'US' })).toBe('hunyuan3d');
		// Paid textured TRELLIS still outranks it when configured.
		expect(await pickRedispatchLane({ attempted: ['hunyuan3d'], country: 'US' })).toBe('trellis');
		expect(await pickRedispatchLane({ attempted: ['hunyuan3d', 'trellis'], country: 'US' })).toBe('triposg');
		expect(await pickRedispatchLane({ attempted: ['hunyuan3d', 'trellis', 'triposg'], country: 'US' })).toBeNull();
	});

	it('keeps TripoSG out of the fresh-retry suggestions (its public lane is sketch-only)', () => {
		process.env.GCP_TRIPOSG_URL = 'https://triposg.example';
		process.env.GCP_RECONSTRUCTION_KEY = 'k';
		expect(retryBackendSuggestions({ hasImage: true, country: 'US' })).not.toContain('triposg');
		expect(retryBackendSuggestions({ hasImage: false, country: 'US' })).not.toContain('triposg');
	});

	it('falls back to the first configured candidate when the health snapshot throws', async () => {
		// No telemetry must never mean no failover: a bad pick just fails over
		// again, but a null pick strands the job on a dead lane.
		process.env.MODEL_TRELLIS_URL = 'https://model-trellis.example';
		process.env.GCP_HUNYUAN3D_URL = 'https://hunyuan.example';
		process.env.GCP_RECONSTRUCTION_KEY = 'k';
		laneHealth.mockRejectedValue(new Error('telemetry down'));
		expect(await pickRedispatchLane({ attempted: [], country: 'US' })).toBe('trellis_selfhost');
		// The fallback respects the attempted filter: candidates[0] is the first
		// UNATTEMPTED configured lane, not the first lane in the static order.
		expect(await pickRedispatchLane({ attempted: ['trellis_selfhost'], country: 'US' })).toBe('hunyuan3d');
	});
});

describe('submitFailoverJob guard rails', () => {
	it('refuses a redispatch with no stored reference image', async () => {
		await expect(submitFailoverJob({ backend: 'trellis_selfhost', imageUrl: null })).rejects.toThrow(
			/reference image/,
		);
	});

	it('refuses lanes that cannot be driven from a poll (blocking or text-only)', async () => {
		await expect(
			submitFailoverJob({ backend: 'huggingface', imageUrl: 'https://x/img.png' }),
		).rejects.toThrow(/cannot be redispatched/);
		await expect(submitFailoverJob({ backend: 'nvidia', imageUrl: 'https://x/img.png' })).rejects.toThrow(
			/cannot be redispatched/,
		);
	});
});

describe('submitFailoverJob handle shapes', () => {
	// The handle is what the poll handler routes on. GCP lanes MUST return a
	// signed f1.* job token (provider gcp) and the Replicate lane MUST return the
	// bare prediction id: a shape regression on either side makes clients poll a
	// dead handle forever, so the exact encoding is pinned here.
	const IMG = 'https://cdn.example/ref.png';

	it('trellis_selfhost returns a gcp job token wrapping the worker task id', async () => {
		gcpSubmit.mockResolvedValue({ extJobId: 'gcp-task-1' });
		const out = await submitFailoverJob({ backend: 'trellis_selfhost', imageUrl: IMG, prompt: 'a knight' });

		expect(out.extJobId).toBe('gcp-task-1');
		expect(out.handle).toMatch(/^f1\./);
		expect(decodeJobToken(out.handle)).toEqual({ provider: 'gcp', kind: null, taskId: 'gcp-task-1' });

		// Submitted through the gcp provider in trellis mode with the stored inputs.
		expect(gcpSubmit).toHaveBeenCalledTimes(1);
		expect(gcpSubmit).toHaveBeenCalledWith({
			mode: 'trellis',
			sourceUrl: IMG,
			params: { images: [IMG], prompt: 'a knight' },
		});
		expect(replicateSubmit).not.toHaveBeenCalled();
	});

	it('hunyuan3d returns a gcp job token and carries the tier polycount target', async () => {
		gcpSubmit.mockResolvedValue({ extJobId: 'hy-task-9' });
		const out = await submitFailoverJob({ backend: 'hunyuan3d', imageUrl: IMG, tierId: 'high' });

		expect(out.extJobId).toBe('hy-task-9');
		expect(out.handle).toMatch(/^f1\./);
		expect(decodeJobToken(out.handle)).toEqual({ provider: 'gcp', kind: null, taskId: 'hy-task-9' });

		expect(gcpSubmit).toHaveBeenCalledTimes(1);
		const call = gcpSubmit.mock.calls[0][0];
		// Hunyuan3D speaks the standard /infer task shape, not /reconstruct.
		expect(call.mode).toBe('hunyuan');
		expect(call.sourceUrl).toBe(IMG);
		expect(call.params.images).toEqual([IMG]);
		expect(call.params.prompt).toBeUndefined();
		expect(call.params.target_polycount).toBe(resolveTier('high').polycount);
	});

	it('an unknown tier falls back to the default tier polycount', async () => {
		gcpSubmit.mockResolvedValue({ extJobId: 'hy-task-10' });
		await submitFailoverJob({ backend: 'hunyuan3d', imageUrl: IMG, tierId: 'no-such-tier' });
		expect(gcpSubmit.mock.calls[0][0].params.target_polycount).toBe(resolveTier(undefined).polycount);
	});

	it('triposg returns a gcp job token and submits the photo pipeline, not scribble', async () => {
		gcpSubmit.mockResolvedValue({ extJobId: 'tsg-task-3' });
		const out = await submitFailoverJob({ backend: 'triposg', imageUrl: IMG, prompt: 'a knight', tierId: 'high' });

		expect(out.extJobId).toBe('tsg-task-3');
		expect(decodeJobToken(out.handle)).toEqual({ provider: 'gcp', kind: null, taskId: 'tsg-task-3' });
		expect(gcpSubmit).toHaveBeenCalledWith({
			mode: 'triposg',
			sourceUrl: IMG,
			params: { target_polycount: resolveTier('high').polycount },
		});
		expect(replicateSubmit).not.toHaveBeenCalled();
	});

	it('trellis (Replicate) returns the bare prediction id, never an f1 token', async () => {
		replicateSubmit.mockResolvedValue({ extJobId: 'r8-pred-abc123' });
		const out = await submitFailoverJob({ backend: 'trellis', imageUrl: IMG, prompt: 'a knight' });

		expect(out).toEqual({ extJobId: 'r8-pred-abc123', handle: 'r8-pred-abc123' });
		expect(out.handle.startsWith('f1.')).toBe(false);
		expect(decodeJobToken(out.handle)).toBeNull();

		expect(replicateSubmit).toHaveBeenCalledTimes(1);
		expect(replicateSubmit).toHaveBeenCalledWith({
			mode: 'reconstruct',
			sourceUrl: IMG,
			params: { images: [IMG], prompt: 'a knight' },
		});
		expect(gcpSubmit).not.toHaveBeenCalled();
	});
});

describe('failover budget', () => {
	it('allows exactly one primary + three backup lanes', () => {
		expect(MAX_FAILOVER_HOPS).toBe(3);
	});
});