Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.
JavaScript 3d-generationai-agentsanimationavatardigital-humangenerative-aiglbgltfimage-to-3dllmmcpmcp-servermodel-context-protocolsolanatext-to-3dthreejsweb-componentswebglwebxrx402
// Regression tests for the rate limiter's behaviour when Redis cannot answer.
//
// Incident (2026-07-09): the shared Upstash store hit its plan-wide command
// allowance ("max requests limit exceeded. Limit: 500000"). That is not a blip —
// the counter resets only on the plan's billing boundary — so for the rest of the
// period every Redis command was rejected. Two consequences fell out of the old
// design, and each is pinned by a test below:
//
// 1. `critical` money buckets FAILED CLOSED, taking every checkout, withdrawal,
// mint and trade down for the length of the outage. They must now degrade
// onto a durable Postgres counter instead.
// 2. Every rate-limited request paid a full failing Redis round-trip, because
// rate-limit.js — unlike cache.js — had no circuit breaker.
//
// The Postgres path is exercised against a stubbed `sql` tag: these are unit
// tests, and the live-database proof (exact enforcement + durability across a
// second module instance) is covered separately by the deploy-time check.
import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
const QUOTA_ERROR = 'ERR max requests limit exceeded. Limit: 500000, Usage: 500000';
// In-memory stand-in for the rate_limit_counters table. Mirrors the migration's
// (bucket, window_start) primary key and the INSERT … ON CONFLICT DO UPDATE
// semantics: one atomic increment returning the post-increment count.
function makeSqlStub() {
const rows = new Map();
const calls = { inserts: 0, deletes: 0, deleteCutoffs: [] };
const sql = (strings, ...values) => {
const text = strings.join('?');
if (/DELETE FROM rate_limit_counters/i.test(text)) {
calls.deletes++;
calls.deleteCutoffs.push(values[0]);
return Promise.resolve([]);
}
if (/INSERT INTO rate_limit_counters/i.test(text)) {
calls.inserts++;
const [bucket, windowStart] = values;
const key = `${bucket}\u0000${windowStart}`;
const hits = (rows.get(key) || 0) + 1;
rows.set(key, hits);
return Promise.resolve([{ hits }]);
}
return Promise.resolve([]);
};
return { sql, calls, rows };
}
async function loadRateLimit({ redis, sqlStub, production = true }) {
vi.resetModules();
vi.doMock('../api/_lib/db.js', () => ({ sql: sqlStub.sql }));
vi.doMock('../api/_lib/redis.js', () => ({
getRedis: () => redis,
isRedisAuthError: () => false,
}));
vi.doMock('../api/_lib/env.js', () => ({
env: {
NODE_ENV: production ? 'production' : 'test',
VERCEL_ENV: production ? 'production' : 'development',
DATABASE_URL: 'postgres://stub/db',
CACHE_REDIS_CMD_TIMEOUT_MS: 3000,
},
}));
return import('../api/_lib/rate-limit.js');
}
// A fake @upstash/ratelimit-shaped client whose commands always reject the way an
// over-quota Upstash store does.
function overQuotaRedis(counter) {
return {
// The Ratelimit constructor only stores this object; our stub throws when
// @upstash/ratelimit issues its script/eval command through it.
eval: () => {
counter.attempts++;
return Promise.reject(new Error(QUOTA_ERROR));
},
evalsha: () => {
counter.attempts++;
return Promise.reject(new Error(QUOTA_ERROR));
},
scriptLoad: () => Promise.reject(new Error(QUOTA_ERROR)),
};
}
const SHEBANG_ERROR = 'Command failed: ERR Unexpected flag in script shebang: allow-key-locking';
function shebangRedis() {
const reject = () => Promise.reject(new Error(SHEBANG_ERROR));
return { eval: reject, evalsha: reject, scriptLoad: reject };
}
beforeEach(() => {
vi.stubEnv('VITEST', '1');
});
afterEach(() => {
vi.unstubAllEnvs();
vi.doUnmock('../api/_lib/db.js');
vi.doUnmock('../api/_lib/redis.js');
vi.doUnmock('../api/_lib/env.js');
});
describe('isRedisQuotaError', () => {
it('recognises the Upstash over-quota rejection', async () => {
const { isRedisQuotaError } = await loadRateLimit({ redis: null, sqlStub: makeSqlStub() });
expect(isRedisQuotaError(new Error(QUOTA_ERROR))).toBe(true);
});
it('does not mistake an ordinary timeout for an exhausted allowance', async () => {
const { isRedisQuotaError } = await loadRateLimit({ redis: null, sqlStub: makeSqlStub() });
expect(isRedisQuotaError(new Error('operation aborted due to timeout'))).toBe(false);
expect(isRedisQuotaError(undefined)).toBe(false);
});
});
describe('isRedisShebangIncompatibilityError', () => {
it('recognises the OSS Redis rejection of the allow-key-locking script flag', async () => {
const { isRedisShebangIncompatibilityError } = await loadRateLimit({ redis: null, sqlStub: makeSqlStub() });
expect(isRedisShebangIncompatibilityError(new Error(SHEBANG_ERROR))).toBe(true);
});
it('does not mistake a timeout or an exhausted allowance for it', async () => {
const { isRedisShebangIncompatibilityError } = await loadRateLimit({ redis: null, sqlStub: makeSqlStub() });
expect(isRedisShebangIncompatibilityError(new Error('operation aborted due to timeout'))).toBe(false);
expect(isRedisShebangIncompatibilityError(new Error(QUOTA_ERROR))).toBe(false);
expect(isRedisShebangIncompatibilityError(undefined)).toBe(false);
});
it('still decides requests and reports the incompatibility through rateLimiterHealth', async () => {
const stub = makeSqlStub();
const { limits, rateLimiterHealth } = await loadRateLimit({ redis: shebangRedis(), sqlStub: stub });
const open = await limits.cohortsIp('203.0.113.9');
expect(open.success).toBe(true);
expect(open.reason).toBe('rate_limiter_degraded');
const durable = await limits.withdrawalPerUser('u');
expect(durable.reason).toBe('rate_limiter_degraded_postgres');
const h = rateLimiterHealth();
expect(h.scriptIncompatible).toBe(true);
expect(h.quotaExhausted).toBe(false);
});
});
describe('durable Postgres fallback (no Redis configured)', () => {
it('a critical money bucket enforces its ceiling instead of failing closed', async () => {
const stub = makeSqlStub();
const { limits } = await loadRateLimit({ redis: null, sqlStub: stub });
// withdrawalPerUser: 5 per day, critical, not degradeToMemory — the exact
// shape that used to answer "denied, rate_limiter_unavailable" on request #1.
const seen = [];
for (let i = 0; i < 7; i++) seen.push(await limits.withdrawalPerUser('user-1'));
expect(seen[0].success).toBe(true);
expect(seen[0].reason).toBe('rate_limiter_degraded_postgres');
expect(seen.filter((r) => r.success)).toHaveLength(5);
expect(seen[5].success).toBe(false);
expect(seen[6].success).toBe(false);
expect(stub.calls.inserts).toBe(7);
});
it('counts each identity in its own bucket', async () => {
const stub = makeSqlStub();
const { limits } = await loadRateLimit({ redis: null, sqlStub: stub });
for (let i = 0; i < 5; i++) await limits.withdrawalPerUser('user-a');
const other = await limits.withdrawalPerUser('user-b');
expect(other.success).toBe(true);
expect(other.remaining).toBe(4);
});
it('namespaces buckets so a spent limiter cannot starve an unrelated one', async () => {
const stub = makeSqlStub();
const { limits } = await loadRateLimit({ redis: null, sqlStub: stub });
for (let i = 0; i < 6; i++) await limits.withdrawalPerUser('same-id');
// voiceClone: 3/day, also critical — same id, different bucket.
const voice = await limits.voiceClone('same-id');
expect(voice.success).toBe(true);
});
it('prunes on the widest window any bucket can use, not the widest this process built', async () => {
// rate_limit_counters is shared by every instance, and the prune is a
// blind DELETE. It used to size its retention floor from the widest window
// a limiter had been CONSTRUCTED with in the current process, which starts
// at one minute. An instance whose first durable bucket was short-windowed
// therefore deleted every other instance's live daily row, silently
// resetting the 5/day and 3/day spend caps for everyone.
const stub = makeSqlStub();
const { limits } = await loadRateLimit({ redis: null, sqlStub: stub });
// inscribeIp: critical (so it lands on Postgres) with a 10-minute window.
await limits.inscribeIp('203.0.113.9');
// The prune reads its own clock partway through that call, so the reference
// point has to be taken AFTER it. Measuring from before the call subtracts
// the elapsed milliseconds from the window and fails on any machine slow
// enough to notice.
const after = Date.now();
expect(stub.calls.deletes).toBe(1);
expect(after - stub.calls.deleteCutoffs[0]).toBeGreaterThanOrEqual(2 * 86_400_000);
});
it('cheap non-critical buckets never touch Postgres', async () => {
const stub = makeSqlStub();
const { limits } = await loadRateLimit({ redis: null, sqlStub: stub });
for (let i = 0; i < 20; i++) await limits.imgProxyIp('203.0.113.1');
expect(stub.calls.inserts).toBe(0);
});
it('the hottest read guards are local — zero Postgres, zero Redis', async () => {
const stub = makeSqlStub();
const { limits } = await loadRateLimit({ redis: null, sqlStub: stub });
for (let i = 0; i < 30; i++) {
await limits.authedReadIp('203.0.113.2');
await limits.mcpIp('203.0.113.2');
await limits.walletRead('user-x');
}
expect(stub.calls.inserts).toBe(0);
});
it('a Postgres outage on top of a Redis outage still denies a money bucket', async () => {
const stub = makeSqlStub();
stub.sql = () => Promise.reject(new Error('neon unreachable'));
const { limits } = await loadRateLimit({ redis: null, sqlStub: stub });
const r = await limits.withdrawalPerUser('user-1');
expect(r.success).toBe(false);
expect(r.reason).toBe('rate_limiter_unavailable');
});
it('an auth bucket stays usable when both Redis and Postgres are down', async () => {
// degradeToMemory: locking every user out of login is worse than a weaker,
// per-instance brute-force cap.
const stub = makeSqlStub();
stub.sql = () => Promise.reject(new Error('neon unreachable'));
const { limits } = await loadRateLimit({ redis: null, sqlStub: stub });
const r = await limits.authIp('203.0.113.9');
expect(r.success).toBe(true);
});
});
describe('circuit breaker in front of Redis', () => {
it('stops hammering an over-quota store and serves from the durable fallback', async () => {
const counter = { attempts: 0 };
const stub = makeSqlStub();
const { limits } = await loadRateLimit({ redis: overQuotaRedis(counter), sqlStub: stub });
// Enough calls that, without a breaker, every one would hit Redis.
const results = [];
for (let i = 0; i < 25; i++) results.push(await limits.withdrawalPerUser(`user-${i}`));
// The breaker opens after 5 consecutive failures, so Redis is attempted a
// bounded number of times — not once per request.
expect(counter.attempts).toBeLessThanOrEqual(6);
expect(counter.attempts).toBeGreaterThan(0);
// And the requests were still decided — durably, in Postgres.
expect(results.every((r) => r.reason === 'rate_limiter_degraded_postgres')).toBe(true);
expect(results.every((r) => r.success)).toBe(true);
expect(stub.calls.inserts).toBe(25);
});
it('reports the exhausted allowance through rateLimiterHealth', async () => {
const counter = { attempts: 0 };
const stub = makeSqlStub();
const { limits, rateLimiterHealth } = await loadRateLimit({
redis: overQuotaRedis(counter),
sqlStub: stub,
});
for (let i = 0; i < 6; i++) await limits.withdrawalPerUser('u');
const h = rateLimiterHealth();
expect(h.configured).toBe(true);
expect(h.quotaExhausted).toBe(true);
expect(h.durableFallback).toBe(true);
expect(h.circuitOpen).toBe(true);
});
it('a non-critical bucket fails OPEN when Redis is blind', async () => {
// A limiter outage must never take down a read endpoint.
const counter = { attempts: 0 };
const stub = makeSqlStub();
const { limits } = await loadRateLimit({ redis: overQuotaRedis(counter), sqlStub: stub });
const r = await limits.cohortsIp('203.0.113.4');
expect(r.success).toBe(true);
expect(r.reason).toBe('rate_limiter_degraded');
expect(stub.calls.inserts).toBe(0);
});
});
describe('bucket declaration invariants', () => {
// `local` means per-instance memory and nothing else — never a Redis or
// Postgres command. That is right for a flood guard and catastrophic for a
// spend control: across a Cloud Run fan-out it bounds one container, not the
// money. Reducing Redis burn is a standing pressure on this file, so pin the
// rule mechanically rather than trusting review.
it('no `critical` bucket is declared `local`', async () => {
const fs = await import('node:fs');
const url = await import('node:url');
const src = fs.readFileSync(
url.fileURLToPath(new URL('../api/_lib/rate-limit.js', import.meta.url)),
'utf8',
);
const offenders = [];
for (const m of src.matchAll(/getLimiter\(\s*'([^']+)'\s*,\s*\{([^}]*)\}/g)) {
const [, name, opts] = m;
if (/local:\s*true/.test(opts) && /critical:\s*true/.test(opts)) offenders.push(name);
}
expect(offenders).toEqual([]);
});
});