7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.
TypeScript
fix(moderation): enforce monthly caps and mark a served moderation as key evidence (#1625)
tashdroidb06de14b6a0d
diff --git a/server/src/__tests__/routes/moderation.test.ts b/server/src/__tests__/routes/moderation.test.tsindex 0b0c7694..4d176c20 100644--- a/server/src/__tests__/routes/moderation.test.ts+++ b/server/src/__tests__/routes/moderation.test.ts@@ -4,6 +4,7 @@ import { createApp } from '../../app.js';import { initDb, getDb, getUnifiedApiKey } from '../../db/index.js';import { encrypt } from '../../lib/crypto.js';import { listModerationModels, resolveModel } from '../../services/moderation.js';+import { clearLastHealthyStampState } from '../../services/health.js';// POST /v1/moderations (#…): OpenAI-compatible content-moderation shim that// routes to the first enabled OpenAI/OpenRouter/NVIDIA key and fails over@@ -186,4 +187,39 @@ describe('POST /v1/moderations route', () => {expect(status).toBe(503);expect(body.error.message).toContain('No moderation providers available');});++ it('a served moderation marks the key as working (#1493)', async () => {+ clearLastHealthyStampState();+ const key = insertKey('openai');+ getDb().prepare("UPDATE api_keys SET status = 'unknown' WHERE id = ?").run(key);+ mockModeration([{ platform: 'openai', response: { results: [{ flagged: false }] } }]);+ const { status } = await postModeration({ input: 'fine' }, getUnifiedApiKey());+ expect(status).toBe(200);+ expect(getDb().prepare('SELECT status, last_healthy_at FROM api_keys WHERE id = ?').get(key))+ .toMatchObject({ status: 'healthy', last_healthy_at: expect.any(String) });+ });++ it('a key at its monthly cap is not used for moderation; the reply is 429 with Retry-After (#1158)', async () => {+ const key = insertKey('openai');+ getDb().prepare('UPDATE api_keys SET monthly_request_cap = 1 WHERE id = ?').run(key);+ const month = new Date().toISOString().slice(0, 7);+ getDb().prepare(+ 'INSERT INTO key_monthly_usage (key_id, month, requests, tokens) VALUES (?, ?, 1, 0)',+ ).run(key, month);+ const calls = mockModeration([{ platform: 'openai', response: { results: [{ flagged: false }] } }]);+ const server = app.listen(0, '127.0.0.1');+ if (!server.listening) await new Promise<void>(r => server.once('listening', () => r()));+ const addr = server.address() as { port: number };+ const res = await fetch(`http://127.0.0.1:${addr.port}/v1/moderations`, {+ method: 'POST',+ headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${getUnifiedApiKey()}` },+ body: JSON.stringify({ input: 'x' }),+ });+ server.close();+ expect(res.status).toBe(429);+ expect(await res.json()).toMatchObject({ error: { type: 'rate_limit_error' } });+ expect(Number(res.headers.get('retry-after'))).toBeGreaterThan(0);+ // The capped key must never be called.+ expect(calls).toEqual([]);+ });});diff --git a/server/src/routes/moderation.ts b/server/src/routes/moderation.tsindex 059d9b3a..956ace7b 100644--- a/server/src/routes/moderation.ts+++ b/server/src/routes/moderation.ts@@ -58,6 +58,11 @@ moderationRouter.post('/moderations', async (req: Request, res: Response) => {});} catch (err: any) {const status = err instanceof ModerationError ? err.status : 502;+ // #1158: a monthly-cap exhaustion carries the seconds until the reset so a+ // client backs off to the right moment instead of hammering a bare 429.+ if (err instanceof ModerationError && err.retryAfterSec !== undefined) {+ res.setHeader('Retry-After', String(Math.max(1, Math.ceil(err.retryAfterSec))));+ }res.status(status >= 400 && status < 600 ? status : 502).json({error: {message: err?.message ?? 'moderation request failed',diff --git a/server/src/services/moderation.ts b/server/src/services/moderation.tsindex edafc64b..400a0f71 100644--- a/server/src/services/moderation.ts+++ b/server/src/services/moderation.ts@@ -15,6 +15,8 @@ import { getDb } from '../db/index.js';import { getClientContext } from '../lib/client-context.js';import { decrypt } from '../lib/crypto.js';import { proxyFetch } from '../lib/proxy.js';+import { checkMonthlyBudget, secondsUntilNextMonth } from './key-budget.js';+import { markKeyHealthyFromRequest } from './health.js';/* ------------------------------------------------------------------ *//* Types */@@ -38,9 +40,12 @@ export interface ModerationResult {export class ModerationError extends Error {status: number;- constructor(message: string, status: number) {+ /** Seconds the caller should wait before retrying (429 exhaustion). */+ retryAfterSec?: number;+ constructor(message: string, status: number, retryAfterSec?: number) {super(message);this.status = status;+ this.retryAfterSec = retryAfterSec;}}@@ -194,6 +199,11 @@ export async function runModerations(}let lastError: ModerationError | null = null;+ // #1158: a moderation call spends real provider quota, so a key at its+ // monthly cap must not serve one. Every other inference surface checks the+ // caps before dispatch (router, embeddings, media, search); moderation was+ // the only one that read api_keys directly and skipped the gate.+ let budgetBlocked = false;for (const row of chain) {let apiKey: string;try {@@ -201,10 +211,18 @@ export async function runModerations(} catch {continue;}+ if (!checkMonthlyBudget(row.keyId, 0).allowed) {+ budgetBlocked = true;+ continue;+ }const started = Date.now();try {const json = await callProvider(row, apiKey, inputs);logModerationRequest(row, 'success', Date.now() - started, null);+ // A served moderation is the evidence the key works, the same as a+ // served chat request (#1493): it stamps last_healthy_at and lifts an+ // 'unknown' or 'error' key.+ markKeyHealthyFromRequest(row.keyId);return {provider: row.platform,model: (json.model as string) ?? row.modelId,@@ -217,8 +235,20 @@ export async function runModerations(}}+ // A monthly-cap exhaustion outranks a transport failure in the reply: the+ // caller needs the month-reset Retry-After, not a "try another provider" 502.+ if (budgetBlocked && (!lastError || lastError.status === 429)) {+ const exhausted = new ModerationError(+ 'Monthly key budget exhausted.',+ 429,+ secondsUntilNextMonth(),+ );+ throw exhausted;+ }+throw new ModerationError(`All moderation providers failed${lastError ? ` (last: ${lastError.message.slice(0, 160)})` : ' (no usable keys)'}.`,lastError && lastError.status === 429 ? 429 : 502,+ lastError?.retryAfterSec,);}