Skip to content

7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.

TypeScript

fix(moderation): enforce monthly caps and mark a served moderation as key evidence (#1625)

tashdroidb06de14b6a0d
diff --git a/server/src/__tests__/routes/moderation.test.ts b/server/src/__tests__/routes/moderation.test.ts
index 0b0c7694..4d176c20 100644
--- a/server/src/__tests__/routes/moderation.test.ts
+++ b/server/src/__tests__/routes/moderation.test.ts
@@ -4,6 +4,7 @@ import { createApp } from '../../app.js';
import { initDb, getDb, getUnifiedApiKey } from '../../db/index.js';
import { encrypt } from '../../lib/crypto.js';
import { listModerationModels, resolveModel } from '../../services/moderation.js';
+import { clearLastHealthyStampState } from '../../services/health.js';
// POST /v1/moderations (#…): OpenAI-compatible content-moderation shim that
// routes to the first enabled OpenAI/OpenRouter/NVIDIA key and fails over
@@ -186,4 +187,39 @@ describe('POST /v1/moderations route', () => {
expect(status).toBe(503);
expect(body.error.message).toContain('No moderation providers available');
});
+
+ it('a served moderation marks the key as working (#1493)', async () => {
+ clearLastHealthyStampState();
+ const key = insertKey('openai');
+ getDb().prepare("UPDATE api_keys SET status = 'unknown' WHERE id = ?").run(key);
+ mockModeration([{ platform: 'openai', response: { results: [{ flagged: false }] } }]);
+ const { status } = await postModeration({ input: 'fine' }, getUnifiedApiKey());
+ expect(status).toBe(200);
+ expect(getDb().prepare('SELECT status, last_healthy_at FROM api_keys WHERE id = ?').get(key))
+ .toMatchObject({ status: 'healthy', last_healthy_at: expect.any(String) });
+ });
+
+ it('a key at its monthly cap is not used for moderation; the reply is 429 with Retry-After (#1158)', async () => {
+ const key = insertKey('openai');
+ getDb().prepare('UPDATE api_keys SET monthly_request_cap = 1 WHERE id = ?').run(key);
+ const month = new Date().toISOString().slice(0, 7);
+ getDb().prepare(
+ 'INSERT INTO key_monthly_usage (key_id, month, requests, tokens) VALUES (?, ?, 1, 0)',
+ ).run(key, month);
+ const calls = mockModeration([{ platform: 'openai', response: { results: [{ flagged: false }] } }]);
+ const server = app.listen(0, '127.0.0.1');
+ if (!server.listening) await new Promise<void>(r => server.once('listening', () => r()));
+ const addr = server.address() as { port: number };
+ const res = await fetch(`http://127.0.0.1:${addr.port}/v1/moderations`, {
+ method: 'POST',
+ headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${getUnifiedApiKey()}` },
+ body: JSON.stringify({ input: 'x' }),
+ });
+ server.close();
+ expect(res.status).toBe(429);
+ expect(await res.json()).toMatchObject({ error: { type: 'rate_limit_error' } });
+ expect(Number(res.headers.get('retry-after'))).toBeGreaterThan(0);
+ // The capped key must never be called.
+ expect(calls).toEqual([]);
+ });
});
diff --git a/server/src/routes/moderation.ts b/server/src/routes/moderation.ts
index 059d9b3a..956ace7b 100644
--- a/server/src/routes/moderation.ts
+++ b/server/src/routes/moderation.ts
@@ -58,6 +58,11 @@ moderationRouter.post('/moderations', async (req: Request, res: Response) => {
});
} catch (err: any) {
const status = err instanceof ModerationError ? err.status : 502;
+ // #1158: a monthly-cap exhaustion carries the seconds until the reset so a
+ // client backs off to the right moment instead of hammering a bare 429.
+ if (err instanceof ModerationError && err.retryAfterSec !== undefined) {
+ res.setHeader('Retry-After', String(Math.max(1, Math.ceil(err.retryAfterSec))));
+ }
res.status(status >= 400 && status < 600 ? status : 502).json({
error: {
message: err?.message ?? 'moderation request failed',
diff --git a/server/src/services/moderation.ts b/server/src/services/moderation.ts
index edafc64b..400a0f71 100644
--- a/server/src/services/moderation.ts
+++ b/server/src/services/moderation.ts
@@ -15,6 +15,8 @@ import { getDb } from '../db/index.js';
import { getClientContext } from '../lib/client-context.js';
import { decrypt } from '../lib/crypto.js';
import { proxyFetch } from '../lib/proxy.js';
+import { checkMonthlyBudget, secondsUntilNextMonth } from './key-budget.js';
+import { markKeyHealthyFromRequest } from './health.js';
/* ------------------------------------------------------------------ */
/* Types */
@@ -38,9 +40,12 @@ export interface ModerationResult {
export class ModerationError extends Error {
status: number;
- constructor(message: string, status: number) {
+ /** Seconds the caller should wait before retrying (429 exhaustion). */
+ retryAfterSec?: number;
+ constructor(message: string, status: number, retryAfterSec?: number) {
super(message);
this.status = status;
+ this.retryAfterSec = retryAfterSec;
}
}
@@ -194,6 +199,11 @@ export async function runModerations(
}
let lastError: ModerationError | null = null;
+ // #1158: a moderation call spends real provider quota, so a key at its
+ // monthly cap must not serve one. Every other inference surface checks the
+ // caps before dispatch (router, embeddings, media, search); moderation was
+ // the only one that read api_keys directly and skipped the gate.
+ let budgetBlocked = false;
for (const row of chain) {
let apiKey: string;
try {
@@ -201,10 +211,18 @@ export async function runModerations(
} catch {
continue;
}
+ if (!checkMonthlyBudget(row.keyId, 0).allowed) {
+ budgetBlocked = true;
+ continue;
+ }
const started = Date.now();
try {
const json = await callProvider(row, apiKey, inputs);
logModerationRequest(row, 'success', Date.now() - started, null);
+ // A served moderation is the evidence the key works, the same as a
+ // served chat request (#1493): it stamps last_healthy_at and lifts an
+ // 'unknown' or 'error' key.
+ markKeyHealthyFromRequest(row.keyId);
return {
provider: row.platform,
model: (json.model as string) ?? row.modelId,
@@ -217,8 +235,20 @@ export async function runModerations(
}
}
+ // A monthly-cap exhaustion outranks a transport failure in the reply: the
+ // caller needs the month-reset Retry-After, not a "try another provider" 502.
+ if (budgetBlocked && (!lastError || lastError.status === 429)) {
+ const exhausted = new ModerationError(
+ 'Monthly key budget exhausted.',
+ 429,
+ secondsUntilNextMonth(),
+ );
+ throw exhausted;
+ }
+
throw new ModerationError(
`All moderation providers failed${lastError ? ` (last: ${lastError.message.slice(0, 160)})` : ' (no usable keys)'}.`,
lastError && lastError.status === 429 ? 429 : 502,
+ lastError?.retryAfterSec,
);
}