7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.
TypeScript
feat(moderations): /v1/moderations endpoint (OpenAI-compatible) (#1155)
suantea (asuan_suan)94a59dd608eb
diff --git a/README.md b/README.mdindex 29f04145..d8976f1e 100644--- a/README.md+++ b/README.md@@ -335,7 +335,7 @@ Stacking free tiers has real trade-offs: no frontier models, variable latency, nContributors very welcome! See [CONTRIBUTING.md](CONTRIBUTING.md) for the dev loop, PR expectations, and the policy on AI/LLM-assisted contributions (short version: welcome, same quality bar as any other PR). Good first PRs:- **Add a provider** — copy `server/src/providers/openai-compat.ts` as a template, wire it into `server/src/providers/index.ts`, seed its models in `server/src/db/index.ts`, add a test in `server/src/__tests__/providers/`.-- **Add an endpoint** — moderations and other OpenAI-compatible surfaces. The provider base class can grow new methods; adapters declare which they support.+- **Add an endpoint** — other OpenAI-compatible surfaces (e.g. `/v1/realtime`). The provider base class can grow new methods; adapters declare which they support.- **Improve the router** — cost-aware routing (cheapest-healthy-fastest tradeoffs), better latency-weighted priority, regional pinning.- **Dashboard polish** — charts on the Analytics page, key rotation UX, batch import of keys from `.env`.- **Docs** — more examples, client library snippets for Go/Rust/etc., a deployment recipe for Docker or Fly.diff --git a/server/src/__tests__/routes/moderation.test.ts b/server/src/__tests__/routes/moderation.test.tsnew file mode 100644index 00000000..0b0c7694--- /dev/null+++ b/server/src/__tests__/routes/moderation.test.ts@@ -0,0 +1,189 @@+import { describe, it, expect, beforeAll, beforeEach, afterEach, vi } from 'vitest';+import type { Express } from 'express';+import { createApp } from '../../app.js';+import { initDb, getDb, getUnifiedApiKey } from '../../db/index.js';+import { encrypt } from '../../lib/crypto.js';+import { listModerationModels, resolveModel } from '../../services/moderation.js';++// POST /v1/moderations (#…): OpenAI-compatible content-moderation shim that+// routes to the first enabled OpenAI/OpenRouter/NVIDIA key and fails over+// across platforms. Covers the model-resolution rules, the failover chain,+// and the route's auth/validation envelope — without hitting any provider.++let app: Express;++async function postModeration(body: unknown, token?: string) {+ const server = app.listen(0, '127.0.0.1');+ if (!server.listening) await new Promise<void>(r => server.once('listening', () => r()));+ const addr = server.address() as { port: number };+ const res = await fetch(`http://127.0.0.1:${addr.port}/v1/moderations`, {+ method: 'POST',+ headers: {+ 'Content-Type': 'application/json',+ ...(token ? { Authorization: `Bearer ${token}` } : {}),+ },+ body: JSON.stringify(body),+ });+ const json = await res.json().catch(() => null);+ server.close();+ return { status: res.status, body: json as any };+}++function insertKey(platform: string): number {+ const { encrypted, iv, authTag } = encrypt('sk-test');+ const result = getDb().prepare(`+ INSERT INTO api_keys (platform, label, encrypted_key, iv, auth_tag, status, enabled)+ VALUES (?, 'test', ?, ?, ?, 'healthy', 1)+ `).run(platform, encrypted, iv, authTag);+ return Number(result.lastInsertRowid);+}++/** Mock provider moderation calls for one or more platforms in one spy.+ * Failover walks several platforms, so a single mock must answer all of+ * them (each mockModeration() call would otherwise replace the previous spy). */+function mockModeration(+ handlers: Array<{ platform: string; response: Record<string, unknown>; status?: number }>,+) {+ const urlPatterns: Record<string, string> = {+ openai: 'https://api.openai.com/v1/moderations',+ openrouter: 'https://openrouter.ai/api/v1/moderations',+ nvidia: 'https://integrate.api.nvidia.com/v1/moderations',+ };+ const origFetch = global.fetch;+ const calls: string[] = [];+ vi.spyOn(global, 'fetch').mockImplementation(async (url: any, init?: any) => {+ const urlStr = typeof url === 'string' ? url : url.toString();+ // Test-harness requests to the ephemeral app server must pass through.+ if (urlStr.startsWith('http://127.0.0.1')) return origFetch(url, init);+ const handler = handlers.find(h => urlStr === urlPatterns[h.platform]);+ if (handler) {+ calls.push(handler.platform);+ return new Response(JSON.stringify({ id: 'modr-test', model: 'omni-moderation-latest', ...handler.response }), {+ status: handler.status ?? 200,+ headers: { 'content-type': 'application/json' },+ });+ }+ throw new Error(`unexpected URL in moderation test: ${urlStr}`);+ });+ return calls;+}++function restoreFetch() {+ vi.restoreAllMocks();+}++describe('moderation service — model resolution', () => {+ beforeAll(() => {+ process.env.ENCRYPTION_KEY = '0'.repeat(64);+ initDb(':memory:');+ });++ beforeEach(() => {+ getDb().prepare('DELETE FROM api_keys').run();+ });++ afterEach(restoreFetch);++ it('lists one entry per enabled platform, first key per platform', () => {+ insertKey('openai');+ insertKey('openai'); // second openai key must be ignored+ insertKey('nvidia');+ const models = listModerationModels();+ expect(models.map(m => m.platform)).toEqual(['openai', 'nvidia']);+ const openai = models.find(m => m.platform === 'openai')!;+ expect(openai.modelId).toBe('omni-moderation-latest');+ expect(openai.baseUrl).toBe('https://api.openai.com');+ });++ it('ignores disabled or unhealthy keys', () => {+ const { encrypted, iv, authTag } = encrypt('sk-test');+ getDb().prepare(`+ INSERT INTO api_keys (platform, label, encrypted_key, iv, auth_tag, status, enabled)+ VALUES ('openai', 'off', ?, ?, ?, 'healthy', 0)+ `).run(encrypted, iv, authTag);+ expect(listModerationModels()).toHaveLength(0);+ });++ it('resolveModel maps OpenAI legacy names and unknown models to the default', () => {+ insertKey('openai');+ insertKey('nvidia');+ expect(resolveModel(undefined)).toBe('omni-moderation-latest');+ expect(resolveModel('text-moderation-latest')).toBe('omni-moderation-latest');+ expect(resolveModel('text-moderation-stable')).toBe('omni-moderation-latest');+ // An unknown requested model falls back to the first available default.+ expect(resolveModel('nonsense-model')).toBe('omni-moderation-latest');+ });+});++describe('POST /v1/moderations route', () => {+ beforeAll(() => {+ process.env.ENCRYPTION_KEY = '0'.repeat(64);+ initDb(':memory:');+ app = createApp();+ });++ beforeEach(() => {+ getDb().prepare('DELETE FROM api_keys').run();+ });++ afterEach(restoreFetch);++ it('rejects a request without a valid key', async () => {+ const { status } = await postModeration({ input: 'kill all humans' });+ expect(status).toBe(401);+ });++ it('validates the body shape', async () => {+ const key = getUnifiedApiKey();+ const missing = await postModeration({}, key);+ expect(missing.status).toBe(400);+ const empty = await postModeration({ input: [] }, key);+ expect(empty.status).toBe(400);+ });++ it('proxies to the first enabled platform and returns OpenAI-shaped results', async () => {+ insertKey('openai');+ const calls = mockModeration([+ {+ platform: 'openai',+ response: {+ results: [{ flagged: false, categories: { violence: false }, category_scores: { violence: 0.01 } }],+ },+ },+ ]);+ const { status, body } = await postModeration(+ { input: 'a perfectly fine sentence' },+ getUnifiedApiKey(),+ );+ expect(status).toBe(200);+ expect(calls).toEqual(['openai']);+ expect(body._provider).toBe('openai');+ expect(body.model).toBe('omni-moderation-latest');+ expect(body.results).toHaveLength(1);+ expect(body.results[0].flagged).toBe(false);+ });++ it('fails over to the next platform when the first errors', async () => {+ // openai and openrouter share the default 'omni-moderation-latest'+ // model, so the chain walks openai → openrouter.+ insertKey('openai');+ insertKey('openrouter');+ const calls = mockModeration([+ { platform: 'openai', response: {}, status: 500 },+ { platform: 'openrouter', response: { results: [{ flagged: true }] }, status: 200 },+ ]);+ const { status, body } = await postModeration(+ { input: 'flagged content' },+ getUnifiedApiKey(),+ );+ expect(status).toBe(200);+ expect(body._provider).toBe('openrouter');+ expect(calls).toEqual(['openai', 'openrouter']);+ });++ it('returns 503 when no moderation provider has an enabled key', async () => {+ const { status, body } = await postModeration({ input: 'x' }, getUnifiedApiKey());+ expect(status).toBe(503);+ expect(body.error.message).toContain('No moderation providers available');+ });+});diff --git a/server/src/app.ts b/server/src/app.tsindex f0303459..833e4501 100644--- a/server/src/app.ts+++ b/server/src/app.ts@@ -32,6 +32,7 @@ import { geminiRouter } from './routes/gemini.js';import { ollamaRouter } from './routes/ollama.js';import { urlTokenRouter } from './routes/url-tokens.js';import { updateRouter } from './routes/update.js';+import { moderationRouter } from './routes/moderation.js';import { requireAuth } from './middleware/requireAuth.js';import { createProxyRateLimiter, createAdminRateLimiter } from './middleware/rateLimit.js';@@ -302,6 +303,8 @@ export function createApp(config?: Config) {// paths it doesn't own fall through to the OpenAI router untouched.app.use('/v1', anthropicRouter);app.use('/v1', proxyRouter);+ // OpenAI Moderation API shim (POST /v1/moderations)+ app.use('/v1', moderationRouter);// OpenAI Responses API shim (Codex CLI requires wire_api="responses"; see #96)app.use('/v1', responsesRouter);diff --git a/server/src/lib/proxy.ts b/server/src/lib/proxy.tsindex 9d49203f..454710f9 100644--- a/server/src/lib/proxy.ts+++ b/server/src/lib/proxy.ts@@ -560,7 +560,7 @@ async function resolveDispatcher(): Promise<{ dispatcher: unknown; isSocks: bool* written to `requests.request_type` so the abort message and the row* column agree on terminology.*/-export type ProxyRequestType = 'chat' | 'embedding' | 'image' | 'video' | 'audio' | 'transcription' | 'unknown';+export type ProxyRequestType = 'chat' | 'embedding' | 'image' | 'video' | 'audio' | 'transcription' | 'moderation' | 'unknown';/*** Build an AbortError DOMException whose `message` carries a compact triagediff --git a/server/src/routes/moderation.ts b/server/src/routes/moderation.tsnew file mode 100644index 00000000..059d9b3a--- /dev/null+++ b/server/src/routes/moderation.ts@@ -0,0 +1,68 @@+import { Router } from 'express';+import type { Request, Response } from 'express';+import { z } from 'zod';+import { extractApiToken } from './proxy.js';+import { resolveAuth } from '../lib/system-prompt.js';+import { runModerations, ModerationError } from '../services/moderation.js';++export const moderationRouter = Router();++function requireInferenceAuth(req: Request, res: Response): boolean {+ const token = extractApiToken(req);+ const auth = resolveAuth(token);+ if (!auth) {+ res.status(401).json({ error: { message: 'Invalid API key', type: 'authentication_error' } });+ return false;+ }+ return true;+}++const moderationBody = z.object({+ model: z.string().optional(),+ input: z.union([z.string(), z.array(z.string())]),+});++moderationRouter.post('/moderations', async (req: Request, res: Response) => {+ if (!requireInferenceAuth(req, res)) return;++ const parsed = moderationBody.safeParse(req.body);+ if (!parsed.success) {+ res.status(400).json({+ error: {+ message: parsed.error.errors.map(e => e.message).join(', '),+ type: 'invalid_request_error',+ },+ });+ return;+ }++ const { model, input } = parsed.data;+ const inputs = Array.isArray(input) ? input : [input];++ if (inputs.length === 0) {+ res.status(400).json({+ error: { message: 'input must not be empty', type: 'invalid_request_error' },+ });+ return;+ }++ try {+ const { provider, model: usedModel, result } = await runModerations(model, inputs);+ const upstream = result as { id?: string; model?: string; results?: unknown[] };++ res.json({+ id: upstream.id ?? `modr-${Date.now()}`,+ model: usedModel,+ results: upstream.results ?? [],+ _provider: provider,+ });+ } catch (err: any) {+ const status = err instanceof ModerationError ? err.status : 502;+ res.status(status >= 400 && status < 600 ? status : 502).json({+ error: {+ message: err?.message ?? 'moderation request failed',+ type: status === 429 ? 'rate_limit_error' : 'server_error',+ },+ });+ }+});diff --git a/server/src/services/moderation.ts b/server/src/services/moderation.tsnew file mode 100644index 00000000..edafc64b--- /dev/null+++ b/server/src/services/moderation.ts@@ -0,0 +1,224 @@+/**+ * Moderation service — proxy /v1/moderations requests to providers that+ * expose OpenAI-compatible moderation endpoints.+ *+ * Supported platforms (first key per platform wins):+ * - openai → https://api.openai.com/v1/moderations+ * - openrouter → https://openrouter.ai/api/v1/moderations+ * - nvidia → https://integrate.api.nvidia.com/v1/moderations+ *+ * The failover walks the first available key for each platform in order.+ * No separate `moderation_models` table is needed for v1 — the set of+ * platforms is small and the endpoint is always /v1/moderations.+ */+import { getDb } from '../db/index.js';+import { getClientContext } from '../lib/client-context.js';+import { decrypt } from '../lib/crypto.js';+import { proxyFetch } from '../lib/proxy.js';++/* ------------------------------------------------------------------ */+/* Types */+/* ------------------------------------------------------------------ */++export interface ModerationModelRow {+ platform: string;+ modelId: string;+ baseUrl: string;+ keyId: number;+ encryptedKey: string;+ iv: string;+ authTag: string;+}++export interface ModerationResult {+ provider: string;+ model: string;+ result: Record<string, unknown>;+}++export class ModerationError extends Error {+ status: number;+ constructor(message: string, status: number) {+ super(message);+ this.status = status;+ }+}++/* ------------------------------------------------------------------ */+/* Provider registry (hardcoded for v1) */+/* ------------------------------------------------------------------ */++interface ModerationEndpoint {+ baseUrl: string;+ modelId: string;+}++const MODERATION_MODELS: Record<string, ModerationEndpoint> = {+ openai: { baseUrl: 'https://api.openai.com', modelId: 'omni-moderation-latest' },+ openrouter: { baseUrl: 'https://openrouter.ai/api', modelId: 'openai/omni-moderation-latest' },+ nvidia: { baseUrl: 'https://integrate.api.nvidia.com', modelId: 'nvidia/llama-3.1-nemoguard-8b-content-safety' },+};++const PLATFORM_ORDER = ['openai', 'openrouter', 'nvidia'] as const;++/* ------------------------------------------------------------------ */+/* Helpers */+/* ------------------------------------------------------------------ */++export function listModerationModels(): ModerationModelRow[] {+ const db = getDb();+ const rows: ModerationModelRow[] = [];+ for (const platform of PLATFORM_ORDER) {+ const ep = MODERATION_MODELS[platform];+ const key = db.prepare(`+ SELECT id, encrypted_key, iv, auth_tag+ FROM api_keys+ WHERE platform = ?+ AND enabled = 1+ AND status IN ('healthy', 'unknown')+ ORDER BY id+ LIMIT 1+ `).get(platform) as { id: number; encrypted_key: string; iv: string; auth_tag: string } | undefined;+ if (key) {+ rows.push({+ platform,+ modelId: ep.modelId,+ baseUrl: ep.baseUrl,+ keyId: key.id,+ encryptedKey: key.encrypted_key,+ iv: key.iv,+ authTag: key.auth_tag,+ });+ }+ }+ return rows;+}++function getDefaultModel(): string | null {+ const models = listModerationModels();+ return models.length > 0 ? models[0].modelId : null;+}++export function resolveModel(requestedModel: string | undefined): string | null {+ if (!requestedModel || requestedModel === 'text-moderation-stable' || requestedModel === 'text-moderation-latest') {+ return getDefaultModel();+ }+ const models = listModerationModels();+ const match = models.find(m => m.modelId === requestedModel);+ return match ? match.modelId : getDefaultModel();+}++/* ------------------------------------------------------------------ */+/* Provider call */+/* ------------------------------------------------------------------ */++async function callProvider(+ row: ModerationModelRow,+ apiKey: string,+ inputs: string[],+): Promise<Record<string, unknown>> {+ const url = `${row.baseUrl}/v1/moderations`;+ const res = await proxyFetch(url, {+ method: 'POST',+ headers: {+ 'Content-Type': 'application/json',+ Authorization: `Bearer ${apiKey}`,+ },+ body: JSON.stringify({ model: row.modelId, input: inputs.length === 1 ? inputs[0] : inputs }),+ signal: AbortSignal.timeout(30_000),+ }, row.platform, 'moderation');++ if (!res.ok) {+ const body = await res.text().catch(() => '');+ throw new ModerationError(`moderation provider ${row.platform} returned ${res.status}: ${body.slice(0, 200)}`, res.status);+ }++ const json = await res.json() as Record<string, unknown>;+ if (!json.results || !Array.isArray(json.results)) {+ throw new ModerationError('upstream returned malformed moderation response', 502);+ }+ return json;+}++/* ------------------------------------------------------------------ */+/* Logging */+/* ------------------------------------------------------------------ */++function logModerationRequest(+ row: ModerationModelRow,+ status: 'success' | 'error',+ latencyMs: number,+ error: string | null,+): void {+ try {+ const client = getClientContext();+ getDb().prepare(`+ INSERT INTO requests (platform, model_id, key_id, status, input_tokens, output_tokens, latency_ms, error, request_type, client_ip, client_user_agent, client_agent)+ VALUES (?, ?, ?, ?, 0, 0, ?, ?, 'moderation', ?, ?, ?)+ `).run(row.platform, row.modelId, row.keyId, status, latencyMs, error, client.ip, client.userAgent, client.agent);+ } catch (e) {+ console.error('Failed to log moderation request:', e);+ }+}++/* ------------------------------------------------------------------ */+/* Main entry point */+/* ------------------------------------------------------------------ */++export async function runModerations(+ model: string | undefined,+ inputs: string[],+): Promise<ModerationResult> {+ const resolved = resolveModel(model);+ if (!resolved) {+ throw new ModerationError(+ 'No moderation providers available. Add an OpenAI, OpenRouter, or NVIDIA API key.',+ 503,+ );+ }++ const models = listModerationModels();+ // A default/legacy request walks EVERY available platform in order — each+ // provider calls its own catalog model, which is what makes the cross-+ // platform failover below real (OpenRouter's omni-moderation id carries an+ // `openai/` prefix, so an exact-modelId filter would silently drop it).+ // An explicit model narrows the chain to the platform(s) exposing it.+ const isDefaultRequest = !model+ || model === 'text-moderation-stable'+ || model === 'text-moderation-latest';+ const chain = isDefaultRequest+ ? models+ : models.filter(m => m.modelId === resolved);+ if (chain.length === 0) {+ throw new ModerationError(`No enabled providers for moderation model '${resolved}'.`, 503);+ }++ let lastError: ModerationError | null = null;+ for (const row of chain) {+ let apiKey: string;+ try {+ apiKey = decrypt(row.encryptedKey, row.iv, row.authTag);+ } catch {+ continue;+ }+ const started = Date.now();+ try {+ const json = await callProvider(row, apiKey, inputs);+ logModerationRequest(row, 'success', Date.now() - started, null);+ return {+ provider: row.platform,+ model: (json.model as string) ?? row.modelId,+ result: json,+ };+ } catch (err: any) {+ const e = err instanceof ModerationError ? err : new ModerationError(String(err?.message ?? err), 502);+ logModerationRequest(row, 'error', Date.now() - started, e.message.slice(0, 300));+ lastError = e;+ }+ }++ throw new ModerationError(+ `All moderation providers failed${lastError ? ` (last: ${lastError.message.slice(0, 160)})` : ' (no usable keys)'}.`,+ lastError && lastError.status === 429 ? 429 : 502,+ );+}