7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.
TypeScript
fix(modelscope): stop health checks from burning magic-grain quota (#882)
ZER0-auto473d790f4b7d
diff --git a/.env.example b/.env.exampleindex 8120f200..45903dde 100644--- a/.env.example+++ b/.env.example@@ -99,6 +99,15 @@ PORT=3001# Read at startup. Example — give NVIDIA NIM's slow reasoning models 5 minutes:# PROVIDER_TIMEOUT_NVIDIA=300000+# ModelScope (魔搭) validates API keys with a paid 1-token chat completion+# (GET /v1/models does not enforce auth), which spends the account's+# magic-grain (魔粒) quota per probe — observed as 2 魔粒 per ultra-tier+# request. To stop the 5-minute health pass from burning ~288 paid probes+# per key per day, a successful validation is cached per key for this long+# (milliseconds, default 86400000 = 24h). Set to 0 to probe on every pass.+# A revoked key is still caught by the next real request's 401.+# MODELSCOPE_VALIDATE_CACHE_MS=86400000+# Mid-stream inactivity timeout for streaming responses, in milliseconds# (default: 90000). If an SSE stream goes this long without a single byte# AFTER the first byte arrived, the gateway gives up on it (before any outputdiff --git a/server/src/__tests__/providers/modelscope.test.ts b/server/src/__tests__/providers/modelscope.test.tsindex 721e5cec..7a3d7a66 100644--- a/server/src/__tests__/providers/modelscope.test.ts+++ b/server/src/__tests__/providers/modelscope.test.ts@@ -138,4 +138,65 @@ describe('ModelScope validateKey (#581)', () => {const provider = new ModelScopeProvider();await expect(provider.validateKey(GARBAGE_KEY)).rejects.toThrow(/no models/);});++ it('caches a successful validation per key and skips the network on repeat checks', async () => {+ const fetchSpy = vi.spyOn(global, 'fetch').mockImplementation(async (url) => {+ if (String(url).endsWith('/models')) return ROSTER as unknown as Response;+ return {+ ok: true,+ status: 200,+ json: () => Promise.resolve({+ id: 'chatcmpl-1', object: 'chat.completion', created: 1,+ model: 'Qwen/Qwen3-235B-A22B-Instruct-2507',+ choices: [{ index: 0, message: { role: 'assistant', content: 'p' }, finish_reason: 'length' }],+ usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 },+ }),+ } as unknown as Response;+ });++ const provider = new ModelScopeProvider();+ const ctx = {+ platform: 'modelscope' as const,+ keyId: 48,+ quotaPoolKey: 'modelscope::account',+ endpoint: 'models',+ origin: 'health' as const,+ };++ // First validation probes the network (models + chat completion).+ await expect(provider.validateKey('ms-test-valid-shaped', ctx)).resolves.toBe(true);+ expect(fetchSpy).toHaveBeenCalledTimes(2);++ // Second validation within the cache window returns true without any call.+ await expect(provider.validateKey('ms-test-valid-shaped', ctx)).resolves.toBe(true);+ expect(fetchSpy).toHaveBeenCalledTimes(2);++ // A DIFFERENT token on the same key row is a different secret — rotating a+ // key in place must re-probe rather than inherit the old token's verdict.+ await expect(provider.validateKey('ms-test-rotated', ctx)).resolves.toBe(true);+ expect(fetchSpy).toHaveBeenCalledTimes(4);+ });++ it('does not cache an invalid validation (a 401 re-probes on the next check)', async () => {+ const fetchSpy = vi.spyOn(global, 'fetch').mockImplementation(async (url) => {+ if (String(url).endsWith('/models')) return ROSTER as unknown as Response;+ return AUTH_FAILED_401 as unknown as Response;+ });++ const provider = new ModelScopeProvider();+ const ctx = {+ platform: 'modelscope' as const,+ keyId: 48,+ quotaPoolKey: 'modelscope::account',+ endpoint: 'models',+ origin: 'health' as const,+ };++ await expect(provider.validateKey(GARBAGE_KEY, ctx)).resolves.not.toBe(true);+ expect(fetchSpy).toHaveBeenCalledTimes(2);++ // A failed validation must not be cached — the next health pass re-probes.+ await expect(provider.validateKey(GARBAGE_KEY, ctx)).resolves.not.toBe(true);+ expect(fetchSpy).toHaveBeenCalledTimes(4);+ });});diff --git a/server/src/providers/modelscope.ts b/server/src/providers/modelscope.tsindex e8393be5..68cb0a46 100644--- a/server/src/providers/modelscope.ts+++ b/server/src/providers/modelscope.ts@@ -1,9 +1,27 @@+import { createHash } from 'node:crypto';import { OpenAICompatProvider } from './openai-compat.js';import type { KeyValidationResult } from './base.js';import { recordQuotaObservationsFromResponse, type QuotaObservationContext } from '../services/provider-quota.js';const MODELSCOPE_BASE_URL = 'https://api-inference.modelscope.cn/v1';+// A successful validation is cached per key for this long. The 5-minute+// health pass would otherwise burn ~288 paid 1-token completions per key+// per day against the account's magic-grain (魔粒) quota — observed as+// 2 魔粒 per ultra-tier request (2026-08-14). A revoked key is still caught+// by the next real request's 401 handling (error-classify disables it), so+// the cache only delays proactive detection, never hides a live failure.+const DEFAULT_VALIDATE_CACHE_MS = 24 * 60 * 60 * 1000;++function modelscopeValidateCacheMs(): number {+ const raw = process.env.MODELSCOPE_VALIDATE_CACHE_MS;+ if (raw !== undefined && raw.trim() !== '') {+ const n = Number(raw);+ if (Number.isInteger(n) && n >= 0) return n;+ }+ return DEFAULT_VALIDATE_CACHE_MS;+}+/*** ModelScope (魔搭社区, Alibaba) — OpenAI-compatible inference API.*@@ -24,10 +42,15 @@ const MODELSCOPE_BASE_URL = 'https://api-inference.modelscope.cn/v1';* completion. The model id is picked dynamically from GET /v1/models (first* entry) because the roster churns — a hardcoded id would rot.*- * COST: a successful validation burns 1 request (and ~1 output token) of the- * 2000-requests/day account-wide free quota per health check. A failed-auth- * validation is rejected before generation and, per maintainer reports, does- * not count against quota.+ * COST: a successful validation burns 1 paid 1-token completion per health+ * check against the account's magic-grain (魔粒) quota — observed as 2 魔粒+ * per ultra-tier request (2026-08-14), not the historical 2000-requests/day+ * free API quota. With the default 5-minute health pass that would be ~288+ * paid probes per key per day. A successful validation is therefore cached+ * per key for MODELSCOPE_VALIDATE_CACHE_MS (default 24h) — repeat health+ * passes return true without re-probing, taking the steady-state cost to one+ * probe per key per day. A failed-auth validation is rejected before+ * generation and, per maintainer reports, does not count against quota.** Community testers must confirm (we have NO real token for this platform):* the maintainer-documented bad-binding failure is@@ -38,6 +61,11 @@ const MODELSCOPE_BASE_URL = 'https://api-inference.modelscope.cn/v1';* actionable reason instead of a generic "invalid key". See issue #581.*/export class ModelScopeProvider extends OpenAICompatProvider {+ /** key-material fingerprint → last successful validation wall-clock (ms).+ * Keyed on the token itself, not the row id, so editing a key in place+ * re-probes instead of inheriting the old token's verdict. */+ private readonly lastValidatedAt = new Map<string, number>();+constructor() {super({platform: 'modelscope',@@ -52,6 +80,21 @@ export class ModelScopeProvider extends OpenAICompatProvider {}override async validateKey(apiKey: string, quotaContext?: QuotaObservationContext): Promise<KeyValidationResult> {+ // Validation cache: a key validated successfully within+ // MODELSCOPE_VALIDATE_CACHE_MS is trusted without re-probing. The+ // 5-minute health pass would otherwise burn ~288 paid 1-token+ // completions per key per day against the magic-grain quota. A revoked+ // key is still caught by the next real request's 401 handling, so this+ // only delays proactive detection.+ const cacheMs = modelscopeValidateCacheMs();+ const fingerprint = createHash('sha256').update(apiKey).digest('hex');+ if (cacheMs > 0) {+ const last = this.lastValidatedAt.get(fingerprint);+ if (last !== undefined && Date.now() - last < cacheMs) {+ return true;+ }+ }+// Transport errors (DNS / timeout / TLS) propagate — health.ts marks// status='error' without counting toward auto-disable; only a confirmed// 401/403 disables a key.@@ -101,6 +144,10 @@ export class ModelScopeProvider extends OpenAICompatProvider {// (surfaces the alibaba-binding reason); everything else — 200, but also// 400/404 (probe model quirk), 429 (quota) — proves the token// authenticated and returns true.- return this.validationResult(res);+ const result = await this.validationResult(res);+ if (result === true && cacheMs > 0) {+ this.lastValidatedAt.set(fingerprint, Date.now());+ }+ return result;}}