7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.
TypeScript
feat(embeddings): support verified Inferbase catalog routes (#1483)
Tashfeen73a24de33c9f
diff --git a/server/src/__tests__/services/inferbase-embeddings.test.ts b/server/src/__tests__/services/inferbase-embeddings.test.tsnew file mode 100644index 00000000..b7139075--- /dev/null+++ b/server/src/__tests__/services/inferbase-embeddings.test.ts@@ -0,0 +1,97 @@+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';+import { getDb, initDb } from '../../db/index.js';+import { encrypt } from '../../lib/crypto.js';+import { EMBEDDING_PLATFORMS, runEmbeddings } from '../../services/embeddings.js';+import { applyCatalog } from '../../services/catalog-sync.js';++const models = [+ ['nvidia-nemotron-3-embed-8b', 4096],+ ['qwen3-embedding-0-6b', 1024],+ ['qwen3-embedding-4b', 2560],+ ['qwen3-embedding-8b', 4096],+] as const;+const json = (body: unknown) => new Response(JSON.stringify(body));++function seed(model: string, dimensions: number) {+ const db = getDb(), secret = encrypt('inferbase-test-key');+ db.prepare("INSERT INTO api_keys (platform,label,encrypted_key,iv,auth_tag,status,enabled) VALUES ('inferbase','test',?,?,?,'healthy',1)").run(secret.encrypted, secret.iv, secret.authTag);+ const result = applyCatalog(db, {+ version: '2099.01.01', generatedAt: new Date().toISOString(), tier: 'live', models: [], quirks: [],+ embeddings: [{ platform: 'inferbase', modelId: model, family: 'inferbase-test', displayName: model,+ dimensions, maxInputTokens: null, priority: 1, enabled: true, quotaLabel: 'Shared free managed allowance' }],+ });+ expect(result.skippedUnknownPlatform).toBe(0);+}++describe('Inferbase catalog-managed embeddings', () => {+ beforeEach(() => { process.env.ENCRYPTION_KEY = '0'.repeat(64); initDb(':memory:'); });+ afterEach(() => { vi.restoreAllMocks(); getDb().close(); });++ it.each(models)('routes %s with float vectors, input ordering and token usage', async (model, dimensions) => {+ seed(model, dimensions);+ const fetch = vi.spyOn(global, 'fetch').mockResolvedValue(json({ model,+ data: [{ index: 1, embedding: Array(dimensions).fill(0.2) }, { index: 0, embedding: Array(dimensions).fill(0.1) }],+ usage: { prompt_tokens: 10, total_tokens: 10, cost_micros: 0 },+ }));+ const result = await runEmbeddings('inferbase-test', ['first', 'second']);+ expect(EMBEDDING_PLATFORMS.has('inferbase')).toBe(true);+ expect(fetch.mock.calls[0][0]).toBe('https://api.inferbase.ai/v1/embeddings');+ expect(new Headers(fetch.mock.calls[0][1]?.headers).get('authorization')).toBe('Bearer inferbase-test-key');+ expect(JSON.parse(String(fetch.mock.calls[0][1]?.body))).toEqual({ model, input: ['first', 'second'], encoding_format: 'float' });+ expect(result).toMatchObject({ platform: 'inferbase', modelId: model, dimensions, inputTokens: 10 });+ expect(result.vectors.map(vector => vector[0])).toEqual([0.1, 0.2]);+ });++ it('forwards and validates an explicit dimension override', async () => {+ seed('qwen3-embedding-4b', 2560);+ const fetch = vi.spyOn(global, 'fetch').mockResolvedValue(json({ model: 'qwen3-embedding-4b',+ data: [{ index: 0, embedding: [0.1, 0.2] }], usage: { total_tokens: 3 },+ }));+ expect(await runEmbeddings('inferbase-test', ['hello'], 2)).toMatchObject({ dimensions: 2, inputTokens: 3 });+ expect(JSON.parse(String(fetch.mock.calls[0][1]?.body)).dimensions).toBe(2);+ });++ it.each([+ { model: 'other-model', data: [{ index: 0, embedding: [0.1, 0.2] }] },+ { data: [{ index: 0, embedding: [0.1, 0.2] }] },+ { model: 'test', data: [] },+ { model: 'test', data: [{ index: 0, embedding: [0.1] }] },+ { model: 'test', data: [{ index: 0, embedding: [0.1, null] }] },+ { model: 'test', data: [{ index: 0, embedding: ['0.1', '0.2'] }] },+ { model: 'test', data: [{ embedding: [0.1, 0.2] }] },+ { model: 'test', data: [{ index: 1, embedding: [0.1, 0.2] }] },+ { error: { message: 'quota exhausted' } },+ ])('rejects mismatched identities and invalid vectors instead of corrupting the family', async body => {+ seed('test', 2);+ vi.spyOn(global, 'fetch').mockResolvedValue(json(body));+ await expect(runEmbeddings('inferbase-test', ['hello'])).rejects.toMatchObject({ status: 502 });+ });++ it('rejects duplicate vector indices', async () => {+ seed('test', 2);+ vi.spyOn(global, 'fetch').mockResolvedValue(json({ model: 'test', data: [+ { index: 0, embedding: [0.1, 0.2] }, { index: 0, embedding: [0.3, 0.4] },+ ] }));+ await expect(runEmbeddings('inferbase-test', ['first', 'second'])).rejects.toMatchObject({ status: 502 });+ });++ it('preserves upstream rate-limit retry hints', async () => {+ seed('test', 2);+ vi.spyOn(global, 'fetch').mockResolvedValue(new Response('upstream error', { status: 429, headers: { 'Retry-After': '20' } }));+ await expect(runEmbeddings('inferbase-test', ['hello'])).rejects.toMatchObject({ status: 429, retryAfterMs: 20_000 });+ });++ it.each([401, 402, 503])('fails closed on upstream HTTP %s without returning vectors', async status => {+ seed('test', 2);+ vi.spyOn(global, 'fetch').mockResolvedValue(new Response('upstream error', { status }));+ // The existing family router normalizes exhausted non-rate-limit failures+ // to 502, retaining the upstream status in its diagnostic message.+ await expect(runEmbeddings('inferbase-test', ['hello'])).rejects.toMatchObject({+ status: 502, message: expect.stringContaining(`upstream ${status}`),+ });+ });++ it('does not seed catalog models before their signed feed arrives', () => {+ expect(getDb().prepare("SELECT COUNT(*) AS n FROM embedding_models WHERE platform='inferbase'").get()).toEqual({ n: 0 });+ });+});diff --git a/server/src/services/embeddings.ts b/server/src/services/embeddings.tsindex 96190756..8e75fde8 100644--- a/server/src/services/embeddings.ts+++ b/server/src/services/embeddings.ts@@ -19,6 +19,7 @@ import { customEndpointKeyIds } from './custom-endpoint.js';import type { Db } from '../db/types.js';import { SPEKA_BASE_URL } from '../providers/speka.js';import { ELECTRONHUB_BASE_URL } from '../providers/electronhub.js';+import { INFERBASE_BASE_URL } from '../providers/inferbase.js';export interface EmbeddingModelRow {id: number;@@ -117,6 +118,7 @@ export const EMBEDDING_PLATFORMS = new Set(['sealion','speka','electronhub',+ 'inferbase',]);interface ProviderCallResult {@@ -159,7 +161,7 @@ async function openAiStyleEmbed(usage?: { prompt_tokens?: number; total_tokens?: number };};const data = [...(j.data ?? [])].sort((a, b) => (a.index ?? 0) - (b.index ?? 0));- if (platform === 'electronhub' && (+ if ((platform === 'electronhub' || platform === 'inferbase') && (j.model !== modelId || data.length !== inputs.length ||data.some((d, i) => d.index !== i || !Array.isArray(d.embedding) ||d.embedding.length !== (dimensions ?? expectedDimensions) ||@@ -167,7 +169,8 @@ async function openAiStyleEmbed()) {// A successful HTTP response is not enough: wrong model/dimension or// malformed vectors would corrupt the caller's embedding family.- throw new EmbeddingsError('ElectronHub returned mismatched or malformed embeddings', 502);+ const provider = platform === 'electronhub' ? 'ElectronHub' : 'Inferbase';+ throw new EmbeddingsError(`${provider} returned mismatched or malformed embeddings`, 502);}return {vectors: data.map(d => d.embedding),@@ -283,6 +286,8 @@ async function callProvider(row: EmbeddingModelRow, credential: ProviderCredentireturn openAiStyleEmbed(`${SPEKA_BASE_URL}/embeddings`, row.platform, key, row.model_id, inputs, { encoding_format: 'float' }, dimensions);case 'electronhub':return openAiStyleEmbed(`${ELECTRONHUB_BASE_URL}/embeddings`, row.platform, key, row.model_id, inputs, { encoding_format: 'float' }, dimensions, row.dimensions);+ case 'inferbase':+ return openAiStyleEmbed(`${INFERBASE_BASE_URL}/embeddings`, row.platform, key, row.model_id, inputs, { encoding_format: 'float' }, dimensions, row.dimensions);case 'cloudflare': {// Key is stored as "account_id:token".const sep = key.indexOf(':');