7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.
TypeScript
feat: support catalog-managed LLMTR embeddings (#1531)
Tashfeen1b80dc3fafe3
diff --git a/server/src/__tests__/services/llmtr-embeddings.test.ts b/server/src/__tests__/services/llmtr-embeddings.test.tsnew file mode 100644index 00000000..7c59c34d--- /dev/null+++ b/server/src/__tests__/services/llmtr-embeddings.test.ts@@ -0,0 +1,80 @@+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';+import { getDb, initDb } from '../../db/index.js';+import { encrypt } from '../../lib/crypto.js';+import { EMBEDDING_PLATFORMS, runEmbeddings } from '../../services/embeddings.js';+import { applyCatalog } from '../../services/catalog-sync.js';++const models = [+ { id: 'llmtr/embeddingmagibu-200m', dimensions: 768, maxInputTokens: 8192 },+ { id: 'liquid/lfm-2.5-embedding-350m-free', dimensions: 1024, maxInputTokens: 512 },+];+const json = (body: unknown) => new Response(JSON.stringify(body));+function seed() {+ const db = getDb(), secret = encrypt('llmtr-test-key');+ db.prepare("INSERT INTO api_keys (platform,label,encrypted_key,iv,auth_tag,status,enabled) VALUES ('llmtr','test',?,?,?,'healthy',1)").run(secret.encrypted, secret.iv, secret.authTag);+ const result = applyCatalog(db, { version: '2099.01.01', generatedAt: new Date().toISOString(), tier: 'live', models: [], quirks: [],+ embeddings: models.map(model => ({ platform: 'llmtr', modelId: model.id, family: model.id, displayName: model.id, dimensions: model.dimensions,+ maxInputTokens: model.maxInputTokens, priority: 1, enabled: true, quotaLabel: 'Selected zero-price route; provider quota applies' })) });+ expect(result.skippedUnknownPlatform).toBe(0);+}++describe('LLMTR catalog-managed embeddings', () => {+ beforeEach(() => { process.env.ENCRYPTION_KEY = '0'.repeat(64); initDb(':memory:'); });+ afterEach(() => { vi.restoreAllMocks(); getDb().close(); });++ it.each(models)('routes $id with exact model identity, dimensions and usage', async model => {+ seed();+ const fetch = vi.spyOn(global, 'fetch').mockResolvedValue(json({ model: model.id,+ data: [{ index: 1, embedding: Array(model.dimensions).fill(0.2) }, { index: 0, embedding: Array(model.dimensions).fill(0.1) }], usage: { prompt_tokens: 6 } }));+ const result = await runEmbeddings(model.id, ['first', 'second'], model.dimensions);+ expect(EMBEDDING_PLATFORMS.has('llmtr')).toBe(true);+ expect(fetch.mock.calls[0][0]).toBe('https://llmtr.com/v1/embeddings');+ expect(new Headers(fetch.mock.calls[0][1]?.headers).get('authorization')).toBe('Bearer llmtr-test-key');+ expect(JSON.parse(String(fetch.mock.calls[0][1]?.body))).toEqual({ model: model.id, input: ['first', 'second'] });+ expect(result).toMatchObject({ platform: 'llmtr', modelId: model.id, dimensions: model.dimensions, inputTokens: 6 });+ expect(result.vectors.map(v => v[0])).toEqual([0.1, 0.2]);+ });++ it.each([+ { model: 'other', data: [{ index: 0, embedding: Array(768).fill(0.1) }] },+ { data: [{ index: 0, embedding: Array(768).fill(0.1) }] },+ { model: models[0].id, data: [] },+ { model: models[0].id, data: [{ index: 0, embedding: [0.1] }] },+ { model: models[0].id, data: [{ index: 0, embedding: Array(768).fill(null) }] },+ { model: models[0].id, data: [{ index: 0, embedding: Array(768).fill('0.1') }] },+ { model: models[0].id, data: [{ index: 1, embedding: Array(768).fill(0.1) }] },+ { error: { code: 'insufficient_credits' } },+ ])('rejects malformed vectors or substituted model identity', async body => {+ seed(); vi.spyOn(global, 'fetch').mockResolvedValue(json(body));+ await expect(runEmbeddings(models[0].id, ['hello'])).rejects.toMatchObject({ status: 502 });+ });++ it('rejects duplicate indexes even when the batch size matches', async () => {+ seed(); vi.spyOn(global, 'fetch').mockResolvedValue(json({ model: models[0].id,+ data: [{ index: 0, embedding: Array(768).fill(0.1) }, { index: 0, embedding: Array(768).fill(0.2) }] }));+ await expect(runEmbeddings(models[0].id, ['first', 'second'])).rejects.toMatchObject({ status: 502 });+ });++ it('rejects a dimension override before sending a request', async () => {+ seed(); const fetch = vi.spyOn(global, 'fetch');+ await expect(runEmbeddings(models[0].id, ['hello'], 1024)).rejects.toThrow('dimension override');+ expect(fetch).not.toHaveBeenCalled();+ });++ it('preserves upstream rate-limit retry hints', async () => {+ seed(); vi.spyOn(global, 'fetch').mockResolvedValue(new Response('upstream error', { status: 429, headers: { 'Retry-After': '45' } }));+ await expect(runEmbeddings(models[0].id, ['hello'])).rejects.toMatchObject({ status: 429, retryAfterMs: 45_000 });+ });++ it.each([401, 402, 503])('fails closed on upstream HTTP %i without returning vectors', async status => {+ seed(); vi.spyOn(global, 'fetch').mockResolvedValue(new Response('upstream error', { status }));+ // Preserve the existing family router's normalization of exhausted chains.+ await expect(runEmbeddings(models[0].id, ['hello'])).rejects.toMatchObject({+ status: 502, message: expect.stringContaining(`upstream ${status}`),+ });+ });++ it('does not seed rows before the signed catalog arrives', () => {+ expect(getDb().prepare("SELECT COUNT(*) AS n FROM embedding_models WHERE platform='llmtr'").get()).toEqual({ n: 0 });+ });+});diff --git a/server/src/services/embeddings.ts b/server/src/services/embeddings.tsindex 3790d84a..cf7a7cd8 100644--- a/server/src/services/embeddings.ts+++ b/server/src/services/embeddings.ts@@ -21,6 +21,7 @@ import { SPEKA_BASE_URL } from '../providers/speka.js';import { ELECTRONHUB_BASE_URL } from '../providers/electronhub.js';import { INFERBASE_BASE_URL } from '../providers/inferbase.js';import { AURAY_BASE_URL } from '../providers/auray.js';+import { LLMTR_BASE_URL } from '../providers/llmtr.js';export interface EmbeddingModelRow {id: number;@@ -121,6 +122,7 @@ export const EMBEDDING_PLATFORMS = new Set(['electronhub','inferbase','auray',+ 'llmtr',]);interface ProviderCallResult {@@ -164,7 +166,7 @@ async function openAiStyleEmbed(usage?: { prompt_tokens?: number; total_tokens?: number };};const data = [...(j.data ?? [])].sort((a, b) => (a.index ?? 0) - (b.index ?? 0));- if (['electronhub', 'inferbase', 'auray'].includes(platform) && (+ if (['electronhub', 'inferbase', 'auray', 'llmtr'].includes(platform) && (j.model !== modelId || data.length !== inputs.length ||data.some((d, i) => d.index !== i || !Array.isArray(d.embedding) ||d.embedding.length !== (dimensions ?? expectedDimensions) ||@@ -172,7 +174,7 @@ async function openAiStyleEmbed()) {// A successful HTTP response is not enough: wrong model/dimension or// malformed vectors would corrupt the caller's embedding family.- const provider = platform === 'electronhub' ? 'ElectronHub' : platform === 'auray' ? 'Auray' : 'Inferbase';+ const provider = platform === 'electronhub' ? 'ElectronHub' : platform === 'auray' ? 'Auray' : platform === 'llmtr' ? 'LLMTR' : 'Inferbase';throw new EmbeddingsError(`${provider} returned mismatched or malformed embeddings`, 502);}return {@@ -291,6 +293,13 @@ async function callProvider(row: EmbeddingModelRow, credential: ProviderCredentireturn openAiStyleEmbed(`${ELECTRONHUB_BASE_URL}/embeddings`, row.platform, key, row.model_id, inputs, { encoding_format: 'float' }, dimensions, row.dimensions);case 'inferbase':return openAiStyleEmbed(`${INFERBASE_BASE_URL}/embeddings`, row.platform, key, row.model_id, inputs, { encoding_format: 'float' }, dimensions, row.dimensions);+ case 'llmtr':+ // The catalog-approved models return fixed-width vectors. Do not send a+ // dimension override or accept a substituted model/embedding space.+ if (dimensions !== undefined && dimensions !== row.dimensions) {+ throw new EmbeddingsError('LLMTR does not support an embedding dimension override', 400);+ }+ return openAiStyleEmbed(`${LLMTR_BASE_URL}/embeddings`, row.platform, key, row.model_id, inputs, {}, undefined, row.dimensions);case 'auray':// Auray documents fixed-width vectors, not Matryoshka truncation. A cold// request can take longer than the generic 30-second embedding deadline.