🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(topics): support fixed-size embedding models (#18706)
Lysander Kieselb8ccc91458eb
diff --git a/packages/shared/src/server/topics/embeddings.ts b/packages/shared/src/server/topics/embeddings.tsindex fdc4475611..16a60bde4f 100644--- a/packages/shared/src/server/topics/embeddings.ts+++ b/packages/shared/src/server/topics/embeddings.ts@@ -1,25 +1,9 @@-import { embedLLMText } from "../llm/llmText";+import { getLLMErrorInfo } from "../llm/errors";+import { embedLLMText, type LLMProviderOptions } from "../llm/llmText";import { LLMAdapter } from "../llm/types";import { topicEmbeddingConfigSchema } from "../../topics";import type { TopicsModel } from "./model-config";-// The AI SDK validates the whole Bedrock options object, and each family names-// its size option differently, so only the family's own fields are sent.-function bedrockEmbeddingOptions(- modelId: string,- dimensions: number,-): Record<string, string | number> {- if (modelId.includes("nova"))- return { embeddingPurpose: "CLUSTERING", embeddingDimension: dimensions };- if (modelId.includes("cohere"))- return {- inputType: "clustering",- truncate: "NONE",- outputDimension: dimensions,- };- return { dimensions };-}-/** Embeds summaries through the project's LLM connection in as few requests as the provider allows. */export async function generateTopicEmbeddings(params: {model: TopicsModel;@@ -29,35 +13,68 @@ export async function generateTopicEmbeddings(params: {const { embeddingDimensions } = topicEmbeddingConfigSchema.parse({embeddingDimensions: params.dimensions,});- const result = await embedLLMText({- model: { adapter: params.model.adapter, id: params.model.model },- connection: params.model.connection,- values: params.summaries,- // Discovery and later centroid assignment must share one vector space, so- // every provider is asked for clustering vectors of the configured size.- providerOptions:- params.model.adapter === LLMAdapter.Bedrock- ? {- amazonBedrock: bedrockEmbeddingOptions(- params.model.model,- embeddingDimensions,- ),- }- : {- openai: { dimensions: embeddingDimensions },- google: {- taskType: "CLUSTERING",- outputDimensionality: embeddingDimensions,- },- vertex: {- taskType: "CLUSTERING",- outputDimensionality: embeddingDimensions,- },- },- maxRetries: 2,- // Single-text models (Bedrock Titan, Nova) split a batch into one request per text.- maxParallelCalls: 10,- abortSignal: AbortSignal.timeout(60_000),- });+ const result = await embedWithSizeFallback(params, embeddingDimensions);return { embeddings: result.embeddings, tokens: result.usage.tokens };}++async function embedWithSizeFallback(+ params: Parameters<typeof generateTopicEmbeddings>[0],+ embeddingDimensions: number,+) {+ const embed = (dimensions: number | undefined) =>+ embedLLMText({+ model: { adapter: params.model.adapter, id: params.model.model },+ connection: params.model.connection,+ values: params.summaries,+ providerOptions: embeddingProviderOptions(params.model, dimensions),+ maxRetries: 2,+ // Single-text models (Bedrock Titan, Nova) split a batch into one request per text.+ maxParallelCalls: 10,+ abortSignal: AbortSignal.timeout(60_000),+ });+ try {+ return await embed(embeddingDimensions);+ } catch (error) {+ const info = getLLMErrorInfo(error);+ if (info?.kind === "provider" && info.statusCode === 400)+ return embed(undefined);+ throw error;+ }+}++/** Keeps discovery and centroid assignment in the same clustering vector space. */+function embeddingProviderOptions(+ model: TopicsModel,+ dimensions: number | undefined,+): LLMProviderOptions {+ if (model.adapter === LLMAdapter.Bedrock)+ return { amazonBedrock: bedrockEmbeddingOptions(model.model, dimensions) };+ const google = {+ taskType: "CLUSTERING",+ ...(dimensions && { outputDimensionality: dimensions }),+ };+ return {+ openai: dimensions ? { dimensions } : {},+ google,+ vertex: google,+ };+}++/** The Bedrock SDK validates all options, so send only the model family's size field. */+function bedrockEmbeddingOptions(+ modelId: string,+ dimensions: number | undefined,+): Record<string, string | number> {+ if (modelId.includes("nova"))+ return {+ embeddingPurpose: "CLUSTERING",+ ...(dimensions && { embeddingDimension: dimensions }),+ };+ if (modelId.includes("cohere"))+ return {+ inputType: "clustering",+ truncate: "NONE",+ ...(dimensions && { outputDimension: dimensions }),+ };+ return dimensions ? { dimensions } : {};+}diff --git a/packages/shared/src/server/topics/text.test.ts b/packages/shared/src/server/topics/text.test.tsindex f6ed7ce1aa..c35d7b7241 100644--- a/packages/shared/src/server/topics/text.test.ts+++ b/packages/shared/src/server/topics/text.test.ts@@ -273,6 +273,87 @@ describe("generateTopicEmbeddings", () => {});expect(result.embeddings[0]).toHaveLength(768);});++ it("retries without the size option when a fixed-size model rejects it", async () => {+ const vector = Array.from({ length: 1536 }, () => 0.5);+ const fetch = vi.fn<typeof globalThis.fetch>(async (_url, init) =>+ JSON.parse(String(init?.body)).dimensions+ ? Response.json(+ {+ error: {+ message: "This model does not support specifying dimensions.",+ },+ },+ { status: 400 },+ )+ : Response.json({+ data: [{ embedding: vector, index: 0 }],+ usage: { prompt_tokens: 3 },+ }),+ );+ vi.stubGlobal("fetch", fetch);++ const result = await generateTopicEmbeddings({+ model: {+ slot: "embedding",+ provider: "openai-topics",+ adapter: LLMAdapter.OpenAI,+ model: "text-embedding-ada-002",+ connection: { secretKey: encrypt("sk-test") },+ },+ summaries: ["Account access"],+ dimensions: 1536,+ });++ expect(fetch).toHaveBeenCalledTimes(2);+ expect(await new Request(...fetch.mock.calls[1]).json()).not.toHaveProperty(+ "dimensions",+ );+ expect(result.embeddings[0]).toHaveLength(1536);+ });++ it("falls back when a rate-limit retry precedes the size rejection", async () => {+ const vector = Array.from({ length: 1536 }, () => 0.5);+ const fetch = vi.fn<typeof globalThis.fetch>(async (_url, init) => {+ if (fetch.mock.calls.length === 1)+ return Response.json(+ { error: { message: "Rate limit exceeded" } },+ { status: 429 },+ );+ if (JSON.parse(String(init?.body)).dimensions)+ return Response.json(+ {+ error: {+ message: "This model does not support specifying dimensions.",+ },+ },+ { status: 400 },+ );+ return Response.json({+ data: [{ embedding: vector, index: 0 }],+ usage: { prompt_tokens: 3 },+ });+ });+ vi.stubGlobal("fetch", fetch);++ const result = await generateTopicEmbeddings({+ model: {+ slot: "embedding",+ provider: "openai-topics",+ adapter: LLMAdapter.OpenAI,+ model: "text-embedding-ada-002",+ connection: { secretKey: encrypt("sk-test") },+ },+ summaries: ["Account access"],+ dimensions: 1536,+ });++ expect(fetch).toHaveBeenCalledTimes(3);+ expect(await new Request(...fetch.mock.calls[2]).json()).not.toHaveProperty(+ "dimensions",+ );+ expect(result.embeddings[0]).toHaveLength(1536);+ });});describe("allowlisted Cloud Bedrock runtime credentials", () => {diff --git a/worker/src/features/topics/README.md b/worker/src/features/topics/README.mdindex 9bf39deff9..6e9a378e3a 100644--- a/worker/src/features/topics/README.md+++ b/worker/src/features/topics/README.md@@ -113,7 +113,9 @@ On Bedrock, Cohere Embed v4 uses `eu.cohere.embed-v4:0` for EU routing or`us.cohere.embed-v4:0` for US routing. Avoid the global profile when geographicresidency matters. Every provider is asked for clustering vectors of theconfigured size (Cohere `clustering` input type with truncation disabled, Google-`CLUSTERING` task type, OpenAI `dimensions`). Default: 1,024+`CLUSTERING` task type, OpenAI `dimensions`). Fixed-size models such as `text-embedding-ada-002` reject+the size option; a request rejected with HTTP 400 is retried once without it, and+the vector must still have the configured length. Default: 1,024dimensions; any whole number from 64 to 4,096 is accepted, and a provider thatreturns a vector of another length fails the call. Embeddings are batched per trace: each trace'sapplicable facets are embedded in one call, in trace-batch and in embedding jobs