Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

fix(topics): support fixed-size embedding models (#18706)

Lysander Kieselb8ccc91458eb
diff --git a/packages/shared/src/server/topics/embeddings.ts b/packages/shared/src/server/topics/embeddings.ts
index fdc4475611..16a60bde4f 100644
--- a/packages/shared/src/server/topics/embeddings.ts
+++ b/packages/shared/src/server/topics/embeddings.ts
@@ -1,25 +1,9 @@
-import { embedLLMText } from "../llm/llmText";
+import { getLLMErrorInfo } from "../llm/errors";
+import { embedLLMText, type LLMProviderOptions } from "../llm/llmText";
import { LLMAdapter } from "../llm/types";
import { topicEmbeddingConfigSchema } from "../../topics";
import type { TopicsModel } from "./model-config";
-// The AI SDK validates the whole Bedrock options object, and each family names
-// its size option differently, so only the family's own fields are sent.
-function bedrockEmbeddingOptions(
- modelId: string,
- dimensions: number,
-): Record<string, string | number> {
- if (modelId.includes("nova"))
- return { embeddingPurpose: "CLUSTERING", embeddingDimension: dimensions };
- if (modelId.includes("cohere"))
- return {
- inputType: "clustering",
- truncate: "NONE",
- outputDimension: dimensions,
- };
- return { dimensions };
-}
-
/** Embeds summaries through the project's LLM connection in as few requests as the provider allows. */
export async function generateTopicEmbeddings(params: {
model: TopicsModel;
@@ -29,35 +13,68 @@ export async function generateTopicEmbeddings(params: {
const { embeddingDimensions } = topicEmbeddingConfigSchema.parse({
embeddingDimensions: params.dimensions,
});
- const result = await embedLLMText({
- model: { adapter: params.model.adapter, id: params.model.model },
- connection: params.model.connection,
- values: params.summaries,
- // Discovery and later centroid assignment must share one vector space, so
- // every provider is asked for clustering vectors of the configured size.
- providerOptions:
- params.model.adapter === LLMAdapter.Bedrock
- ? {
- amazonBedrock: bedrockEmbeddingOptions(
- params.model.model,
- embeddingDimensions,
- ),
- }
- : {
- openai: { dimensions: embeddingDimensions },
- google: {
- taskType: "CLUSTERING",
- outputDimensionality: embeddingDimensions,
- },
- vertex: {
- taskType: "CLUSTERING",
- outputDimensionality: embeddingDimensions,
- },
- },
- maxRetries: 2,
- // Single-text models (Bedrock Titan, Nova) split a batch into one request per text.
- maxParallelCalls: 10,
- abortSignal: AbortSignal.timeout(60_000),
- });
+ const result = await embedWithSizeFallback(params, embeddingDimensions);
return { embeddings: result.embeddings, tokens: result.usage.tokens };
}
+
+async function embedWithSizeFallback(
+ params: Parameters<typeof generateTopicEmbeddings>[0],
+ embeddingDimensions: number,
+) {
+ const embed = (dimensions: number | undefined) =>
+ embedLLMText({
+ model: { adapter: params.model.adapter, id: params.model.model },
+ connection: params.model.connection,
+ values: params.summaries,
+ providerOptions: embeddingProviderOptions(params.model, dimensions),
+ maxRetries: 2,
+ // Single-text models (Bedrock Titan, Nova) split a batch into one request per text.
+ maxParallelCalls: 10,
+ abortSignal: AbortSignal.timeout(60_000),
+ });
+ try {
+ return await embed(embeddingDimensions);
+ } catch (error) {
+ const info = getLLMErrorInfo(error);
+ if (info?.kind === "provider" && info.statusCode === 400)
+ return embed(undefined);
+ throw error;
+ }
+}
+
+/** Keeps discovery and centroid assignment in the same clustering vector space. */
+function embeddingProviderOptions(
+ model: TopicsModel,
+ dimensions: number | undefined,
+): LLMProviderOptions {
+ if (model.adapter === LLMAdapter.Bedrock)
+ return { amazonBedrock: bedrockEmbeddingOptions(model.model, dimensions) };
+ const google = {
+ taskType: "CLUSTERING",
+ ...(dimensions && { outputDimensionality: dimensions }),
+ };
+ return {
+ openai: dimensions ? { dimensions } : {},
+ google,
+ vertex: google,
+ };
+}
+
+/** The Bedrock SDK validates all options, so send only the model family's size field. */
+function bedrockEmbeddingOptions(
+ modelId: string,
+ dimensions: number | undefined,
+): Record<string, string | number> {
+ if (modelId.includes("nova"))
+ return {
+ embeddingPurpose: "CLUSTERING",
+ ...(dimensions && { embeddingDimension: dimensions }),
+ };
+ if (modelId.includes("cohere"))
+ return {
+ inputType: "clustering",
+ truncate: "NONE",
+ ...(dimensions && { outputDimension: dimensions }),
+ };
+ return dimensions ? { dimensions } : {};
+}
diff --git a/packages/shared/src/server/topics/text.test.ts b/packages/shared/src/server/topics/text.test.ts
index f6ed7ce1aa..c35d7b7241 100644
--- a/packages/shared/src/server/topics/text.test.ts
+++ b/packages/shared/src/server/topics/text.test.ts
@@ -273,6 +273,87 @@ describe("generateTopicEmbeddings", () => {
});
expect(result.embeddings[0]).toHaveLength(768);
});
+
+ it("retries without the size option when a fixed-size model rejects it", async () => {
+ const vector = Array.from({ length: 1536 }, () => 0.5);
+ const fetch = vi.fn<typeof globalThis.fetch>(async (_url, init) =>
+ JSON.parse(String(init?.body)).dimensions
+ ? Response.json(
+ {
+ error: {
+ message: "This model does not support specifying dimensions.",
+ },
+ },
+ { status: 400 },
+ )
+ : Response.json({
+ data: [{ embedding: vector, index: 0 }],
+ usage: { prompt_tokens: 3 },
+ }),
+ );
+ vi.stubGlobal("fetch", fetch);
+
+ const result = await generateTopicEmbeddings({
+ model: {
+ slot: "embedding",
+ provider: "openai-topics",
+ adapter: LLMAdapter.OpenAI,
+ model: "text-embedding-ada-002",
+ connection: { secretKey: encrypt("sk-test") },
+ },
+ summaries: ["Account access"],
+ dimensions: 1536,
+ });
+
+ expect(fetch).toHaveBeenCalledTimes(2);
+ expect(await new Request(...fetch.mock.calls[1]).json()).not.toHaveProperty(
+ "dimensions",
+ );
+ expect(result.embeddings[0]).toHaveLength(1536);
+ });
+
+ it("falls back when a rate-limit retry precedes the size rejection", async () => {
+ const vector = Array.from({ length: 1536 }, () => 0.5);
+ const fetch = vi.fn<typeof globalThis.fetch>(async (_url, init) => {
+ if (fetch.mock.calls.length === 1)
+ return Response.json(
+ { error: { message: "Rate limit exceeded" } },
+ { status: 429 },
+ );
+ if (JSON.parse(String(init?.body)).dimensions)
+ return Response.json(
+ {
+ error: {
+ message: "This model does not support specifying dimensions.",
+ },
+ },
+ { status: 400 },
+ );
+ return Response.json({
+ data: [{ embedding: vector, index: 0 }],
+ usage: { prompt_tokens: 3 },
+ });
+ });
+ vi.stubGlobal("fetch", fetch);
+
+ const result = await generateTopicEmbeddings({
+ model: {
+ slot: "embedding",
+ provider: "openai-topics",
+ adapter: LLMAdapter.OpenAI,
+ model: "text-embedding-ada-002",
+ connection: { secretKey: encrypt("sk-test") },
+ },
+ summaries: ["Account access"],
+ dimensions: 1536,
+ });
+
+ expect(fetch).toHaveBeenCalledTimes(3);
+ expect(await new Request(...fetch.mock.calls[2]).json()).not.toHaveProperty(
+ "dimensions",
+ );
+ expect(result.embeddings[0]).toHaveLength(1536);
+ });
});
describe("allowlisted Cloud Bedrock runtime credentials", () => {
diff --git a/worker/src/features/topics/README.md b/worker/src/features/topics/README.md
index 9bf39deff9..6e9a378e3a 100644
--- a/worker/src/features/topics/README.md
+++ b/worker/src/features/topics/README.md
@@ -113,7 +113,9 @@ On Bedrock, Cohere Embed v4 uses `eu.cohere.embed-v4:0` for EU routing or
`us.cohere.embed-v4:0` for US routing. Avoid the global profile when geographic
residency matters. Every provider is asked for clustering vectors of the
configured size (Cohere `clustering` input type with truncation disabled, Google
-`CLUSTERING` task type, OpenAI `dimensions`). Default: 1,024
+`CLUSTERING` task type, OpenAI `dimensions`). Fixed-size models such as `text-embedding-ada-002` reject
+the size option; a request rejected with HTTP 400 is retried once without it, and
+the vector must still have the configured length. Default: 1,024
dimensions; any whole number from 64 to 4,096 is accepted, and a provider that
returns a vector of another length fails the call. Embeddings are batched per trace: each trace's
applicable facets are embedded in one call, in trace-batch and in embedding jobs