🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
import { expect, it } from "vitest";
import {
canLeaveTopicQuestions,
followsTopicRecommendations,
initialTopicSetupDraft,
isRecommendedTopicModel,
pickRecommendedTopicModel,
setupWaitMinutes,
topicSetupRun,
topicSetupStage,
topicConnectionCaption,
} from "./topic-setup-flow";
const connections = [
{
id: "bedrock",
provider: "Amazon Bedrock",
adapter: "bedrock",
customModels: [
"us.anthropic.claude-haiku-5-5",
"us.anthropic.claude-sonnet-5-5",
"us.cohere.embed-v4:0",
],
withDefaultModels: false,
},
];
it("recommends only the listed models for each job", () => {
expect(isRecommendedTopicModel("summary", "us.openai.gpt-6-luna")).toBe(true);
expect(
isRecommendedTopicModel("summary", "eu.anthropic.claude-haiku-4-5"),
).toBe(false);
expect(isRecommendedTopicModel("summary", "gemini-2.5-pro")).toBe(false);
expect(isRecommendedTopicModel("embedding", "us.cohere.embed-v4:0")).toBe(
true,
);
expect(
isRecommendedTopicModel("embedding", "amazon.titan-embed-text-v2:0"),
).toBe(false);
expect(
isRecommendedTopicModel("clustering", "us.anthropic.claude-sonnet-5-5"),
).toBe(true);
expect(
isRecommendedTopicModel("clustering", "us.anthropic.claude-opus-5-5"),
).toBe(false);
});
it("preselects the highest-ranked recommended model across connections", () => {
expect(
pickRecommendedTopicModel("summary", [
["Anthropic", ["claude-sonnet-5-5", "claude-haiku-5-5"]],
["Amazon Bedrock", ["us.openai.gpt-6-luna"]],
]),
).toEqual({ provider: "Amazon Bedrock", model: "us.openai.gpt-6-luna" });
expect(
pickRecommendedTopicModel("clustering", [
["OpenAI", ["gpt-6-sol", "gpt-6.1-sol", "claude-sonnet-5-5"]],
]),
).toEqual({ provider: "OpenAI", model: "gpt-6.1-sol" });
});
it("starts from recommended models and a 5 minute wait when the project has no topics setup", () => {
const draft = initialTopicSetupDraft({
stored: { summary: null, embedding: null, clustering: null },
connections,
facets: [],
enabledFacetIds: null,
traceIdleMs: null,
});
expect(draft.questions.every((question) => question.enabled)).toBe(true);
expect(
draft.questions.find((question) => question.name === "Intent")?.description,
).toBe("What the run was asked to do.");
expect(draft.summary?.model).toBe("us.anthropic.claude-haiku-5-5");
expect(draft.embedding?.model).toBe("us.cohere.embed-v4:0");
expect(draft.clustering?.model).toBe("us.anthropic.claude-sonnet-5-5");
expect(draft.waitMinutes).toBe(5);
expect(followsTopicRecommendations(draft)).toBe(true);
});
it("keeps a stored 10 minute idle time and a stored model", () => {
expect(setupWaitMinutes(600_000)).toBe(10);
expect(setupWaitMinutes(90_000)).toBe(5);
const draft = initialTopicSetupDraft({
stored: {
summary: {
provider: "Amazon Bedrock",
model: "eu.anthropic.claude-sonnet-4-5",
},
embedding: null,
clustering: null,
},
connections,
facets: [
{
id: "intent",
name: "Intent",
description: "Custom intent description.",
isBuiltIn: true,
},
],
enabledFacetIds: new Set(["intent"]),
traceIdleMs: 600_000,
});
expect(draft.summary?.model).toBe("eu.anthropic.claude-sonnet-4-5");
expect(draft.waitMinutes).toBe(10);
expect(
draft.questions.find((question) => question.name === "Intent")?.description,
).toBe("Custom intent description.");
expect(
draft.questions.find((question) => question.name === "Sentiment")?.enabled,
).toBe(false);
});
it("requires one question and all three models before the first step can save", () => {
const model = {
provider: "Amazon Bedrock",
model: "eu.anthropic.claude-haiku-4-5",
};
expect(
canLeaveTopicQuestions({
canWrite: true,
questions: [
{ enabled: false },
{ enabled: false },
{ enabled: false },
{ enabled: false },
],
summary: model,
embedding: model,
clustering: model,
}),
).toBe(false);
expect(
canLeaveTopicQuestions({
canWrite: true,
questions: [{ enabled: true }],
summary: model,
embedding: null,
clustering: model,
}),
).toBe(false);
});
it("follows a setup run from summaries through its topic update", () => {
const now = Date.parse("2026-10-22T08:00:00Z");
const process = (status: string, updatedAt = now) => ({
id: "setup",
status,
phase: status,
updatedAt: new Date(updatedAt).toISOString(),
input: {
operation: "process" as const,
requestId: "r",
thenUpdate: { timeRange: {} },
},
});
const update = (status: string, phase = status) => ({
id: "topics",
status,
phase,
updatedAt: new Date(now).toISOString(),
input: { operation: "update" as const, requestId: "setup-topics" },
});
const stage = (executions: Parameters<typeof topicSetupRun>[0]) =>
topicSetupStage(topicSetupRun(executions)!, now);
expect(topicSetupRun([update("completed")])).toBeNull();
expect(stage([process("running")]).stage).toBe("summarizing");
expect(stage([process("completed")]).stage).toBe("grouping");
expect(stage([process("completed", now - 5 * 60_000)])).toEqual({
stage: "failed",
retryId: null,
});
expect(
stage([update("running", "clustering"), process("completed")]).stage,
).toBe("grouping");
expect(stage([update("running", "naming"), process("completed")]).stage).toBe(
"naming",
);
expect(
stage([update("completed"), process("completed_with_errors")]),
).toEqual({ stage: "done", retryId: null });
expect(stage([process("failed")])).toEqual({
stage: "failed",
retryId: "setup",
});
expect(stage([update("failed"), process("completed")])).toEqual({
stage: "failed",
retryId: "topics",
});
});
it("names the connection when the project has one provider", () => {
expect(topicConnectionCaption(connections)).toBe("Amazon Bedrock");
expect(topicConnectionCaption([])).toBe("No AI connection");
});