🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
import { describe, expect, it } from "vitest";
import {
getApiMigrationGuidance,
getCodingAgentName,
} from "./apiMigrationGuidance";
describe("getApiMigrationGuidance", () => {
it("provides bare Python trace migration methods", () => {
expect(
getApiMigrationGuidance("GET /api/public/traces/{id}", "python", "3.9.0"),
).toEqual({
currentMethod: "client.api.trace.get",
replacementMethod: "client.api.observations.get_many",
latestMajor: 5,
replacement: "GET /api/public/v2/observations",
minimumVersion: "4.0.0",
requiresUpgrade: true,
});
});
it("recommends the JS v6 scores method with the v5 Scores v3 client as fallback", () => {
expect(
getApiMigrationGuidance(
"GET /api/public/v2/scores",
"javascript",
"5.4.0",
),
).toMatchObject({
currentMethod: "client.api.scores.getMany",
replacementMethod: "client.api.scores.getMany",
previousMajorReplacementMethod: "client.api.scoresV3.getManyV3",
latestMajor: 6,
minimumVersion: "5.5.0",
requiresUpgrade: true,
});
});
it("recommends the Python v5 scores lookup by ID for the legacy score endpoint", () => {
expect(
getApiMigrationGuidance("GET /api/public/scores/{id}", "python", "4.8.1"),
).toMatchObject({
currentMethod: "client.api.scores.get_by_id",
replacementMethod: "client.api.scores.get_many(id=...)",
previousMajorReplacementMethod: "client.api.scores_v3.get_many_v3",
latestMajor: 5,
replacement: "GET /api/public/v3/scores",
requiresUpgrade: false,
});
});
it("recommends the JS v6 scores lookup by ID for the v2 score endpoint", () => {
expect(
getApiMigrationGuidance(
"GET /api/public/v2/scores/{id}",
"javascript",
"5.5.0",
),
).toMatchObject({
replacementMethod: "client.api.scores.getMany({ id })",
previousMajorReplacementMethod: "client.api.scoresV3.getManyV3",
});
});
it("uses the bare v3 route when migrating the v2 score endpoint", () => {
expect(
getApiMigrationGuidance(
"GET /api/public/v2/scores/{id}",
undefined,
undefined,
),
).toEqual({
replacement: "GET /api/public/v3/scores",
});
});
it("uses bare observation guidance for the legacy ID lookup", () => {
expect(
getApiMigrationGuidance(
"GET /api/public/observations/{id}",
"javascript",
"5.5.0",
),
).toMatchObject({
replacementMethod: "client.api.observations.getMany",
replacement: "GET /api/public/v2/observations",
});
});
it("points daily metrics at the v2 metrics endpoint", () => {
expect(
getApiMigrationGuidance(
"GET /api/public/metrics/daily",
undefined,
undefined,
),
).toEqual({
replacement: "GET /api/public/v2/metrics",
});
});
it("uses bare metrics guidance", () => {
expect(
getApiMigrationGuidance("GET /api/public/metrics", "python", "4.8.1"),
).toMatchObject({
replacementMethod: "client.api.metrics.metrics",
replacement: "GET /api/public/v2/metrics",
});
});
it("uses bare observation guidance when migrating session listing", () => {
expect(
getApiMigrationGuidance(
"GET /api/public/sessions",
"javascript",
"5.5.0",
),
).toMatchObject({
currentMethod: "client.api.sessions.list",
replacementMethod: "client.api.observations.getMany",
replacement: "GET /api/public/v2/observations",
});
});
it("uses bare observation guidance when migrating a session lookup", () => {
expect(
getApiMigrationGuidance(
"GET /api/public/sessions/{id}",
"python",
"4.8.1",
),
).toMatchObject({
currentMethod: "client.api.sessions.get",
replacementMethod: "client.api.observations.get_many",
replacement: "GET /api/public/v2/observations",
});
});
it("recognizes a v-prefixed outdated SDK version", () => {
expect(
getApiMigrationGuidance(
"GET /api/public/traces/{id}",
"python",
"v3.9.0",
),
).toMatchObject({ requiresUpgrade: true, minimumVersion: "4.0.0" });
});
it("shows the minimum SDK version when the attributed version is unknown", () => {
expect(
getApiMigrationGuidance(
"GET /api/public/v2/scores",
"javascript",
undefined,
),
).toMatchObject({ requiresUpgrade: true, minimumVersion: "5.5.0" });
});
it("returns REST guidance without inventing an SDK method", () => {
expect(
getApiMigrationGuidance("GET /api/public/traces", undefined, undefined),
).toEqual({
replacement: "GET /api/public/v2/observations",
});
});
it.each([
"GET /api/public/dataset-run-items",
"GET /api/public/datasets/{datasetName}/runs/{runName}",
])("uses the bare experiment-items method for %s", (entrypoint) => {
expect(
getApiMigrationGuidance(entrypoint, "javascript", "5.5.0"),
).toMatchObject({
replacementMethod: "client.api.experiments.listItems",
minimumVersion: "5.10.0",
requiresUpgrade: true,
});
});
it("uses the bare experiments method when listing experiments", () => {
expect(
getApiMigrationGuidance(
"GET /api/public/datasets/{datasetName}/runs",
"python",
"4.8.1",
),
).toMatchObject({
replacementMethod: "client.api.experiments.list",
minimumVersion: "4.13.1",
requiresUpgrade: true,
});
});
it.each([
["python", "4.13.1"],
["javascript", "5.10.0"],
] as const)(
"does not ask %s %s experiments callers to upgrade first",
(sdkName, sdkVersion) => {
expect(
getApiMigrationGuidance(
"GET /api/public/dataset-run-items",
sdkName,
sdkVersion,
),
).toMatchObject({ requiresUpgrade: false });
},
);
});
describe("getCodingAgentName", () => {
it.each([
["codex-cli/1.2.3", "Codex"],
["claude-code/1.0", "Claude Code"],
["cursor-agent/0.9", "Cursor"],
])("recognizes %s", (userAgent, expected) => {
expect(getCodingAgentName(userAgent)).toBe(expected);
});
});