🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(ci): make the model price audit validator accept real audit reports (#18376)
Hassieb Pakzad0b2b1235f8a4
diff --git a/.github/workflows/model-price-audit.yml b/.github/workflows/model-price-audit.ymlindex fa16b37e21..5b733ea7c8 100644--- a/.github/workflows/model-price-audit.yml+++ b/.github/workflows/model-price-audit.yml@@ -238,7 +238,7 @@ jobs:- Set `usageKeyCoverageConfirmed` to `yes` only after comparing the provider usage object, Langfuse ingestion normalization, and the provider usage-key matrix. Use `no` for a gap or unresolved capability and do not change that model entry.- Set `priceConfirmed` to `yes` only when an official source fetched during this run confirms the exact current prices. Otherwise set it to `no` and explain why in `comments`.- Set `tieringCorrect` to `yes` only when all applicable price tiers, thresholds, and tier-specific prices are confirmed. Use `not_applicable` only when the provider has no applicable tiering dimension, otherwise use `no` and explain the gap in `comments`.- - For every model removed from a selectable model array, add exactly one row with `change` set to `removed`, the provider's own retirement, shutdown, or model-listing URL in `officialSources`, and a comment naming the arrays it was removed from, its retirement status or date, and its successor if one exists. Price, usage-key, and tiering confirmations may be `no` for removed rows. A model listed in several arrays gets one row.+ - For every model removed from a selectable model array, add exactly one row with `change` set to `removed`, the provider's own retirement, shutdown, or model-listing URL in `officialSources`, and a comment naming the arrays it was removed from, its retirement status or date, and its successor if one exists. Price, usage-key, and tiering confirmations may be `no` for removed rows. A model listed in several arrays gets one row. Set `model` to the exact string from the selectable array, even when its pricing `modelName` is spelled differently (for example `claude-3-7-sonnet-20250219` in the array versus `claude-3.7-sonnet-20250219` in the pricing file), and set `provider` to `OpenAI`, `Anthropic`, or `Google` to match the array.- Put the official evidence URLs used for each model in that row. Comments should capture price units and conversions, tier thresholds, corrections, uncertainty, or why no change was made.- Treat every allowed repository edit as a diff, including an audit snapshot written to `model-audit-memory.md` when no model prices changed. If there is no repository diff at all, set `pullRequestTitle` to an empty string, say that no pricing changes were made, and list notable unresolved findings.- If there is a diff, set `pullRequestTitle` to a Conventional Commit title starting with `chore(pricing): ` that names the affected model families or the specific audit behavior changed, for example `chore(pricing): remove retired gemini-2.0 models from pickers`. Never use a generic title such as `update default model prices`.diff --git a/packages/shared/src/server/llm/types.ts b/packages/shared/src/server/llm/types.tsindex de479c968a..ebe57d9e48 100644--- a/packages/shared/src/server/llm/types.ts+++ b/packages/shared/src/server/llm/types.ts@@ -415,9 +415,9 @@ export type OpenAIModel = (typeof openAIModels)[number];// NOTE: Update docs page when changing this! https://langfuse.com/docs/prompt-management/features/playground#openai-playground--anthropic-playground// WARNING: The first entry in the array is chosen as the default model to add LLM API keysexport const anthropicModels = [- "claude-sonnet-4-5-20250929","claude-sonnet-5","claude-sonnet-5-5",+ "claude-sonnet-4-5-20250929","claude-fable-5","claude-fable-5-1","claude-mythos-5",diff --git a/scripts/model-price-audit/audit-output-contract.mjs b/scripts/model-price-audit/audit-output-contract.mjsindex 7d7eb9921d..e4f6b7ed5b 100644--- a/scripts/model-price-audit/audit-output-contract.mjs+++ b/scripts/model-price-audit/audit-output-contract.mjs@@ -89,6 +89,33 @@ const selectableModelArrays = [},];+// The audit reports free-form provider labels, and the pricing skill refers to+// Google as "Google Gemini (AI Studio)" and "Google Gemini (Vertex AI)".+const providerAliases = new Map([+ [+ "google",+ new Set([+ "google",+ "google gemini",+ "gemini",+ "google ai studio",+ "google gemini (ai studio)",+ "google vertex ai",+ "google gemini (vertex ai)",+ "vertex ai",+ ]),+ ],+]);++export const reportedProviderMatches = (reportedProvider, expectedProvider) => {+ const reported = normalize(reportedProvider).replace(/\s+/gu, " ");+ const expected = normalize(expectedProvider);+ return (+ reported === expected ||+ (providerAliases.get(expected)?.has(reported) ?? false)+ );+};+const readSelectableModelArrays = (source, label) =>new Map(selectableModelArrays.map(({ name, pattern, provider }) => {@@ -406,11 +433,24 @@ export function reconcileAuditOutput(expectedModelChanges.map((change) => [normalize(change.modelName), change]),);+ // A selectable model ID can also carry a standing finding from another+ // provider, such as an AWS Bedrock SKU note. Those informational rows do not+ // report the selectable-model change, so they are left out of the match.+ const rowsForChange = (change) => {+ const rows = rowsByModel.get(normalize(change.modelName)) ?? [];+ if (!change.provider || rows.length < 2) return rows;+ return rows.filter(+ (row) =>+ reportedProviderMatches(row.provider, change.provider) ||+ !["none", "unresolved"].includes(row.change),+ );+ };+for (const change of typeModelChanges) {- const matchingRows = rowsByModel.get(normalize(change.modelName)) ?? [];+ const matchingRows = rowsForChange(change);if (matchingRows.length === 1 &&- normalize(matchingRows[0].provider) !== normalize(change.provider)+ !reportedProviderMatches(matchingRows[0].provider, change.provider)) {throw new Error(`${change.arrayName} changes require provider ${change.provider}: ${change.modelName}`,@@ -419,7 +459,7 @@ export function reconcileAuditOutput(}for (const change of expectedChangesByModel.values()) {- const matchingRows = rowsByModel.get(normalize(change.modelName)) ?? [];+ const matchingRows = rowsForChange(change);if (matchingRows.length === 0) {throw new Error(`modelsChecked must report the actual ${change.expectedChange} model entry: ${change.modelName}`,diff --git a/scripts/model-price-audit/audit-output-contract.test.mjs b/scripts/model-price-audit/audit-output-contract.test.mjsindex b8f40db667..49372ed6f9 100644--- a/scripts/model-price-audit/audit-output-contract.test.mjs+++ b/scripts/model-price-audit/audit-output-contract.test.mjs@@ -408,6 +408,68 @@ test("removes a retired model listed in several selectable arrays", () => {assert.deepEqual(normalizedOutput.changedModels, [`${model} (removed)`]);});+test("accepts a Google provider alias for a Gemini selectable-model removal", () => {+ const model = "gemini-1";+ const { normalizedOutput, result } = runContract({+ baseTypes: typesSource({+ googleAIStudio: ["gemini-2", model],+ vertexAI: ["gemini-2", model],+ }),+ currentTypes: typesSource(),+ output: {+ ...structuredOutput([+ {+ ...retiredRow(model, {+ officialSources: ["https://ai.google.dev/gemini-api/docs/models"],+ }),+ provider: "Google Gemini",+ },+ ]),+ pullRequestTitle: "chore(pricing): remove retired gemini-1 from pickers",+ },+ typesDiff: `- "${model}",\n- "${model}",\n`,+ });++ assert.equal(result.status, 0, result.stderr);+ assert.deepEqual(normalizedOutput.changedModels, [`${model} (removed)`]);+});++test("ignores an unresolved other-provider note for a removed selectable model", () => {+ const model = "claude-3-5-sonnet-20240620";+ const anthropicSource =+ "https://platform.claude.com/docs/en/about-claude/model-deprecations";+ const { normalizedOutput, result } = runContract({+ baseTypes: typesSource({ anthropic: ["claude-sonnet-5", model] }),+ basePrices: [pricingEntry(model)],+ currentTypes: typesSource({ anthropic: ["claude-sonnet-5"] }),+ output: {+ ...structuredOutput([+ {+ ...retiredRow(model, { officialSources: [anthropicSource] }),+ provider: "Anthropic",+ },+ {+ ...retiredRow(model, {+ change: "unresolved",+ officialSources: ["https://aws.amazon.com/bedrock/pricing/"],+ comments: "Bedrock extended-access SKU is not representable.",+ }),+ provider: "AWS Bedrock",+ },+ ]),+ pullRequestTitle: "chore(pricing): remove retired claude-3-5-sonnet",+ },+ typesDiff: `- "${model}",\n`,+ });++ assert.equal(result.status, 0, result.stderr);+ assert.deepEqual(normalizedOutput.changedModels, [`${model} (removed)`]);+ const bedrockRow = normalizedOutput.modelsChecked.find(+ (row) => row.provider === "AWS Bedrock",+ );+ assert.equal(bedrockRow.change, "unresolved");+});+test("rejects a selectable-model removal without official evidence", () => {const model = "gpt-4o";const { result } = runContract({diff --git a/scripts/model-price-audit/prepare-metadata.mjs b/scripts/model-price-audit/prepare-metadata.mjsindex adb984673a..2a21a80fae 100644--- a/scripts/model-price-audit/prepare-metadata.mjs+++ b/scripts/model-price-audit/prepare-metadata.mjs@@ -7,6 +7,7 @@ import {collectTypeModelChanges,mergeModelChanges,normalizeReportedModel,+ reportedProviderMatches,validateChangedModelRow,validateOfficialSources,} from "./audit-output-contract.mjs";@@ -111,7 +112,7 @@ const changedRowsByModel = new Map();for (const change of typeModelChanges) {const row = changedRowsByModel.get(normalize(change.modelName));- if (row && normalize(row.provider) !== normalize(change.provider)) {+ if (row && !reportedProviderMatches(row.provider, change.provider)) {throw new Error(`${change.arrayName} changes require provider ${change.provider}: ${change.modelName}`,);