🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(pricing): validate added model audit metadata (#17029)
Hassieb Pakzad8a44d9e83d2b
diff --git a/.github/workflows/model-price-audit.yml b/.github/workflows/model-price-audit.ymlindex 8603a3d6c1..af24478865 100644--- a/.github/workflows/model-price-audit.yml+++ b/.github/workflows/model-price-audit.yml@@ -606,12 +606,8 @@ jobs:esacdone- CURRENT_PRICING_FILE="worker/src/constants/default-model-prices.json"BASE_PRICING_FILE="$RUNNER_TEMP/default-model-prices-base.json"- TYPES_DIFF_FILE="$RUNNER_TEMP/llm-types.diff"- git show "HEAD:$CURRENT_PRICING_FILE" > "$BASE_PRICING_FILE"- git diff --unified=0 -- "packages/shared/src/server/llm/types.ts" > "$TYPES_DIFF_FILE"- export BASE_PRICING_FILE CHANGED_MODEL_TYPES CHANGED_PRICING_JSON CURRENT_PRICING_FILE TYPES_DIFF_FILE+ git show "HEAD:worker/src/constants/default-model-prices.json" > "$BASE_PRICING_FILE"mapfile -t untracked_files < <(git ls-files --others --exclude-standard)if [ "${#untracked_files[@]}" -gt 0 ]; then@@ -623,16 +619,6 @@ jobs:node scripts/agents/sync-agent-shims.mjs --checkgit diff --check -- "${changed_files[@]}"- git config user.name "langfuse-bot"- git config user.email "[email protected]"- git -c core.hooksPath=/dev/null checkout -b "$BOT_BRANCH"-- STRUCTURED_OUTPUT_PATH="$RUNNER_TEMP/claude-model-price-audit.json"- PR_TITLE_PATH="$RUNNER_TEMP/model-price-audit-pr-title.txt"- export STRUCTURED_OUTPUT_PATH- node scripts/model-price-audit/prepare-metadata.mjs > "$PR_TITLE_PATH"- PR_TITLE="$(cat "$PR_TITLE_PATH")"-GIT_CONFIG_GLOBAL=/dev/null GIT_CONFIG_SYSTEM=/dev/null git -c core.hooksPath=/dev/null add -- "${changed_files[@]}"mapfile -t staged_files < <(git diff --cached --name-only)if [ "${#staged_files[@]}" -eq 0 ]; then@@ -660,10 +646,24 @@ jobs:staged_pricing_file="$RUNNER_TEMP/default-model-prices-staged.json"git show ":worker/src/constants/default-model-prices.json" > "$staged_pricing_file"+ git diff --cached --binary > "$RUNNER_TEMP/model-price-audit-diagnostic.patch"+ test -s "$RUNNER_TEMP/model-price-audit-diagnostic.patch"+ CURRENT_PRICING_FILE="$staged_pricing_file"+ TYPES_DIFF_FILE="$RUNNER_TEMP/llm-types.diff"+ git diff --cached --unified=0 -- "packages/shared/src/server/llm/types.ts" > "$TYPES_DIFF_FILE"++ STRUCTURED_OUTPUT_PATH="$RUNNER_TEMP/claude-model-price-audit.json"+ PR_TITLE_PATH="$RUNNER_TEMP/model-price-audit-pr-title.txt"+ export BASE_PRICING_FILE CHANGED_MODEL_TYPES CHANGED_PRICING_JSON CURRENT_PRICING_FILE STRUCTURED_OUTPUT_PATH TYPES_DIFF_FILE+ node scripts/model-price-audit/prepare-metadata.mjs > "$PR_TITLE_PATH"+ PR_TITLE="$(cat "$PR_TITLE_PATH")"+node .agents/skills/add-model-price/scripts/validate-pricing-file.mjs \"$staged_pricing_file" --base "$BASE_PRICING_FILE"git diff --cached --check -- "${staged_files[@]}"+ git config user.name "langfuse-bot"+ git config user.email "[email protected]"git -c core.hooksPath=/dev/null commit --no-verify -m "$PR_TITLE"node .agents/skills/add-model-price/scripts/validate-pricing-file.mjs \--base "$BASE_PRICING_FILE"@@ -718,6 +718,17 @@ jobs:if-no-files-found: errorretention-days: 1+ - name: Upload failure diagnostics+ if: failure()+ uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1+ with:+ name: model-price-audit-failure-${{ github.run_attempt }}+ path: |+ ${{ runner.temp }}/model-price-audit-diagnostic.patch+ ${{ runner.temp }}/claude-model-price-audit.txt+ if-no-files-found: ignore+ retention-days: 1+publish:needs: auditif: needs.audit.outputs.has_changes == 'true' && needs.audit.outputs.dry_run_mode == 'disabled'diff --git a/scripts/model-price-audit/prepare-metadata.mjs b/scripts/model-price-audit/prepare-metadata.mjsindex 258ea7057f..3fba18281a 100644--- a/scripts/model-price-audit/prepare-metadata.mjs+++ b/scripts/model-price-audit/prepare-metadata.mjs@@ -1,5 +1,6 @@#!/usr/bin/env node+import { createHash } from "node:crypto";import fs from "node:fs";process.on("uncaughtException", (error) => {@@ -45,12 +46,16 @@ if ();}-const basePrices = JSON.parse(- fs.readFileSync(process.env.BASE_PRICING_FILE, "utf8"),+const basePricingText = fs.readFileSync(+ process.env.BASE_PRICING_FILE,+ "utf8",);-const currentPrices = JSON.parse(- fs.readFileSync(process.env.CURRENT_PRICING_FILE, "utf8"),+const currentPricingText = fs.readFileSync(+ process.env.CURRENT_PRICING_FILE,+ "utf8",);+const basePrices = JSON.parse(basePricingText);+const currentPrices = JSON.parse(currentPricingText);const basePricesByName = new Map(basePrices.map((item) => [normalize(item.modelName), item]),);@@ -58,9 +63,10 @@ const currentPricesByName = new Map(currentPrices.map((item) => [normalize(item.modelName), item]),);const pricingModelChanges = [];-for (const modelName of new Set(- Array.from(basePricesByName.keys()).concat(currentPricesByName.keys()),-)) {+for (const modelName of new Set([+ ...basePricesByName.keys(),+ ...currentPricesByName.keys(),+])) {const before = basePricesByName.get(modelName);const after = currentPricesByName.get(modelName);if (JSON.stringify(before) !== JSON.stringify(after)) {@@ -83,7 +89,13 @@ if (process.env.CHANGED_PRICING_JSON === "true" &&pricingModelChanges.length === 0) {- throw new Error("Pricing JSON changed without a concrete model-entry change");+ const digest = (value) =>+ createHash("sha256").update(value).digest("hex").slice(0, 12);+ throw new Error(+ "Pricing JSON changed without a concrete model-entry change " ++ `(base entries: ${basePrices.length}, current entries: ${currentPrices.length}, ` ++ `base sha256: ${digest(basePricingText)}, current sha256: ${digest(currentPricingText)})`,+ );}if (process.env.CHANGED_MODEL_TYPES === "true" && typeModelChanges.size === 0) {throw new Error(diff --git a/scripts/model-price-audit/prepare-metadata.test.mjs b/scripts/model-price-audit/prepare-metadata.test.mjsindex c5591f9871..bcedc0b376 100644--- a/scripts/model-price-audit/prepare-metadata.test.mjs+++ b/scripts/model-price-audit/prepare-metadata.test.mjs@@ -6,6 +6,10 @@ import { spawnSync } from "node:child_process";import test from "node:test";const scriptPath = path.join(import.meta.dirname, "prepare-metadata.mjs");+const workflowPath = path.join(+ import.meta.dirname,+ "../../.github/workflows/model-price-audit.yml",+);function runValidator({basePrices = [],@@ -51,6 +55,24 @@ const changedPrices = {after: [{ modelName, pricingTiers: [{ prices: { input: 10 } }] }],};+test("accepts an added pricing entry from the staged snapshot", () => {+ const addedModel = {+ modelName: "gpt-6-astra",+ pricingTiers: [{ prices: { input: 10, output: 50 } }],+ };+ const result = runValidator({+ basePrices: changedPrices.before,+ currentPrices: [...changedPrices.before, addedModel],+ changedPricingJson: true,+ output: {+ pullRequestTitle: "chore(pricing): add OpenAI gpt-6-astra pricing",+ modelsChecked: [{ model: addedModel.modelName, change: "added" }],+ },+ });++ assert.equal(result.status, 0, result.stderr);+});+test("accepts a unique trailing match-alias annotation", () => {const result = runValidator({basePrices: changedPrices.before,@@ -123,3 +145,28 @@ test("rejects a generic pull request title", () => {/must identify the specific models or audit behavior changed/,);});++test("validates metadata against the exact staged pricing blob", () => {+ const workflow = fs.readFileSync(workflowPath, "utf8");+ const prepareStep = workflow.slice(+ workflow.indexOf(" - name: Prepare pull request artifact"),+ workflow.indexOf(" - name: Upload pull request artifact"),+ );+ const stageIndex = prepareStep.indexOf(+ 'git -c core.hooksPath=/dev/null add -- "${changed_files[@]}"',+ );+ const metadataIndex = prepareStep.indexOf(+ "node scripts/model-price-audit/prepare-metadata.mjs",+ );++ assert.ok(stageIndex >= 0, "prepare step must stage the validated file list");+ assert.ok(+ stageIndex < metadataIndex,+ "metadata validation must run after staging",+ );+ assert.match(+ prepareStep,+ /CURRENT_PRICING_FILE="\$staged_pricing_file"/,+ );+ assert.doesNotMatch(prepareStep, /checkout -b "\$BOT_BRANCH"/);+});