Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

fix(pricing): validate added model audit metadata (#17029)

Hassieb Pakzad8a44d9e83d2b
diff --git a/.github/workflows/model-price-audit.yml b/.github/workflows/model-price-audit.yml
index 8603a3d6c1..af24478865 100644
--- a/.github/workflows/model-price-audit.yml
+++ b/.github/workflows/model-price-audit.yml
@@ -606,12 +606,8 @@ jobs:
esac
done
- CURRENT_PRICING_FILE="worker/src/constants/default-model-prices.json"
BASE_PRICING_FILE="$RUNNER_TEMP/default-model-prices-base.json"
- TYPES_DIFF_FILE="$RUNNER_TEMP/llm-types.diff"
- git show "HEAD:$CURRENT_PRICING_FILE" > "$BASE_PRICING_FILE"
- git diff --unified=0 -- "packages/shared/src/server/llm/types.ts" > "$TYPES_DIFF_FILE"
- export BASE_PRICING_FILE CHANGED_MODEL_TYPES CHANGED_PRICING_JSON CURRENT_PRICING_FILE TYPES_DIFF_FILE
+ git show "HEAD:worker/src/constants/default-model-prices.json" > "$BASE_PRICING_FILE"
mapfile -t untracked_files < <(git ls-files --others --exclude-standard)
if [ "${#untracked_files[@]}" -gt 0 ]; then
@@ -623,16 +619,6 @@ jobs:
node scripts/agents/sync-agent-shims.mjs --check
git diff --check -- "${changed_files[@]}"
- git config user.name "langfuse-bot"
- git config user.email "[email protected]"
- git -c core.hooksPath=/dev/null checkout -b "$BOT_BRANCH"
-
- STRUCTURED_OUTPUT_PATH="$RUNNER_TEMP/claude-model-price-audit.json"
- PR_TITLE_PATH="$RUNNER_TEMP/model-price-audit-pr-title.txt"
- export STRUCTURED_OUTPUT_PATH
- node scripts/model-price-audit/prepare-metadata.mjs > "$PR_TITLE_PATH"
- PR_TITLE="$(cat "$PR_TITLE_PATH")"
-
GIT_CONFIG_GLOBAL=/dev/null GIT_CONFIG_SYSTEM=/dev/null git -c core.hooksPath=/dev/null add -- "${changed_files[@]}"
mapfile -t staged_files < <(git diff --cached --name-only)
if [ "${#staged_files[@]}" -eq 0 ]; then
@@ -660,10 +646,24 @@ jobs:
staged_pricing_file="$RUNNER_TEMP/default-model-prices-staged.json"
git show ":worker/src/constants/default-model-prices.json" > "$staged_pricing_file"
+ git diff --cached --binary > "$RUNNER_TEMP/model-price-audit-diagnostic.patch"
+ test -s "$RUNNER_TEMP/model-price-audit-diagnostic.patch"
+ CURRENT_PRICING_FILE="$staged_pricing_file"
+ TYPES_DIFF_FILE="$RUNNER_TEMP/llm-types.diff"
+ git diff --cached --unified=0 -- "packages/shared/src/server/llm/types.ts" > "$TYPES_DIFF_FILE"
+
+ STRUCTURED_OUTPUT_PATH="$RUNNER_TEMP/claude-model-price-audit.json"
+ PR_TITLE_PATH="$RUNNER_TEMP/model-price-audit-pr-title.txt"
+ export BASE_PRICING_FILE CHANGED_MODEL_TYPES CHANGED_PRICING_JSON CURRENT_PRICING_FILE STRUCTURED_OUTPUT_PATH TYPES_DIFF_FILE
+ node scripts/model-price-audit/prepare-metadata.mjs > "$PR_TITLE_PATH"
+ PR_TITLE="$(cat "$PR_TITLE_PATH")"
+
node .agents/skills/add-model-price/scripts/validate-pricing-file.mjs \
"$staged_pricing_file" --base "$BASE_PRICING_FILE"
git diff --cached --check -- "${staged_files[@]}"
+ git config user.name "langfuse-bot"
+ git config user.email "[email protected]"
git -c core.hooksPath=/dev/null commit --no-verify -m "$PR_TITLE"
node .agents/skills/add-model-price/scripts/validate-pricing-file.mjs \
--base "$BASE_PRICING_FILE"
@@ -718,6 +718,17 @@ jobs:
if-no-files-found: error
retention-days: 1
+ - name: Upload failure diagnostics
+ if: failure()
+ uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+ with:
+ name: model-price-audit-failure-${{ github.run_attempt }}
+ path: |
+ ${{ runner.temp }}/model-price-audit-diagnostic.patch
+ ${{ runner.temp }}/claude-model-price-audit.txt
+ if-no-files-found: ignore
+ retention-days: 1
+
publish:
needs: audit
if: needs.audit.outputs.has_changes == 'true' && needs.audit.outputs.dry_run_mode == 'disabled'
diff --git a/scripts/model-price-audit/prepare-metadata.mjs b/scripts/model-price-audit/prepare-metadata.mjs
index 258ea7057f..3fba18281a 100644
--- a/scripts/model-price-audit/prepare-metadata.mjs
+++ b/scripts/model-price-audit/prepare-metadata.mjs
@@ -1,5 +1,6 @@
#!/usr/bin/env node
+import { createHash } from "node:crypto";
import fs from "node:fs";
process.on("uncaughtException", (error) => {
@@ -45,12 +46,16 @@ if (
);
}
-const basePrices = JSON.parse(
- fs.readFileSync(process.env.BASE_PRICING_FILE, "utf8"),
+const basePricingText = fs.readFileSync(
+ process.env.BASE_PRICING_FILE,
+ "utf8",
);
-const currentPrices = JSON.parse(
- fs.readFileSync(process.env.CURRENT_PRICING_FILE, "utf8"),
+const currentPricingText = fs.readFileSync(
+ process.env.CURRENT_PRICING_FILE,
+ "utf8",
);
+const basePrices = JSON.parse(basePricingText);
+const currentPrices = JSON.parse(currentPricingText);
const basePricesByName = new Map(
basePrices.map((item) => [normalize(item.modelName), item]),
);
@@ -58,9 +63,10 @@ const currentPricesByName = new Map(
currentPrices.map((item) => [normalize(item.modelName), item]),
);
const pricingModelChanges = [];
-for (const modelName of new Set(
- Array.from(basePricesByName.keys()).concat(currentPricesByName.keys()),
-)) {
+for (const modelName of new Set([
+ ...basePricesByName.keys(),
+ ...currentPricesByName.keys(),
+])) {
const before = basePricesByName.get(modelName);
const after = currentPricesByName.get(modelName);
if (JSON.stringify(before) !== JSON.stringify(after)) {
@@ -83,7 +89,13 @@ if (
process.env.CHANGED_PRICING_JSON === "true" &&
pricingModelChanges.length === 0
) {
- throw new Error("Pricing JSON changed without a concrete model-entry change");
+ const digest = (value) =>
+ createHash("sha256").update(value).digest("hex").slice(0, 12);
+ throw new Error(
+ "Pricing JSON changed without a concrete model-entry change " +
+ `(base entries: ${basePrices.length}, current entries: ${currentPrices.length}, ` +
+ `base sha256: ${digest(basePricingText)}, current sha256: ${digest(currentPricingText)})`,
+ );
}
if (process.env.CHANGED_MODEL_TYPES === "true" && typeModelChanges.size === 0) {
throw new Error(
diff --git a/scripts/model-price-audit/prepare-metadata.test.mjs b/scripts/model-price-audit/prepare-metadata.test.mjs
index c5591f9871..bcedc0b376 100644
--- a/scripts/model-price-audit/prepare-metadata.test.mjs
+++ b/scripts/model-price-audit/prepare-metadata.test.mjs
@@ -6,6 +6,10 @@ import { spawnSync } from "node:child_process";
import test from "node:test";
const scriptPath = path.join(import.meta.dirname, "prepare-metadata.mjs");
+const workflowPath = path.join(
+ import.meta.dirname,
+ "../../.github/workflows/model-price-audit.yml",
+);
function runValidator({
basePrices = [],
@@ -51,6 +55,24 @@ const changedPrices = {
after: [{ modelName, pricingTiers: [{ prices: { input: 10 } }] }],
};
+test("accepts an added pricing entry from the staged snapshot", () => {
+ const addedModel = {
+ modelName: "gpt-6-astra",
+ pricingTiers: [{ prices: { input: 10, output: 50 } }],
+ };
+ const result = runValidator({
+ basePrices: changedPrices.before,
+ currentPrices: [...changedPrices.before, addedModel],
+ changedPricingJson: true,
+ output: {
+ pullRequestTitle: "chore(pricing): add OpenAI gpt-6-astra pricing",
+ modelsChecked: [{ model: addedModel.modelName, change: "added" }],
+ },
+ });
+
+ assert.equal(result.status, 0, result.stderr);
+});
+
test("accepts a unique trailing match-alias annotation", () => {
const result = runValidator({
basePrices: changedPrices.before,
@@ -123,3 +145,28 @@ test("rejects a generic pull request title", () => {
/must identify the specific models or audit behavior changed/,
);
});
+
+test("validates metadata against the exact staged pricing blob", () => {
+ const workflow = fs.readFileSync(workflowPath, "utf8");
+ const prepareStep = workflow.slice(
+ workflow.indexOf(" - name: Prepare pull request artifact"),
+ workflow.indexOf(" - name: Upload pull request artifact"),
+ );
+ const stageIndex = prepareStep.indexOf(
+ 'git -c core.hooksPath=/dev/null add -- "${changed_files[@]}"',
+ );
+ const metadataIndex = prepareStep.indexOf(
+ "node scripts/model-price-audit/prepare-metadata.mjs",
+ );
+
+ assert.ok(stageIndex >= 0, "prepare step must stage the validated file list");
+ assert.ok(
+ stageIndex < metadataIndex,
+ "metadata validation must run after staging",
+ );
+ assert.match(
+ prepareStep,
+ /CURRENT_PRICING_FILE="\$staged_pricing_file"/,
+ );
+ assert.doesNotMatch(prepareStep, /checkout -b "\$BOT_BRANCH"/);
+});