Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

fix(pricing): ignore formatting-only audit diffs (#16971)

Hassieb Pakzad2ecf6a763dca
diff --git a/.github/workflows/model-price-audit.yml b/.github/workflows/model-price-audit.yml
index 00659f9ca6..e649d03b1b 100644
--- a/.github/workflows/model-price-audit.yml
+++ b/.github/workflows/model-price-audit.yml
@@ -503,6 +503,16 @@ jobs:
cat "$RUNNER_TEMP/claude-model-price-audit.txt"
} >> "$GITHUB_STEP_SUMMARY"
+ - name: Discard pricing-only formatting changes
+ run: |
+ set -euo pipefail
+
+ base_pricing_file="$RUNNER_TEMP/default-model-prices-before-audit.json"
+ git show "HEAD:worker/src/constants/default-model-prices.json" > "$base_pricing_file"
+ node scripts/model-price-audit/discard-formatting-only-pricing-diff.mjs \
+ --base "$base_pricing_file" \
+ --current worker/src/constants/default-model-prices.json
+
- name: Validate audit diff
run: |
set -euo pipefail
diff --git a/scripts/model-price-audit/discard-formatting-only-pricing-diff.mjs b/scripts/model-price-audit/discard-formatting-only-pricing-diff.mjs
new file mode 100644
index 0000000000..5e71aaf1c1
--- /dev/null
+++ b/scripts/model-price-audit/discard-formatting-only-pricing-diff.mjs
@@ -0,0 +1,61 @@
+#!/usr/bin/env node
+
+import fs from "node:fs";
+
+const args = process.argv.slice(2);
+const readArgument = (name) => {
+ const index = args.indexOf(name);
+ if (index === -1 || !args[index + 1]) {
+ throw new Error(`Missing required argument: ${name}`);
+ }
+ return args[index + 1];
+};
+
+const basePath = readArgument("--base");
+const currentPath = readArgument("--current");
+const base = JSON.parse(fs.readFileSync(basePath, "utf8"));
+const current = JSON.parse(fs.readFileSync(currentPath, "utf8"));
+
+const canonicalize = (value) => {
+ if (Array.isArray(value)) {
+ return value.map(canonicalize);
+ }
+ if (value !== null && typeof value === "object") {
+ return Object.fromEntries(
+ Object.keys(value)
+ .sort()
+ .map((key) => [key, canonicalize(value[key])]),
+ );
+ }
+ return value;
+};
+
+const entriesByModel = (entries) => {
+ const result = new Map();
+ for (const entry of entries) {
+ const modelName = entry.modelName.trim().toLowerCase();
+ if (result.has(modelName)) return null;
+ result.set(modelName, JSON.stringify(canonicalize(entry)));
+ }
+ return result;
+};
+
+const baseByModel = entriesByModel(base);
+const currentByModel = entriesByModel(current);
+const sameEntries =
+ baseByModel !== null &&
+ currentByModel !== null &&
+ baseByModel.size === currentByModel.size &&
+ [...baseByModel].every(
+ ([modelName, entry]) => currentByModel.get(modelName) === entry,
+ );
+
+if (
+ sameEntries &&
+ fs.readFileSync(basePath, "utf8") !== fs.readFileSync(currentPath, "utf8")
+) {
+ fs.copyFileSync(basePath, currentPath);
+ console.log(
+ "Discarded pricing JSON formatting or ordering changes without model-entry changes.",
+ );
+}
diff --git a/scripts/model-price-audit/discard-formatting-only-pricing-diff.test.mjs b/scripts/model-price-audit/discard-formatting-only-pricing-diff.test.mjs
new file mode 100644
index 0000000000..746a9184af
--- /dev/null
+++ b/scripts/model-price-audit/discard-formatting-only-pricing-diff.test.mjs
@@ -0,0 +1,83 @@
+import assert from "node:assert/strict";
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+import { spawnSync } from "node:child_process";
+import test from "node:test";
+
+const scriptPath = path.join(
+ import.meta.dirname,
+ "discard-formatting-only-pricing-diff.mjs",
+);
+
+function runCleanup(basePrices, currentPrices) {
+ const fixtureDirectory = fs.mkdtempSync(
+ path.join(os.tmpdir(), "model-price-audit-formatting-"),
+ );
+ const basePath = path.join(fixtureDirectory, "base.json");
+ const currentPath = path.join(fixtureDirectory, "current.json");
+ const baseText = JSON.stringify(basePrices, null, 2) + "\n";
+ const currentText = JSON.stringify(currentPrices, null, 4) + "\n";
+ fs.writeFileSync(basePath, baseText);
+ fs.writeFileSync(currentPath, currentText);
+
+ const result = spawnSync(
+ process.execPath,
+ [scriptPath, "--base", basePath, "--current", currentPath],
+ { encoding: "utf8" },
+ );
+ const finalText = fs.readFileSync(currentPath, "utf8");
+ fs.rmSync(fixtureDirectory, { force: true, recursive: true });
+ return { baseText, currentText, finalText, result };
+}
+
+const alpha = {
+ modelName: "alpha",
+ pricingTiers: [{ prices: { input: 1, output: 2 }, priority: 0 }],
+};
+const beta = {
+ modelName: "beta",
+ pricingTiers: [{ prices: { input: 3, output: 4 }, priority: 0 }],
+};
+
+test("restores the checked-in file after entry and object-key reordering", () => {
+ const reorderedAlpha = {
+ pricingTiers: [{ priority: 0, prices: { output: 2, input: 1 } }],
+ modelName: "alpha",
+ };
+ const { baseText, finalText, result } = runCleanup(
+ [alpha, beta],
+ [beta, reorderedAlpha],
+ );
+
+ assert.equal(result.status, 0, result.stderr);
+ assert.equal(finalText, baseText);
+ assert.match(
+ result.stdout,
+ /Discarded pricing JSON formatting or ordering changes/,
+ );
+});
+
+test("preserves a real pricing value change", () => {
+ const updatedAlpha = {
+ ...alpha,
+ pricingTiers: [{ prices: { input: 10, output: 2 }, priority: 0 }],
+ };
+ const { currentText, finalText, result } = runCleanup(
+ [alpha, beta],
+ [updatedAlpha, beta],
+ );
+
+ assert.equal(result.status, 0, result.stderr);
+ assert.equal(finalText, currentText);
+ assert.equal(result.stdout, "");
+});
+
+test("preserves nested array reordering as a potentially meaningful change", () => {
+ const base = [{ ...alpha, inputPriceKeys: ["input", "input_tokens"] }];
+ const current = [{ ...alpha, inputPriceKeys: ["input_tokens", "input"] }];
+ const { currentText, finalText, result } = runCleanup(base, current);
+
+ assert.equal(result.status, 0, result.stderr);
+ assert.equal(finalText, currentText);
+});