🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(pricing): ignore formatting-only audit diffs (#16971)
Hassieb Pakzad2ecf6a763dca
diff --git a/.github/workflows/model-price-audit.yml b/.github/workflows/model-price-audit.ymlindex 00659f9ca6..e649d03b1b 100644--- a/.github/workflows/model-price-audit.yml+++ b/.github/workflows/model-price-audit.yml@@ -503,6 +503,16 @@ jobs:cat "$RUNNER_TEMP/claude-model-price-audit.txt"} >> "$GITHUB_STEP_SUMMARY"+ - name: Discard pricing-only formatting changes+ run: |+ set -euo pipefail++ base_pricing_file="$RUNNER_TEMP/default-model-prices-before-audit.json"+ git show "HEAD:worker/src/constants/default-model-prices.json" > "$base_pricing_file"+ node scripts/model-price-audit/discard-formatting-only-pricing-diff.mjs \+ --base "$base_pricing_file" \+ --current worker/src/constants/default-model-prices.json+- name: Validate audit diffrun: |set -euo pipefaildiff --git a/scripts/model-price-audit/discard-formatting-only-pricing-diff.mjs b/scripts/model-price-audit/discard-formatting-only-pricing-diff.mjsnew file mode 100644index 0000000000..5e71aaf1c1--- /dev/null+++ b/scripts/model-price-audit/discard-formatting-only-pricing-diff.mjs@@ -0,0 +1,61 @@+#!/usr/bin/env node++import fs from "node:fs";++const args = process.argv.slice(2);+const readArgument = (name) => {+ const index = args.indexOf(name);+ if (index === -1 || !args[index + 1]) {+ throw new Error(`Missing required argument: ${name}`);+ }+ return args[index + 1];+};++const basePath = readArgument("--base");+const currentPath = readArgument("--current");+const base = JSON.parse(fs.readFileSync(basePath, "utf8"));+const current = JSON.parse(fs.readFileSync(currentPath, "utf8"));++const canonicalize = (value) => {+ if (Array.isArray(value)) {+ return value.map(canonicalize);+ }+ if (value !== null && typeof value === "object") {+ return Object.fromEntries(+ Object.keys(value)+ .sort()+ .map((key) => [key, canonicalize(value[key])]),+ );+ }+ return value;+};++const entriesByModel = (entries) => {+ const result = new Map();+ for (const entry of entries) {+ const modelName = entry.modelName.trim().toLowerCase();+ if (result.has(modelName)) return null;+ result.set(modelName, JSON.stringify(canonicalize(entry)));+ }+ return result;+};++const baseByModel = entriesByModel(base);+const currentByModel = entriesByModel(current);+const sameEntries =+ baseByModel !== null &&+ currentByModel !== null &&+ baseByModel.size === currentByModel.size &&+ [...baseByModel].every(+ ([modelName, entry]) => currentByModel.get(modelName) === entry,+ );++if (+ sameEntries &&+ fs.readFileSync(basePath, "utf8") !== fs.readFileSync(currentPath, "utf8")+) {+ fs.copyFileSync(basePath, currentPath);+ console.log(+ "Discarded pricing JSON formatting or ordering changes without model-entry changes.",+ );+}diff --git a/scripts/model-price-audit/discard-formatting-only-pricing-diff.test.mjs b/scripts/model-price-audit/discard-formatting-only-pricing-diff.test.mjsnew file mode 100644index 0000000000..746a9184af--- /dev/null+++ b/scripts/model-price-audit/discard-formatting-only-pricing-diff.test.mjs@@ -0,0 +1,83 @@+import assert from "node:assert/strict";+import fs from "node:fs";+import os from "node:os";+import path from "node:path";+import { spawnSync } from "node:child_process";+import test from "node:test";++const scriptPath = path.join(+ import.meta.dirname,+ "discard-formatting-only-pricing-diff.mjs",+);++function runCleanup(basePrices, currentPrices) {+ const fixtureDirectory = fs.mkdtempSync(+ path.join(os.tmpdir(), "model-price-audit-formatting-"),+ );+ const basePath = path.join(fixtureDirectory, "base.json");+ const currentPath = path.join(fixtureDirectory, "current.json");+ const baseText = JSON.stringify(basePrices, null, 2) + "\n";+ const currentText = JSON.stringify(currentPrices, null, 4) + "\n";+ fs.writeFileSync(basePath, baseText);+ fs.writeFileSync(currentPath, currentText);++ const result = spawnSync(+ process.execPath,+ [scriptPath, "--base", basePath, "--current", currentPath],+ { encoding: "utf8" },+ );+ const finalText = fs.readFileSync(currentPath, "utf8");+ fs.rmSync(fixtureDirectory, { force: true, recursive: true });+ return { baseText, currentText, finalText, result };+}++const alpha = {+ modelName: "alpha",+ pricingTiers: [{ prices: { input: 1, output: 2 }, priority: 0 }],+};+const beta = {+ modelName: "beta",+ pricingTiers: [{ prices: { input: 3, output: 4 }, priority: 0 }],+};++test("restores the checked-in file after entry and object-key reordering", () => {+ const reorderedAlpha = {+ pricingTiers: [{ priority: 0, prices: { output: 2, input: 1 } }],+ modelName: "alpha",+ };+ const { baseText, finalText, result } = runCleanup(+ [alpha, beta],+ [beta, reorderedAlpha],+ );++ assert.equal(result.status, 0, result.stderr);+ assert.equal(finalText, baseText);+ assert.match(+ result.stdout,+ /Discarded pricing JSON formatting or ordering changes/,+ );+});++test("preserves a real pricing value change", () => {+ const updatedAlpha = {+ ...alpha,+ pricingTiers: [{ prices: { input: 10, output: 2 }, priority: 0 }],+ };+ const { currentText, finalText, result } = runCleanup(+ [alpha, beta],+ [updatedAlpha, beta],+ );++ assert.equal(result.status, 0, result.stderr);+ assert.equal(finalText, currentText);+ assert.equal(result.stdout, "");+});++test("preserves nested array reordering as a potentially meaningful change", () => {+ const base = [{ ...alpha, inputPriceKeys: ["input", "input_tokens"] }];+ const current = [{ ...alpha, inputPriceKeys: ["input_tokens", "input"] }];+ const { currentText, finalText, result } = runCleanup(base, current);++ assert.equal(result.status, 0, result.stderr);+ assert.equal(finalText, currentText);+});