Skip to content

armgit / mso

Public Imported from GitHub
0

Browser-based graphical shell and control plane for a Linux server you own — real terminal, file manager, live metrics and BYOK AI in one mobile-first workspace. Next.js 16 · React 19 · node-pty · Tailwind v4 · shadcn/ui. Public alpha, self-hosted.

fix(agent): compare normalized token proofs per row

Rahman6a0f21753958
diff --git a/scripts/bench-agent-cache.mjs b/scripts/bench-agent-cache.mjs
index bd0d7643..4f19356d 100755
--- a/scripts/bench-agent-cache.mjs
+++ b/scripts/bench-agent-cache.mjs
@@ -4,7 +4,7 @@ import { existsSync, mkdirSync, readFileSync, rmSync } from "node:fs";
import os from "node:os";
import path from "node:path";
import { spawnSync } from "node:child_process";
-import { aggregateAgent, extractModelEvidence, extractUsage, modelFamily } from "./bench-agent-metrics.mjs";
+import { aggregateAgent, extractModelEvidence, extractUsage, modelFamily, normalizedTokenUsageComparable } from "./bench-agent-metrics.mjs";
import { normalizeCorpusSeed, rotateAgentOrder } from "./bench-agent-seed.mjs";
const AGENTS = ["mso", "hermes"];
@@ -91,7 +91,10 @@ export function summarizeCacheRows(rows, requested) {
};
});
const evidenceComplete = rows.length > 0 && rows.every((row) => row.modelEvidence?.modelFamily === expectedFamily && row.modelEvidence?.provider === expectedProvider);
- const tokenSemanticsComparable = aggregates.length >= 2 && evidenceComplete && aggregates.every((row) => row.tokenCoveragePct === 100 && row.tokenAccountingMode === "inclusive-input-output-total" && !["unknown", "mixed"].includes(row.tokenAccountingProof));
+ const tokenSemanticsComparable = aggregates.length >= 2 && evidenceComplete && aggregates.every((aggregate) => {
+ const agentRows = rows.filter((row) => row.agent === aggregate.agent);
+ return aggregate.tokenCoveragePct === 100 && normalizedTokenUsageComparable(agentRows);
+ });
const costPairs = new Set(aggregates.map((row) => `${row.costStatus}:${row.costSource}`));
const costSemanticsComparable = aggregates.length >= 2 && evidenceComplete && aggregates.every((row) => row.costCoveragePct === 100 && !["unknown", "mixed"].includes(row.costStatus) && !["unknown", "mixed"].includes(row.costSource)) && costPairs.size === 1;
return {
diff --git a/scripts/bench-agent-cache.test.ts b/scripts/bench-agent-cache.test.ts
index 16313bfa..759eb50a 100644
--- a/scripts/bench-agent-cache.test.ts
+++ b/scripts/bench-agent-cache.test.ts
@@ -43,6 +43,13 @@ describe("P8 calibration", () => {
expect(out.efficiencyComparability.costSemanticsComparable).toBe(false);
});
+ it("keeps token semantics comparable when one agent uses multiple independently valid accounting proofs", () => {
+ const rows: any[] = [row("mso", 0, 0), row("hermes", 0, 0), row("mso", 1, 0), row("hermes", 1, 30)];
+ rows[1].usage.accountingProof = "exact-input-output-identity";
+ rows[3].usage.accountingProof = "exact-exclusive-cache-sum";
+ expect(summarizeCacheRows(rows, { model: "x", provider: "openai-codex" }).efficiencyComparability.tokenSemanticsComparable).toBe(true);
+ });
+
it("allows cost comparability only when both agents expose the same attributed contract", () => {
const rows = [row("mso", 0, 0, true), row("hermes", 0, 30, true), row("mso", 1, 20, true), row("hermes", 1, 40, true)];
expect(summarizeCacheRows(rows as any, { model: "x", provider: "openai-codex" }).efficiencyComparability.costSemanticsComparable).toBe(true);
diff --git a/scripts/bench-agent-corpus.mjs b/scripts/bench-agent-corpus.mjs
index dfd47b2c..dc6da208 100644
--- a/scripts/bench-agent-corpus.mjs
+++ b/scripts/bench-agent-corpus.mjs
@@ -5,7 +5,7 @@ import { existsSync, mkdirSync, readFileSync, rmSync } from "node:fs";
import os from "node:os";
import path from "node:path";
import { spawnSync } from "node:child_process";
-import { aggregateAgent, comparabilityLevel, eligibleRanking, extractModelEvidence, extractToolTelemetry, extractUsage, modelFamily } from "./bench-agent-metrics.mjs";
+import { aggregateAgent, comparabilityLevel, eligibleRanking, extractModelEvidence, extractToolTelemetry, extractUsage, modelFamily, normalizedTokenUsageComparable } from "./bench-agent-metrics.mjs";
import { createCorpus, scratchIsPrivate } from "./bench-agent-corpus-fixtures.mjs";
export const CORPUS_VERSION = "mso-agent-quality-v2";
@@ -70,10 +70,10 @@ export function summarizeCorpus(rows, requested) {
const expectedScenarios = new Set(rows.map((row) => row.scenarioId)).size;
const tokenModes = new Set(aggregates.map((row) => row.tokenAccountingMode));
const tokenSemanticsComparable = aggregates.length >= 2
- && aggregates.every((row) => row.tokenCoveragePct === 100
- && row.tokenAccountingMode === "inclusive-input-output-total"
- && !["unknown", "mixed"].includes(row.tokenAccountingProof))
- && tokenModes.size === 1;
+ && aggregates.every((aggregate) => {
+ const agentRows = rows.filter((row) => row.agent === aggregate.agent);
+ return aggregate.tokenCoveragePct === 100 && normalizedTokenUsageComparable(agentRows);
+ });
const costPairs = new Set(aggregates.map((row) => `${row.costStatus}:${row.costSource}`));
const costSemanticsComparable = aggregates.length >= 2
&& aggregates.every((row) => row.costCoveragePct === 100 && !["unknown", "mixed"].includes(row.costStatus) && !["unknown", "mixed"].includes(row.costSource))
diff --git a/scripts/bench-agent-corpus.test.ts b/scripts/bench-agent-corpus.test.ts
index 89f76b59..ea1ef5cd 100644
--- a/scripts/bench-agent-corpus.test.ts
+++ b/scripts/bench-agent-corpus.test.ts
@@ -198,6 +198,21 @@ describe("agent-quality corpus", () => {
expect(summarizeCorpus(rows as any, { model: "x", provider: "p" }).efficiencyComparability.tokenSemanticsComparable).toBe(true);
});
+ it("keeps token semantics comparable when valid normalized proof shapes vary across scenarios", () => {
+ const explicit = extractUsage({ inputTokens: 100, outputTokens: 20, totalTokens: 120, cacheReadTokens: 0, accountingMode: "inclusive-input-output-total" })!;
+ const identity = extractUsage({ input_tokens: 100, output_tokens: 20, total_tokens: 120, cache_read_tokens: 0 })!;
+ const cached = extractUsage({ input_tokens: 40, cache_read_tokens: 60, output_tokens: 20, total_tokens: 120 })!;
+ const rows = [
+ { agent: "a", scenarioId: "x", fullSuccess: true, taskSuccess: true, policyCompliant: true, latencyMs: 10, usage: explicit, modelEvidence: { modelFamily: "x", provider: "p" } },
+ { agent: "b", scenarioId: "x", fullSuccess: true, taskSuccess: true, policyCompliant: true, latencyMs: 12, usage: identity, modelEvidence: { modelFamily: "x", provider: "p" } },
+ { agent: "a", scenarioId: "y", fullSuccess: true, taskSuccess: true, policyCompliant: true, latencyMs: 11, usage: explicit, modelEvidence: { modelFamily: "x", provider: "p" } },
+ { agent: "b", scenarioId: "y", fullSuccess: true, taskSuccess: true, policyCompliant: true, latencyMs: 13, usage: cached, modelEvidence: { modelFamily: "x", provider: "p" } },
+ ];
+ const out = summarizeCorpus(rows as any, { model: "x", provider: "p" });
+ expect(out.aggregates.find((row: any) => row.agent === "b")?.tokenAccountingProof).toBe("mixed");
+ expect(out.efficiencyComparability.tokenSemanticsComparable).toBe(true);
+ });
+
it("can also normalize an exact representation where reasoning is outside output", () => {
const usage = extractUsage({ input_tokens: 40, cache_read_tokens: 60, output_tokens: 13, reasoning_tokens: 7, total_tokens: 120 })!;
expect(usage).toMatchObject({ reportedAccountingMode: "exclusive-cache-reasoning", accountingProof: "exact-exclusive-cache-reasoning-sum", normalizedInputTokens: 100, normalizedOutputTokens: 20, normalizedTotalTokens: 120 });
diff --git a/scripts/bench-agent-metrics.mjs b/scripts/bench-agent-metrics.mjs
index 17f4a77a..ce28e972 100644
--- a/scripts/bench-agent-metrics.mjs
+++ b/scripts/bench-agent-metrics.mjs
@@ -105,6 +105,17 @@ export function extractUsage(value) {
});
}
+export function normalizedTokenUsageComparable(rows) {
+ return rows.length > 0 && rows.every((row) => {
+ const usage = row?.usage;
+ return usage?.normalizedTotalTokens !== undefined
+ && usage?.accountingMode === "inclusive-input-output-total"
+ && typeof usage?.accountingProof === "string"
+ && usage.accountingProof !== "none"
+ && usage.accountingProof !== "unknown";
+ });
+}
+
/**
* @param {Array<any>} rows
* @param {string} requestedFamily