Browser-based graphical shell and control plane for a Linux server you own — real terminal, file manager, live metrics and BYOK AI in one mobile-first workspace. Next.js 16 · React 19 · node-pty · Tailwind v4 · shadcn/ui. Public alpha, self-hosted.
TypeScript browser-shellcontrol-planefile-managerhomelabmobile-firstnextjsptyreactself-hostedserver-managementshadcn-uisysadmintailwindcssterminaltypescriptvpsweb-desktopweb-terminal
fix(agent): compare normalized token proofs per row
Rahman6a0f21753958
diff --git a/scripts/bench-agent-cache.mjs b/scripts/bench-agent-cache.mjsindex bd0d7643..4f19356d 100755--- a/scripts/bench-agent-cache.mjs+++ b/scripts/bench-agent-cache.mjs@@ -4,7 +4,7 @@ import { existsSync, mkdirSync, readFileSync, rmSync } from "node:fs";import os from "node:os";import path from "node:path";import { spawnSync } from "node:child_process";-import { aggregateAgent, extractModelEvidence, extractUsage, modelFamily } from "./bench-agent-metrics.mjs";+import { aggregateAgent, extractModelEvidence, extractUsage, modelFamily, normalizedTokenUsageComparable } from "./bench-agent-metrics.mjs";import { normalizeCorpusSeed, rotateAgentOrder } from "./bench-agent-seed.mjs";const AGENTS = ["mso", "hermes"];@@ -91,7 +91,10 @@ export function summarizeCacheRows(rows, requested) {};});const evidenceComplete = rows.length > 0 && rows.every((row) => row.modelEvidence?.modelFamily === expectedFamily && row.modelEvidence?.provider === expectedProvider);- const tokenSemanticsComparable = aggregates.length >= 2 && evidenceComplete && aggregates.every((row) => row.tokenCoveragePct === 100 && row.tokenAccountingMode === "inclusive-input-output-total" && !["unknown", "mixed"].includes(row.tokenAccountingProof));+ const tokenSemanticsComparable = aggregates.length >= 2 && evidenceComplete && aggregates.every((aggregate) => {+ const agentRows = rows.filter((row) => row.agent === aggregate.agent);+ return aggregate.tokenCoveragePct === 100 && normalizedTokenUsageComparable(agentRows);+ });const costPairs = new Set(aggregates.map((row) => `${row.costStatus}:${row.costSource}`));const costSemanticsComparable = aggregates.length >= 2 && evidenceComplete && aggregates.every((row) => row.costCoveragePct === 100 && !["unknown", "mixed"].includes(row.costStatus) && !["unknown", "mixed"].includes(row.costSource)) && costPairs.size === 1;return {diff --git a/scripts/bench-agent-cache.test.ts b/scripts/bench-agent-cache.test.tsindex 16313bfa..759eb50a 100644--- a/scripts/bench-agent-cache.test.ts+++ b/scripts/bench-agent-cache.test.ts@@ -43,6 +43,13 @@ describe("P8 calibration", () => {expect(out.efficiencyComparability.costSemanticsComparable).toBe(false);});+ it("keeps token semantics comparable when one agent uses multiple independently valid accounting proofs", () => {+ const rows: any[] = [row("mso", 0, 0), row("hermes", 0, 0), row("mso", 1, 0), row("hermes", 1, 30)];+ rows[1].usage.accountingProof = "exact-input-output-identity";+ rows[3].usage.accountingProof = "exact-exclusive-cache-sum";+ expect(summarizeCacheRows(rows, { model: "x", provider: "openai-codex" }).efficiencyComparability.tokenSemanticsComparable).toBe(true);+ });+it("allows cost comparability only when both agents expose the same attributed contract", () => {const rows = [row("mso", 0, 0, true), row("hermes", 0, 30, true), row("mso", 1, 20, true), row("hermes", 1, 40, true)];expect(summarizeCacheRows(rows as any, { model: "x", provider: "openai-codex" }).efficiencyComparability.costSemanticsComparable).toBe(true);diff --git a/scripts/bench-agent-corpus.mjs b/scripts/bench-agent-corpus.mjsindex dfd47b2c..dc6da208 100644--- a/scripts/bench-agent-corpus.mjs+++ b/scripts/bench-agent-corpus.mjs@@ -5,7 +5,7 @@ import { existsSync, mkdirSync, readFileSync, rmSync } from "node:fs";import os from "node:os";import path from "node:path";import { spawnSync } from "node:child_process";-import { aggregateAgent, comparabilityLevel, eligibleRanking, extractModelEvidence, extractToolTelemetry, extractUsage, modelFamily } from "./bench-agent-metrics.mjs";+import { aggregateAgent, comparabilityLevel, eligibleRanking, extractModelEvidence, extractToolTelemetry, extractUsage, modelFamily, normalizedTokenUsageComparable } from "./bench-agent-metrics.mjs";import { createCorpus, scratchIsPrivate } from "./bench-agent-corpus-fixtures.mjs";export const CORPUS_VERSION = "mso-agent-quality-v2";@@ -70,10 +70,10 @@ export function summarizeCorpus(rows, requested) {const expectedScenarios = new Set(rows.map((row) => row.scenarioId)).size;const tokenModes = new Set(aggregates.map((row) => row.tokenAccountingMode));const tokenSemanticsComparable = aggregates.length >= 2- && aggregates.every((row) => row.tokenCoveragePct === 100- && row.tokenAccountingMode === "inclusive-input-output-total"- && !["unknown", "mixed"].includes(row.tokenAccountingProof))- && tokenModes.size === 1;+ && aggregates.every((aggregate) => {+ const agentRows = rows.filter((row) => row.agent === aggregate.agent);+ return aggregate.tokenCoveragePct === 100 && normalizedTokenUsageComparable(agentRows);+ });const costPairs = new Set(aggregates.map((row) => `${row.costStatus}:${row.costSource}`));const costSemanticsComparable = aggregates.length >= 2&& aggregates.every((row) => row.costCoveragePct === 100 && !["unknown", "mixed"].includes(row.costStatus) && !["unknown", "mixed"].includes(row.costSource))diff --git a/scripts/bench-agent-corpus.test.ts b/scripts/bench-agent-corpus.test.tsindex 89f76b59..ea1ef5cd 100644--- a/scripts/bench-agent-corpus.test.ts+++ b/scripts/bench-agent-corpus.test.ts@@ -198,6 +198,21 @@ describe("agent-quality corpus", () => {expect(summarizeCorpus(rows as any, { model: "x", provider: "p" }).efficiencyComparability.tokenSemanticsComparable).toBe(true);});+ it("keeps token semantics comparable when valid normalized proof shapes vary across scenarios", () => {+ const explicit = extractUsage({ inputTokens: 100, outputTokens: 20, totalTokens: 120, cacheReadTokens: 0, accountingMode: "inclusive-input-output-total" })!;+ const identity = extractUsage({ input_tokens: 100, output_tokens: 20, total_tokens: 120, cache_read_tokens: 0 })!;+ const cached = extractUsage({ input_tokens: 40, cache_read_tokens: 60, output_tokens: 20, total_tokens: 120 })!;+ const rows = [+ { agent: "a", scenarioId: "x", fullSuccess: true, taskSuccess: true, policyCompliant: true, latencyMs: 10, usage: explicit, modelEvidence: { modelFamily: "x", provider: "p" } },+ { agent: "b", scenarioId: "x", fullSuccess: true, taskSuccess: true, policyCompliant: true, latencyMs: 12, usage: identity, modelEvidence: { modelFamily: "x", provider: "p" } },+ { agent: "a", scenarioId: "y", fullSuccess: true, taskSuccess: true, policyCompliant: true, latencyMs: 11, usage: explicit, modelEvidence: { modelFamily: "x", provider: "p" } },+ { agent: "b", scenarioId: "y", fullSuccess: true, taskSuccess: true, policyCompliant: true, latencyMs: 13, usage: cached, modelEvidence: { modelFamily: "x", provider: "p" } },+ ];+ const out = summarizeCorpus(rows as any, { model: "x", provider: "p" });+ expect(out.aggregates.find((row: any) => row.agent === "b")?.tokenAccountingProof).toBe("mixed");+ expect(out.efficiencyComparability.tokenSemanticsComparable).toBe(true);+ });+it("can also normalize an exact representation where reasoning is outside output", () => {const usage = extractUsage({ input_tokens: 40, cache_read_tokens: 60, output_tokens: 13, reasoning_tokens: 7, total_tokens: 120 })!;expect(usage).toMatchObject({ reportedAccountingMode: "exclusive-cache-reasoning", accountingProof: "exact-exclusive-cache-reasoning-sum", normalizedInputTokens: 100, normalizedOutputTokens: 20, normalizedTotalTokens: 120 });diff --git a/scripts/bench-agent-metrics.mjs b/scripts/bench-agent-metrics.mjsindex 17f4a77a..ce28e972 100644--- a/scripts/bench-agent-metrics.mjs+++ b/scripts/bench-agent-metrics.mjs@@ -105,6 +105,17 @@ export function extractUsage(value) {});}+export function normalizedTokenUsageComparable(rows) {+ return rows.length > 0 && rows.every((row) => {+ const usage = row?.usage;+ return usage?.normalizedTotalTokens !== undefined+ && usage?.accountingMode === "inclusive-input-output-total"+ && typeof usage?.accountingProof === "string"+ && usage.accountingProof !== "none"+ && usage.accountingProof !== "unknown";+ });+}+/*** @param {Array<any>} rows* @param {string} requestedFamily