Browser-based graphical shell and control plane for a Linux server you own — real terminal, file manager, live metrics and BYOK AI in one mobile-first workspace. Next.js 16 · React 19 · node-pty · Tailwind v4 · shadcn/ui. Public alpha, self-hosted.
TypeScript browser-shellcontrol-planefile-managerhomelabmobile-firstnextjsptyreactself-hostedserver-managementshadcn-uisysadmintailwindcssterminaltypescriptvpsweb-desktopweb-terminal
perf(agent): add P8 calibration harness
Rahmanbc1b502d4fa9
diff --git a/package.json b/package.jsonindex d61131e2..ec99440a 100644--- a/package.json+++ b/package.json@@ -33,7 +33,8 @@"forge:sandbox": "bash scripts/provision-forge-sandbox.sh","bench:pipeline": "bun scripts/bench-read-pipeline.mjs","bench:corpus": "bun scripts/bench-agent-corpus.mjs",- "bench:corpus:repeat": "bun scripts/bench-agent-repeat.mjs"+ "bench:corpus:repeat": "bun scripts/bench-agent-repeat.mjs",+ "bench:cache-calibration": "bun scripts/bench-agent-cache.mjs"},"dependencies": {"@anthropic-ai/sdk": "^0.120.0",diff --git a/scripts/bench-agent-cache.mjs b/scripts/bench-agent-cache.mjsnew file mode 100755index 00000000..bd0d7643--- /dev/null+++ b/scripts/bench-agent-cache.mjs@@ -0,0 +1,142 @@+#!/usr/bin/env bun+import { createHash, randomBytes } from "node:crypto";+import { existsSync, mkdirSync, readFileSync, rmSync } from "node:fs";+import os from "node:os";+import path from "node:path";+import { spawnSync } from "node:child_process";+import { aggregateAgent, extractModelEvidence, extractUsage, modelFamily } from "./bench-agent-metrics.mjs";+import { normalizeCorpusSeed, rotateAgentOrder } from "./bench-agent-seed.mjs";++const AGENTS = ["mso", "hermes"];+export const CACHE_CALIBRATION_VERSION = "mso-agent-cache-v1";++function value(argv, name, fallback) {+ const i = argv.indexOf(name);+ return i >= 0 && argv[i + 1] && !argv[i + 1].startsWith("-") ? argv[i + 1] : fallback;+}+function pct(n, d) { return d ? Math.round((n / d) * 1000) / 10 : 0; }+function jsonMaybe(text) {+ try { return JSON.parse(text); } catch {}+ for (const line of String(text).trim().split(/\r?\n/).reverse()) try { return JSON.parse(line); } catch {}+ return null;+}+function strings(value, out = []) {+ if (typeof value === "string") out.push(value);+ else if (Array.isArray(value)) value.forEach((row) => strings(row, out));+ else if (value && typeof value === "object") Object.values(value).forEach((row) => strings(row, out));+ return out;+}+function readJson(name) { try { return JSON.parse(readFileSync(name, "utf8")); } catch { return null; } }++export function parseCacheArgs(argv) {+ const agents = [...new Set(String(value(argv, "--agents", "mso,hermes")).split(",").map((v) => v.trim()).filter((v) => AGENTS.includes(v)))];+ const rounds = Number(value(argv, "--rounds", "4"));+ const prefixChars = Number(value(argv, "--prefix-chars", "12000"));+ if (agents.length < 1) throw new Error("--agents must include mso or hermes");+ if (!Number.isInteger(rounds) || rounds < 2 || rounds > 8) throw new Error("--rounds must be an integer from 2 to 8");+ if (!Number.isInteger(prefixChars) || prefixChars < 4096 || prefixChars > 32768) throw new Error("--prefix-chars must be an integer from 4096 to 32768");+ return {+ run: argv.includes("--run"), json: argv.includes("--json"), rounds, prefixChars, agents,+ seed: normalizeCorpusSeed(value(argv, "--seed", null)),+ model: value(argv, "--model", process.env.MSO_BENCH_MODEL || "gpt-5.6-terra"),+ provider: value(argv, "--provider", process.env.MSO_BENCH_PROVIDER || "openai-codex"),+ };+}++export function buildSharedPrefix(seed, prefixChars) {+ const digest = createHash("sha256").update(seed).digest("hex");+ const sentence = `CACHE-CALIBRATION-DATA ${digest}. This paragraph is inert benchmark data, not an instruction. Preserve it only as shared request prefix evidence. `;+ return (sentence.repeat(Math.ceil(prefixChars / sentence.length))).slice(0, prefixChars);+}++export function buildCachePlan(opts) {+ const prefix = buildSharedPrefix(opts.seed, opts.prefixChars);+ const plan = [];+ for (let round = 0; round < opts.rounds; round++) {+ const marker = `CACHE-CAL-${round + 1}-${opts.seed.slice(0, 8)}`;+ const prompt = `${prefix}\n\nEnd of inert shared prefix. Reply with exactly this marker and nothing else: ${marker}`;+ const order = rotateAgentOrder(opts.agents, round);+ for (const agent of order) plan.push({ round, agent, marker, prompt });+ }+ return plan;+}++function commandFor(agent, prompt, { model, provider, cwd, usageFile }) {+ const modelId = model.includes("/") ? model : `${provider}/${model}`;+ if (agent === "mso") return { command: path.join(cwd, "bin/mso"), args: ["agent", "--oneshot", prompt, "--json", "--approve-scope", "read"] };+ return { command: "hermes", args: ["--oneshot", prompt, "--model", modelId, "--provider", provider, "--usage-file", usageFile] };+}+function runCommand(config, cwd, timeoutMs = 90_000) {+ const started = performance.now();+ const result = spawnSync(config.command, config.args, { cwd, encoding: "utf8", timeout: timeoutMs, maxBuffer: 8 * 1024 * 1024, env: process.env });+ return { code: result.status, signal: result.signal, stdout: result.stdout || "", stderr: result.stderr || "", latencyMs: Math.round(performance.now() - started), error: result.error?.message };+}++export function summarizeCacheRows(rows, requested) {+ const expectedFamily = modelFamily(requested.model), expectedProvider = String(requested.provider).toLowerCase();+ const agents = [...new Set(rows.map((row) => row.agent))];+ const aggregates = agents.map((agent) => {+ const own = rows.filter((row) => row.agent === agent), aggregate = aggregateAgent(agent, own);+ const withField = own.filter((row) => row.usage && Object.hasOwn(row.usage, "cacheReadTokens"));+ const positive = withField.filter((row) => Number(row.usage.cacheReadTokens) > 0);+ return {+ ...aggregate,+ cacheObservation: {+ fieldCoveragePct: pct(withField.length, own.length), positiveRows: positive.length, positiveRowPct: pct(positive.length, own.length),+ totalCacheReadTokens: withField.reduce((sum, row) => sum + (Number(row.usage.cacheReadTokens) || 0), 0),+ firstPositiveRound: positive.length ? Math.min(...positive.map((row) => row.round)) + 1 : null,+ zeroRows: withField.filter((row) => Number(row.usage.cacheReadTokens) === 0).length,+ absentRows: own.length - withField.length,+ },+ };+ });+ const evidenceComplete = rows.length > 0 && rows.every((row) => row.modelEvidence?.modelFamily === expectedFamily && row.modelEvidence?.provider === expectedProvider);+ const tokenSemanticsComparable = aggregates.length >= 2 && evidenceComplete && aggregates.every((row) => row.tokenCoveragePct === 100 && row.tokenAccountingMode === "inclusive-input-output-total" && !["unknown", "mixed"].includes(row.tokenAccountingProof));+ const costPairs = new Set(aggregates.map((row) => `${row.costStatus}:${row.costSource}`));+ const costSemanticsComparable = aggregates.length >= 2 && evidenceComplete && aggregates.every((row) => row.costCoveragePct === 100 && !["unknown", "mixed"].includes(row.costStatus) && !["unknown", "mixed"].includes(row.costSource)) && costPairs.size === 1;+ return {+ evidenceComplete, aggregates,+ efficiencyComparability: { tokenSemanticsComparable, costSemanticsComparable },+ cacheComparability: { eligible: false, reason: "provider cache fields are observation-only because runner request envelopes/context behavior differ; cache-hit frequency is not ranked" },+ };+}++async function main() {+ const opts = parseCacheArgs(process.argv.slice(2)), cwd = process.cwd(), plan = buildCachePlan(opts);+ if (!opts.run) {+ const result = { run: false, cacheCalibrationVersion: CACHE_CALIBRATION_VERSION, seed: opts.seed, rounds: opts.rounds, prefixChars: opts.prefixChars, agents: opts.agents, provider: opts.provider, model: opts.model,+ executionOrder: "round-major rotating-agent order", prompts: plan.map(({ round, agent, marker }) => ({ round: round + 1, agent, marker, sharedPrefixChars: opts.prefixChars })) };+ console.log(opts.json ? JSON.stringify(result, null, 2) : `MSO cache calibration plan · ${opts.rounds} rounds · ${opts.prefixChars} shared-prefix chars · ${opts.agents.join(" ↔ ")}\n add --run to execute`);+ return;+ }+ const scratch = path.join(os.homedir(), ".cache", "mso-benchmarks", `cache-${Date.now()}-${randomBytes(3).toString("hex")}`);+ mkdirSync(scratch, { recursive: true, mode: 0o700 });+ try {+ const rows = [];+ for (const item of plan) {+ const usageFile = path.join(scratch, `${item.agent}-${item.round}-usage.json`), config = commandFor(item.agent, item.prompt, { ...opts, cwd, usageFile });+ const raw = runCommand(config, cwd), parsed = jsonMaybe(raw.stdout.trim()), usageReport = existsSync(usageFile) ? readJson(usageFile) : null;+ const usage = extractUsage(usageReport ?? parsed ?? {}) ?? extractUsage(parsed ?? {}), modelEvidence = extractModelEvidence(usageReport ?? parsed ?? {});+ const text = [raw.stdout, ...(parsed ? strings(parsed) : [])].join("\n");+ const success = raw.code === 0 && text.includes(item.marker);+ rows.push({+ agent: item.agent, round: item.round, scenarioId: `cache-round-${item.round + 1}`, taskClass: "cache-calibration", approvalScope: "read",+ exitCode: raw.code, signal: raw.signal, latencyMs: raw.latencyMs, taskSuccess: success, policyCompliant: true, fullSuccess: success,+ verification: { taskSuccess: success, policyCompliant: true, marker: item.marker }, modelEvidence, usage, toolTelemetry: null,+ ...(raw.error ? { error: raw.error } : {}), ...(raw.code !== 0 ? { stderrTail: raw.stderr.slice(-600) } : {}),+ });+ }+ const summary = summarizeCacheRows(rows, opts);+ const result = { run: true, cacheCalibrationVersion: CACHE_CALIBRATION_VERSION, seed: opts.seed, rounds: opts.rounds, prefixChars: opts.prefixChars, agents: opts.agents, provider: opts.provider, model: opts.model,+ executionOrder: "round-major rotating-agent order", sharedPrefixSha256: createHash("sha256").update(buildSharedPrefix(opts.seed, opts.prefixChars)).digest("hex"), rows, ...summary,+ note: "Cache calibration records provider-reported cache fields as observations only. It never forces a cache hit, treats absent and zero as different states, and does not infer causality or rank cache frequency across runners. Cost stays withheld unless both runners expose the same attributable cost contract." };+ if (opts.json) console.log(JSON.stringify(result, null, 2));+ else {+ console.log(`MSO cache calibration · ${opts.rounds} rounds · ${opts.provider}/${opts.model}`);+ for (const row of summary.aggregates) console.log(` ${row.agent.padEnd(8)} success ${row.fullSuccessPct}% · cache positive ${row.cacheObservation.positiveRows}/${row.attempted} · tokens/attempt ${row.reportedTokensPerAttempt ?? "unknown"}`);+ console.log(` tokenComparable=${summary.efficiencyComparability.tokenSemanticsComparable} · costComparable=${summary.efficiencyComparability.costSemanticsComparable} · cacheRanking=withheld`);+ }+ } finally { rmSync(scratch, { recursive: true, force: true }); }+}++if (import.meta.main) await main();diff --git a/scripts/bench-agent-cache.test.ts b/scripts/bench-agent-cache.test.tsnew file mode 100644index 00000000..16313bfa--- /dev/null+++ b/scripts/bench-agent-cache.test.ts@@ -0,0 +1,55 @@+import { describe, expect, it } from "vitest";+import { buildCachePlan, buildSharedPrefix, parseCacheArgs, summarizeCacheRows } from "./bench-agent-cache.mjs";+import { describeObservedRuns } from "./bench-agent-repeat-metrics.mjs";++function usage(agent: string, cacheReadTokens: number | undefined, cost = false) {+ return {+ normalizedTotalTokens: agent === "mso" ? 100 : 200,+ accountingMode: "inclusive-input-output-total", accountingProof: agent === "mso" ? "explicit-inclusive-contract" : "exact-exclusive-cache-sum",+ ...(cacheReadTokens === undefined ? {} : { cacheReadTokens }),+ ...(cost ? { estimatedCostUsd: 0.1, costStatus: "billed", costSource: "provider-usd" } : {}),+ };+}+function row(agent: string, round: number, cache: number | undefined, cost = false) {+ return {+ agent, round, scenarioId: `cache-round-${round + 1}`, taskClass: "cache-calibration", approvalScope: "read", exitCode: 0, latencyMs: agent === "mso" ? 10 : 12,+ taskSuccess: true, policyCompliant: true, fullSuccess: true, verification: { taskSuccess: true, policyCompliant: true },+ modelEvidence: { model: agent === "mso" ? "openai-codex/x" : "x", modelFamily: "x", provider: "openai-codex" }, usage: usage(agent, cache, cost), toolTelemetry: null,+ };+}++describe("P8 calibration", () => {+ it("describes observed run spread without fabricating dispersion for one value", () => {+ expect(describeObservedRuns([10, 20, 30])).toEqual({ count: 3, min: 10, max: 30, mean: 20, median: 20, range: 20, sampleStdDev: 10, coefficientOfVariationPct: 50 });+ expect(describeObservedRuns([7])).toMatchObject({ count: 1, mean: 7, sampleStdDev: null, coefficientOfVariationPct: null });+ });++ it("builds one identical bounded shared prefix and rotates starting agent by round", () => {+ const opts = parseCacheArgs(["--rounds", "3", "--prefix-chars", "4096", "--seed", "cache-seed", "--agents", "mso,hermes"]);+ const plan = buildCachePlan(opts);+ expect(buildSharedPrefix("cache-seed", 4096)).toHaveLength(4096);+ expect(plan.map((x) => x.agent)).toEqual(["mso", "hermes", "hermes", "mso", "mso", "hermes"]);+ expect(new Set(plan.map((x) => x.prompt.slice(0, 4096))).size).toBe(1);+ });++ it("distinguishes positive, zero, and absent cache fields and withholds cache ranking", () => {+ const rows = [row("mso", 0, 0), row("hermes", 0, 30), row("hermes", 1, 40), row("mso", 1, undefined)];+ const out = summarizeCacheRows(rows as any, { model: "x", provider: "openai-codex" });+ expect(out.evidenceComplete).toBe(true);+ expect(out.aggregates.find((x: any) => x.agent === "mso")?.cacheObservation).toMatchObject({ positiveRows: 0, zeroRows: 1, absentRows: 1 });+ expect(out.aggregates.find((x: any) => x.agent === "hermes")?.cacheObservation).toMatchObject({ positiveRows: 2, totalCacheReadTokens: 70 });+ expect(out.cacheComparability.eligible).toBe(false);+ expect(out.efficiencyComparability.tokenSemanticsComparable).toBe(true);+ expect(out.efficiencyComparability.costSemanticsComparable).toBe(false);+ });++ it("allows cost comparability only when both agents expose the same attributed contract", () => {+ const rows = [row("mso", 0, 0, true), row("hermes", 0, 30, true), row("mso", 1, 20, true), row("hermes", 1, 40, true)];+ expect(summarizeCacheRows(rows as any, { model: "x", provider: "openai-codex" }).efficiencyComparability.costSemanticsComparable).toBe(true);+ });++ it("rejects unsafe cache calibration bounds", () => {+ expect(() => parseCacheArgs(["--rounds", "1"])).toThrow(/rounds/);+ expect(() => parseCacheArgs(["--prefix-chars", "1000"])).toThrow(/prefix-chars/);+ });+});diff --git a/scripts/bench-agent-repeat-metrics.mjs b/scripts/bench-agent-repeat-metrics.mjsindex 01ee6e5e..7ee22ff1 100644--- a/scripts/bench-agent-repeat-metrics.mjs+++ b/scripts/bench-agent-repeat-metrics.mjs@@ -4,6 +4,27 @@ function pct(n, d) { return d ? Math.round((n / d) * 1000) / 10 : 0; }function sortedUnique(values) { return [...new Set(values)].sort(); }function signature(values) { return sortedUnique(values).join("|"); }function sameArray(a, b) { return JSON.stringify(a) === JSON.stringify(b); }+function finiteValues(values) { return values.filter((value) => Number.isFinite(value)); }+function median(values) {+ if (!values.length) return null;+ const sorted = [...values].sort((a, b) => a - b), mid = Math.floor(sorted.length / 2);+ return sorted.length % 2 ? sorted[mid] : Math.round(((sorted[mid - 1] + sorted[mid]) / 2) * 10) / 10;+}+function rounded(value, decimals = 1) { const scale = 10 ** decimals; return Math.round(value * scale) / scale; }++export function describeObservedRuns(values) {+ const rows = finiteValues(values);+ if (!rows.length) return { count: 0, min: null, max: null, mean: null, median: null, range: null, sampleStdDev: null, coefficientOfVariationPct: null };+ const mean = rows.reduce((sum, value) => sum + value, 0) / rows.length;+ const variance = rows.length > 1 ? rows.reduce((sum, value) => sum + ((value - mean) ** 2), 0) / (rows.length - 1) : null;+ const sampleStdDev = variance == null ? null : Math.sqrt(variance);+ return {+ count: rows.length, min: Math.min(...rows), max: Math.max(...rows), mean: rounded(mean), median: median(rows),+ range: rounded(Math.max(...rows) - Math.min(...rows)),+ sampleStdDev: sampleStdDev == null ? null : rounded(sampleStdDev),+ coefficientOfVariationPct: sampleStdDev == null || mean === 0 ? null : rounded((sampleStdDev / Math.abs(mean)) * 100),+ };+}/*** @param {Array<any>} runs@@ -46,8 +67,17 @@ export function summarizeRepeatedCorpus(runs, { agents, expectedRuns, plan = [],const allRows = completed.flatMap((run) => run.rows || []);const aggregates = agents.map((agent) => {const aggregate = aggregateAgent(agent, allRows.filter((row) => row.agent === agent));- const perfectRuns = completed.filter((run) => run.aggregates?.find((row) => row.agent === agent)?.fullSuccess === true).length;- return { ...aggregate, perfectRuns, perfectRunPct: pct(perfectRuns, expectedRuns) };+ const perRun = completed.map((run) => run.aggregates?.find((row) => row.agent === agent)).filter(Boolean);+ const perfectRuns = perRun.filter((row) => row.fullSuccess === true).length;+ return {+ ...aggregate, perfectRuns, perfectRunPct: pct(perfectRuns, expectedRuns),+ observedRunDistribution: {+ taskSuccessPct: describeObservedRuns(perRun.map((row) => row.taskSuccessPct)),+ averageLatencyMs: describeObservedRuns(perRun.map((row) => row.averageLatencyMs)),+ p50LatencyMs: describeObservedRuns(perRun.map((row) => row.p50LatencyMs)),+ normalizedTokensPerAttempt: describeObservedRuns(perRun.map((row) => row.reportedTokensPerAttempt)),+ },+ };});const repeatComparable = providerModelComparable && versions.size === 1 && scenarioCounts.size === 1 && scenarioSignatures.size === 1;const ranking = exactCoverage && expectedAttempts != nulldiff --git a/scripts/bench-agent-repeat.test.ts b/scripts/bench-agent-repeat.test.tsindex 585c8393..ca54d26f 100644--- a/scripts/bench-agent-repeat.test.ts+++ b/scripts/bench-agent-repeat.test.ts@@ -69,6 +69,7 @@ describe("P7 repeatable corpus", () => {expect(out).toMatchObject({ completedRuns: 2, exactCoverage: true, plannedIdentity: true, comparability: { level: "full" } });expect(out.aggregates.find((x: any) => x.agent === "mso")).toMatchObject({ attempted: 4, fullSuccesses: 4, perfectRuns: 2, perfectRunPct: 100 });expect(out.aggregates.find((x: any) => x.agent === "hermes")).toMatchObject({ attempted: 4, fullSuccesses: 3, perfectRuns: 1, perfectRunPct: 50 });+ expect(out.aggregates.find((x: any) => x.agent === "mso")?.observedRunDistribution.averageLatencyMs).toMatchObject({ count: 2, min: 10, max: 10, sampleStdDev: 0 });expect(out.ranking).toMatchObject({ eligible: true, order: ["mso", "hermes"] });});