Skip to content

armgit / mso

Public Imported from GitHub
0

Browser-based graphical shell and control plane for a Linux server you own — real terminal, file manager, live metrics and BYOK AI in one mobile-first workspace. Next.js 16 · React 19 · node-pty · Tailwind v4 · shadcn/ui. Public alpha, self-hosted.

fix(perf): compare TBT only on the same machine, not just the same load

Abdurrahman Fakhrulc5667ac2af26
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index e20e12e7..bf35957b 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -15,6 +15,7 @@ this is the *what*, and it is what Settings → Account → About shows as “Wh
**Fixed**
+- `perf` compare TBT only on the same machine, not just the same load
- `shell, gates` restore the artwork reveal rule, and make the instance guard actually read files (#171)
- `install` make bun install compile node-pty without a global node-gyp
- `api` restore the models provider name the route extraction dropped (#169)
diff --git a/scripts/check-perf.mjs b/scripts/check-perf.mjs
index e2c14cda..a6e588bf 100644
--- a/scripts/check-perf.mjs
+++ b/scripts/check-perf.mjs
@@ -75,7 +75,7 @@ try {
// A skipped timing check must be visible, not silent: the whole point of the load gate
// is that nobody mistakes a busy host for a clean result.
- const skipped = timingSkipReason(report, budget);
+ const skipped = timingSkipReason(report, budget, baseline);
if (skipped && !values.json) console.log(`perf: ${skipped}`);
if (values["report-only"]) {
diff --git a/scripts/lib/perf-report.gate.test.ts b/scripts/lib/perf-report.gate.test.ts
index f8780b7f..29b110ff 100644
--- a/scripts/lib/perf-report.gate.test.ts
+++ b/scripts/lib/perf-report.gate.test.ts
@@ -67,18 +67,35 @@ describe("timingSkipReason", () => {
const budgetWithLoad = { ...sampleBudget, enforce: { ...sampleBudget.enforce, maxLoadAvg1ForTiming: 4 } };
it("returns null when the host is idle enough to compare", () => {
- expect(timingSkipReason(idleReport, budgetWithLoad)).toBeNull();
+ expect(timingSkipReason(idleReport, budgetWithLoad, null)).toBeNull();
});
it("explains the skip when load exceeds the budget ceiling", () => {
- const reason = timingSkipReason(busyReport, budgetWithLoad);
+ const reason = timingSkipReason(busyReport, budgetWithLoad, null);
expect(reason).toContain("9.5");
expect(reason).toContain("idle host");
});
it("returns null when no ceiling or no measured load exists", () => {
- expect(timingSkipReason(busyReport, sampleBudget)).toBeNull();
- expect(timingSkipReason({ runs: [] }, budgetWithLoad)).toBeNull();
+ expect(timingSkipReason(busyReport, sampleBudget, null)).toBeNull();
+ expect(timingSkipReason({ runs: [] }, budgetWithLoad, null)).toBeNull();
+ });
+
+ it("skips the comparison when the runner has a different core count", () => {
+ // A 2-4 core GitHub runner and a 16-core workstation are different instruments for a
+ // wall-clock metric. CI measured 271-321 ms against a 164/132 ms baseline and failed,
+ // which said nothing about the commit.
+ const ciRun = { runs: [run({ tbtMs: 321 })], host: { cpus: 4, loadAvg1: 1.2 } };
+ const baseline16 = { host: { cpus: 16 }, runs: [{ label: "desktop", tbtMs: 146 }] };
+ expect(timingSkipReason(ciRun, sampleBudget, baseline16)).toContain("4 cpus");
+ expect(perfViolations(ciRun, sampleBudget, baseline16)).toEqual([]);
+ });
+
+ it("still compares when the core count matches", () => {
+ const sameHost = { runs: [run({ tbtMs: 900 })], host: { cpus: 16, loadAvg1: 1.2 } };
+ const baseline16 = { host: { cpus: 16 }, runs: [{ label: "desktop", tbtMs: 146 }] };
+ expect(timingSkipReason(sameHost, sampleBudget, baseline16)).toBeNull();
+ expect(perfViolations(sameHost, sampleBudget, baseline16)).toContain("desktop: tbtMs 900ms > 2x baseline 146ms");
});
it("still reports a real regression at idle, so the gate is not disarmed", () => {
diff --git a/scripts/lib/perf-report.mjs b/scripts/lib/perf-report.mjs
index e5229120..62e29494 100644
--- a/scripts/lib/perf-report.mjs
+++ b/scripts/lib/perf-report.mjs
@@ -156,8 +156,14 @@ export function perfViolations(report, budget, baseline = null) {
const reportLoad = data?.host?.loadAvg1;
const loadComparable =
!Number.isFinite(maxLoad) || !Number.isFinite(reportLoad) || reportLoad <= maxLoad;
+ // Same instrument check: a different core count is a different machine, and TBT is
+ // wall-clock. Comparing across it produces a failure that says nothing about the commit.
+ const baseCpus = Number(baseline?.host?.cpus);
+ const runCpus = Number(data?.host?.cpus);
+ const hostComparable =
+ !Number.isFinite(baseCpus) || !Number.isFinite(runCpus) || baseCpus === runCpus;
const blowupFactor = budget?.enforce?.tbtMsBlowupFactor;
- if (loadComparable && baseline && blowupFactor !== null && blowupFactor !== undefined && Number.isFinite(blowupFactor)) {
+ if (loadComparable && hostComparable && baseline && blowupFactor !== null && blowupFactor !== undefined && Number.isFinite(blowupFactor)) {
for (const run of runs) {
if (!run?.label) continue;
const baseTbt = getBaselineTbt(baseline, run.label);
diff --git a/scripts/lib/perf-timing-load.mjs b/scripts/lib/perf-timing-load.mjs
index 5ddffed2..67e4d9f6 100644
--- a/scripts/lib/perf-timing-load.mjs
+++ b/scripts/lib/perf-timing-load.mjs
@@ -19,9 +19,20 @@
* @param {any} budget
* @returns {string|null}
*/
-export function timingSkipReason(report, budget) {
+export function timingSkipReason(report, budget, baseline) {
+ const r = typeof report === "string" ? JSON.parse(report) : report;
+ const load = r?.host?.loadAvg1;
const maxLoad = Number(budget?.enforce?.maxLoadAvg1ForTiming);
- const load = (typeof report === "string" ? JSON.parse(report) : report)?.host?.loadAvg1;
- if (!Number.isFinite(maxLoad) || !Number.isFinite(load) || load <= maxLoad) return null;
- return `tbt not compared: host load1=${load} exceeds ${maxLoad}, and the committed baseline was measured on an idle host`;
+ if (Number.isFinite(maxLoad) && Number.isFinite(load) && load > maxLoad) {
+ return `tbt not compared: host load1=${load} exceeds ${maxLoad}, and the committed baseline was measured on an idle host`;
+ }
+ // Core count matters as much as load: TBT is wall-clock, so a 2-core CI runner and a
+ // 16-core workstation are different instruments. The baseline records the machine it
+ // was taken on; a run elsewhere is skipped rather than allowed to fail or pass.
+ const baseCpus = Number(baseline?.host?.cpus);
+ const runCpus = Number(r?.host?.cpus);
+ if (Number.isFinite(baseCpus) && Number.isFinite(runCpus) && baseCpus !== runCpus) {
+ return `tbt not compared: this host has ${runCpus} cpus, the committed baseline was taken on ${baseCpus}`;
+ }
+ return null;
}
diff --git a/scripts/perf-baseline.json b/scripts/perf-baseline.json
index 920a5da9..67351038 100644
--- a/scripts/perf-baseline.json
+++ b/scripts/perf-baseline.json
@@ -1,6 +1,6 @@
{
"schemaVersion": 1,
- "note": "Committed measured baseline for the catch-all shell, Slow 4G + 4x CPU, commit 1c74603 (2026-10-28). Values are the median of 3 runs against the synthetic releaseFixture (the path CI takes); the first run of a session is discarded as warm-up because a cold page cache inflates TBT by ~2x. Observed spread across the stable runs: desktop TBT 150-184 ms, mobile 99-148 ms, LCP 3.02-3.22 s, so host load moves these by roughly +/-25%. Only the 2x blowup guard reads them, which is why they are allowed to be approximate; bundle bytes are gated deterministically elsewhere (check-bundle.mjs). Regenerate with: node scripts/perf-probe.mjs --json. TBT here is the standard definition: sum over long tasks of max(0, duration - 50ms), NOT the raw sum of task durations.",
+ "note": "Committed measured baseline for the catch-all shell, Slow 4G + 4x CPU, commit 1c74603 (2026-10-28). Values are the median of 3 runs against the synthetic releaseFixture (the path CI takes); the first run of a session is discarded as warm-up because a cold page cache inflates TBT by ~2x. Observed spread across the stable runs: desktop TBT 150-184 ms, mobile 99-148 ms, LCP 3.02-3.22 s, so host load moves these by roughly +/-25%. Only the 2x blowup guard reads them, which is why they are allowed to be approximate; bundle bytes are gated deterministically elsewhere (check-bundle.mjs). Regenerate with: node scripts/perf-probe.mjs --json. TBT here is the standard definition: sum over long tasks of max(0, duration - 50ms), NOT the raw sum of task durations. host records the machine the baseline was taken on: TBT is wall-clock and scales with core count as well as load, so a run on a different core count is not comparable and the gate skips it. The GitHub runner is ubuntu-latest (2-4 cores) and measured 271-321 ms against this 16-core baseline of 164/132 ms.",
"commit": "1c74603",
"profile": {
"name": "cold-slow4g-4xcpu",
@@ -12,7 +12,10 @@
"runs": [
{
"label": "desktop",
- "viewport": { "width": 1440, "height": 900 },
+ "viewport": {
+ "width": 1440,
+ "height": 900
+ },
"fcpMs": 1180,
"lcpMs": 3222,
"tbtMs": 164,
@@ -25,7 +28,10 @@
},
{
"label": "mobile",
- "viewport": { "width": 390, "height": 844 },
+ "viewport": {
+ "width": 390,
+ "height": 844
+ },
"fcpMs": 1166,
"lcpMs": 3054,
"tbtMs": 132,
@@ -36,5 +42,11 @@
"firstLoadJsFiles": 21,
"domNodes": 1165
}
- ]
+ ],
+ "host": {
+ "cpus": 16,
+ "node": "26.7.0",
+ "platform": "linux",
+ "loadAvg1AtMeasure": 0.32
+ }
}