The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.
TypeScript 3d-modelingagent-frameworkagent-harnessai-agentscadclaude-codecodexcoding-agentsdeveloper-toolsesp32game-developmentmcp-servermulti-agentopen-hardwareopencodeopenharnesspcbroboticsterminaltmux
Run complete Desktop VM coverage in parallel CI shards (#662)
Dee94125c357e3e
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.ymlindex 6cf27db13..ddb720d01 100644--- a/.github/workflows/ci.yml+++ b/.github/workflows/ci.yml@@ -19,7 +19,7 @@ on:scope:description: 'Affected components (full for shared protocols or uncertain impact)'type: choice- options: [full, cli, tui, backend, process]+ options: [full, cli, tui, backend, desktop, process]default: full# Callable as a reusable workflow. release.yml used to gate on this and take the Node pin from# its output; it no longer does (see the note above `publish` there) and pins the same version@@ -27,7 +27,7 @@ on:workflow_call:inputs:scope:- description: 'full, cli, tui, backend, or process (repository tooling only)'+ description: 'full, cli, tui, backend, desktop (VM tests), or process (repository tooling only)'type: stringdefault: fulloutputs:@@ -50,7 +50,7 @@ concurrency:jobs:process-checks:- if: ${{ contains(fromJSON('["full","cli","tui","backend","process"]'), inputs.scope || 'full') }}+ if: ${{ contains(fromJSON('["full","cli","tui","backend","desktop","process"]'), inputs.scope || 'full') }}runs-on: ubuntu-latesttimeout-minutes: 3steps:@@ -61,10 +61,76 @@ jobs:# workflow_dispatch restricts the choice; reusable callers can pass any string.# Fail invalid calls rather than reporting success after skipping every check.invalid-scope:- if: ${{ !contains(fromJSON('["full","cli","tui","backend","process"]'), inputs.scope || 'full') }}+ if: ${{ !contains(fromJSON('["full","cli","tui","backend","desktop","process"]'), inputs.scope || 'full') }}runs-on: ubuntu-lateststeps:- - run: echo '::error::scope must be full, cli, tui, backend, or process'; exit 1+ - run: echo '::error::scope must be full, cli, tui, backend, desktop, or process'; exit 1++ desktop-tests:+ if: ${{ contains(fromJSON('["full","desktop"]'), inputs.scope || 'full') }}+ name: desktop-tests (${{ matrix.os }}, ${{ matrix.shard }})+ runs-on: ${{ matrix.os }}+ timeout-minutes: 15+ strategy:+ fail-fast: false+ matrix:+ os: [ubuntu-22.04, macos-15]+ shard: [1, 2, 3, 4]+ steps:+ - uses: actions/checkout@v7+ - id: toolchain+ run: |+ version="$(sed -n "s/^ FLUTTER_VERSION: '\([^']*\)'.*/\1/p" .github/workflows/release-desktop.yml)"+ test -n "$version"+ echo "version=$version" >> "$GITHUB_OUTPUT"+ - uses: ./.github/actions/desktop-flutter+ with:+ version: ${{ steps.toolchain.outputs.version }}+ - name: Run bounded Desktop VM shard+ env:+ DESKTOP_SHARD: ${{ matrix.shard }}+ run: python3 scripts/test-desktop.py --shard "$DESKTOP_SHARD/4" --workers 4 --timeout 600+ - name: Preserve complete test logs and receipts+ if: ${{ !cancelled() }}+ uses: actions/upload-artifact@v4+ with:+ name: desktop-shard-${{ matrix.os }}-${{ matrix.shard }}+ path: .harness/validation/+ include-hidden-files: true+ if-no-files-found: error+ overwrite: true+ retention-days: 7++ desktop-test-summary:+ needs: desktop-tests+ if: ${{ !cancelled() && contains(fromJSON('["full","desktop"]'), inputs.scope || 'full') }}+ runs-on: ubuntu-22.04+ timeout-minutes: 3+ steps:+ - uses: actions/checkout@v7+ with:+ sparse-checkout: |+ /desktop/test/+ /scripts/+ sparse-checkout-cone-mode: false+ - uses: actions/download-artifact@v4+ with:+ pattern: desktop-shard-*+ path: ${{ runner.temp }}/desktop-shards+ - name: Verify every VM file and its complete raw reporter on both platforms+ env:+ DESKTOP_TEST_RESULT: ${{ needs.desktop-tests.result }}+ run: >-+ python3 scripts/verify-desktop-test-shards.py "$RUNNER_TEMP/desktop-shards"+ --root "$GITHUB_WORKSPACE" --shards 4 --tests-result "$DESKTOP_TEST_RESULT"+ --output "$RUNNER_TEMP/desktop-test-summary.json"+ - uses: actions/upload-artifact@v4+ with:+ name: desktop-test-summary+ path: ${{ runner.temp }}/desktop-test-summary.json+ if-no-files-found: error+ overwrite: true+ retention-days: 7# tty.ReadStream uses libuv's native TTY handle. Exercise descriptor ownership,# binary backpressure and unplug behavior on both supported OSes and runtimes.diff --git a/CONTRIBUTING.md b/CONTRIBUTING.mdindex efa7cc041..deb2527cf 100644--- a/CONTRIBUTING.md+++ b/CONTRIBUTING.md@@ -196,9 +196,12 @@ or installed dependencies change. Run `npm run typecheck` and the affected testwhile developing. Shared state, authentication, protocols, dependencies, and changeswith an unclear impact also need the full CLI suite. A passing CI run on the testedsource satisfies that full-suite check; do not run it again locally before release.-Manual CI offers `cli`, `tui`, `backend`, and `full` scopes; `full` remains the default.+Manual CI offers `cli`, `tui`, `backend`, `desktop`, and `full` scopes; `full` remains the default.CLI scope includes the supported OS/Node shell and serial matrix. Cross-componentchanges need all affected scopes or `full`.+Desktop scope runs the complete VM suite in four shards on both macOS and Linux,+with a verified coverage summary. Changed Dart analysis, browser tests and native+integration checks remain separate requirements when relevant.Two further suites exist and are**opt-in**, because they need software the machine may not have — they skip themselves rather thandiff --git a/desktop/test/desktop_workspace_tab_test.dart b/desktop/test/desktop_workspace_tab_test.dartindex bfe9584cd..abb6083e7 100644--- a/desktop/test/desktop_workspace_tab_test.dart+++ b/desktop/test/desktop_workspace_tab_test.dart@@ -239,13 +239,20 @@ void main() {expect(tester.getRect(_label('Desktop')), title);expect(tester.getRect(_tab('Desktop')), tabBounds);final shortcut = tester.getRect(_hint('Desktop'));- expect(shortcut.left - title.right, closeTo(6, .5));+ final indicator = tester.getRect(+ find.ancestor(of: _hint('Desktop'), matching: find.byType(Center)),+ );+ // The stable slot also holds the 16px activity mark. A narrower hint+ // remains centered inside it; its font-dependent glyph edge can be+ // farther from the label than the slot's six-pixel gap.+ expect(indicator.left - title.right, closeTo(6, .5));+ expect(shortcut.center.dx, closeTo(indicator.center.dx, .01));expect(- shortcut.right,+ indicator.right,lessThanOrEqualTo(tester.getRect(_close('Desktop')).left),);expect(- (title.left + shortcut.right) / 2,+ (title.left + indicator.right) / 2,closeTo(tabBounds.center.dx, .5),);await _capture(@@ -264,6 +271,40 @@ void main() {}}+ testWidgets('a narrow hint preserves the centered activity slot', (+ tester,+ ) async {+ tester.view.devicePixelRatio = 1;+ tester.view.physicalSize = const Size(640, 100);+ addTearDown(tester.view.reset);+ final hints = ValueNotifier(false);+ addTearDown(hints.dispose);+ await tester.pumpWidget(+ _host(hints, tabs: [(name: 'Desktop', hint: '1', working: true)]),+ );+ final title = tester.getRect(_label('Desktop'));+ final slot = tester.getRect(+ find.ancestor(+ of: find.byKey(const ValueKey('status:Desktop')),+ matching: find.byType(Center),+ ),+ );+ expect(slot.width, 16);+ expect(slot.left - title.right, closeTo(6, .01));+ hints.value = true;+ await tester.pump();+ final shortcut = tester.getRect(_hint('Desktop'));+ expect(shortcut.width, lessThan(slot.width));+ expect(shortcut.left - title.right, greaterThan(6));+ expect(shortcut.center.dx, closeTo(slot.center.dx, .01));+ expect(tester.getRect(_label('Desktop')), title);+ expect(+ (title.left + slot.right) / 2,+ closeTo(tester.getCenter(_tab('Desktop')).dx, .01),+ );+ expect(tester.takeException(), isNull);+ });+testWidgets('resolved remaps replace status and preserve purposeful actions',(tester) async {diff --git a/docs/validation-and-release.md b/docs/validation-and-release.mdindex b9a73eefa..0c11359df 100644--- a/docs/validation-and-release.md+++ b/docs/validation-and-release.md@@ -25,12 +25,27 @@ compiling, or starting a broad suite. Do not clean other sessions' files to makeRun independent checks concurrently, with enough capacity for their workers;two commands each spawning every CPU is not useful parallelism.-Manual **CI → Run workflow** accepts `scope`: `cli`, `tui`, `backend`, or `full`+Manual **CI → Run workflow** accepts `scope`: `cli`, `tui`, `backend`, `desktop`, or `full`(the default). `cli` includes typecheck, all CLI tests, updater coverage, releasebundle checks, and the serial/login-shell OS/Node matrix. `tui` includes its nativeCLI integration tests. Select `full` for cross-component changes or uncertain impact.The workflow remains on demand; this change does not introduce new required gates.+`desktop` runs all Desktop VM files on both macOS and Linux, split into four+isolated runners per platform. It uses the release's pinned Flutter SDK and shared+dependency caches. Each shard has four test workers and the bounded runner's+ten-minute budget; startup recovery retains the same narrow rules below. `full`+includes this matrix alongside the existing component checks. Changed Dart+analysis and browser/native integration checks remain separate where relevant.++The Desktop aggregate reads each raw test log, verifies its hash and complete-file+result, and compares every shard's inventory with the checked-out test files.+Every file must finish exactly once per platform. Failed jobs, missing or duplicate+files, changed source/environment, malformed reports and incomplete tests fail it.+Platform-specific skips and any pre-test loader recovery remain explicit in the+summary. Logs, receipts and the summary are retained for seven days. Collect the+result with `scripts/record-ci-validation.py RUN_ID --scope desktop --pr PR_NUMBER`.+CLI's default Vitest suite runs as four file shards on separate runners, retainingits worker cap and isolation. Typecheck, lockfile checks, guard fuzz, registry andrelease-bundle integration, and updater coverage run alongside them. Guard fuzz@@ -193,6 +208,8 @@ point, not a fixed cap for a full suite. Choose and record a worker count that fthe host and other running checks. The full-suite command below selects moreworkers on larger hosts; use an explicit lower count when other checks are active.Chrome and native integration tests ignore Flutter's concurrency option.+Use `make desktop-test ARGS="--shard 1/4 --workers 4"` to reproduce one CI+assignment locally. A shard alone does not validate the complete suite.Tests legitimately needing longer can declare that explicitly. Start a necessarybroad desktop run early, withan outer limit (initial budget: 15 minutes); investigate a timeout instead of waiting@@ -252,13 +269,15 @@ collect its record with one read-only command:python3 scripts/record-ci-validation.py RUN_ID --scope cli --pr PR_NUMBER```-Use the required CI scope (`cli`, `tui`, `backend`, `process`, or `full`). The command+Use the required CI scope (`cli`, `tui`, `backend`, `desktop`, `process`, or `full`). The commandwrites a short `validation.md` and machine-readable `receipt.json` under ignored`.harness/validation/`; use the paragraph/table in the PR's verification section.It checks the exact run/repository/workflow, required jobs, completed steps, sourcetrees, and an optional PR's head/base stability. For CLI it also downloads thecoverage summary by immutable artifact ID and verifies its archive checksum and-file/case totals. Network calls are bounded; the default collection budget is 90s.+file/case totals. Desktop scope similarly verifies the summary for both platforms+and preserves explicit startup-recovery counts. Network calls are bounded; the+default collection budget is 90s.Exit 0 means the requested CI scope passed and its committed source tree matchesthe selected target (`HEAD` by default). Exit 3 still saves the record but flagsdiff --git a/scripts/record-ci-validation.py b/scripts/record-ci-validation.pyindex d7f21eee9..ca1c69d06 100644--- a/scripts/record-ci-validation.py+++ b/scripts/record-ci-validation.py@@ -22,8 +22,10 @@ CLI_JOBS = {"cli-contracts", "typecheck-test"} | {f"cli-tests ({i})" for i in raf"serial-native ({system}, {node})" for system in ["ubuntu-latest", "macos-latest"] for node in ["20.19.0", "22.23.2"]}TUI_JOBS = {"tui-test (ubuntu-latest, x86_64-unknown-linux-musl)", "tui-test (ubuntu-24.04-arm, aarch64-unknown-linux-musl)"}-SCOPES = {"cli": CLI_JOBS, "tui": TUI_JOBS, "backend": {"backend-desk"}, "process": set()}-SCOPES["full"] = CLI_JOBS | TUI_JOBS | SCOPES["backend"]+DESKTOP_PLATFORMS = {"ubuntu-22.04", "macos-15"}+DESKTOP_JOBS = {"desktop-test-summary"} | {f"desktop-tests ({platform}, {index})" for platform in DESKTOP_PLATFORMS for index in range(1, 5)}+SCOPES = {"cli": CLI_JOBS, "tui": TUI_JOBS, "backend": {"backend-desk"}, "desktop": DESKTOP_JOBS, "process": set()}+SCOPES["full"] = CLI_JOBS | TUI_JOBS | SCOPES["backend"] | DESKTOP_JOBSdef utc_now():@@ -101,20 +103,25 @@ def check_jobs(run, jobs, scope):raise ValueError(f"job contains a failed or unfinished step: {job['name']}")-def read_cli_summary(artifact, archive, run):+def summary_contents(artifact, archive, run, name):provenance = artifact.get("workflow_run", {})- if (artifact.get("name") != "cli-test-summary" or artifact.get("expired") is not False+ if (artifact.get("name") != name or artifact.get("expired") is not Falseor provenance.get("id") != run["id"] or provenance.get("head_sha") != run["head_sha"]):- raise ValueError("CLI summary artifact is expired or belongs to another source/run")+ raise ValueError("summary artifact is expired or belongs to another source/run")digest = "sha256:" + hashlib.sha256(archive).hexdigest()if artifact.get("digest") != digest:- raise ValueError("CLI summary archive digest mismatch")+ raise ValueError("summary archive digest mismatch")with zipfile.ZipFile(io.BytesIO(archive)) as bundle:- if bundle.namelist() != ["cli-test-summary.json"] or bundle.infolist()[0].file_size > 8 * 1024 ** 2:- raise ValueError("unexpected CLI summary archive contents")- summary = json.loads(bundle.read("cli-test-summary.json"))+ if bundle.namelist() != [name + ".json"] or bundle.infolist()[0].file_size > 8 * 1024 ** 2:+ raise ValueError("unexpected summary archive contents")+ summary = json.loads(bundle.read(name + ".json"))if not isinstance(summary, dict):- raise ValueError("invalid CLI summary object")+ raise ValueError("invalid summary object")+ return summary+++def read_cli_summary(artifact, archive, run):+ summary = summary_contents(artifact, archive, run, "cli-test-summary")for key in ["files", "passed", "skipped", "todo"]:if type(summary.get(key)) is not int or summary[key] < 0:raise ValueError("invalid CLI summary counts")@@ -128,6 +135,43 @@ def read_cli_summary(artifact, archive, run):return summary+def read_desktop_summary(artifact, archive, run):+ summary = summary_contents(artifact, archive, run, "desktop-test-summary")+ source = summary.get("source", {})+ platforms = summary.get("platforms", [])+ if (summary.get("schema") != 1 or summary.get("kind") != "desktop-vm-ci" or summary.get("status") != "passed"+ or source.get("commit") != run["head_sha"] or source.get("dirty") is not False+ or not re.fullmatch(r"[0-9a-f]{40}", source.get("tree", ""))+ or not isinstance(platforms, list) or len(platforms) != len(DESKTOP_PLATFORMS)+ or {p.get("platform") for p in platforms} != DESKTOP_PLATFORMS):+ raise ValueError("Desktop summary source/platform coverage differs")+ inventory = None+ for platform in platforms:+ for key in ["files", "passed", "skipped"]:+ if type(platform.get(key)) is not int or platform[key] < 0:+ raise ValueError("invalid Desktop summary count")+ files, shards = platform.get("verified_files", []), platform.get("shards", [])+ if (not platform["files"] or len(files) != platform["files"] or len(set(files)) != len(files)+ or any(not isinstance(p, str) or not p.startswith("test/") or not p.endswith("_test.dart")+ or ".." in p.split("/") or p.startswith("test/web/") for p in files)+ or len(shards) != 4 or {s["shard"] for s in shards} != {1, 2, 3, 4}):+ raise ValueError("Desktop summary file/shard coverage is incomplete")+ if inventory is not None and set(files) != inventory:+ raise ValueError("Desktop platform inventories disagree")+ inventory = set(files)+ for shard in shards:+ if (any(type(shard.get(k)) is not int or shard[k] < 0 for k in ["files", "verified_files", "passed", "skipped"])+ or not shard["files"] or shard["verified_files"] != shard["files"]+ or not re.fullmatch(r"[0-9a-f]{64}", shard.get("receipt_sha256", ""))+ or not isinstance(shard.get("recovered_files"), list)+ or len(set(shard["recovered_files"])) != len(shard["recovered_files"])+ or not set(shard["recovered_files"]) <= inventory):+ raise ValueError("Desktop shard counts or recovery record differ")+ if any(sum(s[k] for s in shards) != platform[k] for k in ["files", "passed", "skipped"]):+ raise ValueError("Desktop summary totals disagree")+ return summary++def source_comparison(root, tested, target):def git(*args):return subprocess.check_output(["git", *args], cwd=root, stderr=subprocess.PIPE, timeout=10).decode().strip()@@ -145,7 +189,7 @@ def collect(client, root, run_id, scope, target, output, pr_number=None):check_run(run, client.repository, run_id)with ThreadPoolExecutor(max_workers=3) as pool:job_future = pool.submit(client.pages, f"actions/runs/{run_id}/jobs?filter=latest", "jobs")- artifact_future = pool.submit(client.pages, f"actions/runs/{run_id}/artifacts", "artifacts") if scope in {"cli", "full"} else None+ artifact_future = pool.submit(client.pages, f"actions/runs/{run_id}/artifacts", "artifacts") if scope in {"cli", "desktop", "full"} else Nonepr_future = pool.submit(client.api, f"pulls/{pr_number}") if pr_number else Nonejobs = job_future.result()artifacts = artifact_future.result() if artifact_future else []@@ -170,6 +214,19 @@ def collect(client, root, run_id, scope, target, output, pr_number=None):record["cli_summary"] = read_cli_summary(artifact, archive, run)record["artifact"] = {key: artifact[key] for key in ["id", "name", "digest", "size_in_bytes"]}(output / "cli-test-summary.zip").write_bytes(archive)+ if scope in {"desktop", "full"}:+ matches = [artifact for artifact in artifacts if artifact["name"] == "desktop-test-summary"]+ if len(matches) != 1:+ raise ValueError("expected one nonexpired Desktop coverage summary artifact")+ artifact = matches[0]+ if artifact.get("size_in_bytes", 16 * 1024 ** 2 + 1) > 16 * 1024 ** 2:+ raise ValueError("Desktop coverage artifact is unexpectedly large")+ archive = client.command("api", f"repos/{client.repository}/actions/artifacts/{artifact['id']}/zip", binary=True)+ record["desktop_summary"] = read_desktop_summary(artifact, archive, run)+ if record["desktop_summary"]["source"]["tree"] != source["tested_tree"]:+ raise ValueError("Desktop summary Git tree differs from the tested source")+ record["desktop_artifact"] = {key: artifact[key] for key in ["id", "name", "digest", "size_in_bytes"]}+ (output / "desktop-test-summary.zip").write_bytes(archive)if pr:record["pr"] = {"number": pr_number, "url": pr["html_url"], "head": pr["head"]["sha"], "base": pr["base"]["sha"]}# Detect reruns and moving PR heads after the independent downloads. Never@@ -202,6 +259,12 @@ def markdown(record):if "cli_summary" in record:c = record["cli_summary"]lines.append(f"CLI default suite: **{c['files']} files verified exactly once; {c['passed']:,} passed, {c['skipped']} skipped, {c['todo']} todo**. Summary artifact digest verified.")+ if "desktop_summary" in record:+ for platform in record["desktop_summary"]["platforms"]:+ recovered = sum(len(s["recovered_files"]) for s in platform["shards"])+ recovery = f" Includes {recovered} explicitly recorded pre-test loader recoveries." if recovered else ""+ lines.append(f"Desktop VM `{platform['platform']}`: **{platform['files']} files verified exactly once; "+ f"{platform['passed']:,} passed, {platform['skipped']} skipped**. Summary artifact digest verified.{recovery}")lines += ["", "| Job | Result | Time |", "| --- | --- | ---: |"]for job in record["jobs"]:name = job["name"].replace("|", "\\|").replace("\n", " ")diff --git a/scripts/test-desktop.py b/scripts/test-desktop.pyindex da9f2068d..f850c58bc 100644--- a/scripts/test-desktop.py+++ b/scripts/test-desktop.py@@ -9,6 +9,7 @@ import importlib.utilimport jsonimport osfrom pathlib import Path+import reimport shutilimport subprocessimport sys@@ -148,11 +149,29 @@ def select_files(desktop, paths):return sorted(set(selected))+def shard_pair(value):+ if not re.fullmatch(r"[1-9][0-9]*/[1-9][0-9]*", value):+ raise argparse.ArgumentTypeError("shard must be INDEX/TOTAL, starting at 1")+ index, total = map(int, value.split("/"))+ if index > total:+ raise argparse.ArgumentTypeError("shard index exceeds total")+ return index, total+++def partition(files, shard):+ index, total = shard+ selected = files[index - 1::total]+ if not selected:+ raise ValueError("shard has no test files; reduce the shard count")+ return selected++def main(argv=None):parser = argparse.ArgumentParser(description=__doc__)parser.add_argument("files", nargs="*", help="desktop-relative test/foo_test.dart paths; default: every VM test")parser.add_argument("--flutter", default="flutter", help="Flutter executable (dependencies must already be installed)")parser.add_argument("--workers", type=int, default=default_workers())+ parser.add_argument("--shard", type=shard_pair, default=(1, 1), help="run INDEX/TOTAL of the complete selected file inventory")parser.add_argument("--timeout", type=float, default=900, help="total attempt budget in seconds, including recovery (default: 900)")parser.add_argument("--no-loader-retry", action="store_true", help="retain the first failure without recovery")args = parser.parse_args(argv)@@ -162,7 +181,8 @@ def main(argv=None):parser.error("workers and timeout must be positive and finite")root = Path(subprocess.check_output(["git", "rev-parse", "--show-toplevel"], text=True).strip()).resolve()desktop = root / "desktop"- selected = select_files(desktop, args.files)+ inventory = select_files(desktop, args.files)+ selected = partition(inventory, args.shard)flutter = shutil.which(args.flutter)if flutter is None:raise ValueError(f"Flutter executable unavailable: {args.flutter}")@@ -173,7 +193,10 @@ def main(argv=None):deadline = started + args.timeoutreceipt = dict(schema=1, kind="desktop-vm", started_at=validation.utc_now(), source=validation.source_state(root),workers=workers, logical_cpus=os.cpu_count(), timeout_seconds=args.timeout,- selected_files=selected, checks=[], status="blocked")+ selected_files=selected, checks=[], status="blocked", platform=sys.platform,+ desktop_root=str(desktop.resolve()),+ shard=dict(index=args.shard[0], total=args.shard[1],+ inventory=[str(Path(path).relative_to(desktop)) for path in inventory]))# Reuse the common runner's source/toolchain/environment identity and owned# process cleanup. This driver never reuses results from an earlier invocation.identity_plan = {"checks": [{"name": "desktop-vm", "argv": [flutter, "test"], "cwd": "desktop",diff --git a/scripts/tests/test_desktop_test_shards.py b/scripts/tests/test_desktop_test_shards.pynew file mode 100644index 000000000..22c93f7c1--- /dev/null+++ b/scripts/tests/test_desktop_test_shards.py@@ -0,0 +1,148 @@+import copy+import importlib.util+import json+from pathlib import Path+import subprocess+import tempfile+import unittest++from desktop_vm_fixture import suite++spec = importlib.util.spec_from_file_location("desktop_shards", Path(__file__).resolve().parents[1] / "verify-desktop-test-shards.py")+verifier = importlib.util.module_from_spec(spec)+spec.loader.exec_module(verifier)+++class DesktopShardTests(unittest.TestCase):+ def setUp(self):+ self.tmp = tempfile.TemporaryDirectory()+ self.addCleanup(self.tmp.cleanup)+ self.root = Path(self.tmp.name).resolve() / "source"+ tests = self.root / "desktop/test"+ tests.mkdir(parents=True)+ for name in ["one", "two"]:+ (tests / f"{name}_test.dart").write_text("// fixture\n")+ for args in [["init", "-q"], ["add", "."], ["-c", "user.name=Fixture", "-c", "[email protected]", "commit", "-qm", "fixture"]]:+ subprocess.run(["git", *args], cwd=self.root, capture_output=True, check=True)+ self.source = verifier.driver.validation.source_state(self.root)+ self.directory = self.root.parent / "reports"+ self.records = []+ self.inventory = ["test/one_test.dart", "test/two_test.dart"]+ for label, platform in verifier.PLATFORMS.items():+ for index in [1, 2]:+ folder = self.directory / f"desktop-shard-{label}-{index}" / "run"+ folder.mkdir(parents=True)+ desktop = f"/producer/{label}/desktop"+ selected = [desktop + "/" + self.inventory[index - 1]]+ check = self.check(folder, selected, "desktop-vm")+ record = dict(schema=1, kind="desktop-vm", platform=platform, source=self.source,+ source_after=self.source, identity={"tool": "same"}, identity_after={"tool": "same"},+ desktop_root=desktop, selected_files=selected, checks=[check], status="passed",+ coverage=dict(files=1, verified_files=1, passed=1, skipped=1),+ shard=dict(index=index, total=2, inventory=self.inventory))+ path = folder / "receipt.json"+ self.write(path, record)+ self.records.append((path, record))++ def write(self, path, record):+ path.write_text(json.dumps(record))++ def check(self, folder, selected, name, behavior="pass"):+ log = folder / (name + ".log")+ events = suite(selected[0], behavior=behavior) + [dict(type="done", success=behavior == "pass")]+ log.write_text("\n".join(json.dumps(e) for e in events) + "\n")+ return dict(name=name, status="passed" if behavior == "pass" else "failed",+ exit_code=0 if behavior == "pass" else 1, log="/producer/" + log.name,+ log_sha256=verifier.driver.validation.file_sha256(log),+ report=verifier.driver.summarize(log, selected))++ def verify(self, result="success"):+ return verifier.verify(self.directory, self.root, 2, result)++ def test_complete_logs_cover_each_platform_once_with_separate_counts(self):+ result = self.verify()+ self.assertEqual(result["status"], "passed")+ self.assertEqual(result["source"], self.source)+ self.assertEqual(len(result["platforms"]), 2)+ for platform in result["platforms"]:+ self.assertEqual(platform["verified_files"], self.inventory)+ self.assertEqual((platform["files"], platform["passed"], platform["skipped"]), (2, 2, 2))++ def test_failed_matrix_cannot_be_hidden_by_passing_reports(self):+ for result in ["failure", "cancelled", "skipped", ""]:+ with self.subTest(result=result), self.assertRaises(ValueError):+ self.verify(result)++ def test_missing_and_extra_artifacts_are_rejected(self):+ extra = self.directory / "desktop-shard-macos-15-3"+ extra.mkdir()+ with self.assertRaises(ValueError):+ self.verify()+ extra.rmdir()+ self.records[0][0].unlink()+ with self.assertRaises(ValueError):+ self.verify()++ def test_source_platform_assignment_inventory_and_counts_must_agree(self):+ path, original = self.records[0]+ changes = [dict(source=dict(self.source, commit="f" * 40)), dict(platform="other"),+ dict(identity_after={"tool": "changed"}), dict(selected_files=["/outside/test.dart"]),+ dict(shard=dict(index=2, total=2, inventory=self.inventory)),+ dict(shard=dict(index=1, total=2, inventory=self.inventory[:1])),+ dict(coverage=dict(files=1, verified_files=1, passed=999, skipped=1))]+ for change in changes:+ with self.subTest(change=change):+ self.write(path, dict(original, **change))+ with self.assertRaises(ValueError):+ self.verify()+ self.write(path, original)++ def test_changed_log_or_summary_does_not_pass(self):+ path, original = self.records[0]+ modified = copy.deepcopy(original)+ modified["checks"][0]["report"]["reported_success"] = False+ self.write(path, modified)+ with self.assertRaises(ValueError):+ self.verify()+ self.write(path, original)+ (path.parent / "desktop-vm.log").write_text("truncated\n")+ with self.assertRaises(ValueError):+ self.verify()++ def test_raw_failure_or_incomplete_report_cannot_claim_success(self):+ path, original = self.records[0]+ for behavior in ["assertion", "incomplete", "loader"]:+ with self.subTest(behavior=behavior):+ record = copy.deepcopy(original)+ check = self.check(path.parent, record["selected_files"], "desktop-vm", behavior)+ check.update(status="passed", exit_code=0)+ record["checks"] = [check]+ self.write(path, record)+ with self.assertRaises(ValueError):+ self.verify()++ def test_only_the_existing_narrow_loader_recovery_is_accepted(self):+ path, record = self.records[0]+ record = copy.deepcopy(record)+ record["checks"] = [self.check(path.parent, record["selected_files"], "desktop-vm", "loader"),+ self.check(path.parent, record["selected_files"], "desktop-vm-loader-retry")]+ record.update(status="passed_after_startup_retry", retried_files=record["selected_files"])+ self.write(path, record)+ result = self.verify()+ self.assertEqual(result["platforms"][0]["shards"][0]["recovered_files"], ["test/one_test.dart"])+ record["checks"][0] = self.check(path.parent, record["selected_files"], "desktop-vm", "assertion")+ self.write(path, record)+ with self.assertRaises(ValueError):+ self.verify()++ def test_cleanup_failure_blocks_aggregation(self):+ path, record = self.records[0]+ record = copy.deepcopy(record)+ record["checks"][0]["cleanup_error"] = "child remained"+ self.write(path, record)+ with self.assertRaises(ValueError):+ self.verify()+++if __name__ == "__main__":+ unittest.main()diff --git a/scripts/tests/test_desktop_tests.py b/scripts/tests/test_desktop_tests.pyindex 2fc7dbb0d..385a98974 100644--- a/scripts/tests/test_desktop_tests.py+++ b/scripts/tests/test_desktop_tests.py@@ -23,6 +23,18 @@ class WorkerCountTests(unittest.TestCase):with self.subTest(cpus=cpus), mock.patch.object(driver.os, "cpu_count", return_value=cpus):self.assertEqual(driver.default_workers(), expected)+ def test_shards_cover_every_file_once_and_reject_invalid_or_empty_assignments(self):+ files = [f"test/{i}_test.dart" for i in range(11)]+ groups = [driver.partition(files, driver.shard_pair(f"{i}/4")) for i in range(1, 5)]+ selected = [path for group in groups for path in group]+ self.assertEqual(set(selected), set(files))+ self.assertEqual(len(selected), len(set(selected)))+ for value in ["0/4", "1/0", "5/4", "1", "-1/4", "1/4/2", "01/4"]:+ with self.subTest(value=value), self.assertRaises(driver.argparse.ArgumentTypeError):+ driver.shard_pair(value)+ with self.assertRaises(ValueError):+ driver.partition(files[:1], (2, 2))+class ReporterTests(unittest.TestCase):def setUp(self):@@ -137,6 +149,14 @@ class DesktopProcessTests(unittest.TestCase):self.assertEqual(receipt["source"], receipt["source_after"])self.assertEqual(receipt["identity"], receipt["identity_after"])+ def test_shard_executes_only_its_files_and_records_the_full_inventory(self):+ result, receipt, calls = self.run_driver(extra=("--shard", "2/2"))+ self.assertEqual(result.returncode, 0, result.stdout + result.stderr)+ self.assertEqual(receipt["shard"], dict(index=2, total=2, inventory=["test/one_test.dart", "test/two_test.dart"]))+ self.assertEqual([Path(p).name for p in receipt["selected_files"]], ["two_test.dart"])+ self.assertEqual([Path(p).name for p in calls[0] if p.endswith("_test.dart")], ["two_test.dart"])+ self.assertEqual(receipt["coverage"], dict(files=1, verified_files=1, passed=1, skipped=1))+def test_recovery_runs_only_the_failed_loader_once_and_preserves_both_logs(self):result, receipt, calls = self.run_driver("once")self.assertEqual(result.returncode, 0, result.stdout + result.stderr)diff --git a/scripts/tests/test_record_ci_validation.py b/scripts/tests/test_record_ci_validation.pyindex 43c2f590c..755c176a0 100644--- a/scripts/tests/test_record_ci_validation.py+++ b/scripts/tests/test_record_ci_validation.py@@ -73,11 +73,11 @@ class EvidenceTests(unittest.TestCase):raise AssertionError(path)def pages(self, path, key):- return copy.deepcopy(fixture.jobs if key == "jobs" else [fixture.artifact])+ return copy.deepcopy(fixture.jobs if key == "jobs" else getattr(fixture, "artifacts", [fixture.artifact]))def command(self, *args, binary=False):fixture.command_args = args- return fixture.archive+ return fixture.archives[args[-1]] if hasattr(fixture, "archives") else fixture.archivereturn recorder.collect(FakeClient(), self.root, 123, scope, target, self.output, 5 if with_pr else None)@@ -98,6 +98,75 @@ class EvidenceTests(unittest.TestCase):self.assertNotIn("cli_summary", result)self.assertFalse((self.output / "cli-test-summary.zip").exists())+ def desktop_fixture(self):+ self.jobs = [dict(self.jobs[0], id=i, name=name) for i, name in+ enumerate(sorted(recorder.DESKTOP_JOBS | {"process-checks"}), 1)]+ files = [f"test/{i}_test.dart" for i in range(4)]+ self.summary = dict(schema=1, kind="desktop-vm-ci", status="passed",+ source=dict(commit=self.sha, tree=self.git("rev-parse", "HEAD^{tree}"), dirty=False),+ platforms=[dict(platform=name, files=4, passed=4, skipped=0, verified_files=files,+ shards=[dict(shard=i, files=1, verified_files=1, passed=1, skipped=0,+ recovered_files=[], receipt_sha256="a" * 64) for i in range(1, 5)])+ for name in sorted(recorder.DESKTOP_PLATFORMS)])+ self.desktop_archive()++ def desktop_archive(self):+ self.archive = self.zip_summary(self.summary, "desktop-test-summary.json")+ self.artifact.update(name="desktop-test-summary", size_in_bytes=len(self.archive),+ digest="sha256:" + hashlib.sha256(self.archive).hexdigest())++ def test_desktop_scope_verifies_both_platforms_and_preserves_the_artifact(self):+ self.desktop_fixture()+ result = self.collect(scope="desktop", with_pr=True)+ self.assertEqual(result["status"], "passed")+ self.assertEqual(len(result["desktop_summary"]["platforms"]), 2)+ self.assertIn("Desktop VM `macos-15`", recorder.markdown(result))+ self.assertEqual((self.output / "desktop-test-summary.zip").read_bytes(), self.archive)+ self.assertNotIn("cli_summary", result)++ def test_desktop_missing_platform_bad_counts_and_wrong_tree_are_rejected(self):+ self.desktop_fixture()+ original = copy.deepcopy(self.summary)+ for change in [lambda s: s["platforms"].pop(),+ lambda s: s["source"].update(tree="b" * 40),+ lambda s: s["platforms"][0].update(passed=True),+ lambda s: s["platforms"][0]["shards"].pop(),+ lambda s: s["platforms"][0]["shards"][0].update(verified_files=0),+ lambda s: s["platforms"][0]["shards"][0].update(recovered_files=["test/absent_test.dart"])]:+ self.summary = copy.deepcopy(original)+ change(self.summary)+ self.desktop_archive()+ with self.assertRaises(ValueError):+ self.collect(scope="desktop")++ def test_desktop_recovery_is_explicit_and_every_matrix_job_is_required(self):+ self.desktop_fixture()+ self.summary["platforms"][0]["shards"][0]["recovered_files"] = ["test/0_test.dart"]+ self.desktop_archive()+ self.assertIn("1 explicitly recorded pre-test loader recoveries", recorder.markdown(self.collect(scope="desktop")))+ self.jobs.pop()+ with self.assertRaisesRegex(ValueError, "missing jobs"):+ self.collect(scope="desktop")++ def test_full_scope_requires_and_retains_cli_and_desktop_summaries(self):+ cli_archive, cli_artifact = self.archive, copy.deepcopy(self.artifact)+ self.desktop_fixture()+ self.artifact["id"] = 988+ self.artifacts = [cli_artifact, self.artifact]+ self.archives = {"repos/owner/repo/actions/artifacts/987/zip": cli_archive,+ "repos/owner/repo/actions/artifacts/988/zip": self.archive}+ self.jobs = [dict(self.jobs[0], id=i, name=name) for i, name in+ enumerate(sorted(recorder.SCOPES["full"] | {"process-checks"}), 1)]+ result = self.collect(scope="full")+ self.assertEqual(result["status"], "passed")+ self.assertIn("cli_summary", result)+ self.assertIn("desktop_summary", result)+ self.assertEqual(result["artifact"]["id"], 987)+ self.assertEqual(result["desktop_artifact"]["id"], 988)+ self.artifacts.pop()+ with self.assertRaisesRegex(ValueError, "Desktop coverage summary"):+ self.collect(scope="full")+def test_partial_scope_cannot_be_called_full_ci(self):with self.assertRaisesRegex(ValueError, "missing jobs"):self.collect(scope="full")diff --git a/scripts/verify-desktop-test-shards.py b/scripts/verify-desktop-test-shards.pynew file mode 100644index 000000000..19c4ecb8d--- /dev/null+++ b/scripts/verify-desktop-test-shards.py@@ -0,0 +1,138 @@+#!/usr/bin/env python3+"""Verify complete Desktop VM coverage from bounded runner receipts and raw logs."""+import argparse+from collections import Counter+import importlib.util+import json+from pathlib import Path+import sys++spec = importlib.util.spec_from_file_location("desktop_tests", Path(__file__).with_name("test-desktop.py"))+driver = importlib.util.module_from_spec(spec)+spec.loader.exec_module(driver)+PLATFORMS = {"ubuntu-22.04": "linux", "macos-15": "darwin"}+++def verify_receipt(path, source, inventory, platform, index, shards):+ receipt = json.loads(path.read_text())+ if (receipt.get("schema") != 1 or receipt.get("kind") != "desktop-vm"+ or receipt.get("platform") != platform or source.get("dirty")+ or receipt.get("source") != source or receipt.get("source_after") != source+ or not receipt.get("identity") or receipt.get("identity") != receipt.get("identity_after")):+ raise ValueError("receipt source, platform or environment differs")+ shard = receipt.get("shard", {})+ if (type(shard.get("index")) is not int or type(shard.get("total")) is not int+ or shard != dict(index=index, total=shards, inventory=inventory)):+ raise ValueError("shard identity or full discovery inventory differs")+ desktop = Path(receipt["desktop_root"])+ if not desktop.is_absolute() or ".." in desktop.parts:+ raise ValueError("invalid producer checkout path")++ def relative(value):+ path = Path(value)+ if not path.is_absolute() or ".." in path.parts:+ raise ValueError("invalid reported test path")+ return path.relative_to(desktop).as_posix()++ selected = receipt["selected_files"]+ if [relative(p) for p in selected] != driver.partition(inventory, (index, shards)):+ raise ValueError("selected files do not match the assigned shard")+ checks = receipt.get("checks", [])+ if not isinstance(checks, list) or len(checks) not in {1, 2}:+ raise ValueError("missing or unexpected test attempts")++ def report(check, paths, name):+ if check.get("cleanup_error"):+ raise ValueError("test process cleanup failed")+ if check.get("name") != name or Path(check["log"]).name != name + ".log":+ raise ValueError("unexpected attempt or log path")+ log = path.parent / (name + ".log")+ if log.is_symlink() or driver.validation.file_sha256(log) != check.get("log_sha256"):+ raise ValueError("test log digest differs")+ actual = driver.summarize(log, paths)+ if actual != check.get("report"):+ raise ValueError("receipt disagrees with its raw test log")+ return actual++ first = report(checks[0], selected, "desktop-vm")+ results = dict(first["files"])+ retried = []+ if len(checks) == 1:+ if (receipt.get("status") != "passed" or checks[0].get("exit_code") != 0+ or not driver.passed(checks[0], first)):+ raise ValueError("test attempt did not pass completely")+ else:+ candidates = driver.retry_files(checks[0], first)+ if (not candidates or receipt.get("retried_files") != candidates+ or receipt.get("status") != "passed_after_startup_retry"):+ raise ValueError("invalid loader recovery")+ retried = [relative(p) for p in candidates]+ recovered = report(checks[1], candidates, "desktop-vm-loader-retry")+ if checks[1].get("exit_code") != 0 or not driver.passed(checks[1], recovered):+ raise ValueError("loader recovery did not pass completely")+ results.update(recovered["files"])+ if any(item["status"] != "passed" for item in results.values()):+ raise ValueError("a test file remains incomplete")+ coverage = dict(files=len(results), verified_files=len(results),+ passed=sum(item["passed"] for item in results.values()),+ skipped=sum(item["skipped"] for item in results.values()))+ if coverage != receipt.get("coverage"):+ raise ValueError("receipt coverage totals disagree with its logs")+ return {relative(p): result for p, result in results.items()}, dict(+ shard=index, **coverage, recovered_files=retried,+ receipt_sha256=driver.validation.file_sha256(path))+++def verify(directory, root, shards, tests_result):+ if tests_result != "success":+ raise ValueError(f"the complete Desktop matrix must pass: {tests_result}")+ expected_folders = {f"desktop-shard-{label}-{index}" for label in PLATFORMS for index in range(1, shards + 1)}+ if {path.name for path in directory.glob("desktop-shard-*")} != expected_folders:+ raise ValueError("missing or unexpected shard artifacts")+ source = driver.validation.source_state(root)+ inventory = [Path(p).relative_to(root / "desktop").as_posix()+ for p in driver.select_files(root / "desktop", [])]+ platforms = []+ for label, platform in PLATFORMS.items():+ seen, counts = Counter(), Counter(passed=0, skipped=0)+ records = []+ for index in range(1, shards + 1):+ folder = directory / f"desktop-shard-{label}-{index}"+ receipts = list(folder.rglob("receipt.json"))+ if len(receipts) != 1 or receipts[0].is_symlink():+ raise ValueError(f"{label} shard {index}: expected exactly one receipt")+ results, record = verify_receipt(receipts[0], source, inventory, platform, index, shards)+ seen.update(results.keys())+ counts.update({key: record[key] for key in counts})+ records.append(record)+ if set(seen) != set(inventory) or any(count != 1 for count in seen.values()):+ raise ValueError(f"{label}: files are missing, unexpected or duplicated")+ platforms.append(dict(platform=label, files=len(seen), **dict(counts),+ verified_files=sorted(seen), shards=records))+ return dict(schema=1, kind="desktop-vm-ci", status="passed", source=source,+ platforms=platforms)+++def main():+ parser = argparse.ArgumentParser(description=__doc__)+ parser.add_argument("directory", type=Path)+ parser.add_argument("--root", type=Path, required=True)+ parser.add_argument("--shards", type=int, default=4)+ parser.add_argument("--tests-result", required=True)+ parser.add_argument("--output", type=Path, required=True)+ args = parser.parse_args()+ if args.shards < 1:+ parser.error("shards must be positive")+ result = verify(args.directory, args.root.resolve(), args.shards, args.tests_result)+ args.output.write_text(json.dumps(result, indent=2) + "\n")+ for platform in result["platforms"]:+ print(f"{platform['platform']}: {platform['files']} files verified exactly once; "+ f"{platform['passed']} passed, {platform['skipped']} skipped")+++if __name__ == "__main__":+ try:+ main()+ except (OSError, ValueError, KeyError, TypeError) as error:+ print(f"Desktop shard verification failed: {error}", file=sys.stderr)+ sys.exit(1)