Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

Run complete Desktop VM coverage in parallel CI shards (#662)

Dee94125c357e3e
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 6cf27db13..ddb720d01 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -19,7 +19,7 @@ on:
scope:
description: 'Affected components (full for shared protocols or uncertain impact)'
type: choice
- options: [full, cli, tui, backend, process]
+ options: [full, cli, tui, backend, desktop, process]
default: full
# Callable as a reusable workflow. release.yml used to gate on this and take the Node pin from
# its output; it no longer does (see the note above `publish` there) and pins the same version
@@ -27,7 +27,7 @@ on:
workflow_call:
inputs:
scope:
- description: 'full, cli, tui, backend, or process (repository tooling only)'
+ description: 'full, cli, tui, backend, desktop (VM tests), or process (repository tooling only)'
type: string
default: full
outputs:
@@ -50,7 +50,7 @@ concurrency:
jobs:
process-checks:
- if: ${{ contains(fromJSON('["full","cli","tui","backend","process"]'), inputs.scope || 'full') }}
+ if: ${{ contains(fromJSON('["full","cli","tui","backend","desktop","process"]'), inputs.scope || 'full') }}
runs-on: ubuntu-latest
timeout-minutes: 3
steps:
@@ -61,10 +61,76 @@ jobs:
# workflow_dispatch restricts the choice; reusable callers can pass any string.
# Fail invalid calls rather than reporting success after skipping every check.
invalid-scope:
- if: ${{ !contains(fromJSON('["full","cli","tui","backend","process"]'), inputs.scope || 'full') }}
+ if: ${{ !contains(fromJSON('["full","cli","tui","backend","desktop","process"]'), inputs.scope || 'full') }}
runs-on: ubuntu-latest
steps:
- - run: echo '::error::scope must be full, cli, tui, backend, or process'; exit 1
+ - run: echo '::error::scope must be full, cli, tui, backend, desktop, or process'; exit 1
+
+ desktop-tests:
+ if: ${{ contains(fromJSON('["full","desktop"]'), inputs.scope || 'full') }}
+ name: desktop-tests (${{ matrix.os }}, ${{ matrix.shard }})
+ runs-on: ${{ matrix.os }}
+ timeout-minutes: 15
+ strategy:
+ fail-fast: false
+ matrix:
+ os: [ubuntu-22.04, macos-15]
+ shard: [1, 2, 3, 4]
+ steps:
+ - uses: actions/checkout@v7
+ - id: toolchain
+ run: |
+ version="$(sed -n "s/^ FLUTTER_VERSION: '\([^']*\)'.*/\1/p" .github/workflows/release-desktop.yml)"
+ test -n "$version"
+ echo "version=$version" >> "$GITHUB_OUTPUT"
+ - uses: ./.github/actions/desktop-flutter
+ with:
+ version: ${{ steps.toolchain.outputs.version }}
+ - name: Run bounded Desktop VM shard
+ env:
+ DESKTOP_SHARD: ${{ matrix.shard }}
+ run: python3 scripts/test-desktop.py --shard "$DESKTOP_SHARD/4" --workers 4 --timeout 600
+ - name: Preserve complete test logs and receipts
+ if: ${{ !cancelled() }}
+ uses: actions/upload-artifact@v4
+ with:
+ name: desktop-shard-${{ matrix.os }}-${{ matrix.shard }}
+ path: .harness/validation/
+ include-hidden-files: true
+ if-no-files-found: error
+ overwrite: true
+ retention-days: 7
+
+ desktop-test-summary:
+ needs: desktop-tests
+ if: ${{ !cancelled() && contains(fromJSON('["full","desktop"]'), inputs.scope || 'full') }}
+ runs-on: ubuntu-22.04
+ timeout-minutes: 3
+ steps:
+ - uses: actions/checkout@v7
+ with:
+ sparse-checkout: |
+ /desktop/test/
+ /scripts/
+ sparse-checkout-cone-mode: false
+ - uses: actions/download-artifact@v4
+ with:
+ pattern: desktop-shard-*
+ path: ${{ runner.temp }}/desktop-shards
+ - name: Verify every VM file and its complete raw reporter on both platforms
+ env:
+ DESKTOP_TEST_RESULT: ${{ needs.desktop-tests.result }}
+ run: >-
+ python3 scripts/verify-desktop-test-shards.py "$RUNNER_TEMP/desktop-shards"
+ --root "$GITHUB_WORKSPACE" --shards 4 --tests-result "$DESKTOP_TEST_RESULT"
+ --output "$RUNNER_TEMP/desktop-test-summary.json"
+ - uses: actions/upload-artifact@v4
+ with:
+ name: desktop-test-summary
+ path: ${{ runner.temp }}/desktop-test-summary.json
+ if-no-files-found: error
+ overwrite: true
+ retention-days: 7
# tty.ReadStream uses libuv's native TTY handle. Exercise descriptor ownership,
# binary backpressure and unplug behavior on both supported OSes and runtimes.
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index efa7cc041..deb2527cf 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -196,9 +196,12 @@ or installed dependencies change. Run `npm run typecheck` and the affected test
while developing. Shared state, authentication, protocols, dependencies, and changes
with an unclear impact also need the full CLI suite. A passing CI run on the tested
source satisfies that full-suite check; do not run it again locally before release.
-Manual CI offers `cli`, `tui`, `backend`, and `full` scopes; `full` remains the default.
+Manual CI offers `cli`, `tui`, `backend`, `desktop`, and `full` scopes; `full` remains the default.
CLI scope includes the supported OS/Node shell and serial matrix. Cross-component
changes need all affected scopes or `full`.
+Desktop scope runs the complete VM suite in four shards on both macOS and Linux,
+with a verified coverage summary. Changed Dart analysis, browser tests and native
+integration checks remain separate requirements when relevant.
Two further suites exist and are
**opt-in**, because they need software the machine may not have — they skip themselves rather than
diff --git a/desktop/test/desktop_workspace_tab_test.dart b/desktop/test/desktop_workspace_tab_test.dart
index bfe9584cd..abb6083e7 100644
--- a/desktop/test/desktop_workspace_tab_test.dart
+++ b/desktop/test/desktop_workspace_tab_test.dart
@@ -239,13 +239,20 @@ void main() {
expect(tester.getRect(_label('Desktop')), title);
expect(tester.getRect(_tab('Desktop')), tabBounds);
final shortcut = tester.getRect(_hint('Desktop'));
- expect(shortcut.left - title.right, closeTo(6, .5));
+ final indicator = tester.getRect(
+ find.ancestor(of: _hint('Desktop'), matching: find.byType(Center)),
+ );
+ // The stable slot also holds the 16px activity mark. A narrower hint
+ // remains centered inside it; its font-dependent glyph edge can be
+ // farther from the label than the slot's six-pixel gap.
+ expect(indicator.left - title.right, closeTo(6, .5));
+ expect(shortcut.center.dx, closeTo(indicator.center.dx, .01));
expect(
- shortcut.right,
+ indicator.right,
lessThanOrEqualTo(tester.getRect(_close('Desktop')).left),
);
expect(
- (title.left + shortcut.right) / 2,
+ (title.left + indicator.right) / 2,
closeTo(tabBounds.center.dx, .5),
);
await _capture(
@@ -264,6 +271,40 @@ void main() {
}
}
+ testWidgets('a narrow hint preserves the centered activity slot', (
+ tester,
+ ) async {
+ tester.view.devicePixelRatio = 1;
+ tester.view.physicalSize = const Size(640, 100);
+ addTearDown(tester.view.reset);
+ final hints = ValueNotifier(false);
+ addTearDown(hints.dispose);
+ await tester.pumpWidget(
+ _host(hints, tabs: [(name: 'Desktop', hint: '1', working: true)]),
+ );
+ final title = tester.getRect(_label('Desktop'));
+ final slot = tester.getRect(
+ find.ancestor(
+ of: find.byKey(const ValueKey('status:Desktop')),
+ matching: find.byType(Center),
+ ),
+ );
+ expect(slot.width, 16);
+ expect(slot.left - title.right, closeTo(6, .01));
+ hints.value = true;
+ await tester.pump();
+ final shortcut = tester.getRect(_hint('Desktop'));
+ expect(shortcut.width, lessThan(slot.width));
+ expect(shortcut.left - title.right, greaterThan(6));
+ expect(shortcut.center.dx, closeTo(slot.center.dx, .01));
+ expect(tester.getRect(_label('Desktop')), title);
+ expect(
+ (title.left + slot.right) / 2,
+ closeTo(tester.getCenter(_tab('Desktop')).dx, .01),
+ );
+ expect(tester.takeException(), isNull);
+ });
+
testWidgets(
'resolved remaps replace status and preserve purposeful actions',
(tester) async {
diff --git a/docs/validation-and-release.md b/docs/validation-and-release.md
index b9a73eefa..0c11359df 100644
--- a/docs/validation-and-release.md
+++ b/docs/validation-and-release.md
@@ -25,12 +25,27 @@ compiling, or starting a broad suite. Do not clean other sessions' files to make
Run independent checks concurrently, with enough capacity for their workers;
two commands each spawning every CPU is not useful parallelism.
-Manual **CI → Run workflow** accepts `scope`: `cli`, `tui`, `backend`, or `full`
+Manual **CI → Run workflow** accepts `scope`: `cli`, `tui`, `backend`, `desktop`, or `full`
(the default). `cli` includes typecheck, all CLI tests, updater coverage, release
bundle checks, and the serial/login-shell OS/Node matrix. `tui` includes its native
CLI integration tests. Select `full` for cross-component changes or uncertain impact.
The workflow remains on demand; this change does not introduce new required gates.
+`desktop` runs all Desktop VM files on both macOS and Linux, split into four
+isolated runners per platform. It uses the release's pinned Flutter SDK and shared
+dependency caches. Each shard has four test workers and the bounded runner's
+ten-minute budget; startup recovery retains the same narrow rules below. `full`
+includes this matrix alongside the existing component checks. Changed Dart
+analysis and browser/native integration checks remain separate where relevant.
+
+The Desktop aggregate reads each raw test log, verifies its hash and complete-file
+result, and compares every shard's inventory with the checked-out test files.
+Every file must finish exactly once per platform. Failed jobs, missing or duplicate
+files, changed source/environment, malformed reports and incomplete tests fail it.
+Platform-specific skips and any pre-test loader recovery remain explicit in the
+summary. Logs, receipts and the summary are retained for seven days. Collect the
+result with `scripts/record-ci-validation.py RUN_ID --scope desktop --pr PR_NUMBER`.
+
CLI's default Vitest suite runs as four file shards on separate runners, retaining
its worker cap and isolation. Typecheck, lockfile checks, guard fuzz, registry and
release-bundle integration, and updater coverage run alongside them. Guard fuzz
@@ -193,6 +208,8 @@ point, not a fixed cap for a full suite. Choose and record a worker count that f
the host and other running checks. The full-suite command below selects more
workers on larger hosts; use an explicit lower count when other checks are active.
Chrome and native integration tests ignore Flutter's concurrency option.
+Use `make desktop-test ARGS="--shard 1/4 --workers 4"` to reproduce one CI
+assignment locally. A shard alone does not validate the complete suite.
Tests legitimately needing longer can declare that explicitly. Start a necessary
broad desktop run early, with
an outer limit (initial budget: 15 minutes); investigate a timeout instead of waiting
@@ -252,13 +269,15 @@ collect its record with one read-only command:
python3 scripts/record-ci-validation.py RUN_ID --scope cli --pr PR_NUMBER
```
-Use the required CI scope (`cli`, `tui`, `backend`, `process`, or `full`). The command
+Use the required CI scope (`cli`, `tui`, `backend`, `desktop`, `process`, or `full`). The command
writes a short `validation.md` and machine-readable `receipt.json` under ignored
`.harness/validation/`; use the paragraph/table in the PR's verification section.
It checks the exact run/repository/workflow, required jobs, completed steps, source
trees, and an optional PR's head/base stability. For CLI it also downloads the
coverage summary by immutable artifact ID and verifies its archive checksum and
-file/case totals. Network calls are bounded; the default collection budget is 90s.
+file/case totals. Desktop scope similarly verifies the summary for both platforms
+and preserves explicit startup-recovery counts. Network calls are bounded; the
+default collection budget is 90s.
Exit 0 means the requested CI scope passed and its committed source tree matches
the selected target (`HEAD` by default). Exit 3 still saves the record but flags
diff --git a/scripts/record-ci-validation.py b/scripts/record-ci-validation.py
index d7f21eee9..ca1c69d06 100644
--- a/scripts/record-ci-validation.py
+++ b/scripts/record-ci-validation.py
@@ -22,8 +22,10 @@ CLI_JOBS = {"cli-contracts", "typecheck-test"} | {f"cli-tests ({i})" for i in ra
f"serial-native ({system}, {node})" for system in ["ubuntu-latest", "macos-latest"] for node in ["20.19.0", "22.23.2"]
}
TUI_JOBS = {"tui-test (ubuntu-latest, x86_64-unknown-linux-musl)", "tui-test (ubuntu-24.04-arm, aarch64-unknown-linux-musl)"}
-SCOPES = {"cli": CLI_JOBS, "tui": TUI_JOBS, "backend": {"backend-desk"}, "process": set()}
-SCOPES["full"] = CLI_JOBS | TUI_JOBS | SCOPES["backend"]
+DESKTOP_PLATFORMS = {"ubuntu-22.04", "macos-15"}
+DESKTOP_JOBS = {"desktop-test-summary"} | {f"desktop-tests ({platform}, {index})" for platform in DESKTOP_PLATFORMS for index in range(1, 5)}
+SCOPES = {"cli": CLI_JOBS, "tui": TUI_JOBS, "backend": {"backend-desk"}, "desktop": DESKTOP_JOBS, "process": set()}
+SCOPES["full"] = CLI_JOBS | TUI_JOBS | SCOPES["backend"] | DESKTOP_JOBS
def utc_now():
@@ -101,20 +103,25 @@ def check_jobs(run, jobs, scope):
raise ValueError(f"job contains a failed or unfinished step: {job['name']}")
-def read_cli_summary(artifact, archive, run):
+def summary_contents(artifact, archive, run, name):
provenance = artifact.get("workflow_run", {})
- if (artifact.get("name") != "cli-test-summary" or artifact.get("expired") is not False
+ if (artifact.get("name") != name or artifact.get("expired") is not False
or provenance.get("id") != run["id"] or provenance.get("head_sha") != run["head_sha"]):
- raise ValueError("CLI summary artifact is expired or belongs to another source/run")
+ raise ValueError("summary artifact is expired or belongs to another source/run")
digest = "sha256:" + hashlib.sha256(archive).hexdigest()
if artifact.get("digest") != digest:
- raise ValueError("CLI summary archive digest mismatch")
+ raise ValueError("summary archive digest mismatch")
with zipfile.ZipFile(io.BytesIO(archive)) as bundle:
- if bundle.namelist() != ["cli-test-summary.json"] or bundle.infolist()[0].file_size > 8 * 1024 ** 2:
- raise ValueError("unexpected CLI summary archive contents")
- summary = json.loads(bundle.read("cli-test-summary.json"))
+ if bundle.namelist() != [name + ".json"] or bundle.infolist()[0].file_size > 8 * 1024 ** 2:
+ raise ValueError("unexpected summary archive contents")
+ summary = json.loads(bundle.read(name + ".json"))
if not isinstance(summary, dict):
- raise ValueError("invalid CLI summary object")
+ raise ValueError("invalid summary object")
+ return summary
+
+
+def read_cli_summary(artifact, archive, run):
+ summary = summary_contents(artifact, archive, run, "cli-test-summary")
for key in ["files", "passed", "skipped", "todo"]:
if type(summary.get(key)) is not int or summary[key] < 0:
raise ValueError("invalid CLI summary counts")
@@ -128,6 +135,43 @@ def read_cli_summary(artifact, archive, run):
return summary
+def read_desktop_summary(artifact, archive, run):
+ summary = summary_contents(artifact, archive, run, "desktop-test-summary")
+ source = summary.get("source", {})
+ platforms = summary.get("platforms", [])
+ if (summary.get("schema") != 1 or summary.get("kind") != "desktop-vm-ci" or summary.get("status") != "passed"
+ or source.get("commit") != run["head_sha"] or source.get("dirty") is not False
+ or not re.fullmatch(r"[0-9a-f]{40}", source.get("tree", ""))
+ or not isinstance(platforms, list) or len(platforms) != len(DESKTOP_PLATFORMS)
+ or {p.get("platform") for p in platforms} != DESKTOP_PLATFORMS):
+ raise ValueError("Desktop summary source/platform coverage differs")
+ inventory = None
+ for platform in platforms:
+ for key in ["files", "passed", "skipped"]:
+ if type(platform.get(key)) is not int or platform[key] < 0:
+ raise ValueError("invalid Desktop summary count")
+ files, shards = platform.get("verified_files", []), platform.get("shards", [])
+ if (not platform["files"] or len(files) != platform["files"] or len(set(files)) != len(files)
+ or any(not isinstance(p, str) or not p.startswith("test/") or not p.endswith("_test.dart")
+ or ".." in p.split("/") or p.startswith("test/web/") for p in files)
+ or len(shards) != 4 or {s["shard"] for s in shards} != {1, 2, 3, 4}):
+ raise ValueError("Desktop summary file/shard coverage is incomplete")
+ if inventory is not None and set(files) != inventory:
+ raise ValueError("Desktop platform inventories disagree")
+ inventory = set(files)
+ for shard in shards:
+ if (any(type(shard.get(k)) is not int or shard[k] < 0 for k in ["files", "verified_files", "passed", "skipped"])
+ or not shard["files"] or shard["verified_files"] != shard["files"]
+ or not re.fullmatch(r"[0-9a-f]{64}", shard.get("receipt_sha256", ""))
+ or not isinstance(shard.get("recovered_files"), list)
+ or len(set(shard["recovered_files"])) != len(shard["recovered_files"])
+ or not set(shard["recovered_files"]) <= inventory):
+ raise ValueError("Desktop shard counts or recovery record differ")
+ if any(sum(s[k] for s in shards) != platform[k] for k in ["files", "passed", "skipped"]):
+ raise ValueError("Desktop summary totals disagree")
+ return summary
+
+
def source_comparison(root, tested, target):
def git(*args):
return subprocess.check_output(["git", *args], cwd=root, stderr=subprocess.PIPE, timeout=10).decode().strip()
@@ -145,7 +189,7 @@ def collect(client, root, run_id, scope, target, output, pr_number=None):
check_run(run, client.repository, run_id)
with ThreadPoolExecutor(max_workers=3) as pool:
job_future = pool.submit(client.pages, f"actions/runs/{run_id}/jobs?filter=latest", "jobs")
- artifact_future = pool.submit(client.pages, f"actions/runs/{run_id}/artifacts", "artifacts") if scope in {"cli", "full"} else None
+ artifact_future = pool.submit(client.pages, f"actions/runs/{run_id}/artifacts", "artifacts") if scope in {"cli", "desktop", "full"} else None
pr_future = pool.submit(client.api, f"pulls/{pr_number}") if pr_number else None
jobs = job_future.result()
artifacts = artifact_future.result() if artifact_future else []
@@ -170,6 +214,19 @@ def collect(client, root, run_id, scope, target, output, pr_number=None):
record["cli_summary"] = read_cli_summary(artifact, archive, run)
record["artifact"] = {key: artifact[key] for key in ["id", "name", "digest", "size_in_bytes"]}
(output / "cli-test-summary.zip").write_bytes(archive)
+ if scope in {"desktop", "full"}:
+ matches = [artifact for artifact in artifacts if artifact["name"] == "desktop-test-summary"]
+ if len(matches) != 1:
+ raise ValueError("expected one nonexpired Desktop coverage summary artifact")
+ artifact = matches[0]
+ if artifact.get("size_in_bytes", 16 * 1024 ** 2 + 1) > 16 * 1024 ** 2:
+ raise ValueError("Desktop coverage artifact is unexpectedly large")
+ archive = client.command("api", f"repos/{client.repository}/actions/artifacts/{artifact['id']}/zip", binary=True)
+ record["desktop_summary"] = read_desktop_summary(artifact, archive, run)
+ if record["desktop_summary"]["source"]["tree"] != source["tested_tree"]:
+ raise ValueError("Desktop summary Git tree differs from the tested source")
+ record["desktop_artifact"] = {key: artifact[key] for key in ["id", "name", "digest", "size_in_bytes"]}
+ (output / "desktop-test-summary.zip").write_bytes(archive)
if pr:
record["pr"] = {"number": pr_number, "url": pr["html_url"], "head": pr["head"]["sha"], "base": pr["base"]["sha"]}
# Detect reruns and moving PR heads after the independent downloads. Never
@@ -202,6 +259,12 @@ def markdown(record):
if "cli_summary" in record:
c = record["cli_summary"]
lines.append(f"CLI default suite: **{c['files']} files verified exactly once; {c['passed']:,} passed, {c['skipped']} skipped, {c['todo']} todo**. Summary artifact digest verified.")
+ if "desktop_summary" in record:
+ for platform in record["desktop_summary"]["platforms"]:
+ recovered = sum(len(s["recovered_files"]) for s in platform["shards"])
+ recovery = f" Includes {recovered} explicitly recorded pre-test loader recoveries." if recovered else ""
+ lines.append(f"Desktop VM `{platform['platform']}`: **{platform['files']} files verified exactly once; "
+ f"{platform['passed']:,} passed, {platform['skipped']} skipped**. Summary artifact digest verified.{recovery}")
lines += ["", "| Job | Result | Time |", "| --- | --- | ---: |"]
for job in record["jobs"]:
name = job["name"].replace("|", "\\|").replace("\n", " ")
diff --git a/scripts/test-desktop.py b/scripts/test-desktop.py
index da9f2068d..f850c58bc 100644
--- a/scripts/test-desktop.py
+++ b/scripts/test-desktop.py
@@ -9,6 +9,7 @@ import importlib.util
import json
import os
from pathlib import Path
+import re
import shutil
import subprocess
import sys
@@ -148,11 +149,29 @@ def select_files(desktop, paths):
return sorted(set(selected))
+def shard_pair(value):
+ if not re.fullmatch(r"[1-9][0-9]*/[1-9][0-9]*", value):
+ raise argparse.ArgumentTypeError("shard must be INDEX/TOTAL, starting at 1")
+ index, total = map(int, value.split("/"))
+ if index > total:
+ raise argparse.ArgumentTypeError("shard index exceeds total")
+ return index, total
+
+
+def partition(files, shard):
+ index, total = shard
+ selected = files[index - 1::total]
+ if not selected:
+ raise ValueError("shard has no test files; reduce the shard count")
+ return selected
+
+
def main(argv=None):
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("files", nargs="*", help="desktop-relative test/foo_test.dart paths; default: every VM test")
parser.add_argument("--flutter", default="flutter", help="Flutter executable (dependencies must already be installed)")
parser.add_argument("--workers", type=int, default=default_workers())
+ parser.add_argument("--shard", type=shard_pair, default=(1, 1), help="run INDEX/TOTAL of the complete selected file inventory")
parser.add_argument("--timeout", type=float, default=900, help="total attempt budget in seconds, including recovery (default: 900)")
parser.add_argument("--no-loader-retry", action="store_true", help="retain the first failure without recovery")
args = parser.parse_args(argv)
@@ -162,7 +181,8 @@ def main(argv=None):
parser.error("workers and timeout must be positive and finite")
root = Path(subprocess.check_output(["git", "rev-parse", "--show-toplevel"], text=True).strip()).resolve()
desktop = root / "desktop"
- selected = select_files(desktop, args.files)
+ inventory = select_files(desktop, args.files)
+ selected = partition(inventory, args.shard)
flutter = shutil.which(args.flutter)
if flutter is None:
raise ValueError(f"Flutter executable unavailable: {args.flutter}")
@@ -173,7 +193,10 @@ def main(argv=None):
deadline = started + args.timeout
receipt = dict(schema=1, kind="desktop-vm", started_at=validation.utc_now(), source=validation.source_state(root),
workers=workers, logical_cpus=os.cpu_count(), timeout_seconds=args.timeout,
- selected_files=selected, checks=[], status="blocked")
+ selected_files=selected, checks=[], status="blocked", platform=sys.platform,
+ desktop_root=str(desktop.resolve()),
+ shard=dict(index=args.shard[0], total=args.shard[1],
+ inventory=[str(Path(path).relative_to(desktop)) for path in inventory]))
# Reuse the common runner's source/toolchain/environment identity and owned
# process cleanup. This driver never reuses results from an earlier invocation.
identity_plan = {"checks": [{"name": "desktop-vm", "argv": [flutter, "test"], "cwd": "desktop",
diff --git a/scripts/tests/test_desktop_test_shards.py b/scripts/tests/test_desktop_test_shards.py
new file mode 100644
index 000000000..22c93f7c1
--- /dev/null
+++ b/scripts/tests/test_desktop_test_shards.py
@@ -0,0 +1,148 @@
+import copy
+import importlib.util
+import json
+from pathlib import Path
+import subprocess
+import tempfile
+import unittest
+
+from desktop_vm_fixture import suite
+
+spec = importlib.util.spec_from_file_location("desktop_shards", Path(__file__).resolve().parents[1] / "verify-desktop-test-shards.py")
+verifier = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(verifier)
+
+
+class DesktopShardTests(unittest.TestCase):
+ def setUp(self):
+ self.tmp = tempfile.TemporaryDirectory()
+ self.addCleanup(self.tmp.cleanup)
+ self.root = Path(self.tmp.name).resolve() / "source"
+ tests = self.root / "desktop/test"
+ tests.mkdir(parents=True)
+ for name in ["one", "two"]:
+ (tests / f"{name}_test.dart").write_text("// fixture\n")
+ for args in [["init", "-q"], ["add", "."], ["-c", "user.name=Fixture", "-c", "[email protected]", "commit", "-qm", "fixture"]]:
+ subprocess.run(["git", *args], cwd=self.root, capture_output=True, check=True)
+ self.source = verifier.driver.validation.source_state(self.root)
+ self.directory = self.root.parent / "reports"
+ self.records = []
+ self.inventory = ["test/one_test.dart", "test/two_test.dart"]
+ for label, platform in verifier.PLATFORMS.items():
+ for index in [1, 2]:
+ folder = self.directory / f"desktop-shard-{label}-{index}" / "run"
+ folder.mkdir(parents=True)
+ desktop = f"/producer/{label}/desktop"
+ selected = [desktop + "/" + self.inventory[index - 1]]
+ check = self.check(folder, selected, "desktop-vm")
+ record = dict(schema=1, kind="desktop-vm", platform=platform, source=self.source,
+ source_after=self.source, identity={"tool": "same"}, identity_after={"tool": "same"},
+ desktop_root=desktop, selected_files=selected, checks=[check], status="passed",
+ coverage=dict(files=1, verified_files=1, passed=1, skipped=1),
+ shard=dict(index=index, total=2, inventory=self.inventory))
+ path = folder / "receipt.json"
+ self.write(path, record)
+ self.records.append((path, record))
+
+ def write(self, path, record):
+ path.write_text(json.dumps(record))
+
+ def check(self, folder, selected, name, behavior="pass"):
+ log = folder / (name + ".log")
+ events = suite(selected[0], behavior=behavior) + [dict(type="done", success=behavior == "pass")]
+ log.write_text("\n".join(json.dumps(e) for e in events) + "\n")
+ return dict(name=name, status="passed" if behavior == "pass" else "failed",
+ exit_code=0 if behavior == "pass" else 1, log="/producer/" + log.name,
+ log_sha256=verifier.driver.validation.file_sha256(log),
+ report=verifier.driver.summarize(log, selected))
+
+ def verify(self, result="success"):
+ return verifier.verify(self.directory, self.root, 2, result)
+
+ def test_complete_logs_cover_each_platform_once_with_separate_counts(self):
+ result = self.verify()
+ self.assertEqual(result["status"], "passed")
+ self.assertEqual(result["source"], self.source)
+ self.assertEqual(len(result["platforms"]), 2)
+ for platform in result["platforms"]:
+ self.assertEqual(platform["verified_files"], self.inventory)
+ self.assertEqual((platform["files"], platform["passed"], platform["skipped"]), (2, 2, 2))
+
+ def test_failed_matrix_cannot_be_hidden_by_passing_reports(self):
+ for result in ["failure", "cancelled", "skipped", ""]:
+ with self.subTest(result=result), self.assertRaises(ValueError):
+ self.verify(result)
+
+ def test_missing_and_extra_artifacts_are_rejected(self):
+ extra = self.directory / "desktop-shard-macos-15-3"
+ extra.mkdir()
+ with self.assertRaises(ValueError):
+ self.verify()
+ extra.rmdir()
+ self.records[0][0].unlink()
+ with self.assertRaises(ValueError):
+ self.verify()
+
+ def test_source_platform_assignment_inventory_and_counts_must_agree(self):
+ path, original = self.records[0]
+ changes = [dict(source=dict(self.source, commit="f" * 40)), dict(platform="other"),
+ dict(identity_after={"tool": "changed"}), dict(selected_files=["/outside/test.dart"]),
+ dict(shard=dict(index=2, total=2, inventory=self.inventory)),
+ dict(shard=dict(index=1, total=2, inventory=self.inventory[:1])),
+ dict(coverage=dict(files=1, verified_files=1, passed=999, skipped=1))]
+ for change in changes:
+ with self.subTest(change=change):
+ self.write(path, dict(original, **change))
+ with self.assertRaises(ValueError):
+ self.verify()
+ self.write(path, original)
+
+ def test_changed_log_or_summary_does_not_pass(self):
+ path, original = self.records[0]
+ modified = copy.deepcopy(original)
+ modified["checks"][0]["report"]["reported_success"] = False
+ self.write(path, modified)
+ with self.assertRaises(ValueError):
+ self.verify()
+ self.write(path, original)
+ (path.parent / "desktop-vm.log").write_text("truncated\n")
+ with self.assertRaises(ValueError):
+ self.verify()
+
+ def test_raw_failure_or_incomplete_report_cannot_claim_success(self):
+ path, original = self.records[0]
+ for behavior in ["assertion", "incomplete", "loader"]:
+ with self.subTest(behavior=behavior):
+ record = copy.deepcopy(original)
+ check = self.check(path.parent, record["selected_files"], "desktop-vm", behavior)
+ check.update(status="passed", exit_code=0)
+ record["checks"] = [check]
+ self.write(path, record)
+ with self.assertRaises(ValueError):
+ self.verify()
+
+ def test_only_the_existing_narrow_loader_recovery_is_accepted(self):
+ path, record = self.records[0]
+ record = copy.deepcopy(record)
+ record["checks"] = [self.check(path.parent, record["selected_files"], "desktop-vm", "loader"),
+ self.check(path.parent, record["selected_files"], "desktop-vm-loader-retry")]
+ record.update(status="passed_after_startup_retry", retried_files=record["selected_files"])
+ self.write(path, record)
+ result = self.verify()
+ self.assertEqual(result["platforms"][0]["shards"][0]["recovered_files"], ["test/one_test.dart"])
+ record["checks"][0] = self.check(path.parent, record["selected_files"], "desktop-vm", "assertion")
+ self.write(path, record)
+ with self.assertRaises(ValueError):
+ self.verify()
+
+ def test_cleanup_failure_blocks_aggregation(self):
+ path, record = self.records[0]
+ record = copy.deepcopy(record)
+ record["checks"][0]["cleanup_error"] = "child remained"
+ self.write(path, record)
+ with self.assertRaises(ValueError):
+ self.verify()
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/scripts/tests/test_desktop_tests.py b/scripts/tests/test_desktop_tests.py
index 2fc7dbb0d..385a98974 100644
--- a/scripts/tests/test_desktop_tests.py
+++ b/scripts/tests/test_desktop_tests.py
@@ -23,6 +23,18 @@ class WorkerCountTests(unittest.TestCase):
with self.subTest(cpus=cpus), mock.patch.object(driver.os, "cpu_count", return_value=cpus):
self.assertEqual(driver.default_workers(), expected)
+ def test_shards_cover_every_file_once_and_reject_invalid_or_empty_assignments(self):
+ files = [f"test/{i}_test.dart" for i in range(11)]
+ groups = [driver.partition(files, driver.shard_pair(f"{i}/4")) for i in range(1, 5)]
+ selected = [path for group in groups for path in group]
+ self.assertEqual(set(selected), set(files))
+ self.assertEqual(len(selected), len(set(selected)))
+ for value in ["0/4", "1/0", "5/4", "1", "-1/4", "1/4/2", "01/4"]:
+ with self.subTest(value=value), self.assertRaises(driver.argparse.ArgumentTypeError):
+ driver.shard_pair(value)
+ with self.assertRaises(ValueError):
+ driver.partition(files[:1], (2, 2))
+
class ReporterTests(unittest.TestCase):
def setUp(self):
@@ -137,6 +149,14 @@ class DesktopProcessTests(unittest.TestCase):
self.assertEqual(receipt["source"], receipt["source_after"])
self.assertEqual(receipt["identity"], receipt["identity_after"])
+ def test_shard_executes_only_its_files_and_records_the_full_inventory(self):
+ result, receipt, calls = self.run_driver(extra=("--shard", "2/2"))
+ self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
+ self.assertEqual(receipt["shard"], dict(index=2, total=2, inventory=["test/one_test.dart", "test/two_test.dart"]))
+ self.assertEqual([Path(p).name for p in receipt["selected_files"]], ["two_test.dart"])
+ self.assertEqual([Path(p).name for p in calls[0] if p.endswith("_test.dart")], ["two_test.dart"])
+ self.assertEqual(receipt["coverage"], dict(files=1, verified_files=1, passed=1, skipped=1))
+
def test_recovery_runs_only_the_failed_loader_once_and_preserves_both_logs(self):
result, receipt, calls = self.run_driver("once")
self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
diff --git a/scripts/tests/test_record_ci_validation.py b/scripts/tests/test_record_ci_validation.py
index 43c2f590c..755c176a0 100644
--- a/scripts/tests/test_record_ci_validation.py
+++ b/scripts/tests/test_record_ci_validation.py
@@ -73,11 +73,11 @@ class EvidenceTests(unittest.TestCase):
raise AssertionError(path)
def pages(self, path, key):
- return copy.deepcopy(fixture.jobs if key == "jobs" else [fixture.artifact])
+ return copy.deepcopy(fixture.jobs if key == "jobs" else getattr(fixture, "artifacts", [fixture.artifact]))
def command(self, *args, binary=False):
fixture.command_args = args
- return fixture.archive
+ return fixture.archives[args[-1]] if hasattr(fixture, "archives") else fixture.archive
return recorder.collect(FakeClient(), self.root, 123, scope, target, self.output, 5 if with_pr else None)
@@ -98,6 +98,75 @@ class EvidenceTests(unittest.TestCase):
self.assertNotIn("cli_summary", result)
self.assertFalse((self.output / "cli-test-summary.zip").exists())
+ def desktop_fixture(self):
+ self.jobs = [dict(self.jobs[0], id=i, name=name) for i, name in
+ enumerate(sorted(recorder.DESKTOP_JOBS | {"process-checks"}), 1)]
+ files = [f"test/{i}_test.dart" for i in range(4)]
+ self.summary = dict(schema=1, kind="desktop-vm-ci", status="passed",
+ source=dict(commit=self.sha, tree=self.git("rev-parse", "HEAD^{tree}"), dirty=False),
+ platforms=[dict(platform=name, files=4, passed=4, skipped=0, verified_files=files,
+ shards=[dict(shard=i, files=1, verified_files=1, passed=1, skipped=0,
+ recovered_files=[], receipt_sha256="a" * 64) for i in range(1, 5)])
+ for name in sorted(recorder.DESKTOP_PLATFORMS)])
+ self.desktop_archive()
+
+ def desktop_archive(self):
+ self.archive = self.zip_summary(self.summary, "desktop-test-summary.json")
+ self.artifact.update(name="desktop-test-summary", size_in_bytes=len(self.archive),
+ digest="sha256:" + hashlib.sha256(self.archive).hexdigest())
+
+ def test_desktop_scope_verifies_both_platforms_and_preserves_the_artifact(self):
+ self.desktop_fixture()
+ result = self.collect(scope="desktop", with_pr=True)
+ self.assertEqual(result["status"], "passed")
+ self.assertEqual(len(result["desktop_summary"]["platforms"]), 2)
+ self.assertIn("Desktop VM `macos-15`", recorder.markdown(result))
+ self.assertEqual((self.output / "desktop-test-summary.zip").read_bytes(), self.archive)
+ self.assertNotIn("cli_summary", result)
+
+ def test_desktop_missing_platform_bad_counts_and_wrong_tree_are_rejected(self):
+ self.desktop_fixture()
+ original = copy.deepcopy(self.summary)
+ for change in [lambda s: s["platforms"].pop(),
+ lambda s: s["source"].update(tree="b" * 40),
+ lambda s: s["platforms"][0].update(passed=True),
+ lambda s: s["platforms"][0]["shards"].pop(),
+ lambda s: s["platforms"][0]["shards"][0].update(verified_files=0),
+ lambda s: s["platforms"][0]["shards"][0].update(recovered_files=["test/absent_test.dart"])]:
+ self.summary = copy.deepcopy(original)
+ change(self.summary)
+ self.desktop_archive()
+ with self.assertRaises(ValueError):
+ self.collect(scope="desktop")
+
+ def test_desktop_recovery_is_explicit_and_every_matrix_job_is_required(self):
+ self.desktop_fixture()
+ self.summary["platforms"][0]["shards"][0]["recovered_files"] = ["test/0_test.dart"]
+ self.desktop_archive()
+ self.assertIn("1 explicitly recorded pre-test loader recoveries", recorder.markdown(self.collect(scope="desktop")))
+ self.jobs.pop()
+ with self.assertRaisesRegex(ValueError, "missing jobs"):
+ self.collect(scope="desktop")
+
+ def test_full_scope_requires_and_retains_cli_and_desktop_summaries(self):
+ cli_archive, cli_artifact = self.archive, copy.deepcopy(self.artifact)
+ self.desktop_fixture()
+ self.artifact["id"] = 988
+ self.artifacts = [cli_artifact, self.artifact]
+ self.archives = {"repos/owner/repo/actions/artifacts/987/zip": cli_archive,
+ "repos/owner/repo/actions/artifacts/988/zip": self.archive}
+ self.jobs = [dict(self.jobs[0], id=i, name=name) for i, name in
+ enumerate(sorted(recorder.SCOPES["full"] | {"process-checks"}), 1)]
+ result = self.collect(scope="full")
+ self.assertEqual(result["status"], "passed")
+ self.assertIn("cli_summary", result)
+ self.assertIn("desktop_summary", result)
+ self.assertEqual(result["artifact"]["id"], 987)
+ self.assertEqual(result["desktop_artifact"]["id"], 988)
+ self.artifacts.pop()
+ with self.assertRaisesRegex(ValueError, "Desktop coverage summary"):
+ self.collect(scope="full")
+
def test_partial_scope_cannot_be_called_full_ci(self):
with self.assertRaisesRegex(ValueError, "missing jobs"):
self.collect(scope="full")
diff --git a/scripts/verify-desktop-test-shards.py b/scripts/verify-desktop-test-shards.py
new file mode 100644
index 000000000..19c4ecb8d
--- /dev/null
+++ b/scripts/verify-desktop-test-shards.py
@@ -0,0 +1,138 @@
+#!/usr/bin/env python3
+"""Verify complete Desktop VM coverage from bounded runner receipts and raw logs."""
+import argparse
+from collections import Counter
+import importlib.util
+import json
+from pathlib import Path
+import sys
+
+spec = importlib.util.spec_from_file_location("desktop_tests", Path(__file__).with_name("test-desktop.py"))
+driver = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(driver)
+PLATFORMS = {"ubuntu-22.04": "linux", "macos-15": "darwin"}
+
+
+def verify_receipt(path, source, inventory, platform, index, shards):
+ receipt = json.loads(path.read_text())
+ if (receipt.get("schema") != 1 or receipt.get("kind") != "desktop-vm"
+ or receipt.get("platform") != platform or source.get("dirty")
+ or receipt.get("source") != source or receipt.get("source_after") != source
+ or not receipt.get("identity") or receipt.get("identity") != receipt.get("identity_after")):
+ raise ValueError("receipt source, platform or environment differs")
+ shard = receipt.get("shard", {})
+ if (type(shard.get("index")) is not int or type(shard.get("total")) is not int
+ or shard != dict(index=index, total=shards, inventory=inventory)):
+ raise ValueError("shard identity or full discovery inventory differs")
+ desktop = Path(receipt["desktop_root"])
+ if not desktop.is_absolute() or ".." in desktop.parts:
+ raise ValueError("invalid producer checkout path")
+
+ def relative(value):
+ path = Path(value)
+ if not path.is_absolute() or ".." in path.parts:
+ raise ValueError("invalid reported test path")
+ return path.relative_to(desktop).as_posix()
+
+ selected = receipt["selected_files"]
+ if [relative(p) for p in selected] != driver.partition(inventory, (index, shards)):
+ raise ValueError("selected files do not match the assigned shard")
+ checks = receipt.get("checks", [])
+ if not isinstance(checks, list) or len(checks) not in {1, 2}:
+ raise ValueError("missing or unexpected test attempts")
+
+ def report(check, paths, name):
+ if check.get("cleanup_error"):
+ raise ValueError("test process cleanup failed")
+ if check.get("name") != name or Path(check["log"]).name != name + ".log":
+ raise ValueError("unexpected attempt or log path")
+ log = path.parent / (name + ".log")
+ if log.is_symlink() or driver.validation.file_sha256(log) != check.get("log_sha256"):
+ raise ValueError("test log digest differs")
+ actual = driver.summarize(log, paths)
+ if actual != check.get("report"):
+ raise ValueError("receipt disagrees with its raw test log")
+ return actual
+
+ first = report(checks[0], selected, "desktop-vm")
+ results = dict(first["files"])
+ retried = []
+ if len(checks) == 1:
+ if (receipt.get("status") != "passed" or checks[0].get("exit_code") != 0
+ or not driver.passed(checks[0], first)):
+ raise ValueError("test attempt did not pass completely")
+ else:
+ candidates = driver.retry_files(checks[0], first)
+ if (not candidates or receipt.get("retried_files") != candidates
+ or receipt.get("status") != "passed_after_startup_retry"):
+ raise ValueError("invalid loader recovery")
+ retried = [relative(p) for p in candidates]
+ recovered = report(checks[1], candidates, "desktop-vm-loader-retry")
+ if checks[1].get("exit_code") != 0 or not driver.passed(checks[1], recovered):
+ raise ValueError("loader recovery did not pass completely")
+ results.update(recovered["files"])
+ if any(item["status"] != "passed" for item in results.values()):
+ raise ValueError("a test file remains incomplete")
+ coverage = dict(files=len(results), verified_files=len(results),
+ passed=sum(item["passed"] for item in results.values()),
+ skipped=sum(item["skipped"] for item in results.values()))
+ if coverage != receipt.get("coverage"):
+ raise ValueError("receipt coverage totals disagree with its logs")
+ return {relative(p): result for p, result in results.items()}, dict(
+ shard=index, **coverage, recovered_files=retried,
+ receipt_sha256=driver.validation.file_sha256(path))
+
+
+def verify(directory, root, shards, tests_result):
+ if tests_result != "success":
+ raise ValueError(f"the complete Desktop matrix must pass: {tests_result}")
+ expected_folders = {f"desktop-shard-{label}-{index}" for label in PLATFORMS for index in range(1, shards + 1)}
+ if {path.name for path in directory.glob("desktop-shard-*")} != expected_folders:
+ raise ValueError("missing or unexpected shard artifacts")
+ source = driver.validation.source_state(root)
+ inventory = [Path(p).relative_to(root / "desktop").as_posix()
+ for p in driver.select_files(root / "desktop", [])]
+ platforms = []
+ for label, platform in PLATFORMS.items():
+ seen, counts = Counter(), Counter(passed=0, skipped=0)
+ records = []
+ for index in range(1, shards + 1):
+ folder = directory / f"desktop-shard-{label}-{index}"
+ receipts = list(folder.rglob("receipt.json"))
+ if len(receipts) != 1 or receipts[0].is_symlink():
+ raise ValueError(f"{label} shard {index}: expected exactly one receipt")
+ results, record = verify_receipt(receipts[0], source, inventory, platform, index, shards)
+ seen.update(results.keys())
+ counts.update({key: record[key] for key in counts})
+ records.append(record)
+ if set(seen) != set(inventory) or any(count != 1 for count in seen.values()):
+ raise ValueError(f"{label}: files are missing, unexpected or duplicated")
+ platforms.append(dict(platform=label, files=len(seen), **dict(counts),
+ verified_files=sorted(seen), shards=records))
+ return dict(schema=1, kind="desktop-vm-ci", status="passed", source=source,
+ platforms=platforms)
+
+
+def main():
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("directory", type=Path)
+ parser.add_argument("--root", type=Path, required=True)
+ parser.add_argument("--shards", type=int, default=4)
+ parser.add_argument("--tests-result", required=True)
+ parser.add_argument("--output", type=Path, required=True)
+ args = parser.parse_args()
+ if args.shards < 1:
+ parser.error("shards must be positive")
+ result = verify(args.directory, args.root.resolve(), args.shards, args.tests_result)
+ args.output.write_text(json.dumps(result, indent=2) + "\n")
+ for platform in result["platforms"]:
+ print(f"{platform['platform']}: {platform['files']} files verified exactly once; "
+ f"{platform['passed']} passed, {platform['skipped']} skipped")
+
+
+if __name__ == "__main__":
+ try:
+ main()
+ except (OSError, ValueError, KeyError, TypeError) as error:
+ print(f"Desktop shard verification failed: {error}", file=sys.stderr)
+ sys.exit(1)