Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust
Rust ai-agentsai-toolsbrowserbrowser-automationcloud-browserheadlessheadless-browserkitesurfplaywrightpuppeteerrustservoweb-crawlerweb-crawlingweb-scraperweb-scraping
fix(ci): stabilize frontend reference observations
ldm0d2afd104ab65
diff --git a/.github/scripts/render-ci-regression-comment.cjs b/.github/scripts/render-ci-regression-comment.cjsindex c6a12d7b4..5e06dcf1c 100644--- a/.github/scripts/render-ci-regression-comment.cjs+++ b/.github/scripts/render-ci-regression-comment.cjs@@ -355,9 +355,10 @@ function frontendOverview(section) {const total = summary.results.length;const matches = (count(summary.counts.match) ?? 0) + (count(summary.counts.reference_ok) ?? 0);const problems = Math.max(0, total - matches);+ const recoveries = count(objectAt(summary, 'referenceGate').recoveredCases) ?? 0;return {status: summary.ok === true && problems === 0 ? '✅' : '❌',- signal: `${formatInteger(matches)}/${formatInteger(total)} cases matched; ${formatInteger(problems)} issues`,+ signal: `${formatInteger(matches)}/${formatInteger(total)} cases matched; ${formatInteger(problems)} issues${recoveries ? `; ${formatInteger(recoveries)} Chromium reference recoveries` : ''}`,};}@@ -375,6 +376,24 @@ function renderFrontend(section) {'| ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |',`| ${formatInteger(count(summary.counts.match) ?? 0)} | ${formatInteger(count(summary.counts.dom_mismatch) ?? 0)} | ${formatInteger(count(summary.counts.diagnostic_mismatch) ?? 0)} | ${formatInteger(count(summary.counts.moli_error) ?? 0)} | ${formatInteger(count(summary.counts.reference_error) ?? 0)} | ${formatInteger(count(summary.counts.infrastructure_error) ?? 0)} | ${formatInteger(timeline.mismatchedFrames)} | ${formatDurationMilliseconds(summary.durationMs)} |`);+ const recoveredReferences = summary.results+ .filter((result) => {+ const chromium = objectAt(result, 'chromium');+ return Array.isArray(chromium.previous_failures) && chromium.previous_failures.length !== 0;+ })+ .slice(0, MAX_DETAIL_ROWS);+ if (recoveredReferences.length !== 0) {+ lines.push('', '**Recovered Chromium reference infrastructure failures**', '');+ for (const result of recoveredReferences) {+ const chromium = objectAt(result, 'chromium');+ const failures = chromium.previous_failures;+ const first = isObject(failures[0]) ? failures[0] : {};+ const error = typeof first.error === 'string' ? first.error : 'unknown';+ lines.push(+ `- ${code(result.id, 160)} — ${formatInteger(failures.length)} failed attempt(s) before recovery · ${code(error, 160)}`+ );+ }+ }const issues = summary.results.filter((result) => isObject(result) && !['match', 'reference_ok'].includes(result.status)).slice(0, MAX_DETAIL_ROWS);diff --git a/.github/scripts/render-ci-regression-comment.test.cjs b/.github/scripts/render-ci-regression-comment.test.cjsindex a24adc11b..4ef1765c3 100644--- a/.github/scripts/render-ci-regression-comment.test.cjs+++ b/.github/scripts/render-ci-regression-comment.test.cjs@@ -116,11 +116,21 @@ function createArtifacts(root) {const frontendRoot = path.join(root, 'frontend');writeJson(frontendRoot, 'summary.json', {ok: false,+ referenceGate: { recoveredCases: 1 },counts: { match: 1, dom_mismatch: 1 },durationMs: 1_250,timeline: { chromiumFrames: 5, moliFrames: 5, mismatchedFrames: 1 },results: [- { id: 'safe', status: 'match', mismatchedFrames: [] },+ {+ id: 'safe',+ status: 'match',+ mismatchedFrames: [],+ chromium: {+ previous_failures: [+ { error: 'Target.createBrowserContext failed: unknown' },+ ],+ },+ },{id: 'bad|case<',status: 'dom_mismatch',@@ -197,6 +207,9 @@ test('renders all five trusted artifact sections into one bounded report', (t) =assert.match(report, /artifacts: `5\/5`/);assert.match(report, /Raw binary \| 100 B \| 110 B \| \+10 B \| \+10\.000000%/);assert.match(report, /Frontend differential/);+ assert.match(report, /1 Chromium reference recoveries/);+ assert.match(report, /Recovered Chromium reference infrastructure failures/);+ assert.match(report, /Target\.createBrowserContext failed: unknown/);assert.match(report, /`bad\\\|case<`/);assert.match(report, /first difference `\$\.body\\\|<`/);assert.match(report, /frames `ready\\\|<`/);diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.ymlindex 5bd7cfc65..199b92756 100644--- a/.github/workflows/ci.yml+++ b/.github/workflows/ci.yml@@ -446,6 +446,7 @@ jobs:--chromium-bin "${{ steps.chromium.outputs.binary }}"--moli-bin "$PWD/target/ci-bin/moli-release-head/moli"--timeout-ms 30000+ --reference-infrastructure-retries 1--output target/frontend-smoke- name: Validate frontend artifactsid: analysisdiff --git a/moli-frontend-smoke/README.md b/moli-frontend-smoke/README.mdindex 115994104..91fcc141a 100644--- a/moli-frontend-smoke/README.md+++ b/moli-frontend-smoke/README.md@@ -95,6 +95,11 @@ an explicit concurrency-boundary probe. Chromium-only baseline calibration can uconcurrency, but a release conclusion should also repeat the reference run and compare every framehash across scheduling modes.+Each worker keeps one browser-level CDP connection and creates an isolated browser context per+case. CI may pass `--reference-infrastructure-retries 1` to re-observe a case after a Chromium+connection or protocol failure. The retry does not apply to fixture errors, normalization errors,+Moli failures, or DOM differences, and the original failure remains recorded in `summary.json`.+Run Chromium as a reference-only fixture gate:```bashdiff --git a/moli-frontend-smoke/moli_frontend_smoke/analysis.py b/moli-frontend-smoke/moli_frontend_smoke/analysis.pyindex 41093b215..bf19470c9 100644--- a/moli-frontend-smoke/moli_frontend_smoke/analysis.py+++ b/moli-frontend-smoke/moli_frontend_smoke/analysis.py@@ -1003,6 +1003,28 @@ def _validate_observation_contract(errors: list[str] = []if not isinstance(observation, dict) or observation.get("ok") is not True:return [f"{prefix}: {engine} observation is not successful"], 0+ attempt_count = observation.get("attempt_count", 1)+ previous_failures = observation.get("previous_failures", [])+ if (+ not isinstance(attempt_count, int)+ or isinstance(attempt_count, bool)+ or attempt_count < 1+ ):+ errors.append(f"{prefix}: attempt count is invalid")+ if not isinstance(previous_failures, list):+ errors.append(f"{prefix}: previous failures is not a list")+ elif isinstance(attempt_count, int) and not isinstance(attempt_count, bool):+ if attempt_count != len(previous_failures) + 1:+ errors.append(f"{prefix}: attempt count does not match previous failures")+ for index, failure in enumerate(previous_failures, start=1):+ if (+ not isinstance(failure, dict)+ or failure.get("attempt") != index+ or failure.get("kind") != "infrastructure"+ ):+ errors.append(+ f"{prefix}: previous failure[{index - 1}] is invalid"+ )if not _is_nonempty_string(observation.get("dom_hash")):errors.append(f"{prefix}: final DOM hash is missing")if not _is_node_count(observation.get("node_count")):@@ -1133,6 +1155,8 @@ def _validate_reference_summary(ids: set[str] = set()frame_total = 0+ retried_cases = 0+ recovered_cases = 0for position, result in enumerate(results):case_id = result.get("id")prefix = f"{label}: result[{position}]"@@ -1151,15 +1175,36 @@ def _validate_reference_summary(None,}:errors.append(f"{prefix}: status invalidates the Chromium reference phase")+ chromium_observation = result.get("chromium")+ if isinstance(chromium_observation, dict):+ attempt_count = chromium_observation.get("attempt_count", 1)+ if (+ isinstance(attempt_count, int)+ and not isinstance(attempt_count, bool)+ and attempt_count > 1+ ):+ retried_cases += 1+ previous_failures = chromium_observation.get("previous_failures", [])+ if (+ chromium_observation.get("ok") is True+ and isinstance(previous_failures, list)+ and previous_failures+ ):+ recovered_cases += 1observation_errors, observation_frames = _validate_observation_contract(case_id=case_id,- observation=result.get("chromium"),+ observation=chromium_observation,prefix=prefix,engine="Chromium",)errors.extend(observation_errors)frame_total += observation_frames+ if "retriedCases" in gate and gate.get("retriedCases") != retried_cases:+ errors.append(f"{label}: retried case count does not match results")+ if "recoveredCases" in gate and gate.get("recoveredCases") != recovered_cases:+ errors.append(f"{label}: recovered case count does not match results")+counts = summary.get("counts")if mode == "reference" and counts != {"reference_ok": len(results)}:errors.append(f"{label}: strict counts do not match reference results")diff --git a/moli-frontend-smoke/moli_frontend_smoke/cdp.py b/moli-frontend-smoke/moli_frontend_smoke/cdp.pyindex ba43c3e32..e5492dcd3 100644--- a/moli-frontend-smoke/moli_frontend_smoke/cdp.py+++ b/moli-frontend-smoke/moli_frontend_smoke/cdp.py@@ -17,6 +17,10 @@ class CdpError(RuntimeError):pass+class FixtureObservationError(CdpError):+ """The page or fixture violated its deterministic observation contract."""++class CdpCommandError(CdpError):def __init__(self, method: str, error: dict[str, Any]) -> None:super().__init__(f"{method} failed: {error}")@@ -227,13 +231,13 @@ async def _capture_document(timeout=timeout,)if barrier.get("exceptionDetails"):- raise CdpError(+ raise FixtureObservationError("observable-tree materialization threw: "f"{barrier['exceptionDetails']}")materialized = (barrier.get("result") or {}).get("value")if not isinstance(materialized, int) or materialized < 1:- raise CdpError(+ raise FixtureObservationError("observable-tree materialization returned invalid element count: "f"{materialized!r}")@@ -325,17 +329,19 @@ def _reconcile_expected_diagnostics() -> None:expected_container = ready_state.get("expectedDiagnostics") or {}if not isinstance(expected_container, dict):- raise CdpError("fixture expectedDiagnostics is not an object")+ raise FixtureObservationError("fixture expectedDiagnostics is not an object")expected = expected_container.get("networkFailures") or []if not isinstance(expected, list):- raise CdpError("fixture expected network failures are not a list")+ raise FixtureObservationError("fixture expected network failures are not a list")unmatched = list(diagnostics.get("networkFailures") or [])matched: list[dict[str, Any]] = []missing: list[dict[str, Any]] = []labels: set[str] = set()for position, item in enumerate(expected):if not isinstance(item, dict):- raise CdpError(f"expected network failure {position} is not an object")+ raise FixtureObservationError(+ f"expected network failure {position} is not an object"+ )label = item.get("label")url = item.get("url")resource_type = item.get("type")@@ -350,7 +356,9 @@ def _reconcile_expected_diagnostics(or not resource_typeor not isinstance(canceled, bool)):- raise CdpError(f"invalid expected network failure {position}: {item!r}")+ raise FixtureObservationError(+ f"invalid expected network failure {position}: {item!r}"+ )labels.add(label)match_index = next((@@ -407,9 +415,10 @@ async def observe_case(case: SmokeCase,url: str,timeout_ms: int,+ client: RawCdpClient | None = None,) -> EngineObservation:started = time.perf_counter()- client: RawCdpClient | None = None+ owns_client = client is Nonecontext_id: str | None = Nonesession_id: str | None = Noneready_state: dict[str, Any] | None = None@@ -417,7 +426,10 @@ async def observe_case(final_root: dict[str, Any] | None = Nonediagnostics: dict[str, Any] = {}try:- client = await connect(endpoint)+ if client is None:+ client = await connect(endpoint)+ else:+ client.events.clear()context = await client.command("Target.createBrowserContext")context_id = context.get("browserContextId")if not isinstance(context_id, str) or not context_id:@@ -471,29 +483,37 @@ async def observe_case(timeout=command_timeout,)if evaluation.get("exceptionDetails"):- raise CdpError(+ raise FixtureObservationError(f"state Runtime.evaluate threw: {evaluation['exceptionDetails']}")state_value = (evaluation.get("result") or {}).get("value")if not isinstance(state_value, dict):- raise CdpError(f"fixture returned invalid state: {state_value!r}")+ raise FixtureObservationError(+ f"fixture returned invalid state: {state_value!r}"+ )ready_state = state_valueif ready_state.get("id") != case.id:- raise CdpError(f"fixture state id mismatch: {ready_state!r}")+ raise FixtureObservationError(+ f"fixture state id mismatch: {ready_state!r}"+ )phase = ready_state.get("phase")if phase == "checkpoint":pending = ready_state.get("pendingFrame")if not isinstance(pending, dict):- raise CdpError(f"checkpoint has no pending frame: {ready_state!r}")+ raise FixtureObservationError(+ f"checkpoint has no pending frame: {ready_state!r}"+ )index = pending.get("index")name = pending.get("name")token = pending.get("token")if index != len(frames) or not isinstance(name, str) or not isinstance(token, str):- raise CdpError(+ raise FixtureObservationError(f"invalid checkpoint sequence at frame {len(frames)}: {pending!r}")if len(frames) >= 32:- raise CdpError("fixture exceeded the 32-frame safety limit")+ raise FixtureObservationError(+ "fixture exceeded the 32-frame safety limit"+ )root = await _capture_document(client,session_id=session_id,@@ -512,21 +532,27 @@ async def observe_case(timeout=command_timeout,)if resumed.get("exceptionDetails"):- raise CdpError(+ raise FixtureObservationError(f"frame resume Runtime.evaluate threw: {resumed['exceptionDetails']}")if (resumed.get("result") or {}).get("value") is not True:- raise CdpError(f"fixture refused frame resume token {token!r}")+ raise FixtureObservationError(+ f"fixture refused frame resume token {token!r}"+ )after_token = tokencontinueif phase != "ready":- raise CdpError(f"fixture did not become ready: {ready_state!r}")+ raise FixtureObservationError(+ f"fixture did not become ready: {ready_state!r}"+ )if ready_state.get("errors"):- raise CdpError(f"fixture reported errors: {ready_state!r}")+ raise FixtureObservationError(+ f"fixture reported errors: {ready_state!r}"+ )reported_frames = ready_state.get("frames")observed_names = [frame.name for frame in frames]if reported_frames != observed_names:- raise CdpError(+ raise FixtureObservationError("fixture frame history mismatch: "f"reported {reported_frames!r}, observed {observed_names!r}")@@ -548,7 +574,7 @@ async def observe_case(diagnostics = _diagnostics(client.events, session_id)_reconcile_expected_diagnostics(diagnostics, ready_state)if _diagnostics_have_errors(diagnostics):- raise CdpError(+ raise FixtureObservationError("fixture emitted browser diagnostics: "+ json.dumps(diagnostics, ensure_ascii=False, separators=(",", ":")))@@ -574,6 +600,11 @@ async def observe_case(diagnostics=diagnostics,error_type=type(error).__name__,error=str(error),+ failure_kind=(+ "fixture"+ if isinstance(error, FixtureObservationError)+ else "infrastructure"+ ),)finally:if client is not None:@@ -586,4 +617,5 @@ async def observe_case()except Exception:pass- await client.websocket.close()+ if owns_client:+ await client.websocket.close()diff --git a/moli-frontend-smoke/moli_frontend_smoke/models.py b/moli-frontend-smoke/moli_frontend_smoke/models.pyindex 38a449a68..fa0ca1bcc 100644--- a/moli-frontend-smoke/moli_frontend_smoke/models.py+++ b/moli-frontend-smoke/moli_frontend_smoke/models.py@@ -6,6 +6,7 @@ from typing import Any, LiteralFrameworkName = Literal["react", "vue", "angular"]Complexity = Literal["simple", "medium", "complex"]+ObservationFailureKind = Literal["infrastructure", "fixture", "normalization"]CaseStatus = Literal["match","reference_ok",@@ -61,6 +62,15 @@ class DomFrameObservation:return value+@dataclass(frozen=True)+class ObservationFailure:+ attempt: int+ kind: ObservationFailureKind+ duration_ms: float+ error_type: str | None+ error: str | None++@dataclassclass EngineObservation:engine: str@@ -74,6 +84,9 @@ class EngineObservation:diagnostics: dict[str, Any] = field(default_factory=dict)error_type: str | None = Noneerror: str | None = None+ failure_kind: ObservationFailureKind | None = None+ attempt_count: int = 1+ previous_failures: list[ObservationFailure] = field(default_factory=list)def summary_json(self) -> dict[str, Any]:value = asdict(self)diff --git a/moli-frontend-smoke/moli_frontend_smoke/runner.py b/moli-frontend-smoke/moli_frontend_smoke/runner.pyindex 9b8dc8afa..b9221d358 100644--- a/moli-frontend-smoke/moli_frontend_smoke/runner.py+++ b/moli-frontend-smoke/moli_frontend_smoke/runner.py@@ -21,7 +21,7 @@ from .browsers import (start_moli,wait_for_cdp_endpoint,)-from .cdp import observe_case+from .cdp import RawCdpClient, connect, observe_casefrom .config import (PROJECT_ROOT,REPO_ROOT,@@ -33,7 +33,7 @@ from .config import (from .dom import dom_hash, first_difference, iter_nodes, normalize_dom_node, unified_dom_difffrom .fixture_server import FixtureServerfrom .manifest import SmokeManifest, load_manifest, select_cases-from .models import CaseResult, EngineObservation, SmokeCase+from .models import CaseResult, EngineObservation, ObservationFailure, SmokeCasedef _split_values(values: Iterable[str]) -> set[str]:@@ -257,6 +257,25 @@ def _write_failure_artifact(return str(case_dir.relative_to(output_dir))+async def _close_cdp_client(client: RawCdpClient | None) -> None:+ if client is None:+ return+ try:+ await client.websocket.close()+ except Exception:+ pass+++def _attempt_failure(attempt: int, observation: EngineObservation) -> ObservationFailure:+ return ObservationFailure(+ attempt=attempt,+ kind=observation.failure_kind or "infrastructure",+ duration_ms=observation.duration_ms,+ error_type=observation.error_type,+ error=observation.error,+ )++async def _observe_phase_case(*,engine: str,@@ -264,33 +283,90 @@ async def _observe_phase_case(case: SmokeCase,fixture_url: str,timeout_ms: int,- semaphore: asyncio.Semaphore,+ client: RawCdpClient | None,+ infrastructure_retries: int,progress: dict[str, int],total: int,-) -> EngineObservation:- async with semaphore:- url = fixture_url.rstrip("/") + case.path- observation = await observe_case(- engine=engine,- endpoint=endpoint,- case=case,- url=url,- timeout_ms=timeout_ms,- )+) -> tuple[EngineObservation, RawCdpClient | None]:+ url = fixture_url.rstrip("/") + case.path+ previous_failures: list[ObservationFailure] = []+ attempt = 0+ while True:+ attempt += 1+ if client is None:+ connect_started = time.perf_counter()+ try:+ client = await connect(endpoint)+ except Exception as error:+ observation = EngineObservation(+ engine=engine,+ ok=False,+ duration_ms=(time.perf_counter() - connect_started) * 1000,+ error_type=type(error).__name__,+ error=f"CDP connection failed: {error}",+ failure_kind="infrastructure",+ )+ else:+ observation = await observe_case(+ engine=engine,+ endpoint=endpoint,+ case=case,+ url=url,+ timeout_ms=timeout_ms,+ client=client,+ )+ else:+ observation = await observe_case(+ engine=engine,+ endpoint=endpoint,+ case=case,+ url=url,+ timeout_ms=timeout_ms,+ client=client,+ )try:_normalize_observation(observation)except Exception as error:observation.ok = Falseobservation.error_type = type(error).__name__observation.error = f"DOM normalization failed: {error}"- progress["completed"] += 1- state = "ok" if observation.ok else "error"+ observation.failure_kind = "normalization"++ if observation.failure_kind == "infrastructure":+ await _close_cdp_client(client)+ client = None++ should_retry = (+ engine == "chromium"+ and observation.failure_kind == "infrastructure"+ and attempt <= infrastructure_retries+ )+ if not should_retry:+ break+ previous_failures.append(_attempt_failure(attempt, observation))+ error = (observation.error or observation.error_type or "unknown").replace(+ "\n", " "+ )print(- f"[{engine} {progress['completed']:>3}/{total}] {state:<5} {case.id}",+ f"[{engine} retry {attempt}/{infrastructure_retries}] "+ f"{case.id}: {error[:240]}",file=sys.stderr,flush=True,)- return observation++ observation.duration_ms += sum(+ failure.duration_ms for failure in previous_failures+ )+ observation.attempt_count = attempt+ observation.previous_failures = previous_failures+ progress["completed"] += 1+ state = "ok" if observation.ok else "error"+ print(+ f"[{engine} {progress['completed']:>3}/{total}] {state:<5} {case.id}",+ file=sys.stderr,+ flush=True,+ )+ return observation, clientasync def _observe_phase(@@ -301,27 +377,43 @@ async def _observe_phase(fixture_url: str,timeout_ms: int,jobs: int,+ infrastructure_retries: int = 0,) -> dict[str, EngineObservation]:- semaphore = asyncio.Semaphore(jobs)+ if not cases:+ return {}+ queue: asyncio.Queue[SmokeCase] = asyncio.Queue()+ for case in cases:+ queue.put_nowait(case)progress = {"completed": 0}- observations = await asyncio.gather(- *[- asyncio.create_task(- _observe_phase_case(+ observations: dict[str, EngineObservation] = {}++ async def worker() -> None:+ client: RawCdpClient | None = None+ try:+ while True:+ try:+ case = queue.get_nowait()+ except asyncio.QueueEmpty:+ return+ observation, client = await _observe_phase_case(engine=engine,endpoint=endpoint,case=case,fixture_url=fixture_url,timeout_ms=timeout_ms,- semaphore=semaphore,+ client=client,+ infrastructure_retries=infrastructure_retries,progress=progress,total=len(cases),)- )- for case in cases- ]+ observations[case.id] = observation+ finally:+ await _close_cdp_client(client)++ await asyncio.gather(+ *(asyncio.create_task(worker()) for _ in range(min(jobs, len(cases)))))- return {case.id: observation for case, observation in zip(cases, observations, strict=True)}+ return {case.id: observations[case.id] for case in cases}def _case_result(@@ -402,6 +494,9 @@ def _default_output_dir() -> Path:async def run(args: argparse.Namespace) -> tuple[int, dict[str, Any]]:+ reference_infrastructure_retries = getattr(+ args, "reference_infrastructure_retries", 0+ )manifest_path = Path(args.manifest).expanduser().resolve()manifest = load_manifest(manifest_path)if not args.allow_partial_manifest:@@ -452,6 +547,7 @@ async def run(args: argparse.Namespace) -> tuple[int, dict[str, Any]]:fixture_url=fixture.url,timeout_ms=args.timeout_ms,jobs=args.jobs,+ infrastructure_retries=reference_infrastructure_retries,)reference_gate_ok = all(observation.ok for observation in chromium_observations.values()@@ -479,6 +575,7 @@ async def run(args: argparse.Namespace) -> tuple[int, dict[str, Any]]:fixture_url=fixture.url,timeout_ms=args.timeout_ms,jobs=args.jobs,+ infrastructure_retries=0,)effective_reference_only = args.reference_only or not reference_gate_ok@@ -531,6 +628,15 @@ async def run(args: argparse.Namespace) -> tuple[int, dict[str, Any]]:1 for observation in chromium_observations.values() if not observation.ok),"moliPhaseStarted": bool(moli_observations),+ "infrastructureRetryLimit": reference_infrastructure_retries,+ "retriedCases": sum(+ observation.attempt_count > 1+ for observation in chromium_observations.values()+ ),+ "recoveredCases": sum(+ observation.ok and bool(observation.previous_failures)+ for observation in chromium_observations.values()+ ),},"fixture": fixture.url,"engines": {@@ -622,6 +728,15 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace:default=15_000,help="Per-page fixture ready timeout.",)+ parser.add_argument(+ "--reference-infrastructure-retries",+ type=int,+ default=0,+ help=(+ "Retry only Chromium CDP/infrastructure observation failures; fixture, "+ "normalization, and Moli failures remain single-attempt gates."+ ),+ )parser.add_argument("--output", help="Artifact output directory.")parser.add_argument("--chromium-bin", help="Chromium executable.")parser.add_argument("--moli-bin", help="Moli executable.")@@ -643,6 +758,8 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace:parser.error("--jobs must be between 1 and 32")if args.timeout_ms < 100 or args.timeout_ms > 120_000:parser.error("--timeout-ms must be between 100 and 120000")+ if not 0 <= args.reference_infrastructure_retries <= 3:+ parser.error("--reference-infrastructure-retries must be between 0 and 3")if args.reference_only and args.moli_endpoint:parser.error("--reference-only cannot be combined with --moli-endpoint")return argsdiff --git a/moli-frontend-smoke/tests/test_analysis.py b/moli-frontend-smoke/tests/test_analysis.pyindex 526b2ab73..d67e86d42 100644--- a/moli-frontend-smoke/tests/test_analysis.py+++ b/moli-frontend-smoke/tests/test_analysis.py@@ -138,6 +138,41 @@ def test_reference_stability_ignores_endpoint_but_requires_every_frame_hash() ->]+def test_reference_contract_preserves_recovered_infrastructure_failures() -> None:+ first = _reference_summary()+ first["referenceGate"].update( # type: ignore[union-attr]+ {+ "infrastructureRetryLimit": 1,+ "retriedCases": 1,+ "recoveredCases": 1,+ }+ )+ first["results"][0]["chromium"].update( # type: ignore[index,union-attr]+ {+ "attempt_count": 2,+ "previous_failures": [+ {+ "attempt": 1,+ "kind": "infrastructure",+ "duration_ms": 1.0,+ "error_type": "CdpCommandError",+ "error": "Target.createBrowserContext failed: unknown",+ }+ ],+ }+ )+ second = deepcopy(first)++ assert compare_reference_data(first, second)["ok"] # type: ignore[arg-type]++ second["referenceGate"]["recoveredCases"] = 0 # type: ignore[index]+ result = compare_reference_data(first, second) # type: ignore[arg-type]+ assert not result["ok"]+ assert result["secondValidationErrors"] == [+ "second: recovered case count does not match results"+ ]++def test_projection_is_explicitly_non_gating_and_can_isolate_ua_edges() -> None:case = {"id": "react/family/case", "framework": "react"}chromium = {diff --git a/moli-frontend-smoke/tests/test_cdp.py b/moli-frontend-smoke/tests/test_cdp.pyindex 796186103..a55a133c3 100644--- a/moli-frontend-smoke/tests/test_cdp.py+++ b/moli-frontend-smoke/tests/test_cdp.py@@ -2,7 +2,11 @@ from __future__ import annotationsimport asyncio+import pytest+from moli_frontend_smoke.cdp import (+ CdpCommandError,+ FixtureObservationError,_DOM_ENABLE_PARAMS,_OBSERVABLE_TREE_BARRIER_EXPRESSION,_capture_document,@@ -11,7 +15,106 @@ from moli_frontend_smoke.cdp import (_reconcile_expected_diagnostics,_resume_expression,_state_expression,+ observe_case,)+from moli_frontend_smoke.models import SmokeCase+++def _case() -> SmokeCase:+ return SmokeCase(+ id="vue/web-platform-integration/formdata-searchparams-pipeline",+ framework="vue",+ family="web-platform-integration",+ complexity="complex",+ slug="formdata-searchparams-pipeline",+ title="FormData and URLSearchParams pipeline",+ variant=0,+ seed=1,+ size=4,+ path="/cases/vue/web-platform-integration/formdata-searchparams-pipeline/index.html",+ )+++class _FakeWebSocket:+ def __init__(self) -> None:+ self.close_count = 0++ async def close(self) -> None:+ self.close_count += 1+++class _FixtureFailureClient:+ def __init__(self) -> None:+ self.websocket = _FakeWebSocket()+ self.events: list[dict[str, object]] = []+ self.methods: list[str] = []++ async def command(self, method, params=None, **_kwargs):+ self.methods.append(method)+ if method == "Target.createBrowserContext":+ return {"browserContextId": "context"}+ if method == "Target.createTarget":+ return {"targetId": "target"}+ if method == "Target.attachToTarget":+ return {"sessionId": "session"}+ if method == "Runtime.evaluate":+ return {"exceptionDetails": {"text": "fixture failed"}}+ return {}++ async def wait_event(self, *_args, **_kwargs):+ return {}+++def test_observe_case_distinguishes_fixture_failures_from_infrastructure() -> None:+ client = _FixtureFailureClient()++ observation = asyncio.run(+ observe_case(+ engine="chromium",+ endpoint="http://127.0.0.1:9222",+ case=_case(),+ url="http://127.0.0.1:3000/case",+ timeout_ms=1000,+ client=client, # type: ignore[arg-type]+ )+ )++ assert not observation.ok+ assert observation.failure_kind == "fixture"+ assert observation.error_type == "FixtureObservationError"+ assert "state Runtime.evaluate threw" in (observation.error or "")+ assert client.methods[-1] == "Target.disposeBrowserContext"+ assert client.websocket.close_count == 0+++class _InfrastructureFailureClient:+ def __init__(self) -> None:+ self.websocket = _FakeWebSocket()+ self.events: list[dict[str, object]] = []++ async def command(self, method, _params=None, **_kwargs):+ raise CdpCommandError(method, {"code": -32000, "message": "unknown"})+++def test_observe_case_marks_cdp_command_errors_as_infrastructure() -> None:+ client = _InfrastructureFailureClient()++ observation = asyncio.run(+ observe_case(+ engine="chromium",+ endpoint="http://127.0.0.1:9222",+ case=_case(),+ url="http://127.0.0.1:3000/case",+ timeout_ms=1000,+ client=client, # type: ignore[arg-type]+ )+ )++ assert not observation.ok+ assert observation.failure_kind == "infrastructure"+ assert observation.error_type == "CdpCommandError"+ assert "unknown" in (observation.error or "")+ assert client.websocket.close_count == 0def test_diagnostics_preserve_runtime_console_and_network_failures() -> None:@@ -222,6 +325,21 @@ def test_missing_expected_network_failure_remains_a_diagnostic_error() -> None:assert _diagnostics_have_errors(diagnostics)+def test_invalid_expected_diagnostics_is_a_fixture_failure() -> None:+ diagnostics = {+ "exceptions": [],+ "consoleErrors": [],+ "networkFailures": [],+ "httpErrors": [],+ }++ with pytest.raises(FixtureObservationError):+ _reconcile_expected_diagnostics(+ diagnostics,+ {"expectedDiagnostics": {"networkFailures": "invalid"}},+ )++def test_state_expression_waits_for_a_new_checkpoint_or_terminal_state() -> None:expression = _state_expression("react/family/case", 12_345, "old-token")diff --git a/moli-frontend-smoke/tests/test_runner.py b/moli-frontend-smoke/tests/test_runner.pyindex 9473e95c0..49db3f163 100644--- a/moli-frontend-smoke/tests/test_runner.py+++ b/moli-frontend-smoke/tests/test_runner.py@@ -3,6 +3,7 @@ from __future__ import annotationsimport asyncioimport jsonfrom collections import deque+from dataclasses import replacefrom types import SimpleNamespaceimport moli_frontend_smoke.runner as runner@@ -14,6 +15,7 @@ from moli_frontend_smoke.models import (from moli_frontend_smoke.runner import (_case_result,_normalize_observation,+ _observe_phase,parse_args,)@@ -77,6 +79,191 @@ def test_default_differential_is_serial() -> None:assert parse_args([]).jobs == 1+def test_reference_infrastructure_retry_is_opt_in() -> None:+ assert parse_args([]).reference_infrastructure_retries == 0+ assert (+ parse_args(["--reference-infrastructure-retries", "1"])+ .reference_infrastructure_retries+ == 1+ )+++class _FakeWebSocket:+ def __init__(self) -> None:+ self.close_count = 0++ async def close(self) -> None:+ self.close_count += 1+++class _FakeClient:+ def __init__(self) -> None:+ self.websocket = _FakeWebSocket()+ self.events: list[dict[str, object]] = []+++def test_observation_worker_reuses_one_cdp_connection(monkeypatch) -> None:+ clients: list[_FakeClient] = []+ observed_clients: list[_FakeClient] = []++ async def fake_connect(_endpoint):+ client = _FakeClient()+ clients.append(client)+ return client++ async def fake_observe_case(*, engine, client, **_kwargs):+ observed_clients.append(client)+ return _observation(engine, {"nodeType": 9, "nodeName": "#document"})++ monkeypatch.setattr(runner, "connect", fake_connect)+ monkeypatch.setattr(runner, "observe_case", fake_observe_case)+ first = _case()+ second = replace(first, id="react/family/second", slug="second")++ observations = asyncio.run(+ _observe_phase(+ engine="chromium",+ endpoint="http://127.0.0.1:9222",+ cases=(first, second),+ fixture_url="http://127.0.0.1:3000",+ timeout_ms=1000,+ jobs=1,+ )+ )++ assert list(observations) == [first.id, second.id]+ assert len(clients) == 1+ assert observed_clients == [clients[0], clients[0]]+ assert clients[0].websocket.close_count == 1+++def test_chromium_retries_only_infrastructure_failures(monkeypatch) -> None:+ clients: list[_FakeClient] = []+ attempts = 0++ async def fake_connect(_endpoint):+ client = _FakeClient()+ clients.append(client)+ return client++ async def fake_observe_case(*, engine, **_kwargs):+ nonlocal attempts+ attempts += 1+ if attempts == 1:+ return EngineObservation(+ engine=engine,+ ok=False,+ duration_ms=2.0,+ error_type="CdpCommandError",+ error="Target.createBrowserContext failed: unknown",+ failure_kind="infrastructure",+ )+ return _observation(engine, {"nodeType": 9, "nodeName": "#document"})++ monkeypatch.setattr(runner, "connect", fake_connect)+ monkeypatch.setattr(runner, "observe_case", fake_observe_case)++ observation = asyncio.run(+ _observe_phase(+ engine="chromium",+ endpoint="http://127.0.0.1:9222",+ cases=(_case(),),+ fixture_url="http://127.0.0.1:3000",+ timeout_ms=1000,+ jobs=1,+ infrastructure_retries=1,+ )+ )[_case().id]++ assert observation.ok+ assert observation.attempt_count == 2+ assert observation.duration_ms == 3.0+ assert [failure.error for failure in observation.previous_failures] == [+ "Target.createBrowserContext failed: unknown"+ ]+ assert len(clients) == 2+ assert all(client.websocket.close_count == 1 for client in clients)+++def test_fixture_failure_is_never_retried(monkeypatch) -> None:+ attempts = 0++ async def fake_connect(_endpoint):+ return _FakeClient()++ async def fake_observe_case(*, engine, **_kwargs):+ nonlocal attempts+ attempts += 1+ return EngineObservation(+ engine=engine,+ ok=False,+ duration_ms=1.0,+ error_type="FixtureObservationError",+ error="fixture assertion failed",+ failure_kind="fixture",+ )++ monkeypatch.setattr(runner, "connect", fake_connect)+ monkeypatch.setattr(runner, "observe_case", fake_observe_case)++ observation = asyncio.run(+ _observe_phase(+ engine="chromium",+ endpoint="http://127.0.0.1:9222",+ cases=(_case(),),+ fixture_url="http://127.0.0.1:3000",+ timeout_ms=1000,+ jobs=1,+ infrastructure_retries=1,+ )+ )[_case().id]++ assert not observation.ok+ assert observation.failure_kind == "fixture"+ assert observation.attempt_count == 1+ assert observation.previous_failures == []+ assert attempts == 1+++def test_moli_failure_is_never_retried(monkeypatch) -> None:+ attempts = 0++ async def fake_connect(_endpoint):+ return _FakeClient()++ async def fake_observe_case(*, engine, **_kwargs):+ nonlocal attempts+ attempts += 1+ return EngineObservation(+ engine=engine,+ ok=False,+ duration_ms=1.0,+ error_type="CdpCommandError",+ error="candidate connection failed",+ failure_kind="infrastructure",+ )++ monkeypatch.setattr(runner, "connect", fake_connect)+ monkeypatch.setattr(runner, "observe_case", fake_observe_case)++ observation = asyncio.run(+ _observe_phase(+ engine="moli",+ endpoint="http://127.0.0.1:9222",+ cases=(_case(),),+ fixture_url="http://127.0.0.1:3000",+ timeout_ms=1000,+ jobs=1,+ infrastructure_retries=1,+ )+ )[_case().id]++ assert not observation.ok+ assert observation.attempt_count == 1+ assert observation.previous_failures == []+ assert attempts == 1++def test_any_moli_tree_difference_is_a_failure(tmp_path) -> None:chromium = _observation("chromium", {"nodeType": 3, "nodeValue": "reference"})moli = _observation("moli", {"nodeType": 3, "nodeValue": "different"})@@ -378,4 +565,7 @@ def test_failed_reference_gate_never_starts_moli(monkeypatch, tmp_path) -> None:"cases": 1,"errors": 1,"moliPhaseStarted": False,+ "infrastructureRetryLimit": 0,+ "retriedCases": 0,+ "recoveredCases": 0,}