Skip to content

armgit / moli

Public Imported from GitHub
0

Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust

fix(ci): stabilize frontend reference observations

ldm0d2afd104ab65
diff --git a/.github/scripts/render-ci-regression-comment.cjs b/.github/scripts/render-ci-regression-comment.cjs
index c6a12d7b4..5e06dcf1c 100644
--- a/.github/scripts/render-ci-regression-comment.cjs
+++ b/.github/scripts/render-ci-regression-comment.cjs
@@ -355,9 +355,10 @@ function frontendOverview(section) {
const total = summary.results.length;
const matches = (count(summary.counts.match) ?? 0) + (count(summary.counts.reference_ok) ?? 0);
const problems = Math.max(0, total - matches);
+ const recoveries = count(objectAt(summary, 'referenceGate').recoveredCases) ?? 0;
return {
status: summary.ok === true && problems === 0 ? '✅' : '❌',
- signal: `${formatInteger(matches)}/${formatInteger(total)} cases matched; ${formatInteger(problems)} issues`,
+ signal: `${formatInteger(matches)}/${formatInteger(total)} cases matched; ${formatInteger(problems)} issues${recoveries ? `; ${formatInteger(recoveries)} Chromium reference recoveries` : ''}`,
};
}
@@ -375,6 +376,24 @@ function renderFrontend(section) {
'| ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |',
`| ${formatInteger(count(summary.counts.match) ?? 0)} | ${formatInteger(count(summary.counts.dom_mismatch) ?? 0)} | ${formatInteger(count(summary.counts.diagnostic_mismatch) ?? 0)} | ${formatInteger(count(summary.counts.moli_error) ?? 0)} | ${formatInteger(count(summary.counts.reference_error) ?? 0)} | ${formatInteger(count(summary.counts.infrastructure_error) ?? 0)} | ${formatInteger(timeline.mismatchedFrames)} | ${formatDurationMilliseconds(summary.durationMs)} |`
);
+ const recoveredReferences = summary.results
+ .filter((result) => {
+ const chromium = objectAt(result, 'chromium');
+ return Array.isArray(chromium.previous_failures) && chromium.previous_failures.length !== 0;
+ })
+ .slice(0, MAX_DETAIL_ROWS);
+ if (recoveredReferences.length !== 0) {
+ lines.push('', '**Recovered Chromium reference infrastructure failures**', '');
+ for (const result of recoveredReferences) {
+ const chromium = objectAt(result, 'chromium');
+ const failures = chromium.previous_failures;
+ const first = isObject(failures[0]) ? failures[0] : {};
+ const error = typeof first.error === 'string' ? first.error : 'unknown';
+ lines.push(
+ `- ${code(result.id, 160)} — ${formatInteger(failures.length)} failed attempt(s) before recovery · ${code(error, 160)}`
+ );
+ }
+ }
const issues = summary.results
.filter((result) => isObject(result) && !['match', 'reference_ok'].includes(result.status))
.slice(0, MAX_DETAIL_ROWS);
diff --git a/.github/scripts/render-ci-regression-comment.test.cjs b/.github/scripts/render-ci-regression-comment.test.cjs
index a24adc11b..4ef1765c3 100644
--- a/.github/scripts/render-ci-regression-comment.test.cjs
+++ b/.github/scripts/render-ci-regression-comment.test.cjs
@@ -116,11 +116,21 @@ function createArtifacts(root) {
const frontendRoot = path.join(root, 'frontend');
writeJson(frontendRoot, 'summary.json', {
ok: false,
+ referenceGate: { recoveredCases: 1 },
counts: { match: 1, dom_mismatch: 1 },
durationMs: 1_250,
timeline: { chromiumFrames: 5, moliFrames: 5, mismatchedFrames: 1 },
results: [
- { id: 'safe', status: 'match', mismatchedFrames: [] },
+ {
+ id: 'safe',
+ status: 'match',
+ mismatchedFrames: [],
+ chromium: {
+ previous_failures: [
+ { error: 'Target.createBrowserContext failed: unknown' },
+ ],
+ },
+ },
{
id: 'bad|case<',
status: 'dom_mismatch',
@@ -197,6 +207,9 @@ test('renders all five trusted artifact sections into one bounded report', (t) =
assert.match(report, /artifacts: `5\/5`/);
assert.match(report, /Raw binary \| 100 B \| 110 B \| \+10 B \| \+10\.000000%/);
assert.match(report, /Frontend differential/);
+ assert.match(report, /1 Chromium reference recoveries/);
+ assert.match(report, /Recovered Chromium reference infrastructure failures/);
+ assert.match(report, /Target\.createBrowserContext failed: unknown/);
assert.match(report, /`bad\\\|case&lt;`/);
assert.match(report, /first difference `\$\.body\\\|&lt;`/);
assert.match(report, /frames `ready\\\|&lt;`/);
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 5bd7cfc65..199b92756 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -446,6 +446,7 @@ jobs:
--chromium-bin "${{ steps.chromium.outputs.binary }}"
--moli-bin "$PWD/target/ci-bin/moli-release-head/moli"
--timeout-ms 30000
+ --reference-infrastructure-retries 1
--output target/frontend-smoke
- name: Validate frontend artifacts
id: analysis
diff --git a/moli-frontend-smoke/README.md b/moli-frontend-smoke/README.md
index 115994104..91fcc141a 100644
--- a/moli-frontend-smoke/README.md
+++ b/moli-frontend-smoke/README.md
@@ -95,6 +95,11 @@ an explicit concurrency-boundary probe. Chromium-only baseline calibration can u
concurrency, but a release conclusion should also repeat the reference run and compare every frame
hash across scheduling modes.
+Each worker keeps one browser-level CDP connection and creates an isolated browser context per
+case. CI may pass `--reference-infrastructure-retries 1` to re-observe a case after a Chromium
+connection or protocol failure. The retry does not apply to fixture errors, normalization errors,
+Moli failures, or DOM differences, and the original failure remains recorded in `summary.json`.
+
Run Chromium as a reference-only fixture gate:
```bash
diff --git a/moli-frontend-smoke/moli_frontend_smoke/analysis.py b/moli-frontend-smoke/moli_frontend_smoke/analysis.py
index 41093b215..bf19470c9 100644
--- a/moli-frontend-smoke/moli_frontend_smoke/analysis.py
+++ b/moli-frontend-smoke/moli_frontend_smoke/analysis.py
@@ -1003,6 +1003,28 @@ def _validate_observation_contract(
errors: list[str] = []
if not isinstance(observation, dict) or observation.get("ok") is not True:
return [f"{prefix}: {engine} observation is not successful"], 0
+ attempt_count = observation.get("attempt_count", 1)
+ previous_failures = observation.get("previous_failures", [])
+ if (
+ not isinstance(attempt_count, int)
+ or isinstance(attempt_count, bool)
+ or attempt_count < 1
+ ):
+ errors.append(f"{prefix}: attempt count is invalid")
+ if not isinstance(previous_failures, list):
+ errors.append(f"{prefix}: previous failures is not a list")
+ elif isinstance(attempt_count, int) and not isinstance(attempt_count, bool):
+ if attempt_count != len(previous_failures) + 1:
+ errors.append(f"{prefix}: attempt count does not match previous failures")
+ for index, failure in enumerate(previous_failures, start=1):
+ if (
+ not isinstance(failure, dict)
+ or failure.get("attempt") != index
+ or failure.get("kind") != "infrastructure"
+ ):
+ errors.append(
+ f"{prefix}: previous failure[{index - 1}] is invalid"
+ )
if not _is_nonempty_string(observation.get("dom_hash")):
errors.append(f"{prefix}: final DOM hash is missing")
if not _is_node_count(observation.get("node_count")):
@@ -1133,6 +1155,8 @@ def _validate_reference_summary(
ids: set[str] = set()
frame_total = 0
+ retried_cases = 0
+ recovered_cases = 0
for position, result in enumerate(results):
case_id = result.get("id")
prefix = f"{label}: result[{position}]"
@@ -1151,15 +1175,36 @@ def _validate_reference_summary(
None,
}:
errors.append(f"{prefix}: status invalidates the Chromium reference phase")
+ chromium_observation = result.get("chromium")
+ if isinstance(chromium_observation, dict):
+ attempt_count = chromium_observation.get("attempt_count", 1)
+ if (
+ isinstance(attempt_count, int)
+ and not isinstance(attempt_count, bool)
+ and attempt_count > 1
+ ):
+ retried_cases += 1
+ previous_failures = chromium_observation.get("previous_failures", [])
+ if (
+ chromium_observation.get("ok") is True
+ and isinstance(previous_failures, list)
+ and previous_failures
+ ):
+ recovered_cases += 1
observation_errors, observation_frames = _validate_observation_contract(
case_id=case_id,
- observation=result.get("chromium"),
+ observation=chromium_observation,
prefix=prefix,
engine="Chromium",
)
errors.extend(observation_errors)
frame_total += observation_frames
+ if "retriedCases" in gate and gate.get("retriedCases") != retried_cases:
+ errors.append(f"{label}: retried case count does not match results")
+ if "recoveredCases" in gate and gate.get("recoveredCases") != recovered_cases:
+ errors.append(f"{label}: recovered case count does not match results")
+
counts = summary.get("counts")
if mode == "reference" and counts != {"reference_ok": len(results)}:
errors.append(f"{label}: strict counts do not match reference results")
diff --git a/moli-frontend-smoke/moli_frontend_smoke/cdp.py b/moli-frontend-smoke/moli_frontend_smoke/cdp.py
index ba43c3e32..e5492dcd3 100644
--- a/moli-frontend-smoke/moli_frontend_smoke/cdp.py
+++ b/moli-frontend-smoke/moli_frontend_smoke/cdp.py
@@ -17,6 +17,10 @@ class CdpError(RuntimeError):
pass
+class FixtureObservationError(CdpError):
+ """The page or fixture violated its deterministic observation contract."""
+
+
class CdpCommandError(CdpError):
def __init__(self, method: str, error: dict[str, Any]) -> None:
super().__init__(f"{method} failed: {error}")
@@ -227,13 +231,13 @@ async def _capture_document(
timeout=timeout,
)
if barrier.get("exceptionDetails"):
- raise CdpError(
+ raise FixtureObservationError(
"observable-tree materialization threw: "
f"{barrier['exceptionDetails']}"
)
materialized = (barrier.get("result") or {}).get("value")
if not isinstance(materialized, int) or materialized < 1:
- raise CdpError(
+ raise FixtureObservationError(
"observable-tree materialization returned invalid element count: "
f"{materialized!r}"
)
@@ -325,17 +329,19 @@ def _reconcile_expected_diagnostics(
) -> None:
expected_container = ready_state.get("expectedDiagnostics") or {}
if not isinstance(expected_container, dict):
- raise CdpError("fixture expectedDiagnostics is not an object")
+ raise FixtureObservationError("fixture expectedDiagnostics is not an object")
expected = expected_container.get("networkFailures") or []
if not isinstance(expected, list):
- raise CdpError("fixture expected network failures are not a list")
+ raise FixtureObservationError("fixture expected network failures are not a list")
unmatched = list(diagnostics.get("networkFailures") or [])
matched: list[dict[str, Any]] = []
missing: list[dict[str, Any]] = []
labels: set[str] = set()
for position, item in enumerate(expected):
if not isinstance(item, dict):
- raise CdpError(f"expected network failure {position} is not an object")
+ raise FixtureObservationError(
+ f"expected network failure {position} is not an object"
+ )
label = item.get("label")
url = item.get("url")
resource_type = item.get("type")
@@ -350,7 +356,9 @@ def _reconcile_expected_diagnostics(
or not resource_type
or not isinstance(canceled, bool)
):
- raise CdpError(f"invalid expected network failure {position}: {item!r}")
+ raise FixtureObservationError(
+ f"invalid expected network failure {position}: {item!r}"
+ )
labels.add(label)
match_index = next(
(
@@ -407,9 +415,10 @@ async def observe_case(
case: SmokeCase,
url: str,
timeout_ms: int,
+ client: RawCdpClient | None = None,
) -> EngineObservation:
started = time.perf_counter()
- client: RawCdpClient | None = None
+ owns_client = client is None
context_id: str | None = None
session_id: str | None = None
ready_state: dict[str, Any] | None = None
@@ -417,7 +426,10 @@ async def observe_case(
final_root: dict[str, Any] | None = None
diagnostics: dict[str, Any] = {}
try:
- client = await connect(endpoint)
+ if client is None:
+ client = await connect(endpoint)
+ else:
+ client.events.clear()
context = await client.command("Target.createBrowserContext")
context_id = context.get("browserContextId")
if not isinstance(context_id, str) or not context_id:
@@ -471,29 +483,37 @@ async def observe_case(
timeout=command_timeout,
)
if evaluation.get("exceptionDetails"):
- raise CdpError(
+ raise FixtureObservationError(
f"state Runtime.evaluate threw: {evaluation['exceptionDetails']}"
)
state_value = (evaluation.get("result") or {}).get("value")
if not isinstance(state_value, dict):
- raise CdpError(f"fixture returned invalid state: {state_value!r}")
+ raise FixtureObservationError(
+ f"fixture returned invalid state: {state_value!r}"
+ )
ready_state = state_value
if ready_state.get("id") != case.id:
- raise CdpError(f"fixture state id mismatch: {ready_state!r}")
+ raise FixtureObservationError(
+ f"fixture state id mismatch: {ready_state!r}"
+ )
phase = ready_state.get("phase")
if phase == "checkpoint":
pending = ready_state.get("pendingFrame")
if not isinstance(pending, dict):
- raise CdpError(f"checkpoint has no pending frame: {ready_state!r}")
+ raise FixtureObservationError(
+ f"checkpoint has no pending frame: {ready_state!r}"
+ )
index = pending.get("index")
name = pending.get("name")
token = pending.get("token")
if index != len(frames) or not isinstance(name, str) or not isinstance(token, str):
- raise CdpError(
+ raise FixtureObservationError(
f"invalid checkpoint sequence at frame {len(frames)}: {pending!r}"
)
if len(frames) >= 32:
- raise CdpError("fixture exceeded the 32-frame safety limit")
+ raise FixtureObservationError(
+ "fixture exceeded the 32-frame safety limit"
+ )
root = await _capture_document(
client,
session_id=session_id,
@@ -512,21 +532,27 @@ async def observe_case(
timeout=command_timeout,
)
if resumed.get("exceptionDetails"):
- raise CdpError(
+ raise FixtureObservationError(
f"frame resume Runtime.evaluate threw: {resumed['exceptionDetails']}"
)
if (resumed.get("result") or {}).get("value") is not True:
- raise CdpError(f"fixture refused frame resume token {token!r}")
+ raise FixtureObservationError(
+ f"fixture refused frame resume token {token!r}"
+ )
after_token = token
continue
if phase != "ready":
- raise CdpError(f"fixture did not become ready: {ready_state!r}")
+ raise FixtureObservationError(
+ f"fixture did not become ready: {ready_state!r}"
+ )
if ready_state.get("errors"):
- raise CdpError(f"fixture reported errors: {ready_state!r}")
+ raise FixtureObservationError(
+ f"fixture reported errors: {ready_state!r}"
+ )
reported_frames = ready_state.get("frames")
observed_names = [frame.name for frame in frames]
if reported_frames != observed_names:
- raise CdpError(
+ raise FixtureObservationError(
"fixture frame history mismatch: "
f"reported {reported_frames!r}, observed {observed_names!r}"
)
@@ -548,7 +574,7 @@ async def observe_case(
diagnostics = _diagnostics(client.events, session_id)
_reconcile_expected_diagnostics(diagnostics, ready_state)
if _diagnostics_have_errors(diagnostics):
- raise CdpError(
+ raise FixtureObservationError(
"fixture emitted browser diagnostics: "
+ json.dumps(diagnostics, ensure_ascii=False, separators=(",", ":"))
)
@@ -574,6 +600,11 @@ async def observe_case(
diagnostics=diagnostics,
error_type=type(error).__name__,
error=str(error),
+ failure_kind=(
+ "fixture"
+ if isinstance(error, FixtureObservationError)
+ else "infrastructure"
+ ),
)
finally:
if client is not None:
@@ -586,4 +617,5 @@ async def observe_case(
)
except Exception:
pass
- await client.websocket.close()
+ if owns_client:
+ await client.websocket.close()
diff --git a/moli-frontend-smoke/moli_frontend_smoke/models.py b/moli-frontend-smoke/moli_frontend_smoke/models.py
index 38a449a68..fa0ca1bcc 100644
--- a/moli-frontend-smoke/moli_frontend_smoke/models.py
+++ b/moli-frontend-smoke/moli_frontend_smoke/models.py
@@ -6,6 +6,7 @@ from typing import Any, Literal
FrameworkName = Literal["react", "vue", "angular"]
Complexity = Literal["simple", "medium", "complex"]
+ObservationFailureKind = Literal["infrastructure", "fixture", "normalization"]
CaseStatus = Literal[
"match",
"reference_ok",
@@ -61,6 +62,15 @@ class DomFrameObservation:
return value
+@dataclass(frozen=True)
+class ObservationFailure:
+ attempt: int
+ kind: ObservationFailureKind
+ duration_ms: float
+ error_type: str | None
+ error: str | None
+
+
@dataclass
class EngineObservation:
engine: str
@@ -74,6 +84,9 @@ class EngineObservation:
diagnostics: dict[str, Any] = field(default_factory=dict)
error_type: str | None = None
error: str | None = None
+ failure_kind: ObservationFailureKind | None = None
+ attempt_count: int = 1
+ previous_failures: list[ObservationFailure] = field(default_factory=list)
def summary_json(self) -> dict[str, Any]:
value = asdict(self)
diff --git a/moli-frontend-smoke/moli_frontend_smoke/runner.py b/moli-frontend-smoke/moli_frontend_smoke/runner.py
index 9b8dc8afa..b9221d358 100644
--- a/moli-frontend-smoke/moli_frontend_smoke/runner.py
+++ b/moli-frontend-smoke/moli_frontend_smoke/runner.py
@@ -21,7 +21,7 @@ from .browsers import (
start_moli,
wait_for_cdp_endpoint,
)
-from .cdp import observe_case
+from .cdp import RawCdpClient, connect, observe_case
from .config import (
PROJECT_ROOT,
REPO_ROOT,
@@ -33,7 +33,7 @@ from .config import (
from .dom import dom_hash, first_difference, iter_nodes, normalize_dom_node, unified_dom_diff
from .fixture_server import FixtureServer
from .manifest import SmokeManifest, load_manifest, select_cases
-from .models import CaseResult, EngineObservation, SmokeCase
+from .models import CaseResult, EngineObservation, ObservationFailure, SmokeCase
def _split_values(values: Iterable[str]) -> set[str]:
@@ -257,6 +257,25 @@ def _write_failure_artifact(
return str(case_dir.relative_to(output_dir))
+async def _close_cdp_client(client: RawCdpClient | None) -> None:
+ if client is None:
+ return
+ try:
+ await client.websocket.close()
+ except Exception:
+ pass
+
+
+def _attempt_failure(attempt: int, observation: EngineObservation) -> ObservationFailure:
+ return ObservationFailure(
+ attempt=attempt,
+ kind=observation.failure_kind or "infrastructure",
+ duration_ms=observation.duration_ms,
+ error_type=observation.error_type,
+ error=observation.error,
+ )
+
+
async def _observe_phase_case(
*,
engine: str,
@@ -264,33 +283,90 @@ async def _observe_phase_case(
case: SmokeCase,
fixture_url: str,
timeout_ms: int,
- semaphore: asyncio.Semaphore,
+ client: RawCdpClient | None,
+ infrastructure_retries: int,
progress: dict[str, int],
total: int,
-) -> EngineObservation:
- async with semaphore:
- url = fixture_url.rstrip("/") + case.path
- observation = await observe_case(
- engine=engine,
- endpoint=endpoint,
- case=case,
- url=url,
- timeout_ms=timeout_ms,
- )
+) -> tuple[EngineObservation, RawCdpClient | None]:
+ url = fixture_url.rstrip("/") + case.path
+ previous_failures: list[ObservationFailure] = []
+ attempt = 0
+ while True:
+ attempt += 1
+ if client is None:
+ connect_started = time.perf_counter()
+ try:
+ client = await connect(endpoint)
+ except Exception as error:
+ observation = EngineObservation(
+ engine=engine,
+ ok=False,
+ duration_ms=(time.perf_counter() - connect_started) * 1000,
+ error_type=type(error).__name__,
+ error=f"CDP connection failed: {error}",
+ failure_kind="infrastructure",
+ )
+ else:
+ observation = await observe_case(
+ engine=engine,
+ endpoint=endpoint,
+ case=case,
+ url=url,
+ timeout_ms=timeout_ms,
+ client=client,
+ )
+ else:
+ observation = await observe_case(
+ engine=engine,
+ endpoint=endpoint,
+ case=case,
+ url=url,
+ timeout_ms=timeout_ms,
+ client=client,
+ )
try:
_normalize_observation(observation)
except Exception as error:
observation.ok = False
observation.error_type = type(error).__name__
observation.error = f"DOM normalization failed: {error}"
- progress["completed"] += 1
- state = "ok" if observation.ok else "error"
+ observation.failure_kind = "normalization"
+
+ if observation.failure_kind == "infrastructure":
+ await _close_cdp_client(client)
+ client = None
+
+ should_retry = (
+ engine == "chromium"
+ and observation.failure_kind == "infrastructure"
+ and attempt <= infrastructure_retries
+ )
+ if not should_retry:
+ break
+ previous_failures.append(_attempt_failure(attempt, observation))
+ error = (observation.error or observation.error_type or "unknown").replace(
+ "\n", " "
+ )
print(
- f"[{engine} {progress['completed']:>3}/{total}] {state:<5} {case.id}",
+ f"[{engine} retry {attempt}/{infrastructure_retries}] "
+ f"{case.id}: {error[:240]}",
file=sys.stderr,
flush=True,
)
- return observation
+
+ observation.duration_ms += sum(
+ failure.duration_ms for failure in previous_failures
+ )
+ observation.attempt_count = attempt
+ observation.previous_failures = previous_failures
+ progress["completed"] += 1
+ state = "ok" if observation.ok else "error"
+ print(
+ f"[{engine} {progress['completed']:>3}/{total}] {state:<5} {case.id}",
+ file=sys.stderr,
+ flush=True,
+ )
+ return observation, client
async def _observe_phase(
@@ -301,27 +377,43 @@ async def _observe_phase(
fixture_url: str,
timeout_ms: int,
jobs: int,
+ infrastructure_retries: int = 0,
) -> dict[str, EngineObservation]:
- semaphore = asyncio.Semaphore(jobs)
+ if not cases:
+ return {}
+ queue: asyncio.Queue[SmokeCase] = asyncio.Queue()
+ for case in cases:
+ queue.put_nowait(case)
progress = {"completed": 0}
- observations = await asyncio.gather(
- *[
- asyncio.create_task(
- _observe_phase_case(
+ observations: dict[str, EngineObservation] = {}
+
+ async def worker() -> None:
+ client: RawCdpClient | None = None
+ try:
+ while True:
+ try:
+ case = queue.get_nowait()
+ except asyncio.QueueEmpty:
+ return
+ observation, client = await _observe_phase_case(
engine=engine,
endpoint=endpoint,
case=case,
fixture_url=fixture_url,
timeout_ms=timeout_ms,
- semaphore=semaphore,
+ client=client,
+ infrastructure_retries=infrastructure_retries,
progress=progress,
total=len(cases),
)
- )
- for case in cases
- ]
+ observations[case.id] = observation
+ finally:
+ await _close_cdp_client(client)
+
+ await asyncio.gather(
+ *(asyncio.create_task(worker()) for _ in range(min(jobs, len(cases))))
)
- return {case.id: observation for case, observation in zip(cases, observations, strict=True)}
+ return {case.id: observations[case.id] for case in cases}
def _case_result(
@@ -402,6 +494,9 @@ def _default_output_dir() -> Path:
async def run(args: argparse.Namespace) -> tuple[int, dict[str, Any]]:
+ reference_infrastructure_retries = getattr(
+ args, "reference_infrastructure_retries", 0
+ )
manifest_path = Path(args.manifest).expanduser().resolve()
manifest = load_manifest(manifest_path)
if not args.allow_partial_manifest:
@@ -452,6 +547,7 @@ async def run(args: argparse.Namespace) -> tuple[int, dict[str, Any]]:
fixture_url=fixture.url,
timeout_ms=args.timeout_ms,
jobs=args.jobs,
+ infrastructure_retries=reference_infrastructure_retries,
)
reference_gate_ok = all(
observation.ok for observation in chromium_observations.values()
@@ -479,6 +575,7 @@ async def run(args: argparse.Namespace) -> tuple[int, dict[str, Any]]:
fixture_url=fixture.url,
timeout_ms=args.timeout_ms,
jobs=args.jobs,
+ infrastructure_retries=0,
)
effective_reference_only = args.reference_only or not reference_gate_ok
@@ -531,6 +628,15 @@ async def run(args: argparse.Namespace) -> tuple[int, dict[str, Any]]:
1 for observation in chromium_observations.values() if not observation.ok
),
"moliPhaseStarted": bool(moli_observations),
+ "infrastructureRetryLimit": reference_infrastructure_retries,
+ "retriedCases": sum(
+ observation.attempt_count > 1
+ for observation in chromium_observations.values()
+ ),
+ "recoveredCases": sum(
+ observation.ok and bool(observation.previous_failures)
+ for observation in chromium_observations.values()
+ ),
},
"fixture": fixture.url,
"engines": {
@@ -622,6 +728,15 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
default=15_000,
help="Per-page fixture ready timeout.",
)
+ parser.add_argument(
+ "--reference-infrastructure-retries",
+ type=int,
+ default=0,
+ help=(
+ "Retry only Chromium CDP/infrastructure observation failures; fixture, "
+ "normalization, and Moli failures remain single-attempt gates."
+ ),
+ )
parser.add_argument("--output", help="Artifact output directory.")
parser.add_argument("--chromium-bin", help="Chromium executable.")
parser.add_argument("--moli-bin", help="Moli executable.")
@@ -643,6 +758,8 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
parser.error("--jobs must be between 1 and 32")
if args.timeout_ms < 100 or args.timeout_ms > 120_000:
parser.error("--timeout-ms must be between 100 and 120000")
+ if not 0 <= args.reference_infrastructure_retries <= 3:
+ parser.error("--reference-infrastructure-retries must be between 0 and 3")
if args.reference_only and args.moli_endpoint:
parser.error("--reference-only cannot be combined with --moli-endpoint")
return args
diff --git a/moli-frontend-smoke/tests/test_analysis.py b/moli-frontend-smoke/tests/test_analysis.py
index 526b2ab73..d67e86d42 100644
--- a/moli-frontend-smoke/tests/test_analysis.py
+++ b/moli-frontend-smoke/tests/test_analysis.py
@@ -138,6 +138,41 @@ def test_reference_stability_ignores_endpoint_but_requires_every_frame_hash() ->
]
+def test_reference_contract_preserves_recovered_infrastructure_failures() -> None:
+ first = _reference_summary()
+ first["referenceGate"].update( # type: ignore[union-attr]
+ {
+ "infrastructureRetryLimit": 1,
+ "retriedCases": 1,
+ "recoveredCases": 1,
+ }
+ )
+ first["results"][0]["chromium"].update( # type: ignore[index,union-attr]
+ {
+ "attempt_count": 2,
+ "previous_failures": [
+ {
+ "attempt": 1,
+ "kind": "infrastructure",
+ "duration_ms": 1.0,
+ "error_type": "CdpCommandError",
+ "error": "Target.createBrowserContext failed: unknown",
+ }
+ ],
+ }
+ )
+ second = deepcopy(first)
+
+ assert compare_reference_data(first, second)["ok"] # type: ignore[arg-type]
+
+ second["referenceGate"]["recoveredCases"] = 0 # type: ignore[index]
+ result = compare_reference_data(first, second) # type: ignore[arg-type]
+ assert not result["ok"]
+ assert result["secondValidationErrors"] == [
+ "second: recovered case count does not match results"
+ ]
+
+
def test_projection_is_explicitly_non_gating_and_can_isolate_ua_edges() -> None:
case = {"id": "react/family/case", "framework": "react"}
chromium = {
diff --git a/moli-frontend-smoke/tests/test_cdp.py b/moli-frontend-smoke/tests/test_cdp.py
index 796186103..a55a133c3 100644
--- a/moli-frontend-smoke/tests/test_cdp.py
+++ b/moli-frontend-smoke/tests/test_cdp.py
@@ -2,7 +2,11 @@ from __future__ import annotations
import asyncio
+import pytest
+
from moli_frontend_smoke.cdp import (
+ CdpCommandError,
+ FixtureObservationError,
_DOM_ENABLE_PARAMS,
_OBSERVABLE_TREE_BARRIER_EXPRESSION,
_capture_document,
@@ -11,7 +15,106 @@ from moli_frontend_smoke.cdp import (
_reconcile_expected_diagnostics,
_resume_expression,
_state_expression,
+ observe_case,
)
+from moli_frontend_smoke.models import SmokeCase
+
+
+def _case() -> SmokeCase:
+ return SmokeCase(
+ id="vue/web-platform-integration/formdata-searchparams-pipeline",
+ framework="vue",
+ family="web-platform-integration",
+ complexity="complex",
+ slug="formdata-searchparams-pipeline",
+ title="FormData and URLSearchParams pipeline",
+ variant=0,
+ seed=1,
+ size=4,
+ path="/cases/vue/web-platform-integration/formdata-searchparams-pipeline/index.html",
+ )
+
+
+class _FakeWebSocket:
+ def __init__(self) -> None:
+ self.close_count = 0
+
+ async def close(self) -> None:
+ self.close_count += 1
+
+
+class _FixtureFailureClient:
+ def __init__(self) -> None:
+ self.websocket = _FakeWebSocket()
+ self.events: list[dict[str, object]] = []
+ self.methods: list[str] = []
+
+ async def command(self, method, params=None, **_kwargs):
+ self.methods.append(method)
+ if method == "Target.createBrowserContext":
+ return {"browserContextId": "context"}
+ if method == "Target.createTarget":
+ return {"targetId": "target"}
+ if method == "Target.attachToTarget":
+ return {"sessionId": "session"}
+ if method == "Runtime.evaluate":
+ return {"exceptionDetails": {"text": "fixture failed"}}
+ return {}
+
+ async def wait_event(self, *_args, **_kwargs):
+ return {}
+
+
+def test_observe_case_distinguishes_fixture_failures_from_infrastructure() -> None:
+ client = _FixtureFailureClient()
+
+ observation = asyncio.run(
+ observe_case(
+ engine="chromium",
+ endpoint="http://127.0.0.1:9222",
+ case=_case(),
+ url="http://127.0.0.1:3000/case",
+ timeout_ms=1000,
+ client=client, # type: ignore[arg-type]
+ )
+ )
+
+ assert not observation.ok
+ assert observation.failure_kind == "fixture"
+ assert observation.error_type == "FixtureObservationError"
+ assert "state Runtime.evaluate threw" in (observation.error or "")
+ assert client.methods[-1] == "Target.disposeBrowserContext"
+ assert client.websocket.close_count == 0
+
+
+class _InfrastructureFailureClient:
+ def __init__(self) -> None:
+ self.websocket = _FakeWebSocket()
+ self.events: list[dict[str, object]] = []
+
+ async def command(self, method, _params=None, **_kwargs):
+ raise CdpCommandError(method, {"code": -32000, "message": "unknown"})
+
+
+def test_observe_case_marks_cdp_command_errors_as_infrastructure() -> None:
+ client = _InfrastructureFailureClient()
+
+ observation = asyncio.run(
+ observe_case(
+ engine="chromium",
+ endpoint="http://127.0.0.1:9222",
+ case=_case(),
+ url="http://127.0.0.1:3000/case",
+ timeout_ms=1000,
+ client=client, # type: ignore[arg-type]
+ )
+ )
+
+ assert not observation.ok
+ assert observation.failure_kind == "infrastructure"
+ assert observation.error_type == "CdpCommandError"
+ assert "unknown" in (observation.error or "")
+ assert client.websocket.close_count == 0
def test_diagnostics_preserve_runtime_console_and_network_failures() -> None:
@@ -222,6 +325,21 @@ def test_missing_expected_network_failure_remains_a_diagnostic_error() -> None:
assert _diagnostics_have_errors(diagnostics)
+def test_invalid_expected_diagnostics_is_a_fixture_failure() -> None:
+ diagnostics = {
+ "exceptions": [],
+ "consoleErrors": [],
+ "networkFailures": [],
+ "httpErrors": [],
+ }
+
+ with pytest.raises(FixtureObservationError):
+ _reconcile_expected_diagnostics(
+ diagnostics,
+ {"expectedDiagnostics": {"networkFailures": "invalid"}},
+ )
+
+
def test_state_expression_waits_for_a_new_checkpoint_or_terminal_state() -> None:
expression = _state_expression("react/family/case", 12_345, "old-token")
diff --git a/moli-frontend-smoke/tests/test_runner.py b/moli-frontend-smoke/tests/test_runner.py
index 9473e95c0..49db3f163 100644
--- a/moli-frontend-smoke/tests/test_runner.py
+++ b/moli-frontend-smoke/tests/test_runner.py
@@ -3,6 +3,7 @@ from __future__ import annotations
import asyncio
import json
from collections import deque
+from dataclasses import replace
from types import SimpleNamespace
import moli_frontend_smoke.runner as runner
@@ -14,6 +15,7 @@ from moli_frontend_smoke.models import (
from moli_frontend_smoke.runner import (
_case_result,
_normalize_observation,
+ _observe_phase,
parse_args,
)
@@ -77,6 +79,191 @@ def test_default_differential_is_serial() -> None:
assert parse_args([]).jobs == 1
+def test_reference_infrastructure_retry_is_opt_in() -> None:
+ assert parse_args([]).reference_infrastructure_retries == 0
+ assert (
+ parse_args(["--reference-infrastructure-retries", "1"])
+ .reference_infrastructure_retries
+ == 1
+ )
+
+
+class _FakeWebSocket:
+ def __init__(self) -> None:
+ self.close_count = 0
+
+ async def close(self) -> None:
+ self.close_count += 1
+
+
+class _FakeClient:
+ def __init__(self) -> None:
+ self.websocket = _FakeWebSocket()
+ self.events: list[dict[str, object]] = []
+
+
+def test_observation_worker_reuses_one_cdp_connection(monkeypatch) -> None:
+ clients: list[_FakeClient] = []
+ observed_clients: list[_FakeClient] = []
+
+ async def fake_connect(_endpoint):
+ client = _FakeClient()
+ clients.append(client)
+ return client
+
+ async def fake_observe_case(*, engine, client, **_kwargs):
+ observed_clients.append(client)
+ return _observation(engine, {"nodeType": 9, "nodeName": "#document"})
+
+ monkeypatch.setattr(runner, "connect", fake_connect)
+ monkeypatch.setattr(runner, "observe_case", fake_observe_case)
+ first = _case()
+ second = replace(first, id="react/family/second", slug="second")
+
+ observations = asyncio.run(
+ _observe_phase(
+ engine="chromium",
+ endpoint="http://127.0.0.1:9222",
+ cases=(first, second),
+ fixture_url="http://127.0.0.1:3000",
+ timeout_ms=1000,
+ jobs=1,
+ )
+ )
+
+ assert list(observations) == [first.id, second.id]
+ assert len(clients) == 1
+ assert observed_clients == [clients[0], clients[0]]
+ assert clients[0].websocket.close_count == 1
+
+
+def test_chromium_retries_only_infrastructure_failures(monkeypatch) -> None:
+ clients: list[_FakeClient] = []
+ attempts = 0
+
+ async def fake_connect(_endpoint):
+ client = _FakeClient()
+ clients.append(client)
+ return client
+
+ async def fake_observe_case(*, engine, **_kwargs):
+ nonlocal attempts
+ attempts += 1
+ if attempts == 1:
+ return EngineObservation(
+ engine=engine,
+ ok=False,
+ duration_ms=2.0,
+ error_type="CdpCommandError",
+ error="Target.createBrowserContext failed: unknown",
+ failure_kind="infrastructure",
+ )
+ return _observation(engine, {"nodeType": 9, "nodeName": "#document"})
+
+ monkeypatch.setattr(runner, "connect", fake_connect)
+ monkeypatch.setattr(runner, "observe_case", fake_observe_case)
+
+ observation = asyncio.run(
+ _observe_phase(
+ engine="chromium",
+ endpoint="http://127.0.0.1:9222",
+ cases=(_case(),),
+ fixture_url="http://127.0.0.1:3000",
+ timeout_ms=1000,
+ jobs=1,
+ infrastructure_retries=1,
+ )
+ )[_case().id]
+
+ assert observation.ok
+ assert observation.attempt_count == 2
+ assert observation.duration_ms == 3.0
+ assert [failure.error for failure in observation.previous_failures] == [
+ "Target.createBrowserContext failed: unknown"
+ ]
+ assert len(clients) == 2
+ assert all(client.websocket.close_count == 1 for client in clients)
+
+
+def test_fixture_failure_is_never_retried(monkeypatch) -> None:
+ attempts = 0
+
+ async def fake_connect(_endpoint):
+ return _FakeClient()
+
+ async def fake_observe_case(*, engine, **_kwargs):
+ nonlocal attempts
+ attempts += 1
+ return EngineObservation(
+ engine=engine,
+ ok=False,
+ duration_ms=1.0,
+ error_type="FixtureObservationError",
+ error="fixture assertion failed",
+ failure_kind="fixture",
+ )
+
+ monkeypatch.setattr(runner, "connect", fake_connect)
+ monkeypatch.setattr(runner, "observe_case", fake_observe_case)
+
+ observation = asyncio.run(
+ _observe_phase(
+ engine="chromium",
+ endpoint="http://127.0.0.1:9222",
+ cases=(_case(),),
+ fixture_url="http://127.0.0.1:3000",
+ timeout_ms=1000,
+ jobs=1,
+ infrastructure_retries=1,
+ )
+ )[_case().id]
+
+ assert not observation.ok
+ assert observation.failure_kind == "fixture"
+ assert observation.attempt_count == 1
+ assert observation.previous_failures == []
+ assert attempts == 1
+
+
+def test_moli_failure_is_never_retried(monkeypatch) -> None:
+ attempts = 0
+
+ async def fake_connect(_endpoint):
+ return _FakeClient()
+
+ async def fake_observe_case(*, engine, **_kwargs):
+ nonlocal attempts
+ attempts += 1
+ return EngineObservation(
+ engine=engine,
+ ok=False,
+ duration_ms=1.0,
+ error_type="CdpCommandError",
+ error="candidate connection failed",
+ failure_kind="infrastructure",
+ )
+
+ monkeypatch.setattr(runner, "connect", fake_connect)
+ monkeypatch.setattr(runner, "observe_case", fake_observe_case)
+
+ observation = asyncio.run(
+ _observe_phase(
+ engine="moli",
+ endpoint="http://127.0.0.1:9222",
+ cases=(_case(),),
+ fixture_url="http://127.0.0.1:3000",
+ timeout_ms=1000,
+ jobs=1,
+ infrastructure_retries=1,
+ )
+ )[_case().id]
+
+ assert not observation.ok
+ assert observation.attempt_count == 1
+ assert observation.previous_failures == []
+ assert attempts == 1
+
+
def test_any_moli_tree_difference_is_a_failure(tmp_path) -> None:
chromium = _observation("chromium", {"nodeType": 3, "nodeValue": "reference"})
moli = _observation("moli", {"nodeType": 3, "nodeValue": "different"})
@@ -378,4 +565,7 @@ def test_failed_reference_gate_never_starts_moli(monkeypatch, tmp_path) -> None:
"cases": 1,
"errors": 1,
"moliPhaseStarted": False,
+ "infrastructureRetryLimit": 0,
+ "retriedCases": 0,
+ "recoveredCases": 0,
}