Skip to content

armgit / moli

Public Imported from GitHub
0

Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust

ci: report 200-navigation memory soak

ldm0ac31b289d4dd
diff --git a/.github/workflows/sequential-navigation-soak-comment.yml b/.github/workflows/sequential-navigation-soak-comment.yml
new file mode 100644
index 000000000..66ce5fa36
--- /dev/null
+++ b/.github/workflows/sequential-navigation-soak-comment.yml
@@ -0,0 +1,143 @@
+name: Sequential Navigation Soak Comment
+
+on:
+ workflow_run:
+ workflows: [Sequential Navigation Soak]
+ types: [completed]
+
+# This workflow runs from the default branch. It never executes the measured
+# PR binary and only renders bounded numeric fields from the downloaded artifact.
+permissions:
+ actions: read
+ contents: read
+ pull-requests: write
+
+jobs:
+ comment:
+ name: Update PR soak comment
+ if: >-
+ github.event.workflow_run.event == 'pull_request' &&
+ github.event.workflow_run.conclusion != 'cancelled' &&
+ github.event.workflow_run.pull_requests[0].head.repo.id == github.event.repository.id &&
+ github.event.workflow_run.actor.login != 'dependabot[bot]'
+ runs-on: ubuntu-latest
+ timeout-minutes: 10
+ steps:
+ - name: Checkout trusted comment renderer
+ uses: actions/checkout@v4
+ with:
+ persist-credentials: false
+ ref: ${{ github.event.repository.default_branch }}
+
+ - name: Download soak evidence
+ id: evidence
+ continue-on-error: true
+ uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
+ with:
+ name: sequential-navigation-soak-results
+ path: sequential-navigation-soak-results
+ github-token: ${{ secrets.GITHUB_TOKEN }}
+ run-id: ${{ github.event.workflow_run.id }}
+
+ - name: Render trusted PR comment
+ id: render
+ env:
+ PYTHONPATH: moli-benchmark
+ RUN_URL: https://github.com/${{ github.repository }}/actions/runs/${{ github.event.workflow_run.id }}
+ CONCLUSION: ${{ github.event.workflow_run.conclusion }}
+ EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }}
+ run: |
+ if [[ "$EVIDENCE_OUTCOME" == success && -f sequential-navigation-soak-results/comparison.json ]]; then
+ python3 -m moli_benchmark.sequential_navigate_ci comment \
+ --input sequential-navigation-soak-results/comparison.json \
+ --run-url "$RUN_URL" \
+ --conclusion "$CONCLUSION" \
+ --output sequential-navigation-soak-comment.md
+ echo "has_comparison=true" >> "$GITHUB_OUTPUT"
+ else
+ python3 -m moli_benchmark.sequential_navigate_ci infrastructure-comment \
+ --run-url "$RUN_URL" \
+ --conclusion "$CONCLUSION" \
+ --output sequential-navigation-soak-comment.md
+ echo "has_comparison=false" >> "$GITHUB_OUTPUT"
+ fi
+
+ - name: Post PR soak comment
+ uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7
+ env:
+ COMMENT_PATH: sequential-navigation-soak-comment.md
+ COMPARISON_PATH: sequential-navigation-soak-results/comparison.json
+ HAS_COMPARISON: ${{ steps.render.outputs.has_comparison }}
+ with:
+ script: |
+ const fs = require('fs');
+ const run = context.payload.workflow_run;
+ const association = run.pull_requests?.[0];
+ if (!association) {
+ core.notice('No pull request is associated with this workflow run.');
+ return;
+ }
+
+ const pull = await github.rest.pulls.get({
+ owner: context.repo.owner,
+ repo: context.repo.repo,
+ pull_number: association.number,
+ });
+ if (pull.data.head.sha !== association.head.sha) {
+ core.notice('Skipping a stale Sequential Navigation Soak result.');
+ return;
+ }
+
+ if (process.env.HAS_COMPARISON === 'true') {
+ const comparison = JSON.parse(fs.readFileSync(process.env.COMPARISON_PATH, 'utf8'));
+ const compared = await github.rest.repos.compareCommitsWithBasehead({
+ owner: context.repo.owner,
+ repo: context.repo.repo,
+ basehead: `${association.base.sha}...${association.head.sha}`,
+ });
+ const expectedCommonAncestor = compared.data.merge_base_commit?.sha;
+ if (
+ comparison.schema !== 'moli.sequential-navigation.comparison.v1' ||
+ comparison.expected_navigations !== 200 ||
+ comparison.head?.sha !== association.head.sha ||
+ comparison.base?.sha !== expectedCommonAncestor
+ ) {
+ core.setFailed(
+ 'Sequential Navigation Soak artifact identities do not match the PR.'
+ );
+ return;
+ }
+ }
+
+ const body = fs.readFileSync(process.env.COMMENT_PATH, 'utf8');
+ const marker = '<!-- moli-sequential-navigation-soak -->';
+ if (!body.includes(marker) || Buffer.byteLength(body, 'utf8') > 32 * 1024) {
+ core.setFailed('Rendered soak comment is missing its marker or exceeds 32 KiB.');
+ return;
+ }
+ const comments = await github.paginate(github.rest.issues.listComments, {
+ owner: context.repo.owner,
+ repo: context.repo.repo,
+ issue_number: association.number,
+ per_page: 100,
+ });
+ const existing = comments.find(
+ (comment) =>
+ comment.user?.login === 'github-actions[bot]' &&
+ comment.body?.includes(marker)
+ );
+ if (existing) {
+ await github.rest.issues.updateComment({
+ owner: context.repo.owner,
+ repo: context.repo.repo,
+ comment_id: existing.id,
+ body,
+ });
+ } else {
+ await github.rest.issues.createComment({
+ owner: context.repo.owner,
+ repo: context.repo.repo,
+ issue_number: association.number,
+ body,
+ });
+ }
diff --git a/.github/workflows/sequential-navigation-soak.yml b/.github/workflows/sequential-navigation-soak.yml
new file mode 100644
index 000000000..0d60f5df7
--- /dev/null
+++ b/.github/workflows/sequential-navigation-soak.yml
@@ -0,0 +1,211 @@
+name: Sequential Navigation Soak
+
+on:
+ pull_request:
+ types: [opened, synchronize, reopened]
+ workflow_dispatch:
+ inputs:
+ base_ref:
+ description: Base commit or ref
+ required: false
+ default: main
+ head_ref:
+ description: Head commit or ref (defaults to the dispatched commit)
+ required: false
+ default: ""
+
+concurrency:
+ group: sequential-navigation-soak-${{ github.event.pull_request.number || github.ref }}
+ cancel-in-progress: true
+
+# Candidate code builds and runs without a token capable of writing to GitHub.
+# A separate workflow_run workflow renders the artifact from the default branch.
+permissions:
+ contents: read
+
+env:
+ CARGO_TERM_COLOR: always
+ CARGO_INCREMENTAL: "0"
+
+jobs:
+ compare:
+ name: 200 navigations · Base vs HEAD
+ if: >-
+ github.event_name != 'pull_request' ||
+ (github.event.pull_request.head.repo.full_name == github.repository &&
+ github.actor != 'dependabot[bot]')
+ runs-on: [self-hosted, moli-ci]
+ timeout-minutes: 90
+ container: rust:1.96.1-bookworm
+ env:
+ BENCHMARK_OUTPUT: sequential-navigation-soak-results
+ BINARY_DIR: /tmp/moli-sequential-navigation-binaries
+ TARGET_BASE_REF: ${{ github.event.pull_request.base.sha || inputs.base_ref || github.event.repository.default_branch }}
+ HEAD_REF: ${{ github.event.pull_request.head.sha || inputs.head_ref || github.sha }}
+ steps:
+ # The exact HEAD harness drives both binaries, so an A/B run uses one
+ # report schema and one URL order. This job has read-only permissions.
+ - name: Checkout exact HEAD harness
+ uses: actions/checkout@v4
+ with:
+ fetch-depth: 0
+ persist-credentials: false
+ ref: ${{ github.event.pull_request.head.sha || inputs.head_ref || github.sha }}
+
+ - name: Resolve commits and initialize evidence
+ id: refs
+ shell: bash
+ run: |
+ set -euo pipefail
+ git config --global --add safe.directory "$(pwd -P)"
+
+ mkdir -p "$BENCHMARK_OUTPUT"
+ printf '%s\n' '{"schema":"moli.sequential-navigation.run-metadata.v1","status":"initializing"}' \
+ > "$BENCHMARK_OUTPUT/run-metadata.json"
+
+ target_base_sha=$(git rev-parse "${TARGET_BASE_REF}^{commit}")
+ head_sha=$(git rev-parse "${HEAD_REF}^{commit}")
+ base_sha=$(git merge-base "$target_base_sha" "$head_sha")
+ git merge-base --is-ancestor "$base_sha" "$target_base_sha"
+ git merge-base --is-ancestor "$base_sha" "$head_sha"
+ echo "target_base_sha=$target_base_sha" >> "$GITHUB_OUTPUT"
+ echo "base_sha=$base_sha" >> "$GITHUB_OUTPUT"
+ echo "head_sha=$head_sha" >> "$GITHUB_OUTPUT"
+
+ if (( GITHUB_RUN_NUMBER % 2 == 0 )); then
+ execution_order=head-first
+ else
+ execution_order=base-first
+ fi
+ echo "execution_order=$execution_order" >> "$GITHUB_OUTPUT"
+
+ printf '{\n "schema": "moli.sequential-navigation.run-metadata.v1",\n "status": "resolved",\n "target_base": { "sha": "%s" },\n "base": { "sha": "%s", "role": "common-ancestor" },\n "head": { "sha": "%s" },\n "harness": { "sha": "%s", "role": "exact-head" },\n "execution_order": "%s",\n "navigation_count": 200\n}\n' \
+ "$target_base_sha" "$base_sha" "$head_sha" "$head_sha" \
+ "$execution_order" > "$BENCHMARK_OUTPUT/run-metadata.json"
+
+ - name: Build common-ancestor base and exact HEAD release binaries
+ shell: bash
+ env:
+ BASE_SHA: ${{ steps.refs.outputs.base_sha }}
+ HEAD_SHA: ${{ steps.refs.outputs.head_sha }}
+ run: |
+ set -euo pipefail
+ mkdir -p "$BINARY_DIR"
+
+ restore_harness() {
+ git checkout --detach "$HEAD_SHA"
+ }
+ trap restore_harness EXIT
+
+ git checkout --detach "$BASE_SHA"
+ cargo build --release --locked -p moli
+ install -m 755 target/release/moli "$BINARY_DIR/moli-base"
+
+ git checkout --detach "$HEAD_SHA"
+ cargo build --release --locked -p moli
+ install -m 755 target/release/moli "$BINARY_DIR/moli-head"
+
+ restore_harness
+ trap - EXIT
+
+ - name: Install uv
+ uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0
+ with:
+ version: 0.11.14
+ python-version: "3.12"
+ enable-cache: true
+ cache-dependency-glob: moli-benchmark/uv.lock
+
+ - name: Install benchmark dependencies
+ run: uv sync --project moli-benchmark --locked
+
+ - name: Run one-session public-web A/B
+ id: soak
+ shell: bash
+ env:
+ EXECUTION_ORDER: ${{ steps.refs.outputs.execution_order }}
+ run: |
+ set -euo pipefail
+
+ run_side() {
+ local side=$1
+ local binary=$2
+ local report="$BENCHMARK_OUTPUT/$side.json"
+ set +e
+ uv run --project moli-benchmark --no-sync python \
+ moli-benchmark/scripts/fuzz-cdp-sequential-navigate.py \
+ --engine moli \
+ --moli-bin "$binary" \
+ --url https://www.csdn.net/ \
+ --url https://segmentfault.com/ \
+ --url https://huaban.com/ \
+ --url https://example.com/ \
+ --rounds 50 \
+ --network-diagnostics \
+ --navigation-resource-samples \
+ --response-timeout 15 \
+ --dcl-timeout 14 \
+ --load-timeout 20 \
+ --postcheck-timeout 5 \
+ --recovery-timeout 8 \
+ --output "$report"
+ local status=$?
+ set -e
+ echo "${side}_exit_code=$status" >> "$GITHUB_OUTPUT"
+ }
+
+ if [[ "$EXECUTION_ORDER" == head-first ]]; then
+ run_side head "$BINARY_DIR/moli-head"
+ run_side base "$BINARY_DIR/moli-base"
+ else
+ run_side base "$BINARY_DIR/moli-base"
+ run_side head "$BINARY_DIR/moli-head"
+ fi
+
+ - name: Compare resilience and memory evidence
+ id: comparison
+ if: ${{ always() && !cancelled() && steps.soak.outcome == 'success' }}
+ env:
+ PYTHONPATH: moli-benchmark
+ run: >-
+ uv run --project moli-benchmark --no-sync python
+ -m moli_benchmark.sequential_navigate_ci compare
+ --base-report "$BENCHMARK_OUTPUT/base.json"
+ --head-report "$BENCHMARK_OUTPUT/head.json"
+ --base-exit-code "${{ steps.soak.outputs.base_exit_code }}"
+ --head-exit-code "${{ steps.soak.outputs.head_exit_code }}"
+ --base-sha "${{ steps.refs.outputs.base_sha }}"
+ --head-sha "${{ steps.refs.outputs.head_sha }}"
+ --execution-order "${{ steps.refs.outputs.execution_order }}"
+ --expected-navigations 200
+ --output "$BENCHMARK_OUTPUT/comparison.json"
+ --comment-output "$BENCHMARK_OUTPUT/comment.md"
+ --run-url "https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+ --conclusion local
+
+ - name: Render Actions job summary
+ if: ${{ always() && !cancelled() }}
+ continue-on-error: true
+ shell: bash
+ run: |
+ if [[ -f "$BENCHMARK_OUTPUT/comment.md" ]]; then
+ cat "$BENCHMARK_OUTPUT/comment.md" >> "$GITHUB_STEP_SUMMARY"
+ fi
+
+ - name: Upload full soak evidence
+ if: ${{ always() && !cancelled() }}
+ uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+ with:
+ name: sequential-navigation-soak-results
+ path: ${{ env.BENCHMARK_OUTPUT }}
+ if-no-files-found: error
+ retention-days: 14
+
+ - name: Require complete HEAD resilience evidence
+ if: ${{ always() && !cancelled() && steps.comparison.outcome == 'success' }}
+ env:
+ PYTHONPATH: moli-benchmark
+ run: >-
+ uv run --project moli-benchmark --no-sync python
+ -m moli_benchmark.sequential_navigate_ci check
+ --input "$BENCHMARK_OUTPUT/comparison.json"
diff --git a/moli-benchmark/moli_benchmark/sequential_navigate.py b/moli-benchmark/moli_benchmark/sequential_navigate.py
index 7cbf1233d..bbf4d3353 100644
--- a/moli-benchmark/moli_benchmark/sequential_navigate.py
+++ b/moli-benchmark/moli_benchmark/sequential_navigate.py
@@ -23,6 +23,7 @@ from .raw_cdp import (
RoutedRawCdpClient,
connect_routed_raw_cdp,
)
+from .sampling import snapshot_resources
from .target_serve import start_target_serve, stop_target_serve
@@ -45,6 +46,15 @@ NETWORK_EVENT_METHODS = {
"Network.loadingFinished",
"Network.loadingFailed",
}
+NAVIGATION_RESOURCE_FIELDS = (
+ "process_count",
+ "pss_bytes",
+ "pss_process_count",
+ "rss_bytes",
+ "rss_process_count",
+ "thread_count",
+ "fd_count",
+)
@dataclass(frozen=True)
@@ -910,6 +920,157 @@ def repository_state() -> dict[str, Any]:
}
+def capture_navigation_resource_sample(
+ root_pid: int,
+ *,
+ index: int,
+ url: str | None,
+) -> dict[str, Any]:
+ sample: dict[str, Any] = {
+ "index": index,
+ "url": url,
+ "captured_at": datetime.now(UTC).isoformat(),
+ "error": None,
+ }
+ try:
+ observed = snapshot_resources(root_pid, include_lifetime_cpu=False)
+ except Exception as error:
+ sample["error"] = f"{type(error).__name__}: {error}"
+ return sample
+ sample.update({field: observed.get(field) for field in NAVIGATION_RESOURCE_FIELDS})
+ return sample
+
+
+def _average(values: Iterable[float | int]) -> float | None:
+ values = list(values)
+ return sum(values) / len(values) if values else None
+
+
+def _linear_slope(points: list[tuple[int, float]]) -> float | None:
+ if len(points) < 2:
+ return None
+ mean_x = sum(point[0] for point in points) / len(points)
+ mean_y = sum(point[1] for point in points) / len(points)
+ denominator = sum((point[0] - mean_x) ** 2 for point in points)
+ if denominator == 0:
+ return None
+ return sum(
+ (point[0] - mean_x) * (point[1] - mean_y) for point in points
+ ) / denominator
+
+
+def _resource_metric_summary(
+ samples: list[dict[str, Any]],
+ metric: str,
+) -> dict[str, Any]:
+ initial = next(
+ (
+ sample.get(metric)
+ for sample in samples
+ if sample.get("index") == 0
+ and isinstance(sample.get(metric), (int, float))
+ ),
+ None,
+ )
+ points = [
+ (int(sample["index"]), float(sample[metric]))
+ for sample in samples
+ if isinstance(sample.get("index"), int)
+ and sample["index"] > 0
+ and isinstance(sample.get(metric), (int, float))
+ ]
+ values = [value for _, value in points]
+ window_size = min(10, len(values))
+ first_window = values[:window_size]
+ last_window = values[-window_size:] if window_size else []
+ first_window_average = _average(first_window)
+ last_window_average = _average(last_window)
+ warm_start = min(len(points), max(1, len(points) // 10))
+ warm_slope = _linear_slope(points[warm_start:])
+ return {
+ "observed_samples": len(points),
+ "initial": initial,
+ "first": values[0] if values else None,
+ "final": values[-1] if values else None,
+ "minimum": min(values) if values else None,
+ "peak": max(values) if values else None,
+ "window_size": window_size,
+ "first_window_average": first_window_average,
+ "last_window_average": last_window_average,
+ "first_to_last_window_delta": (
+ last_window_average - first_window_average
+ if first_window_average is not None and last_window_average is not None
+ else None
+ ),
+ "warm_slope_per_navigation": warm_slope,
+ "warm_slope_per_100_navigations": (
+ warm_slope * 100.0 if warm_slope is not None else None
+ ),
+ }
+
+
+def summarize_navigation_resource_samples(
+ samples: list[dict[str, Any]],
+ periodic_resources: dict[str, Any],
+) -> dict[str, Any]:
+ navigation_samples = [
+ sample
+ for sample in samples
+ if isinstance(sample.get("index"), int) and sample["index"] > 0
+ ]
+ quarters = []
+ for offset in range(4):
+ start = len(navigation_samples) * offset // 4
+ end = len(navigation_samples) * (offset + 1) // 4
+ quarter_samples = navigation_samples[start:end]
+
+ def quarter_metric(metric: str) -> dict[str, Any]:
+ values = [
+ float(sample[metric])
+ for sample in quarter_samples
+ if isinstance(sample.get(metric), (int, float))
+ ]
+ return {
+ "observed_samples": len(values),
+ "average": _average(values),
+ "final": values[-1] if values else None,
+ "peak": max(values) if values else None,
+ }
+
+ quarters.append(
+ {
+ "quarter": offset + 1,
+ "start_index": quarter_samples[0]["index"] if quarter_samples else None,
+ "end_index": quarter_samples[-1]["index"] if quarter_samples else None,
+ "sample_count": len(quarter_samples),
+ "rss_bytes": quarter_metric("rss_bytes"),
+ "pss_bytes": quarter_metric("pss_bytes"),
+ "fd_count": quarter_metric("fd_count"),
+ }
+ )
+
+ return {
+ "sample_count": len(navigation_samples),
+ "initial_sample_present": any(sample.get("index") == 0 for sample in samples),
+ "sample_errors": sum(bool(sample.get("error")) for sample in samples),
+ "rss_bytes": _resource_metric_summary(samples, "rss_bytes"),
+ "pss_bytes": _resource_metric_summary(samples, "pss_bytes"),
+ "fd_count": _resource_metric_summary(samples, "fd_count"),
+ "thread_count": _resource_metric_summary(samples, "thread_count"),
+ "process_count": _resource_metric_summary(samples, "process_count"),
+ "periodic": {
+ "sample_count": periodic_resources.get("sample_count"),
+ "peak_rss_bytes": periodic_resources.get("peak_rss_bytes"),
+ "peak_pss_bytes": periodic_resources.get("peak_pss_bytes"),
+ "peak_fd_count": periodic_resources.get("peak_fd_count"),
+ "peak_thread_count": periodic_resources.get("peak_thread_count"),
+ "observer_error": periodic_resources.get("observer_error"),
+ "late_sample_count": periodic_resources.get("late_sample_count"),
+ },
+ "quarters": quarters,
+ }
+
+
def selected_engines(engine: str) -> tuple[str, ...]:
if engine == "both":
return ("moli", "chromium")
@@ -927,6 +1088,7 @@ async def run_engine(
startup_timeout: float,
recovery_timeout: float,
network_diagnostics: bool,
+ navigation_resource_samples: bool,
) -> dict[str, Any]:
started_at = datetime.now(UTC).isoformat()
serve = start_target_serve(target, binary, startup_timeout)
@@ -934,6 +1096,7 @@ async def run_engine(
session_id: str | None = None
rows: list[dict[str, Any]] = []
network_order_violations: list[dict[str, Any]] = []
+ resource_samples: list[dict[str, Any]] = []
aborted_after_index: int | None = None
try:
client = await connect_routed_raw_cdp(serve.endpoint)
@@ -942,6 +1105,14 @@ async def run_engine(
startup_timeout,
network_diagnostics=network_diagnostics,
)
+ if navigation_resource_samples:
+ resource_samples.append(
+ capture_navigation_resource_sample(
+ serve.process.pid,
+ index=0,
+ url=None,
+ )
+ )
for index, url in enumerate(urls, 1):
row = await navigate_once(
client,
@@ -975,6 +1146,14 @@ async def run_engine(
if not recovery["ok"]:
aborted_after_index = index
rows.append(row)
+ if navigation_resource_samples:
+ resource_samples.append(
+ capture_navigation_resource_sample(
+ serve.process.pid,
+ index=index,
+ url=url,
+ )
+ )
print(
json.dumps(
{
@@ -1036,6 +1215,17 @@ async def run_engine(
"ready_ms": serve.ready_ms,
"summary": summary,
"network_order_violations": network_order_violations,
+ "navigation_resources": (
+ {
+ "samples": resource_samples,
+ "summary": summarize_navigation_resource_samples(
+ resource_samples,
+ serve_result.get("resources", {}),
+ ),
+ }
+ if navigation_resource_samples
+ else None
+ ),
"rows": rows,
"process": serve_result,
}
@@ -1069,6 +1259,15 @@ def _parse_args(argv: list[str] | None) -> argparse.Namespace:
"overhead comparable."
),
)
+ parser.add_argument(
+ "--navigation-resource-samples",
+ action="store_true",
+ help=(
+ "Capture a process-tree RSS/PSS/FD checkpoint before the first "
+ "navigation and after every navigation, then report windows, "
+ "quarters, and a warm memory slope."
+ ),
+ )
parser.add_argument("--startup-timeout", type=float, default=20.0)
parser.add_argument("--response-timeout", type=float, default=15.0)
parser.add_argument("--dcl-timeout", type=float, default=15.0)
@@ -1142,10 +1341,11 @@ async def _run(args: argparse.Namespace) -> tuple[dict[str, Any], int]:
startup_timeout=args.startup_timeout,
recovery_timeout=args.recovery_timeout,
network_diagnostics=args.network_diagnostics,
+ navigation_resource_samples=args.navigation_resource_samples,
)
)
payload = {
- "schema_version": 4,
+ "schema_version": 5,
"repository": repository_state(),
"engine_selection": args.engine,
"seed": args.seed,
@@ -1153,6 +1353,7 @@ async def _run(args: argparse.Namespace) -> tuple[dict[str, Any], int]:
"urls": urls,
"timeouts": asdict(timeouts),
"network_diagnostics": args.network_diagnostics,
+ "navigation_resource_samples": args.navigation_resource_samples,
"results": results,
}
failed = any(
diff --git a/moli-benchmark/moli_benchmark/sequential_navigate_ci.py b/moli-benchmark/moli_benchmark/sequential_navigate_ci.py
new file mode 100644
index 000000000..bb487353d
--- /dev/null
+++ b/moli-benchmark/moli_benchmark/sequential_navigate_ci.py
@@ -0,0 +1,679 @@
+from __future__ import annotations
+
+import argparse
+import json
+import math
+import re
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import Any
+
+
+COMPARISON_SCHEMA = "moli.sequential-navigation.comparison.v1"
+COMMENT_MARKER = "<!-- moli-sequential-navigation-soak -->"
+EXPECTED_REPORT_SCHEMA_VERSION = 5
+SHA_PATTERN = re.compile(r"^[0-9a-f]{40}$")
+NUMERIC_METRICS = (
+ "planned",
+ "attempted",
+ "observable_passes",
+ "failures",
+ "recovery_attempts",
+ "recovery_passes",
+ "recovery_failures",
+ "order_violations",
+ "network_order_violations",
+ "resource_sample_count",
+ "resource_sample_errors",
+ "rss_observed_samples",
+ "pss_observed_samples",
+ "fd_observed_samples",
+ "duration_seconds",
+ "periodic_peak_rss_bytes",
+ "periodic_peak_pss_bytes",
+ "periodic_peak_fd_count",
+ "periodic_peak_thread_count",
+ "rss_first_window_average",
+ "rss_last_window_average",
+ "rss_first_to_last_window_delta",
+ "rss_warm_slope_per_100_navigations",
+ "pss_first_window_average",
+ "pss_last_window_average",
+ "pss_first_to_last_window_delta",
+ "pss_warm_slope_per_100_navigations",
+)
+
+
+def _finite_number(value: Any) -> int | float | None:
+ if isinstance(value, bool) or not isinstance(value, (int, float)):
+ return None
+ number = float(value)
+ if not math.isfinite(number) or abs(number) > 1e18:
+ return None
+ return value
+
+
+def _integer(value: Any) -> int | None:
+ value = _finite_number(value)
+ if value is None or int(value) != value:
+ return None
+ return int(value)
+
+
+def _nested(mapping: Any, *keys: str) -> Any:
+ current = mapping
+ for key in keys:
+ if not isinstance(current, dict):
+ return None
+ current = current.get(key)
+ return current
+
+
+def _duration_seconds(started_at: Any, finished_at: Any) -> float | None:
+ if not isinstance(started_at, str) or not isinstance(finished_at, str):
+ return None
+ try:
+ started = datetime.fromisoformat(started_at)
+ finished = datetime.fromisoformat(finished_at)
+ except ValueError:
+ return None
+ duration = (finished - started).total_seconds()
+ return duration if math.isfinite(duration) and duration >= 0 else None
+
+
+def _unavailable_run(exit_code: int, reason: str) -> dict[str, Any]:
+ return {
+ "available": False,
+ "exit_code": exit_code,
+ "reason": reason[:240],
+ "metrics": None,
+ }
+
+
+def _sanitize_quarters(value: Any) -> list[dict[str, Any]]:
+ if not isinstance(value, list):
+ return []
+ quarters = []
+ for raw in value[:4]:
+ if not isinstance(raw, dict):
+ continue
+ quarter = _integer(raw.get("quarter"))
+ if quarter not in {1, 2, 3, 4}:
+ continue
+ quarters.append(
+ {
+ "quarter": quarter,
+ "start_index": _integer(raw.get("start_index")),
+ "end_index": _integer(raw.get("end_index")),
+ "sample_count": _integer(raw.get("sample_count")),
+ "rss_average": _finite_number(_nested(raw, "rss_bytes", "average")),
+ "rss_final": _finite_number(_nested(raw, "rss_bytes", "final")),
+ "rss_peak": _finite_number(_nested(raw, "rss_bytes", "peak")),
+ "pss_average": _finite_number(_nested(raw, "pss_bytes", "average")),
+ "pss_final": _finite_number(_nested(raw, "pss_bytes", "final")),
+ "pss_peak": _finite_number(_nested(raw, "pss_bytes", "peak")),
+ "fd_peak": _finite_number(_nested(raw, "fd_count", "peak")),
+ }
+ )
+ return sorted(quarters, key=lambda quarter: quarter["quarter"])
+
+
+def read_sequential_navigation_report(path: Path, exit_code: int) -> dict[str, Any]:
+ try:
+ payload = json.loads(path.read_text(encoding="utf-8"))
+ except (OSError, json.JSONDecodeError) as error:
+ return _unavailable_run(exit_code, f"report unavailable: {type(error).__name__}")
+ if not isinstance(payload, dict):
+ return _unavailable_run(exit_code, "report root is not an object")
+ if payload.get("schema_version") != EXPECTED_REPORT_SCHEMA_VERSION:
+ return _unavailable_run(exit_code, "unsupported sequential navigation report schema")
+ results = payload.get("results")
+ if not isinstance(results, list):
+ return _unavailable_run(exit_code, "report has no result list")
+ result = next(
+ (
+ candidate
+ for candidate in results
+ if isinstance(candidate, dict)
+ and isinstance(candidate.get("target"), str)
+ and candidate["target"].startswith("moli")
+ ),
+ None,
+ )
+ if result is None:
+ return _unavailable_run(exit_code, "report has no Moli result")
+ summary = result.get("summary")
+ navigation_resources = result.get("navigation_resources")
+ memory = _nested(navigation_resources, "summary")
+ if not isinstance(summary, dict) or not isinstance(memory, dict):
+ return _unavailable_run(exit_code, "report is missing summary or navigation resources")
+
+ metrics = {
+ "planned": _integer(summary.get("planned")),
+ "attempted": _integer(summary.get("attempted")),
+ "observable_passes": _integer(summary.get("observable_passes")),
+ "failures": _integer(summary.get("failures")),
+ "recovery_attempts": _integer(summary.get("recovery_attempts")),
+ "recovery_passes": _integer(summary.get("recovery_passes")),
+ "recovery_failures": _integer(summary.get("recovery_failures")),
+ "order_violations": _integer(summary.get("order_violations")),
+ "network_order_violations": _integer(summary.get("network_order_violations")),
+ "aborted_after_index": _integer(summary.get("aborted_after_index")),
+ "resource_sample_count": _integer(memory.get("sample_count")),
+ "resource_sample_errors": _integer(memory.get("sample_errors")),
+ "resource_initial_sample_present": memory.get("initial_sample_present") is True,
+ "rss_observed_samples": _integer(
+ _nested(memory, "rss_bytes", "observed_samples")
+ ),
+ "pss_observed_samples": _integer(
+ _nested(memory, "pss_bytes", "observed_samples")
+ ),
+ "fd_observed_samples": _integer(
+ _nested(memory, "fd_count", "observed_samples")
+ ),
+ "duration_seconds": _duration_seconds(
+ result.get("started_at"), result.get("finished_at")
+ ),
+ "process_returncode": _integer(_nested(result, "process", "returncode")),
+ "periodic_observer_error": _nested(memory, "periodic", "observer_error"),
+ "periodic_peak_rss_bytes": _finite_number(
+ _nested(memory, "periodic", "peak_rss_bytes")
+ ),
+ "periodic_peak_pss_bytes": _finite_number(
+ _nested(memory, "periodic", "peak_pss_bytes")
+ ),
+ "periodic_peak_fd_count": _finite_number(
+ _nested(memory, "periodic", "peak_fd_count")
+ ),
+ "periodic_peak_thread_count": _finite_number(
+ _nested(memory, "periodic", "peak_thread_count")
+ ),
+ "rss_first_window_average": _finite_number(
+ _nested(memory, "rss_bytes", "first_window_average")
+ ),
+ "rss_last_window_average": _finite_number(
+ _nested(memory, "rss_bytes", "last_window_average")
+ ),
+ "rss_first_to_last_window_delta": _finite_number(
+ _nested(memory, "rss_bytes", "first_to_last_window_delta")
+ ),
+ "rss_warm_slope_per_100_navigations": _finite_number(
+ _nested(memory, "rss_bytes", "warm_slope_per_100_navigations")
+ ),
+ "pss_first_window_average": _finite_number(
+ _nested(memory, "pss_bytes", "first_window_average")
+ ),
+ "pss_last_window_average": _finite_number(
+ _nested(memory, "pss_bytes", "last_window_average")
+ ),
+ "pss_first_to_last_window_delta": _finite_number(
+ _nested(memory, "pss_bytes", "first_to_last_window_delta")
+ ),
+ "pss_warm_slope_per_100_navigations": _finite_number(
+ _nested(memory, "pss_bytes", "warm_slope_per_100_navigations")
+ ),
+ "quarters": _sanitize_quarters(memory.get("quarters")),
+ }
+ return {
+ "available": True,
+ "exit_code": exit_code,
+ "reason": None,
+ "metrics": metrics,
+ }
+
+
+def _validated_sha(value: str) -> str:
+ if not SHA_PATTERN.fullmatch(value):
+ raise ValueError(f"invalid commit SHA: {value!r}")
+ return value
+
+
+def build_comparison(
+ *,
+ base_sha: str,
+ head_sha: str,
+ execution_order: str,
+ expected_navigations: int,
+ base_run: dict[str, Any],
+ head_run: dict[str, Any],
+) -> dict[str, Any]:
+ if execution_order not in {"base-first", "head-first"}:
+ raise ValueError(f"invalid execution order: {execution_order!r}")
+ if expected_navigations <= 0 or expected_navigations > 10_000:
+ raise ValueError("expected navigations must be between 1 and 10000")
+ delta: dict[str, int | float | None] = {}
+ base_metrics = base_run.get("metrics") if base_run.get("available") else None
+ head_metrics = head_run.get("metrics") if head_run.get("available") else None
+ for metric in NUMERIC_METRICS:
+ base_value = _finite_number(
+ base_metrics.get(metric) if isinstance(base_metrics, dict) else None
+ )
+ head_value = _finite_number(
+ head_metrics.get(metric) if isinstance(head_metrics, dict) else None
+ )
+ delta[metric] = (
+ head_value - base_value
+ if base_value is not None and head_value is not None
+ else None
+ )
+ return {
+ "schema": COMPARISON_SCHEMA,
+ "created_at": datetime.now(UTC).isoformat(),
+ "base": {"sha": _validated_sha(base_sha)},
+ "head": {"sha": _validated_sha(head_sha)},
+ "execution_order": execution_order,
+ "expected_navigations": expected_navigations,
+ "workload": {
+ "single_process": True,
+ "single_target": True,
+ "single_session": True,
+ "cycle_size": 4,
+ "cycle_count": expected_navigations // 4,
+ },
+ "runs": {"base": base_run, "head": head_run},
+ "delta": delta,
+ }
+
+
+def comparison_health_issues(comparison: dict[str, Any]) -> list[str]:
+ if comparison.get("schema") != COMPARISON_SCHEMA:
+ return ["unsupported comparison schema"]
+ expected = _integer(comparison.get("expected_navigations"))
+ if expected is None or expected <= 0 or expected > 10_000:
+ return ["missing expected navigation count"]
+ issues = []
+ for side in ("base", "head"):
+ run = _nested(comparison, "runs", side)
+ if not isinstance(run, dict) or run.get("available") is not True:
+ issues.append(f"{side} report unavailable")
+ head = _nested(comparison, "runs", "head", "metrics")
+ if not isinstance(head, dict):
+ return issues or ["HEAD metrics unavailable"]
+ required_equal = {
+ "planned": expected,
+ "attempted": expected,
+ "resource_sample_count": expected,
+ "rss_observed_samples": expected,
+ "pss_observed_samples": expected,
+ "fd_observed_samples": expected,
+ "recovery_failures": 0,
+ "order_violations": 0,
+ "network_order_violations": 0,
+ "resource_sample_errors": 0,
+ }
+ for metric, required in required_equal.items():
+ if _integer(head.get(metric)) != required:
+ issues.append(f"HEAD {metric} is not {required}")
+ if head.get("aborted_after_index") is not None:
+ issues.append("HEAD session aborted before all navigations")
+ if head.get("resource_initial_sample_present") is not True:
+ issues.append("HEAD initial resource sample is unavailable")
+ if _integer(head.get("process_returncode")) not in {0, 143, -15}:
+ issues.append("HEAD browser process had an unexpected exit status")
+ observer_error = head.get("periodic_observer_error")
+ if observer_error is not None and observer_error != "":
+ issues.append("HEAD periodic resource observer failed")
+ for metric in ("periodic_peak_rss_bytes", "periodic_peak_pss_bytes"):
+ if _finite_number(head.get(metric)) is None:
+ issues.append(f"HEAD {metric} is unavailable")
+ return issues
+
+
+def _format_integer(value: Any) -> str:
+ value = _integer(value)
+ return f"{value:,}" if value is not None else "—"
+
+
+def _format_number(value: Any, digits: int = 1) -> str:
+ value = _finite_number(value)
+ return f"{float(value):,.{digits}f}" if value is not None else "—"
+
+
+def _format_mib(value: Any) -> str:
+ value = _finite_number(value)
+ return f"{float(value) / 1024 / 1024:,.1f} MiB" if value is not None else "—"
+
+
+def _format_signed_mib(value: Any) -> str:
+ value = _finite_number(value)
+ return f"{float(value) / 1024 / 1024:+,.1f} MiB" if value is not None else "—"
+
+
+def _format_seconds(value: Any) -> str:
+ value = _finite_number(value)
+ return f"{float(value):,.1f} s" if value is not None else "—"
+
+
+def _format_ratio(numerator: Any, denominator: Any) -> str:
+ return f"{_format_integer(numerator)} / {_format_integer(denominator)}"
+
+
+def _short_sha(value: Any) -> str:
+ return value[:10] if isinstance(value, str) and SHA_PATTERN.fullmatch(value) else "unknown"
+
+
+def _safe_run_url(value: str | None) -> str | None:
+ if not isinstance(value, str):
+ return None
+ match = re.fullmatch(
+ r"https://github\.com/[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+/actions/runs/[0-9]+",
+ value,
+ )
+ return value if match else None
+
+
+def _safe_conclusion(value: str) -> str:
+ return value if value in {"success", "failure", "cancelled", "timed_out", "local"} else "unknown"
+
+
+def _run_metrics(comparison: dict[str, Any], side: str) -> dict[str, Any] | None:
+ run = _nested(comparison, "runs", side)
+ if not isinstance(run, dict) or run.get("available") is not True:
+ return None
+ metrics = run.get("metrics")
+ return metrics if isinstance(metrics, dict) else None
+
+
+def _memory_table(comparison: dict[str, Any]) -> str:
+ base = _run_metrics(comparison, "base")
+ head = _run_metrics(comparison, "head")
+ delta = comparison.get("delta") if isinstance(comparison.get("delta"), dict) else {}
+
+ def value(metrics: dict[str, Any] | None, key: str) -> Any:
+ return metrics.get(key) if metrics is not None else None
+
+ rows = [
+ ("Boundary samples", "resource_sample_count", _format_integer, _format_integer),
+ ("Boundary RSS samples", "rss_observed_samples", _format_integer, _format_integer),
+ ("Boundary PSS samples", "pss_observed_samples", _format_integer, _format_integer),
+ ("Periodic peak RSS", "periodic_peak_rss_bytes", _format_mib, _format_signed_mib),
+ ("Periodic peak PSS", "periodic_peak_pss_bytes", _format_mib, _format_signed_mib),
+ ("RSS · first 10 avg", "rss_first_window_average", _format_mib, _format_signed_mib),
+ ("RSS · last 10 avg", "rss_last_window_average", _format_mib, _format_signed_mib),
+ ("RSS · last − first window", "rss_first_to_last_window_delta", _format_signed_mib, _format_signed_mib),
+ ("RSS · warm slope / 100 nav", "rss_warm_slope_per_100_navigations", _format_signed_mib, _format_signed_mib),
+ ("PSS · first 10 avg", "pss_first_window_average", _format_mib, _format_signed_mib),
+ ("PSS · last 10 avg", "pss_last_window_average", _format_mib, _format_signed_mib),
+ ("PSS · last − first window", "pss_first_to_last_window_delta", _format_signed_mib, _format_signed_mib),
+ ("PSS · warm slope / 100 nav", "pss_warm_slope_per_100_navigations", _format_signed_mib, _format_signed_mib),
+ ("Peak file descriptors", "periodic_peak_fd_count", _format_integer, _format_integer),
+ ("Peak threads", "periodic_peak_thread_count", _format_integer, _format_integer),
+ ]
+ lines = ["| Metric | Base | HEAD | Δ |", "| --- | ---: | ---: | ---: |"]
+ for label, key, formatter, delta_formatter in rows:
+ lines.append(
+ f"| {label} | {formatter(value(base, key))} | "
+ f"{formatter(value(head, key))} | {delta_formatter(delta.get(key))} |"
+ )
+ return "\n".join(lines)
+
+
+def _correctness_table(comparison: dict[str, Any]) -> str:
+ base = _run_metrics(comparison, "base")
+ head = _run_metrics(comparison, "head")
+
+ def value(metrics: dict[str, Any] | None, key: str) -> Any:
+ return metrics.get(key) if metrics is not None else None
+
+ return "\n".join(
+ [
+ "| Metric | Base | HEAD |",
+ "| --- | ---: | ---: |",
+ (
+ "| Attempted / planned | "
+ f"{_format_ratio(value(base, 'attempted'), value(base, 'planned'))} | "
+ f"{_format_ratio(value(head, 'attempted'), value(head, 'planned'))} |"
+ ),
+ (
+ "| Direct observable pass | "
+ f"{_format_ratio(value(base, 'observable_passes'), value(base, 'attempted'))} | "
+ f"{_format_ratio(value(head, 'observable_passes'), value(head, 'attempted'))} |"
+ ),
+ (
+ "| Failures / recovered | "
+ f"{_format_ratio(value(base, 'failures'), value(base, 'recovery_passes'))} | "
+ f"{_format_ratio(value(head, 'failures'), value(head, 'recovery_passes'))} |"
+ ),
+ (
+ "| Recovery failures | "
+ f"{_format_integer(value(base, 'recovery_failures'))} | "
+ f"{_format_integer(value(head, 'recovery_failures'))} |"
+ ),
+ (
+ "| Lifecycle order violations | "
+ f"{_format_integer(value(base, 'order_violations'))} | "
+ f"{_format_integer(value(head, 'order_violations'))} |"
+ ),
+ (
+ "| Network order violations | "
+ f"{_format_integer(value(base, 'network_order_violations'))} | "
+ f"{_format_integer(value(head, 'network_order_violations'))} |"
+ ),
+ (
+ "| Wall time | "
+ f"{_format_seconds(value(base, 'duration_seconds'))} | "
+ f"{_format_seconds(value(head, 'duration_seconds'))} |"
+ ),
+ ]
+ )
+
+
+def _head_quarter_table(comparison: dict[str, Any]) -> str:
+ head = _run_metrics(comparison, "head")
+ quarters = head.get("quarters") if isinstance(head, dict) else None
+ if not isinstance(quarters, list) or not quarters:
+ return "_HEAD navigation-quarter memory is unavailable._"
+ lines = [
+ "| HEAD range | Avg RSS | Final RSS | Peak RSS | Avg PSS | Final PSS | Peak FD |",
+ "| --- | ---: | ---: | ---: | ---: | ---: | ---: |",
+ ]
+ rendered_quarters = 0
+ for quarter in quarters[:4]:
+ if not isinstance(quarter, dict):
+ continue
+ rendered_quarters += 1
+ start = _format_integer(quarter.get("start_index"))
+ end = _format_integer(quarter.get("end_index"))
+ lines.append(
+ "| "
+ f"{start}–{end} | "
+ f"{_format_mib(quarter.get('rss_average'))} | "
+ f"{_format_mib(quarter.get('rss_final'))} | "
+ f"{_format_mib(quarter.get('rss_peak'))} | "
+ f"{_format_mib(quarter.get('pss_average'))} | "
+ f"{_format_mib(quarter.get('pss_final'))} | "
+ f"{_format_integer(quarter.get('fd_peak'))} |"
+ )
+ return (
+ "\n".join(lines)
+ if rendered_quarters
+ else "_HEAD navigation-quarter memory is unavailable._"
+ )
+
+
+def render_comparison_comment(
+ comparison: dict[str, Any],
+ *,
+ run_url: str | None,
+ conclusion: str,
+) -> str:
+ if comparison.get("schema") != COMPARISON_SCHEMA:
+ raise ValueError("unsupported sequential navigation comparison schema")
+ expected = _integer(comparison.get("expected_navigations"))
+ if expected is None or expected <= 0 or expected > 10_000:
+ raise ValueError("comparison is missing expected navigation count")
+ run_link = _safe_run_url(run_url)
+ artifact = (
+ f"[workflow run and full `sequential-navigation-soak-results` artifact]({run_link})"
+ if run_link
+ else "local output or `sequential-navigation-soak-results` artifact"
+ )
+ issues = comparison_health_issues(comparison)
+ assessment = (
+ f"✅ HEAD completed the {expected}-navigation resilience and memory observation."
+ if not issues
+ else "⚠️ HEAD did not produce a complete resilience/memory observation: "
+ + "; ".join(issues[:6])
+ + "."
+ )
+ order = comparison.get("execution_order")
+ rendered_order = order if order in {"base-first", "head-first"} else "unknown"
+ return "\n".join(
+ [
+ COMMENT_MARKER,
+ "## Sequential Navigation Soak A/B",
+ "",
+ assessment,
+ "",
+ "One browser process, one target, and one CDP session navigate "
+ "CSDN → SegmentFault → Huaban → example.com repeatedly. "
+ f"This run issued {expected} `Page.navigate` commands per binary.",
+ "",
+ (
+ f"Common ancestor `{_short_sha(_nested(comparison, 'base', 'sha'))}` → "
+ f"HEAD `{_short_sha(_nested(comparison, 'head', 'sha'))}`; "
+ f"benchmark order: `{rendered_order}`; "
+ f"workflow: `{_safe_conclusion(conclusion)}`."
+ ),
+ "",
+ "### Session resilience",
+ "",
+ _correctness_table(comparison),
+ "",
+ "A public-page failure is reported separately from an unrecoverable session. "
+ "The soak requires all navigation attempts, zero failed recovery, zero "
+ "lifecycle/network ordering violations, and complete resource evidence.",
+ "",
+ "### Process-tree memory",
+ "",
+ _memory_table(comparison),
+ "",
+ "### HEAD memory by 50-navigation quarter",
+ "",
+ _head_quarter_table(comparison),
+ "",
+ f"Raw reports and per-navigation boundary samples: {artifact}.",
+ "",
+ "_Public-site timing and memory are observational. Use the A/B deltas and "
+ "trend shape as evidence, not as a deterministic performance threshold._",
+ ]
+ )
+
+
+def render_infrastructure_comment(*, run_url: str | None, conclusion: str) -> str:
+ run_link = _safe_run_url(run_url)
+ workflow = f"[workflow run]({run_link})" if run_link else "workflow run"
+ return "\n".join(
+ [
+ COMMENT_MARKER,
+ "## Sequential Navigation Soak A/B",
+ "",
+ "⚠️ Benchmark infrastructure failed before a comparison could be produced.",
+ "",
+ f"Workflow: `{_safe_conclusion(conclusion)}`. Inspect the {workflow} for the "
+ "failing build, harness, or artifact step.",
+ "",
+ "_No navigation resilience or memory conclusion is available for this run._",
+ ]
+ )
+
+
+def _write_json(path: Path, payload: Any) -> None:
+ path.parent.mkdir(parents=True, exist_ok=True)
+ path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8")
+
+
+def _write_text(path: Path, text: str) -> None:
+ path.parent.mkdir(parents=True, exist_ok=True)
+ path.write_text(text.rstrip() + "\n", encoding="utf-8")
+
+
+def _parse_args(argv: list[str] | None) -> argparse.Namespace:
+ parser = argparse.ArgumentParser(description="Compare and render sequential navigation soak evidence.")
+ commands = parser.add_subparsers(dest="command", required=True)
+
+ compare = commands.add_parser("compare")
+ compare.add_argument("--base-report", type=Path, required=True)
+ compare.add_argument("--head-report", type=Path, required=True)
+ compare.add_argument("--base-exit-code", type=int, required=True)
+ compare.add_argument("--head-exit-code", type=int, required=True)
+ compare.add_argument("--base-sha", required=True)
+ compare.add_argument("--head-sha", required=True)
+ compare.add_argument("--execution-order", choices=("base-first", "head-first"), required=True)
+ compare.add_argument("--expected-navigations", type=int, default=200)
+ compare.add_argument("--output", type=Path, required=True)
+ compare.add_argument("--comment-output", type=Path, required=True)
+ compare.add_argument("--run-url")
+ compare.add_argument("--conclusion", default="local")
+
+ comment = commands.add_parser("comment")
+ comment.add_argument("--input", type=Path, required=True)
+ comment.add_argument("--output", type=Path, required=True)
+ comment.add_argument("--run-url")
+ comment.add_argument("--conclusion", required=True)
+
+ infrastructure = commands.add_parser("infrastructure-comment")
+ infrastructure.add_argument("--output", type=Path, required=True)
+ infrastructure.add_argument("--run-url")
+ infrastructure.add_argument("--conclusion", required=True)
+
+ check = commands.add_parser("check")
+ check.add_argument("--input", type=Path, required=True)
+ return parser.parse_args(argv)
+
+
+def main(argv: list[str] | None = None) -> int:
+ args = _parse_args(argv)
+ if args.command == "compare":
+ comparison = build_comparison(
+ base_sha=args.base_sha,
+ head_sha=args.head_sha,
+ execution_order=args.execution_order,
+ expected_navigations=args.expected_navigations,
+ base_run=read_sequential_navigation_report(
+ args.base_report, args.base_exit_code
+ ),
+ head_run=read_sequential_navigation_report(
+ args.head_report, args.head_exit_code
+ ),
+ )
+ _write_json(args.output, comparison)
+ _write_text(
+ args.comment_output,
+ render_comparison_comment(
+ comparison,
+ run_url=args.run_url,
+ conclusion=args.conclusion,
+ ),
+ )
+ return 0
+ if args.command == "comment":
+ comparison = json.loads(args.input.read_text(encoding="utf-8"))
+ _write_text(
+ args.output,
+ render_comparison_comment(
+ comparison,
+ run_url=args.run_url,
+ conclusion=args.conclusion,
+ ),
+ )
+ return 0
+ if args.command == "infrastructure-comment":
+ _write_text(
+ args.output,
+ render_infrastructure_comment(
+ run_url=args.run_url,
+ conclusion=args.conclusion,
+ ),
+ )
+ return 0
+ if args.command == "check":
+ comparison = json.loads(args.input.read_text(encoding="utf-8"))
+ issues = comparison_health_issues(comparison)
+ print(json.dumps({"healthy": not issues, "issues": issues}, indent=2))
+ return 1 if issues else 0
+ raise AssertionError(f"unhandled command: {args.command}")
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/moli-benchmark/tests/test_sequential_navigate.py b/moli-benchmark/tests/test_sequential_navigate.py
index 3824a2197..ff858d43a 100644
--- a/moli-benchmark/tests/test_sequential_navigate.py
+++ b/moli-benchmark/tests/test_sequential_navigate.py
@@ -14,6 +14,7 @@ from moli_benchmark.sequential_navigate import (
parse_top_100_domains,
select_seed_urls,
selected_engines,
+ summarize_navigation_resource_samples,
summarize_network_activity,
)
@@ -99,6 +100,67 @@ def network_event(
class SequentialNavigateTests(unittest.TestCase):
+ def test_navigation_resource_summary_reports_windows_slope_and_quarters(self) -> None:
+ mib = 1024 * 1024
+ samples = [
+ {
+ "index": 0,
+ "rss_bytes": 100 * mib,
+ "pss_bytes": 90 * mib,
+ "fd_count": 20,
+ "thread_count": 8,
+ "process_count": 1,
+ "error": None,
+ }
+ ]
+ samples.extend(
+ {
+ "index": index,
+ "rss_bytes": (100 + index) * mib,
+ "pss_bytes": (90 + index / 2) * mib,
+ "fd_count": 20 + index // 50,
+ "thread_count": 8,
+ "process_count": 1,
+ "error": None,
+ }
+ for index in range(1, 201)
+ )
+
+ summary = summarize_navigation_resource_samples(
+ samples,
+ {
+ "sample_count": 500,
+ "peak_rss_bytes": 305 * mib,
+ "peak_pss_bytes": 195 * mib,
+ "peak_fd_count": 24,
+ "peak_thread_count": 8,
+ "observer_error": None,
+ "late_sample_count": 0,
+ },
+ )
+
+ self.assertEqual(summary["sample_count"], 200)
+ self.assertTrue(summary["initial_sample_present"])
+ self.assertEqual(summary["sample_errors"], 0)
+ self.assertEqual(summary["rss_bytes"]["first_window_average"], 105.5 * mib)
+ self.assertEqual(summary["rss_bytes"]["last_window_average"], 295.5 * mib)
+ self.assertEqual(
+ summary["rss_bytes"]["first_to_last_window_delta"],
+ 190 * mib,
+ )
+ self.assertAlmostEqual(
+ summary["rss_bytes"]["warm_slope_per_100_navigations"],
+ 100 * mib,
+ )
+ self.assertEqual(
+ [
+ (quarter["start_index"], quarter["end_index"], quarter["sample_count"])
+ for quarter in summary["quarters"]
+ ],
+ [(1, 50, 50), (51, 100, 50), (101, 150, 50), (151, 200, 50)],
+ )
+ self.assertEqual(summary["periodic"]["peak_rss_bytes"], 305 * mib)
+
def test_url_normalization_preserves_explicit_data_url(self) -> None:
url = "data:text/html,<title>fixture</title>"
diff --git a/moli-benchmark/tests/test_sequential_navigate_ci.py b/moli-benchmark/tests/test_sequential_navigate_ci.py
new file mode 100644
index 000000000..db94c24ee
--- /dev/null
+++ b/moli-benchmark/tests/test_sequential_navigate_ci.py
@@ -0,0 +1,214 @@
+from __future__ import annotations
+
+import json
+import tempfile
+import unittest
+from pathlib import Path
+
+from moli_benchmark.sequential_navigate_ci import (
+ COMMENT_MARKER,
+ build_comparison,
+ comparison_health_issues,
+ read_sequential_navigation_report,
+ render_comparison_comment,
+ render_infrastructure_comment,
+)
+
+
+MIB = 1024 * 1024
+BASE_SHA = "a" * 40
+HEAD_SHA = "b" * 40
+
+
+def report_payload(*, aborted_after_index: int | None = None) -> dict[str, object]:
+ quarters = []
+ for quarter in range(1, 5):
+ start = (quarter - 1) * 50 + 1
+ end = quarter * 50
+ quarters.append(
+ {
+ "quarter": quarter,
+ "start_index": start,
+ "end_index": end,
+ "sample_count": 50,
+ "rss_bytes": {
+ "observed_samples": 50,
+ "average": (140 + quarter * 5) * MIB,
+ "final": (142 + quarter * 5) * MIB,
+ "peak": (145 + quarter * 5) * MIB,
+ },
+ "pss_bytes": {
+ "observed_samples": 50,
+ "average": (130 + quarter * 5) * MIB,
+ "final": (132 + quarter * 5) * MIB,
+ "peak": (135 + quarter * 5) * MIB,
+ },
+ "fd_count": {
+ "observed_samples": 50,
+ "average": 25,
+ "final": 25,
+ "peak": 26,
+ },
+ }
+ )
+ return {
+ "schema_version": 5,
+ "results": [
+ {
+ "target": "moli-cdp",
+ "started_at": "2026-08-12T00:00:00+00:00",
+ "finished_at": "2026-08-12T00:05:00+00:00",
+ "summary": {
+ "planned": 200,
+ "attempted": 200,
+ "observable_passes": 198,
+ "failures": 2,
+ "recovery_attempts": 2,
+ "recovery_passes": 2,
+ "recovery_failures": 0,
+ "order_violations": 0,
+ "network_order_violations": 0,
+ "aborted_after_index": aborted_after_index,
+ },
+ "navigation_resources": {
+ "summary": {
+ "sample_count": 200,
+ "sample_errors": 0,
+ "initial_sample_present": True,
+ "rss_bytes": {
+ "observed_samples": 200,
+ "first_window_average": 140 * MIB,
+ "last_window_average": 160 * MIB,
+ "first_to_last_window_delta": 20 * MIB,
+ "warm_slope_per_100_navigations": 8 * MIB,
+ },
+ "pss_bytes": {
+ "observed_samples": 200,
+ "first_window_average": 130 * MIB,
+ "last_window_average": 150 * MIB,
+ "first_to_last_window_delta": 20 * MIB,
+ "warm_slope_per_100_navigations": 7 * MIB,
+ },
+ "fd_count": {"observed_samples": 200},
+ "periodic": {
+ "peak_rss_bytes": 170 * MIB,
+ "peak_pss_bytes": 160 * MIB,
+ "peak_fd_count": 26,
+ "peak_thread_count": 12,
+ "observer_error": None,
+ },
+ "quarters": quarters,
+ }
+ },
+ "process": {"returncode": 143},
+ }
+ ],
+ }
+
+
+class SequentialNavigateCiTests(unittest.TestCase):
+ def write_report(self, directory: Path, payload: dict[str, object]) -> Path:
+ path = directory / "report.json"
+ path.write_text(json.dumps(payload), encoding="utf-8")
+ return path
+
+ def test_comparison_comment_reports_200_navigation_memory_shape(self) -> None:
+ with tempfile.TemporaryDirectory() as temporary:
+ report = self.write_report(Path(temporary), report_payload())
+ base = read_sequential_navigation_report(report, 1)
+ head = read_sequential_navigation_report(report, 1)
+
+ comparison = build_comparison(
+ base_sha=BASE_SHA,
+ head_sha=HEAD_SHA,
+ execution_order="head-first",
+ expected_navigations=200,
+ base_run=base,
+ head_run=head,
+ )
+ comment = render_comparison_comment(
+ comparison,
+ run_url="https://github.com/example/moli/actions/runs/123",
+ conclusion="success",
+ )
+
+ self.assertEqual(comparison_health_issues(comparison), [])
+ self.assertIn(COMMENT_MARKER, comment)
+ self.assertIn("200 `Page.navigate` commands", comment)
+ self.assertIn("Periodic peak RSS", comment)
+ self.assertIn("RSS · warm slope / 100 nav", comment)
+ self.assertIn("151–200", comment)
+ self.assertIn("170.0 MiB", comment)
+ self.assertIn("workflow run and full", comment)
+
+ def test_health_requires_complete_session_and_resource_samples(self) -> None:
+ with tempfile.TemporaryDirectory() as temporary:
+ report = self.write_report(
+ Path(temporary),
+ report_payload(aborted_after_index=117),
+ )
+ run = read_sequential_navigation_report(report, 1)
+ run["metrics"]["attempted"] = 117
+ run["metrics"]["resource_sample_count"] = 117
+ comparison = build_comparison(
+ base_sha=BASE_SHA,
+ head_sha=HEAD_SHA,
+ execution_order="base-first",
+ expected_navigations=200,
+ base_run=run,
+ head_run=run,
+ )
+
+ issues = comparison_health_issues(comparison)
+
+ self.assertIn("HEAD attempted is not 200", issues)
+ self.assertIn("HEAD resource_sample_count is not 200", issues)
+ self.assertIn("HEAD session aborted before all navigations", issues)
+
+ def test_reader_rejects_old_report_schema(self) -> None:
+ payload = report_payload()
+ payload["schema_version"] = 4
+ with tempfile.TemporaryDirectory() as temporary:
+ report = self.write_report(Path(temporary), payload)
+ run = read_sequential_navigation_report(report, 2)
+
+ self.assertFalse(run["available"])
+ self.assertEqual(run["exit_code"], 2)
+
+ def test_infrastructure_comment_does_not_render_untrusted_links(self) -> None:
+ comment = render_infrastructure_comment(
+ run_url="javascript:alert(1)",
+ conclusion="invented",
+ )
+
+ self.assertIn(COMMENT_MARKER, comment)
+ self.assertIn("workflow: `unknown`", comment.lower())
+ self.assertNotIn("javascript:", comment)
+
+ def test_trusted_renderer_handles_malformed_optional_artifact_fields(self) -> None:
+ with tempfile.TemporaryDirectory() as temporary:
+ report = self.write_report(Path(temporary), report_payload())
+ run = read_sequential_navigation_report(report, 0)
+ comparison = build_comparison(
+ base_sha=BASE_SHA,
+ head_sha=HEAD_SHA,
+ execution_order="base-first",
+ expected_navigations=200,
+ base_run=run,
+ head_run=run,
+ )
+ comparison["runs"]["head"]["metrics"]["periodic_observer_error"] = {}
+ comparison["runs"]["head"]["metrics"]["quarters"] = ["untrusted"] * 100
+
+ comment = render_comparison_comment(
+ comparison,
+ run_url=None,
+ conclusion="failure",
+ )
+
+ self.assertIn("periodic resource observer failed", comment)
+ self.assertIn("navigation-quarter memory is unavailable", comment)
+
+
+if __name__ == "__main__":
+ unittest.main()