The agent that grows with you
Python aiai-agentai-agentsanthropicchatgptclaudeclaude-codecodexhermeshermes-agentllmnous-researchopenai
test(ci): trim cache coverage to two behavior checks
ethernet63c1d815dde2
diff --git a/tests/ci/test_cache_report.py b/tests/ci/test_cache_report.pyindex 9018777eef5..c344bf8157d 100644--- a/tests/ci/test_cache_report.py+++ b/tests/ci/test_cache_report.py@@ -1,170 +1,38 @@-"""Compressed archive accounting follows cache receipts, not progress or key hashes."""--from __future__ import annotations--import importlib---def _cache(name, key, events, *, action="actions/cache", minute=0):- marker = f"##[start-action display={name};id=forward.{name}]" if name else ""- lines = [marker, f"##[group]Run {action}@55cc8345863c7cc4c66a329aec7e433d2d1c52a9",- "with:", f" key: {key}", " lookup-only: false", "##[endgroup]", *events]- if name:- lines.append(f"##[end-action id=forward.{name};outcome=success]")- return "\n".join(f"2026-10-20T20:{minute:02d}:10.1234567Z {line}" for line in lines if line)---def _post(name, events):- return "\n".join([- "Post job cleanup.",- f"##[start-action display={name};id=__changed-uuid.{name}]",- "Post job cleanup.", *events,- f"##[end-action id=__changed-uuid.{name};outcome=success]",- ])---def test_cache_invocations_account_only_confirmed_phase_local_archives():- report = importlib.import_module("scripts.ci.cache_report")- first = "tools-" + "a" * 64- second = "tools-" + "b" * 64- log = "\n".join([- _cache("PM tools", first, [- "Cache hit for restore-key: tools-main",- "Received 0 of 100 (0.0%), 0.0 MBs/sec",- "Received 100 of 100 (100.0%), 1.0 MBs/sec",- "Cache Size: ~0 MB (100 B)", "Cache restored successfully",- "Cache restored from key: tools-main",- ]),- _cache("PM tools", second, [f"Cache not found for input keys: {second}"]),- _cache("uv", "uv-key", [- "Cache hit for: uv-key", "Cache Size: ~0 MB (40 B)",- "Cache restored successfully", "Cache restored from key: uv-key",- ]),- _cache("npm", "npm-key", [- "Cache found and can be restored from key: npm-key", "Lookup only - skipping download",- ]),- _cache("unknown size", "unknown-key", ["Cache restored from key: unknown-key"]),- _cache("transport", "failed-key", [- "##[warning]Failed to restore: service unavailable",- "Cache not found for input keys: failed-key",- ]),- _cache("race", "race-key", ["Cache not found for input keys: race-key"]),- _cache("failed restore", "broken-key", [- "Cache hit for: broken-key", "Received 90 of 90 (100.0%), 1.0 MBs/sec",- "Cache Size: ~0 MB (90 B)", "##[warning]Failed to restore: tar failed",- ]),- _cache(None, "modules-key", [- "Cache hit for: modules-key", "Cache Size: ~0 MB (7 B)",- "Cache restored from key: modules-key",- ], minute=1),- _cache("unconfirmed", "unconfirmed-key", ["Cache not found for input keys: unconfirmed-key"], minute=2),- "##[group]Run echo 'Cache saved with key: fake'",- "echo 'Cache Size: ~0 MB (99999 B)'", "##[endgroup]",- "Cache saved with key: fake", # Shell output outside a cache action is not a receipt.- _post("PM tools", [- "Sent 0 of 200 (0.0%), 0.0 MBs/sec", "Sent 200 of 200 (100.0%), 1.0 MBs/sec",- f"Cache saved with key: {second}",- ]),- _post("PM tools", [- "Sent 150 of 150 (100.0%), 1.0 MBs/sec", f"Cache saved with key: {first}",- ]),- _post("uv", ["Cache hit occurred on the primary key uv-key, not saving cache."]),- _post("race", [- "Sent 500 of 500 (100.0%), 1.0 MBs/sec",- "##[warning]Failed to save: Unable to reserve cache with key race-key, another job may be creating this cache.",- ]),- _post("unknown size", ["Cache saved with key: unknown-key"]),- _post("unconfirmed", ["Sent 300 of 300 (100.0%), 1.0 MBs/sec"]),- _post("Upload artifact", ["##[warning]unrelated cleanup warning"]),- ])- steps = [{"name": "Restore node_modules", "number": 9, "started_at": "2026-10-20T20:01:10Z",- "completed_at": "2026-10-20T20:01:11Z"}]- entries = report.parse_cache_log(log, steps)- assert len(entries) == 10- assert [(e["requested_key"], e["matched_key"], e["restore_status"], e["restored_bytes"],- e["save_status"], e["saved_bytes"]) for e in entries] == [- (first, "tools-main", "fallback", 100, "saved", 150),- (second, None, "miss", None, "saved", 200),- ("uv-key", "uv-key", "exact", 40, "skipped-exact-hit", None),- ("npm-key", "npm-key", "lookup-only", None, "not-attempted", None),- ("unknown-key", "unknown-key", "exact", None, "saved", None),- ("failed-key", None, "miss", None, "not-attempted", None),- ("race-key", None, "miss", None, "failed", None),- ("broken-key", "broken-key", "failed", None, "not-attempted", None),- ("modules-key", "modules-key", "exact", 7, "not-attempted", None),- ("unconfirmed-key", None, "miss", None, "unknown", None),+"""Protect archive accounting, not report wording or markup."""+from scripts.ci.cache_report import parse_cache_log+++def test_cache_accounting() -> None:+ log = """##[group]Run actions/cache@fixture+##[endgroup]+Cache hit for: hit+Received 0 of 100 (0.0%), 0.0 MBs/sec+Received 100 of 100 (100.0%), 1.0 MBs/sec+Cache restored from key: hit+##[group]Run actions/cache@fixture+##[endgroup]+Cache not found for input keys: miss+Post job cleanup.+Sent 0 of 200 (0.0%), 0.0 MBs/sec+Sent 200 of 200 (100.0%), 1.0 MBs/sec+Cache saved with key: miss+##[group]Run actions/cache@fixture+##[endgroup]+Cache hit for: failed+Received 80 of 80 (100.0%), 1.0 MBs/sec+##[warning]Failed to restore: tar failed+Post job cleanup.+Sent 90 of 90 (100.0%), 1.0 MBs/sec+##[group]Run actions/cache@fixture+##[endgroup]+Cache hit for: unknown+Cache restored from key: unknown+"""+ entries = parse_cache_log(log)+ assert [(e["restore_status"], e["restored_bytes"], e["save_status"], e["saved_bytes"])+ for e in entries] == [+ ("exact", 100, "not-attempted", None),+ ("miss", None, "saved", 200),+ ("failed", None, "unknown", None),+ ("exact", None, "not-attempted", None),]- assert "service unavailable" in entries[5]["warnings"][0]- assert "Unable to reserve" in entries[6]["warnings"][0]- assert "tar failed" in entries[7]["warnings"][0]- assert entries[8]["name"] == "Restore node_modules"- assert len({e["identity"] for e in entries}) == len(entries)- changed = report.parse_cache_log(log.replace(first, "tools-" + "c" * 64)- .replace(second, "tools-" + "d" * 64), steps)- assert [e["identity"] for e in changed] == [e["identity"] for e in entries]-----def test_restore_only_and_combined_actions_keep_the_same_comparison_identity():- report = importlib.import_module("scripts.ci.cache_report")- main = report.parse_cache_log("\n".join([- _cache("Cache verified PM tools", "main-hash", [- "Cache hit for: main-hash", "Cache Size: ~0 MB (100 B)",- "Cache restored from key: main-hash",- ]),- _post("Cache verified PM tools", [- "Cache hit occurred on the primary key main-hash, not saving cache.",- ]),- ]))- current = report.parse_cache_log("\n".join([- _cache("Restore verified PM tools without saving", "pr-hash", [- "Cache hit for: pr-hash", "Cache Size: ~0 MB (120 B)",- "Cache restored from key: pr-hash",- ], action="actions/cache/restore"),- _cache("Save verified PM tools", "pr-hash", [- "Sent 150 of 150 (100.0%), 1.0 MBs/sec", "Cache saved with key: pr-hash",- ], action="actions/cache/save"),- ]))- assert len(current) == 1- assert current[0]["identity"] == main[0]["identity"]- assert current[0]["save_status"] == "saved" and current[0]["saved_bytes"] == 150- markdown = report.render_cache_summary({"cache_data": {"entries": current}},- {"cache_data": {"entries": main}})- assert "+20 B" in markdown and "Saved" not in markdown---def test_failed_save_does_not_obscure_a_clean_restore_miss():- report = importlib.import_module("scripts.ci.cache_report")- entry = report.parse_cache_log("\n".join([- _cache("Cache downloads", "key", ["Cache not found for input keys: key"]),- _post("Cache downloads", ["##[warning]Failed to save: finalization failed"]),- ]))[0]- main = dict(entry, restore_status="exact", restored_bytes=10, save_status="skipped-exact-hit")- markdown = report.render_cache_summary({"is_main": True, "cache_data": {"entries": [entry]}},- {"cache_data": {"entries": [main]}})- assert "0 / 1 cache hits" in markdown and "1 miss" in markdown- assert "failed" in markdown and "finalization failed" in markdown- assert "vs main" not in markdown---def test_content_key_changes_pair_restore_and_save_by_their_transport_paths():- report = importlib.import_module("scripts.ci.cache_report")- restore = _cache("Restore signature candidates", "before-build", [- "Cache hit for restore-key: previous-signatures", "Cache Size: ~0 MB (100 B)",- "Cache restored from key: previous-signatures",- ], action="actions/cache/restore").replace("with:", "with:\n path: /cache/signatures\n/cache/policy")- save = _cache("Save verified signatures", "after-build", [- "Sent 140 of 140 (100.0%), 1.0 MBs/sec", "Cache saved with key: after-build",- ], action="actions/cache/save").replace("with:", "with:\n path: /cache/signatures\n/cache/policy")- unrelated = _cache("Save another cache", "another-key", [- "Sent 10 of 10 (100.0%), 1.0 MBs/sec", "Cache saved with key: another-key",- ], action="actions/cache/save").replace("with:", "with:\n path: /cache/other")- entries = report.parse_cache_log("\n".join([restore, save, unrelated]))- assert len(entries) == 2, "a content-key change still writes the same restored cache paths"- assert entries[0]["restored_bytes"] == 100 and entries[0]["saved_bytes"] == 140- assert entries[0]["saved_key"] == "after-build"- assert entries[1]["restore_status"] == "not-attempted" and entries[1]["saved_bytes"] == 10- summary = report.render_cache_summary({"is_main": True, "cache_data": {"entries": entries}}, None)- assert "140 B" in summary and "100 B" in summary- assert "vs main" not in summarydiff --git a/tests/ci/test_cache_report_collection.py b/tests/ci/test_cache_report_collection.pyindex 97ec3e6ede1..125018cdd04 100644--- a/tests/ci/test_cache_report_collection.py+++ b/tests/ci/test_cache_report_collection.py@@ -1,88 +1,34 @@-"""Exercise Actions log collection through the existing report/comment path."""-from __future__ import annotations--import json-import threading--import pytest+"""The API token must not reach a redirected log download."""from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer-from urllib.parse import urlsplit+import threading-from scripts.ci import assemble_review_comment, timings_report+from scripts.ci import timings_report[email protected]("event,is_main", [("pull_request", False), ("push", True), ("workflow_dispatch", True)])-def test_collected_cache_data_reaches_reports_without_forwarding_auth(monkeypatch, event, is_main):- timestamp = "2026-10-20T12:00:01Z "- messages = [- "##[group]Run actions/cache/restore@fixture", "with:",- " path: /cache", " key: fixture-v1-current", "##[endgroup]",- "Cache hit for restore-key: fixture-v1-previous",- "Received 100 of 100 (100.0%), 1.0 MBs/sec",- "Cache Size: ~0 MB (100 B)", "Cache restored successfully",- "Cache restored from key: fixture-v1-previous",- ]- log = "\n".join(timestamp + line for line in messages).encode()- jobs = [{- "id": number, "name": name, "status": "completed", "conclusion": conclusion,- "started_at": "2026-10-20T12:00:00Z", "completed_at": "2026-10-20T12:00:02Z",- "html_url": f"https://github.com/fixture/cache/actions/runs/42/job/{number}",- "steps": [{"name": "Cache artifacts", "number": 1, "status": "completed", "conclusion": "success",- "started_at": "2026-10-20T12:00:00Z", "completed_at": "2026-10-20T12:00:02Z"}],- } for number, name, conclusion in [(101, "working", "success"), (102, "logs unavailable", "success"),- (103, "not selected", "skipped")]]- prefix = "/repos/fixture/cache/actions"- routes = {- f"{prefix}/runs/42": (200, {}, json.dumps({"created_at": "2026-10-20T12:00:00Z", "head_branch": "main", "event": event}).encode()),- f"{prefix}/runs/42/jobs": (200, {}, json.dumps({"jobs": jobs}).encode()),- f"{prefix}/runs": (200, {}, b'{"workflow_runs": []}'),- "/blob": (200, {}, log),- }+def test_log_redirect_does_not_forward_api_token(monkeypatch) -> None:requests = []class Handler(BaseHTTPRequestHandler):- def do_GET(self):- path = urlsplit(self.path).path- requests.append((path, self.headers.get("Authorization")))- status, headers, body = routes.get(path, (404, {}, b"missing"))- self.send_response(status)- for name, value in headers.items():- self.send_header(name, value)+ def do_GET(self) -> None:+ requests.append((self.path, self.headers.get("Authorization")))+ redirect = self.path.endswith("/logs")+ self.send_response(302 if redirect else 200)+ if redirect:+ self.send_header("Location", "/blob")self.end_headers()- self.wfile.write(body)-- def log_message(self, format: str, *args: object) -> None:- pass+ self.wfile.write(b"log")server = ThreadingHTTPServer(("127.0.0.1", 0), Handler)- address = f"http://127.0.0.1:{server.server_port}"- routes[f"{prefix}/jobs/101/logs"] = (302, {"Location": address + "/blob"}, b"")+ monkeypatch.setattr(timings_report, "API_BASE", f"http://127.0.0.1:{server.server_port}")thread = threading.Thread(target=server.serve_forever, daemon=True)thread.start()- monkeypatch.setattr(timings_report, "API_BASE", address)try:- current = timings_report.collect_timings("fixture-token", "fixture/cache", "42", "head")+ assert timings_report._job_log(1, "fixture/cache", "fixture-token") == "log"finally:server.shutdown()server.server_close()thread.join()-- assert "cache_data" in current, "CI collection must retain per-cache data in the baseline JSON"- assert current["is_main"] is is_main, "a PR from a branch named main is not a main run"- caches = current["cache_data"]- assert caches["unavailable_jobs"] == ["logs unavailable"]- assert len(caches["entries"]) == 1- assert caches["entries"][0]["restored_bytes"] == 100- assert ("/blob", None) in requests, "the signed log-blob hop must not receive the API bearer"- assert (f"{prefix}/jobs/101/logs", "Bearer fixture-token") in requests- assert not any("/jobs/103/" in path for path, _ in requests)-- old_baseline = {"jobs": current["jobs"]}- statuses = timings_report.generate_review_status(current, old_baseline, "https://example.invalid/report")- cache_status, = [s for s in statuses if s["source"] == "ci caches"]- assert cache_status["results"][0]["link"] == "https://example.invalid/report"- comment = assemble_review_comment.assemble(review_statuses_json=json.dumps(statuses))- assert "CI caches" in comment and "logs unavailable" in comment- assert "Cache artifacts" in timings_report.generate_html(current, old_baseline)- assert "#step:1:1" in comment- assert "CI caches" in timings_report.generate_summary(current, old_baseline)+ assert requests == [+ ("/repos/fixture/cache/actions/jobs/1/logs", "Bearer fixture-token"),+ ("/blob", None),+ ]diff --git a/tests/ci/test_cache_report_views.py b/tests/ci/test_cache_report_views.pydeleted file mode 100644index 8801f6cc6bc..00000000000--- a/tests/ci/test_cache_report_views.py+++ /dev/null@@ -1,77 +0,0 @@-"""Cache presentation separates read-only comparisons from main's writes."""-from __future__ import annotations--import pytest--from scripts.ci import cache_report as report-from tests.ci.test_cache_report import _cache, _post---def _snapshot(main: bool, *, baseline: bool = False) -> dict:- log = "\n".join([- _cache("Cache packages", "hit", [- "Cache hit for: hit", f"Cache Size: ~0 MB ({80 if baseline else 100} B)",- "Cache restored from key: hit",- ]),- _cache("Cache tools", "miss", ["Cache not found for input keys: miss"]),- _cache("Cache broken", "broken", ["##[warning]Failed to restore: service | <script>"]),- _cache("Lookup", "lookup", ["Cache found and can be restored from key: lookup"]),- _post("Cache packages", [- f"Sent {90 if baseline else 130} of {90 if baseline else 130} (100.0%), 1.0 MBs/sec",- "Cache saved with key: hit",- ]),- _post("Cache tools", [- f"Sent {70 if baseline else 40} of {70 if baseline else 40} (100.0%), 1.0 MBs/sec",- "Cache saved with key: miss",- ]),- ])- steps = [{"name": "Prepare packages", "number": 4, "started_at": "2026-10-20T20:00:00Z",- "completed_at": "2026-10-20T20:00:59Z"}]- entries = report.parse_cache_log(log, steps)- return {"is_main": main, "cache_data": {"entries": [dict(e, job="job|<svg>",- job_url="https://github.com/fixture/cache/actions/runs/1/job/2") for e in entries],- "jobs_collected": 1, "unavailable_jobs": ["lost|<script>"]}}--[email protected]("is_main", [False, True], ids=["pull-request", "main"])-def test_cache_summary_and_expando_follow_the_run_mode(is_main):- current, baseline = _snapshot(is_main), _snapshot(True, baseline=True)- result = report.cache_review_status(current, baseline)[0]["results"][0]- markdown, html = report.render_cache_summary(current, baseline), report.render_cache_html(current, baseline)- assert "1 / 3 cache hits" in result["summary"]- assert "1 miss" in result["summary"]- assert "100 B restored" in result["summary"]- assert f"{40 if is_main else 70} B not restored" in result["summary"]- assert "unknown" in result["summary"]- for rendered in (result["detail"], markdown, html):- assert "<details>" in rendered and "<summary>" in rendered- assert "hits" in rendered and "misses" in rendered- assert "job" in rendered and "Cache packages" in rendered- assert "#step:4:1" in rendered- assert "lost" in rendered and "<script>" in rendered- assert "<script>" not in rendered and "<svg>" not in rendered- if is_main:- assert "170 B saved" in result["summary"]- assert "Saved" in html and "130 B" in html and "40 B" in html- assert "vs main" not in html and "vs main" not in markdown- assert "+10 B" not in html- else:- assert "saved" not in result["summary"]- assert "Saved" not in html and "130 B" not in html- assert "vs main" in html and "+10 B" in html- misses = result["detail"].split("<h3>misses</h3>", 1)[1].split("<h3>", 1)[0]- assert "Cache tools" in misses- assert "Restored" not in misses and "vs main" not in misses---def test_miss_bytes_remain_unknown_without_a_main_reference():- current = _snapshot(False)- result = report.cache_review_status(current, None)[0]["results"][0]- assert "not restored: unknown" in result["summary"]- assert "0 B not restored" not in result["summary"]- assert "main-size reference" in result["detail"]- assert "lookup-only" in result["detail"]- current["cache_data"]["entries"][0]["restored_bytes"] = None- unknown = report.cache_review_status(current, None)[0]["results"][0]- assert "restored: unknown" in unknown["summary"]- assert "0 B restored" not in unknown["summary"]diff --git a/tests/ci/test_pm_cache_log.py b/tests/ci/test_pm_cache_log.pydeleted file mode 100644index 616dc5f6336..00000000000--- a/tests/ci/test_pm_cache_log.py+++ /dev/null@@ -1,48 +0,0 @@-"""PM cache diagnostics describe the action's actual selected branches."""-from __future__ import annotations--import json-import os-from pathlib import Path-import subprocess-import sys--import pytest--[email protected]("hit,save", [("true", "true"), ("false", "true"), ("", "false")])-def test_cache_log_reports_selected_restore_and_save_policy(tmp_path, hit, save, capsys):- from scripts.ci.pm_cache_log import log_caches- steps: dict[str, dict] = {"prepare": {"outputs": {"store": str(tmp_path / "tools")}},- "install": {"outputs": {"uv-cache-path": str(tmp_path / "uv"),- "npm-cache-path": str(tmp_path / "npm")}}}- inputs = {}- for name in ["tools", "python", "node"]:- inputs[f"save-{name}-cache"] = save- selected = f"{name}-cache" + ("-restore" if save == "false" else "")- steps[selected] = {"outcome": "success", "outputs": {"cache-hit": hit}}- log_caches(inputs, steps, "fixture-ref")- lines = capsys.readouterr().out.splitlines()- assert len(lines) == 3- for family, line in zip(["tools", "uv", "npm"], lines):- assert f"[setup-pm cache] {family}:" in line- assert f"cache-hit={hit!r}" in line- assert ("save=on-success" if save == "true" else "save=disabled") in line- assert str(tmp_path / family) in line---def test_cache_log_does_not_call_a_skipped_restore_a_miss(tmp_path):- script = Path(__file__).resolve().parents[2] / "scripts/ci/pm_cache_log.py"- result = subprocess.run(- [sys.executable, "-S", str(script)],- env={**os.environ, "_PM_CACHE_INPUTS": "{}", "_PM_CACHE_STEPS": json.dumps({- "tools-cache": {"outcome": "skipped", "outputs": {}},- "python-cache-restore": {"outcome": "failure", "outputs": {}},- })},- capture_output=True, text=True, encoding="utf-8", timeout=10, check=False,- )- assert result.returncode == 0, result.stderr- lines = result.stdout.splitlines()- assert "tools: restore=skipped" in lines[0]- assert "uv: restore=failure" in lines[1]- assert "npm: restore=skipped" in lines[2]