Skip to content

The agent that grows with you

test(ci): trim cache coverage to two behavior checks

ethernet63c1d815dde2
diff --git a/tests/ci/test_cache_report.py b/tests/ci/test_cache_report.py
index 9018777eef5..c344bf8157d 100644
--- a/tests/ci/test_cache_report.py
+++ b/tests/ci/test_cache_report.py
@@ -1,170 +1,38 @@
-"""Compressed archive accounting follows cache receipts, not progress or key hashes."""
-
-from __future__ import annotations
-
-import importlib
-
-
-def _cache(name, key, events, *, action="actions/cache", minute=0):
- marker = f"##[start-action display={name};id=forward.{name}]" if name else ""
- lines = [marker, f"##[group]Run {action}@55cc8345863c7cc4c66a329aec7e433d2d1c52a9",
- "with:", f" key: {key}", " lookup-only: false", "##[endgroup]", *events]
- if name:
- lines.append(f"##[end-action id=forward.{name};outcome=success]")
- return "\n".join(f"2026-10-20T20:{minute:02d}:10.1234567Z {line}" for line in lines if line)
-
-
-def _post(name, events):
- return "\n".join([
- "Post job cleanup.",
- f"##[start-action display={name};id=__changed-uuid.{name}]",
- "Post job cleanup.", *events,
- f"##[end-action id=__changed-uuid.{name};outcome=success]",
- ])
-
-
-def test_cache_invocations_account_only_confirmed_phase_local_archives():
- report = importlib.import_module("scripts.ci.cache_report")
- first = "tools-" + "a" * 64
- second = "tools-" + "b" * 64
- log = "\n".join([
- _cache("PM tools", first, [
- "Cache hit for restore-key: tools-main",
- "Received 0 of 100 (0.0%), 0.0 MBs/sec",
- "Received 100 of 100 (100.0%), 1.0 MBs/sec",
- "Cache Size: ~0 MB (100 B)", "Cache restored successfully",
- "Cache restored from key: tools-main",
- ]),
- _cache("PM tools", second, [f"Cache not found for input keys: {second}"]),
- _cache("uv", "uv-key", [
- "Cache hit for: uv-key", "Cache Size: ~0 MB (40 B)",
- "Cache restored successfully", "Cache restored from key: uv-key",
- ]),
- _cache("npm", "npm-key", [
- "Cache found and can be restored from key: npm-key", "Lookup only - skipping download",
- ]),
- _cache("unknown size", "unknown-key", ["Cache restored from key: unknown-key"]),
- _cache("transport", "failed-key", [
- "##[warning]Failed to restore: service unavailable",
- "Cache not found for input keys: failed-key",
- ]),
- _cache("race", "race-key", ["Cache not found for input keys: race-key"]),
- _cache("failed restore", "broken-key", [
- "Cache hit for: broken-key", "Received 90 of 90 (100.0%), 1.0 MBs/sec",
- "Cache Size: ~0 MB (90 B)", "##[warning]Failed to restore: tar failed",
- ]),
- _cache(None, "modules-key", [
- "Cache hit for: modules-key", "Cache Size: ~0 MB (7 B)",
- "Cache restored from key: modules-key",
- ], minute=1),
- _cache("unconfirmed", "unconfirmed-key", ["Cache not found for input keys: unconfirmed-key"], minute=2),
- "##[group]Run echo 'Cache saved with key: fake'",
- "echo 'Cache Size: ~0 MB (99999 B)'", "##[endgroup]",
- "Cache saved with key: fake", # Shell output outside a cache action is not a receipt.
- _post("PM tools", [
- "Sent 0 of 200 (0.0%), 0.0 MBs/sec", "Sent 200 of 200 (100.0%), 1.0 MBs/sec",
- f"Cache saved with key: {second}",
- ]),
- _post("PM tools", [
- "Sent 150 of 150 (100.0%), 1.0 MBs/sec", f"Cache saved with key: {first}",
- ]),
- _post("uv", ["Cache hit occurred on the primary key uv-key, not saving cache."]),
- _post("race", [
- "Sent 500 of 500 (100.0%), 1.0 MBs/sec",
- "##[warning]Failed to save: Unable to reserve cache with key race-key, another job may be creating this cache.",
- ]),
- _post("unknown size", ["Cache saved with key: unknown-key"]),
- _post("unconfirmed", ["Sent 300 of 300 (100.0%), 1.0 MBs/sec"]),
- _post("Upload artifact", ["##[warning]unrelated cleanup warning"]),
- ])
- steps = [{"name": "Restore node_modules", "number": 9, "started_at": "2026-10-20T20:01:10Z",
- "completed_at": "2026-10-20T20:01:11Z"}]
- entries = report.parse_cache_log(log, steps)
- assert len(entries) == 10
- assert [(e["requested_key"], e["matched_key"], e["restore_status"], e["restored_bytes"],
- e["save_status"], e["saved_bytes"]) for e in entries] == [
- (first, "tools-main", "fallback", 100, "saved", 150),
- (second, None, "miss", None, "saved", 200),
- ("uv-key", "uv-key", "exact", 40, "skipped-exact-hit", None),
- ("npm-key", "npm-key", "lookup-only", None, "not-attempted", None),
- ("unknown-key", "unknown-key", "exact", None, "saved", None),
- ("failed-key", None, "miss", None, "not-attempted", None),
- ("race-key", None, "miss", None, "failed", None),
- ("broken-key", "broken-key", "failed", None, "not-attempted", None),
- ("modules-key", "modules-key", "exact", 7, "not-attempted", None),
- ("unconfirmed-key", None, "miss", None, "unknown", None),
+"""Protect archive accounting, not report wording or markup."""
+from scripts.ci.cache_report import parse_cache_log
+
+
+def test_cache_accounting() -> None:
+ log = """##[group]Run actions/cache@fixture
+##[endgroup]
+Cache hit for: hit
+Received 0 of 100 (0.0%), 0.0 MBs/sec
+Received 100 of 100 (100.0%), 1.0 MBs/sec
+Cache restored from key: hit
+##[group]Run actions/cache@fixture
+##[endgroup]
+Cache not found for input keys: miss
+Post job cleanup.
+Sent 0 of 200 (0.0%), 0.0 MBs/sec
+Sent 200 of 200 (100.0%), 1.0 MBs/sec
+Cache saved with key: miss
+##[group]Run actions/cache@fixture
+##[endgroup]
+Cache hit for: failed
+Received 80 of 80 (100.0%), 1.0 MBs/sec
+##[warning]Failed to restore: tar failed
+Post job cleanup.
+Sent 90 of 90 (100.0%), 1.0 MBs/sec
+##[group]Run actions/cache@fixture
+##[endgroup]
+Cache hit for: unknown
+Cache restored from key: unknown
+"""
+ entries = parse_cache_log(log)
+ assert [(e["restore_status"], e["restored_bytes"], e["save_status"], e["saved_bytes"])
+ for e in entries] == [
+ ("exact", 100, "not-attempted", None),
+ ("miss", None, "saved", 200),
+ ("failed", None, "unknown", None),
+ ("exact", None, "not-attempted", None),
]
- assert "service unavailable" in entries[5]["warnings"][0]
- assert "Unable to reserve" in entries[6]["warnings"][0]
- assert "tar failed" in entries[7]["warnings"][0]
- assert entries[8]["name"] == "Restore node_modules"
- assert len({e["identity"] for e in entries}) == len(entries)
- changed = report.parse_cache_log(log.replace(first, "tools-" + "c" * 64)
- .replace(second, "tools-" + "d" * 64), steps)
- assert [e["identity"] for e in changed] == [e["identity"] for e in entries]
-
-
-
-
-def test_restore_only_and_combined_actions_keep_the_same_comparison_identity():
- report = importlib.import_module("scripts.ci.cache_report")
- main = report.parse_cache_log("\n".join([
- _cache("Cache verified PM tools", "main-hash", [
- "Cache hit for: main-hash", "Cache Size: ~0 MB (100 B)",
- "Cache restored from key: main-hash",
- ]),
- _post("Cache verified PM tools", [
- "Cache hit occurred on the primary key main-hash, not saving cache.",
- ]),
- ]))
- current = report.parse_cache_log("\n".join([
- _cache("Restore verified PM tools without saving", "pr-hash", [
- "Cache hit for: pr-hash", "Cache Size: ~0 MB (120 B)",
- "Cache restored from key: pr-hash",
- ], action="actions/cache/restore"),
- _cache("Save verified PM tools", "pr-hash", [
- "Sent 150 of 150 (100.0%), 1.0 MBs/sec", "Cache saved with key: pr-hash",
- ], action="actions/cache/save"),
- ]))
- assert len(current) == 1
- assert current[0]["identity"] == main[0]["identity"]
- assert current[0]["save_status"] == "saved" and current[0]["saved_bytes"] == 150
- markdown = report.render_cache_summary({"cache_data": {"entries": current}},
- {"cache_data": {"entries": main}})
- assert "+20 B" in markdown and "Saved" not in markdown
-
-
-def test_failed_save_does_not_obscure_a_clean_restore_miss():
- report = importlib.import_module("scripts.ci.cache_report")
- entry = report.parse_cache_log("\n".join([
- _cache("Cache downloads", "key", ["Cache not found for input keys: key"]),
- _post("Cache downloads", ["##[warning]Failed to save: finalization failed"]),
- ]))[0]
- main = dict(entry, restore_status="exact", restored_bytes=10, save_status="skipped-exact-hit")
- markdown = report.render_cache_summary({"is_main": True, "cache_data": {"entries": [entry]}},
- {"cache_data": {"entries": [main]}})
- assert "0 / 1 cache hits" in markdown and "1 miss" in markdown
- assert "failed" in markdown and "finalization failed" in markdown
- assert "vs main" not in markdown
-
-
-def test_content_key_changes_pair_restore_and_save_by_their_transport_paths():
- report = importlib.import_module("scripts.ci.cache_report")
- restore = _cache("Restore signature candidates", "before-build", [
- "Cache hit for restore-key: previous-signatures", "Cache Size: ~0 MB (100 B)",
- "Cache restored from key: previous-signatures",
- ], action="actions/cache/restore").replace("with:", "with:\n path: /cache/signatures\n/cache/policy")
- save = _cache("Save verified signatures", "after-build", [
- "Sent 140 of 140 (100.0%), 1.0 MBs/sec", "Cache saved with key: after-build",
- ], action="actions/cache/save").replace("with:", "with:\n path: /cache/signatures\n/cache/policy")
- unrelated = _cache("Save another cache", "another-key", [
- "Sent 10 of 10 (100.0%), 1.0 MBs/sec", "Cache saved with key: another-key",
- ], action="actions/cache/save").replace("with:", "with:\n path: /cache/other")
- entries = report.parse_cache_log("\n".join([restore, save, unrelated]))
- assert len(entries) == 2, "a content-key change still writes the same restored cache paths"
- assert entries[0]["restored_bytes"] == 100 and entries[0]["saved_bytes"] == 140
- assert entries[0]["saved_key"] == "after-build"
- assert entries[1]["restore_status"] == "not-attempted" and entries[1]["saved_bytes"] == 10
- summary = report.render_cache_summary({"is_main": True, "cache_data": {"entries": entries}}, None)
- assert "140 B" in summary and "100 B" in summary
- assert "vs main" not in summary
diff --git a/tests/ci/test_cache_report_collection.py b/tests/ci/test_cache_report_collection.py
index 97ec3e6ede1..125018cdd04 100644
--- a/tests/ci/test_cache_report_collection.py
+++ b/tests/ci/test_cache_report_collection.py
@@ -1,88 +1,34 @@
-"""Exercise Actions log collection through the existing report/comment path."""
-from __future__ import annotations
-
-import json
-import threading
-
-import pytest
+"""The API token must not reach a redirected log download."""
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
-from urllib.parse import urlsplit
+import threading
-from scripts.ci import assemble_review_comment, timings_report
+from scripts.ci import timings_report
[email protected]("event,is_main", [("pull_request", False), ("push", True), ("workflow_dispatch", True)])
-def test_collected_cache_data_reaches_reports_without_forwarding_auth(monkeypatch, event, is_main):
- timestamp = "2026-10-20T12:00:01Z "
- messages = [
- "##[group]Run actions/cache/restore@fixture", "with:",
- " path: /cache", " key: fixture-v1-current", "##[endgroup]",
- "Cache hit for restore-key: fixture-v1-previous",
- "Received 100 of 100 (100.0%), 1.0 MBs/sec",
- "Cache Size: ~0 MB (100 B)", "Cache restored successfully",
- "Cache restored from key: fixture-v1-previous",
- ]
- log = "\n".join(timestamp + line for line in messages).encode()
- jobs = [{
- "id": number, "name": name, "status": "completed", "conclusion": conclusion,
- "started_at": "2026-10-20T12:00:00Z", "completed_at": "2026-10-20T12:00:02Z",
- "html_url": f"https://github.com/fixture/cache/actions/runs/42/job/{number}",
- "steps": [{"name": "Cache artifacts", "number": 1, "status": "completed", "conclusion": "success",
- "started_at": "2026-10-20T12:00:00Z", "completed_at": "2026-10-20T12:00:02Z"}],
- } for number, name, conclusion in [(101, "working", "success"), (102, "logs unavailable", "success"),
- (103, "not selected", "skipped")]]
- prefix = "/repos/fixture/cache/actions"
- routes = {
- f"{prefix}/runs/42": (200, {}, json.dumps({"created_at": "2026-10-20T12:00:00Z", "head_branch": "main", "event": event}).encode()),
- f"{prefix}/runs/42/jobs": (200, {}, json.dumps({"jobs": jobs}).encode()),
- f"{prefix}/runs": (200, {}, b'{"workflow_runs": []}'),
- "/blob": (200, {}, log),
- }
+def test_log_redirect_does_not_forward_api_token(monkeypatch) -> None:
requests = []
class Handler(BaseHTTPRequestHandler):
- def do_GET(self):
- path = urlsplit(self.path).path
- requests.append((path, self.headers.get("Authorization")))
- status, headers, body = routes.get(path, (404, {}, b"missing"))
- self.send_response(status)
- for name, value in headers.items():
- self.send_header(name, value)
+ def do_GET(self) -> None:
+ requests.append((self.path, self.headers.get("Authorization")))
+ redirect = self.path.endswith("/logs")
+ self.send_response(302 if redirect else 200)
+ if redirect:
+ self.send_header("Location", "/blob")
self.end_headers()
- self.wfile.write(body)
-
- def log_message(self, format: str, *args: object) -> None:
- pass
+ self.wfile.write(b"log")
server = ThreadingHTTPServer(("127.0.0.1", 0), Handler)
- address = f"http://127.0.0.1:{server.server_port}"
- routes[f"{prefix}/jobs/101/logs"] = (302, {"Location": address + "/blob"}, b"")
+ monkeypatch.setattr(timings_report, "API_BASE", f"http://127.0.0.1:{server.server_port}")
thread = threading.Thread(target=server.serve_forever, daemon=True)
thread.start()
- monkeypatch.setattr(timings_report, "API_BASE", address)
try:
- current = timings_report.collect_timings("fixture-token", "fixture/cache", "42", "head")
+ assert timings_report._job_log(1, "fixture/cache", "fixture-token") == "log"
finally:
server.shutdown()
server.server_close()
thread.join()
-
- assert "cache_data" in current, "CI collection must retain per-cache data in the baseline JSON"
- assert current["is_main"] is is_main, "a PR from a branch named main is not a main run"
- caches = current["cache_data"]
- assert caches["unavailable_jobs"] == ["logs unavailable"]
- assert len(caches["entries"]) == 1
- assert caches["entries"][0]["restored_bytes"] == 100
- assert ("/blob", None) in requests, "the signed log-blob hop must not receive the API bearer"
- assert (f"{prefix}/jobs/101/logs", "Bearer fixture-token") in requests
- assert not any("/jobs/103/" in path for path, _ in requests)
-
- old_baseline = {"jobs": current["jobs"]}
- statuses = timings_report.generate_review_status(current, old_baseline, "https://example.invalid/report")
- cache_status, = [s for s in statuses if s["source"] == "ci caches"]
- assert cache_status["results"][0]["link"] == "https://example.invalid/report"
- comment = assemble_review_comment.assemble(review_statuses_json=json.dumps(statuses))
- assert "CI caches" in comment and "logs unavailable" in comment
- assert "Cache artifacts" in timings_report.generate_html(current, old_baseline)
- assert "#step:1:1" in comment
- assert "CI caches" in timings_report.generate_summary(current, old_baseline)
+ assert requests == [
+ ("/repos/fixture/cache/actions/jobs/1/logs", "Bearer fixture-token"),
+ ("/blob", None),
+ ]
diff --git a/tests/ci/test_cache_report_views.py b/tests/ci/test_cache_report_views.py
deleted file mode 100644
index 8801f6cc6bc..00000000000
--- a/tests/ci/test_cache_report_views.py
+++ /dev/null
@@ -1,77 +0,0 @@
-"""Cache presentation separates read-only comparisons from main's writes."""
-from __future__ import annotations
-
-import pytest
-
-from scripts.ci import cache_report as report
-from tests.ci.test_cache_report import _cache, _post
-
-
-def _snapshot(main: bool, *, baseline: bool = False) -> dict:
- log = "\n".join([
- _cache("Cache packages", "hit", [
- "Cache hit for: hit", f"Cache Size: ~0 MB ({80 if baseline else 100} B)",
- "Cache restored from key: hit",
- ]),
- _cache("Cache tools", "miss", ["Cache not found for input keys: miss"]),
- _cache("Cache broken", "broken", ["##[warning]Failed to restore: service | <script>"]),
- _cache("Lookup", "lookup", ["Cache found and can be restored from key: lookup"]),
- _post("Cache packages", [
- f"Sent {90 if baseline else 130} of {90 if baseline else 130} (100.0%), 1.0 MBs/sec",
- "Cache saved with key: hit",
- ]),
- _post("Cache tools", [
- f"Sent {70 if baseline else 40} of {70 if baseline else 40} (100.0%), 1.0 MBs/sec",
- "Cache saved with key: miss",
- ]),
- ])
- steps = [{"name": "Prepare packages", "number": 4, "started_at": "2026-10-20T20:00:00Z",
- "completed_at": "2026-10-20T20:00:59Z"}]
- entries = report.parse_cache_log(log, steps)
- return {"is_main": main, "cache_data": {"entries": [dict(e, job="job|<svg>",
- job_url="https://github.com/fixture/cache/actions/runs/1/job/2") for e in entries],
- "jobs_collected": 1, "unavailable_jobs": ["lost|<script>"]}}
-
-
[email protected]("is_main", [False, True], ids=["pull-request", "main"])
-def test_cache_summary_and_expando_follow_the_run_mode(is_main):
- current, baseline = _snapshot(is_main), _snapshot(True, baseline=True)
- result = report.cache_review_status(current, baseline)[0]["results"][0]
- markdown, html = report.render_cache_summary(current, baseline), report.render_cache_html(current, baseline)
- assert "1 / 3 cache hits" in result["summary"]
- assert "1 miss" in result["summary"]
- assert "100 B restored" in result["summary"]
- assert f"{40 if is_main else 70} B not restored" in result["summary"]
- assert "unknown" in result["summary"]
- for rendered in (result["detail"], markdown, html):
- assert "<details>" in rendered and "<summary>" in rendered
- assert "hits" in rendered and "misses" in rendered
- assert "job" in rendered and "Cache packages" in rendered
- assert "#step:4:1" in rendered
- assert "lost" in rendered and "&lt;script&gt;" in rendered
- assert "<script>" not in rendered and "<svg>" not in rendered
- if is_main:
- assert "170 B saved" in result["summary"]
- assert "Saved" in html and "130 B" in html and "40 B" in html
- assert "vs main" not in html and "vs main" not in markdown
- assert "+10 B" not in html
- else:
- assert "saved" not in result["summary"]
- assert "Saved" not in html and "130 B" not in html
- assert "vs main" in html and "+10 B" in html
- misses = result["detail"].split("<h3>misses</h3>", 1)[1].split("<h3>", 1)[0]
- assert "Cache tools" in misses
- assert "Restored" not in misses and "vs main" not in misses
-
-
-def test_miss_bytes_remain_unknown_without_a_main_reference():
- current = _snapshot(False)
- result = report.cache_review_status(current, None)[0]["results"][0]
- assert "not restored: unknown" in result["summary"]
- assert "0 B not restored" not in result["summary"]
- assert "main-size reference" in result["detail"]
- assert "lookup-only" in result["detail"]
- current["cache_data"]["entries"][0]["restored_bytes"] = None
- unknown = report.cache_review_status(current, None)[0]["results"][0]
- assert "restored: unknown" in unknown["summary"]
- assert "0 B restored" not in unknown["summary"]
diff --git a/tests/ci/test_pm_cache_log.py b/tests/ci/test_pm_cache_log.py
deleted file mode 100644
index 616dc5f6336..00000000000
--- a/tests/ci/test_pm_cache_log.py
+++ /dev/null
@@ -1,48 +0,0 @@
-"""PM cache diagnostics describe the action's actual selected branches."""
-from __future__ import annotations
-
-import json
-import os
-from pathlib import Path
-import subprocess
-import sys
-
-import pytest
-
-
[email protected]("hit,save", [("true", "true"), ("false", "true"), ("", "false")])
-def test_cache_log_reports_selected_restore_and_save_policy(tmp_path, hit, save, capsys):
- from scripts.ci.pm_cache_log import log_caches
- steps: dict[str, dict] = {"prepare": {"outputs": {"store": str(tmp_path / "tools")}},
- "install": {"outputs": {"uv-cache-path": str(tmp_path / "uv"),
- "npm-cache-path": str(tmp_path / "npm")}}}
- inputs = {}
- for name in ["tools", "python", "node"]:
- inputs[f"save-{name}-cache"] = save
- selected = f"{name}-cache" + ("-restore" if save == "false" else "")
- steps[selected] = {"outcome": "success", "outputs": {"cache-hit": hit}}
- log_caches(inputs, steps, "fixture-ref")
- lines = capsys.readouterr().out.splitlines()
- assert len(lines) == 3
- for family, line in zip(["tools", "uv", "npm"], lines):
- assert f"[setup-pm cache] {family}:" in line
- assert f"cache-hit={hit!r}" in line
- assert ("save=on-success" if save == "true" else "save=disabled") in line
- assert str(tmp_path / family) in line
-
-
-def test_cache_log_does_not_call_a_skipped_restore_a_miss(tmp_path):
- script = Path(__file__).resolve().parents[2] / "scripts/ci/pm_cache_log.py"
- result = subprocess.run(
- [sys.executable, "-S", str(script)],
- env={**os.environ, "_PM_CACHE_INPUTS": "{}", "_PM_CACHE_STEPS": json.dumps({
- "tools-cache": {"outcome": "skipped", "outputs": {}},
- "python-cache-restore": {"outcome": "failure", "outputs": {}},
- })},
- capture_output=True, text=True, encoding="utf-8", timeout=10, check=False,
- )
- assert result.returncode == 0, result.stderr
- lines = result.stdout.splitlines()
- assert "tools: restore=skipped" in lines[0]
- assert "uv: restore=failure" in lines[1]
- assert "npm: restore=skipped" in lines[2]