Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.
Python agent-evaluationagent-pluginsagent-securityagent-skillsagentic-aibenchmarkci-templatesclaude-codecodexevaluateevaluationharnesspluginplugin-evaluationsecurity-scannerskill-evalskill-evaluationskill-evaluatorskill-gap-analysisskills
Add private MCP execution proof and behavior signals (#321)
Narendran Raghavand31f1a433df2
diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.mdindex c60b08ea..bffb58a8 100644--- a/THIRD_PARTY_NOTICES.md+++ b/THIRD_PARTY_NOTICES.md@@ -6,7 +6,7 @@ records the exact resolved dependency set used for this release.| Dependency group | Packages and licenses || --- | --- |-| Base | Click (BSD-3-Clause), IDNA (BSD-3-Clause), Jinja2 (BSD-3-Clause), Markdown-It-Py (MIT), Packaging (Apache-2.0 or BSD-2-Clause), Pydantic (MIT), PyYAML (MIT), Rich (MIT) |+| Base | Click (BSD-3-Clause), IDNA (BSD-3-Clause), Jinja2 (BSD-3-Clause), Markdown-It-Py (MIT), Packaging (Apache-2.0 or BSD-2-Clause), Pydantic (MIT), PyYAML (MIT), regex (Apache-2.0 AND CNRI-Python), Rich (MIT) || LLM | Anthropic (MIT), Boto3 (Apache-2.0), LiteLLM (MIT), OpenAI (Apache-2.0) || Tier 3 | Harbor (Apache-2.0), MCP (MIT), OAuthLib (BSD-3-Clause), PyJWT (MIT), python-dotenv (BSD-3-Clause) || Security | Bandit (Apache-2.0), pip-audit (Apache-2.0) |diff --git a/pyproject.toml b/pyproject.tomlindex 9a637054..edd44742 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -45,6 +45,8 @@ dependencies = ["packaging>=24","pydantic>=2.11.0","pyyaml>=6.0",+ # Dataset-authored signal patterns require a per-match timeout.+ "regex>=2024.11.6","rich>=13.5.0",]diff --git a/src/skillevaluator/constants.py b/src/skillevaluator/constants.pyindex ea540722..207217c4 100644--- a/src/skillevaluator/constants.py+++ b/src/skillevaluator/constants.py@@ -549,6 +549,14 @@ AGENT_EVAL_VERDICT_FAIL = "fail"TIER3_LIFT_PASS_THRESHOLD = 0.05TIER3_LIFT_FAIL_THRESHOLD = -0.10+# Private paired comparisons resample whole cases with a reproducible seed.+LIFT_BOOTSTRAP_RESAMPLES = 2000+LIFT_BOOTSTRAP_TAIL_RESAMPLES = 200+LIFT_BOOTSTRAP_SEED = 0+LIFT_BOOTSTRAP_CONFIDENCE = 0.95+LIFT_CI_MIN_PAIRED_CASES = 5+LIFT_CI_LOW_PRECISION_WIDTH = 0.20+AGENT_EVAL_DATASET_REQUIRED_FIELDS = ["id","question",diff --git a/src/skillevaluator/reporting/plugin_static.py b/src/skillevaluator/reporting/plugin_static.pynew file mode 100644index 00000000..4a0fb9ef--- /dev/null+++ b/src/skillevaluator/reporting/plugin_static.py@@ -0,0 +1,702 @@+# SPDX-License-Identifier: Apache-2.0+"""Private bounded projection of passive plugin evidence; no discovery or execution."""++from __future__ import annotations++import json+import math+import posixpath+import unicodedata+from collections import Counter+from dataclasses import dataclass+from itertools import islice+from pathlib import PurePosixPath, PureWindowsPath+from typing import TYPE_CHECKING, Any+from urllib.parse import urlsplit, urlunsplit++from skillevaluator.utils.redaction import redact_sensitive_text+from skillevaluator.utils.rich_markup import strip_terminal_controls++if TYPE_CHECKING:+ from skillevaluator.models.result import ValidationResult++MAX_OUTPUT_BYTES = 256 * 1024+MAX_RESULTS = 256+MAX_INPUT_ROWS = 8192+MAX_LIST_ROWS = 4096+MAX_SECTION_ROWS = 64+MAX_TOTAL_ROWS = 256+MAX_TEXT_BYTES = 48 * 1024+MAX_FIELD_INPUT = 16384+MAX_CELL_CHARS = 300+++def _mapping(value: object) -> dict:+ return value if isinstance(value, dict) else {}+++def _number(value: object) -> int | float | None:+ if type(value) not in (int, float):+ return None+ if isinstance(value, int) and value.bit_length() > 53:+ return None+ return value if math.isfinite(value) else None+++def _count(value: object) -> int | None:+ return value if type(value) is int and 0 <= value <= 2**53 - 1 else None+++def _path(value: object) -> str:+ if not isinstance(value, str) or len(value) > 4096:+ return ""+ return PurePosixPath(value.replace("\\", "/")).as_posix()+++def _absolute(path: str) -> bool:+ return PurePosixPath(path).is_absolute() or PureWindowsPath(path).is_absolute()+++def _clean_text(value: str) -> tuple[str, bool]:+ if len(value) > MAX_FIELD_INPUT:+ return "[text exceeds inspection bound]", True+ value = redact_sensitive_text(strip_terminal_controls(value))+ value = "".join(+ "\ufffd"+ if unicodedata.category(char) == "Cs"+ else f"\\u{ord(char):04x}"+ if unicodedata.category(char) == "Cf"+ else char+ for char in value+ )+ value = " ".join(value.split())+ truncated = len(value) > MAX_CELL_CHARS+ return (value[: MAX_CELL_CHARS - 1] + "…" if truncated else value), truncated+++@dataclass(frozen=True)+class StaticPluginReport:+ """Immutable serialized evidence and a bounded lexical component index."""++ _serialized: str+ _root: str+ _components: tuple[tuple[str, str, str], ...]++ def payload(self) -> dict[str, Any]:+ return json.loads(self._serialized)++ def component_for(self, file_path: object, metadata: object = None) -> dict[str, str] | None:+ raw = _path(file_path)+ label = None+ if raw.startswith("[") and "] " in raw:+ label, raw = raw[1:].split("] ", 1)+ if ".." in PurePosixPath(raw).parts:+ return None+ if self._root and (raw == self._root or raw.startswith(self._root.rstrip("/") + "/")):+ raw = raw[len(self._root) :].lstrip("/")+ elif _absolute(raw):+ return None+ tags = _mapping(metadata)+ tagged = _mapping(tags.get("plugin_component"))+ if tagged:+ kind, name = tagged.get("type"), tagged.get("name")+ if kind == "hook" and isinstance(name, str) and name.startswith("monitor:"):+ kind, name = "monitor", name.removeprefix("monitor:")+ candidates = [row for row in self._components if row[:2] == (kind, name)]+ elif isinstance(tags.get("mcp_server"), str):+ candidates = [row for row in self._components if row[:2] == ("mcp", tags["mcp_server"])]+ same_file = [row for row in candidates if row[2] == raw]+ candidates = same_file or candidates+ elif isinstance(tags.get("plugin_component_ref"), str):+ ref = _path(tags["plugin_component_ref"])+ for prefix in ("${CLAUDE_PLUGIN_ROOT}", "${CURSOR_PLUGIN_ROOT}", "${PLUGIN_ROOT}"):+ if ref.startswith(prefix + "/"):+ ref = ref[len(prefix) + 1 :]+ break+ candidates = (+ []+ if _absolute(ref) or ".." in PurePosixPath(ref).parts+ else [row for row in self._components if row[1] == ref or row[2] == ref]+ )+ else:+ if label:+ skills = [+ row+ for row in self._components+ if row[0] == "skill" and (row[1] == label or row[2] == label or row[2].endswith("/" + label))+ ]+ if len(skills) != 1:+ return None+ if not (raw == skills[0][2] or raw.startswith(skills[0][2] + "/")):+ raw = posixpath.join(skills[0][2], raw)+ candidates = [+ row+ for row in self._components+ if row[0] != "mcp" and row[2] and (raw == row[2] or raw.startswith(row[2].rstrip("/") + "/"))+ ]+ if candidates:+ longest = max(len(row[2]) for row in candidates)+ candidates = [row for row in candidates if len(row[2]) == longest]+ if len(candidates) != 1:+ return None+ kind, name, path = candidates[0]+ return {"type": _clean_text(kind)[0], "name": _clean_text(name)[0], "path": _clean_text(path)[0]}+++class _Projection:+ def __init__(self) -> None:+ self.input_remaining = MAX_INPUT_ROWS+ self.text_remaining = MAX_TEXT_BYTES+ self.rows_remaining = MAX_TOTAL_ROWS+ self.truncated = False+ self.sections: list[dict[str, Any]] = []++ def records(self, value: object) -> list[dict]:+ if not isinstance(value, (list, tuple)):+ return []+ length = min(len(value), self.input_remaining, MAX_LIST_ROWS)+ self.input_remaining -= length+ self.truncated |= length < len(value)+ return [row for row in islice(value, length) if isinstance(row, dict)]++ def text(self, value: object, *, charge: bool = True) -> str | int | float | None:+ if value is None:+ return None+ if type(value) in (int, float):+ return _number(value)+ if isinstance(value, bool):+ return "yes" if value else "no"+ if not isinstance(value, str):+ return "unavailable"+ value, truncated = _clean_text(value)+ self.truncated |= truncated+ encoded = value.encode("utf-8")+ if charge:+ if len(encoded) > self.text_remaining:+ value = encoded[: self.text_remaining].decode("utf-8", "ignore")+ self.truncated = True+ self.text_remaining -= len(value.encode("utf-8"))+ return value++ def label(self, value: object) -> str:+ normalized = self.text(value, charge=False)+ return "unknown" if normalized is None else str(normalized)++ def names(self, value: object, limit: int = 12) -> str:+ if isinstance(value, str):+ return self.label(value)+ if not isinstance(value, (list, tuple)):+ return "unknown"+ chosen = [self.label(item) for item in islice(value, limit)]+ if len(value) > limit:+ chosen.append(f"(+{len(value) - limit} more)")+ self.truncated = True+ return ", ".join(chosen) or "none"++ def section(+ self,+ key: str,+ title: str,+ columns: tuple[str, ...],+ rows: list[list],+ *,+ total: int | None = None,+ notes: list | None = None,+ ) -> None:+ # Defer shared budgets until every section is known so risk evidence+ # can take priority without changing the report's display order.+ self.sections.append(+ {+ "key": key,+ "title": title,+ "columns": list(columns),+ "rows": rows[:MAX_SECTION_ROWS],+ "total": len(rows) if total is None else total,+ "notes": (notes or [])[:8],+ }+ )++ def _bounded_cells(self, values: list) -> list | None:+ cells = [self.text(value, charge=False) for value in values]+ size = sum(len(cell.encode("utf-8")) for cell in cells if isinstance(cell, str))+ if size > self.text_remaining:+ self.truncated = True+ return None+ self.text_remaining -= size+ return cells++ def finish(self) -> list[dict[str, Any]]:+ risk_keys = {"hooks", "privileges", "audits", "endpoints"}+ for section in sorted(self.sections, key=lambda item: item["key"] not in risk_keys):+ rows = []+ for row in section["rows"][: self.rows_remaining]:+ cells = self._bounded_cells(row)+ if cells is None:+ break+ rows.append(cells)+ self.rows_remaining -= len(rows)+ section["rows"] = rows+ section["omitted"] = max(0, section["total"] - len(rows))+ self.truncated |= bool(section["omitted"])+ section["notes"] = [+ cells[0] for note in section["notes"] if (cells := self._bounded_cells([note])) is not None+ ]+ return self.sections++ def url(self, value: object) -> str:+ if not isinstance(value, str) or len(value) > MAX_FIELD_INPUT:+ return "unavailable"+ try:+ parsed = urlsplit(value)+ host = parsed.hostname+ port = parsed.port+ if parsed.scheme not in {"http", "https"} or not host or any(char.isspace() for char in parsed.netloc):+ return "unavailable"+ authority = f"[{host}]" if ":" in host else host+ if port is not None:+ authority += f":{port}"+ return urlunsplit((parsed.scheme, authority, parsed.path, "", ""))+ except ValueError:+ return "unavailable"+++def _count_rows(value: object) -> int | None:+ return len(value) if isinstance(value, (list, tuple)) else None+++def project_plugin_static(results: list[ValidationResult]) -> StaticPluginReport | None:+ """Reduce private static results once, keeping unknown and incomplete facts explicit."""+ projection = _Projection()+ merged: dict[str, Any] = {}+ sources = []+ endpoints = []+ conflicts = []+ blocked = False+ incomplete = len(results) > MAX_RESULTS+ known_keys = (+ "root",+ "name",+ "manifest_sha256",+ "manifest_declarations",+ "component_inventory",+ "dependency_resolution",+ "dependency_status_counts",+ "dependency_manifests",+ "mcp",+ "context_cost",+ "hook_risk",+ "privileges",+ "cve_summary",+ "validator_parity",+ "catalog_skill_similarity",+ "inter_plugin_similarity",+ )+ for result in islice(results, MAX_RESULTS):+ metadata = _mapping(result.metadata)+ block = _mapping(metadata.get("plugin"))+ severities: dict[str, Counter] = {}+ incomplete |= result.is_incomplete or len(result.findings) > MAX_INPUT_ROWS+ blocked |= bool(result.gate_failures) or (+ not result.passed and not result.is_incomplete and metadata.get("optional_skip") is not True+ )+ for finding in islice(result.findings, MAX_INPUT_ROWS):+ severity = getattr(finding.severity, "value", finding.severity)+ blocked |= severity in {"critical", "high"}+ ecosystem = _mapping(finding.metadata).get("ecosystem")+ if isinstance(ecosystem, str) and finding.check_name != "dependency-not-audited":+ severities.setdefault(ecosystem, Counter())[severity] += 1+ sources.append((result, block, severities))+ for key in known_keys:+ if key not in block:+ continue+ value = block[key]+ if (+ key in {"root", "manifest_sha256", "name"}+ and isinstance(value, str)+ and key in merged+ and value != merged[key]+ ):+ conflicts.append(f"Conflicting source identity: {key}")+ if key not in merged or merged[key] is None:+ merged[key] = value+ endpoint = _mapping(metadata.get("endpoint_resolution"))+ if endpoint:+ endpoints.append(endpoint)+ projection.truncated |= len(results) > MAX_RESULTS+ if not merged:+ return None+ declarations = merged.get("manifest_declarations")+ if isinstance(declarations, dict):+ declarations = declarations.get("manifests")+ manifests = projection.records(declarations)+ selected = next((row for row in manifests if row.get("selected") is True), {})+ name = projection.text(merged.get("name") or selected.get("name") or "plugin")+ root = _path(merged.get("root"))+ # Gates and typed findings remain authoritative; metadata never declares a pass.+ status = "failed" if blocked else "incomplete" if incomplete or conflicts else "passed"+ if conflicts:+ projection.section(+ "identity_conflicts", "Source identity conflicts", ("Issue",), [[message] for message in conflicts]+ )+ if manifests:+ projection.section(+ "manifests",+ "Plugin manifests",+ ("Selected", "Manifest", "Type", "Name", "Version"),+ [+ [+ row.get("selected"),+ row.get("manifest_filename"),+ row.get("manifest_type"),+ row.get("name"),+ row.get("version"),+ ]+ for row in manifests+ ],+ total=_count_rows(declarations),+ )+ hook_block = _mapping(merged.get("hook_risk"))+ privilege_block = _mapping(merged.get("privileges"))+ hook_records = projection.records(hook_block.get("hooks"))+ privilege_records = projection.records(privilege_block.get("components"))+ checked_types = {"monitor" if row.get("event") == "Monitor" else "hook" for row in hook_records}+ checked_types.update(row.get("type") for row in privilege_records if isinstance(row.get("type"), str))+ checked_types.update({"lsp", "monitor", "settings", "output_style"})++ def support(row: dict) -> object:+ if row.get("type") in checked_types:+ return "statically checked"+ return "inventoried only" if row.get("support") == "unsupported" else row.get("support")++ inventory = _mapping(merged.get("component_inventory"))+ components = projection.records(inventory.get("components"))+ index = tuple(+ (+ str(projection.text(row.get("type"), charge=False)),+ str(projection.text(row.get("name"), charge=False)),+ _path(row.get("path")),+ )+ for row in components+ )+ if inventory:+ complete = inventory.get("complete")+ complete_label = "yes" if complete is True else "no" if complete is False else "unknown"+ ordered = sorted(components, key=lambda row: (not bool(row.get("problem")), not bool(row.get("findings"))))+ projection.section(+ "inventory",+ "Component inventory and support",+ ("Type", "Name", "Origin", "Path", "Static support", "State", "Findings", "Declared by", "Client view"),+ [+ [+ row.get("type"),+ row.get("name"),+ row.get("origin"),+ row.get("path"),+ support(row),+ row.get("problem")+ or row.get("dependency")+ or ("overridden" if row.get("effective") is False else "declared"),+ _count(row.get("findings")),+ row.get("declared_by"),+ row.get("loaded_by"),+ ]+ for row in ordered+ ],+ total=_count_rows(inventory.get("components")),+ notes=[+ "Static evidence does not demonstrate runtime loading or behavior.",+ f"Discovery complete: {complete_label}",+ ],+ )+ counts = _mapping(inventory.get("counts"))+ if counts:+ projection.section(+ "inventory_counts",+ "Component counts",+ ("Type", "Count"),+ [[kind, _count(amount)] for kind, amount in islice(counts.items(), 32)],+ )+ counts = _mapping(merged.get("dependency_status_counts"))+ if counts:+ projection.section(+ "dependency_counts",+ "Dependency state counts",+ ("State", "Count"),+ [[state, _count(amount)] for state, amount in islice(counts.items(), 16)],+ )+ resolution = _mapping(merged.get("dependency_resolution"))+ dependencies = []+ dependency_total = 0+ for kind, raw in islice(resolution.items(), 16):+ dependency_total += _count_rows(raw) or 0+ dependencies.extend((kind, row) for row in projection.records(raw))+ if resolution:+ dependencies.sort(+ key=lambda item: {"missing": 0, "unresolved": 1, "external": 2}.get(str(item[1].get("state")), 3)+ )+ projection.section(+ "dependencies",+ "Dependency resolution",+ ("Kind", "Reference", "State", "Path", "Declared by", "Reason"),+ [+ [kind, row.get("ref"), row.get("state"), row.get("path"), row.get("declared_by"), row.get("reason")]+ for kind, row in dependencies+ ],+ total=dependency_total,+ )+ mcp = _mapping(merged.get("mcp"))+ if mcp:+ projection.section(+ "mcp",+ "MCP declarations and pinning",+ ("Server", "Source", "Kind", "Transport", "Pinned", "Detail"),+ [+ [+ row.get("name"),+ row.get("source"),+ row.get("kind"),+ row.get("transport"),+ row.get("pinned"),+ row.get("pin_detail"),+ ]+ for row in projection.records(mcp.get("servers"))+ ],+ total=_count_rows(mcp.get("servers")),+ )+ pinning = _mapping(mcp.get("pinning"))+ if pinning:+ projection.section(+ "mcp_pinning",+ "MCP pinning totals",+ ("Total", "Pinned", "Unpinned", "Not applicable", "Ratio"),+ [+ [_count(pinning.get(key)) for key in ("total", "pinned", "unpinned", "not_applicable")]+ + [_number(pinning.get("ratio"))]+ ],+ )+ cost = _mapping(merged.get("context_cost"))+ if cost:+ summary = f"Static estimate {projection.label(cost.get('estimator'))}; {projection.label(cost.get('harness'))}/{projection.label(cost.get('load_mode'))}; always-on {projection.label(_number(cost.get('always_on_tokens')))}; on-demand {projection.label(_number(cost.get('on_demand_tokens')))}; lower bound {cost.get('lower_bound') is True}"+ projection.section(+ "context",+ "Estimated context cost",+ ("Type", "Name", "Always-on tokens", "On-demand tokens", "Basis", "Not counted"),+ [+ [+ row.get("type"),+ row.get("name"),+ _number(row.get("always_on_tokens")),+ _number(row.get("on_demand_tokens")),+ row.get("basis"),+ row.get("not_counted"),+ ]+ for row in projection.records(cost.get("by_component"))+ ],+ total=_count_rows(cost.get("by_component")),+ notes=[summary, projection.names(cost.get("not_counted")), projection.names(cost.get("notes"))],+ )+ projection.section(+ "context_harnesses",+ "Context estimates by harness",+ ("Harness", "Load mode", "Always-on tokens", "On-demand tokens", "Lower bound"),+ [+ [+ row.get("harness"),+ row.get("load_mode"),+ _number(row.get("always_on_tokens")),+ _number(row.get("on_demand_tokens")),+ row.get("lower_bound"),+ ]+ for row in projection.records(cost.get("by_harness"))+ ],+ total=_count_rows(cost.get("by_harness")),+ )+ hooks = hook_block+ if hooks:+ rows = hook_records+ rows.sort(key=lambda row: not bool(row.get("risk_flags")))+ projection.section(+ "hooks",+ "Hook static risk",+ ("Event", "Hook", "Matcher", "Handler", "Target", "Flags"),+ [+ [+ row.get("event"),+ row.get("id"),+ row.get("matcher") or "all",+ row.get("handler_type"),+ row.get("target"),+ projection.names(row.get("risk_flags")),+ ]+ for row in rows+ ],+ total=_count_rows(hooks.get("hooks")),+ )+ privileges = privilege_block+ if privileges:+ rows = privilege_records+ rows.sort(key=lambda row: not bool(row.get("flags")))+ projection.section(+ "privileges",+ "Static component privileges",+ ("Type", "Name", "Path", "Grants", "Model", "Permission mode", "Flags"),+ [+ [+ row.get("type"),+ row.get("name"),+ row.get("path"),+ "inherits all tools"+ if row.get("inherits_all_tools") is True+ else projection.names(row.get("allowed_tools") or row.get("tools")),+ row.get("model"),+ row.get("permission_mode"),+ projection.names(row.get("flags")),+ ]+ for row in rows+ ],+ total=_count_rows(privileges.get("components")),+ )+ audit_rows = []+ parity_rows = []+ for _result, block, severities in sources:+ for ecosystem, raw in islice(_mapping(_mapping(block.get("cve_summary")).get("ecosystems")).items(), 16):+ row = _mapping(raw)+ observed = severities.get(ecosystem, Counter())+ declared = row.get("vulnerabilities")+ if isinstance(declared, dict):+ observed = {key: _count(value) for key, value in islice(declared.items(), 5)}+ total = sum(value for value in observed.values() if value is not None)+ else:+ total = _count(declared)+ partial = row.get("findings_truncated") is True or row.get("status") in {"incomplete", "unverified"}+ if row.get("status") in {"disabled", "not_found", "unverified"} or (partial and total == 0):+ total = None+ severity = (+ ", ".join(f"{key}: {value}" for key, value in sorted(observed.items()) if value is not None)+ or "unknown"+ )+ audit_rows.append(+ [+ ecosystem,+ row.get("status", "unknown"),+ _count(row.get("sources", row.get("declarations"))),+ _count(row.get("audited")),+ total,+ ("partial; " if partial else "") + severity,+ projection.names(row.get("scanners")),+ projection.names(row.get("errors")),+ ]+ )+ parity = _mapping(block.get("validator_parity"))+ if parity:+ parity_rows.append(+ [+ parity.get("status"),+ parity.get("claude_verdict"),+ parity.get("strict_verdict"),+ parity.get("skillevaluator_verdict"),+ parity.get("agree"),+ projection.names(parity.get("errors")),+ projection.names(parity.get("warnings")),+ parity.get("reason"),+ ]+ )+ if audit_rows:+ projection.section(+ "audits",+ "Dependency vulnerability audits",+ (+ "Ecosystem",+ "Status",+ "Sources",+ "Audited",+ "Vulnerabilities",+ "Known severity counts",+ "Scanners",+ "Notes",+ ),+ audit_rows,+ )+ if parity_rows:+ projection.section(+ "parity",+ "Official validator parity",+ (+ "Status",+ "CLI verdict",+ "Strict verdict",+ "Evaluator verdict",+ "Agreement",+ "Errors",+ "Warnings",+ "Reason",+ ),+ parity_rows,+ )+ if endpoints:+ rows = [row for block in endpoints for row in projection.records(block.get("endpoints"))]+ total = sum(_count_rows(block.get("endpoints")) or 0 for block in endpoints)+ projection.section(+ "endpoints",+ "Endpoint checks",+ ("Kind", "Name", "URL", "Status", "HEAD", "Redirect", "Redirect classification", "Reason"),+ [+ [+ row.get("kind"),+ row.get("name"),+ projection.url(row.get("url")),+ row.get("status"),+ _mapping(row.get("head")).get("status") or _mapping(row.get("head")).get("skipped") or "unknown",+ projection.url(_mapping(row.get("redirect")).get("url")),+ _mapping(row.get("redirect")).get("classification"),+ row.get("reason"),+ ]+ for row in rows+ ],+ total=total,+ )+ for source_key, key, title in (+ ("catalog_skill_similarity", "catalog_skills", "Skills versus local catalog"),+ ("inter_plugin_similarity", "catalog_plugins", "Plugin versus local catalog"),+ ):+ comparison = _mapping(merged.get(source_key))+ if comparison:+ projection.section(+ key,+ title,+ ("Subject", "Match", "Similarity", "Member overlap", "Verdict"),+ [+ [+ row.get("skill") or row.get("name"),+ row.get("match"),+ row.get("similarity"),+ row.get("member_overlap"),+ row.get("verdict"),+ ]+ for row in projection.records(comparison.get("matches"))+ ],+ total=_count_rows(comparison.get("matches")),+ notes=[+ f"Advisory; status {projection.text(comparison.get('status'), charge=False)}; catalog entries {projection.label(_count(comparison.get('catalog_entries')))}",+ comparison.get("reason"),+ ],+ )+ sections = projection.finish()+ payload = {+ "schema_version": 1,+ "name": name,+ "status": status,+ "truncated": projection.truncated,+ "sections": sections,+ }+ serialized = json.dumps(payload, ensure_ascii=False, allow_nan=False, separators=(",", ":"))+ # A final byte ceiling also covers structural overhead, not only text cells.+ while len(serialized.encode("utf-8")) > MAX_OUTPUT_BYTES:+ section = next((item for item in reversed(projection.sections) if item["rows"]), None)+ if section is None:+ raise ValueError("Static report structure exceeds output allowance")+ section["rows"].pop()+ section["omitted"] += 1+ payload["truncated"] = True+ serialized = json.dumps(payload, ensure_ascii=False, allow_nan=False, separators=(",", ":"))+ return StaticPluginReport(serialized, root, index)diff --git a/src/skillevaluator/reporting/plugin_static_render.py b/src/skillevaluator/reporting/plugin_static_render.pynew file mode 100644index 00000000..4818a76f--- /dev/null+++ b/src/skillevaluator/reporting/plugin_static_render.py@@ -0,0 +1,138 @@+# SPDX-License-Identifier: Apache-2.0+"""Explicit private adapters; ordinary reporter entrypoints keep their behavior."""++from __future__ import annotations++import io+import json+from typing import TYPE_CHECKING++from skillevaluator.reporting.plugin_static import StaticPluginReport, project_plugin_static++if TYPE_CHECKING:+ from collections.abc import Iterable++ from skillevaluator.models.result import ValidationResult+ from skillevaluator.reporting.base import ReporterBase+++def _display(value: object) -> str:+ return "unknown" if value is None else str(value)+++def _markdown(payload: dict, *, publication: bool = False) -> str:+ if publication:+ from skillevaluator.reporting.benchmark import _md_cell as escape+ else:+ from skillevaluator.reporting.markdown import _markdown_table_cell as escape++ def cell(value: object) -> str:+ encoded = escape(_display(value))+ if publication:+ return encoded+ for character in "[]!*_\\":+ encoded = encoded.replace(character, f"&#{ord(character)};")+ return encoded++ lines = [+ "## Plugin static evidence",+ "",+ f"**{cell(payload['name'])}** — {payload['status']}",+ "",+ "Static evidence does not demonstrate runtime loading or behavior.",+ "",+ ]+ if payload["truncated"]:+ lines.extend(["Some static evidence was omitted by report bounds; see section counts.", ""])+ for section in payload["sections"]:+ lines.extend([f"### {section['title']}", ""])+ lines.extend(f"{cell(note)}\n" for note in section["notes"] if note is not None)+ if section["rows"]:+ lines.extend(+ [+ "| " + " | ".join(section["columns"]) + " |",+ "| " + " | ".join("---" for _ in section["columns"]) + " |",+ ]+ )+ lines.extend("| " + " | ".join(cell(value) for value in row) + " |" for row in section["rows"])+ if section["omitted"]:+ lines.append(f"{section['omitted']} more rows omitted ({section['total']} total).")+ lines.append("")+ return "\n".join(lines)+++def _cli(payload: dict) -> str:+ from rich.console import Console+ from rich.text import Text++ output = io.StringIO()+ console = Console(file=output, color_system=None, emoji=False)+ lines = [+ "Plugin static evidence",+ f"{payload['name']} — {payload['status']}",+ "Static evidence does not demonstrate runtime loading or behavior.",+ ]+ if payload["truncated"]:+ lines.append("Static evidence was omitted by report bounds; see section counts.")+ for section in payload["sections"]:+ lines.extend(["", section["title"], " | ".join(section["columns"])])+ lines.extend(_display(note) for note in section["notes"] if note is not None)+ lines.extend(" | ".join(_display(cell) for cell in row) for row in section["rows"])+ if section["omitted"]:+ lines.append(f"{section['omitted']} more rows omitted ({section['total']} total).")+ for line in lines:+ console.print(Text(line), soft_wrap=True)+ return output.getvalue()+++def _augment(+ reporter: ReporterBase,+ document: str,+ results: list[ValidationResult],+ projection: StaticPluginReport,+ payload: dict,+) -> str:+ if reporter.name == "json":+ data = json.loads(document)+ data["plugin_static"] = payload+ return json.dumps(data, indent=reporter.indent, ensure_ascii=True, allow_nan=False)+ if reporter.name == "sarif":+ data = json.loads(document)+ run = data["runs"][0]+ run.setdefault("properties", {})["pluginStatic"] = payload+ findings = [finding for result in results for finding in result.findings]+ for row, finding in zip(run["results"], findings, strict=True):+ component = projection.component_for(finding.file_path, finding.metadata)+ if component is not None:+ row.setdefault("properties", {})["pluginComponent"] = component+ return json.dumps(data, indent=reporter.indent, ensure_ascii=True, allow_nan=False)+ if reporter.name == "html":+ fragment = reporter._env.get_template("plugin_static_sections.html.j2").render(plugin_static=payload)+ if "</main>" not in document:+ raise ValueError("The HTML reporter did not provide its main content boundary")+ return document.replace("</main>", fragment + "\n</main>", 1)+ if reporter.name in {"markdown", "benchmark"}:+ return document.rstrip() + "\n\n" + _markdown(payload, publication=reporter.name == "benchmark").rstrip() + "\n"+ if reporter.name == "cli":+ return document.rstrip() + "\n\n" + _cli(payload)+ raise ValueError(f"Unsupported private static report format: {reporter.name}")+++def render_plugin_static_reports(+ results: list[ValidationResult], *, reporters: Iterable[ReporterBase]+) -> dict[str, str]:+ """Render a private static bundle through existing reporters with one reduction.++ This function is deliberately absent from CLI dispatch and the public+ reporting package exports. No reporter instance or validation result is+ mutated, and a bundle without plugin evidence uses existing output exactly.+ """+ projection = project_plugin_static(results)+ payload = projection.payload() if projection is not None else None+ rendered = {}+ for reporter in reporters:+ document = reporter.render_all(results)+ rendered[reporter.name] = (+ _augment(reporter, document, results, projection, payload) if projection is not None else document+ )+ return rendereddiff --git a/src/skillevaluator/reporting/templates/plugin_static_sections.html.j2 b/src/skillevaluator/reporting/templates/plugin_static_sections.html.j2new file mode 100644index 00000000..bba30fec--- /dev/null+++ b/src/skillevaluator/reporting/templates/plugin_static_sections.html.j2@@ -0,0 +1,18 @@+<section class="table-section" id="plugin-static-evidence" aria-label="Plugin static evidence">+ <h2>Plugin static evidence: {{ plugin_static.name }}</h2>+ <p>Status: {{ plugin_static.status }}. Static evidence does not demonstrate runtime loading or behavior.</p>+ {% if plugin_static.truncated %}<p>Some static evidence was omitted by report bounds; see section counts.</p>{% endif %}+ {% for section in plugin_static.sections %}+ <section aria-label="{{ section.title }}">+ <h3>{{ section.title }}</h3>+ {% for note in section.notes %}{% if note is not none %}<p>{{ note }}</p>{% endif %}{% endfor %}+ {% if section.rows %}+ <div class="table-scroll" tabindex="0" role="region" aria-label="{{ section.title }}"><table class="contributor-table plugin-static-table">+ <thead><tr>{% for column in section.columns %}<th scope="col">{{ column }}</th>{% endfor %}</tr></thead>+ <tbody>{% for row in section.rows %}<tr>{% for cell in row %}<td>{{ 'unknown' if cell is none else cell }}</td>{% endfor %}</tr>{% endfor %}</tbody>+ </table></div>+ {% endif %}+ {% if section.omitted %}<p>{{ section.omitted }} more rows omitted ({{ section.total }} total).</p>{% endif %}+ </section>+ {% endfor %}+</section>diff --git a/src/skillevaluator/tier3/eval_core/plugin_signals.py b/src/skillevaluator/tier3/eval_core/plugin_signals.pynew file mode 100644index 00000000..3c4074f5--- /dev/null+++ b/src/skillevaluator/tier3/eval_core/plugin_signals.py@@ -0,0 +1,5022 @@+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+# SPDX-License-Identifier: Apache-2.0++"""Report-only plugin component signals computed on the host from ATIF trajectories.++The private runtime observer calls :func:`compute_plugin_signals` once per+logical trial and :func:`summarize_plugin_signals` per arm. Nothing+here changes a trial score, pass/fail result, or verdict: every value is+advisory and is persisted next to the existing reward/summary data.++Component activation conventions+--------------------------------+Each normalized tool call is mapped to zero or more component activations:++* ``skill`` -- the ``Skill`` tool (name from ``skill``/``name``/``command``),+ or a read of a *declared* member's ``<member>/SKILL.md`` through a file-read+ tool (``Read``/``read_file``/``view``...), an MCP filesystem read, or a shell+ reader command (``cat``/``sed``/``head``...). Shell reads are recognized only+ when the manifest is an operand of a known reader verb, so ``rm``/``ls``/+ ``grep``/``printf``/``cp``/``sed -i`` of the same path, or ``<`` input to a+ non-reader, never count. The scored skill-read check is stricter (see+ ``_FILE_READER_VERBS``), so a read credited here may not count toward a+ score. Each read of a shell list has its own outcome: a+ read after a failed ``&&`` part never ran, and a read whose own error line+ names it failed. Skill, subagent and command names are this plugin's+ declared names with or without its ``<plugin>:`` namespace; another plugin's+ namespaced name stays foreign and never matches.+* ``mcp`` -- ``mcp__<server>__<tool>``; for *declared* runnable servers the+ ``<server>__<tool>``/``<server>.<tool>``/``<server>/<tool>`` spellings used by+ other agents, Hermes ``mcp_<server>_<tool>`` and OpenCode ``<server>_<tool>``+ are also recognized and canonicalized to ``mcp__<server>__<tool>``. Claude+ Code's plugin servers (``mcp__plugin_<plugin>_<server>__<tool>``) and+ harness-sanitized names map back to the declared server; Codex bare tool+ names map through the server its log recorded (``mcp_call_servers``).+* ``subagent`` -- ``Task``/``Agent`` (name from ``subagent_type``..., else ``type``).+* ``command`` -- ``SlashCommand`` (name from the first token of ``command``),+ or the ``Skill`` tool naming a declared command (``<plugin>:<command>``),+ which is how Claude Code runs plugin commands.+* ``lsp`` -- Claude Code's ``LSP`` tool, credited to the one plugin LSP+ server staged in this arm whose ``extensionToLanguage`` covers the extension+ of the call's ``filePath`` (``lsp_servers``). An extension that two staged+ servers claim names neither.+* ``rule_read`` is reserved by the output contract but never emitted: plugin+ rules are inlined into the generated wrapper skill, so there is no separate+ rule file for an agent to read.++Native Codex ``exec`` wrappers are unwrapped with+:mod:`~skillevaluator.tier3.eval_core.codex_tool_call_normalizer` first. An+inner call only inherits the outer observation when the normalizer proves it+owns that observation; otherwise its outcome is unknown.++``succeeded`` is tri-state. ``True``: a correlated result came back without a+structured error flag or a failure/unavailable marker in the head of any of its+content blocks or in its tail. ``False``: the correlated result is flagged or+carries such a marker ("no such file" only counts for non-shell calls, whose+output is not a mix of several commands). A file read or skill load returns+file text, so only its first line counts (past a harness's shell status lines),+plus, for a shell read, any shell error line that names a ``SKILL.md``. ``None``:+no result could be attributed to this call (no id match, or an ambiguous+multi-call step), or the correlated body is empty. A sibling call's result never+stands in for this call's outcome.++Case references+---------------+Dataset fields refer to components or tools with *refs*. A ref is a string or a+list of alternative strings. Matching is case-insensitive and supports+``fnmatch`` globs:++* ``Skill:<name>``, ``Agent:<name>`` (aliases ``Subagent:``/``Task:``),+ ``Command:<name>`` (alias ``SlashCommand:``), ``MCP:<server>`` (or+ ``MCP:<server>/<tool>``) and ``LSP:<server>`` match a component activation+ of that type (a bare ``LSP`` ref is the tool itself);+* any other ref matches the canonical tool label (``mcp__server__tool``,+ ``Skill:<name>``...), the raw tool function name (``Bash``, ``Read``...) or+ its harness-neutral alias (``Bash`` is also Codex ``exec_command``; ``Write``+ and ``Edit`` are also any shell call that writes a file), an MCP tool's bare+ name, or a component name -- never a component through the tool that+ carried it, and never the generated wrapper skill (the ``<plugin>-plugin-eval``+ package, or the plugin's own name when no member skill or command has it).++Check 15 (``routing``) scores the refs about skills, subagents, and commands;+check 22 (``tool_selection``) scores the refs about MCP, LSP, and plain tools.++Handoff heuristics (conservative)+---------------------------------+Calls are *attributed* to a ref when they match it directly, or when they run+inside the window opened by a matching skill/command activation (see+``_handoff_calls``: only a successful switch to one declared skill or command+opens or closes a window, and the consumer's own activation does not close the+producer's). A subagent's own calls (Claude Code sidechain steps) keep their+own window and also belong to a ref naming that subagent: the parent call whose+result names the sidechain's ``agentId``, else the parent's latest subagent call.++* ``value``: the literal must appear in the result of a producer-attributed+ call, then in the arguments of a consumer-attributed call in a strictly later+ step (a result is only visible to the model after its step). The value must+ not appear in the task prompt (the user/system messages before the agent's+ first step), since then the consumer could have taken it from the prompt+ rather than from the producer. Later user steps (Claude Code's skill-load+ step, a subagent's prompt) are written by the harness or the agent.+* ``artifact``: a producer-attributed call must write the path (write/edit+ tools, the ``Add File``/``Update File``/``Move to`` headers of an+ ``apply_patch`` body, shell redirects/``tee``/``cp``/``mv``/``touch``/``sed -i``,+ interpreter code given inline or as a here-document, or an MCP tool whose+ name or argument key says it writes; a patch that only deletes the path, or+ moves it away, does not write it),+ and a consumer-attributed call in a later step must read it (read tools,+ shell readers/interpreters/globs, or an MCP tool taking the path as input).+ Paths resolve against the call's working directory and the task's; when a+ directory is unknown a path ending with ``/<artifact>`` matches.++Values are compared as parsed values (argument string leaves, and JSON text+inside results), as whole tokens.+* When both are given, both must hold.++Trajectory content is untrusted: step, call, text and regex-subject sizes are+bounded, dataset patterns run under a deadline, and every persisted string is+redacted (from a bounded window) and truncated. Argument values are never+persisted: failure details describe them by type and size, and a component+name taken from tool arguments is kept only when it is identifier-shaped or a+declared member.+"""++from __future__ import annotations++import fnmatch+import json+import math+import posixpath+import re+import shlex+import time+from collections.abc import Callable, Iterable, Mapping, Sequence+from dataclasses import dataclass, field, replace+from functools import cached_property+from typing import Any, NamedTuple++import regex++from skillevaluator.tier3.eval_core.checks import _APPLY_PATCH_COMMAND_RE, _APPLY_PATCH_HEADER_RE, _FILE_READ_VERBS+from skillevaluator.tier3.eval_core.codex_tool_call_normalizer import (+ MAPPED_OUTER_EXEC_OBSERVATION,+ normalize_tool_call,+)+from skillevaluator.tier3.plugin_native import OPENCODE_BUILTIN_AGENTS+from skillevaluator.tier3.runtime_arms import ARM_LOOSE_PARTS, ARM_SUM_OF_PARTS, ARM_WITH, ARM_WITHOUT+from skillevaluator.utils.redaction import redact_printable_text++COMPONENT_SKILL = "skill"+COMPONENT_MCP = "mcp"+COMPONENT_SUBAGENT = "subagent"+COMPONENT_COMMAND = "command"+COMPONENT_LSP = "lsp"+COMPONENT_RULE_READ = "rule_read"++STATUS_SCORED = "scored"+STATUS_NOT_APPLICABLE = "not_applicable"++# Compatibility aliases for the neutral runtime arm names.+ARM_WITH_SKILL = ARM_WITH+ARM_WITHOUT_SKILL = ARM_WITHOUT+#: The arms that stage the plugin's MCP servers, subagents, and commands: the full plugin and its loose parts.+PLUGIN_PART_ARMS = frozenset({ARM_WITH, ARM_LOOSE_PARTS})++# -- dataset field bounds ----------------------------------------------------+MAX_TOOL_PATTERNS = 64+MAX_REF_CHARS = 256+MAX_REF_ALTERNATIVES = 16+MAX_ARGUMENT_RULES = 32+MAX_ARGUMENT_NAMES = 64+MAX_ARGUMENT_NAME_CHARS = 128+MAX_SCHEMA_DEPTH = 6+MAX_SCHEMA_NODES = 256+MAX_ENUM_ITEMS = 64+MAX_REGEX_CHARS = 512+MAX_LITERAL_JSON_CHARS = 4096+MAX_CONTAINS_CHARS = 1024+MAX_ORDER_EDGES = 64+MAX_HANDOFFS = 32+MAX_HANDOFF_VALUE_CHARS = 1024+MAX_ARTIFACT_CHARS = 512+MAX_CONFLICT_PROBES = 32+MAX_PROBE_ID_CHARS = 128+MAX_DESCRIPTION_CHARS = 1024+MAX_CONTEXT_CASES = 4096+MAX_DECLARED_COMPONENTS = 256+MAX_COMPONENT_NAME_CHARS = 256++# -- trajectory bounds ---------------------------------------------------------+_MAX_STEPS = 10_000+_MAX_CALLS = 4_000+_MAX_STEP_RESULTS = 256+_MAX_CONTENT_BLOCKS = 256+# String values (never key names) read from a JSON argument, result or ``contains`` object.+_MAX_STRING_LEAVES = 1_024+# List elements a ``contains`` argument check looks at.+_MAX_CONTAINS_ITEMS = 1_024+_MAX_OBSERVATION_CHARS = 64 * 1024+_MAX_ARGS_TEXT_CHARS = 64 * 1024+_MAX_ARGS_JSON_CHARS = 256 * 1024+_MAX_SHELL_TEXT_CHARS = 32 * 1024+_MAX_SHELL_TOKENS = 4_096+_MAX_SHELL_DEPTH = 3+_MAX_PATTERN_SUBJECT_CHARS = 4_096+# Dataset patterns run under a deadline: per check, and shared by all checks of one trial.+_PATTERN_TIMEOUT_SECONDS = 0.1+_PATTERN_BUDGET_SECONDS = 5.0+_FAILURE_SCAN_CHARS = 2_048++# -- output bounds -------------------------------------------------------------+_MAX_ACTIVATIONS = 512+_MAX_FAILURES = 50+_MAX_CALLED = 128+_MAX_SERVER_TOOLS = 64+_MAX_LABEL_CHARS = 256+_MAX_DETAIL_CHARS = 240+# How much of an unsupported dataset key a problem message quotes.+_MAX_KEY_PREVIEW_CHARS = 64+_MAX_PREVIEW_CHARS = 80+_MAX_SCHEMA_ERRORS_PER_CALL = 5++_SKILL_TOOLS = frozenset({"skill"})+_SUBAGENT_TOOLS = frozenset({"task", "agent"})+_COMMAND_TOOLS = frozenset({"slashcommand", "slash_command"})+# Claude Code's LSP tool: Claude Code sends it to the server whose ``extensionToLanguage`` maps the file's extension.+_LSP_TOOLS = frozenset({"lsp"})+_LSP_PATH_KEYS = ("filePath", "file_path")+# An ``extensionToLanguage`` key: a dot and one suffix (``.lab``; a file's extension is never ``.d.ts``).+_LSP_EXTENSION_RE = re.compile(r"\.[^./\\\s]{1,32}")+_MAX_LSP_EXTENSIONS = 64+_SHELL_TOOLS = frozenset(+ {+ "bash",+ "shell",+ "exec_command",+ "exec",+ "local_shell",+ "local_shell_call",+ "run",+ "run_command",+ "run_shell_command",+ "execute",+ "execute_command",+ "run_code",+ "command",+ "terminal",+ }+)+_READ_TOOLS = frozenset(+ {"read", "read_file", "read_text_file", "view", "open", "open_file", "cat", "get_file_contents", "view_file"}+)+# Edit tools that take an apply_patch body: Codex ``apply_patch``/``applypatch``+# and the Hermes ``patch`` tool (whose ``replace`` mode names a ``path`` instead).+_PATCH_TOOLS = frozenset({"apply_patch", "applypatch", "patch"})+# Where harnesses put that body: Codex ``input``, OpenCode ``patchText``, Hermes+# ``patch``, and ``raw`` or ``value`` for a tool input that was not a JSON object.+_PATCH_BODY_KEYS = ("input", "patch", "patchText", "content", "raw", "value")+# Text editor tools: they write, except for their ``view`` command, which reads.+_STR_REPLACE_EDITORS = frozenset({"str_replace_editor", "str_replace_based_edit_tool"})+_WRITE_TOOLS = (+ _PATCH_TOOLS+ | _STR_REPLACE_EDITORS+ | frozenset(+ {+ "write",+ "write_file",+ "create_file",+ "edit",+ "edit_file",+ "multiedit",+ "multi_edit",+ "notebookedit",+ "notebook_edit",+ "save_file",+ }+ )+)+_PATH_ARG_KEYS = (+ "file_path",+ "path",+ "filename",+ "target_file",+ "filePath",+ "absolute_path",+ "notebook_path",+ "raw",+)+_SHELL_COMMAND_KEYS = ("cmd", "command", "code", "script", "input")+_TOOL_NAME_SEPARATORS = ("__", ".", ":", "/")+# MCP tool-name spellings (see ``_McpNames.identity``). Hermes names MCP tools+# ``mcp_<server>_<tool>`` with every character outside ``[A-Za-z0-9_]`` as ``_``;+# Claude Code names plugin servers ``plugin_<plugin>_<server>``.+_HERMES_NAME_RE = re.compile(r"[^a-z0-9_]")+# Claude Code and OpenCode write a server name with every character outside ``[A-Za-z0-9_-]`` as ``_``.+_SANITIZED_NAME_RE = re.compile(r"[^a-z0-9_-]")+_MCP_PREFIX = "mcp__"+_CLAUDE_PLUGIN_SERVER_PREFIX = "plugin_"+# Harnesses that never name an MCP tool ``<server>_<tool>`` (only OpenCode does).+_NO_BARE_MCP_PREFIX_AGENTS = frozenset({"claude-code", "codex", "hermes"})+# Built-in tool names that a declared server called ``web``/``read``/... must not claim as ``<server>_<tool>``.+_BUILTIN_TOOL_NAMES = (+ _SKILL_TOOLS+ | _SUBAGENT_TOOLS+ | _COMMAND_TOOLS+ | _SHELL_TOOLS+ | _READ_TOOLS+ | _WRITE_TOOLS+ | frozenset(+ {+ "web_search",+ "web_fetch",+ "web_extract",+ "update_plan",+ "write_stdin",+ "view_image",+ "execute_code",+ "search_files",+ "todo_write",+ "todo_read",+ }+ )+)+_SHELLS = frozenset({"bash", "sh", "zsh", "dash", "ksh"})+_SHELL_PREFIX_WORDS = frozenset({"sudo", "env", "time", "nohup", "command", "exec", "builtin", "nice"})+# Shell commands that read a SKILL.md: the scored check's reader verbs plus a few+# more viewers. This report-only detector is deliberately more lenient than the+# scored checks._cmd_reads_skill_md: it also looks past prefix words (``sudo``,+# ``env``...) and into ``bash -c`` payloads. Unlike the scored check it does not+# expand shell variables, so ``D=skills/x; cat $D/SKILL.md`` counts only there.+_FILE_READER_VERBS = frozenset(_FILE_READ_VERBS) | frozenset({"batcat", "view", "xxd", "od", "tac"})+_ARTIFACT_CONSUMER_VERBS = _FILE_READER_VERBS | frozenset(+ {"jq", "yq", "python", "python3", "node", "grep", "rg", "wc", "sort", "uniq", "cut", "diff", "cmp", "base64"}+)+_SHELL_SEPARATORS = frozenset({"&&", "||", ";", "|", "&", "(", ")", ";;", "|&", ";&"})+_OUTPUT_REDIRECTS = frozenset({">", ">>", ">|", "&>", "&>>", "1>", "2>"})+_ENV_ASSIGNMENT_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*=")+_REPEATED_SLASHES_RE = re.compile(r"/{2,}")+# ``<<EOF``/``<<-'EOF'``/``<<"EOF"`` (not ``<<<`` here-strings); the groups hold the delimiter.+_HEREDOC_RE = re.compile(r"(?<!<)<<(?!<)-?[ \t]*(?:'([^'\n]+)'|\"([^\"\n]+)\"|\\?([A-Za-z_][A-Za-z0-9_]*))")+_WRITE_VERB_RE = re.compile(+ r"(?:^|_|-)(?:write|create|save|export|put|upload|dump|store|generate|render|append)(?:$|_|-)"+)+_OUTPUT_ARG_KEYS = frozenset(+ {"output", "out", "output_path", "out_path", "output_file", "dest", "destination", "target", "save_path"}+)+_READ_VERB_RE = re.compile(r"(?:^|_|-)(?:read|open|get|load|view|fetch|cat)(?:$|_|-)")+# A ``declared`` mapping may name the plugin itself under this key. Claude Code+# namespaces plugin skills, agents, and commands as ``<plugin>:<name>``; a name+# under another plugin's namespace is not this plugin's component.+DECLARED_PLUGIN = "plugin"+# ... and may list the component types the plugin declares but this arm does not stage.+DECLARED_UNSTAGED = "unstaged"+# ... and, as ``<type>:<name>``, the subagents and commands an arm leaves out of its declared names+# entirely, so an untyped ref that names one is still read as a ref to that component.+DECLARED_UNSTAGED_NAMES = "unstaged_names"+_WRAPPER_PACKAGE_SUFFIX = "-plugin-eval"+# A shell tool's error line that says one file operand could not be read.+_READ_FAILURE_PHRASES = (+ "no such file",+ "permission denied",+ "is a directory",+ "not a directory",+ "cannot open",+ "can't open",+ "cannot read",+ "can't read",+ "cannot access",+)+# Harness-neutral names for plain tools, so one unprefixed ref means the same+# thing on every harness: Codex writes files with ``apply_patch``, runs shell+# commands with ``exec_command``, has one hosted web tool (``web_search_call``,+# both search and page open; see ``_WEB_ACTION_ALIASES``), and plans with ``update_plan``.+_TOOL_ALIASES: dict[str, tuple[str, ...]] = {+ "bash": ("exec_command", "shell", "local_shell"),+ "exec_command": ("bash", "shell"),+ "shell": ("bash", "exec_command"),+ "local_shell": ("bash", "exec_command", "shell"),+ "local_shell_call": ("bash", "exec_command", "shell"),+ "shell_command": ("bash", "exec_command", "shell"),+ "apply_patch": ("write", "edit", "multiedit"),+ "write": ("apply_patch",),+ "edit": ("apply_patch",),+ "multiedit": ("apply_patch",),+ "web_search_call": ("websearch", "webfetch", "web_search", "web_fetch"),+ "web_search": ("websearch", "web_search_call"),+ "websearch": ("web_search_call", "web_search"),+ "webfetch": ("web_search_call", "web_fetch"),+ "update_plan": ("todowrite",),+ "todowrite": ("update_plan",),+}+# Codex's hosted web call records its action: a ``search`` is a web search, and+# opening a page or finding text in it is a fetch. Only a call that records no+# known action answers to both names.+_WEB_ACTION_ALIASES: dict[str, tuple[str, ...]] = {+ "search": ("websearch", "web_search"),+ "open_page": ("webfetch", "web_fetch"),+ "find_in_page": ("webfetch", "web_fetch"),+}+# A shell call that writes a file also answers to the file-writing tool names. Codex+# writes most files through ``exec_command`` (``cat > f <<'EOF'``, ``python3 - <<'PY'``),+# so an unprefixed ``Write`` or ``Edit`` ref means "wrote a file" on every harness.+_SHELL_WRITE_ALIASES = ("write", "edit", "multiedit")+# A failed result that says the called tool, skill, command, or agent does not exist.+_MISSING_TOOL_MARKERS = (+ "no such tool",+ "tool not found",+ "unknown tool",+ "unknown skill",+ "skill not found",+ "no such skill",+ "unknown command",+ "no such command",+ "not found. available",+ "unsupported call",+ "unknown agent type",+)+# Check 15 routes work to components; check 22 selects tools (MCP and plain).+_ROUTING_KINDS = frozenset({COMPONENT_SKILL, COMPONENT_SUBAGENT, COMPONENT_COMMAND})+# Harnesses that cannot stage plugin subagents or commands, so refs to those can never match there.+_NO_PLUGIN_AGENT_HARNESSES = frozenset({"codex"})+# Each harness's own agents (casefolded). A bare one of these names reaches the built-in, never+# the plugin's agent: Claude Code names a plugin agent ``<plugin>:<name>``, and OpenCode stages a+# plugin agent named like its own as ``<plugin>-<name>``.+_BUILTIN_AGENTS = {+ "claude-code": frozenset(+ {"claude", "explore", "general-purpose", "output-style-setup", "plan", "statusline-setup"}+ ),+ "opencode": OPENCODE_BUILTIN_AGENTS,+}++_REF_PREFIXES = {+ "skill": COMPONENT_SKILL,+ "agent": COMPONENT_SUBAGENT,+ "subagent": COMPONENT_SUBAGENT,+ "task": COMPONENT_SUBAGENT,+ "command": COMPONENT_COMMAND,+ "slashcommand": COMPONENT_COMMAND,+ "mcp": COMPONENT_MCP,+ "lsp": COMPONENT_LSP,+}+# Rules are inlined into the wrapper skill and hooks run outside the agent's tool calls.+_UNSUPPORTED_REF_PREFIXES = frozenset({"rule", "rules", "hook", "hooks"})+_REF_PREFIX_SPELLING = {+ "skill": "Skill",+ "agent": "Agent",+ "subagent": "Subagent",+ "task": "Task",+ "command": "Command",+ "slashcommand": "SlashCommand",+ "mcp": "MCP",+ "lsp": "LSP",+}+_IDENTITY_PREFIX = {+ COMPONENT_SKILL: "Skill",+ COMPONENT_SUBAGENT: "Agent",+ COMPONENT_COMMAND: "Command",+ COMPONENT_LSP: "LSP",+}+# Skill/subagent/command names come from agent-written tool arguments. Only an+# identifier-shaped name (or a declared member) is persisted; anything else, such+# as a URL with credentials or key material, is reported as ``<non-name>``.+_PERSISTABLE_NAME_RE = re.compile(r"[A-Za-z0-9_.:/-]{1,128}")+_NON_NAME = "<non-name>"++# Failure text is read only when a result has no structured outcome, and only+# on its lead lines (the first line of each block, past harness status lines+# and MCP content wrappers, and the last line): a good answer that talks about+# errors ("Permission denied errors happen when ...", "AT-412: Fix race",+# "400-500 rps", "Error: ENOENT means ...") is not a failed call. Only harness,+# transport and HTTP status text counts; a server's own "Error: ..." does not.+# Harness and transport phrases: a failure anywhere on a lead line.+_UNAVAILABLE_MARKERS = (+ "no such tool",+ "tool not found",+ "unknown tool",+ "unknown skill",+ "skill not found",+ "no such command",+ "command not found",+ "unknown command",+ "failed to connect",+ "connection refused",+ "mcp server not",+ "no mcp server",+ "no lsp server",+)+_FAILURE_PHRASES = (+ *_UNAVAILABLE_MARKERS,+ "<tool_use_error>",+ "mcp error",+ "internal server error",+ "failed to decrypt",+ # Harbor appends this line to a Claude Code result the harness flagged.+ "[error] tool reported failure",+)+# Words that say a call failed only when they open a lead line (after an+# optional ``name:`` chain such as ``cat: <path>:``) and end a clause there:+# "Permission denied: notes_path" fails, "Permission denied errors happen" does not.+_OPENING_FAILURE_RE = re.compile(+ r"(?:permission denied|access denied|unauthorized|forbidden|invalid[_ ]token|token (?:is )?expired)"+ r"(?=\s*(?:$|[:.,;!)\-]))",+ re.IGNORECASE,+)+# A missing file means a failed call only for MCP and file-read tools. A shell+# call's output mixes several commands, so ``cat SKILL.md missing.md`` read the+# manifest even though its output says "No such file or directory".+_FILE_MISSING_RE = re.compile(+ r"(?:no such file or directory|file does not exist)(?=\s*(?:$|[:.,;!)\-]))", re.IGNORECASE+)+# A line about a tool, server, skill, or command that ends with "is not available":+# "Tool 'x' is not available." fails; "Streaming is not available." and+# "Streaming is not available on the free plan" do not.+_UNAVAILABLE_TAIL_RE = re.compile(+ r"\b(?:tool|server|skill|command|mcp)\b.*\b(?:not available|is unavailable)\s*[.!]?\s*$", re.IGNORECASE+)+# A harness's own error prefix opening a lead line: "tool call error: ...". A+# bare "Error: ...", "TypeError: ..." or "ERROR: ..." is not one: a server's+# own error text with no flag is an answer the agent saw ("Error: ENOENT means+# the path does not exist", a log search hit), and Claude Code and Codex both+# report a failed call with a structured flag that wins over any text.+_HARNESS_ERROR_PREFIX_RE = re.compile(r"tool\s+call\s+error\s*:", re.IGNORECASE)+# An HTTP status 400-599 counts only as a status line: it opens the line+# (optionally after "HTTP", "HTTP/1.1", "HTTP Error", "Error", "status" or+# "status code"), a standard reason phrase follows, and the clause ends there.+# "404 Not Found", "HTTP Error 404: Not Found", "401 - Unauthorized: missing+# token" and "403: Forbidden - token lacks scope" fail; "403: Forbidden is+# returned when ...", "404: page moved" and "429 - Too Many Requests means ..."+# do not.+_HTTP_REASONS = (+ "bad request|unauthorized|payment required|forbidden|not found|method not allowed|not acceptable"+ "|request timeout|conflict|gone|precondition failed|payload too large|unprocessable entity|too many requests"+ "|internal server error|not implemented|bad gateway|service unavailable|gateway timeout"+)+_HTTP_STATUS_OPENING_RE = re.compile(+ r"(?:(?:https?(?:/\d(?:\.\d)?)?\s+)?(?:error\s+)?|status(?:\s+code)?\s*[:=]?\s*)"+ rf"[45]\d{{2}}(?:\s*[:\-]\s*|\s+)(?:{_HTTP_REASONS})(?=\s*(?:$|[:.,;!)\-]))",+ re.IGNORECASE,+)+# An HTTP response status line ("HTTP/1.1 404 - x", "HTTP/2 500") is a status+# line whatever its reason text says, so it fails without a standard reason+# phrase when the code ends the line or a ":" or "-" follows it. Prose such+# as "HTTP/1.1 404 responses are cached" does not.+_HTTP_VERSION_STATUS_RE = re.compile(r"https?/\d(?:\.\d)?\s+[45]\d{2}(?=\s*(?:$|[:\-]))", re.IGNORECASE)+# A client's failed-request line opening a lead line: "Request failed with+# status code 404" (also after a ``name:`` prefix such as ``AxiosError:``).+# "Fixed: retries after status code 503" is not one.+_REQUEST_FAILED_RE = re.compile(+ r"(?:request\s+)?failed\s+with\s+status(?:\s+code)?\s*[:=]?\s*[45]\d{2}\b(?=\s*(?:$|[:.,;!)\-]))",+ re.IGNORECASE,+)+# A leading ``name:`` segment (a program, a path, a server, an issue key) that+# can precede the failure words: ``cat: <dir>/SKILL.md: Permission denied``.+_LEAD_SEGMENT_RE = re.compile(r"[^\s:]{1,256}:\s+")+# What can open a lead line before an HTTP status line: a request line ("GET+# /repos/x: 404 - Not Found") or a clause that starts with an error word+# ("Error calling tool: 404: not found"). Each gap is bounded, so a long line is+# scanned in linear time. The status line itself still needs a reason phrase.+_STATUS_LINE_PREFIX_RES = (+ re.compile(r"(?:get|head|post|put|patch|delete|options)[ \t]+\S{1,200}?:?[ \t]+", re.IGNORECASE),+ re.compile(r"(?:error|failed)\b[^:\n]{0,80}:[ \t]+", re.IGNORECASE),+)+# Status lines a harness puts in front of shell output (Codex ``exec_command``).+# They are not file text, so a shell read looks past them for its first line.+_SHELL_STATUS_LINE_RE = re.compile(+ r"chunk id:.*|wall time:.*|process exited with code -?\d+|exit code:\s*-?\d+"+ r"|original token count:.*|total output lines:.*|output:",+ re.IGNORECASE,+)+# A shell tool's error line about a file (``cat: <dir>/SKILL.md: Permission denied``).+_SHELL_ERROR_LINE_RE = re.compile(r"[\w./+-]+:\s")+# The MCP content wrapper a harness keeps around a tool's text: Codex prints+# ``[{"type":"text","text":"..."}]`` and Harbor's Claude converter may keep one+# ``{"type": "text", "text": "..."}`` block as a string.+_MCP_TEXT_WRAPPER_RE = re.compile(r'\[?\s*\{\s*"type"\s*:\s*"text"\s*,\s*"text"\s*:\s*"')+# Wrappers up to this size are parsed whole; a larger one yields its first text only.+_MAX_WRAPPER_JSON_CHARS = 16 * 1024+# Structured per-call flags. Harbor keeps Claude Code's own flag as+# ``extra.tool_result_is_error`` and ``extra.tool_result_metadata.is_error``+# (the raw block under ``raw_tool_result``); other producers use a top-level+# ``is_error``/``isError``. A string or number spelling counts the same way.+_ERROR_FLAG_KEYS = ("is_error", "isError")+_TRUE_FLAGS = frozenset({"true", "1", "yes"})+_FALSE_FLAGS = frozenset({"false", "0", "no"})+#: Result ``extra`` key the collector fills from a harness's own per-call status+#: (Codex ``codex.txt``: ``completed`` or ``failed``).+HARNESS_STATUS_KEY = "harness_status"+_HARNESS_FAILED = frozenset({"failed", "error", "errored"})+_HARNESS_SUCCEEDED = frozenset({"completed", "succeeded", "success"})+_SCHEMA_TYPES = frozenset({"string", "number", "integer", "boolean", "object", "array", "null"})+_SCHEMA_KEYWORDS = frozenset({"type", "properties", "required", "enum", "pattern", "minimum", "maximum"})+_SCHEMA_ANNOTATIONS = frozenset({"description", "title", "$comment", "examples", "default"})+_ARGUMENT_RULE_KEYS = frozenset({"tool", "required", "schema", "equals", "contains", "pattern", "description"})+_HANDOFF_KEYS = frozenset({"producer", "consumer", "value", "artifact", "description"})+_PROBE_KEYS = frozenset({"id", "must_use", "must_not_use", "description"})+_CONTROL_CHARS_RE = re.compile(r"[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]")+_MISSING = object()+++# =============================================================================+# Dataset case fields+# =============================================================================+++class _FieldErrors:+ def __init__(self) -> None:+ self.messages: list[str] = []++ def add(self, where: str, problem: str) -> None:+ self.messages.append(f"{where}: {problem}")+++def _check_description(raw: Mapping[str, Any], where: str, errors: _FieldErrors) -> bool:+ description = raw.get("description")+ if description is not None and (not isinstance(description, str) or len(description) > MAX_DESCRIPTION_CHARS):+ errors.add(f"{where}.description", f"must be a string of at most {MAX_DESCRIPTION_CHARS} characters")+ return False+ return True+++def _check_list(value: Any, where: str, errors: _FieldErrors, *, items: str, limit: int, unit: str) -> list[Any] | None:+ """``value`` if it is a list of at most ``limit`` entries; otherwise the problem is recorded."""+ if not isinstance(value, list):+ errors.add(where, f"must be a list of {items}")+ return None+ if len(value) > limit:+ errors.add(where, f"must list at most {limit} {unit}")+ return None+ return value+++def _check_object(raw: Any, where: str, errors: _FieldErrors, allowed_keys: frozenset[str]) -> bool:+ """Whether ``raw`` is an object with no keys outside ``allowed_keys``; otherwise the problem is recorded."""+ if not isinstance(raw, dict):+ errors.add(where, "must be an object")+ return False+ unknown = sorted(str(key)[:_MAX_KEY_PREVIEW_CHARS] for key in raw if key not in allowed_keys)+ if unknown:+ errors.add(where, f"unsupported keys: {', '.join(unknown)}")+ return False+ return True+++def _is_name(value: Any) -> bool:+ """An argument or property name: a non-empty string of at most ``MAX_ARGUMENT_NAME_CHARS``."""+ return isinstance(value, str) and 0 < len(value) <= MAX_ARGUMENT_NAME_CHARS+++def _is_name_list(value: Any) -> bool:+ return isinstance(value, list) and len(value) <= MAX_ARGUMENT_NAMES and all(_is_name(item) for item in value)+++def _check_ref_string(value: Any, where: str, errors: _FieldErrors) -> str | None:+ if not isinstance(value, str) or not value.strip():+ errors.add(where, "must be a non-empty string")+ return None+ text = value.strip()+ if len(text) > MAX_REF_CHARS:+ errors.add(where, f"must be at most {MAX_REF_CHARS} characters")+ return None+ if _CONTROL_CHARS_RE.search(text):+ errors.add(where, "must not contain control characters")+ return None+ prefix, sep, rest = text.partition(":")+ if sep and prefix.strip().casefold() in _REF_PREFIXES and not rest.strip():+ errors.add(where, f"'{prefix}:' must be followed by a name")+ return None+ if sep and prefix.strip().casefold() in _UNSUPPORTED_REF_PREFIXES:+ errors.add(+ where,+ f"'{_safe_text(prefix, 32)}:' refs are not supported: rules and hooks are not tool calls, so the ref "+ "could never match (the hook census reports hook runs)",+ )+ return None+ near = _near_ref_prefix(prefix.strip().casefold()) if sep else None+ if near is not None:+ errors.add(+ where,+ f"unknown ref prefix '{_safe_text(prefix, 32)}:' (did you mean '{_REF_PREFIX_SPELLING[near]}:'?); "+ "write a namespaced name as an explicit type, such as 'Skill:<plugin>:<name>'",+ )+ return None+ return text+++def _near_ref_prefix(word: str) -> str | None:+ """A known ref type one typo away from ``word`` (``Skil`` -> ``skill``), so the ref could never match."""+ if len(word) < 3 or word in _REF_PREFIXES:+ return None+ return next((prefix for prefix in _REF_PREFIX_SPELLING if _within_one_edit(word, prefix)), None)+++def _within_one_edit(left: str, right: str) -> bool:+ """One insertion, deletion, substitution, or swap of neighbors apart."""+ if left == right or abs(len(left) - len(right)) > 1:+ return False+ if len(left) == len(right):+ diff = [index for index, (a, b) in enumerate(zip(left, right, strict=True)) if a != b]+ if len(diff) == 1:+ return True+ return (+ len(diff) == 2+ and diff[1] == diff[0] + 1+ and left[diff[0]] == right[diff[1]]+ and left[diff[1]] == right[diff[0]]+ )+ short, long = (left, right) if len(left) < len(right) else (right, left)+ return any(long[:index] + long[index + 1 :] == short for index in range(len(long)))+++def _check_ref_alternatives(value: Any, where: str, errors: _FieldErrors) -> list[str] | None:+ if isinstance(value, str):+ ref = _check_ref_string(value, where, errors)+ return [ref] if ref is not None else None+ if not isinstance(value, list) or not value:+ errors.add(where, "must be a ref string or a non-empty list of alternative refs")+ return None+ if len(value) > MAX_REF_ALTERNATIVES:+ errors.add(where, f"must list at most {MAX_REF_ALTERNATIVES} alternatives")+ return None+ refs: list[str] = []+ for index, item in enumerate(value):+ ref = _check_ref_string(item, f"{where}[{index}]", errors)+ if ref is None:+ return None+ refs.append(ref)+ return refs+++def _check_tool_patterns(value: Any, where: str, errors: _FieldErrors) -> list[str] | None:+ items = _check_list(value, where, errors, items="tool refs", limit=MAX_TOOL_PATTERNS, unit="tool refs")+ if items is None:+ return None+ patterns: list[str] = []+ for index, item in enumerate(items):+ ref = _check_ref_string(item, f"{where}[{index}]", errors)+ if ref is None:+ return None+ if ref not in patterns:+ patterns.append(ref)+ return patterns+++_REGEX_COMPILE_TOKENS = re.compile(+ r"\\[pPN]\{[^}]*\}|\\.|\[\^?\]?(?:\\.|[^\]\\])*\]"+ r"|(?P<flags>\(\?[a-zA-Z0-9-]+\))"+ r"|\{(?P<count>[0-9]*)(?:,(?P<upper>[0-9]*))?\}"+ r"|(?P<fuzzy>\{(?:[ides](?=[<>=,}:])|[0-9]+[ides<=>]))|.",+ re.DOTALL,+)+_REGEX_EXTENDED_REPEAT_FLAGS = re.compile(r"\(\?[a-zA-Z0-9-]*(?:x|V1)")+_MAX_REGEX_REPEAT = 100+++def _regex_compile_problem(value: str) -> str | None:+ """Bound counted expansion before entering the engine's untimed compiler.++ Escapes and simple character classes are single tokens. Braces combined+ with comments, POSIX classes, verbose mode or V1 sets are conservatively+ unsupported because those constructs change the token grammar. Fuzzy+ repetition constraints are unsupported too.+ """+ if "{" in value and _REGEX_EXTENDED_REPEAT_FLAGS.search(value):+ return "braces with verbose mode or V1 sets are unsupported"+ if "{" in value and ("(?#" in value or "[:" in value):+ return "braces with regex comments or POSIX classes are unsupported"+ groups = [False]+ atom_counted = False+ for token in _REGEX_COMPILE_TOKENS.finditer(value):+ count = token.group("count")+ if count is not None:+ for bound in (count, token.group("upper")):+ digits = (bound or "0").lstrip("0") or "0"+ if len(digits) > 3 or int(digits) > _MAX_REGEX_REPEAT:+ return f"counted repeats must be at most {_MAX_REGEX_REPEAT}"+ if atom_counted:+ return "nested counted repeats are unsupported"+ groups[-1] = atom_counted = True+ elif token.group("fuzzy"):+ return "fuzzy repetition constraints are unsupported"+ elif token.group("flags"):+ continue+ elif token[0] == "(":+ groups.append(False)+ atom_counted = False+ elif token[0] == ")" and len(groups) > 1:+ atom_counted = groups.pop()+ groups[-1] |= atom_counted+ elif token[0] not in {"*", "+", "?"}:+ atom_counted = False+ return None+++def _check_regex(value: Any, where: str, errors: _FieldErrors) -> str | None:+ if not isinstance(value, str) or not value:+ errors.add(where, "must be a non-empty regular expression string")+ return None+ if len(value) > MAX_REGEX_CHARS:+ errors.add(where, f"must be at most {MAX_REGEX_CHARS} characters")+ return None+ if problem := _regex_compile_problem(value):+ errors.add(where, f"is not a valid regular expression ({problem})")+ return None+ # Checked with the engine that runs it (see _regex_search): ``re`` rejects+ # ``\p{L}``, which ``regex`` runs, and accepts ``[[:alpha:]``, which it does not.+ try:+ regex.compile(value)+ except (regex.error, ValueError, RecursionError, OverflowError) as exc:+ errors.add(where, f"is not a valid regular expression ({exc})")+ return None+ except Exception as exc:+ # Some malformed patterns raise other errors: KeyError for ``(?V0)(?V1)``,+ # RuntimeError for a fuzzy count the engine cannot compile.+ errors.add(where, f"is not a valid regular expression (the regex engine raised {type(exc).__name__})")+ return None+ return value+++def _is_number(value: Any) -> bool:+ # ``float(10**400)`` overflows, so an int is finite by type; floats must be finite.+ return (isinstance(value, int) and not isinstance(value, bool)) or (+ isinstance(value, float) and math.isfinite(value)+ )+++def _bounded_json_text(value: Any, limit: int) -> str | None:+ try:+ text = json.dumps(value, ensure_ascii=False, sort_keys=True, allow_nan=False)+ except (TypeError, ValueError, RecursionError):+ return None+ return text if len(text) <= limit else None+++def _check_schema(schema: Any, where: str, errors: _FieldErrors, *, depth: int, budget: list[int]) -> bool:+ budget[0] += 1+ if budget[0] > MAX_SCHEMA_NODES:+ errors.add(where, f"schema must have at most {MAX_SCHEMA_NODES} nodes")+ return False+ if depth > MAX_SCHEMA_DEPTH:+ errors.add(where, f"schema nesting must be at most {MAX_SCHEMA_DEPTH} levels")+ return False+ if not isinstance(schema, dict):+ errors.add(where, "schema must be an object")+ return False+ ok = True+ for key in schema:+ if key not in _SCHEMA_KEYWORDS and key not in _SCHEMA_ANNOTATIONS:+ errors.add(+ where,+ f"unsupported JSON-Schema keyword {str(key)[:_MAX_KEY_PREVIEW_CHARS]!r}; "+ f"supported: {', '.join(sorted(_SCHEMA_KEYWORDS))}",+ )+ ok = False+ if "type" in schema:+ types = schema["type"]+ type_list = [types] if isinstance(types, str) else types+ if (+ not isinstance(type_list, list)+ or not type_list+ or not all(isinstance(item, str) and item in _SCHEMA_TYPES for item in type_list)+ ):+ errors.add(f"{where}.type", f"must be one of {', '.join(sorted(_SCHEMA_TYPES))} or a list of them")+ ok = False+ if "required" in schema and not _is_name_list(schema["required"]):+ errors.add(f"{where}.required", "must be a list of property names")+ ok = False+ if "enum" in schema:+ enum = schema["enum"]+ if not isinstance(enum, list) or not enum or len(enum) > MAX_ENUM_ITEMS:+ errors.add(f"{where}.enum", f"must be a non-empty list of at most {MAX_ENUM_ITEMS} values")+ ok = False+ elif _bounded_json_text(enum, MAX_LITERAL_JSON_CHARS) is None:+ errors.add(f"{where}.enum", f"must be JSON values of at most {MAX_LITERAL_JSON_CHARS} characters")+ ok = False+ if "pattern" in schema and _check_regex(schema["pattern"], f"{where}.pattern", errors) is None:+ ok = False+ for bound in ("minimum", "maximum"):+ if bound in schema and not _is_number(schema[bound]):+ errors.add(f"{where}.{bound}", "must be a finite number")+ ok = False+ if (+ _is_number(schema.get("minimum"))+ and _is_number(schema.get("maximum"))+ and schema["minimum"] > schema["maximum"]+ ):+ errors.add(where, "minimum must not exceed maximum")+ ok = False+ if "properties" in schema:+ properties = schema["properties"]+ if not isinstance(properties, dict) or len(properties) > MAX_ARGUMENT_NAMES:+ errors.add(f"{where}.properties", f"must be an object with at most {MAX_ARGUMENT_NAMES} properties")+ ok = False+ else:+ for name, sub in properties.items():+ if not _is_name(name):+ errors.add(f"{where}.properties", "property names must be non-empty short strings")+ ok = False+ continue+ if not _check_schema(sub, f"{where}.properties.{name}", errors, depth=depth + 1, budget=budget):+ ok = False+ return ok+++def _check_arg_mapping(value: Any, where: str, errors: _FieldErrors, *, kind: str) -> dict[str, Any] | None:+ if not isinstance(value, dict) or not value:+ errors.add(where, "must be a non-empty object mapping argument names to values")+ return None+ if len(value) > MAX_ARGUMENT_NAMES:+ errors.add(where, f"must name at most {MAX_ARGUMENT_NAMES} arguments")+ return None+ parsed: dict[str, Any] = {}+ for name, item in value.items():+ if not _is_name(name):+ errors.add(where, "argument names must be non-empty strings")+ return None+ item_where = f"{where}.{name}"+ if kind == "equals":+ if _bounded_json_text(item, MAX_LITERAL_JSON_CHARS) is None:+ errors.add(item_where, f"must be a JSON value of at most {MAX_LITERAL_JSON_CHARS} characters")+ return None+ elif kind == "contains":+ if not isinstance(item, str) or not item or len(item) > MAX_CONTAINS_CHARS:+ errors.add(item_where, f"must be a non-empty string of at most {MAX_CONTAINS_CHARS} characters")+ return None+ elif _check_regex(item, item_where, errors) is None:+ return None+ parsed[name] = item+ return parsed+++def _parse_argument_rule(raw: Any, rule_where: str, errors: _FieldErrors) -> dict[str, Any] | None:+ if not _check_object(raw, rule_where, errors, _ARGUMENT_RULE_KEYS):+ return None+ tool = _check_ref_string(raw.get("tool"), f"{rule_where}.tool", errors)+ if tool is None or not _check_description(raw, rule_where, errors):+ return None+ rule: dict[str, Any] = {"tool": tool}+ if "required" in raw:+ required = raw["required"]+ if not _is_name_list(required):+ errors.add(f"{rule_where}.required", "must be a list of argument names")+ return None+ if required:+ rule["required"] = list(dict.fromkeys(required))+ if "schema" in raw:+ if not _check_schema(raw["schema"], f"{rule_where}.schema", errors, depth=1, budget=[0]):+ return None+ rule["schema"] = raw["schema"]+ for kind in ("equals", "contains", "pattern"):+ if kind in raw:+ parsed = _check_arg_mapping(raw[kind], f"{rule_where}.{kind}", errors, kind=kind)+ if parsed is None:+ return None+ rule[kind] = parsed+ if rule.keys() == {"tool"}:+ errors.add(rule_where, "must declare at least one of required, schema, equals, contains, or pattern")+ return None+ return rule+++def _parse_tool_arguments(value: Any, where: str, errors: _FieldErrors) -> list[dict[str, Any]] | None:+ """Every valid rule, or ``None`` when any rule is bad; each bad rule is reported, not only the first."""+ items = _check_list(value, where, errors, items="argument rules", limit=MAX_ARGUMENT_RULES, unit="rules")+ if items is None:+ return None+ rules: list[dict[str, Any]] = []+ valid = True+ for index, raw in enumerate(items):+ rule = _parse_argument_rule(raw, f"{where}[{index}]", errors)+ if rule is None:+ valid = False+ else:+ rules.append(rule)+ return rules if valid else None+++def sanitize_input_schema(schema: Any, *, depth: int = 1, budget: list[int] | None = None) -> dict[str, Any] | None:+ """An MCP server's tool ``inputSchema`` cut to the subset argument checks run, or ``None``.++ Keeps ``type``, ``properties``, ``required``, ``enum``, ``pattern``,+ ``minimum`` and ``maximum`` (the ``tool_arguments`` schema subset) and drops+ everything else (``$ref``, ``anyOf``, ``additionalProperties``, annotations),+ so the check never fails on a keyword it cannot evaluate. Values that would+ fail the dataset schema rules (an invalid regex, a non-finite bound, an+ oversize enum) are dropped too. Bounded by the dataset limits: nesting+ depth, node count, and 64 properties per object.+ """+ budget = budget if budget is not None else [0]+ budget[0] += 1+ if not isinstance(schema, Mapping) or depth > MAX_SCHEMA_DEPTH or budget[0] > MAX_SCHEMA_NODES:+ return None+ errors = _FieldErrors()+ out: dict[str, Any] = {}+ types = schema.get("type")+ type_list = [types] if isinstance(types, str) else types+ if (+ isinstance(type_list, list)+ and type_list+ and all(isinstance(item, str) and item in _SCHEMA_TYPES for item in type_list)+ ):+ out["type"] = types+ required = schema.get("required")+ if isinstance(required, list):+ names = [item for item in required if _is_name(item)]+ if names:+ out["required"] = list(dict.fromkeys(names))[:MAX_ARGUMENT_NAMES]+ enum = schema.get("enum")+ if isinstance(enum, list) and 0 < len(enum) <= MAX_ENUM_ITEMS and _bounded_json_text(enum, MAX_LITERAL_JSON_CHARS):+ out["enum"] = list(enum)+ pattern = schema.get("pattern")+ if isinstance(pattern, str) and _check_regex(pattern, "pattern", errors) is not None:+ out["pattern"] = pattern+ for bound in ("minimum", "maximum"):+ if _is_number(schema.get(bound)):+ out[bound] = schema[bound]+ if _is_number(out.get("minimum")) and _is_number(out.get("maximum")) and out["minimum"] > out["maximum"]:+ out.pop("minimum")+ out.pop("maximum")+ properties = schema.get("properties")+ if isinstance(properties, Mapping):+ kept: dict[str, Any] = {}+ for name, sub in list(properties.items())[:MAX_ARGUMENT_NAMES]:+ if _is_name(name):+ cleaned = sanitize_input_schema(sub, depth=depth + 1, budget=budget)+ if cleaned:+ kept[name] = cleaned+ if kept:+ out["properties"] = kept+ return out or None+++def _parse_expected_order(value: Any, where: str, errors: _FieldErrors) -> list[list[list[str]]] | None:+ items = _check_list(value, where, errors, items="[before, after] edges", limit=MAX_ORDER_EDGES, unit="edges")+ if items is None:+ return None+ edges: list[list[list[str]]] = []+ for index, edge in enumerate(items):+ edge_where = f"{where}[{index}]"+ if not isinstance(edge, list) or len(edge) != 2:+ errors.add(edge_where, "must be a two-item [before, after] list")+ return None+ before = _check_ref_alternatives(edge[0], f"{edge_where}[0]", errors)+ after = _check_ref_alternatives(edge[1], f"{edge_where}[1]", errors)+ if before is None or after is None:+ return None+ before_refs, after_refs = _refs(before), _refs(after)+ if all(_ref_covers(later, earlier) for later in after_refs for earlier in before_refs):+ # Whatever uses ``before`` also uses ``after``, so ``after`` is never strictly later.+ errors.add(edge_where, "can never pass: every 'after' ref also matches every 'before' ref")+ return None+ edges.append([before, after])+ return edges+++def _parse_handoffs(value: Any, where: str, errors: _FieldErrors) -> list[dict[str, Any]] | None:+ items = _check_list(value, where, errors, items="handoff objects", limit=MAX_HANDOFFS, unit="handoffs")+ if items is None:+ return None+ handoffs: list[dict[str, Any]] = []+ for index, raw in enumerate(items):+ item_where = f"{where}[{index}]"+ if not _check_object(raw, item_where, errors, _HANDOFF_KEYS):+ return None+ producer = _check_ref_alternatives(raw.get("producer"), f"{item_where}.producer", errors)+ consumer = _check_ref_alternatives(raw.get("consumer"), f"{item_where}.consumer", errors)+ if producer is None or consumer is None:+ return None+ shared = {ref.casefold() for ref in producer} & {ref.casefold() for ref in consumer}+ if shared:+ errors.add(+ item_where, "producer and consumer must name different components (a self-handoff always passes)"+ )+ return None+ literal = raw.get("value")+ artifact = raw.get("artifact")+ if literal is not None and (+ not isinstance(literal, str) or not literal.strip() or len(literal) > MAX_HANDOFF_VALUE_CHARS+ ):+ errors.add(+ f"{item_where}.value", f"must be a non-empty string of at most {MAX_HANDOFF_VALUE_CHARS} characters"+ )+ return None+ if artifact is not None and (+ not isinstance(artifact, str) or not artifact.strip() or len(artifact) > MAX_ARTIFACT_CHARS+ ):+ errors.add(f"{item_where}.artifact", f"must be a workspace path of at most {MAX_ARTIFACT_CHARS} characters")+ return None+ if artifact is not None and _CONTROL_CHARS_RE.search(artifact):+ errors.add(f"{item_where}.artifact", "must not contain control characters")+ return None+ if literal is None and artifact is None:+ errors.add(item_where, "must declare a value or an artifact as handoff evidence")+ return None+ if not _check_description(raw, item_where, errors):+ return None+ handoff: dict[str, Any] = {"producer": producer, "consumer": consumer}+ if literal is not None:+ handoff["value"] = literal+ if artifact is not None:+ handoff["artifact"] = artifact+ handoffs.append(handoff)+ return handoffs+++def _parse_conflict_probes(value: Any, where: str, errors: _FieldErrors) -> list[dict[str, Any]] | None:+ items = _check_list(value, where, errors, items="probe objects", limit=MAX_CONFLICT_PROBES, unit="probes")+ if items is None:+ return None+ probes: list[dict[str, Any]] = []+ seen_ids: set[str] = set()+ for index, raw in enumerate(items):+ item_where = f"{where}[{index}]"+ if not _check_object(raw, item_where, errors, _PROBE_KEYS):+ return None+ probe_id = raw.get("id")+ if not isinstance(probe_id, str) or not probe_id.strip() or len(probe_id) > MAX_PROBE_ID_CHARS:+ errors.add(f"{item_where}.id", f"must be a non-empty string of at most {MAX_PROBE_ID_CHARS} characters")+ return None+ # Ids are stored trimmed, so ``dup`` and `` dup`` are the same id.+ if probe_id.strip() in seen_ids:+ errors.add(f"{item_where}.id", "must be unique (ids are compared without surrounding spaces)")+ return None+ seen_ids.add(probe_id.strip())+ must_use = _check_ref_alternatives(raw.get("must_use"), f"{item_where}.must_use", errors)+ must_not_use = _check_ref_alternatives(raw.get("must_not_use"), f"{item_where}.must_not_use", errors)+ if must_use is None or must_not_use is None:+ return None+ forbidden = _refs(must_not_use)+ if all(any(_ref_covers(ref, wanted) for ref in forbidden) for wanted in _refs(must_use)):+ errors.add(item_where, "can never pass: every must_use ref is also matched by must_not_use")+ return None+ if not _check_description(raw, item_where, errors):+ return None+ description = raw.get("description")+ probes.append(+ {+ "id": probe_id.strip(),+ "must_use": must_use,+ "must_not_use": must_not_use,+ "description": description or "",+ }+ )+ return probes+++# Each advisory case field and its parser: ``parser(value, field_name, errors)``+# returns the normalized value, or ``None`` after recording the problem.+_FIELD_PARSERS: dict[str, Callable[[Any, str, _FieldErrors], list[Any] | None]] = {+ "expected_tools": _check_tool_patterns,+ "acceptable_tools": _check_tool_patterns,+ "decoy_tools": _check_tool_patterns,+ "tool_arguments": _parse_tool_arguments,+ "expected_order": _parse_expected_order,+ "handoffs": _parse_handoffs,+ "conflict_probes": _parse_conflict_probes,+}+PLUGIN_CASE_FIELDS = tuple(_FIELD_PARSERS)+++def validate_plugin_case_fields(entry: Any) -> list[str]:+ """Return problems with the optional advisory plugin-signal fields of one case.++ Absent fields are fine. ``null`` is treated as absent. Every present field is+ type-checked and size-bounded; see the ``MAX_*`` constants.+ """+ if not isinstance(entry, Mapping):+ return []+ errors = _FieldErrors()+ parsed: dict[str, Any] = {}+ for name, parse in _FIELD_PARSERS.items():+ value = entry.get(name)+ if value is None:+ continue+ before = len(errors.messages)+ result = parse(value, name, errors)+ if result is not None and len(errors.messages) == before:+ parsed[name] = result+ _check_selection_overlap(parsed, errors)+ return errors.messages+++def _ref_covers(general: _Ref, specific: _Ref) -> bool:+ """Whether every call ``specific`` matches is also matched by ``general`` (same type, glob over the text)."""+ if general.kind != specific.kind:+ return False+ if general.kind == COMPONENT_MCP and "/" not in general.pattern and "/" in specific.pattern:+ # ``MCP:<server>`` matches every tool of that server.+ return fnmatch.fnmatchcase(specific.pattern.split("/", 1)[0], general.pattern)+ return fnmatch.fnmatchcase(specific.pattern, general.pattern)+++def _check_selection_overlap(parsed: Mapping[str, Any], errors: _FieldErrors) -> None:+ """A decoy ref must not also name an expected or acceptable call: it would be scored both ways."""+ decoys = parsed.get("decoy_tools") or []+ for field_name in ("expected_tools", "acceptable_tools"):+ for index, raw in enumerate(decoys):+ decoy = _parse_ref(raw)+ for other in parsed.get(field_name) or []:+ wanted = _parse_ref(other)+ if _ref_covers(wanted, decoy) or _ref_covers(decoy, wanted):+ errors.add(+ f"decoy_tools[{index}]",+ f"overlaps {field_name} ref {_safe_text(other, 64)!r}: a call cannot be both wanted and a decoy",+ )+ break+++def plugin_case_spec(entry: Any) -> dict[str, Any]:+ """Return the valid, normalized plugin-signal fields of one case entry.++ Invalid fields are dropped (the dataset validators report them), so a+ malformed advisory field can never break result collection.+ """+ if not isinstance(entry, Mapping):+ return {}+ spec: dict[str, Any] = {}+ for name, parse in _FIELD_PARSERS.items():+ value = entry.get(name)+ if value is None:+ continue+ errors = _FieldErrors()+ parsed = parse(value, name, errors)+ if parsed is not None and not errors.messages:+ spec[name] = parsed+ return spec+++def is_cross_component_case(entry: Any, member_skills: Iterable[str]) -> bool:+ """Whether a case can show the plugin's parts working together.++ A cross-component case is marked ``cross_component: true`` and names two or+ more of the plugin's member skills in ``expected_skills`` (compared without+ case or surrounding spaces). The dataset's cross-component count, the+ Integration readiness gate and the Bundle lift split by case type all use+ this one rule, so a report never shows two different counts.+ """+ if not isinstance(entry, Mapping) or entry.get("cross_component") is not True:+ return False+ expected = entry.get("expected_skills")+ if not isinstance(expected, list):+ return False+ members = {str(name).strip().casefold() for name in member_skills}+ return len({str(name).strip().casefold() for name in expected} & members) >= 2+++# =============================================================================+# Collector context+# =============================================================================+++def _clean_names(values: Iterable[Any]) -> tuple[str, ...]:+ names: list[str] = []+ for value in values:+ if not isinstance(value, str):+ continue+ name = value.strip()+ if not name or len(name) > MAX_COMPONENT_NAME_CHARS or _CONTROL_CHARS_RE.search(name):+ continue+ if name not in names:+ names.append(name)+ if len(names) >= MAX_DECLARED_COMPONENTS:+ break+ return tuple(names)+++@dataclass(frozen=True)+class PluginSignalsContext:+ """The prepared context the private observer uses for one run.++ ``member_skills`` and ``mcp_servers`` are the plugin's declared, locally+ evaluated components (the member skills staged in the with-plugin arm and+ the runnable MCP servers wired into it). ``wrapper_skills`` names the+ generated wrapper skill so it is not scored as a tool selection. ``cases``+ maps case id to :func:`plugin_case_spec` output. The context is shared by+ every agent of a run, so what only one agent's with-plugin arm stages, or+ does not stage, is kept per agent (``agent_mcp_servers``,+ ``agent_unstaged``).+ """++ member_skills: tuple[str, ...] = ()+ mcp_servers: tuple[str, ...] = ()+ wrapper_skills: tuple[str, ...] = ()+ cases: Mapping[str, Mapping[str, Any]] = field(default_factory=dict)+ baseline_has_members: bool = False+ # Declared plugin subagents and commands. They count toward activation+ # coverage in the arms that stage them: the with-plugin and loose-parts arms.+ subagents: tuple[str, ...] = ()+ commands: tuple[str, ...] = ()+ # Staged subagent name (casefolded) -> declared name, for harnesses that+ # rename a plugin agent (OpenCode stages ``build`` as ``<plugin>-build``).+ subagent_aliases: Mapping[str, str] = field(default_factory=dict)+ # Probed URL servers' tool input schemas ({server: {tool: schema}}, from an explicit private host probe), checked as arguments.+ mcp_input_schemas: Mapping[str, Mapping[str, Any]] = field(default_factory=dict)+ # The plugin's own name: the namespace Claude Code gives its skills, agents,+ # and commands (``<plugin>:<name>``).+ plugin_names: tuple[str, ...] = ()+ # Agent -> the MCP servers its with-plugin arm stages beyond ``mcp_servers``: Claude Code's+ # native plugin copy also starts the servers that launch from plugin files.+ agent_mcp_servers: Mapping[str, tuple[str, ...]] = field(default_factory=dict)+ # Selected declarations requiring a projection that this particular arm may not stage.+ unstaged_mcp_servers: tuple[str, ...] = ()+ # Agent -> the declared component types its with-plugin arm does not stage: plugin subagents+ # and commands unless that agent loads them natively.+ agent_unstaged: Mapping[str, tuple[str, ...]] = field(default_factory=dict)+ # Agent -> the plugin LSP servers its with-plugin arm stages natively (Claude Code's ``.lsp.json``),+ # each with the file extensions its ``extensionToLanguage`` maps.+ agent_lsp_servers: Mapping[str, Mapping[str, tuple[str, ...]]] = field(default_factory=dict)+ # Ids of the cross-component cases (:func:`is_cross_component_case`); the Bundle lift is also+ # reported on them alone. ``None`` when no staged case was read, so no split is recorded.+ cross_component_cases: frozenset[str] | None = None++ def arm_enabled(self, arm: str) -> bool:+ if arm in {ARM_WITH, ARM_SUM_OF_PARTS, ARM_LOOSE_PARTS}:+ return True+ return arm == ARM_WITHOUT and self.baseline_has_members++ def declared_for(self, arm: str, agent: str = "") -> dict[str, list[str]]:+ """Declared components staged in ``agent``'s ``arm``.++ MCP servers, subagents, and commands are staged in the with-plugin arm+ and in the loose-parts arm (as plain user-level pieces with bare names;+ the plugin name stays declared, so a namespaced spelling still maps to+ the same component). LSP servers are plugin-only glue: unstaged in the+ loose-parts arm.+ """+ declared: dict[str, list[str]] = {COMPONENT_SKILL: list(self.member_skills)}+ with_plugin = arm in PLUGIN_PART_ARMS+ # Every agent's other arms stage the same members only, so a server any with-plugin arm runs is unstaged there.+ agent_servers = [self.agent_mcp_servers.get(agent, ())] if with_plugin else self.agent_mcp_servers.values()+ mcp_servers = list(dict.fromkeys((*self.mcp_servers, *(name for names in agent_servers for name in names))))+ declared[COMPONENT_MCP] = mcp_servers if with_plugin else []+ if with_plugin and self.subagents:+ declared[COMPONENT_SUBAGENT] = list(self.subagents)+ if with_plugin and self.commands:+ declared[COMPONENT_COMMAND] = list(self.commands)+ lsp_servers = list(self.lsp_servers_for(arm, agent))+ if lsp_servers:+ declared[COMPONENT_LSP] = lsp_servers+ # An LSP server only the native Claude Code arm stages is unstaged in every other arm.+ lsp_elsewhere = not lsp_servers and any(self.agent_lsp_servers.values())+ if self.plugin_names:+ declared[DECLARED_PLUGIN] = list(self.plugin_names)+ # Refs to these can never match in this arm, so graders skip them instead of failing them.+ if not with_plugin:+ unstaged = [+ kind+ for kind, names in (+ (COMPONENT_MCP, mcp_servers),+ (COMPONENT_SUBAGENT, self.subagents),+ (COMPONENT_COMMAND, self.commands),+ (COMPONENT_LSP, lsp_elsewhere),+ )+ if names+ ]+ hidden = [+ f"{kind}:{name}"+ for kind, names in ((COMPONENT_SUBAGENT, self.subagents), (COMPONENT_COMMAND, self.commands))+ for name in names+ ]+ if hidden:+ declared[DECLARED_UNSTAGED_NAMES] = hidden+ else:+ # Still declared, so a call the agent makes to one anyway is recorded. The loose-parts arm runs+ # only where every one of these loads standalone (native Claude Code).+ unstaged = (+ [kind for kind in self.agent_unstaged.get(agent, ()) if declared.get(kind)] if arm == ARM_WITH else []+ )+ if lsp_elsewhere:+ unstaged.append(COMPONENT_LSP)+ if unstaged:+ declared[DECLARED_UNSTAGED] = unstaged+ hidden_mcp = set(self.unstaged_mcp_servers) | set(self.mcp_servers)+ hidden_mcp.update(name for names in self.agent_mcp_servers.values() for name in names)+ hidden_mcp.difference_update(declared[COMPONENT_MCP])+ if hidden_mcp and COMPONENT_MCP not in declared.get(DECLARED_UNSTAGED, ()):+ declared.setdefault(DECLARED_UNSTAGED_NAMES, []).extend(f"mcp:{name}" for name in sorted(hidden_mcp))+ return declared++ def lsp_servers_for(self, arm: str, agent: str = "") -> Mapping[str, tuple[str, ...]]:+ """The LSP servers ``agent``'s ``arm`` stages, with their file extensions (a native with-plugin arm only)."""+ return self.agent_lsp_servers.get(agent, {}) if arm == ARM_WITH else {}++ def aliases_for(self, arm: str) -> Mapping[str, str]:+ """Subagent name aliases for ``arm`` (declared subagents count in the with-plugin and loose-parts arms)."""+ return self.subagent_aliases if arm in PLUGIN_PART_ARMS and self.subagents else {}++ def case_spec(self, case_id: str) -> Mapping[str, Any]:+ return self.cases.get(case_id) or {}+++def build_plugin_signals_context(+ *,+ member_skills: Iterable[Any] = (),+ mcp_servers: Iterable[Any] = (),+ wrapper_skills: Iterable[Any] = (),+ entries: Iterable[Any] = (),+ baseline_has_members: bool = False,+ subagents: Iterable[Any] = (),+ commands: Iterable[Any] = (),+ subagent_aliases: Mapping[str, Any] | None = None,+ plugin_name: str | None = None,+ agent_mcp_servers: Mapping[str, Iterable[Any]] | None = None,+ agent_unstaged: Mapping[str, Iterable[Any]] | None = None,+ agent_lsp_servers: Mapping[str, Mapping[str, Any]] | None = None,+) -> PluginSignalsContext:+ """Build a bounded :class:`PluginSignalsContext` from dataset case entries.++ ``plugin_name`` defaults to the name in the generated wrapper package+ (``<plugin>-plugin-eval``) when ``wrapper_skills`` holds one.+ ``agent_mcp_servers`` maps an agent to the MCP servers only its+ with-plugin arm stages, ``agent_unstaged`` to the declared component+ types its with-plugin arm does not stage, and ``agent_lsp_servers`` to the+ LSP servers its with-plugin arm stages (``{server: extensions}``).+ """+ members = _clean_names(member_skills)+ cases: dict[str, Mapping[str, Any]] = {}+ cross_component: set[str] = set()+ read_case = False+ for count, entry in enumerate(entries):+ if count >= MAX_CONTEXT_CASES:+ break+ if not isinstance(entry, Mapping):+ continue+ case_id = entry.get("id")+ if not isinstance(case_id, str) or not case_id.strip() or case_id in cases:+ continue+ read_case = True+ if is_cross_component_case(entry, members):+ cross_component.add(case_id)+ spec = plugin_case_spec(entry)+ if spec:+ cases[case_id] = spec+ declared_subagents = _clean_names(subagents)+ wrappers = _clean_names(wrapper_skills)+ return PluginSignalsContext(+ member_skills=members,+ mcp_servers=_clean_names(mcp_servers),+ wrapper_skills=wrappers,+ cases=cases,+ baseline_has_members=bool(baseline_has_members),+ subagents=declared_subagents,+ commands=_clean_names(name.lstrip("/") for name in commands if isinstance(name, str)),+ subagent_aliases=_clean_aliases(subagent_aliases, declared_subagents),+ plugin_names=_clean_names([plugin_name] if plugin_name else _plugin_names_from_wrappers(wrappers)),+ agent_mcp_servers={+ agent: names+ for agent, servers in (agent_mcp_servers or {}).items()+ if isinstance(agent, str) and (names := _clean_names(servers))+ },+ agent_unstaged={+ agent: kinds+ for agent, values in (agent_unstaged or {}).items()+ if isinstance(agent, str) and (kinds := _clean_names(values))+ },+ agent_lsp_servers={+ agent: servers+ for agent, value in (agent_lsp_servers or {}).items()+ if isinstance(agent, str) and (servers := _clean_lsp_servers(value))+ },+ # No staged case read: the split is not recorded, which is not the same as zero cases.+ cross_component_cases=frozenset(cross_component) if read_case else None,+ )+++def _plugin_names_from_wrappers(wrapper_skills: Iterable[str]) -> list[str]:+ """The plugin name behind a generated ``<plugin>-plugin-eval`` wrapper package name."""+ return [+ name[: -len(_WRAPPER_PACKAGE_SUFFIX)]+ for name in wrapper_skills+ if isinstance(name, str) and name.casefold().endswith(_WRAPPER_PACKAGE_SUFFIX) and len(name) > 12+ ]+++def _clean_lsp_servers(servers: Any) -> dict[str, tuple[str, ...]]:+ """Bounded ``server -> file extensions`` (casefolded, each with its leading dot) of staged LSP servers."""+ cleaned: dict[str, tuple[str, ...]] = {}+ if not isinstance(servers, Mapping):+ return cleaned+ for raw_name, raw_extensions in servers.items():+ if len(cleaned) >= MAX_DECLARED_COMPONENTS:+ break+ names = _clean_names([raw_name])+ if not names or isinstance(raw_extensions, str | bytes) or not isinstance(raw_extensions, Iterable):+ continue+ extensions = [+ extension.strip().casefold()+ for extension in raw_extensions+ if isinstance(extension, str) and _LSP_EXTENSION_RE.fullmatch(extension.strip())+ ]+ cleaned[names[0]] = tuple(dict.fromkeys(extensions))[:_MAX_LSP_EXTENSIONS]+ return cleaned+++def _lsp_extension_index(lsp_servers: Any, declared: Mapping[str, Sequence[str]]) -> dict[str, str]:+ """File extension -> the one LSP server declared in this arm that serves it.++ An extension two declared servers claim is left out: the call cannot say which one answered.+ """+ staged = {name for name in declared.get(COMPONENT_LSP) or () if isinstance(name, str)}+ owners: dict[str, set[str]] = {}+ for server, extensions in _clean_lsp_servers(lsp_servers).items():+ if server in staged:+ for extension in extensions:+ owners.setdefault(extension, set()).add(server)+ return {extension: next(iter(servers)) for extension, servers in owners.items() if len(servers) == 1}+++def _lsp_ident(fn: str, args: Mapping[str, Any], lsp_extensions: Mapping[str, str]) -> _Ident | None:+ """The plugin LSP server a Claude Code ``LSP`` call reached, from its file's extension, or ``None``."""+ path = _normalize_path(_first_string(args, _LSP_PATH_KEYS))+ suffix = posixpath.splitext(posixpath.basename(path))[1].casefold()+ server = lsp_extensions.get(suffix) if suffix else None+ if server is None:+ return None+ return _Ident(label=f"LSP:{server}", kind=COMPONENT_LSP, name=server, fn=fn, tool_label=fn)+++def _clean_aliases(aliases: Mapping[str, Any] | None, declared: Sequence[str]) -> dict[str, str]:+ """Bounded ``staged name (casefolded) -> declared name`` pairs whose target is a declared subagent."""+ by_folded = {name.casefold(): name for name in declared}+ cleaned: dict[str, str] = {}+ for alias, name in (aliases or {}).items():+ if len(cleaned) >= MAX_DECLARED_COMPONENTS:+ break+ if not isinstance(alias, str) or not isinstance(name, str):+ continue+ target = by_folded.get(name.strip().casefold())+ if target is not None and alias.strip():+ cleaned[alias.strip().casefold()[:_MAX_LABEL_CHARS]] = target+ return cleaned+++# =============================================================================+# Trajectory -> calls+# =============================================================================+++@dataclass(frozen=True)+class _Ident:+ """One identity of a tool call: a component activation or a plain tool.++ ``foreign`` marks a namespaced component name from another plugin+ (``acme-tools:release-notes``): it never matches this plugin's refs.+ ``via_read`` marks a ``SKILL.md`` read (not an explicit activation tool);+ ``position`` orders the reads of one chained shell command; ``failed``+ marks a read whose own error line says it failed. ``wrapper`` marks the+ generated wrapper skill, which is the harness's way in, not a component.+ ``builtin`` marks a harness's own agent called by the bare name of a+ declared plugin agent (Claude Code ``Explore`` for a plugin's ``explore``):+ it is neither that component nor what a ref to it names.+ """++ label: str+ kind: str | None+ name: str+ fn: str+ server: str | None = None+ tool: str | None = None+ tool_label: str = ""+ persist_name: bool = True+ foreign: bool = False+ namespaces: tuple[str, ...] = ()+ via_read: bool = False+ position: int = 0+ failed: bool = False+ wrapper: bool = False+ aliases: tuple[str, ...] = ()+ builtin: bool = False++ @property+ def persisted_name(self) -> str:+ return self.name if self.persist_name else _NON_NAME++ @property+ def persisted_label(self) -> str:+ return self.label if self.persist_name else f"{_IDENTITY_PREFIX[self.kind or '']}:{_NON_NAME}"++ @property+ def name_candidates(self) -> list[str]:+ """Casefolded names this component answers to (another plugin's namespaced name only to itself)."""+ low = self.name.casefold()+ if self.foreign:+ return [low]+ candidates = [low, low.rsplit(":", 1)[-1]] if ":" in low else [low]+ short = candidates[-1]+ candidates.extend(f"{namespace}:{short}" for namespace in self.namespaces)+ return list(dict.fromkeys(candidates))++ @cached_property+ def typed_names(self) -> tuple[str, ...]:+ """What a ``<Type>:<pattern>`` ref of this identity's type is matched against."""+ if self.kind == COMPONENT_MCP:+ return _match_names(_mcp_ref_candidates(self))+ return _match_names(self.name_candidates)++ @cached_property+ def untyped_names(self) -> tuple[str, ...]:+ """What any other ref is matched against (see :func:`_ref_matches`)."""+ if self.kind is None:+ names = [self.label, self.fn, *self.aliases]+ elif self.kind == COMPONENT_MCP:+ # The bare tool name is what Codex and the MCP server call the tool (``list_changes``).+ names = [self.label, self.fn, *_mcp_ref_candidates(self), self.tool or ""]+ else:+ names = [self.label, *self.name_candidates]+ if self.kind == COMPONENT_COMMAND and self.name:+ names.append(f"/{self.name}")+ return _match_names(names)+++class _ShellPaths(NamedTuple):+ reads: list[str]+ writes: list[str]+++class _Result(NamedTuple):+ """One tool result of a step.++ ``scan`` is the part of ``text`` that failure text is checked against, and+ ``flagged`` the structured outcome of :func:`_result_flagged`.+ """++ call_id: str+ text: str+ scan: str+ flagged: bool | None+ uncertain: bool = False+++@dataclass+class _Call:+ """One normalized tool call.++ ``mcp`` is the call's MCP identity, however the harness spelled the tool+ name, and ``fn_base`` is the bare tool name: the MCP tool's for an MCP+ call, so every spelling of one MCP call is treated alike.+ """++ seq: int+ step_index: int+ fn: str+ fn_base: str+ args: dict[str, Any]+ idents: list[_Ident]+ mcp: _Ident | None = None+ observation: str | None = None+ succeeded: bool | None = None+ # The result says the called tool, skill, command, or agent does not exist.+ missing_tool: bool = False+ # A subagent's own call (Claude Code sidechain step), and the parent's subagent call that started it.+ sidechain: bool = False+ # Which subagent run made the call ("" for the main agent): Harbor's ``agent_id``, else one id per run.+ chain: str = ""+ spawner: list[_Ident] = field(default_factory=list)+ # The one declared skill or command this call switched to, when it really did (see ``_window_opener``).+ opener: _Ident | None = None+ # The working directory of the call, when the trajectory says it.+ cwd: str | None = None+ # Components a shell call used without its tool: a JSON-RPC ``tools/call`` piped to an MCP server,+ # or a member skill's script run directly (see ``_side_channel_idents``).+ side_idents: list[_Ident] = field(default_factory=list)++ @property+ def is_shell(self) -> bool:+ """A local shell call (an MCP tool named like a shell is still an MCP call)."""+ return self.fn_base in _SHELL_TOOLS and self.mcp is None++ @property+ def is_content_read(self) -> bool:+ """A skill load or a plain file read: its result is file or skill text, not a status message."""+ if self.mcp is not None:+ return False+ if self.fn.casefold() in _SKILL_TOOLS or any(ident.kind == COMPONENT_SKILL for ident in self.idents):+ return True+ return self.fn_base in _READ_TOOLS++ @cached_property+ def result_texts(self) -> list[str]:+ """The result text plus the string values of JSON inside it, decoded once per call."""+ return _decoded_texts(self.observation) if self.observation else []++ @cached_property+ def argument_texts(self) -> list[str]:+ """The argument string values (and JSON inside them), decoded once per call."""+ return [text for leaf in _string_values(self.args) for text in _decoded_texts(leaf)]++ @property+ def component_idents(self) -> list[_Ident]:+ return [ident for ident in self.idents if ident.kind is not None]++ def ident_ok(self, ident: _Ident) -> bool:+ """Whether this identity's activation did not fail (``None`` outcomes count as not failed)."""+ return self.succeeded is not False and not ident.failed++ @cached_property+ def shell_paths(self) -> _ShellPaths:+ """Distinct shell paths this call reads and writes, resolved against the call's directory.++ A path may be a glob (``out/*.json``); see :func:`_observed_path_matches`.+ """+ reads: dict[str, None] = {}+ writes: dict[str, None] = {}+ if self.is_shell:+ for text in _shell_texts(self.fn_base, self.args):+ text_reads, text_writes = _shell_file_io(text, self.cwd)+ reads.update(dict.fromkeys(text_reads))+ writes.update(dict.fromkeys(text_writes))+ return _ShellPaths(list(reads), list(writes))+++def _safe_text(value: Any, limit: int = _MAX_LABEL_CHARS) -> str:+ # The redactor is superlinear on some inputs, so it only ever sees a bounded+ # window; text past the window would be truncated away anyway. The slack+ # leaves room for redaction to shorten the text and still fill ``limit``.+ return redact_printable_text(+ str(value if value is not None else ""), max_len=limit, max_input_chars=limit * 4 + 256+ )+++def _tool_name(tool_call: Mapping[str, Any]) -> str:+ for key in ("function_name", "name", "tool_name"):+ value = tool_call.get(key)+ if isinstance(value, str) and value.strip():+ return value.strip()[:_MAX_LABEL_CHARS]+ return ""+++def _arguments(tool_call: Mapping[str, Any]) -> dict[str, Any]:+ raw = tool_call.get("arguments")+ if raw is None:+ raw = tool_call.get("input", tool_call.get("args"))+ if isinstance(raw, dict):+ return raw+ if isinstance(raw, str):+ if len(raw) <= _MAX_ARGS_JSON_CHARS:+ try:+ decoded = json.loads(raw)+ except (ValueError, RecursionError):+ decoded = None+ if isinstance(decoded, dict):+ return decoded+ return {"input": raw[:_MAX_ARGS_JSON_CHARS]}+ return {}+++def _content_parts(content: Any) -> list[str]:+ """Bounded text of each content block (one part for a plain string/object)."""+ if content is None:+ return []+ if isinstance(content, str):+ return [content[:_MAX_OBSERVATION_CHARS]]+ if isinstance(content, list):+ parts: list[str] = []+ size = 0+ for block in content[:_MAX_CONTENT_BLOCKS]:+ if isinstance(block, dict) and isinstance(block.get("text"), str):+ part = block["text"]+ elif isinstance(block, str):+ part = block+ else:+ part = _bounded_json_text(block, _MAX_OBSERVATION_CHARS) or ""+ parts.append(part)+ size += len(part)+ if size >= _MAX_OBSERVATION_CHARS:+ break+ return parts+ return [(_bounded_json_text(content, _MAX_OBSERVATION_CHARS) or "")[:_MAX_OBSERVATION_CHARS]]+++def _content_text(content: Any) -> str:+ return "\n".join(_content_parts(content))[:_MAX_OBSERVATION_CHARS]+++def _meaningful_lines(text: str, *, last: bool = False) -> Iterable[str]:+ """Non-blank lines of a bounded window, past a harness's status lines (``Wall time:``, ``Output:``)."""+ window = text[-_FAILURE_SCAN_CHARS:] if last else text[:_FAILURE_SCAN_CHARS]+ lines = window.splitlines()+ for line in reversed(lines) if last else lines:+ stripped = line.strip()+ if stripped and not _SHELL_STATUS_LINE_RE.fullmatch(stripped):+ yield stripped+++def _unwrapped_texts(text: str) -> list[str]:+ """The tool texts inside an MCP content wrapper that opens ``text``, else ``[]``.++ Codex prints an MCP result as ``[{"type":"text","text":"..."}]`` after its+ status lines; Harbor's Claude converter may keep one such block as a+ string. A cut-off wrapper still yields the start of its first text.+ """+ head = _MCP_TEXT_WRAPPER_RE.match(text)+ if head is None:+ return []+ decoded: Any = None+ if len(text) <= _MAX_WRAPPER_JSON_CHARS:+ try:+ decoded = json.loads(text, strict=False)+ except (ValueError, RecursionError):+ decoded = None+ blocks = decoded if isinstance(decoded, list) else [decoded]+ texts = [block["text"] for block in blocks[:8] if isinstance(block, dict) and isinstance(block.get("text"), str)]+ if texts:+ return texts+ literal = re.match(r'(?:[^"\\]|\\.){0,2048}', text[head.end() :])+ try:+ return [json.loads(f'"{literal.group()}"', strict=False)] if literal else []+ except (ValueError, RecursionError):+ return []+++def _lead_lines(part: str) -> list[str]:+ """The lines of one result block that can say the call failed: its first and last lines.++ Harness status lines are skipped, and an MCP content wrapper is opened so+ the tool's own first and last lines are read. The middle of a long answer+ is never read: a good result that talks about errors is not a failure.+ """+ first = next(iter(_meaningful_lines(part)), "")+ if not first:+ return []+ body = part[part.find(first) :] if first in part else first+ texts = _unwrapped_texts(body[:_MAX_OBSERVATION_CHARS])+ if not texts:+ return [first, next(iter(_meaningful_lines(part, last=True)), "")]+ lines: list[str] = []+ for text in texts:+ lines.append(next(iter(_meaningful_lines(text)), ""))+ lines.append(next(iter(_meaningful_lines(text, last=True)), ""))+ return lines+++def _failure_scan_text(parts: Sequence[str], text: str) -> str:+ """The lead lines failure text is checked against: each block's first and last lines, and the result's last."""+ lines = [line for part in parts[:64] for line in _lead_lines(part)]+ if len(parts) > 1:+ lines.append(next(iter(_meaningful_lines(text, last=True)), ""))+ return "\n".join(line[:_FAILURE_SCAN_CHARS] for line in lines if line)+++def _line_failed(line: str, *, shell: bool) -> bool:+ """Whether one lead line says the call failed (see the marker notes above)."""+ lowered = line.casefold()+ if any(phrase in lowered for phrase in _FAILURE_PHRASES):+ return True+ if _UNAVAILABLE_TAIL_RE.search(line[-64:]):+ return True+ for prefix in _STATUS_LINE_PREFIX_RES:+ opened = prefix.match(line)+ if opened is not None and _HTTP_STATUS_OPENING_RE.match(line, opened.end()):+ return True+ # The line, then the line past each leading ``name:`` segment (at most three).+ head = line+ for _ in range(4):+ if any(+ pattern.match(head)+ for pattern in (+ _HARNESS_ERROR_PREFIX_RE,+ _HTTP_STATUS_OPENING_RE,+ _HTTP_VERSION_STATUS_RE,+ _REQUEST_FAILED_RE,+ _OPENING_FAILURE_RE,+ )+ ):+ return True+ if not shell and _FILE_MISSING_RE.match(head):+ return True+ segment = _LEAD_SEGMENT_RE.match(head)+ if segment is None:+ return False+ head = head[segment.end() :]+ return False+++def _flag_value(value: Any) -> bool | None:+ """``True``/``False`` for a boolean flag in any common spelling, else ``None``."""+ if isinstance(value, bool):+ return value+ if isinstance(value, int) and value in (0, 1):+ return value == 1+ if isinstance(value, str):+ lowered = value.strip().casefold()+ if lowered in _TRUE_FLAGS:+ return True+ if lowered in _FALSE_FLAGS:+ return False+ return None+++def _result_flagged(result: Mapping[str, Any]) -> bool | None:+ """A result's structured outcome: ``True`` failed, ``False`` succeeded, ``None`` no signal.++ A failure flag is any ``is_error``/``isError`` (top level or under+ ``extra``), Harbor's Claude Code keys ``extra.tool_result_is_error`` and+ ``extra.tool_result_metadata.is_error`` (and its ``raw_tool_result``), a+ non-empty ``error``, an MCP ``content.isError``, or a failed+ ``extra.harness_status`` (Codex ``codex.txt``). Only the harness's own+ report says a call succeeded: an explicit ``tool_result_is_error: false``+ or a ``completed`` harness status.+ """+ extra = result.get("extra")+ extra = extra if isinstance(extra, Mapping) else {}+ flags: list[Any] = []+ for source in (result, extra):+ flags.extend(source.get(key) for key in _ERROR_FLAG_KEYS)+ error = source.get("error")+ if error not in (None, False, "", {}, []):+ return True+ for key in ("tool_result_metadata", "metadata"):+ metadata = extra.get(key)+ if isinstance(metadata, Mapping):+ flags.extend(metadata.get(name) for name in _ERROR_FLAG_KEYS)+ raw = metadata.get("raw_tool_result")+ if isinstance(raw, Mapping):+ flags.append(raw.get("is_error"))+ content = result.get("content")+ if isinstance(content, Mapping):+ flags.append(content.get("isError"))+ if any(_flag_value(flag) is True for flag in flags):+ return True+ harness = _flag_value(extra.get("tool_result_is_error"))+ if harness is True:+ return True+ status = extra.get(HARNESS_STATUS_KEY)+ status = status.strip().casefold() if isinstance(status, str) else ""+ if status in _HARNESS_FAILED:+ return True+ if harness is False or status in _HARNESS_SUCCEEDED:+ return False+ return None+++def _observations(step: Mapping[str, Any]) -> list[_Result]:+ """Each result of ``step``.++ ``flagged`` is the structured outcome of :func:`_result_flagged`. Harbor+ puts the flag of an orphan Claude Code result on its single-call step's+ ``extra``, so a single-call step's own flags count for its results too.+ """+ observation = step.get("observation")+ if not isinstance(observation, Mapping):+ return []+ results = observation.get("results")+ if not isinstance(results, list):+ return []+ calls = step.get("tool_calls")+ step_extra = step.get("extra")+ step_flag = (+ _result_flagged({"extra": step_extra})+ if isinstance(calls, list) and len(calls) == 1 and isinstance(step_extra, Mapping)+ else None+ )+ entries: list[_Result] = []+ for result in results[:_MAX_STEP_RESULTS]:+ if not isinstance(result, Mapping):+ continue+ parts = _content_parts(result.get("content"))+ text = "\n".join(parts)[:_MAX_OBSERVATION_CHARS]+ flagged = _result_flagged(result)+ if step_flag is True or (flagged is None and step_flag is False):+ flagged = step_flag+ entries.append(+ _Result(+ call_id=str(result.get("source_call_id") or ""),+ text=text,+ scan=_failure_scan_text(parts, text),+ flagged=flagged,+ uncertain=isinstance(result.get("extra"), Mapping)+ and result["extra"].get(HARNESS_STATUS_KEY) == "unknown",+ )+ )+ return entries+++def _results_for_call(results: Sequence[_Result], call_id: str, *, call_count: int) -> list[_Result]:+ """Results attributable to ``call_id``; id matches win, ambiguity stays unknown."""+ if call_id:+ matched = [result for result in results if result.call_id == call_id]+ if matched:+ return matched+ if call_count == 1 and len(results) == 1 and not results[0].call_id:+ return [results[0]]+ return []+++def _first_line(text: str, *, shell: bool = False) -> str:+ for line in text[:_FAILURE_SCAN_CHARS].splitlines():+ stripped = line.strip()+ if stripped and not (shell and _SHELL_STATUS_LINE_RE.fullmatch(stripped)):+ return line+ return ""+++def _content_read_scan(text: str, *, shell: bool) -> str:+ """The lines of a file read or skill load that can say the read failed.++ The first line, past any shell status lines. A shell read can also fail+ after other output, so every shell error line that names a skill manifest+ (``cat: <dir>/SKILL.md: Permission denied``) counts too.+ """+ lines = [_first_line(text, shell=shell)]+ if shell:+ lines.extend(+ line+ for line in text.splitlines()+ if "skill.md" in line.casefold() and _SHELL_ERROR_LINE_RE.match(line.strip())+ )+ return "\n".join(lines)+++def _outcome(+ correlated: Sequence[_Result], *, shell: bool, content_read: bool = False+) -> tuple[str | None, bool | None]:+ """``(observation text, succeeded)`` for one call (see the module docs).++ A structured outcome wins: any failure flag fails the call, and the+ harness's own success report (Claude Code ``tool_result_is_error: false``,+ a Codex ``completed`` status) passes it whatever the text says. Without+ one, failure text is read on the result's lead lines only. For a+ ``content_read`` (a file read or a skill load) the body is file or skill+ text, so only the first line of a result (past a harness's shell status+ lines) or a shell error line that names a skill manifest counts.+ """+ if not correlated:+ return None, None+ text = "".join(result.text for result in correlated)[:_MAX_OBSERVATION_CHARS]+ flags = [result.flagged for result in correlated]+ if any(flag is True for flag in flags):+ return text, False+ if any(result.uncertain for result in correlated):+ return text, None+ if any(flag is False for flag in flags):+ return text, True+ if not text.strip():+ return text, None+ if content_read:+ scan = "\n".join(_content_read_scan(result.text, shell=shell) for result in correlated)+ else:+ scan = "\n".join(result.scan for result in correlated)+ failed = any(_line_failed(line.strip(), shell=shell) for line in scan.splitlines() if line.strip())+ return text, not failed+++def _base_tool_name(fn: str) -> str:+ low = fn.casefold()+ for separator in _TOOL_NAME_SEPARATORS:+ if separator in low:+ low = low.rsplit(separator, 1)[-1]+ return low+++def _norm_server(value: str) -> str:+ return _SANITIZED_NAME_RE.sub("_", value.casefold())+++def _server_spellings(server: str) -> tuple[str, ...]:+ """Casefolded spellings of a declared server name inside harness tool names, exact spelling first.++ Claude Code and OpenCode replace characters outside ``[A-Za-z0-9_-]`` with+ ``_``; Hermes also replaces ``-``.+ """+ low = server.casefold()+ return tuple(dict.fromkeys((low, _norm_server(server), _HERMES_NAME_RE.sub("_", low))))+++class _McpPrefix(NamedTuple):+ """A tool-name prefix that names a declared MCP server (see :meth:`_McpNames.identity`)."""++ order: int # scan order: declared server, then its spelling, then the prefix form+ length: int+ rank: int # 0 for the server's exact spelling, 1 or 2 for a harness-normalized one+ server: str+ spelling: str+ bare: bool # OpenCode's ``<server>_<tool>``, a form other harnesses never use+++class _McpNames:+ """The declared MCP servers of one arm, indexed by every spelling and tool-name prefix harnesses give them.++ Built once per trajectory (or per batch of lookups), so naming the server+ of a call takes a few dictionary lookups however many servers are+ declared. The tool-name prefix table is built on the first+ :meth:`identity`, since :meth:`match` does not use it.+ """++ def __init__(self, declared: Iterable[Any]) -> None:+ self.names = list(dict.fromkeys(name for name in declared if isinstance(name, str) and name))+ self._exact: dict[str, str] = {} # casefolded name -> first declared name+ self._spelled: dict[str, list[str]] = {} # spelling -> the declared names that have it+ for name in self.names:+ self._exact.setdefault(name.casefold(), name)+ for spelling in _server_spellings(name):+ holders = self._spelled.setdefault(spelling, [])+ if name not in holders:+ holders.append(name)++ @cached_property+ def _prefixes(self) -> dict[str, list[_McpPrefix]]:+ """Tool-name prefix -> the declared-server entries it names (see :meth:`identity`)."""+ prefixes: dict[str, list[_McpPrefix]] = {}+ order = 0+ for name in self.names:+ for rank, spelling in enumerate(_server_spellings(name)):+ forms = [(spelling + separator, False) for separator in _TOOL_NAME_SEPARATORS]+ forms += [(f"mcp_{spelling}_", False), (spelling + "_", True)]+ for prefix, bare in forms:+ entry = _McpPrefix(order, len(prefix), rank, name, spelling, bare)+ prefixes.setdefault(prefix, []).append(entry)+ order += 1+ return prefixes++ @cached_property+ def _prefix_ends(self) -> frozenset[str]:+ return frozenset(prefix[-1] for prefix in self._prefixes)++ def match(self, observed: str) -> str | None:+ """See :func:`match_declared_mcp_server`."""+ low = observed.casefold()+ if not low or not self.names:+ return None+ exact = self._exact.get(low)+ if exact is not None:+ return exact+ spelled = self._spelled.get(low, [])+ if len(spelled) == 1:+ return spelled[0]+ if spelled or not low.startswith(_CLAUDE_PLUGIN_SERVER_PREFIX):+ return None+ rest = low[len(_CLAUDE_PLUGIN_SERVER_PREFIX) :]+ slug, sep, server = rest.partition("_")+ if slug and sep and server:+ found = self.match(server)+ if found is not None:+ return found+ # The longest declared spelling ``rest`` ends with, after a ``_`` that is not its first character.+ for index in range(1, len(rest)):+ if rest[index] == "_" and (longest := self._spelled.get(rest[index + 1 :])):+ return longest[0] if len(longest) == 1 else None+ return None++ def identity(self, fn: str, *, agent: str) -> tuple[str, str] | None:+ """``(server, tool)`` of an MCP tool call, with the server mapped to its declared name when known.++ Recognized spellings: ``mcp__<server>__<tool>`` (Claude Code, Codex;+ Claude Code plugin servers appear as ``plugin_<plugin>_<server>``), and+ for declared servers ``<server>__<tool>``/``.``/``/``/``:``, Hermes+ ``mcp_<server>_<tool>``, and OpenCode ``<server>_<tool>`` (not for+ harnesses that never use it). The longest matching prefix wins; on a tie+ an exact spelling beats a normalized one, and a remaining tie between+ servers is left unattributed.+ """+ low = fn.casefold()+ if low.startswith(_MCP_PREFIX):+ server, _, tool = fn[len(_MCP_PREFIX) :].partition("__")+ if not server:+ return None+ return self.match(server) or server, tool+ bare_allowed = agent.casefold() not in _NO_BARE_MCP_PREFIX_AGENTS and low not in _BUILTIN_TOOL_NAMES+ matches: list[_McpPrefix] = []+ for end in range(1, len(low)): # a prefix leaves at least one character of tool name+ if low[end - 1] in self._prefix_ends:+ matches.extend(self._prefixes.get(low[:end], ()))+ best: tuple[int, int] | None = None+ best_spelling = ""+ winners: list[str] = []+ for entry in sorted(matches):+ if entry.bare and not bare_allowed:+ continue+ key = (entry.length, -min(entry.rank, 1))+ if best is None or key > best:+ best, best_spelling, winners = key, entry.spelling, [entry.server]+ elif key == best and entry.server not in winners:+ winners.append(entry.server)+ if best is None:+ return None+ # Two declared servers share this spelling: count the call under the spelling itself.+ return (winners[0] if len(winners) == 1 else best_spelling), fn[best[0] :]+++def match_declared_mcp_server(observed: str, declared: Iterable[str]) -> str | None:+ """The one declared server that ``observed`` (a server name taken from a tool name) refers to.++ An exact (case-insensitive) match wins. Otherwise a harness spelling of a+ declared name counts only when exactly one declared server has it, so+ ``my.docs`` and ``my-docs`` never credit each other. Claude Code names+ plugin servers ``plugin_<plugin>_<server>``; the plugin slug never holds+ ``_``, so the server is what follows the first ``_`` (a longest declared+ suffix is the fallback).+ """+ return _McpNames(declared).match(observed)+++def declared_mcp_server_matcher(declared: Iterable[str]) -> Callable[[str], str | None]:+ """:func:`match_declared_mcp_server` for one set of declared servers, indexed once for many lookups."""+ return _McpNames(declared).match+++def _first_string(args: Mapping[str, Any], keys: Sequence[str]) -> str:+ for key in keys:+ value = args.get(key)+ if isinstance(value, str) and value.strip():+ return value.strip()+ return ""+++def _normalize_path(value: str) -> str:+ text = value.strip().strip("'\"").replace("\\", "/")+ text = _REPEATED_SLASHES_RE.sub("/", text)+ while text.startswith("./"):+ text = text[2:]+ return text.rstrip("/") if len(text) > 1 else text+++def _path_args(args: Mapping[str, Any]) -> list[str]:+ paths: list[str] = []+ for key in _PATH_ARG_KEYS:+ value = args.get(key)+ if isinstance(value, str) and value.strip():+ paths.append(value)+ return paths+++def _shell_texts(fn_base: str, args: Mapping[str, Any]) -> list[str]:+ texts: list[str] = []+ for key in _SHELL_COMMAND_KEYS:+ if key == "input" and fn_base == "exec":+ # Native Codex ``exec`` input is JavaScript, not a shell command.+ continue+ value = args.get(key)+ if isinstance(value, str) and value.strip():+ texts.append(value[:_MAX_SHELL_TEXT_CHARS])+ elif isinstance(value, list) and value and all(isinstance(item, str) for item in value):+ words = [str(item) for item in value[:_MAX_SHELL_TOKENS]]+ texts.append(shlex.join(words)[:_MAX_SHELL_TEXT_CHARS])+ return texts+++def _split_heredocs(text: str) -> tuple[str, list[tuple[str, str]]]:+ """``(text without here-document bodies, [(delimiter, body), ...] in order)``.++ Bodies are data, not commands, and may hold stray quotes, so the shell+ parser never sees them. A body is split off only when its terminator line+ is found, so a ``<<`` that is not really a here-document never swallows+ the rest of the script.+ """+ if "<<" not in text:+ return text, []+ lines = text.split("\n")+ kept: list[str] = []+ bodies: list[tuple[str, str]] = []+ index = 0+ while index < len(lines):+ line = lines[index]+ kept.append(line)+ index += 1+ for match in _HEREDOC_RE.finditer(line):+ delimiter = next(group for group in match.groups() if group)+ end = next((at for at in range(index, len(lines)) if lines[at].strip() == delimiter), None)+ if end is None:+ return text, []+ bodies.append((delimiter, "\n".join(lines[index:end])))+ index = end + 1+ return "\n".join(kept), bodies+++def _strip_heredoc_bodies(text: str) -> str:+ """``text`` without its here-document bodies (see :func:`_split_heredocs`)."""+ return _split_heredocs(text)[0]+++def _take_heredoc_bodies(delimiters: Sequence[str], pending: list[tuple[str, str]]) -> list[str]:+ """Pop the bodies of one command's here-documents (by delimiter, in order) off ``pending``.++ A ``<<`` the shell parser saw with no body (``$((1<<2))``) takes nothing.+ A body passed over belonged to a ``<<`` that was not a real operator.+ """+ bodies: list[str] = []+ for delimiter in delimiters:+ at = next((index for index, (name, _) in enumerate(pending) if name == delimiter), None)+ if at is None:+ continue+ bodies.append(pending[at][1])+ del pending[: at + 1]+ return bodies+++def _shell_tokens(text: str) -> list[str]:+ normalized = _strip_heredoc_bodies(text.replace("\r\n", "\n").replace("\r", "\n")).replace("\n", " ; ")+ lexer = shlex.shlex(normalized, posix=True, punctuation_chars=True)+ lexer.whitespace_split = True+ lexer.commenters = ""+ tokens: list[str] = []+ try:+ for token in lexer:+ tokens.append(token)+ if len(tokens) >= _MAX_SHELL_TOKENS:+ break+ except ValueError:+ # Unbalanced quote: fall back on the same newline-normalized text so a+ # multi-line script still splits into its commands.+ try:+ return shlex.split(normalized, posix=True)[:_MAX_SHELL_TOKENS]+ except ValueError:+ return normalized.split()[:_MAX_SHELL_TOKENS]+ return tokens+++def _shell_payload(command: list[str]) -> str | None:+ """Return the ``-c`` payload of ``bash -lc '...'``-style invocations."""+ for index, token in enumerate(command[1:], start=1):+ if token.startswith("-") and not token.startswith("--") and "c" in token[1:]:+ return command[index + 1] if index + 1 < len(command) else None+ if not token.startswith("-"):+ return None+ return None+++def _shell_chain(text: str, depth: int = 0) -> list[tuple[list[str], str]]:+ """``(command words, joiner before it)`` for each simple command, in order.++ The joiner is the operator that links a command to the one before it+ (``&&``, ``||``, ``;``, ``|``...; ``""`` for the first). ``bash -lc '...'``+ payloads are expanded in place; their first command takes the outer joiner.+ """+ if depth > _MAX_SHELL_DEPTH or len(text) > _MAX_SHELL_TEXT_CHARS:+ return []+ chain: list[tuple[list[str], str]] = []+ current: list[str] = []+ joiner = ""+ for token in _shell_tokens(text):+ if token in _SHELL_SEPARATORS:+ if current:+ chain.append((current, joiner))+ current = []+ # A subshell keeps the operator that leads into it; after it, the next operator links.+ if token == ")":+ joiner = ";"+ elif token != "(":+ joiner = token+ else:+ current.append(token)+ if current:+ chain.append((current, joiner))+ expanded: list[tuple[list[str], str]] = []+ for command, link in chain:+ index = 0+ while index < len(command) and (+ _ENV_ASSIGNMENT_RE.match(command[index]) or command[index] in _SHELL_PREFIX_WORDS+ ):+ index += 1+ command = command[index:]+ if not command:+ continue+ if command[0].rsplit("/", 1)[-1] in _SHELLS:+ payload = _shell_payload(command)+ if payload is not None:+ inner = _shell_chain(payload, depth + 1)+ if inner:+ inner[0] = (inner[0][0], link)+ expanded.extend(inner)+ continue+ expanded.append((command, link))+ return expanded+++def _shell_commands(text: str, depth: int = 0) -> list[list[str]]:+ return [command for command, _ in _shell_chain(text, depth)]+++@dataclass+class _CommandIO:+ """The file operands of one simple shell command."""++ verb: str = ""+ operands: list[str] = field(default_factory=list)+ flags: list[str] = field(default_factory=list)+ inputs: list[str] = field(default_factory=list)+ outputs: list[str] = field(default_factory=list)+ # Here-document delimiters, in order (``cat <<'EOF' > out.json`` has ``EOF``).+ heredocs: list[str] = field(default_factory=list)+++def _command_io(command: Sequence[str]) -> _CommandIO:+ io = _CommandIO()+ index = 0+ while index < len(command):+ token = command[index]+ if token == ">&":+ # ``2>&1``-style descriptor duplication, not a file target.+ index += 2+ continue+ if token in _OUTPUT_REDIRECTS:+ if index + 1 < len(command):+ io.outputs.append(command[index + 1])+ index += 2+ continue+ if token == "<":+ if index + 1 < len(command):+ io.inputs.append(command[index + 1])+ index += 2+ continue+ if token in {"<<", "<<-"}:+ # A here-document: its delimiter is not an operand, and redirects may follow it.+ if index + 1 < len(command):+ io.heredocs.append(command[index + 1].lstrip("-"))+ index += 2+ continue+ if token == "<<<":+ # A here-string is stdin data, not a file operand.+ index += 2+ continue+ if not io.verb:+ io.verb = token.rsplit("/", 1)[-1].casefold()+ elif token.startswith("-"):+ io.flags.append(token)+ else:+ io.operands.append(token)+ index += 1+ return io+++def _sed_in_place(io: _CommandIO) -> bool:+ return io.verb == "sed" and any(flag.startswith("-i") or flag.startswith("--in-place") for flag in io.flags)+++_INTERPRETERS = frozenset({"python", "python3", "node", "ruby", "perl", "deno", "bun"})+_INLINE_CODE_FLAGS = frozenset({"-c", "-e", "--eval", "-p", "--print"})+# Interpreter code that opens a file, given as a path literal (``open('out/a.json', 'w')``,+# ``Path('out/a.json').write_text(...)``) or as a name assigned one path literal+# (``out = Path('out/a.json')`` ... ``out.write_text(...)``). ``{T}`` is the file.+_CODE_LITERAL = r"""(?P<q>['"])(?P<p>[^'"\n]{1,512})(?P=q)"""+_CODE_PATH_LITERAL = r"\b(?:pathlib\.)?Path\(\s*" + _CODE_LITERAL + r"\s*\)"+_CODE_NAME = r"(?<![\w.])(?P<v>[A-Za-z_]\w{0,63})\b"+_CODE_WRITE_MODE = r"""\s*(?:mode\s*=\s*)?['"][^'"\n]{0,8}[wax]"""+_CODE_NOT_WRITE_MODE = r"""(?!\s*(?:mode\s*=\s*)?['"][^'"\n]{0,8}[wax])"""+# ``name = 'out/a.json'``, ``name = Path('out/a.json')``, or ``const name = 'out/a.json';`` on a line of its own.+_CODE_ASSIGN_RE = re.compile(+ r"^[ \t]*(?:(?:const|let|var)\s+)?(?P<name>[A-Za-z_]\w{0,63})\s*=\s*"+ r"(?P<path_call>\b(?:pathlib\.)?Path\(\s*)?" + _CODE_LITERAL + r"(?(path_call)\s*\))\s*;?[ \t]*(?:#[^\n]*)?$",+ re.MULTILINE,+)+++def _code_forms(*forms: tuple[str, str]) -> tuple[tuple[re.Pattern[str], bool], ...]:+ """``(pattern, by_name)`` for each form, once with its literal file and once with a name."""+ compiled: list[tuple[re.Pattern[str], bool]] = []+ for template, literal in forms:+ compiled.append((re.compile(template.replace("{T}", literal)), False))+ compiled.append((re.compile(template.replace("{T}", _CODE_NAME)), True))+ return tuple(compiled)+++_CODE_WRITE_RES = _code_forms(+ (r"\bopen\(\s*{T}\s*," + _CODE_WRITE_MODE, _CODE_LITERAL),+ (r"{T}\s*\.\s*write_(?:text|bytes)\(", _CODE_PATH_LITERAL),+ (r"{T}\s*\.\s*open\(" + _CODE_WRITE_MODE, _CODE_PATH_LITERAL),+ (r"\b(?:writeFileSync|appendFileSync|writeFile|appendFile)\(\s*{T}", _CODE_LITERAL),+ (r"\.to_(?:csv|json|parquet|excel)\(\s*{T}", _CODE_LITERAL),+)+_CODE_READ_RES = _code_forms(+ (r"\bopen\(\s*{T}\s*(?:\)|," + _CODE_NOT_WRITE_MODE + ")", _CODE_LITERAL),+ (r"{T}\s*\.\s*read_(?:text|bytes)\(", _CODE_PATH_LITERAL),+ (r"{T}\s*\.\s*open\(" + _CODE_NOT_WRITE_MODE, _CODE_PATH_LITERAL),+ (+ r"\b(?:readFileSync|readFile|read_csv|read_json|read_table|read_parquet|read_excel|load_workbook)\(\s*{T}",+ _CODE_LITERAL,+ ),+)+++def _resolve_path(path: str, cwd: str | None) -> str:+ """``path`` normalized and, when relative and the directory is known, made absolute."""+ text = _normalize_path(path)+ if not text or text.startswith("~"):+ return text+ if not text.startswith("/") and cwd:+ text = f"{cwd.rstrip('/')}/{text}"+ return posixpath.normpath(text) if text.startswith("/") else text+++def _code_paths(code: str, forms: Sequence[tuple[re.Pattern[str], bool]], names: Mapping[str, str]) -> list[str]:+ paths: list[str] = []+ for pattern, by_name in forms:+ for match in pattern.finditer(code):+ path = names.get(match.group("v")) if by_name else match.group("p")+ if path:+ paths.append(path)+ return paths+++def _inline_code_io(code: str) -> tuple[list[str], list[str]]:+ """``(read, written)`` paths that interpreter code opens.++ The code is a ``-c``/``-e`` program (``python3 -c "open('f','w')..."``) or+ a here-document fed to the interpreter (``python3 - <<'PY' ... PY``). A file+ is a path literal or a name assigned exactly one path literal.+ """+ code = code[:_MAX_SHELL_TEXT_CHARS]+ assigned: dict[str, set[str]] = {}+ for match in _CODE_ASSIGN_RE.finditer(code):+ assigned.setdefault(match.group("name"), set()).add(match.group("p"))+ names = {name: next(iter(paths)) for name, paths in assigned.items() if len(paths) == 1}+ writes = list(dict.fromkeys(_code_paths(code, _CODE_WRITE_RES, names)))+ reads = [path for path in dict.fromkeys(_code_paths(code, _CODE_READ_RES, names)) if path not in writes]+ return reads, writes+++def _shell_file_io(text: str, cwd: str | None) -> tuple[list[str], list[str]]:+ """``(read_paths, written_paths)`` of shell ``text``, resolved against ``cwd`` (``cd`` moves it).++ Reads are operands of reader and consumer verbs (``cat``, ``jq``, ``python3+ script.py <path>``...) and ``<`` input; writes are output redirects,+ ``tee``, ``cp``/``mv``/``install`` targets, ``touch``, and ``sed -i`` (also a+ read). Interpreter code, inline (``-c``/``-e``) or fed as a here-document,+ is scanned for the paths it opens. A glob operand stays a glob. A shell+ ``apply_patch`` (Codex) writes the files its patch headers name (see+ :func:`_patch_written_paths`).+ """+ reads: list[str] = []+ writes: list[str] = []+ pending = _split_heredocs(text.replace("\r\n", "\n").replace("\r", "\n"))[1]+ patched = False+ for command, _ in _shell_chain(text):+ io = _command_io(command)+ verb, operands = io.verb, io.operands+ bodies = _take_heredoc_bodies(io.heredocs, pending) if io.heredocs and pending else []+ if verb == "cd":+ target = operands[0] if operands else ""+ cwd = _resolve_path(target, cwd) if target and not target.startswith(("-", "~", "$")) else None+ continue+ command_reads = list(io.inputs)+ command_writes = list(io.outputs)+ if not patched and any(_APPLY_PATCH_COMMAND_RE.fullmatch(word.rsplit("/", 1)[-1]) for word in command):+ # The patch may be a here-document, an argument or piped text, so its headers are read from+ # the whole script, against the directory this command runs in.+ patched = True+ command_writes += _patch_written_paths(text)+ if verb in _INTERPRETERS and any(flag in _INLINE_CODE_FLAGS for flag in io.flags):+ code_reads, code_writes = _inline_code_io(operands[0] if operands else "")+ command_reads += code_reads+ command_writes += code_writes+ elif verb in _INTERPRETERS and bodies and not operands:+ # ``python3 - <<'PY' ... PY``: the here-document is the program.+ code_reads, code_writes = _inline_code_io("\n".join(bodies))+ command_reads += code_reads+ command_writes += code_writes+ elif _sed_in_place(io):+ edited = operands[1:] if len(operands) > 1 else operands+ command_reads += edited+ command_writes += edited+ elif verb in _ARTIFACT_CONSUMER_VERBS:+ command_reads += operands+ elif verb == "tee":+ command_writes += operands+ elif verb in {"cp", "mv", "install"} and len(operands) >= 2:+ command_writes.append(operands[-1])+ command_reads += operands[:-1]+ elif verb == "touch":+ command_writes += operands+ reads.extend(_resolve_path(path, cwd) for path in command_reads)+ writes.extend(_resolve_path(path, cwd) for path in command_writes)+ return reads, writes+++def _manifest_operands(command: Sequence[str]) -> list[str]:+ """Operands this command really reads as file text: reader verbs only.++ ``cp``/``mv`` sources, ``<`` input to a non-reader (``grep x < SKILL.md``),+ and ``sed -i`` edits do not load a skill.+ """+ io = _command_io(command)+ if io.verb not in _FILE_READER_VERBS or _sed_in_place(io):+ return []+ return [*io.operands, *io.inputs]+++def _is_editor_view(fn_base: str, args: Mapping[str, Any]) -> bool:+ """A text editor tool call running its ``view`` command (a read)."""+ return fn_base in _STR_REPLACE_EDITORS and str(args.get("command") or "").casefold() == "view"+++def _member_manifest_match(path: str, members: Sequence[str]) -> str | None:+ normalized = _normalize_path(path).casefold()+ for member in members:+ suffix = f"{member.casefold()}/skill.md"+ if normalized == suffix or normalized.endswith("/" + suffix):+ return member+ return None+++@dataclass(frozen=True)+class _ManifestRead:+ """One read of a declared member's ``SKILL.md``: the operand as written and its shell command."""++ member: str+ operand: str+ command: int = 0+++@dataclass(frozen=True)+class _ManifestReads:+ reads: tuple[_ManifestRead, ...] = ()+ # Joiner before each shell command of the call (empty for a file-read tool).+ joiners: tuple[str, ...] = ()+++def _manifest_reads(fn_base: str, args: Mapping[str, Any], members: Sequence[str], *, is_mcp: bool) -> _ManifestReads:+ """The declared members whose ``SKILL.md`` this call reads, in order (every one in a chained shell command)."""+ if not members:+ return _ManifestReads()+ if fn_base in _STR_REPLACE_EDITORS:+ reads_file = _is_editor_view(fn_base, args)+ else:+ reads_file = fn_base in _READ_TOOLS or (is_mcp and bool(_READ_VERB_RE.search(fn_base)))+ found: list[_ManifestRead] = []+ joiners: list[str] = []+ if reads_file:+ for path in _path_args(args):+ member = _member_manifest_match(path, members)+ if member is not None:+ found.append(_ManifestRead(member, path))+ elif fn_base in _SHELL_TOOLS and not is_mcp:+ for text in _shell_texts(fn_base, args):+ if "skill.md" not in text.casefold():+ continue+ for command, joiner in _shell_chain(text):+ for operand in _manifest_operands(command):+ member = _member_manifest_match(operand, members)+ if member is not None:+ found.append(_ManifestRead(member, operand, len(joiners)))+ joiners.append(joiner)+ return _ManifestReads(tuple(found), tuple(joiners))+++def _failed_operands(observation: str, operands: Iterable[str]) -> set[str]:+ """Operands that a shell error line in ``observation`` names as unreadable."""+ lines = [+ line.casefold()+ for line in observation[:_MAX_OBSERVATION_CHARS].splitlines()+ if any(phrase in line.casefold() for phrase in _READ_FAILURE_PHRASES)+ ]+ if not lines:+ return set()+ return {operand for operand in operands if operand and any(operand.casefold() in line for line in lines)}+++def _commands_that_ran(joiners: Sequence[str], failed: set[int]) -> list[bool]:+ """Which commands of a shell list ran: ``a && b`` skips ``b`` when ``a`` failed, ``a || b`` when it did not."""+ ran: list[bool] = []+ status_ok = True+ for index, joiner in enumerate(joiners):+ run = not ((joiner == "&&" and not status_ok) or (joiner == "||" and status_ok))+ ran.append(run)+ if run:+ status_ok = index not in failed+ return ran+++def _resolve_manifest_reads(manifest: _ManifestReads, observation: str | None) -> list[tuple[_ManifestRead, bool]]:+ """``(read, failed)`` for each read that ran, in order; a read skipped by a failed ``&&`` part is dropped."""+ if not manifest.reads:+ return []+ if not manifest.joiners:+ return [(read, False) for read in manifest.reads]+ failed_operands = _failed_operands(observation or "", {read.operand for read in manifest.reads})+ failed_commands = {read.command for read in manifest.reads if read.operand in failed_operands}+ ran = _commands_that_ran(manifest.joiners, failed_commands)+ return [+ (read, read.operand in failed_operands)+ for read in manifest.reads+ if read.command >= len(ran) or ran[read.command]+ ]+++def _norm_namespace(value: str) -> str:+ return re.sub(r"[^a-z0-9]+", "-", value.casefold()).strip("-")+++def _plugin_namespaces(declared: Mapping[str, Sequence[str]]) -> tuple[str, ...]:+ """Casefolded names of the plugin under evaluation (its Claude Code namespace)."""+ names = declared.get(DECLARED_PLUGIN) or ()+ return tuple(dict.fromkeys(name.strip().casefold() for name in names if isinstance(name, str) and name.strip()))+++def _foreign_namespace(name: str, namespaces: Sequence[str]) -> bool:+ """Whether ``<namespace>:<name>`` names another plugin's component (unknown plugin name: never)."""+ namespace, sep, _ = name.rpartition(":")+ if not sep or not namespaces:+ return False+ return _norm_namespace(namespace) not in {_norm_namespace(item) for item in namespaces}+++def _component_name(name: str, kind: str, declared: Mapping[str, Sequence[str]]) -> tuple[str, bool]:+ """``(name, foreign)``: a declared component's own spelling when the name is this plugin's.++ ``release-kit:version-bump`` and ``version-bump`` both become the declared+ ``version-bump``, so one component is one identity. A name under another+ plugin's namespace stays as written and is marked foreign.+ """+ if _foreign_namespace(name, _plugin_namespaces(declared)):+ return name, True+ short = name.rpartition(":")[2] if ":" in name else name+ folded = short.casefold()+ for member in declared.get(kind) or ():+ if isinstance(member, str) and member.casefold() == folded:+ return member, False+ return name, False+++def _declared_command(name: str, declared: Mapping[str, Sequence[str]]) -> bool:+ """Whether a ``Skill`` tool name is a declared plugin command (``<plugin>:<command>`` or bare).++ Claude Code runs plugin commands through its ``Skill`` tool. A name that is+ also a declared member skill stays a skill; another plugin's name is never+ this plugin's command.+ """+ commands = {command.casefold() for command in declared.get(COMPONENT_COMMAND) or ()}+ if not name or not commands or _foreign_namespace(name, _plugin_namespaces(declared)):+ return False+ candidates = _name_candidates(name)+ skills = {skill.casefold() for skill in declared.get(COMPONENT_SKILL) or ()}+ return candidates[-1] in commands and not any(candidate in skills for candidate in candidates)+++def _component_ident(+ kind: str,+ name: str,+ fn: str,+ tool_label: str,+ *,+ persist: bool = True,+ declared: Mapping[str, Sequence[str]] | None = None,+ wrapper_skills: Sequence[str] = (),+ **extra: Any,+) -> _Ident:+ foreign = False+ if declared is not None and name:+ name, foreign = _component_name(name, kind, declared)+ label = f"{_IDENTITY_PREFIX[kind]}:{name}" if name else _IDENTITY_PREFIX[kind]+ ident = _Ident(+ label=label,+ kind=kind,+ name=name,+ fn=fn,+ tool_label=tool_label,+ persist_name=persist,+ foreign=foreign,+ namespaces=_plugin_namespaces(declared or {}),+ **extra,+ )+ if kind == COMPONENT_SKILL and wrapper_skills and _is_wrapper(ident, wrapper_skills, declared or {}):+ ident = replace(ident, wrapper=True)+ return ident+++def _persistable_name(name: str, members: Sequence[str]) -> bool:+ """Whether an argument-derived component name may be persisted (identifier-shaped or declared)."""+ if not name or (_PERSISTABLE_NAME_RE.fullmatch(name) and "://" not in name):+ return True+ folded = name.casefold()+ return any(member.casefold() == folded for member in members)+++def _mcp_ident(fn: str, mcp_names: _McpNames, *, agent: str) -> _Ident | None:+ """The MCP identity of a call named ``fn``, labeled ``mcp__<server>__<tool>`` (see :meth:`_McpNames.identity`)."""+ identity = mcp_names.identity(fn, agent=agent)+ if identity is None:+ return None+ server, tool = identity+ canonical = f"mcp__{server}__{tool}" if tool else f"mcp__{server}"+ return _Ident(+ label=canonical,+ kind=COMPONENT_MCP,+ name=server,+ fn=fn,+ server=server,+ tool=tool,+ tool_label=canonical,+ )+++def _builtin_agent(name: str, agent: str, declared: Mapping[str, Sequence[str]]) -> bool:+ """Whether a bare subagent name reaches ``agent``'s own built-in agent, though a declared plugin agent has it."""+ folded = name.casefold()+ if ":" in folded or folded not in _BUILTIN_AGENTS.get(agent.casefold(), ()):+ return False+ return any(isinstance(item, str) and item.casefold() == folded for item in declared.get(COMPONENT_SUBAGENT) or ())+++def _identities(+ fn: str,+ fn_base: str,+ mcp: _Ident | None,+ args: Mapping[str, Any],+ declared: Mapping[str, Sequence[str]],+ *,+ subagent_aliases: Mapping[str, str] | None = None,+ wrapper_skills: Sequence[str] = (),+ manifest: Sequence[tuple[_ManifestRead, bool]] | None = None,+ agent: str = "",+ lsp_extensions: Mapping[str, str] | None = None,+) -> list[_Ident]:+ """The identities of one call. ``manifest`` is the call's resolved ``SKILL.md`` reads, if known.++ ``agent`` names the harness, whose own agents a bare subagent name may reach.+ ``lsp_extensions`` maps a file extension to the staged LSP server that serves it.+ """+ low = fn.casefold()+ idents: list[_Ident] = []+ named = {"declared": declared, "wrapper_skills": wrapper_skills}+ if low in _SKILL_TOOLS:+ name = _first_string(args, ("skill", "name", "command"))+ if _declared_command(name.lstrip("/"), declared):+ name = name.lstrip("/")+ persist = _persistable_name(name, declared.get(COMPONENT_COMMAND) or ())+ idents.append(_component_ident(COMPONENT_COMMAND, name, fn, fn, persist=persist, **named))+ else:+ persist = _persistable_name(name, declared.get(COMPONENT_SKILL) or ())+ idents.append(_component_ident(COMPONENT_SKILL, name, fn, fn, persist=persist, **named))+ elif low in _SUBAGENT_TOOLS:+ # Claude Code 2.1.29x's Agent tool names the subagent in ``type`` (read for these tools only).+ name = _first_string(args, ("subagent_type", "subagent", "agent", "agent_name", "agent_type", "type"))+ # A harness that renamed a plugin agent when staging it calls it by the staged name.+ alias = (subagent_aliases or {}).get(name.casefold())+ persist = _persistable_name(name, ())+ if alias is None and _builtin_agent(name, agent, declared):+ idents.append(_component_ident(COMPONENT_SUBAGENT, name, fn, fn, persist=persist, builtin=True))+ else:+ name = alias or name+ idents.append(_component_ident(COMPONENT_SUBAGENT, name, fn, fn, persist=persist, **named))+ elif low in _COMMAND_TOOLS:+ words = _first_string(args, ("command", "name")).split()+ name = words[0].lstrip("/") if words else ""+ idents.append(_component_ident(COMPONENT_COMMAND, name, fn, fn, persist=_persistable_name(name, ()), **named))+ elif low in _LSP_TOOLS and lsp_extensions and (lsp := _lsp_ident(fn, args, lsp_extensions)) is not None:+ idents.append(lsp)+ if mcp is not None:+ idents.append(mcp)+ if not any(ident.kind in {COMPONENT_SKILL, COMPONENT_COMMAND} for ident in idents):+ if manifest is None:+ reads = _manifest_reads(fn_base, args, declared.get(COMPONENT_SKILL) or (), is_mcp=mcp is not None)+ manifest = [(read, False) for read in reads.reads]+ # One identity per member, in first-read order; it failed only when every read of it failed.+ members: dict[str, bool] = {}+ for read, failed in manifest:+ members[read.member] = members.get(read.member, True) and failed+ for position, (member, failed) in enumerate(members.items()):+ idents.append(+ _component_ident(+ COMPONENT_SKILL,+ member,+ fn,+ f"{fn}:skill-md-read",+ via_read=True,+ position=position,+ failed=failed,+ **named,+ )+ )+ if mcp is None:+ # The tool itself is a plain identity too, so ``Bash`` or ``Read`` refs see a call that also+ # loaded a skill, without that skill answering to the tool's name.+ aliases = _TOOL_ALIASES.get(fn_base, ())+ action = args.get("action_type") if fn_base == "web_search_call" else None+ if isinstance(action, str):+ # A fetch is not a ``WebSearch`` decoy, nor a search a ``WebFetch`` one.+ aliases = _WEB_ACTION_ALIASES.get(action.casefold(), aliases)+ idents.append(_Ident(label=fn, kind=None, name=fn, fn=fn, tool_label=fn, aliases=aliases))+ return idents+++def _trajectory_agent(trajectory: Mapping[str, Any]) -> str:+ agent = trajectory.get("agent")+ name = agent.get("name") if isinstance(agent, Mapping) else None+ return name.strip()[:_MAX_LABEL_CHARS] if isinstance(name, str) else ""+++def _step_tool_calls(step: Any) -> list[Mapping[str, Any]]:+ if not isinstance(step, Mapping) or not isinstance(step.get("tool_calls"), list):+ return []+ return [item for item in step["tool_calls"][: _MAX_CALLS + 1] if isinstance(item, Mapping)]+++def _expanded_tool_calls(raw: Mapping[str, Any], outer_id: str, mcp_call_servers: Mapping[str, str]) -> list[Any]:+ """One raw tool call as normalized calls: a Codex ``exec`` wrapper becomes the calls it made."""+ name = _tool_name(raw)+ server = mcp_call_servers.get(outer_id) if outer_id else None+ if server and name and not name.casefold().startswith(_MCP_PREFIX):+ # The harness log names the server this bare MCP tool name came from (Codex).+ name = f"mcp__{server}__{name}"[:_MAX_LABEL_CHARS]+ prepared = {**raw, "function_name": name, "arguments": _arguments(raw)}+ try:+ return normalize_tool_call(prepared)+ except (TypeError, ValueError, RecursionError):+ return [prepared]+++def _claimable_results(+ tool_call: Mapping[str, Any], results: Sequence[_Result], outer_id: str, *, call_count: int+) -> list[_Result]:+ """The step results this call may claim: an unwrapped inner call only when the normalizer proved it owns them."""+ status = tool_call.get("_atif_observation_status")+ if status is None or status == MAPPED_OUTER_EXEC_OBSERVATION:+ return _results_for_call(results, outer_id, call_count=call_count)+ return []+++def _identify_call(+ tool_call: Mapping[str, Any],+ correlated: Sequence[_Result],+ declared: Mapping[str, Sequence[str]],+ mcp_names: _McpNames,+ *,+ seq: int,+ step_index: int,+ agent: str,+ step_cwd: str | None,+ root_cwd: str | None,+ subagent_aliases: Mapping[str, str] | None,+ wrapper_skills: Sequence[str],+ lsp_extensions: Mapping[str, str] | None = None,+) -> _Call:+ """A normalized tool call with its identities and outcome (its subagent fields are filled in later)."""+ fn = str(tool_call.get("function_name") or "")[:_MAX_LABEL_CHARS]+ args = tool_call.get("arguments")+ args = args if isinstance(args, dict) else {}+ mcp = _mcp_ident(fn, mcp_names, agent=agent)+ fn_base = _base_tool_name(mcp.tool if mcp is not None and mcp.tool else fn)+ manifest = _manifest_reads(fn_base, args, declared.get(COMPONENT_SKILL) or (), is_mcp=mcp is not None)+ named = {+ "subagent_aliases": subagent_aliases,+ "wrapper_skills": wrapper_skills,+ "agent": agent,+ "lsp_extensions": lsp_extensions,+ }+ unresolved = [(read, False) for read in manifest.reads]+ call = _Call(+ seq=seq,+ step_index=step_index,+ fn=fn,+ fn_base=fn_base,+ args=args,+ idents=_identities(fn, fn_base, mcp, args, declared, manifest=unresolved, **named),+ mcp=mcp,+ cwd=_call_cwd(args, step_cwd, root_cwd),+ )+ call.observation, call.succeeded = _outcome(correlated, shell=call.is_shell, content_read=call.is_content_read)+ if call.is_shell and manifest.reads:+ # Each SKILL.md read of a shell list has its own outcome: its own error line, or a+ # failed ``&&`` part before it (then it never ran).+ resolved = _resolve_manifest_reads(manifest, call.observation)+ if resolved != unresolved:+ call.idents = _identities(fn, fn_base, mcp, args, declared, manifest=resolved, **named)+ call.missing_tool = _says_missing_tool(call.observation, call.succeeded)+ call.opener = _window_opener(call, declared)+ if call.is_shell:+ call.side_idents = _side_channel_idents(fn_base, args, declared)+ if _writes_a_file(call):+ # Codex runs its file tool as a shell ``apply_patch``, so that call is ``apply_patch`` too.+ aliases = (*_SHELL_WRITE_ALIASES, "apply_patch") if _runs_apply_patch(call) else _SHELL_WRITE_ALIASES+ call.idents = [+ replace(ident, aliases=(*ident.aliases, *aliases)) if ident.kind is None else ident+ for ident in call.idents+ ]+ return call+++class _ExtractedCalls(list[_Call]):+ """Normalized calls and the input limits that prevent a complete negative claim."""++ def __init__(self) -> None:+ super().__init__()+ self.limits: set[str] = set()+++def _executed_trajectory(trajectory: Mapping[str, Any]) -> tuple[Mapping[str, Any], set[str]]:+ """Flatten only executed child references, preserving distinct ownership scopes.++ File references are resolved by the host adapter. Direct pure callers keep+ unresolved references explicit; this traversal performs no filesystem I/O.+ """+ flattened: list[Mapping[str, Any]] = []+ limits: set[str] = set()+ scopes: dict[int, str] = {}+ owners: dict[int, tuple[str, str] | None] = {}++ def visit(doc: Mapping[str, Any], owner: tuple[str, str] | None, depth: int, active: frozenset[int]) -> None:+ identity = id(doc)+ if identity in active or depth > 16 or len(scopes) >= 65 or len(flattened) >= _MAX_STEPS:+ limits.add("subagent_graph")+ return+ if identity in scopes:+ if owners[identity] != owner:+ owners[identity] = None+ limits.add("subagent_ownership")+ return+ scope = "" if not scopes else f"child:{len(scopes)}"+ scopes[identity], owners[identity] = scope, owner+ if doc.get("continued_trajectory_ref"):+ limits.add("trajectory_references")+ embedded = doc.get("subagent_trajectories", [])+ children: dict[str, Mapping[str, Any]] = {}+ if not isinstance(embedded, list):+ limits.add("subagent_graph")+ embedded = []+ if len(embedded) > 64:+ limits.add("subagent_graph")+ for child in embedded[:64]:+ child_id = child.get("trajectory_id") if isinstance(child, Mapping) else None+ if not isinstance(child_id, str) or not child_id or child_id in children:+ limits.add("subagent_graph")+ else:+ children[child_id] = child+ steps = doc.get("steps")+ if not isinstance(steps, list):+ limits.add("subagent_graph")+ return+ if len(steps) > _MAX_STEPS:+ limits.add("steps")+ for step in steps[:_MAX_STEPS]:+ if len(flattened) >= _MAX_STEPS:+ limits.add("steps")+ break+ if not isinstance(step, Mapping) or step.get("is_copied_context") is True:+ continue+ extra = {**(step.get("extra") or {})} if isinstance(step.get("extra"), Mapping) else {}+ extra.update(+ _plugin_signal_scope=scope,+ _plugin_signal_document=identity,+ _plugin_signal_agent=_trajectory_agent(doc),+ )+ if scope:+ extra.update(is_sidechain=True, cwd=extra.get("cwd") or _trajectory_cwd(doc))+ flattened.append({**step, "extra": extra})+ observation = step.get("observation")+ results = observation.get("results") if isinstance(observation, Mapping) else None+ for result in results[:_MAX_STEP_RESULTS] if isinstance(results, list) else ():+ refs = result.get("subagent_trajectory_ref") if isinstance(result, Mapping) else None+ if refs is None:+ continue+ if not isinstance(refs, list):+ limits.add("subagent_references")+ continue+ if len(refs) > 64:+ limits.add("subagent_graph")+ for ref in refs[:64]:+ child_id = ref.get("trajectory_id") if isinstance(ref, Mapping) else None+ child = children.get(child_id) if isinstance(child_id, str) else None+ if child is None:+ limits.add("subagent_references")+ continue+ source = result.get("source_call_id")+ parent = (scope, source) if isinstance(source, str) and source else None+ visit(child, parent, depth + 1, active | {identity})++ visit(trajectory, None, 0, frozenset())+ for step in flattened:+ extra = step["extra"]+ extra["_plugin_signal_owner"] = owners[extra.pop("_plugin_signal_document")]+ return {**trajectory, "steps": flattened}, limits+++def _extract_calls(+ trajectory: Mapping[str, Any],+ declared: Mapping[str, Sequence[str]],+ mcp_names: _McpNames,+ mcp_call_servers: Mapping[str, str] | None = None,+ subagent_aliases: Mapping[str, str] | None = None,+ wrapper_skills: Sequence[str] = (),+ lsp_extensions: Mapping[str, str] | None = None,+) -> _ExtractedCalls | None:+ steps = trajectory.get("steps")+ if not isinstance(steps, list):+ return None+ agent = _trajectory_agent(trajectory)+ root_cwd = _trajectory_cwd(trajectory)+ named = {"subagent_aliases": subagent_aliases, "wrapper_skills": wrapper_skills, "lsp_extensions": lsp_extensions}+ calls = _ExtractedCalls()+ if len(steps) > _MAX_STEPS:+ calls.limits.add("steps")+ # The parent's latest subagent call: a sidechain run that follows it is that subagent's work,+ # unless the parent's call result names the subagent's ``agentId`` (see ``_spawned_agent_id``).+ last_spawn: list[_Ident] = []+ spawned_by: dict[str, list[_Ident]] = {}+ spawn_by_agent: dict[str, list[_Ident]] = {}+ spawn_by_call: dict[tuple[str, str], list[_Ident]] = {}+ runs = 0+ previous_sidechain = False+ for step_index, step in enumerate(steps[:_MAX_STEPS]):+ if not isinstance(step, Mapping):+ continue+ extra = step.get("extra")+ scope = extra.get("_plugin_signal_scope", "") if isinstance(extra, Mapping) else ""+ sidechain = isinstance(extra, Mapping) and extra.get("is_sidechain") is True+ chain = ""+ if sidechain:+ # Concurrent subagents are told apart by Harbor's ``agent_id``; without it, each run of+ # consecutive sidechain steps is one subagent.+ agent_id = extra.get("agent_id") if isinstance(extra, Mapping) else None+ if not previous_sidechain:+ runs += 1+ known = isinstance(agent_id, str) and bool(agent_id)+ chain = scope or (f"agent:{agent_id}"[:_MAX_LABEL_CHARS] if known else f"run:{runs}")+ spawner = spawn_by_agent.get(str(agent_id)[:_MAX_LABEL_CHARS]) if known else None+ if scope:+ spawner = spawn_by_call.get(extra.get("_plugin_signal_owner"), [])+ if not spawner:+ calls.limits.add("subagent_ownership")+ spawned_by.setdefault(chain, list(last_spawn if spawner is None else spawner))+ previous_sidechain = sidechain+ raw_calls = _step_tool_calls(step)+ if isinstance(step.get("tool_calls"), list) and len(step["tool_calls"]) > _MAX_CALLS:+ calls.limits.add("calls")+ if not raw_calls:+ continue+ observation = step.get("observation")+ raw_results = observation.get("results") if isinstance(observation, Mapping) else None+ if isinstance(raw_results, list) and len(raw_results) > _MAX_STEP_RESULTS:+ calls.limits.add("results")+ results = _observations(step)+ step_cwd = extra.get("cwd") if isinstance(extra, Mapping) and isinstance(extra.get("cwd"), str) else None+ for raw in raw_calls:+ outer_id = str(raw.get("tool_call_id") or raw.get("id") or "")+ call_servers = {} if scope else mcp_call_servers or {}+ for tool_call in _expanded_tool_calls(raw, outer_id, call_servers):+ if len(calls) >= _MAX_CALLS:+ calls.limits.add("calls")+ return calls+ call = _identify_call(+ tool_call,+ _claimable_results(tool_call, results, outer_id, call_count=len(raw_calls)),+ declared,+ mcp_names,+ seq=len(calls),+ step_index=step_index,+ agent=extra.get("_plugin_signal_agent") or agent if isinstance(extra, Mapping) else agent,+ step_cwd=step_cwd,+ root_cwd=root_cwd,+ **named,+ )+ call.sidechain = sidechain+ call.chain = chain+ call.spawner = list(spawned_by.get(chain, [])) if sidechain else []+ if any(ident.kind == COMPONENT_SUBAGENT for ident in call.idents):+ spawn_by_call[(scope, outer_id)] = [+ ident for ident in call.idents if ident.kind == COMPONENT_SUBAGENT+ ]+ if not sidechain and any(ident.kind == COMPONENT_SUBAGENT for ident in call.idents):+ last_spawn = [ident for ident in call.idents if ident.kind == COMPONENT_SUBAGENT]+ spawned_id = _spawned_agent_id(step, outer_id)+ if spawned_id:+ spawn_by_agent[spawned_id] = last_spawn+ calls.append(call)+ return calls+++def _writes_a_file(call: _Call) -> bool:+ """Whether a shell call writes a file (a redirect, ``tee``, ``cp``, ``sed -i``, interpreter code...).++ Writes to devices such as ``/dev/null`` do not count.+ """+ return any(path and not path.startswith("/dev/") for path in call.shell_paths.writes)+++def _runs_apply_patch(call: _Call) -> bool:+ """Whether a shell call runs ``apply_patch`` on a patch (``apply_patch <<'PATCH'``, as Codex does)."""+ return any(+ _APPLY_PATCH_HEADER_RE.search(text)+ and any(+ _APPLY_PATCH_COMMAND_RE.fullmatch(word.rsplit("/", 1)[-1])+ for command, _ in _shell_chain(text)+ for word in command+ )+ for text in _shell_texts(call.fn_base, call.args)+ )+++_AGENT_ID_RE = re.compile(r'"agentId"\s*:\s*"([A-Za-z0-9_.:-]{1,128})"')+_METADATA_MARKER = "[metadata] "+++def _spawned_agent_id(step: Mapping[str, Any], call_id: str) -> str | None:+ """The ``agentId`` of the subagent a parent ``Agent``/``Task`` call started, when its result says it.++ Claude Code returns the subagent's ``agentId`` in the call's result, and+ Harbor keeps it in ``tool_result_metadata.tool_use_result`` and in the+ ``[metadata]`` line of the result text. Sidechain steps carry the same id+ as ``agent_id``, so each subagent's calls are credited to the call that+ started it, even when the parent starts several subagents at once.+ """+ observation = step.get("observation")+ results = observation.get("results") if isinstance(observation, Mapping) else None+ if not call_id or not isinstance(results, list):+ return None+ for result in results[:_MAX_STEP_RESULTS]:+ if not isinstance(result, Mapping) or str(result.get("source_call_id") or "") != call_id:+ continue+ extra = result.get("extra")+ metadata = extra.get("tool_result_metadata") if isinstance(extra, Mapping) else None+ use = metadata.get("tool_use_result") if isinstance(metadata, Mapping) else None+ agent_id = use.get("agentId") if isinstance(use, Mapping) else None+ if isinstance(agent_id, str) and agent_id.strip():+ return agent_id.strip()[:_MAX_LABEL_CHARS]+ content = result.get("content")+ texts = [content] if isinstance(content, str) else _content_parts(content)+ for text in texts:+ at = text.rfind(_METADATA_MARKER) if isinstance(text, str) else -1+ if at < 0 or (at and text[at - 1] != "\n"):+ continue+ line = text[at : at + _MAX_OBSERVATION_CHARS].split("\n", 1)[0]+ match = _AGENT_ID_RE.search(line)+ if match:+ return match.group(1)[:_MAX_LABEL_CHARS]+ return None+++def _trajectory_cwd(trajectory: Mapping[str, Any]) -> str | None:+ """The task's working directory: the agent's recorded cwd, else the first step's."""+ agent = trajectory.get("agent")+ extra = agent.get("extra") if isinstance(agent, Mapping) else None+ if isinstance(extra, Mapping):+ if isinstance(extra.get("cwd"), str) and extra["cwd"].startswith("/"):+ return _normalize_path(extra["cwd"])+ cwds = extra.get("cwds")+ if isinstance(cwds, list) and cwds and isinstance(cwds[0], str) and cwds[0].startswith("/"):+ return _normalize_path(cwds[0])+ for step in (trajectory.get("steps") or [])[:_MAX_STEPS]:+ extra = step.get("extra") if isinstance(step, Mapping) else None+ if isinstance(extra, Mapping) and isinstance(extra.get("cwd"), str) and extra["cwd"].startswith("/"):+ return _normalize_path(extra["cwd"])+ return None+++def _call_cwd(args: Mapping[str, Any], step_cwd: str | None, root_cwd: str | None) -> str | None:+ for value in (args.get("workdir"), args.get("cwd"), step_cwd):+ if isinstance(value, str) and value.startswith("/"):+ return _normalize_path(value)+ return root_cwd+++def _declared_component(ident: _Ident, declared: Mapping[str, Sequence[str]]) -> bool:+ folded = ident.name.casefold()+ return any(isinstance(name, str) and name.casefold() == folded for name in declared.get(ident.kind or "") or ())+++def _window_opener(call: _Call, declared: Mapping[str, Sequence[str]]) -> _Ident | None:+ """The declared skill or command a call switched the agent to, or ``None``.++ Only a call that activated exactly one declared plugin skill or command,+ and did not fail, opens a window. A load of a skill this plugin does not+ declare (a built-in, a made-up ``mcp_tool``, another plugin's skill), the+ wrapper, a failed load, and one read of several ``SKILL.md`` files (which+ of them is the agent following?) neither open nor close a window.+ """+ if call.succeeded is False:+ return None+ openers = {+ ident.label: ident+ for ident in call.idents+ if ident.kind in {COMPONENT_SKILL, COMPONENT_COMMAND}+ and not ident.failed+ and not ident.wrapper+ and not ident.foreign+ and _declared_component(ident, declared)+ }+ return next(iter(openers.values())) if len(openers) == 1 else None+++# A JSON-RPC MCP ``tools/call`` written into a shell command (quotes may be backslash-escaped).+_JSONRPC_TOOL_CALL_RE = re.compile(+ r'\\?"method\\?"\s*:\s*\\?"tools/call\\?"[^\n]{0,512}?\\?"name\\?"\s*:\s*\\?"([A-Za-z0-9_.:/-]{1,128})\\?"'+)+_SCRIPT_SUFFIXES = (".py", ".sh", ".bash", ".js", ".mjs", ".cjs", ".ts", ".rb", ".pl")+++def _side_channel_idents(fn_base: str, args: Mapping[str, Any], declared: Mapping[str, Sequence[str]]) -> list[_Ident]:+ """Plugin components a shell call used without their own tool.++ * An MCP ``tools/call`` sent as JSON-RPC through the shell to a declared+ server's program (``printf '{..."method":"tools/call"...}' | /x/reltools``)+ is that server's tool; with one declared server, it is that server.+ * Running a file inside a member skill's folder (``python3+ <skills>/version-bump/scripts/bump.py``) uses that skill.+ """+ idents: list[_Ident] = []+ servers = [name for name in declared.get(COMPONENT_MCP) or () if isinstance(name, str) and name]+ members = [name for name in declared.get(COMPONENT_SKILL) or () if isinstance(name, str) and name]+ for text in _shell_texts(fn_base, args):+ chain = _shell_chain(text)+ tools = _JSONRPC_TOOL_CALL_RE.findall(text) if "tools/call" in text and servers else []+ if tools:+ words = {word.rsplit("/", 1)[-1].casefold() for command, _ in chain for word in command}+ named = [name for name in servers if words & set(_server_spellings(name))]+ server = named[0] if len(named) == 1 else (servers[0] if len(servers) == 1 else None)+ for tool in dict.fromkeys(tools):+ if server is not None:+ label = f"mcp__{server}__{tool}"+ idents.append(+ _Ident(+ label=label,+ kind=COMPONENT_MCP,+ name=server,+ fn=label,+ server=server,+ tool=tool,+ tool_label=f"shell:{label}",+ )+ )+ for command, _ in chain:+ programs = [command[0]]+ if command[0].rsplit("/", 1)[-1].casefold() in _INTERPRETERS | _SHELLS:+ # The script is the interpreter's first operand, unless inline code or a module comes first.+ for token in command[1:]:+ if token in _INLINE_CODE_FLAGS or token == "-m":+ break+ if not token.startswith("-"):+ programs.append(token)+ break+ for program in programs:+ path = _normalize_path(program)+ if not path.casefold().endswith(_SCRIPT_SUFFIXES):+ continue+ for member in members:+ if f"/{member.casefold()}/" in f"/{path.casefold()}" and not path.casefold().endswith("skill.md"):+ idents.append(+ _component_ident(+ COMPONENT_SKILL, member, fn_base, f"{fn_base}:skill-script", declared=declared+ )+ )+ return list({ident.label: ident for ident in idents}.values())+++def _says_missing_tool(observation: str | None, succeeded: bool | None) -> bool:+ """Whether a failed result says the called tool (or skill, command, agent) does not exist."""+ if succeeded is not False or not observation:+ return False+ head = observation[:_FAILURE_SCAN_CHARS].casefold()+ return any(marker in head for marker in _MISSING_TOOL_MARKERS)+++def _prompt_texts(trajectory: Mapping[str, Any]) -> list[str]:+ """The task prompt: the user and system messages before the agent's first step.++ Every harness puts the task (and context such as ``AGENTS.md``) in front of+ the agent's first step. A user or system step after that is not the task:+ Claude Code injects one after each ``Skill`` call that repeats the agent's+ own Skill arguments, and a subagent's prompt (a sidechain step) is written+ by the parent agent. Counting those as prompt would reject a value the+ agent really carried from the producer.+ """+ texts: list[str] = []+ steps = trajectory.get("steps")+ if not isinstance(steps, list):+ return texts+ for step in steps[:_MAX_STEPS]:+ source = step.get("source") if isinstance(step, Mapping) else None+ if not isinstance(source, str):+ continue+ if source not in {"user", "system"}:+ break+ extra = step.get("extra")+ if isinstance(extra, Mapping) and extra.get("is_sidechain") is True:+ continue+ message = step.get("message")+ text = message if isinstance(message, str) else _content_text(message)+ if text:+ texts.append(text[:_MAX_OBSERVATION_CHARS])+ return texts+++# =============================================================================+# Ref matching+# =============================================================================+++@dataclass(frozen=True)+class _Ref:+ raw: str+ kind: str | None+ pattern: str # a casefolded ``fnmatch`` glob++ @cached_property+ def glob(self) -> re.Pattern[str]:+ """The compiled glob, as :func:`fnmatch.fnmatchcase` compiles it."""+ return re.compile(fnmatch.translate(self.pattern))+++def _parse_ref(raw: str) -> _Ref:+ text = raw.strip()+ prefix, sep, rest = text.partition(":")+ kind = _REF_PREFIXES.get(prefix.strip().casefold()) if sep else None+ if kind is not None and rest.strip():+ pattern = rest.strip().casefold()+ return _Ref(raw=text, kind=kind, pattern=pattern.lstrip("/") if kind == COMPONENT_COMMAND else pattern)+ return _Ref(raw=text, kind=None, pattern=text.casefold())+++def _match_names(values: Iterable[str]) -> tuple[str, ...]:+ """The non-empty ``values``, casefolded: refs match case-insensitively."""+ return tuple(value.casefold() for value in values if value)+++def _name_candidates(name: str) -> list[str]:+ low = name.casefold()+ return [low, low.rsplit(":", 1)[-1]] if ":" in low else [low]+++def _ref_matches(ref: _Ref, ident: _Ident) -> bool:+ """Whether ``ref`` names ``ident`` (see the module docs).++ The generated wrapper skill is the harness's way in, not a component, so+ no ref matches it (not even a ``Skill:<plugin>*`` glob). An untyped ref+ matches a component by its label or name, never through the tool that+ carried it: the underlying tool (``Read``, ``Bash``...) is a separate+ plain identity of the same call. A harness's own agent called by a+ declared agent's bare name is not that agent, so no ref matches it either.+ """+ if ident.wrapper or ident.builtin:+ return False+ if ref.kind is None:+ names = ident.untyped_names+ elif ref.kind == ident.kind:+ names = ident.typed_names+ else:+ return False+ return any(ref.glob.match(name) for name in names)+++def _mcp_ref_candidates(ident: _Ident) -> list[str]:+ """``<server>`` and ``<server>/<tool>``/``<server>__<tool>``, in the declared and the harness spelling.++ Claude Code writes the declared ``docs.v2`` as ``docs_v2``; both spellings+ match, alone or with a tool.+ """+ server = ident.server or ""+ spellings = list(dict.fromkeys((server.casefold(), _norm_server(server))))+ candidates = list(spellings)+ if ident.tool:+ tool = ident.tool.casefold()+ for spelling in spellings:+ candidates += [f"{spelling}/{tool}", f"{spelling}__{tool}"]+ return candidates+++def _refs(values: Sequence[str]) -> list[_Ref]:+ return [_parse_ref(value) for value in values]+++def _call_matches(refs: Sequence[_Ref], call: _Call) -> bool:+ return any(_ref_matches(ref, ident) for ref in refs for ident in call.idents)+++def _ref_label(values: Sequence[str]) -> str:+ return _safe_text(" | ".join(values))+++def _handoff_calls(+ producer: Sequence[_Ref], consumer: Sequence[_Ref], calls: Sequence[_Call]+) -> tuple[list[_Call], list[_Call]]:+ """``(producer calls, consumer calls)``: calls matching a side directly or running in its window.++ A window opens at a call that switched to a declared skill or command (see+ :func:`_window_opener`) and lasts until a call switches to another one.+ The consumer's own activation does not close the producer's window: the+ agent may look up the consumer's instructions before it saves the+ producer's output. Any other switch closes it; any switch, including to the+ producer, closes the consumer's. A subagent's own calls (sidechain steps)+ keep their own windows, start from the parent's, never change the parent's,+ and also belong to a side that names the subagent that started them.+ """+ states: dict[str, list[bool]] = {"": [False, False]}+ producer_calls: list[_Call] = []+ consumer_calls: list[_Call] = []+ for call in calls:+ # A subagent run starts from the parent's window at the time it first acts.+ state = states.setdefault(call.chain, list(states[""]))+ if call.opener is not None:+ is_producer = any(_ref_matches(ref, call.opener) for ref in producer)+ is_consumer = any(_ref_matches(ref, call.opener) for ref in consumer)+ state[0] = is_producer or (state[0] and is_consumer)+ state[1] = is_consumer+ spawned = [ident for ident in call.spawner if call.sidechain]+ if state[0] or _call_matches(producer, call) or any(_ref_matches(ref, i) for ref in producer for i in spawned):+ producer_calls.append(call)+ if state[1] or _call_matches(consumer, call) or any(_ref_matches(ref, i) for ref in consumer for i in spawned):+ consumer_calls.append(call)+ return producer_calls, consumer_calls+++def _is_wrapper(ident: _Ident, wrapper_skills: Sequence[str], declared: Mapping[str, Sequence[str]]) -> bool:+ """Whether a skill identity is the generated wrapper, not a member skill.++ The wrapper package is ``<plugin>-plugin-eval``, and that name always+ means the wrapper. Its ``SKILL.md`` is named after the plugin, so the+ plugin name means the wrapper too, but only when no declared member skill+ or command has that name: a ``frontend-design`` plugin may ship a+ ``frontend-design`` skill, and that skill is a component. A ``SKILL.md``+ read names its folder, so only the package name marks a read as the wrapper.+ """+ if ident.kind != COMPONENT_SKILL:+ return False+ names = set(_name_candidates(ident.name))+ members = {+ name.casefold()+ for kind in (COMPONENT_SKILL, COMPONENT_COMMAND)+ for name in declared.get(kind) or ()+ if isinstance(name, str)+ }+ for wrapper in wrapper_skills:+ folded = wrapper.casefold()+ if folded not in names:+ continue+ if folded.endswith(_WRAPPER_PACKAGE_SUFFIX) or (folded not in members and not ident.via_read):+ return True+ return False+++# =============================================================================+# Graders+# =============================================================================+++def _ratio(numerator: float, denominator: float) -> float | None:+ return round(numerator / denominator, 4) if denominator else None+++def _unavailable_kinds(declared: Mapping[str, Sequence[str]], agent: str = "") -> frozenset[str]:+ """Component types this arm cannot carry: declared by the plugin but not staged here, or not on this harness."""+ kinds = {kind for kind in declared.get(DECLARED_UNSTAGED) or () if isinstance(kind, str)}+ if agent.casefold() in _NO_PLUGIN_AGENT_HARNESSES:+ kinds |= {COMPONENT_SUBAGENT, COMPONENT_COMMAND}+ return frozenset(kinds)+++def _component_names(+ declared: Mapping[str, Sequence[str]],+ kinds: Iterable[str] = (COMPONENT_SKILL, COMPONENT_SUBAGENT, COMPONENT_COMMAND),+) -> list[str]:+ """Casefolded declared skill, subagent, and command names, bare and under the plugin namespace.++ The subagents and commands an arm does not declare (``DECLARED_UNSTAGED_NAMES``) count too, so an+ untyped ref that names one is a ref to that component in every arm.+ """+ namespaces = _plugin_namespaces(declared)+ hidden = [entry.partition(":") for entry in declared.get(DECLARED_UNSTAGED_NAMES) or () if isinstance(entry, str)]+ names: list[str] = []+ for kind in kinds:+ for name in (*(declared.get(kind) or ()), *(name for hidden_kind, _, name in hidden if hidden_kind == kind)):+ if isinstance(name, str) and name:+ names.append(name.casefold())+ names.extend(f"{namespace}:{name.casefold()}" for namespace in namespaces)+ return names+++def _names_by_kind(declared: Mapping[str, Sequence[str]], unavailable: frozenset[str]) -> dict[str, list[str]]:+ """Declared skill, subagent, and command names per type (see :func:`_ref_unavailable`).++ Empty when the arm carries every type, since then no ref is skipped.+ """+ names = {kind: _component_names(declared, [kind]) for kind in _ROUTING_KINDS} if unavailable else {}+ names[COMPONENT_MCP] = list(declared.get(COMPONENT_MCP) or ())+ names[DECLARED_UNSTAGED_NAMES] = [+ value[4:]+ for value in declared.get(DECLARED_UNSTAGED_NAMES) or ()+ if isinstance(value, str) and value.startswith("mcp:")+ ]+ return names+++def _ref_unavailable(ref: _Ref, unavailable: frozenset[str], names_by_kind: Mapping[str, Sequence[str]]) -> bool:+ """Whether ``ref`` names only component types this arm cannot carry.++ A typed ref names its own type. An untyped ref names the type of each+ declared skill, subagent, or command it matches (``names_by_kind``), so a+ bare ``reviewer`` that matches only a declared subagent is a subagent ref;+ one that matches none names a tool.+ """+ mcp_pattern = (+ ref.pattern+ if ref.kind == COMPONENT_MCP+ else (ref.pattern[5:] if ref.kind is None and ref.pattern.startswith("mcp__") else None)+ )+ if mcp_pattern:+ server_pattern = re.split(r"/|__", mcp_pattern, maxsplit=1)[0]++ def matches(name: str) -> bool:+ return fnmatch.fnmatchcase(name.casefold(), server_pattern) or fnmatch.fnmatchcase(+ _norm_server(name), server_pattern+ )++ if any(matches(name) for name in names_by_kind.get(DECLARED_UNSTAGED_NAMES, ())) and not any(+ matches(name) for name in names_by_kind.get(COMPONENT_MCP, ())+ ):+ return True+ if ref.kind is not None:+ return ref.kind in unavailable+ kinds = {kind for kind, names in names_by_kind.items() if any(ref.glob.match(name) for name in names)}+ return bool(kinds) and kinds <= unavailable+++def _routing_ref(ref: _Ref, component_names: Sequence[str]) -> bool:+ """Whether a ref is about routing (check 15) rather than tool selection (check 22).++ Typed ``Skill:``/``Agent:``/``Command:`` refs are routing and ``MCP:`` refs+ are tool selection. An untyped ref is routing when it names a declared+ skill, subagent, or command; otherwise it names a tool.+ """+ if ref.kind is not None:+ return ref.kind in _ROUTING_KINDS+ return any(ref.glob.match(name) for name in component_names)+++def _grade_selection(+ calls: Sequence[_Call],+ spec: Mapping[str, Any],+ *,+ routing: bool,+ component_names: Sequence[str],+ unavailable: frozenset[str],+ declared: Mapping[str, Sequence[str]],+) -> dict[str, Any]:+ def _family(values: Sequence[str]) -> list[str]:+ return [value for value in values if _routing_ref(_parse_ref(value), component_names) == routing]++ expected = _family(spec.get("expected_tools", []))+ acceptable = _family(spec.get("acceptable_tools", []))+ decoys = _family(spec.get("decoy_tools", []))+ expected_refs, acceptable_refs, decoy_refs = _refs(expected), _refs(acceptable), _refs(decoys)+ # Expected refs to a component type this arm cannot carry are reported, not counted against recall.+ names_by_kind = _names_by_kind(declared, unavailable)+ missing = [_ref_unavailable(ref, unavailable, names_by_kind) for ref in expected_refs]+ applicable = [ref for ref, absent in zip(expected_refs, missing, strict=True) if not absent]+ skipped = [value for value, absent in zip(expected, missing, strict=True) if absent]+ allowed = [*expected_refs, *acceptable_refs]+ all_refs = [*allowed, *decoy_refs]++ def _in_family(ident: _Ident) -> bool:+ return (ident.kind in _ROUTING_KINDS) == routing and not ident.failed and not ident.wrapper++ # label -> (identity, credited): a call to a tool that does not exist is a wrong choice, never a hit.+ called: dict[str, list[Any]] = {}+ # Labels a decoy ref matched: a wrong choice even when an allowed ref matches them too, as a shell call+ # that wrote a file matches both an acceptable ``Bash`` and a decoy ``Write``.+ decoyed: set[str] = set()+ decoy_calls = 0+ for call in calls:+ idents = [ident for ident in call.idents if _in_family(ident)]+ matched = {ident.label for ident in idents if any(_ref_matches(ref, ident) for ref in decoy_refs)}+ if matched:+ decoy_calls += 1+ decoyed |= matched+ credited = not call.missing_tool+ for ident in idents:+ if ident.kind is None and not any(_ref_matches(ref, ident) for ref in all_refs):+ continue+ entry = called.get(ident.label)+ if entry is not None:+ entry[1] = entry[1] or credited+ if not set(ident.aliases) <= set(entry[0].aliases):+ # One label, several calls: a shell call that wrote a file also answers to ``Write``.+ entry[0] = replace(entry[0], aliases=tuple(dict.fromkeys((*entry[0].aliases, *ident.aliases))))+ elif len(called) < _MAX_CALLED:+ called[ident.label] = [ident, credited]++ precise = sum(+ 1+ for label, (ident, credited) in called.items()+ if credited and label not in decoyed and any(_ref_matches(ref, ident) for ref in allowed)+ )+ # Precision is undefined when nothing in scope was called (a miss, not a wrong choice).+ precision = _ratio(precise, len(called)) if called else None+ recall: float | None = None+ f1: float | None = None+ if applicable:+ hits = sum(+ 1 for ref in applicable if any(credited and _ref_matches(ref, ident) for ident, credited in called.values())+ )+ recall = _ratio(hits, len(applicable))+ if precision is None or recall is None or precision + recall == 0:+ f1 = 0.0+ else:+ f1 = round(2 * precision * recall / (precision + recall), 4)+ scored = bool(applicable or acceptable or decoys)+ return {+ "expected": [_safe_text(item) for item in expected],+ "acceptable": [_safe_text(item) for item in acceptable],+ "decoys": [_safe_text(item) for item in decoys],+ "skipped": [_safe_text(item) for item in skipped],+ "called": [_safe_text(ident.persisted_label) for ident, _ in called.values()],+ "precision": precision if scored else None,+ "recall": recall,+ "f1": f1,+ "decoy_calls": decoy_calls,+ "status": STATUS_SCORED if scored else STATUS_NOT_APPLICABLE,+ }+++def _grade_routing(+ calls: Sequence[_Call],+ spec: Mapping[str, Any],+ *,+ declared: Mapping[str, Sequence[str]] | None = None,+ agent: str = "",+) -> dict[str, Any]:+ """Check 15: precision/recall/F1 of skill, subagent, and command routing.++ Only routing refs count (typed ``Skill:``/``Agent:``/``Command:`` refs and+ untyped refs naming a declared component), and only skill, subagent, and+ command activations are routing choices, so MCP calls never dilute it. The+ wrapper skill, a ``SKILL.md`` read that failed by itself, and a call the+ harness says does not exist never earn credit; the last still counts as a+ wrong choice. ``precision`` is ``None`` when nothing was routed.+ """+ declared_map = declared or {}+ return _grade_selection(+ calls,+ spec,+ routing=True,+ component_names=_component_names(declared_map),+ unavailable=_unavailable_kinds(declared_map, agent),+ declared=declared_map,+ )+++def _grade_tool_selection(+ calls: Sequence[_Call],+ spec: Mapping[str, Any],+ *,+ declared: Mapping[str, Sequence[str]] | None = None,+ agent: str = "",+) -> dict[str, Any]:+ """Check 22: precision/recall/F1 of MCP and plain tool selection.++ Only tool refs count (``MCP:`` refs and untyped refs that do not name a+ declared component); ``called`` holds every distinct MCP identity plus any+ plain tool a ref names, so ordinary unlisted tools (``Read``, ``Bash``...)+ do not dilute precision, and skills never do. A call to a tool that does+ not exist is a wrong choice. ``precision`` is ``None`` when nothing in+ scope was called; ``recall``/``f1`` are ``None`` when nothing applicable is+ expected. The wrapper skill is marked when calls are extracted.+ """+ declared_map = declared or {}+ return _grade_selection(+ calls,+ spec,+ routing=False,+ component_names=_component_names(declared_map),+ unavailable=_unavailable_kinds(declared_map, agent),+ declared=declared_map,+ )+++def _lookup(args: Any, name: str) -> Any:+ if isinstance(args, Mapping) and name in args:+ return args[name]+ current = args+ for part in name.split("."):+ if isinstance(current, Mapping) and part in current:+ current = current[part]+ elif isinstance(current, list) and part.isascii() and part.isdigit() and int(part) < len(current):+ current = current[int(part)]+ else:+ return _MISSING+ return current+++def _json_equal(left: Any, right: Any) -> bool:+ if isinstance(left, bool) or isinstance(right, bool):+ return isinstance(left, bool) and isinstance(right, bool) and left == right+ if _is_number(left) and _is_number(right):+ # int/float comparison is exact in Python and never overflows.+ return bool(left == right)+ if isinstance(left, Mapping) and isinstance(right, Mapping):+ return left.keys() == right.keys() and all(_json_equal(left[key], right[key]) for key in left)+ if isinstance(left, list) and isinstance(right, list):+ return len(left) == len(right) and all(_json_equal(a, b) for a, b in zip(left, right, strict=True))+ return type(left) is type(right) and left == right+++def _json_type(value: Any) -> str:+ if value is None:+ return "null"+ if isinstance(value, bool):+ return "boolean"+ if isinstance(value, int):+ return "integer"+ if isinstance(value, float):+ return "number"+ if isinstance(value, str):+ return "string"+ if isinstance(value, list):+ return "array"+ if isinstance(value, Mapping):+ return "object"+ return type(value).__name__+++def _type_ok(value: Any, expected: str) -> bool:+ actual = _json_type(value)+ if expected == "number":+ return actual in {"integer", "number"}+ if expected == "integer":+ return actual == "integer" or (actual == "number" and float(value).is_integer())+ return actual == expected+++def _preview(value: Any) -> str:+ """Short redacted rendering of a dataset-authored value (never of agent arguments).++ Token shapes the general redactor does not know (GitHub ``ghp_``/``gho_``+ and ``github_pat_`` tokens, among others) are masked first.+ """+ from skillevaluator.tier3.eval_core.secret_redaction import redact_secrets_in_log_line++ text = value if isinstance(value, str) else (_bounded_json_text(value, _MAX_ARGS_TEXT_CHARS) or _json_type(value))+ return _safe_text(redact_secrets_in_log_line(text[: _MAX_PREVIEW_CHARS * 4 + 256]), _MAX_PREVIEW_CHARS)+++def _describe(value: Any) -> str:+ """Type and size of an agent argument value: failure details never echo the value itself.++ Redaction cannot recognize every credential shape (URL userinfo, PEM+ bodies, ``mysql -p...``), so argument values stay out of persisted signals.+ """+ if value is None or isinstance(value, bool):+ return json.dumps(value)+ if isinstance(value, str | list):+ return f"{_json_type(value)}(len={len(value)})"+ if isinstance(value, Mapping):+ return f"object(keys={len(value)})"+ return _json_type(value)+++def _regex_search(pattern: str, value: Any, budget: list[float]) -> bool | str:+ """Search a bounded subject under a deadline; returns whether it matched, or why it was not checked.++ Patterns are dataset-authored and subjects agent-written, so a pattern that+ backtracks catastrophically (``^(\\w+\\s?)+$``, ``[a-z]*[a-z0-9]*!``...) must+ fail the check rather than hang collection. ``re`` cannot be interrupted, so+ matching uses the ``regex`` engine (``re``-compatible syntax) with a+ per-check timeout drawn from the trial's shared ``budget`` of seconds.+ """+ text = value if isinstance(value, str) else _bounded_json_text(value, _MAX_PATTERN_SUBJECT_CHARS)+ if text is None or len(text) > _MAX_PATTERN_SUBJECT_CHARS:+ return "value exceeds the pattern-check size limit"+ timeout = min(_PATTERN_TIMEOUT_SECONDS, budget[0])+ if timeout <= 0:+ return "pattern check timed out"+ started = time.monotonic()+ try:+ return regex.search(pattern, text, timeout=timeout) is not None+ except TimeoutError:+ return "pattern check timed out"+ except Exception:+ # A pattern the engine cannot run fails the check: besides regex.error and+ # ValueError it can raise KeyError, RuntimeError..., and grading never raises.+ return False+ finally:+ budget[0] -= time.monotonic() - started+++def _schema_errors(+ value: Any, schema: Mapping[str, Any], path: str, errors: list[tuple[str, str]], depth: int, budget: list[float]+) -> None:+ if len(errors) >= _MAX_SCHEMA_ERRORS_PER_CALL or depth > MAX_SCHEMA_DEPTH + 1:+ return+ if "type" in schema:+ types = schema["type"]+ type_list = [types] if isinstance(types, str) else list(types)+ if not any(_type_ok(value, item) for item in type_list):+ errors.append((path, f"expected type {'|'.join(type_list)}, got {_json_type(value)}"))+ return+ if "enum" in schema and not any(_json_equal(value, item) for item in schema["enum"]):+ errors.append((path, f"value {_describe(value)} is not one of the allowed values"))+ if "pattern" in schema and isinstance(value, str):+ found = _regex_search(str(schema["pattern"]), value, budget)+ if isinstance(found, str):+ errors.append((path, found))+ elif not found:+ errors.append((path, f"value {_describe(value)} does not match the schema pattern"))+ if _is_number(value):+ if _is_number(schema.get("minimum")) and value < schema["minimum"]:+ errors.append((path, f"value is below minimum {_preview(schema['minimum'])}"))+ if _is_number(schema.get("maximum")) and value > schema["maximum"]:+ errors.append((path, f"value is above maximum {_preview(schema['maximum'])}"))+ if isinstance(value, Mapping):+ for name in schema.get("required") or ():+ if len(errors) >= _MAX_SCHEMA_ERRORS_PER_CALL:+ return+ if name not in value:+ errors.append((f"{path}.{name}", "required property is missing"))+ properties = schema.get("properties")+ if isinstance(properties, Mapping):+ for name, sub in properties.items():+ if name in value and isinstance(sub, Mapping):+ _schema_errors(value[name], sub, f"{path}.{name}", errors, depth + 1, budget)+++def _string_values(value: Any, *, limit: int = _MAX_STRING_LEAVES) -> list[str]:+ """String values inside a JSON object or list (bounded, depth first), never key names."""+ found: list[str] = []+ stack = [value]+ while stack and len(found) < limit:+ current = stack.pop()+ if isinstance(current, str):+ found.append(current)+ elif isinstance(current, Mapping):+ stack.extend(reversed(list(current.values())[:limit]))+ elif isinstance(current, list):+ stack.extend(reversed(current[:limit]))+ return found+++def _argument_failures(+ args: Mapping[str, Any], rule: Mapping[str, Any], budget: list[float]+) -> list[tuple[str, str, str]]:+ failures: list[tuple[str, str, str]] = []+ for name in rule.get("required") or ():+ if _lookup(args, name) is _MISSING:+ failures.append((name, "required", "argument is missing"))+ schema = rule.get("schema")+ if isinstance(schema, Mapping):+ schema_errors: list[tuple[str, str]] = []+ _schema_errors(args, schema, "$", schema_errors, 1, budget)+ for path, message in schema_errors:+ failures.append((path.removeprefix("$.") if path != "$" else "$", "schema", message))+ for name, expected in (rule.get("equals") or {}).items():+ actual = _lookup(args, name)+ if actual is _MISSING:+ failures.append((name, "equals", "argument is missing"))+ elif not _json_equal(actual, expected):+ failures.append((name, "equals", f"expected {_preview(expected)}, got {_describe(actual)}"))+ for name, needle in (rule.get("contains") or {}).items():+ actual = _lookup(args, name)+ if actual is _MISSING:+ failures.append((name, "contains", "argument is missing"))+ continue+ if isinstance(actual, str):+ found = needle in actual+ elif isinstance(actual, list):+ found = any(+ item == needle or (isinstance(item, str) and needle in item) for item in actual[:_MAX_CONTAINS_ITEMS]+ )+ elif isinstance(actual, Mapping):+ # The object's string values, never its key names.+ found = any(needle in text for text in _string_values(actual))+ else:+ found = needle in (_bounded_json_text(actual, _MAX_ARGS_TEXT_CHARS) or "")+ if not found:+ failures.append((name, "contains", f"value {_describe(actual)} does not contain {_preview(needle)}"))+ for name, pattern in (rule.get("pattern") or {}).items():+ actual = _lookup(args, name)+ if actual is _MISSING:+ failures.append((name, "pattern", "argument is missing"))+ continue+ found = _regex_search(pattern, actual, budget)+ if isinstance(found, str):+ failures.append((name, "pattern", found))+ elif not found:+ failures.append((name, "pattern", f"value {_describe(actual)} does not match {_preview(pattern)}"))+ return failures+++#: Case-spec key the private observer fills with the probed servers' tool input schemas+#: (``{server: {tool: schema}}``, from the private host probe).+MCP_INPUT_SCHEMAS_KEY = "mcp_input_schemas"+_CALL_FAILED_DETAIL = "the call failed (the harness refused it or the server rejected it), so its arguments do not pass"+++def _server_input_schemas(spec: Mapping[str, Any] | None) -> dict[str, dict[str, Mapping[str, Any]]]:+ raw = spec.get(MCP_INPUT_SCHEMAS_KEY) if isinstance(spec, Mapping) else None+ schemas: dict[str, dict[str, Mapping[str, Any]]] = {}+ for server, tools in list(raw.items())[:64] if isinstance(raw, Mapping) else ():+ if isinstance(server, str) and isinstance(tools, Mapping):+ kept = {+ tool: schema for tool, schema in tools.items() if isinstance(tool, str) and isinstance(schema, Mapping)+ }+ if kept:+ schemas[server] = kept+ return schemas+++def _call_input_schema(+ call: _Call, schemas: Mapping[str, Mapping[str, Mapping[str, Any]]]+) -> tuple[str, Mapping[str, Any]] | None:+ """``(label, schema)`` when the server this MCP call went to published an input schema for its tool."""+ if call.mcp is None or not call.mcp.server:+ return None+ schema = schemas.get(call.mcp.server, {}).get(call.mcp.tool or "")+ return (call.mcp.persisted_label, schema) if schema is not None else None+++def _grade_arguments(+ calls: Sequence[_Call], spec: Mapping[str, Any], schemas: Mapping[str, Mapping[str, Mapping[str, Any]]]+) -> dict[str, Any]:+ """Check ``tool_arguments`` rules, and the servers' own input schemas, against every matching call.++ ``checked``/``passed`` count (call, rule) pairs. A rule that matched no+ call counts as one checked pair that did not pass, with a ``not_called``+ failure, so a rate can never be 100% while a rule's tool was never called.+ A call that failed (the harness refused it, or the server rejected it)+ never passes, with a ``call_failed`` failure. When the probe recorded a+ server's ``inputSchema`` for a tool (``schemas``, from ``MCP_INPUT_SCHEMAS_KEY``),+ every call to that tool is also checked against it, as an ``input_schema``+ pair. Pattern checks share one time budget; a check that runs out of time is+ a failure, never a pass.+ """+ rules = spec.get("tool_arguments", [])+ failures: list[dict[str, str]] = []+ checked = 0+ passed = 0+ budget = [_PATTERN_BUDGET_SECONDS]++ def _record(tool: str, arg: str, rule_name: str, detail: str) -> None:+ if len(failures) < _MAX_FAILURES:+ failures.append(+ {+ "tool": _safe_text(tool),+ "arg": _safe_text(arg, MAX_ARGUMENT_NAME_CHARS),+ "rule": rule_name,+ "detail": _safe_text(detail, _MAX_DETAIL_CHARS),+ }+ )++ def _grade(label: str, call: _Call, call_failures: list[tuple[str, str, str]]) -> None:+ nonlocal checked, passed+ checked += 1+ if call.succeeded is False:+ call_failures = [*call_failures, ("", "call_failed", _CALL_FAILED_DETAIL)]+ if not call_failures:+ passed += 1+ for arg, rule_name, detail in call_failures:+ _record(label, arg, rule_name, detail)++ for rule in rules:+ refs = _refs([rule["tool"]])+ matched = [call for call in calls if _call_matches(refs, call)]+ if not matched:+ checked += 1+ _record(rule["tool"], "", "not_called", "no call matched this rule")+ continue+ for call in matched:+ label = next(+ (ident.persisted_label for ident in call.idents if any(_ref_matches(r, ident) for r in refs)),+ call.fn,+ )+ _grade(label, call, _argument_failures(call.args, rule, budget))+ schema_checked = False+ for call in calls if schemas else ():+ found = _call_input_schema(call, schemas)+ if found is None:+ continue+ label, schema = found+ schema_errors: list[tuple[str, str]] = []+ _schema_errors(call.args, schema, "$", schema_errors, 1, budget)+ _grade(+ label,+ call,+ [(path.removeprefix("$.") if path != "$" else "$", "input_schema", msg) for path, msg in schema_errors],+ )+ schema_checked = True+ return {+ "checked": checked,+ "passed": passed,+ "failures": failures,+ "status": STATUS_SCORED if rules or schema_checked else STATUS_NOT_APPLICABLE,+ }+++def top_argument_failures(signals: Iterable[Any], limit: int = 5) -> list[dict[str, Any]]:+ """The most frequent ``arguments.failures`` rows over per-trial signals, with exact counts.++ Every recorded row counts (a trial keeps at most 50), keyed by tool,+ argument, and rule; ties sort by tool and argument.+ """+ counts: dict[tuple[str, str, str], dict[str, Any]] = {}+ for item in signals:+ arguments = item.get("arguments") if isinstance(item, Mapping) else None+ rows = arguments.get("failures") if isinstance(arguments, Mapping) else None+ for failure in rows[:_MAX_FAILURES] if isinstance(rows, list) else ():+ if not isinstance(failure, Mapping):+ continue+ key = (+ str(failure.get("tool") or "")[:200],+ str(failure.get("arg") or "")[:200],+ str(failure.get("rule") or "")[:200],+ )+ entry = counts.setdefault(+ key, {"tool": key[0], "arg": key[1], "rule": key[2], "detail": str(failure.get("detail") or "")[:300]}+ )+ entry["count"] = entry.get("count", 0) + 1+ return sorted(counts.values(), key=lambda row: (-row["count"], row["tool"], row["arg"]))[:limit]+++_OUTCOME_KEYS = ("total", "succeeded", "failed", "unknown")+++def _outcome_counts() -> dict[str, Any]:+ return dict.fromkeys(_OUTCOME_KEYS, 0)+++def _count_outcome(bucket: dict[str, Any], succeeded: bool | None) -> None:+ bucket["total"] += 1+ if succeeded is True:+ bucket["succeeded"] += 1+ elif succeeded is False:+ bucket["failed"] += 1+ else:+ bucket["unknown"] += 1+++def _with_success_rate(bucket: dict[str, Any]) -> dict[str, Any]:+ """``bucket`` with ``success_rate``, ``succeeded / (succeeded + failed)``: an unknown outcome counts as neither."""+ bucket["success_rate"] = _ratio(bucket["succeeded"], bucket["succeeded"] + bucket["failed"])+ return bucket+++def _server_bucket(by_server: dict[str, dict[str, Any]], server: str) -> dict[str, Any]:+ """The outcome counts, tool labels and per-tool counts of ``server``, added empty the first time."""+ return by_server.setdefault(server, {**_outcome_counts(), "tools": [], "by_tool": {}})+++def _add_server_tool(bucket: dict[str, Any], label: str) -> None:+ if label not in bucket["tools"] and len(bucket["tools"]) < _MAX_SERVER_TOOLS:+ bucket["tools"].append(label)+++def _grade_mcp_calls(calls: Sequence[_Call]) -> dict[str, Any]:+ """Outcome counts across every MCP call (any server), with per-server and per-tool breakdowns.++ ``success_rate`` is ``succeeded / (succeeded + failed)``: calls whose outcome+ is unknown are reported separately rather than counted as either. Each+ server keeps its tool labels (``tools``) and the same counts per tool+ (``by_tool``, at most 64 tools).+ """+ totals = _outcome_counts()+ by_server: dict[str, dict[str, Any]] = {}+ for call in calls:+ if call.mcp is None:+ continue+ _count_outcome(totals, call.succeeded)+ bucket = _server_bucket(by_server, _safe_text(call.mcp.server or ""))+ _count_outcome(bucket, call.succeeded)+ label = _safe_text(call.mcp.label)+ _add_server_tool(bucket, label)+ if label in bucket["tools"]:+ _count_outcome(bucket["by_tool"].setdefault(label, _outcome_counts()), call.succeeded)+ for bucket in by_server.values():+ _with_success_rate(bucket)+ for tool_counts in bucket["by_tool"].values():+ _with_success_rate(tool_counts)+ return _with_success_rate({**totals, "by_server": by_server})+++# Why an ``expected_order`` edge did not pass.+ORDER_NEVER_CALLED = "never_called"+ORDER_BEFORE_NEVER_CALLED = "before_never_called"+ORDER_AFTER_NEVER_CALLED = "after_never_called"+ORDER_SAME_CALL = "same_call"+ORDER_SAME_STEP = "same_step"+ORDER_REVERSED = "reversed"+_UNORDERED_REASONS = frozenset({ORDER_SAME_CALL, ORDER_SAME_STEP})+++def _uses(refs: Sequence[_Ref], call: _Call) -> bool:+ """Whether ``call`` really used something ``refs`` names: a matching identity that did not fail.++ A shell side channel (an MCP tool called through the shell, a skill's+ script run directly) is a use too.+ """+ return any(+ call.ident_ok(ident) and _ref_matches(ref, ident) for ref in refs for ident in (*call.idents, *call.side_idents)+ )+++def _attempts(refs: Sequence[_Ref], call: _Call) -> bool:+ """Whether ``call`` tried to use something ``refs`` names, whatever the outcome.++ A ``SKILL.md`` read that failed by itself is not an attempt (the file was+ not the component), and neither is one read of several ``SKILL.md`` files:+ looking through instructions together is not choosing one of them.+ """+ batched = sum(ident.via_read and not ident.failed for ident in call.idents) > 1+ return any(+ _ref_matches(ref, ident)+ for ref in refs+ for ident in (*call.idents, *call.side_idents)+ if not ident.failed and not (batched and ident.via_read)+ )+++def _first_use(refs: Sequence[_Ref], calls: Sequence[_Call]) -> _Call | None:+ return next((call for call in calls if _uses(refs, call)), None)+++def _side_unavailable(+ refs: Sequence[_Ref], unavailable: frozenset[str], names_by_kind: Mapping[str, Sequence[str]]+) -> bool:+ """Every alternative names a component type this arm cannot carry (see :func:`_ref_unavailable`)."""+ return bool(refs) and all(_ref_unavailable(ref, unavailable, names_by_kind) for ref in refs)+++def _edge_reason(before: Sequence[_Ref], after: Sequence[_Ref], calls: Sequence[_Call]) -> str:+ """``""`` when the edge holds, else why not.++ The first use of ``before`` (any alternative) must come in an earlier step+ than the first use of some ``after`` alternative. Each ``after``+ alternative is checked on its own, so an early look at one alternative+ (reading its ``SKILL.md`` while planning) does not hide the real work done+ in order through another. Calls in one step were issued together (parallel+ calls), and reads in one shell command were loaded together, so neither+ has an order.+ """+ first_before = _first_use(before, calls)+ firsts = [_first_use([ref], calls) for ref in after]+ found = [call for call in firsts if call is not None]+ if first_before is None:+ return ORDER_NEVER_CALLED if not found else ORDER_BEFORE_NEVER_CALLED+ if not found:+ return ORDER_AFTER_NEVER_CALLED+ if any(call.step_index > first_before.step_index for call in found):+ return ""+ if any(call.seq == first_before.seq for call in found):+ return ORDER_SAME_CALL+ if any(call.step_index == first_before.step_index for call in found):+ return ORDER_SAME_STEP+ return ORDER_REVERSED+++def _grade_order(+ calls: Sequence[_Call],+ spec: Mapping[str, Any],+ *,+ declared: Mapping[str, Sequence[str]] | None = None,+ agent: str = "",+) -> dict[str, Any]:+ """Check ``expected_order`` precedence edges by first use (see :func:`_edge_reason`).++ Failed calls and failed ``SKILL.md`` reads are not uses. Every edge that+ does not hold is listed in ``violated`` with its ``reason``+ (``never_called``, ``before_never_called``, ``after_never_called``,+ ``same_call``, ``same_step``, or ``reversed``); ``unordered`` counts the+ ``same_call``/``same_step`` ones. An edge one side of which names only a+ component type this arm cannot carry is listed in ``skipped`` and not+ counted in ``edges``.+ """+ unavailable = _unavailable_kinds(declared or {}, agent)+ names_by_kind = _names_by_kind(declared or {}, unavailable)+ satisfied = 0+ unordered = 0+ counted = 0+ violated: list[dict[str, str]] = []+ skipped: list[dict[str, str]] = []+ for before, after in spec.get("expected_order", []):+ before_refs, after_refs = _refs(before), _refs(after)+ labels = {"before": _ref_label(before), "after": _ref_label(after)}+ if any(_side_unavailable(side, unavailable, names_by_kind) for side in (before_refs, after_refs)):+ skipped.append({**labels, "reason": "this arm cannot carry that component type"})+ continue+ counted += 1+ reason = _edge_reason(before_refs, after_refs, calls)+ if not reason:+ satisfied += 1+ continue+ unordered += reason in _UNORDERED_REASONS+ if len(violated) < MAX_ORDER_EDGES:+ violated.append({**labels, "reason": reason})+ return {+ "edges": counted,+ "satisfied": satisfied,+ "unordered": unordered,+ "violated": violated,+ "skipped": skipped,+ "status": STATUS_SCORED if counted else STATUS_NOT_APPLICABLE,+ }+++_GLOB_CHARS = frozenset("*?[")+# MCP tools whose path argument is a destination they write or remove, so naming a path is not reading it.+# (``upload``/``export``/``put`` send a local file somewhere else: they do read it.)+_LOCAL_WRITE_VERB_RE = re.compile(+ r"(?:^|_|-)(?:write|create|save|dump|store|generate|render|append|overwrite"+ r"|delete|remove|rm|unlink|drop|erase|move|rename|trash)(?:$|_|-)"+)+_INPUT_ARG_KEYS = frozenset(+ {"path", "file", "file_path", "filepath", "filename", "input", "input_path", "input_file", "source", "src", "from"}+)+++def _observed_path_matches(observed: str, artifact: str) -> bool:+ """``observed`` and ``artifact`` are resolved (:func:`_resolve_path`); ``observed`` may be a glob.++ Two absolute paths must be equal. A relative path is only left when its+ directory is unknown; then a path ending with ``/<relative>`` matches.+ """+ if not observed or not artifact:+ return False+ if _GLOB_CHARS & set(observed):+ return fnmatch.fnmatchcase(artifact, observed) or (+ not observed.startswith("/") and fnmatch.fnmatchcase(artifact, "*/" + observed)+ )+ if observed.startswith("/") and artifact.startswith("/"):+ return observed == artifact+ if artifact.startswith("/"):+ return artifact.endswith("/" + observed)+ return observed == artifact or observed.endswith("/" + artifact)+++def _path_matches(observed: str, artifact: str, cwd: str | None = None) -> bool:+ """``artifact`` is resolved; ``observed`` is raw and resolved against ``cwd``."""+ return _observed_path_matches(_resolve_path(observed, cwd), artifact)+++def _patch_written_paths(text: str) -> list[str]:+ """Files an apply_patch body writes: the ``Add File``, ``Update File`` and ``Move to`` headers.++ A file the patch only deletes is not written, and neither is a file it+ moves away: ``*** Update File: a`` followed by ``*** Move to: b`` writes+ only ``b``. A file the patch deletes and adds again is still written.+ """+ written: dict[str, None] = {}+ pending_update = "" # an ``Update File`` path, written unless the next header moves it away+ for match in _APPLY_PATCH_HEADER_RE.finditer(text):+ operation = text[match.start() : match.start(1)]+ path = match.group(1).strip()+ if pending_update and "Move to" not in operation:+ written[pending_update] = None+ pending_update = ""+ if "Update File" in operation:+ pending_update = path+ elif "Delete File" not in operation and path:+ written[path] = None+ if pending_update:+ written[pending_update] = None+ return list(written)+++def _patch_targets(args: Mapping[str, Any]) -> list[str]:+ """Paths an apply_patch tool call writes (see :func:`_patch_written_paths`)."""+ paths: list[str] = []+ for key in _PATCH_BODY_KEYS:+ body = args.get(key)+ if isinstance(body, str):+ paths.extend(_patch_written_paths(body[:_MAX_ARGS_TEXT_CHARS]))+ return paths+++def _call_writes(call: _Call, artifact: str) -> bool:+ """Whether a call that did not fail wrote ``artifact`` (resolved)."""+ if call.succeeded is False:+ return False+ fn_base = call.fn_base+ if fn_base in _WRITE_TOOLS:+ if _is_editor_view(fn_base, call.args):+ return False+ if any(_path_matches(path, artifact, call.cwd) for path in _path_args(call.args)):+ return True+ if fn_base in _PATCH_TOOLS and any(+ _path_matches(path, artifact, call.cwd) for path in _patch_targets(call.args)+ ):+ return True+ # An MCP write tool (``write_file``...) may name its file under another key, such as+ # ``destination``: the MCP rules below still apply to it.+ elif call.is_shell:+ return any(_observed_path_matches(path, artifact) for path in call.shell_paths.writes)+ if call.mcp is None:+ return False+ tool = (call.mcp.tool or "").casefold()+ for key, value in call.args.items():+ if not isinstance(value, str) or not _path_matches(value, artifact, call.cwd):+ continue+ if _WRITE_VERB_RE.search(tool) or str(key).casefold() in _OUTPUT_ARG_KEYS:+ return True+ return False+++def _call_reads(call: _Call, artifact: str) -> bool:+ """Whether a call that did not fail read ``artifact`` (resolved).++ An MCP tool reads it when the tool is not a write or delete tool and the+ path is an input argument (or the tool name says it reads). A subagent+ call that only names the path is not a read: the subagent's own reads are+ its sidechain calls.+ """+ if call.succeeded is False:+ return False+ fn_base = call.fn_base+ if (fn_base in _READ_TOOLS or _is_editor_view(fn_base, call.args)) and any(+ _path_matches(path, artifact, call.cwd) for path in _path_args(call.args)+ ):+ return True+ if call.is_shell:+ # A shell read whose own error line names the file did not read it (``cat: x: No such file``).+ name = artifact.rsplit("/", 1)[-1]+ return any(_observed_path_matches(path, artifact) for path in call.shell_paths.reads) and not _failed_operands(+ call.observation or "", {name}+ )+ if call.mcp is None:+ return False+ tool = (call.mcp.tool or "").casefold()+ if _LOCAL_WRITE_VERB_RE.search(tool):+ return False+ reads_by_name = bool(_READ_VERB_RE.search(tool))+ return any(+ isinstance(value, str)+ and (reads_by_name or str(key).casefold() in _INPUT_ARG_KEYS)+ and _path_matches(value, artifact, call.cwd)+ for key, value in call.args.items()+ )+++def _value_in_text(value: str, text: str) -> bool:+ """``value`` occurs in ``text`` as a whole token: ``AT-48`` is not in ``AT-4821``.++ An occurrence counts unless a letter continues a letter, or a digit a+ digit, across its edge (``1.5.0`` is in ``v1.5.0``; ``abc`` is not in ``abcd``).+ """+ start = text.find(value)+ while start >= 0:+ end = start + len(value)+ before = text[start - 1] if start else ""+ after = text[end] if end < len(text) else ""+ if not (_continues(before, value[0]) or _continues(after, value[-1])):+ return True+ start = text.find(value, start + 1)+ return False+++def _continues(neighbor: str, edge: str) -> bool:+ return bool(neighbor) and ((neighbor.isalpha() and edge.isalpha()) or (neighbor.isdigit() and edge.isdigit()))+++def _decoded_texts(text: str, *, depth: int = 0) -> list[str]:+ """``text`` plus the string values of any JSON in it, recursively (MCP results are JSON in text)."""+ texts = [text]+ if depth >= 3 or len(text) > _MAX_OBSERVATION_CHARS:+ return texts+ start = min((index for index in (text.find("{"), text.find("[")) if index >= 0), default=-1)+ if start < 0:+ return texts+ try:+ decoded, _ = json.JSONDecoder().raw_decode(text[start:])+ except (ValueError, RecursionError):+ return texts+ for leaf in _string_values(decoded):+ texts.extend(_decoded_texts(leaf, depth=depth + 1))+ return texts+++def _value_produced(value: str, call: _Call) -> bool:+ return call.succeeded is not False and any(_value_in_text(value, text) for text in call.result_texts)+++def _value_received(value: str, call: _Call) -> bool:+ """The value is in the consumer's arguments, compared as parsed values (quotes and backslashes are data)."""+ return any(_value_in_text(value, text) for text in call.argument_texts)+++def _only_batched(refs: Sequence[_Ref], calls: Sequence[_Call]) -> bool:+ """Whether ``refs`` were used, but never by a call that opened a window (one read of several SKILL.md files)."""+ used = [call for call in calls if _uses(refs, call)]+ return bool(used) and all(+ call.opener is None and sum(ident.via_read and not ident.failed for ident in call.idents) > 1 for call in used+ )+++def _handoff_value_failure(+ value: str, producer_calls: Sequence[_Call], consumer_calls: Sequence[_Call], prompts: Sequence[str]+) -> str:+ if any(_value_in_text(value, text) for text in prompts):+ return "value also appears in the task prompt, so the handoff cannot be attributed"+ produced = next((call for call in producer_calls if _value_produced(value, call)), None)+ if produced is None:+ return "value was not observed in producer output"+ # A call in a later step is never the producing call itself.+ received = any(call.step_index > produced.step_index and _value_received(value, call) for call in consumer_calls)+ if not received:+ return "value did not reach consumer input after the producer produced it"+ return ""+++def _handoff_artifact_failure(+ artifact: str, producer_calls: Sequence[_Call], consumer_calls: Sequence[_Call], root_cwd: str | None+) -> str:+ target = _resolve_path(artifact, root_cwd)+ written = next((call for call in producer_calls if _call_writes(call, target)), None)+ if written is None:+ return "artifact was not written by the producer"+ # Like a value, a file is only there for the consumer after the step that wrote it.+ if not any(call.step_index > written.step_index and _call_reads(call, target) for call in consumer_calls):+ return "artifact was not read by the consumer after the producer wrote it"+ return ""+++def _grade_handoff(+ calls: Sequence[_Call],+ spec: Mapping[str, Any],+ *,+ prompts: Sequence[str] = (),+ root_cwd: str | None = None,+ declared: Mapping[str, Sequence[str]] | None = None,+ agent: str = "",+) -> dict[str, Any]:+ """Verify each ``handoffs[i]`` carried producer output into consumer input (see module docs).++ A handoff whose producer or consumer names only component types this arm+ cannot carry is listed in ``skipped`` and not checked.+ """+ unavailable = _unavailable_kinds(declared or {}, agent)+ names_by_kind = _names_by_kind(declared or {}, unavailable)+ failures: list[dict[str, str]] = []+ skipped: list[dict[str, str]] = []+ checked = 0+ passed = 0+ for handoff in spec.get("handoffs", []):+ producer_refs = _refs(handoff["producer"])+ consumer_refs = _refs(handoff["consumer"])+ if any(_side_unavailable(side, unavailable, names_by_kind) for side in (producer_refs, consumer_refs)):+ skipped.append(+ {+ "producer": _ref_label(handoff["producer"]),+ "consumer": _ref_label(handoff["consumer"]),+ "reason": "this arm cannot carry that component type",+ }+ )+ continue+ checked += 1+ producer_calls, consumer_calls = _handoff_calls(producer_refs, consumer_refs, calls)+ problems: list[str] = []+ if not any(_uses(producer_refs, call) for call in calls):+ problems.append("producer was not activated")+ if not any(_uses(consumer_refs, call) for call in calls):+ problems.append("consumer was not activated")+ if not problems:+ if handoff.get("value") is not None:+ problem = _handoff_value_failure(handoff["value"], producer_calls, consumer_calls, prompts)+ if problem:+ problems.append(problem)+ if handoff.get("artifact") is not None:+ problem = _handoff_artifact_failure(handoff["artifact"], producer_calls, consumer_calls, root_cwd)+ if problem:+ problems.append(problem)+ if problems and _only_batched(producer_refs, calls):+ problems.append(+ "the producer was only read together with other skills in one call, so no later call is its"+ )+ if not problems:+ passed += 1+ elif len(failures) < _MAX_FAILURES:+ failures.append(+ {+ "producer": _ref_label(handoff["producer"]),+ "consumer": _ref_label(handoff["consumer"]),+ "detail": _safe_text("; ".join(problems), _MAX_DETAIL_CHARS),+ }+ )+ return {+ "checked": checked,+ "passed": passed,+ "failures": failures,+ "skipped": skipped,+ "status": STATUS_SCORED if checked else STATUS_NOT_APPLICABLE,+ }+++def _grade_conflict(+ calls: Sequence[_Call],+ spec: Mapping[str, Any],+ *,+ declared: Mapping[str, Sequence[str]] | None = None,+ agent: str = "",+) -> dict[str, Any]:+ """Each probe passes when ``must_use`` was used and ``must_not_use`` was not tried.++ ``must_use`` needs a use that did not fail (a failed call or a missing+ tool is not a use); ``must_not_use`` counts any attempt, except a failed or+ batched ``SKILL.md`` read (see :func:`_attempts`). Shell side channels+ count on both sides. A probe whose ``must_use`` names only component types+ this arm cannot carry is listed in ``skipped`` and not checked. The check+ is about presence over the whole trial, not which one ran first.+ """+ unavailable = _unavailable_kinds(declared or {}, agent)+ names_by_kind = _names_by_kind(declared or {}, unavailable)+ failures: list[dict[str, str]] = []+ skipped: list[str] = []+ checked = 0+ passed = 0+ for probe in spec.get("conflict_probes", []):+ must_use, must_not_use = _refs(probe["must_use"]), _refs(probe["must_not_use"])+ if _side_unavailable(must_use, unavailable, names_by_kind):+ skipped.append(_safe_text(probe["id"]))+ continue+ checked += 1+ used = any(_uses(must_use, call) and not call.missing_tool for call in calls)+ forbidden = any(_attempts(must_not_use, call) for call in calls)+ if used and not forbidden:+ passed += 1+ continue+ problems = []+ if not used:+ problems.append(f"must_use {_ref_label(probe['must_use'])} was not activated")+ if forbidden:+ problems.append(f"must_not_use {_ref_label(probe['must_not_use'])} was activated")+ if len(failures) < _MAX_FAILURES:+ failures.append(+ {"probe": _safe_text(probe["id"]), "detail": _safe_text("; ".join(problems), _MAX_DETAIL_CHARS)}+ )+ return {+ "checked": checked,+ "passed": passed,+ "failures": failures,+ "skipped": skipped,+ "status": STATUS_SCORED if checked else STATUS_NOT_APPLICABLE,+ }+++def _declared_keys(declared: Mapping[str, Sequence[str]] | None) -> list[tuple[str, str]]:+ keys: dict[tuple[str, str], None] = {}+ for kind in (COMPONENT_SKILL, COMPONENT_MCP, COMPONENT_SUBAGENT, COMPONENT_COMMAND, COMPONENT_LSP):+ for name in (declared or {}).get(kind) or ():+ if isinstance(name, str) and name:+ keys[kind, name] = None+ return list(keys)+++def _ident_components(+ ident: _Ident, mcp_names: _McpNames, by_folded_name: Mapping[tuple[str, str], Sequence[str]]+) -> set[tuple[str, str]]:+ """The declared ``(type, name)`` components that ``ident`` activates (the wrapper skill activates none)."""+ if ident.wrapper or ident.builtin:+ return set()+ if ident.kind == COMPONENT_MCP:+ # ``_McpNames.identity`` already maps a recognizable server to its declared name.+ server = mcp_names.match(ident.server or "")+ return {(COMPONENT_MCP, server)} if server is not None else set()+ # Another plugin's ``<plugin>:<name>`` only answers to its full name, never to this plugin's.+ kind = ident.kind or ""+ return {(kind, name) for candidate in ident.name_candidates for name in by_folded_name.get((kind, candidate), ())}+++def _grade_activation_coverage(+ calls: Sequence[_Call], declared: Mapping[str, Sequence[str]], mcp_names: _McpNames+) -> dict[str, Any]:+ """Declared components exercised, never activated, or whose every activation failed.++ Entries are ``"<type>:<name>"``. The three outcome lists do not overlap: a+ component whose every activation failed (a failed call, or a ``SKILL.md``+ read whose own error line says it failed) is ``unavailable``, not+ ``exercised``. ``mcp_names`` indexes the declared MCP servers.+ """+ keys = _declared_keys(declared)+ # A skill, subagent or command activation names its component case-insensitively.+ by_folded_name: dict[tuple[str, str], list[str]] = {}+ for kind, name in keys:+ if kind != COMPONENT_MCP:+ by_folded_name.setdefault((kind, name.casefold()), []).append(name)+ activated: set[tuple[str, str]] = set()+ worked: set[tuple[str, str]] = set() # activated by at least one identity that did not fail+ for call in calls:+ for ident in call.component_idents:+ components = _ident_components(ident, mcp_names, by_folded_name)+ activated |= components+ if call.ident_ok(ident):+ worked |= components+ declared_labels: list[str] = []+ exercised: list[str] = []+ unverified: list[str] = []+ unavailable: list[str] = []+ for kind, name in keys:+ label = _safe_text(f"{kind}:{name}")+ declared_labels.append(label)+ if (kind, name) not in activated:+ unverified.append(label)+ elif (kind, name) in worked:+ exercised.append(label)+ else:+ unavailable.append(label)+ return {"declared": declared_labels, "exercised": exercised, "unverified": unverified, "unavailable": unavailable}+++def _activations(calls: Sequence[_Call]) -> list[dict[str, Any]]:+ activations: list[dict[str, Any]] = []+ for call in calls:+ for ident in call.component_idents:+ if len(activations) >= _MAX_ACTIVATIONS:+ return activations+ activations.append(+ {+ "type": ident.kind,+ "name": _safe_text(ident.persisted_name),+ "tool": _safe_text(ident.tool_label or ident.fn),+ "server": _safe_text(ident.server) if ident.server is not None else None,+ "step_index": call.step_index,+ "succeeded": False if ident.failed else call.succeeded,+ }+ )+ return activations+++def _with_plugin_names(+ declared: Mapping[str, Sequence[str]], wrapper_skills: Sequence[str]+) -> Mapping[str, Sequence[str]]:+ """``declared`` with the plugin name, taken from the wrapper package name when not given."""+ if declared.get(DECLARED_PLUGIN):+ return declared+ names = _plugin_names_from_wrappers(wrapper_skills)+ return {**declared, DECLARED_PLUGIN: names} if names else declared+++def compute_plugin_signals(+ trajectory: Any,+ case: Mapping[str, Any] | None = None,+ *,+ declared: Mapping[str, Sequence[str]] | None = None,+ wrapper_skills: Sequence[str] = (),+ mcp_call_servers: Mapping[str, str] | None = None,+ subagent_aliases: Mapping[str, str] | None = None,+ lsp_servers: Mapping[str, Sequence[str]] | None = None,+) -> dict[str, Any] | None:+ """Per-trial C3 ``plugin_signals`` for one ATIF trajectory, or ``None`` if unreadable.++ ``case`` is a dataset case entry or a :func:`plugin_case_spec` result; it is+ normalized once here, and invalid fields are ignored. Its+ ``MCP_INPUT_SCHEMAS_KEY`` holds the probed servers' tool input schemas.+ ``declared`` maps ``skill``/``mcp`` (and, in the with-plugin+ arm, ``subagent``/``command``) to the declared component names in this arm.+ ``mcp_call_servers`` maps a tool call id to the MCP server the harness log+ says it went to, for harnesses whose trajectory keeps only the bare tool+ name (Codex). ``subagent_aliases`` maps a staged subagent name (casefolded)+ to the declared name, for harnesses that rename a plugin agent (OpenCode).+ ``lsp_servers`` maps each LSP server staged in this arm (also listed under+ ``declared['lsp']``) to the file extensions its ``extensionToLanguage`` maps.+ """+ if not isinstance(trajectory, Mapping):+ return None+ if not isinstance(trajectory.get("steps"), list):+ return None+ trajectory, graph_limits = _executed_trajectory(trajectory)+ declared_map = _with_plugin_names(declared or {}, wrapper_skills)+ mcp_names = _McpNames(declared_map.get(COMPONENT_MCP) or ())+ calls = _extract_calls(+ trajectory,+ declared_map,+ mcp_names,+ mcp_call_servers,+ subagent_aliases,+ wrapper_skills,+ lsp_extensions=_lsp_extension_index(lsp_servers, declared_map),+ )+ if calls is None:+ return None+ calls.limits.update(graph_limits)+ spec = plugin_case_spec(case)+ # The prompts only matter to rule out a handoff value the consumer could have copied from them.+ has_value = any(handoff.get("value") is not None for handoff in spec.get("handoffs", []))+ agent = _trajectory_agent(trajectory)+ return {+ "evidence_complete": not calls.limits,+ "limits": sorted(calls.limits),+ "activations": _activations(calls),+ "routing": _grade_routing(calls, spec, declared=declared_map, agent=agent),+ "tool_selection": _grade_tool_selection(calls, spec, declared=declared_map, agent=agent),+ "arguments": _grade_arguments(calls, spec, _server_input_schemas(case)),+ "mcp_calls": _grade_mcp_calls(calls),+ "order": _grade_order(calls, spec, declared=declared_map, agent=agent),+ "handoff": _grade_handoff(+ calls,+ spec,+ prompts=_prompt_texts(trajectory) if has_value else (),+ root_cwd=_trajectory_cwd(trajectory),+ declared=declared_map,+ agent=agent,+ ),+ "conflict": _grade_conflict(calls, spec, declared=declared_map, agent=agent),+ "activation_coverage": _grade_activation_coverage(calls, declared_map, mcp_names),+ }+++# =============================================================================+# Per-arm summary+# =============================================================================+++def _mean(values: Sequence[float]) -> float | None:+ return round(sum(values) / len(values), 4) if values else None+++def _scored(signals: Sequence[Mapping[str, Any]], key: str) -> list[Mapping[str, Any]]:+ return [+ block+ for item in signals+ if isinstance(block := item.get(key), Mapping) and block.get("status") == STATUS_SCORED+ ]+++def _int(value: Any) -> int:+ return value if isinstance(value, int) and not isinstance(value, bool) else 0+++def _summarize_selection(signals: Sequence[Mapping[str, Any]], key: str) -> dict[str, Any]:+ """Mean precision/recall/F1 over the trials that have a number; ``decoy_call_rate`` is a share of trials."""+ blocks = _scored(signals, key)++ def _values(name: str) -> list[float]:+ return [float(block[name]) for block in blocks if _is_number(block.get(name))]++ return {+ "n_scored": len(blocks),+ "precision": _mean(_values("precision")),+ "recall": _mean(_values("recall")),+ "f1": _mean(_values("f1")),+ "decoy_calls": sum(_int(block.get("decoy_calls")) for block in blocks),+ # The share of scored trials with at least one decoy call (not a share of calls).+ "decoy_call_rate": _ratio(sum(1 for block in blocks if _int(block.get("decoy_calls")) > 0), len(blocks)),+ "status": STATUS_SCORED if blocks else STATUS_NOT_APPLICABLE,+ }+++_MAX_SUMMARY_EDGES = 20+++def _skipped(signals: Sequence[Mapping[str, Any]], key: str) -> int:+ """How many items the trials skipped because this arm cannot carry their component type (scored or not)."""+ return sum(+ len(skipped)+ for item in signals+ if isinstance(block := item.get(key), Mapping) and isinstance(skipped := block.get("skipped"), list)+ )+++def _summarize_order(signals: Sequence[Mapping[str, Any]]) -> dict[str, Any]:+ """Edge totals, trials with every edge in order, and the most common edges that did not hold, with why."""+ blocks = _scored(signals, "order")+ edges = sum(_int(block.get("edges")) for block in blocks)+ satisfied = sum(_int(block.get("satisfied")) for block in blocks)+ counts: dict[tuple[str, str, str], int] = {}+ for block in blocks:+ for item in block.get("violated") or ():+ if isinstance(item, Mapping):+ key = (str(item.get("before") or ""), str(item.get("after") or ""), str(item.get("reason") or ""))+ counts[key] = counts.get(key, 0) + 1+ top = sorted(counts.items(), key=lambda pair: -pair[1])[:_MAX_SUMMARY_EDGES]+ return {+ "n_scored": len(blocks),+ "edges": edges,+ "satisfied": satisfied,+ "unordered": sum(_int(block.get("unordered")) for block in blocks),+ "satisfaction_rate": _ratio(satisfied, edges),+ # The unit above is edges; this is the trial-level picture.+ "trials_in_order": sum(+ 1 for block in blocks if _int(block.get("edges")) and block.get("satisfied") == block.get("edges")+ ),+ "violated_edges": [+ {"before": before, "after": after, "reason": reason, "trials": count}+ for (before, after, reason), count in top+ ],+ "skipped": _skipped(signals, "order"),+ "status": STATUS_SCORED if blocks else STATUS_NOT_APPLICABLE,+ }+++def _summarize_conflict(signals: Sequence[Mapping[str, Any]]) -> dict[str, Any]:+ """The pooled probe pass rate, plus which probes failed in how many trials (the per-probe view)."""+ summary = _summarize_checked(signals, "conflict")+ counts: dict[str, int] = {}+ for block in _scored(signals, "conflict"):+ for item in block.get("failures") or ():+ if isinstance(item, Mapping) and isinstance(item.get("probe"), str):+ counts[item["probe"]] = counts.get(item["probe"], 0) + 1+ summary["failed_probes"] = [+ {"probe": probe, "trials": count}+ for probe, count in sorted(counts.items(), key=lambda pair: -pair[1])[:_MAX_SUMMARY_EDGES]+ ]+ summary["skipped"] = _skipped(signals, "conflict")+ return summary+++def _summarize_checked(signals: Sequence[Mapping[str, Any]], key: str) -> dict[str, Any]:+ blocks = _scored(signals, key)+ checked = sum(_int(block.get("checked")) for block in blocks)+ passed = sum(_int(block.get("passed")) for block in blocks)+ return {+ "n_scored": len(blocks),+ "checked": checked,+ "passed": passed,+ "pass_rate": _ratio(passed, checked),+ "status": STATUS_SCORED if blocks else STATUS_NOT_APPLICABLE,+ }+++def _summarize_mcp(signals: Sequence[Mapping[str, Any]]) -> dict[str, Any]:+ totals = _outcome_counts()+ by_server: dict[str, dict[str, Any]] = {}+ for item in signals:+ block = item.get("mcp_calls")+ if not isinstance(block, Mapping):+ continue+ for key in _OUTCOME_KEYS:+ totals[key] += _int(block.get(key))+ servers = block.get("by_server")+ if not isinstance(servers, Mapping):+ continue+ for server, counts in servers.items():+ if not isinstance(counts, Mapping):+ continue+ bucket = _server_bucket(by_server, str(server))+ for key in _OUTCOME_KEYS:+ bucket[key] += _int(counts.get(key))+ for tool in counts.get("tools") or ():+ if isinstance(tool, str):+ _add_server_tool(bucket, tool)+ tools = counts.get("by_tool")+ for tool, tool_counts in tools.items() if isinstance(tools, Mapping) else ():+ if not isinstance(tool_counts, Mapping) or tool not in bucket["tools"]:+ continue+ tool_bucket = bucket["by_tool"].setdefault(tool, _outcome_counts())+ for key in _OUTCOME_KEYS:+ tool_bucket[key] += _int(tool_counts.get(key))+ for bucket in by_server.values():+ _with_success_rate(bucket)+ for tool_counts in bucket["by_tool"].values():+ _with_success_rate(tool_counts)+ return _with_success_rate({**totals, "by_server": by_server})+++def _summarize_coverage(signals: Sequence[Mapping[str, Any]]) -> dict[str, Any]:+ """Union of per-trial activation coverage.++ A component is ``exercised`` when it was exercised in at least one trial,+ ``unavailable`` when it was tried but every try failed in every trial, and+ ``unverified`` otherwise. ``exercise_rate`` is the share of trials that+ exercised it. A trial block that lists a component as both exercised and+ unavailable (the older shape) counts as unavailable for that trial.+ """+ declared: list[str] = []+ exercised_counts: dict[str, int] = {}+ unavailable_counts: dict[str, int] = {}+ for item in signals:+ block = item.get("activation_coverage")+ if not isinstance(block, Mapping):+ continue+ for label in block.get("declared") or ():+ if isinstance(label, str) and label not in declared:+ declared.append(label)+ unavailable = {label for label in block.get("unavailable") or () if isinstance(label, str)}+ for label in unavailable:+ unavailable_counts[label] = unavailable_counts.get(label, 0) + 1+ for label in dict.fromkeys(block.get("exercised") or ()):+ if isinstance(label, str) and label not in unavailable:+ exercised_counts[label] = exercised_counts.get(label, 0) + 1+ n = len(signals)+ return {+ "declared": declared,+ "exercised": [label for label in declared if exercised_counts.get(label)],+ "unverified": [+ label for label in declared if not exercised_counts.get(label) and not unavailable_counts.get(label)+ ],+ "unavailable": [+ label for label in declared if unavailable_counts.get(label) and not exercised_counts.get(label)+ ],+ "exercise_rate": {label: _ratio(exercised_counts.get(label, 0), n) for label in declared},+ }+++def summarize_plugin_signals(signals: Sequence[Mapping[str, Any] | None]) -> dict[str, Any]:+ """Aggregate per-trial ``plugin_signals`` into one per-arm ``plugin_signals_summary``.++ ``None`` entries are trials whose trajectory could not be read; they are+ counted in ``n_missing_trajectory`` and otherwise ignored.+ """+ present = [item for item in signals if isinstance(item, Mapping)]+ n_trials = len(present)+ activations: dict[str, int] = {}+ total_activations = 0+ for item in present:+ for activation in item.get("activations") or ():+ if isinstance(activation, Mapping):+ kind = str(activation.get("type") or "")+ activations[kind] = activations.get(kind, 0) + 1+ total_activations += 1++ return {+ "n_trials": n_trials,+ "n_missing_trajectory": len(signals) - n_trials,+ "n_incomplete_trajectory": sum(item.get("evidence_complete") is False for item in present),+ "activations": {+ "total": total_activations,+ "mean_per_trial": _ratio(total_activations, n_trials),+ "by_type": activations,+ },+ "routing": _summarize_selection(present, "routing"),+ "tool_selection": _summarize_selection(present, "tool_selection"),+ "arguments": _summarize_checked(present, "arguments"),+ "mcp_calls": _summarize_mcp(present),+ "order": _summarize_order(present),+ "handoff": {**_summarize_checked(present, "handoff"), "skipped": _skipped(present, "handoff")},+ "conflict": _summarize_conflict(present),+ "activation_coverage": _summarize_coverage(present),+ }+++__all__ = [+ "ARM_SUM_OF_PARTS",+ "ARM_WITHOUT_SKILL",+ "ARM_WITH_SKILL",+ "COMPONENT_COMMAND",+ "COMPONENT_LSP",+ "COMPONENT_MCP",+ "COMPONENT_RULE_READ",+ "COMPONENT_SKILL",+ "COMPONENT_SUBAGENT",+ "PLUGIN_CASE_FIELDS",+ "STATUS_NOT_APPLICABLE",+ "STATUS_SCORED",+ "PluginSignalsContext",+ "build_plugin_signals_context",+ "compute_plugin_signals",+ "declared_mcp_server_matcher",+ "is_cross_component_case",+ "match_declared_mcp_server",+ "plugin_case_spec",+ "summarize_plugin_signals",+ "validate_plugin_case_fields",+]diff --git a/src/skillevaluator/tier3/harbor/collector.py b/src/skillevaluator/tier3/harbor/collector.pyindex e5d00d33..99a467eb 100644--- a/src/skillevaluator/tier3/harbor/collector.py+++ b/src/skillevaluator/tier3/harbor/collector.py@@ -24,6 +24,7 @@ from collections.abc import Callable, Iterable, Mapping, Sequencefrom dataclasses import dataclassfrom dataclasses import field as dataclass_fieldfrom fractions import Fraction+from itertools import islicefrom pathlib import Path, PurePosixPath, PureWindowsPathfrom typing import Any@@ -3516,17 +3517,47 @@ def _staged_task_entry_id_map(return mapping+def _private_reward_inputs(job_dir: Path, trial_roots: Sequence[Path]) -> tuple[list[Path], list[Path]]:+ """Known verifier layouts of a bounded, explicitly selected set of regular trial roots."""+ if len(trial_roots) > 512:+ raise ValueError("selected trial roots exceed the private evidence bound")+ result_files, reward_files = [], []+ with SecureRoot(job_dir):+ for root in trial_roots:+ if root.parent.absolute() != job_dir.absolute() or _safe_trial_path_component(root.name) != root.name:+ raise ValueError("selected trial root is outside the Harbor job")+ with SecureRoot(root):+ result_files.append(root / "result.json")+ reward_files.append(root / "verifier" / "reward.json")+ steps = root / "steps"+ if not os.path.lexists(steps):+ continue+ with SecureRoot(steps), os.scandir(steps) as iterator:+ children = list(islice(iterator, _MAX_TRAJECTORY_STEP_DIRECTORIES + 1))+ if len(children) > _MAX_TRAJECTORY_STEP_DIRECTORIES:+ raise ValueError("selected trial step directory bound exceeded")+ for child in sorted(children, key=lambda item: item.name):+ metadata = child.stat(follow_symlinks=False)+ if stat.S_ISDIR(metadata.st_mode) and not stat_is_link_or_reparse(metadata):+ reward_files.append(Path(child.path) / "verifier" / "reward.json")+ return ([path for path in result_files if path.exists()], [path for path in reward_files if path.exists()])++def _extract_rewards(job_dir: Path,case_id_by_task_selector: dict[str, str] | None = None,*,arm_suffix: str = "",task_entry_id_map: Mapping[str, str] | None = None,+ selected_trial_roots: Sequence[Path] | None = None,) -> list[dict[str, Any]]:"""Extract reward.json from all trials in a job directory."""rewards: list[dict[str, Any]] = []scored_trial_roots: set[Path] = set()authoritative_trial_roots: set[Path] = set()+ selected_results, selected_rewards = (+ _private_reward_inputs(job_dir, selected_trial_roots) if selected_trial_roots is not None else (None, None)+ )def _resolve_trial_name_entry_id(trial_name_val: str) -> str:if not task_entry_id_map or not trial_name_val:@@ -3583,7 +3614,7 @@ def _extract_rewards(# Native multi-step Harbor tasks can persist an authoritative aggregate at# the trial root in addition to one reward file per step. Materialize that# single logical row first so both averages and pass@k use the same score.- for result_file in sorted(job_dir.glob("*/result.json")):+ for result_file in selected_results if selected_results is not None else sorted(job_dir.glob("*/result.json")):trial_dir = result_file.parentif _trial_failure_reason(trial_dir) or _is_agent_runtime_failure_trial(trial_dir):continue@@ -3611,7 +3642,7 @@ def _extract_rewards(rewards.append(data)authoritative_trial_roots.add(trial_dir)- for reward_file in sorted(job_dir.rglob("reward.json")):+ for reward_file in selected_rewards if selected_rewards is not None else sorted(job_dir.rglob("reward.json")):if reward_file.parent.name == "verifier":try:trial_dir, trial_name, step_name = _reward_trial_context(reward_file)@@ -3653,7 +3684,7 @@ def _extract_rewards(except OSError as e:logger.warning("Failed to read %s: %s", reward_file, e)- for result_file in sorted(job_dir.glob("*/result.json")):+ for result_file in selected_results if selected_results is not None else sorted(job_dir.glob("*/result.json")):trial_dir = result_file.parentif (trial_dir in authoritative_trial_roots@@ -6855,6 +6886,18 @@ def _aggregate_execution(summaries: list[dict[str, Any]]) -> dict[str, Any]:}+def _observe_private_trials(agent, arm, job_dir, collected, scoreable, selector_map) -> None:+ """Hand existing observations to an active private hook without affecting ordinary collection."""+ runtime = sys.modules.get("skillevaluator.tier3.plugin_execution")+ hooks = runtime.active_runtime() if runtime is not None else None+ observe = getattr(hooks, "observe_collected_trials", None)+ if observe is not None:+ try:+ observe(agent, arm, job_dir, collected, scoreable, selector_map)+ except Exception as exc:+ logger.warning("Private evidence handoff failed: %s", _safe_diagnostic_text(exc, max_len=300))++def collect_harbor_results(skill_name: str,agents: list[str],@@ -6967,6 +7010,9 @@ def collect_harbor_results(else [])with_rewards, invalid_score_failures = _partition_scoreable_rewards(with_collected_rewards)+ _observe_private_trials(+ agent, "with_skill", with_job_dir, with_collected_rewards, with_rewards, case_id_by_task_selector+ )with_recovered = (_recovered_timeout_attempts(with_job_dir, with_rewards, n_attempts=n_attempts) if stop_on_pass else [])@@ -7195,6 +7241,14 @@ def collect_harbor_results(else [])without_rewards, invalid_score_failures = _partition_scoreable_rewards(without_collected_rewards)+ _observe_private_trials(+ agent,+ "without_skill",+ without_job_dir,+ without_collected_rewards,+ without_rewards,+ case_id_by_task_selector,+ )without_recovered = (_recovered_timeout_attempts(without_job_dir, without_rewards, n_attempts=n_attempts)if stop_on_passdiff --git a/src/skillevaluator/tier3/harbor/loose_agents.py b/src/skillevaluator/tier3/harbor/loose_agents.pynew file mode 100644index 00000000..2992d0b1--- /dev/null+++ b/src/skillevaluator/tier3/harbor/loose_agents.py@@ -0,0 +1,46 @@+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+# SPDX-License-Identifier: Apache-2.0++"""Harbor Claude Code wrappers for the loose-parts arm (see :mod:`skillevaluator.tier3.loose_parts`).++The launch is rewritten like the native plugin arm's+(:mod:`skillevaluator.tier3.harbor.native_agents`): ``/skilleval/native/setup.sh``+runs first and writes the loose parts under ``$CLAUDE_CONFIG_DIR``, and the same+launch-shape checks apply (exactly one Harbor launch per run). The differences+are what make the parts loose: no ``--plugin-dir``; the plain MCP config is passed+with ``--mcp-config``; and the launch exports ``CLAUDE_PLUGIN_ROOT`` and+``CLAUDE_PLUGIN_DATA`` pointing at the loose directories, the values a plugin's+scripts see in the plugin arm. No wrapper adds a permission-bypass flag.+"""++from __future__ import annotations++import re+import shlex++from harbor.agents.installed.claude_code import ClaudeCode++from skillevaluator.tier3.harbor.local_agents import SkillEvaluatorClaudeCode, SkillEvaluatorNvidiaBuildClaudeCode+from skillevaluator.tier3.harbor.native_agents import _NativeClaudeCodeMixin+from skillevaluator.tier3.loose_parts import LOOSE_MCP_CONFIG, loose_parts_launch_env+++class _LoosePartsClaudeCodeMixin(_NativeClaudeCodeMixin):+ def _skilleval_rewrite_launcher(self, launcher: str, match: re.Match[str]) -> str:+ flag = f" --mcp-config {shlex.quote(LOOSE_MCP_CONFIG)}"+ return launcher[: match.end("cli")] + flag + launcher[match.end("cli") :]++ def _skilleval_launch_env(self) -> dict[str, str]:+ return loose_parts_launch_env()+++class LoosePartsClaudeCode(_LoosePartsClaudeCodeMixin, ClaudeCode):+ """Claude Code with the plugin's parts staged loose: user-level pieces and ``--mcp-config``."""+++class LoosePartsContainerClaudeCode(_LoosePartsClaudeCodeMixin, SkillEvaluatorClaudeCode):+ """The container Claude Code wrapper the baseline arms run, with the plugin's parts staged loose."""+++class LoosePartsNvidiaBuildClaudeCode(_LoosePartsClaudeCodeMixin, SkillEvaluatorNvidiaBuildClaudeCode):+ """NVIDIA Build-routed Claude Code with the plugin's parts staged loose."""diff --git a/src/skillevaluator/tier3/harbor/loose_parts_arm.py b/src/skillevaluator/tier3/harbor/loose_parts_arm.pynew file mode 100644index 00000000..0ea534b7--- /dev/null+++ b/src/skillevaluator/tier3/harbor/loose_parts_arm.py@@ -0,0 +1,265 @@+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+# SPDX-License-Identifier: Apache-2.0++"""Private loose-parts arm planning, emitted-task hashes, and cached load summaries.++The staging itself is :mod:`skillevaluator.tier3.loose_parts`. This module is+the one seam the runner and the collector call:++* :func:`plan_loose_parts` decides per agent whether the arm runs (native+ Claude Code only in v1) and builds its native task staging and Harbor+ wrapper, or records why it does not (``not_measured_reason``);+* :meth:`LoosePartsPlan.hash_tasks` hashes the Harbor tasks the runner+ emitted for the full-plugin, loose-parts and skills-only arms, so the+ capability check reads what the agents actually get;+* :meth:`LooseAgentArm.info` combines measured hashes with cached load+ evidence supplied by the comparison coordinator for durable provenance.++Nothing here assumes Docker: the loose arm uses the same native staging route+as the native full-plugin arm (the bundle is part of each task's+``environment/``), so it runs wherever that arm runs.+"""++from __future__ import annotations++import json+from collections.abc import Mapping, Sequence+from dataclasses import dataclass, field+from pathlib import Path+from typing import Any++from skillevaluator.tier3.loose_parts import (+ LOOSE_AGENT,+ NOT_SUPPORTED_WRAPPER,+ LooseParts,+ build_loose_parts,+ load_failures_by_case,+ loose_parts_agent_import_path,+ not_supported_reason,+)+from skillevaluator.tier3.plugin_native import PluginLoadError++#: The loose arm's Harbor job-name suffix and staged task directory name.+LOOSE_PARTS_VARIANT = "looseparts"+#: Where the full per-trial load evidence is saved as indexed tables, next to statistics.json.+LOAD_EVIDENCE_FILENAME = "loose_parts_load_evidence.json"+#: Why the ladder cannot run at all without the no-plugin and skills-only arms beside it.+NEEDS_BASE_ARMS = (+ "the lift ladder needs the no-plugin and skills-only arms beside the loose-parts arm (a baseline and the "+ "plugin's member skills)"+)+LOAD_EVIDENCE_SOURCE = (+ "Claude Code's own system/init report; an MCP server init reported as pending (or not at all) counts as loaded "+ "only after a successful call of one of its tools in the trial's trajectory. Rules, hooks and scripts are not "+ "in that report (unverifiable, never a failure)."+)+# The summary's bounded copy of the failure records; the indexed file keeps every trial.+_MAX_FAILURE_CASES = 128+_MAX_FAILURE_RECORDS = 12+++@dataclass+class LooseAgentArm:+ """One agent's measured loose-parts arm: the staged parts, the task staging, and the Harbor wrapper."""++ agent: str+ parts: LooseParts+ staging: Any # NativeTaskStaging; the type lives in harbor.native_staging, imported lazily there+ import_path: str+ #: :meth:`LooseParts.content_report_for_tasks` of the emitted tasks, set by :meth:`LoosePartsPlan.hash_tasks`.+ content: dict[str, Any] = field(default_factory=dict)++ def info(+ self,+ evidence: Mapping[str, Mapping[str, Sequence[Mapping[str, Any]]]],+ *,+ load_census: Mapping[str, Any] | None = None,+ ) -> dict[str, Any]:+ """The staging's ``loose_parts`` info block with the emitted tasks' hashes and a load-evidence summary."""+ content = (+ dict(self.content)+ if self.content+ else {+ "content_hashes": {},+ "equal_across_arms": None,+ "mismatches": [],+ "content_basis": "unavailable",+ "error": "Emitted task parity was not recorded",+ }+ )+ block = self.parts.info(content=content, load_evidence=load_evidence_summary(evidence))+ block["measured"] = True+ block["content_basis"] = content.get("content_basis")+ block["tasks_hashed"] = content.get("tasks_hashed")+ if content.get("error"):+ block["content_error"] = str(content["error"])[:500]+ if load_census is not None:+ block["load_census"] = dict(load_census)+ return block+++@dataclass+class LoosePartsPlan:+ """The run's loose-parts arms: the measured agents, and why every other agent is not measured."""++ requested: bool+ order: tuple[str, ...] = ()+ arms: dict[str, LooseAgentArm] = field(default_factory=dict)+ not_measured: dict[str, str] = field(default_factory=dict)++ def arm_for(self, agent: str) -> LooseAgentArm | None:+ return self.arms.get(agent)++ def run_config(self) -> dict[str, Any]:+ """``run_config["loose_parts"]``: per agent, whether its loose arm runs and why not."""+ agents: dict[str, dict[str, Any]] = {}+ for agent in self.order:+ if agent in self.arms:+ agents[agent] = {"measured": True, "import_path": self.arms[agent].import_path}+ elif agent in self.not_measured:+ agents[agent] = {"measured": False, "not_measured_reason": self.not_measured[agent]}+ return {"requested": self.requested, "variant": LOOSE_PARTS_VARIANT, "agents": agents}++ def hash_tasks(self, agent: str, task_dirs_by_arm: Mapping[str, Sequence[Path]]) -> None:+ """Hash the tasks the runner emitted for ``agent`` (``{arm: [task dir, ...]}``) into its arm's ``content``."""+ arm = self.arms.get(agent)+ if arm is None:+ return+ try:+ report = arm.parts.content_report_for_tasks(task_dirs_by_arm)+ except (OSError, ValueError) as exc:+ # Report-only: an unreadable task leaves the capability unverified, never fails the run.+ report = {+ "content_hashes": {},+ "equal_across_arms": None,+ "mismatches": [],+ "tasks_hashed": 0,+ "error": f"{type(exc).__name__}: {exc}",+ }+ arm.content = {**report, "content_basis": "emitted Harbor tasks"}+++def plan_loose_parts(+ *,+ requested: bool,+ agents: Sequence[str],+ decisions: Mapping[str, Any],+ source: Any,+ agent_import_paths: Mapping[str, str],+ unavailable: str | None = None,+) -> LoosePartsPlan:+ """Decide per agent whether its loose-parts arm runs, and build its staging and Harbor wrapper.++ ``decisions`` are the run's private loading decisions (after any native+ fallback), ``source`` the native plugin snapshot (``None`` in wrapper+ mode), and ``agent_import_paths`` each agent's base Harbor import path.+ ``unavailable`` marks every agent not measured with that reason (the+ ladder cannot run in this run at all). Never raises for an agent: every+ refusal of the staging becomes that agent's ``not_measured_reason``.+ """+ plan = LoosePartsPlan(requested=requested, order=tuple(agents))+ if not requested:+ return plan+ for agent in agents:+ if unavailable:+ plan.not_measured[agent] = unavailable+ continue+ reason, arm = _agent_arm(agent, decisions.get(agent), source, agent_import_paths.get(agent))+ if arm is not None:+ plan.arms[agent] = arm+ else:+ plan.not_measured[agent] = reason or "not supported: the loose-parts arm could not be staged"+ return plan+++def _agent_arm(+ agent: str, decision: Any, source: Any, base_import_path: str | None+) -> tuple[str | None, LooseAgentArm | None]:++ if agent != LOOSE_AGENT:+ return not_supported_reason(agent), None+ if source is None:+ return NOT_SUPPORTED_WRAPPER, None+ if decision is None or not getattr(decision, "native", False):+ return f"{NOT_SUPPORTED_WRAPPER} ({agent}'s plugin arm does not load the plugin natively in this run)", None+ try:+ parts = build_loose_parts(source, agent, plugin_load="native")+ if parts.error is not None:+ return parts.error, None+ staging = parts.task_staging()+ import_path = loose_parts_agent_import_path(agent, base_import_path)+ except PluginLoadError as exc: # includes LoosePartsUnsupported ("not supported: ...")+ return str(exc), None+ except (ValueError, OSError) as exc:+ return f"loose parts cannot be staged: {exc}", None+ return None, LooseAgentArm(agent=agent, parts=parts, staging=staging, import_path=import_path)+++def load_evidence_summary(evidence: Mapping[str, Mapping[str, Sequence[Mapping[str, Any]]]]) -> dict[str, Any]:+ """Per arm and piece, how many scored trials loaded it, and the cases dropped for a load failure.++ ``pieces[arm][piece]`` counts ``loaded``, ``not_loaded`` and+ ``unverifiable`` trials and lists the evidence ``sources`` seen+ (``system/init``, ``mcp-call``, ``unverifiable``).+ ``load_failures_by_case`` is :func:`load_failures_by_case` (bounded), the+ records the statistics drop cases by. The full per-trial evidence is saved+ in :data:`LOAD_EVIDENCE_FILENAME`.+ """+ failures = load_failures_by_case(evidence)+ pieces: dict[str, dict[str, dict[str, Any]]] = {}+ trials: dict[str, int] = {}+ for arm, cases in sorted(evidence.items()):+ rows = pieces.setdefault(arm, {})+ for attempts in cases.values():+ trials[arm] = trials.get(arm, 0) + len(attempts)+ for trial in attempts:+ for piece, row in trial.items():+ counts = rows.setdefault(piece, {"loaded": 0, "not_loaded": 0, "unverifiable": 0, "sources": []})+ loaded = row.get("loaded") if isinstance(row, Mapping) else None+ counts["loaded" if loaded is True else "not_loaded" if loaded is False else "unverifiable"] += 1+ source = str(row.get("source") or "") if isinstance(row, Mapping) else ""+ if source and source not in counts["sources"]:+ counts["sources"] = sorted([*counts["sources"], source])+ bounded = {case: records[:_MAX_FAILURE_RECORDS] for case, records in sorted(failures.items())[:_MAX_FAILURE_CASES]}+ return {+ "source": LOAD_EVIDENCE_SOURCE,+ "trials": trials,+ "pieces": pieces,+ "load_failures_by_case": bounded,+ "dropped_cases": sorted(failures),+ "file": LOAD_EVIDENCE_FILENAME,+ }+++def compact_load_evidence(evidence: Mapping[str, Mapping[str, Sequence[Mapping[str, Any]]]]) -> dict[str, Any]:+ """Retain every piece result and attempt without repeating their JSON trees.++ ``arms[arm][case]`` lists attempt-table indices in the original attempt+ order. Each ``attempts[index]`` maps piece ids to ``results`` indices; a+ result retains the complete original evidence row, including unknown load+ states, reasons, MCP status and tools. Both tables share identical values+ across arms and cases. The normal generated-artifact bounds still apply.+ """+ results: list[dict[str, Any]] = []+ attempts: list[dict[str, int]] = []+ result_ids: dict[str, int] = {}+ attempt_ids: dict[tuple[tuple[str, int], ...], int] = {}+ arms: dict[str, dict[str, list[int]]] = {}+ for arm, cases in sorted(evidence.items()):+ arms[arm] = {}+ for case, trials in sorted(cases.items()):+ indices = arms[arm][case] = []+ for trial in trials:+ rows = {}+ for piece, row in sorted(trial.items()):+ key = json.dumps(row, sort_keys=True, separators=(",", ":"), allow_nan=False)+ if key not in result_ids:+ result_ids[key] = len(results)+ results.append(dict(row))+ rows[piece] = result_ids[key]+ attempt_key = tuple(rows.items())+ if attempt_key not in attempt_ids:+ attempt_ids[attempt_key] = len(attempts)+ attempts.append(rows)+ indices.append(attempt_ids[attempt_key])+ return {"format": "indexed-v1", "results": results, "attempts": attempts, "arms": arms}diff --git a/src/skillevaluator/tier3/harbor/runner.py b/src/skillevaluator/tier3/harbor/runner.pyindex 0bfd2def..8ccc2d7d 100644--- a/src/skillevaluator/tier3/harbor/runner.py+++ b/src/skillevaluator/tier3/harbor/runner.py@@ -4368,15 +4368,17 @@ def _private_runtime_hooks() -> Any:def _native_plugin_agent_import_paths() -> frozenset[str]:- """Private wrappers that do not inherit host-only local execution behavior."""+ """The private native plugin-loading wrappers, which build on the agents allowed on every backend."""+ from skillevaluator.tier3.loose_parts import LOOSE_PARTS_AGENT_IMPORT_PATHSfrom skillevaluator.tier3.plugin_native import NATIVE_AGENT_IMPORT_PATHS, PRIVATE_AGENT_IMPORT_PATHS- remote_private_paths = (- path- for (_, base), path in PRIVATE_AGENT_IMPORT_PATHS.items()- if not (base and base.startswith("skillevaluator.tier3.harbor.local_agents:SkillEvaluatorLocal"))+ # Host-only wrappers remain outside the container backend allowlist.+ return frozenset(+ value+ for paths in (NATIVE_AGENT_IMPORT_PATHS, PRIVATE_AGENT_IMPORT_PATHS, LOOSE_PARTS_AGENT_IMPORT_PATHS)+ for (_, base), value in paths.items()+ if not (base or "").startswith("skillevaluator.tier3.harbor.local_agents:SkillEvaluatorLocal"))- return frozenset((*NATIVE_AGENT_IMPORT_PATHS.values(), *remote_private_paths))def _private_agent_paths(@@ -4432,21 +4434,34 @@ def _run_agent_pair(verifier_env: Mapping[str, str] | None = None,environment_kwargs: Mapping[str, Any] | None = None,with_agent_import_path: str | None = None,+ additional_arms: Sequence[tuple[str, Path, str | None]] = (),+ additional_arm_errors: dict[str, str] | None = None,) -> list[str]:- """Run the with-skill arm and its baseline; ``with_agent_import_path`` replaces the agent of the with arm only."""- # A baseline-only run (``skip_with_skill``) stages no with-skill dataset.+ """Run ordinary and privately requested arms under one per-agent trial budget."""jobs = [] if with_skill is None else [("with", with_skill, with_agent_import_path or agent_import_path)]if baseline is not None:jobs.append(("without", baseline, agent_import_path))+ extra_variants = {variant for variant, _, _ in additional_arms}+ if len(extra_variants) != len(additional_arms) or extra_variants - {"sumofparts", "looseparts"}:+ raise ValueError("Invalid or duplicate private comparison variant")+ jobs.extend(additional_arms)if not jobs:return [f"{agent} has no staged Harbor dataset to run"]++ def record_failure(variant: str, failures: list[str]) -> list[str]:+ if variant not in extra_variants:+ return failures+ if failures and additional_arm_errors is not None:+ additional_arm_errors[variant] = _bounded_execution_error("; ".join(failures))+ return []+if stop_on_pass:# A later attempt is launched only after the previous one scored, so# stop-on-pass runs each condition sequentially, one attempt at a time.sequential_errors: list[str] = []for variant, dataset, job_import_path in jobs:- sequential_errors.extend(- _run_stop_on_pass_variant(+ try:+ failures = _run_stop_on_pass_variant(skill_name=skill_name,agent=agent,variant=variant,@@ -4466,7 +4481,11 @@ def _run_agent_pair(verifier_env=verifier_env,environment_kwargs=environment_kwargs,)- )+ except Exception as exc:+ if variant not in extra_variants:+ raise+ failures = [f"{type(exc).__name__}: {exc}"]+ sequential_errors.extend(record_failure(variant, failures))return sequential_errors# The advertised concurrency is one per-agent trial budget. Split it# across concurrently running conditions instead of multiplying it by two.@@ -4502,9 +4521,24 @@ def _run_agent_pair(for (variant, dataset, job_import_path), condition_concurrency in zip(jobs, job_concurrency, strict=True)}for future in as_completed(futures):- ok, detail = future.result()+ variant = futures[future]+ try:+ ok, detail = future.result()+ except Exception as exc:+ if variant not in extra_variants:+ raise+ ok, detail = False, f"{type(exc).__name__}: {exc}"if not ok:- errors.append(f"{agent} {futures[future]}-skill Harbor run failed: {detail}")+ errors.extend(+ record_failure(+ variant,+ [+ detail+ if variant in extra_variants+ else f"{agent} {variant}-skill Harbor run failed: {detail}"+ ],+ )+ )return errors@@ -5297,6 +5331,26 @@ def _run_harbor_eval_impl(detail="base image build failed; falling back to per-task Dockerfiles",))+ private_hooks = _private_runtime_hooks()+ comparison_plan = getattr(private_hooks, "comparison_plan", None)+ if comparison_plan is not None:+ comparison_plan.configure_agent_routes({agent: paths[0] for agent, paths in private_agent_paths.items()})+ additional_arms: dict[str, tuple[tuple[str, Path, str | None], ...]] = {}+ emitted_tasks: dict[str, dict[str, Sequence[Path]]] = {}+ shared_task_inputs = {+ "reference_skills_dir": reference_skills_dir,+ "workspace_mode": workspace_mode,+ "grading_mode": grading_mode,+ "base_image": base_image,+ "custom_dockerfile_mode": dockerfile_mode,+ "copy_repo": copy_repo,+ "repo_context_exclude_paths": effective_repo_context_exclude_paths,+ "verifier_env": staged_verifier_env,+ "pre_agent_setup": harbor_config.get("pre_agent_setup", []),+ "task_resources": resource_config,+ "agent_workdir": harbor_config.get("agent_workdir"),+ "evaluator_skill_path": evaluator_skill_path,+ }agent_task_dirs: dict[str, tuple[Path | None, Path | None]] = {}# The first staged arm per agent: with-skill tasks, or the baseline tasks# of a baseline-only run. Timeout planning and the runtime preflight use it.@@ -5323,6 +5377,7 @@ def _run_harbor_eval_impl(with_dir = None if skip_with_skill else tasks_dir / agent / "with"without_dir = None if skip_baseline else tasks_dir / agent / "without"agent_task_dirs[agent] = (with_dir, without_dir)+ emitted_tasks[agent] = {}if with_dir is None:continueprimary_task_dirs[agent] = with_dir@@ -5330,21 +5385,10 @@ def _run_harbor_eval_impl(skill_path,with_dir,with_skill=True,- reference_skills_dir=reference_skills_dir,workspace_skill_paths=workspace_skills,- workspace_mode=workspace_mode,- grading_mode=grading_mode,- base_image=base_image,- custom_dockerfile_mode=dockerfile_mode,- copy_repo=copy_repo,- repo_context_exclude_paths=effective_repo_context_exclude_paths,runtime_env=dict(runtime_plans[agent].staged_env),- verifier_env=staged_verifier_env,- pre_agent_setup=harbor_config.get("pre_agent_setup", []),- task_resources=resource_config,- agent_workdir=harbor_config.get("agent_workdir"),- evaluator_skill_path=evaluator_skill_path,arm_suffix=with_arm_suffix,+ **shared_task_inputs,)staged_task_paths.extend(task_paths)if staged_environment_error := _staged_task_environment_error(env_mode, task_paths):@@ -5362,6 +5406,7 @@ def _run_harbor_eval_impl(raise ValueError(f"Generated task identities differ for agent {agent}")if (private_hooks := _private_runtime_hooks()) is not None:private_hooks.stage_with_arm(agent, task_paths)+ emitted_tasks[agent]["with_skill"] = tuple(task_paths)if not skip_with_skill:reporter.emit(ProgressEvent(stage="with-skill-tasks", state="ready", detail="task inputs staged"))if not skip_baseline:@@ -5384,23 +5429,13 @@ def _run_harbor_eval_impl(skill_path,without_dir,with_skill=False,- reference_skills_dir=reference_skills_dir,workspace_skill_paths=workspace_skills,- workspace_mode=workspace_mode,- grading_mode=grading_mode,- base_image=base_image,- custom_dockerfile_mode=dockerfile_mode,- copy_repo=copy_repo,- repo_context_exclude_paths=effective_repo_context_exclude_paths,runtime_env=dict(runtime_plans[agent].staged_env),- verifier_env=staged_verifier_env,- pre_agent_setup=harbor_config.get("pre_agent_setup", []),- task_resources=resource_config,- agent_workdir=harbor_config.get("agent_workdir"),- evaluator_skill_path=evaluator_skill_path,- _baseline_alias_validation=baseline_alias_validation,arm_suffix=without_arm_suffix,+ _baseline_alias_validation=baseline_alias_validation,+ **shared_task_inputs,)+ emitted_tasks[agent]["without_skill"] = tuple(baseline_task_paths)baseline_task_selectors = validate_case_ids(task.name for task in baseline_task_paths)baseline_logical_case_ids = validate_case_ids(_native_entry_id(task) for task in baseline_task_paths)baseline_case_id_by_task_selector = dict(@@ -5419,6 +5454,44 @@ def _run_harbor_eval_impl(or baseline_case_id_by_task_selector != expected_case_id_by_task_selector):raise ValueError(f"Baseline task identities differ for agent {agent}")+ if comparison_plan is not None:+ for agent in agents:++ def emit_control(variant: str, members: Sequence[Path], *, _agent: str = agent) -> Sequence[Path]:+ target = tasks_dir / _agent / variant+ proof = _prevalidate_baseline_skill_candidates(+ skill_path,+ reference_skills_dir,+ list(members),+ excluded_roots=effective_repo_context_exclude_paths,+ )+ paths = emitter(+ skill_path,+ target,+ with_skill=False,+ workspace_skill_paths=list(members),+ runtime_env=dict(runtime_plans[_agent].staged_env),+ arm_suffix="-without-skill",+ _baseline_alias_validation=proof,+ **shared_task_inputs,+ )+ selectors = validate_case_ids(path.name for path in paths)+ identities = dict(+ zip(selectors, validate_case_ids(_native_entry_id(path) for path in paths), strict=True)+ )+ if selectors != expected_task_selectors or identities != expected_case_id_by_task_selector:+ raise ValueError(f"Private comparison task identities differ for {_agent}/{variant}")+ if error := _staged_task_environment_error(env_mode, paths):+ raise ValueError(error)+ staged_task_paths.extend(paths)+ return paths++ additional_arms[agent] = comparison_plan.stage_additional(+ agent,+ emit_control,+ emitted_tasks[agent],+ private_agent_paths[agent][0],+ )if not skip_baseline:reporter.emit(ProgressEvent(stage="baseline-tasks", state="ready", detail="baseline inputs staged"))else:@@ -5476,7 +5549,28 @@ def _run_harbor_eval_impl(return _persist_pre_execution_failure([str(exc)])expected_trials = len(task_selectors) * n_attemptsvariants = 2 - int(skip_baseline) - int(skip_with_skill)- matrix_trials = expected_trials * len(agents) * variants+ matrix_trials = expected_trials * sum(variants + len(additional_arms.get(agent, ())) for agent in agents)+ try:+ if comparison_plan is not None:+ comparison_plan.bind_execution(+ case_ids, n_attempts=n_attempts, stop_on_pass=bool(stop_on_pass), pass_threshold=float(pass_threshold)+ )+ jobs = {+ agent: {+ **({"with_skill": jobs_dir / f"{skill_path.name}-{agent}-with"} if not skip_with_skill else {}),+ **({"without_skill": jobs_dir / f"{skill_path.name}-{agent}-without"} if not skip_baseline else {}),+ **{+ {"sumofparts": "sum_of_parts", "looseparts": "loose_parts"}[variant]: jobs_dir+ / f"{skill_path.name}-{agent}-{variant}"+ for variant, _, _ in additional_arms.get(agent, ())+ },+ }+ for agent in agents+ }+ private_hooks.configure_run(case_id_by_task_selector=case_id_by_task_selector, arm_jobs=jobs)+ run_config["private_comparisons"] = comparison_plan.run_config()+ except (OSError, ValueError) as exc:+ return _persist_pre_execution_failure([str(exc)])preflight_trials = len(agents) if agent_runtime_preflight else 0try:task_timeout_seconds = _task_timeout_plan(@@ -5591,6 +5685,14 @@ def _run_harbor_eval_impl(task_names=task_selectors,verifier_env=job_judge_verifier_env,environment_kwargs=effective_environment_kwargs,+ **(+ {+ "additional_arms": additional_arms[agent],+ "additional_arm_errors": comparison_plan.launch_errors[agent],+ }+ if additional_arms.get(agent)+ else {}+ ),)active_agents: set[str] = set()diff --git a/src/skillevaluator/tier3/harbor/stats.py b/src/skillevaluator/tier3/harbor/stats.pynew file mode 100644index 00000000..d97781ec--- /dev/null+++ b/src/skillevaluator/tier3/harbor/stats.py@@ -0,0 +1,1805 @@+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+# SPDX-License-Identifier: Apache-2.0++"""Report-only Tier 3 statistics computed from collected Harbor trials.++Everything in this module is advisory. None of it changes a metric, a+dimension, ``overall_score``, pass@k, or a pass/fail gate. The collector does+all file I/O and hands this module plain observations, so the functions here+are pure and deterministic.++The blocks produced here (``lift_uncertainty``, ``reliability``, ``cost``,+``token_efficiency``, ``context_cost_measured``, ``lift_ladder`` and+``integration_completeness``) follow the shared C4 statistics contract.+"""++from __future__ import annotations++import math+import random+from collections.abc import Collection, Mapping, Sequence+from dataclasses import dataclass, field+from functools import partial+from typing import Any++from skillevaluator.constants import (+ DIMENSION_MAPPING,+ LIFT_BOOTSTRAP_CONFIDENCE,+ LIFT_BOOTSTRAP_RESAMPLES,+ LIFT_BOOTSTRAP_SEED,+ LIFT_BOOTSTRAP_TAIL_RESAMPLES,+ LIFT_CI_LOW_PRECISION_WIDTH,+ LIFT_CI_MIN_PAIRED_CASES,+ TOKEN_EFFICIENCY_HALF_LIFE,+)+from skillevaluator.tier3.harbor.lift_basis import pair_metric_scores+from skillevaluator.tier3.harbor.metrics import (+ CUSTOM_SCORE_POLICY,+ LEGACY_SCORE_POLICY,+ SUPPORTED_SCORE_POLICIES,+ finite_number,+ metric_set_for_reward,+ metric_value,+ overall_score,+ weighted_dimension_score,+)+from skillevaluator.tier3.runtime_arms import ARM_LOOSE_PARTS, ARM_SUM_OF_PARTS, ARM_WITH, ARM_WITHOUT++LIFT_UNCERTAINTY_METHOD = "paired_case_bootstrap"+CONTEXT_COST_METHOD = "paired_first_turn_prompt_tokens"++PRECISION_ADEQUATE = "adequate"+PRECISION_LOW = "low"+PRECISION_INSUFFICIENT = "insufficient"++MAX_COMPARISON_CASES = 1000+MAX_COMPARISON_TRIALS = 100_000+MAX_BOOTSTRAP_RESAMPLES = 1_000_000+_JUDGE_METRICS = ("accuracy", "goal_accuracy", "behavior_check")++# The statistics blocks of one agent, in the order build_agent_statistics()+# returns them. Each is persisted in statistics.json and in the agent's results.+# ``lift_ladder`` is ``None`` unless the agent ran a loose-parts arm.+STATISTICS_BLOCKS = (+ "lift_uncertainty",+ "reliability",+ "cost",+ "token_efficiency",+ "context_cost_measured",+ "lift_ladder",+ "integration_completeness",+ "comparison_eligibility",+)++_SUCCEEDED = "succeeded"+++@dataclass(frozen=True)+class TrialObservation:+ """One logical Harbor attempt: its case, reward payload and resource usage.++ ``usage`` holds optional non-negative counters read from the trial's ATIF+ trajectory or Harbor ``result.json``: ``prompt_tokens`` (including cached+ tokens), ``cached_tokens``, ``completion_tokens``, ``cost_usd`` and+ ``first_turn_prompt_tokens``. Missing counters are simply absent.++ ``logical_overall`` is set for a multi-step attempt whose steps use+ different metric contracts (standard and custom-only rows). Its dimension+ scores cover only the standard rows, so the attempt is scored by its+ logical overall instead, the score pass@k and the report headline use.+ """++ case_id: str+ reward: Mapping[str, Any]+ usage: Mapping[str, Any] = field(default_factory=dict)+ logical_overall: float | None = None+++@dataclass(frozen=True)+class ArmObservations:+ """All logical attempts of one evaluation arm, in attempt order per case.++ ``unscored`` holds the attempts with no score (a judge error, say). They+ feed only statistics that read usage, such as the measured context cost.+ """++ execution_status: str+ trials: Sequence[TrialObservation] = ()+ pass_summary: Mapping[str, Any] = field(default_factory=dict)+ job_failure: str = ""+ unscored: Sequence[TrialObservation] = ()+ observations_complete: bool = True++ @property+ def succeeded(self) -> bool:+ return self.execution_status == _SUCCEEDED+++def _case_id(value: Any) -> str | None:+ return value if isinstance(value, str) and value not in {"", "unknown"} and len(value) <= 256 else None+++def _logical_score(value: Any) -> float | None:+ number = finite_number(value)+ return number if number is not None and 0 <= number <= 1 and not isinstance(value, bool) else None+++def _reward_policy(reward: Mapping[str, Any], *, logical: bool = False) -> str | None:+ payload = dict(reward)+ declared = payload.get("score_policy")+ if declared is not None and (not isinstance(declared, str) or declared not in SUPPORTED_SCORE_POLICIES):+ return None+ if not logical and overall_score(payload) is None:+ return None+ _, metrics = metric_set_for_reward(payload)+ if not metrics:+ return CUSTOM_SCORE_POLICY+ from skillevaluator.tier3.harbor.collector import _score_policy_for_rewards++ return _score_policy_for_rewards([payload], metrics)[0]+++class _CaseIndex:+ """One bounded index of logical attempts; comparisons reuse its rows and dimension projections."""++ def __init__(+ self, arms: Mapping[str, ArmObservations], judge_mixed_cases: Mapping[str, Collection[str]] | None = None+ ):+ if len(arms) > 4:+ raise ValueError("Comparison arm limit exceeded")+ if judge_mixed_cases is not None:+ for metric, cases in judge_mixed_cases.items():+ if metric not in _JUDGE_METRICS or isinstance(cases, str | bytes) or len(cases) > MAX_COMPARISON_CASES:+ raise ValueError("Invalid bounded judge exclusions")+ self.rows: dict[str, dict[str, list[TrialObservation]]] = {}+ self.models: dict[str, dict[str, dict[str, set[str]]]] = {}+ self.trial_policies: dict[int, str | None] = {}+ self.projected: dict[tuple[str, ...], Any] = {}+ self.policies: dict[str, dict[str, set[str]]] = {}+ self.conflicts: dict[tuple[str, ...], tuple[str, ...]] = {}+ self.external_mixed = judge_mixed_cases+ from skillevaluator.tier3.harbor.collector import _reward_judge_models++ count = 0+ for arm, observed in arms.items():+ cases = self.rows.setdefault(arm, {})+ for trial in observed.trials:+ count += 1+ if count > MAX_COMPARISON_TRIALS:+ raise ValueError("Comparison trial limit exceeded")+ case = _case_id(trial.case_id)+ if case is None:+ continue+ cases.setdefault(case, []).append(trial)+ policy = _reward_policy(trial.reward, logical=_logical_score(trial.logical_overall) is not None)+ self.trial_policies[id(trial)] = policy+ if policy is not None:+ self.policies.setdefault(arm, {}).setdefault(case, set()).add(policy)+ if len(cases) > MAX_COMPARISON_CASES:+ raise ValueError("Comparison case limit exceeded")+ for metric in _JUDGE_METRICS if judge_mixed_cases is None else ():+ models = _reward_judge_models(dict(trial.reward), metric)+ if models:+ self.models.setdefault(case, {}).setdefault(metric, {}).setdefault(arm, set()).update(models)++ def dimensions(self, names: Sequence[str]) -> tuple[dict[str, dict[str, dict[str, float]]], dict[str, int]]:+ projection_key = tuple(names)+ if projection_key in self.projected:+ return self.projected[projection_key]+ selected = set(names)+ excluded: dict[str, set[str]] = {}+ counts: dict[str, int] = {}+ universe = set().union(*(set(self.rows.get(arm, {})) for arm in names))+ conflicts = {+ case+ for case in universe+ if len(set().union(*(self.policies.get(arm, {}).get(case, set()) for arm in names))) > 1+ }+ self.conflicts[projection_key] = tuple(sorted(conflicts))+ if self.external_mixed is not None:+ for metric, cases in self.external_mixed.items():+ valid = {case for case in cases if _case_id(case) is not None}+ if valid:+ counts[metric] = len(valid)+ for case in valid:+ excluded.setdefault(case, set()).add(metric)+ for case, metrics in self.models.items():+ for metric, by_arm in metrics.items():+ models = [models for arm, models in by_arm.items() if arm in selected]+ if models and len(set().union(*models)) > 1:+ excluded.setdefault(case, set()).add(metric)+ counts[metric] = counts.get(metric, 0) + 1+ result: dict[str, dict[str, dict[str, float]]] = {arm: {} for arm in names}+ for case in universe - conflicts:+ rows = {+ arm: [+ trial+ for trial in self.rows.get(arm, {}).get(case, ())+ if self.trial_policies.get(id(trial)) is not None+ and (trial.logical_overall is None or _logical_score(trial.logical_overall) is not None)+ ]+ for arm in names+ }+ present = {arm: trials for arm, trials in rows.items() if trials}+ for arm, dimensions in self._case_projection(case, present, excluded.get(case, set())).items():+ if dimensions:+ result[arm][case] = dimensions+ self.projected[projection_key] = (result, counts)+ return result, counts++ def _case_projection(+ self, case: str, rows: Mapping[str, Sequence[TrialObservation]], excluded: set[str]+ ) -> dict[str, dict[str, float]]:+ """Pair raw metrics first, using the current collector's pairing kernel."""+ if not rows:+ return {}+ policies = {self.trial_policies[id(trial)] for trials in rows.values() for trial in trials}+ if policies == {CUSTOM_SCORE_POLICY}:+ return (+ {+ arm: {+ OVERALL_DIMENSION: math.fsum(+ _logical_score(trial.logical_overall)+ if trial.logical_overall is not None+ else overall_score(dict(trial.reward))+ for trial in trials+ )+ / len(trials)+ }+ for arm, trials in rows.items()+ }+ if not excluded+ else {}+ )+ metric_names = tuple(+ dict.fromkeys(+ metric+ for trials in rows.values()+ for trial in trials+ for metric in metric_set_for_reward(dict(trial.reward))[1]+ )+ )+ first = next(iter(rows))+ reference = [(case, dict(trial.reward)) for trial in rows[first]]+ values: dict[str, dict[str, float]] = {}+ for arm in tuple(rows)[1:] or (first,):+ other = [(case, dict(trial.reward)) for trial in rows[arm]]+ paired = pair_metric_scores(+ reference,+ other,+ arms=("reference", "control"),+ metrics=metric_names,+ judge_mixed_cases=dict.fromkeys(excluded, (case,)),+ )+ values[first] = paired.treatment+ values[arm] = paired.control+ shared = set(metric_names).intersection(*(set(scores) for scores in values.values()))+ if any(trial.logical_overall is not None for trials in rows.values() for trial in trials):+ # An indivisible mixed logical overall cannot be recomputed on a reduced metric basis.+ if excluded or shared != set(metric_names):+ return {}+ return {+ arm: {+ OVERALL_DIMENSION: math.fsum(+ _logical_score(trial.logical_overall)+ if trial.logical_overall is not None+ else overall_score(dict(trial.reward))+ for trial in trials+ )+ / len(trials)+ }+ for arm, trials in rows.items()+ }+ projected: dict[str, dict[str, float]] = {}+ for arm, scores in values.items():+ if policies == {LEGACY_SCORE_POLICY}:+ projected[arm] = (+ {OVERALL_DIMENSION: math.fsum(scores[metric] for metric in shared) / len(shared)} if shared else {}+ )+ continue+ dimensions = {}+ for dimension, config in DIMENSION_MAPPING.items():+ value = weighted_dimension_score(+ lambda metric, scores=scores: scores.get(metric) if metric in shared else None,+ config,+ active_metrics=tuple(shared),+ )+ if value is not None:+ dimensions[dimension] = value+ projected[arm] = dimensions+ return projected+++# ---------------------------------------------------------------------------+# Numeric helpers+# ---------------------------------------------------------------------------+++def _mean(values: Sequence[float]) -> float | None:+ return math.fsum(values) / len(values) if values else None+++def _round(value: float | None, digits: int = 4) -> float | None:+ return round(value, digits) if value is not None else None+++# ---------------------------------------------------------------------------+# Per-trial quality score (the basis of the reported lift)+# ---------------------------------------------------------------------------+++OVERALL_DIMENSION = "overall"+++def trial_dimension_scores(reward: Mapping[str, Any], *, excluded_metrics: Collection[str] = ()) -> dict[str, float]:+ """Return one trial's score per dimension, skipping dimensions it cannot score.++ Dimension scores are linear in the metrics. A legacy reward without the+ ``security`` metric uses the same ``behavior_check`` fallback as the+ report. Metrics that are missing, non-finite or N/A are skipped, so a+ dimension whose metrics are all N/A (for example Discoverability in an arm+ that has no skill to discover) is left out. A custom-only reward, or one+ without any dimension, is scored by its ``overall`` value alone.+ """+ payload = dict(reward)+ policy = _reward_policy(payload)+ if policy is None:+ return {}+ if policy in {LEGACY_SCORE_POLICY, CUSTOM_SCORE_POLICY}:+ value = overall_score(payload)+ return {OVERALL_DIMENSION: value} if value is not None and not excluded_metrics else {}+ _, active_metrics = metric_set_for_reward(payload)+ if excluded_metrics:+ active_metrics = tuple(metric for metric in active_metrics if metric not in excluded_metrics)+ scores: dict[str, float] = {}+ if active_metrics:++ def value_of(metric: str) -> float | None:+ return None if metric in excluded_metrics else metric_value(payload, metric)++ for dimension, config in DIMENSION_MAPPING.items():+ value = weighted_dimension_score(value_of, config, active_metrics=active_metrics)+ if value is not None:+ scores[dimension] = value+ if not scores and not excluded_metrics:+ overall = overall_score(payload)+ if overall is not None:+ scores[OVERALL_DIMENSION] = overall+ return scores+++def trial_quality_score(reward: Mapping[str, Any], dimensions: Sequence[str] | None = None) -> float | None:+ """Return one trial's score on the same basis as the reported lift.++ The score is the mean of the trial's dimension scores. With *dimensions*,+ only those dimensions count, which is how two arms are compared on the+ dimensions both can score. ``None`` means the trial carries no usable score.+ """+ if dimensions is None:+ return overall_score(dict(reward)) if _reward_policy(reward) is not None else None+ scores = trial_dimension_scores(reward)+ if dimensions is not None:+ wanted = set(dimensions)+ scores = {dimension: value for dimension, value in scores.items() if dimension in wanted}+ return _mean(list(scores.values())) if scores else None+++def case_dimension_means(trials: Sequence[TrialObservation]) -> dict[str, dict[str, float]]:+ """Average each dimension over the scored attempts of each case (attempts nest in cases)."""+ by_case: dict[str, dict[str, list[float]]] = {}+ for trial in trials:+ if _case_id(trial.case_id) is None:+ continue+ scores = (+ {OVERALL_DIMENSION: score}+ if trial.logical_overall is not None+ and (score := _logical_score(trial.logical_overall)) is not None+ and _reward_policy(trial.reward, logical=True) is not None+ else {}+ if trial.logical_overall is not None+ else trial_dimension_scores(trial.reward)+ )+ for dimension, value in scores.items():+ by_case.setdefault(trial.case_id, {}).setdefault(dimension, []).append(value)+ return {+ case_id: {dimension: math.fsum(values) / len(values) for dimension, values in dimensions.items()}+ for case_id, dimensions in by_case.items()+ if dimensions+ }+++_DIMENSION_ORDER = (*DIMENSION_MAPPING, OVERALL_DIMENSION)+++def _overall_aligned(+ treatment: dict[str, float], control: dict[str, float]+) -> tuple[dict[str, float], dict[str, float]]:+ """A case scored only by its overall on one side compares with the other side's overall (dimension mean)."""+ if set(treatment) == {OVERALL_DIMENSION} and OVERALL_DIMENSION not in control and control:+ return treatment, {OVERALL_DIMENSION: math.fsum(control.values()) / len(control)}+ if set(control) == {OVERALL_DIMENSION} and OVERALL_DIMENSION not in treatment and treatment:+ return {OVERALL_DIMENSION: math.fsum(treatment.values()) / len(treatment)}, control+ return treatment, control+++def shared_case_scores(+ treatment: Sequence[TrialObservation],+ control: Sequence[TrialObservation],+ *,+ judge_mixed_cases: Mapping[str, Collection[str]] | None = None,+ load_failures_by_case: LoadFailures | None = None,+ _index: _CaseIndex | None = None,+ _arms: tuple[str, str] = ("treatment", "control"),+) -> tuple[dict[str, float], dict[str, float], list[str]]:+ """Score two arms per case on the dimensions both arms scored for that case.++ An arm without the skill under test has no Discoverability or Efficiency+ (the verifier records them as N/A), so a lift over every dimension would+ credit skill activation alone. Each paired case is scored on the+ dimensions both arms have for it, the same set on both sides. Returns the+ treatment and control case scores and the dimensions that were compared.+ """+ index = _index or _CaseIndex(+ {_arms[0]: ArmObservations("succeeded", treatment), _arms[1]: ArmObservations("succeeded", control)},+ judge_mixed_cases,+ )+ cases, _ = index.dimensions(_arms)+ treatment_cases, control_cases = cases[_arms[0]], cases[_arms[1]]+ treatment_scores: dict[str, float] = {}+ control_scores: dict[str, float] = {}+ compared: set[str] = set()+ failures = _clean_load_failures(load_failures_by_case)+ for case_id in sorted(set(treatment_cases) & set(control_cases)):+ if set(failures.get(case_id, {})) & set(_arms):+ continue+ treatment_case, control_case = _overall_aligned(treatment_cases[case_id], control_cases[case_id])+ shared = sorted(set(treatment_case) & set(control_case))+ if not shared:+ continue+ treatment_scores[case_id] = math.fsum(treatment_case[dim] for dim in shared) / len(shared)+ control_scores[case_id] = math.fsum(control_case[dim] for dim in shared) / len(shared)+ compared.update(shared)+ return treatment_scores, control_scores, [dim for dim in _DIMENSION_ORDER if dim in compared]+++# ---------------------------------------------------------------------------+# Lift uncertainty+# ---------------------------------------------------------------------------+++def _percentile(sorted_values: Sequence[float], quantile: float) -> float:+ """Linearly interpolated percentile of an already sorted sequence."""+ if len(sorted_values) == 1:+ return sorted_values[0]+ position = quantile * (len(sorted_values) - 1)+ lower = math.floor(position)+ upper = min(lower + 1, len(sorted_values) - 1)+ fraction = position - lower+ return sorted_values[lower] + (sorted_values[upper] - sorted_values[lower]) * fraction+++# A percentile bootstrap over a handful of cases is too narrow (about 82%+# coverage at 5 cases for a nominal 95%), and at small n the narrow intervals+# are the overconfident ones. "adequate" therefore also needs this many cases.+LIFT_CI_ADEQUATE_MIN_CASES = 10+LIFT_INTERVAL_METHOD = "expanded_percentile"+++def lift_precision(n_cases: int, ci_low: float | None, ci_high: float | None) -> str:+ """Classify how informative a lift interval is, by case count and width."""+ if n_cases < LIFT_CI_MIN_PAIRED_CASES or ci_low is None or ci_high is None:+ return PRECISION_INSUFFICIENT+ if n_cases < LIFT_CI_ADEQUATE_MIN_CASES or ci_high - ci_low > LIFT_CI_LOW_PRECISION_WIDTH:+ return PRECISION_LOW+ return PRECISION_ADEQUATE+++def _normal_cdf(value: float) -> float:+ return 0.5 * math.erfc(-value / math.sqrt(2.0))+++def _incomplete_beta_fraction(a: float, b: float, x: float) -> float:+ """Continued fraction for the regularized incomplete beta function (modified Lentz)."""+ tiny = 1e-300+ c = 1.0+ d = 1.0 - (a + b) * x / (a + 1.0)+ d = 1.0 / (d if abs(d) > tiny else tiny)+ fraction = d+ for m in range(1, 300):+ for numerator in (+ m * (b - m) * x / ((a + 2 * m - 1) * (a + 2 * m)),+ -(a + m) * (a + b + m) * x / ((a + 2 * m) * (a + 2 * m + 1)),+ ):+ d = 1.0 + numerator * d+ d = 1.0 / (d if abs(d) > tiny else tiny)+ c = 1.0 + numerator / c+ c = c if abs(c) > tiny else tiny+ fraction *= c * d+ if abs(c * d - 1.0) < 1e-15:+ break+ return fraction+++def _student_t_cdf(value: float, degrees_of_freedom: int) -> float:+ """Student's t cumulative distribution, via the regularized incomplete beta function."""+ df = float(degrees_of_freedom)+ x = df / (df + value * value)+ a, b = df / 2.0, 0.5+ log_front = math.lgamma(a + b) - math.lgamma(a) - math.lgamma(b) + a * math.log(x) + b * math.log1p(-x)+ if x < (a + 1.0) / (a + b + 2.0):+ incomplete = math.exp(log_front) * _incomplete_beta_fraction(a, b, x) / a+ else:+ incomplete = 1.0 - math.exp(log_front) * _incomplete_beta_fraction(b, a, 1.0 - x) / b+ tail = 0.5 * incomplete+ return 1.0 - tail if value > 0 else tail+++def _student_t_quantile(probability: float, degrees_of_freedom: int) -> float:+ """Upper quantile (probability above 0.5) of Student's t, by bisection."""+ high = 1.0+ while _student_t_cdf(high, degrees_of_freedom) < probability and high < 1e6:+ high *= 2.0+ low = 0.0+ for _ in range(200):+ middle = (low + high) / 2.0+ if _student_t_cdf(middle, degrees_of_freedom) < probability:+ low = middle+ else:+ high = middle+ return (low + high) / 2.0+++def expanded_tail(n_cases: int, confidence: float) -> float:+ """Lower-tail probability of an expanded percentile bootstrap interval.++ A plain percentile bootstrap undercovers at small n. Hesterberg's expanded+ percentile interval uses the percentiles at ``Phi(-sqrt(n / (n - 1)) *+ t_{alpha/2, n-1})`` instead of ``alpha / 2``, which widens the interval at+ small n and converges to the plain interval as n grows.+ """+ alpha = 1.0 - confidence+ if n_cases < 2:+ return alpha / 2.0+ t_value = _student_t_quantile(1.0 - alpha / 2.0, n_cases - 1)+ return _normal_cdf(-math.sqrt(n_cases / (n_cases - 1)) * t_value)+++def _resample_means(deltas: Sequence[float], *, resamples: int, seed: int) -> list[float]:+ """The mean of each case resample, in draw order, from a ``random.Random(seed)`` stream.++ ``rng.choices`` picks every index from the seed and ``len(deltas)`` alone,+ never from the values, so two delta lists of the same length are resampled+ at the same case indices. That is why the lift ladder's steps add up in+ every resample, not only on their means.+ """+ n_cases = len(deltas)+ rng = random.Random(seed)+ return [math.fsum(rng.choices(deltas, k=n_cases)) / n_cases for _ in range(resamples)]+++def paired_case_bootstrap(+ treatment: Mapping[str, float],+ control: Mapping[str, float],+ *,+ resamples: int = LIFT_BOOTSTRAP_RESAMPLES,+ seed: int = LIFT_BOOTSTRAP_SEED,+ confidence: float = LIFT_BOOTSTRAP_CONFIDENCE,+ _means: Sequence[float] | None = None,+) -> dict[str, Any]:+ """Expanded percentile bootstrap CI for the mean paired per-case lift.++ ``treatment`` and ``control`` map case ids to case-level mean scores. Only+ cases present in both arms are paired. Whole cases are resampled with+ replacement from a ``random.Random(seed)`` stream, so the interval is fully+ deterministic for a given input. The percentiles are widened for small n+ (:func:`expanded_tail`), because a plain percentile bootstrap over 5-10+ cases is overconfident. ``resamples`` is a minimum: with at least+ ``LIFT_CI_MIN_PAIRED_CASES`` cases, enough are drawn that each bound rests on+ ``LIFT_BOOTSTRAP_TAIL_RESAMPLES`` resampled means, so the bounds do not move+ with the seed. With fewer cases the bounds are the extreme resampled means,+ which the minimum already reaches.+ """+ if type(resamples) is not int or not 1 <= resamples <= MAX_BOOTSTRAP_RESAMPLES:+ raise ValueError("resamples must be positive")+ if not 0.0 < confidence < 1.0:+ raise ValueError("confidence must be between 0 and 1")+ paired = sorted(+ case+ for case in set(treatment) & set(control)+ if _case_id(case) is not None+ and _logical_score(treatment[case]) is not None+ and _logical_score(control[case]) is not None+ )+ deltas = [float(treatment[case_id]) - float(control[case_id]) for case_id in paired]+ n_cases = len(deltas)+ result: dict[str, Any] = {+ "estimate": None,+ "ci_low": None,+ "ci_high": None,+ "confidence": confidence,+ "method": LIFT_UNCERTAINTY_METHOD,+ "interval": LIFT_INTERVAL_METHOD,+ "resamples": resamples,+ "seed": seed,+ "n_cases": n_cases,+ "precision": PRECISION_INSUFFICIENT,+ "ci_includes_zero": None,+ }+ if not n_cases:+ return result+ if n_cases < 2:+ # One case cannot be resampled: a zero-width interval would look final.+ result["estimate"] = round(deltas[0], 4)+ return result++ tail = expanded_tail(n_cases, confidence)+ resamples = _effective_resamples(n_cases, resamples, confidence)+ means = sorted(_resample_means(deltas, resamples=resamples, seed=seed) if _means is None else _means)+ ci_low = _percentile(means, tail)+ ci_high = _percentile(means, 1.0 - tail)+ rounded_low = round(ci_low, 4)+ rounded_high = round(ci_high, 4)+ result.update(+ {+ "estimate": round(math.fsum(deltas) / n_cases, 4),+ "ci_low": rounded_low,+ "ci_high": rounded_high,+ "resamples": resamples,+ "precision": lift_precision(n_cases, ci_low, ci_high),+ "ci_includes_zero": rounded_low <= 0.0 <= rounded_high,+ }+ )+ return result+++def _effective_resamples(+ n_cases: int, resamples: int = LIFT_BOOTSTRAP_RESAMPLES, confidence: float = LIFT_BOOTSTRAP_CONFIDENCE+) -> int:+ if n_cases < LIFT_CI_MIN_PAIRED_CASES:+ return resamples+ tail = expanded_tail(n_cases, confidence)+ required = math.ceil(LIFT_BOOTSTRAP_TAIL_RESAMPLES / tail) if tail > 0 else MAX_BOOTSTRAP_RESAMPLES+ return min(MAX_BOOTSTRAP_RESAMPLES, max(resamples, required))+++LIFT_BASIS_SHARED_DIMENSIONS = "shared_dimensions_case_weighted"+++def paired_lift(+ treatment: ArmObservations | None,+ control: ArmObservations | None,+ *,+ treatment_arm: str,+ control_arm: str,+ expected_case_ids: Sequence[str] | None = None,+ judge_mixed_cases: Mapping[str, Collection[str]] | None = None,+ load_failures_by_case: LoadFailures | None = None,+ _index: _CaseIndex | None = None,+) -> dict[str, Any] | None:+ """Paired lift of *treatment* over *control*, with its interval, on one basis.++ Both arms are scored per case on the dimensions both scored for that case+ (see :func:`shared_case_scores`), so the headline lift, the arm scores+ shown next to it and the interval are all the same case-weighted paired+ mean. ``treatment_score - control_score`` equals ``estimate`` up to rounding.+ Returns ``None`` when no case was scored in both arms.+ """+ if treatment is None or control is None:+ return None+ treatment_scores, control_scores, dimensions = shared_case_scores(+ treatment.trials,+ control.trials,+ judge_mixed_cases=judge_mixed_cases,+ load_failures_by_case=load_failures_by_case,+ _index=_index,+ _arms=(treatment_arm, control_arm),+ )+ if not treatment_scores:+ return None+ result = paired_case_bootstrap(treatment_scores, control_scores)+ expected_ids = [str(case_id) for case_id in (expected_case_ids or []) if str(case_id)]+ observed_ids = {trial.case_id for trial in (*treatment.trials, *control.trials)}+ expected_cases = len(set(expected_ids)) if expected_ids else len(observed_ids)+ failed_arms = [+ arm+ for arm, observed in ((treatment_arm, treatment), (control_arm, control))+ if not observed.succeeded or observed.job_failure or not observed.observations_complete+ ]+ result.update(+ {+ "basis": LIFT_BASIS_SHARED_DIMENSIONS,+ "dimensions": dimensions,+ "treatment_arm": treatment_arm,+ "control_arm": control_arm,+ "treatment_score": _round(_mean(list(treatment_scores.values()))),+ "control_score": _round(_mean(list(control_scores.values()))),+ "expected_cases": expected_cases,+ "partial": bool(failed_arms) or int(result["n_cases"]) < expected_cases,+ "failed_arms": failed_arms,+ }+ )+ return result+++CASE_TYPE_CROSS_COMPONENT = "cross_component"+CASE_TYPE_OTHER = "other"+++def lift_by_case_type(+ treatment: ArmObservations | None,+ control: ArmObservations | None,+ cross_component_case_ids: Collection[str],+ *,+ _index: _CaseIndex | None = None,+ load_failures_by_case: LoadFailures | None = None,+ _arms: tuple[str, str] = ("treatment", "control"),+) -> dict[str, Any] | None:+ """Split a paired lift into ``cross_component`` cases and all ``other`` cases.++ Report-only. Each side reuses the paired per-case scores and the paired+ case bootstrap of :func:`paired_lift`, so the case-weighted mean of the two+ sides equals the whole lift. A side with fewer than+ ``LIFT_CI_MIN_PAIRED_CASES`` pairs gets a point estimate and a reason, no interval.+ """+ if treatment is None or control is None:+ return None+ treatment_scores, control_scores, _ = shared_case_scores(+ treatment.trials, control.trials, load_failures_by_case=load_failures_by_case, _index=_index, _arms=_arms+ )+ if not treatment_scores:+ return None+ cross = {str(case_id) for case_id in cross_component_case_ids}+ split: dict[str, Any] = {}+ for case_type, label in ((CASE_TYPE_CROSS_COMPONENT, "cross-component"), (CASE_TYPE_OTHER, "other")):+ is_cross = case_type == CASE_TYPE_CROSS_COMPONENT+ ids = [case_id for case_id in treatment_scores if (case_id in cross) is is_cross]+ result = paired_case_bootstrap(+ {case_id: treatment_scores[case_id] for case_id in ids},+ {case_id: control_scores[case_id] for case_id in ids},+ )+ n_cases = int(result["n_cases"])+ reason = None+ if not n_cases:+ reason = f"No {label} case was scored in both arms."+ elif n_cases < LIFT_CI_MIN_PAIRED_CASES:+ reason = (+ f"Only {n_cases} paired {label} case(s); "+ f"at least {LIFT_CI_MIN_PAIRED_CASES} are needed for an interval."+ )+ interval = reason is None+ split[case_type] = {+ "n": n_cases,+ "lift": result["estimate"],+ "ci_low": result["ci_low"] if interval else None,+ "ci_high": result["ci_high"] if interval else None,+ "reason": reason,+ }+ return split+++# ---------------------------------------------------------------------------+# Lift ladder: four arms on one common basis+# ---------------------------------------------------------------------------++LADDER_SCHEMA_VERSION = "1.0"+LADDER_BASIS = "common_dimensions_all_arms"+GLUE_BASIS = "common_dimensions_skill_arms"+# Every arm can score these, the no-plugin arm included (it has no skill to discover or use).+COMMON_BASIS_DIMENSIONS = ("security", "correctness", "effectiveness")+# The three arms that stage skills can all score every dimension, routing and wasted work included.+GLUE_BASIS_DIMENSIONS = tuple(DIMENSION_MAPPING)+# The ladder's arms, cheapest first: role -> arm key.+LADDER_ARMS = {"none": ARM_WITHOUT, "skills": ARM_SUM_OF_PARTS, "loose": ARM_LOOSE_PARTS, "full": ARM_WITH}+# step -> (treatment arm, control arm). skill + capability + integration = plugin, per case.+LADDER_STEPS = {+ "skill": (ARM_SUM_OF_PARTS, ARM_WITHOUT),+ "capability": (ARM_LOOSE_PARTS, ARM_SUM_OF_PARTS),+ "integration": (ARM_WITH, ARM_LOOSE_PARTS),+ "plugin": (ARM_WITH, ARM_WITHOUT),+}+# Not a rung: capability + integration, the Bundle lift on the ladder's common basis.+LADDER_BUNDLE = (ARM_WITH, ARM_SUM_OF_PARTS)+_GLUE_ARMS = (ARM_SUM_OF_PARTS, ARM_LOOSE_PARTS, ARM_WITH)++EXCLUDED_LOAD_FAILED = "load_failed"+EXCLUDED_NOT_SCORED = "not_scored"+EXCLUDED_NO_COMMON_DIMENSION = "no_common_dimension"++# Per case, the arms whose staged parts did not load: ``{arm: reason}``, or the loose-parts staging's own records+# ``[{arm, attempt, piece, reason}]`` (``skillevaluator.tier3.loose_parts.load_failures_by_case``), folded into one+# bounded reason per arm.+LoadFailures = Mapping[str, Mapping[str, str] | Sequence[Mapping[str, Any]]]+_LOAD_FAILURES_PER_ARM = 4+_LOAD_FAILURE_PIECE_CHARS = 100+_LOAD_FAILURE_REASON_CHARS = 120+++@dataclass(frozen=True)+class CommonCaseScores:+ """Every arm scored on one basis: the same cases, and per case the same dimensions.++ ``scores[arm][case_id]`` is the arm's mean over the dimensions every arm+ scored for that case. Every arm holds exactly ``case_ids``. ``dimensions``+ lists the dimensions used for at least one case, in report order.+ ``excluded_cases`` holds one entry per dropped case: ``case``, ``reason``+ (``load_failed``, ``not_scored`` or ``no_common_dimension``, in that+ precedence), ``missing_arms`` (arms with no score for it) and+ ``load_failures`` (``{arm: reason}``).+ """++ scores: Mapping[str, Mapping[str, float]]+ case_ids: tuple[str, ...]+ dimensions: tuple[str, ...]+ excluded_cases: tuple[Mapping[str, Any], ...] = ()+ judge_mixed: Mapping[str, int] = field(default_factory=dict)+ case_dimensions: Mapping[str, Mapping[str, Mapping[str, float]]] = field(default_factory=dict, repr=False)+ policy_conflicts: tuple[str, ...] = ()+++def _shared_case_score(+ case_dimensions: Mapping[str, Mapping[str, Mapping[str, float]]],+ case_id: str,+ dimensions: Collection[str],+) -> tuple[list[str], dict[str, float]]:+ """The *dimensions* every arm scored for one case, and each arm's mean over them."""+ shared = set(dimensions)+ for by_case in case_dimensions.values():+ shared &= set(by_case.get(case_id, {}))+ ordered = [dimension for dimension in _DIMENSION_ORDER if dimension in shared]+ if not ordered or not case_dimensions:+ return [], {}+ return ordered, {+ arm: math.fsum(by_case[case_id][dimension] for dimension in ordered) / len(ordered)+ for arm, by_case in case_dimensions.items()+ }+++def _failure_records_by_arm(records: Sequence[Any]) -> dict[str, str]:+ """The staging's failure records (``{arm, attempt, piece, reason}``) as one bounded reason per arm.++ One piece that failed in several attempts is listed once with all of them+ (its first reason), so a second failing piece is not hidden behind repeats.+ """+ by_arm: dict[str, dict[str, tuple[list[int], str]]] = {}+ for record in records:+ if not isinstance(record, Mapping):+ continue+ piece = str(record.get("piece") or "a staged part")[:_LOAD_FAILURE_PIECE_CHARS]+ reason = str(record.get("reason") or "not loaded")[:_LOAD_FAILURE_REASON_CHARS]+ attempts, _ = by_arm.setdefault(str(record.get("arm") or "unknown"), {}).setdefault(piece, ([], reason))+ attempt = record.get("attempt")+ if type(attempt) is int and attempt not in attempts:+ attempts.append(attempt)+ folded: dict[str, str] = {}+ for arm, pieces in by_arm.items():+ items = []+ for piece, (attempts, reason) in pieces.items():+ label = "attempt" if len(attempts) == 1 else "attempts"+ where = f"{piece} ({label} {', '.join(map(str, sorted(attempts)))})" if attempts else piece+ items.append(f"{where}: {reason}")+ more = len(items) - _LOAD_FAILURES_PER_ARM+ folded[arm] = "; ".join(items[:_LOAD_FAILURES_PER_ARM]) + (f" (+{more} more)" if more > 0 else "")+ return folded+++def _clean_load_failures(load_failures_by_case: LoadFailures | None) -> dict[str, dict[str, str]]:+ """``{case: {arm: reason}}`` from either shape :data:`LoadFailures` accepts; an empty entry is no failure."""+ failures: dict[str, dict[str, str]] = {}+ for case_id, recorded in (load_failures_by_case or {}).items():+ if isinstance(recorded, Mapping):+ by_arm = {str(arm): str(reason) for arm, reason in recorded.items()}+ elif isinstance(recorded, Sequence) and not isinstance(recorded, str | bytes):+ by_arm = _failure_records_by_arm(recorded)+ else:+ continue+ if by_arm:+ failures[str(case_id)] = by_arm+ return failures+++def common_case_scores(+ arms: Mapping[str, ArmObservations],+ *,+ dimensions: Collection[str] = COMMON_BASIS_DIMENSIONS,+ expected_case_ids: Sequence[str] | None = None,+ load_failures_by_case: LoadFailures | None = None,+ judge_mixed_cases: Mapping[str, Collection[str]] | None = None,+ _index: _CaseIndex | None = None,+) -> CommonCaseScores:+ """Score every arm per case on the *dimensions* that every arm scored for that case.++ Listwise: a case is kept only when every arm scored it, no arm failed its+ load verification for it (``load_failures_by_case``, see+ :data:`LoadFailures`; any arm counts), and the arms share at least one of+ *dimensions* for it. Otherwise it is dropped from every arm and listed in+ ``excluded_cases``. Correctness is N/A in every arm for a case without+ ``ground_truth``, so it drops out of that case for all arms alike. An+ expected case no arm scored is listed with every arm missing.+ """+ names = list(arms)+ index = _index or _CaseIndex(arms, judge_mixed_cases)+ case_dimensions, judge_mixed = index.dimensions(names)+ policy_conflicts = index.conflicts.get(tuple(names), ())+ failures = _clean_load_failures(load_failures_by_case)+ universe = sorted(+ {+ *(case_id for by_case in case_dimensions.values() for case_id in by_case),+ *(case_id for by_case in index.rows.values() for case_id in by_case),+ *(case_id for case_id in expected_case_ids or [] if _case_id(case_id) is not None),+ }+ )+ scores: dict[str, dict[str, float]] = {name: {} for name in names}+ kept: list[str] = []+ used: set[str] = set()+ excluded: list[Mapping[str, Any]] = []+ for case_id in universe:+ missing = [name for name in names if not case_dimensions[name].get(case_id)]+ failed = failures.get(case_id, {})+ shared, case_scores = ([], {}) if missing else _shared_case_score(case_dimensions, case_id, dimensions)+ if (+ not missing+ and not shared+ and all(set(by_case[case_id]) == {OVERALL_DIMENSION} for by_case in case_dimensions.values())+ ):+ shared, case_scores = _shared_case_score(case_dimensions, case_id, (OVERALL_DIMENSION,))+ if failed or missing or not shared:+ if failed:+ reason = EXCLUDED_LOAD_FAILED+ elif case_id in policy_conflicts:+ reason = "score_policy_mismatch"+ elif missing:+ reason = EXCLUDED_NOT_SCORED+ else:+ reason = EXCLUDED_NO_COMMON_DIMENSION+ excluded.append({"case": case_id, "reason": reason, "missing_arms": missing, "load_failures": dict(failed)})+ continue+ kept.append(case_id)+ used.update(shared)+ for name, value in case_scores.items():+ scores[name][case_id] = value+ return CommonCaseScores(+ scores=scores,+ case_ids=tuple(kept),+ dimensions=tuple(dimension for dimension in _DIMENSION_ORDER if dimension in used),+ excluded_cases=tuple(excluded),+ judge_mixed=judge_mixed,+ case_dimensions=case_dimensions,+ policy_conflicts=policy_conflicts,+ )+++_STEP_FIELDS = ("estimate", "ci_low", "ci_high", "n_cases", "precision")+++def _ladder_step(+ scores: Mapping[str, Mapping[str, float]], arms: tuple[str, str], case_ids: Sequence[str]+) -> dict[str, Any]:+ """One step's paired bootstrap over *case_ids* (default seed and resamples, like every lift)."""+ treatment, control = arms+ return paired_case_bootstrap(+ {case_id: scores[treatment][case_id] for case_id in case_ids},+ {case_id: scores[control][case_id] for case_id in case_ids},+ )+++def _step_fields(result: Mapping[str, Any], *, split: bool) -> dict[str, Any]:+ """A step's reported fields. A case-type group below the pair minimum keeps its estimate but no interval."""+ step = {field: result[field] for field in _STEP_FIELDS}+ if split and int(result["n_cases"]) < LIFT_CI_MIN_PAIRED_CASES:+ step["ci_low"] = step["ci_high"] = None+ return step+++def _ladder_rungs(+ scores: Mapping[str, Mapping[str, float]], case_ids: Sequence[str], *, split: bool = False+) -> tuple[dict[str, dict[str, Any]], dict[str, Any], dict[str, Any]]:+ """The four steps and the common-basis Bundle lift over one case set, plus the first step's full result.++ *split* marks a case-type group: as in :func:`lift_by_case_type`, fewer+ than ``LIFT_CI_MIN_PAIRED_CASES`` pairs give a point estimate only.+ """+ definitions = {**LADDER_STEPS, "bundle": LADDER_BUNDLE}+ deltas = {+ name: [scores[treatment][case] - scores[control][case] for case in case_ids]+ for name, (treatment, control) in definitions.items()+ }+ resamples = _effective_resamples(len(case_ids))+ means: dict[str, list[float]] = {name: [] for name in definitions}+ if len(case_ids) >= 2:+ rng = random.Random(LIFT_BOOTSTRAP_SEED)+ for _ in range(resamples):+ indices = rng.choices(range(len(case_ids)), k=len(case_ids))+ for name, values in deltas.items():+ means[name].append(math.fsum(values[index] for index in indices) / len(indices))+ results = {+ name: paired_case_bootstrap(+ {case: scores[treatment][case] for case in case_ids},+ {case: scores[control][case] for case in case_ids},+ _means=means[name],+ )+ for name, (treatment, control) in definitions.items()+ }+ bundle = results.pop("bundle")+ steps = {name: _step_fields(result, split=split) for name, result in results.items()}+ return steps, _step_fields(bundle, split=split), results["skill"]+++def _ladder_glue(+ arms: Mapping[str, ArmObservations],+ basis: CommonCaseScores,+ cross: set[str] | None,+ failed_arms: Sequence[str],+ *,+ index: _CaseIndex,+) -> dict[str, Any]:+ """Full plugin minus loose parts on the cross-component common cases, on all five dimensions.++ The cases are the ladder's own (so a case dropped anywhere is dropped+ here too); per case the dimensions are the ones the three skill arms all+ scored, which keeps routing (discoverability) and wasted work+ (efficiency), where glue mostly shows.+ """+ glue_failed = [arm for arm in failed_arms if arm in _GLUE_ARMS]+ if cross is None:+ case_ids: list[str] = []+ excluded: list[str] = []+ reason: str | None = "The run recorded no case types, so no case is known to be cross-component."+ else:+ case_ids = [case_id for case_id in basis.case_ids if case_id in cross]+ excluded = [str(entry["case"]) for entry in basis.excluded_cases if entry["case"] in cross]+ reason = None+ # Baseline N/A metrics do not erase routing/efficiency from the skill arms.+ case_dimensions, _ = index.dimensions(_GLUE_ARMS)+ full: dict[str, float] = {}+ loose: dict[str, float] = {}+ used: set[str] = set()+ for case_id in case_ids:+ shared, case_scores = _shared_case_score(case_dimensions, case_id, GLUE_BASIS_DIMENSIONS)+ if not shared: # cannot happen for a ladder case: every arm shares a common-basis dimension+ continue+ used.update(shared)+ full[case_id], loose[case_id] = case_scores[ARM_WITH], case_scores[ARM_LOOSE_PARTS]+ result = paired_case_bootstrap(full, loose)+ n_cases = int(result["n_cases"])+ # The glue is read on one case type, so it follows the case-type split: no interval below the pair minimum.+ interval = n_cases >= LIFT_CI_MIN_PAIRED_CASES+ if reason is None and not n_cases:+ reason = "No cross-component case was scored in every arm."+ elif reason is None and n_cases < LIFT_CI_MIN_PAIRED_CASES:+ reason = (+ f"Only {n_cases} paired cross-component case(s); "+ f"at least {LIFT_CI_MIN_PAIRED_CASES} are needed for an interval."+ )+ return {+ "case_type": CASE_TYPE_CROSS_COMPONENT,+ "basis": GLUE_BASIS,+ "arms": list(_GLUE_ARMS),+ "basis_dimensions": [dimension for dimension in _DIMENSION_ORDER if dimension in used],+ "n": n_cases,+ "lift": result["estimate"],+ "ci_low": result["ci_low"] if interval else None,+ "ci_high": result["ci_high"] if interval else None,+ "precision": result["precision"],+ "confidence": result["confidence"],+ "partial": bool(excluded or glue_failed),+ "failed_arms": glue_failed,+ "excluded_cases": excluded,+ "reason": reason,+ }+++def lift_ladder(+ arms: Mapping[str, ArmObservations],+ cross_component_case_ids: Collection[str] | None,+ *,+ expected_case_ids: Sequence[str] | None = None,+ load_failures_by_case: LoadFailures | None = None,+ judge_mixed_cases: Mapping[str, Collection[str]] | None = None,+ _index: _CaseIndex | None = None,+) -> dict[str, Any]:+ """Split the plugin lift into four steps that add up, on one common basis.++ Report-only. ``arms`` is keyed like :data:`LADDER_ARMS`; an arm that is+ absent leaves every case unscored and is listed in ``failed_arms``. All+ four arms are scored by :func:`common_case_scores`, so a case is kept only+ when every arm scored it and passed load verification. Each step+ (:data:`LADDER_STEPS`) and the common-basis Bundle lift go through+ :func:`paired_case_bootstrap` over the same cases with the same seed, so+ ``skill + capability + integration = plugin`` per case, on the means, and+ in every resample (the intervals themselves are percentiles and do not+ add). ``by_case_type`` repeats the steps on the cross-component cases and+ on the others; ``None`` when the case types were not recorded. ``glue``+ is the Integration lift on cross-component cases on the five-dimension+ basis of the three skill arms; the report reads its verdict there.+ """+ observed = {arm: arms.get(arm) for arm in LADDER_ARMS.values()}+ failed_arms = [+ arm+ for arm, observations in observed.items()+ if observations is None+ or not observations.succeeded+ or observations.job_failure+ or not observations.observations_complete+ ]+ ladder_arms = {arm: observations or ArmObservations("missing") for arm, observations in observed.items()}+ index = _index or _CaseIndex(ladder_arms, judge_mixed_cases)+ basis = common_case_scores(+ ladder_arms,+ expected_case_ids=expected_case_ids,+ load_failures_by_case=load_failures_by_case,+ judge_mixed_cases=judge_mixed_cases,+ _index=index,+ )+ steps, bundle, first = _ladder_rungs(basis.scores, basis.case_ids)+ cross = None if cross_component_case_ids is None else {str(case_id) for case_id in cross_component_case_ids}+ by_case_type: dict[str, Any] | None = None+ if cross is not None:+ by_case_type = {}+ for case_type in (CASE_TYPE_CROSS_COMPONENT, CASE_TYPE_OTHER):+ is_cross = case_type == CASE_TYPE_CROSS_COMPONENT+ ids = [case_id for case_id in basis.case_ids if (case_id in cross) is is_cross]+ group_steps, group_bundle, _ = _ladder_rungs(basis.scores, ids, split=True)+ by_case_type[case_type] = {"n_cases": len(ids), "steps": group_steps, "bundle": group_bundle}+ expected_ids = {str(case_id) for case_id in expected_case_ids or [] if str(case_id)}+ expected_cases = len(expected_ids) if expected_ids else len(basis.case_ids) + len(basis.excluded_cases)+ return {+ "schema_version": LADDER_SCHEMA_VERSION,+ "advisory": True,+ "report_only": True,+ "arms": dict(LADDER_ARMS),+ "basis": LADDER_BASIS,+ "dimensions": list(basis.dimensions),+ "n_cases": len(basis.case_ids),+ "expected_cases": expected_cases,+ "partial": bool(basis.excluded_cases or failed_arms or len(basis.case_ids) < expected_cases),+ "failed_arms": failed_arms,+ "excluded_cases": [dict(entry) for entry in basis.excluded_cases],+ "judge_mixed_cases": dict(basis.judge_mixed),+ "score_policy_mismatches": list(basis.policy_conflicts),+ "bootstrap": {field: first[field] for field in ("method", "interval", "confidence", "seed", "resamples")},+ "steps": steps,+ "bundle": bundle,+ "by_case_type": by_case_type,+ "glue": _ladder_glue(ladder_arms, basis, cross, failed_arms, index=index),+ }+++# ---------------------------------------------------------------------------+# Reliability (pass^k)+# ---------------------------------------------------------------------------+++def reliability_summary(pass_summary: Mapping[str, Any], *, stop_on_pass: bool) -> dict[str, Any] | None:+ """Report pass@k next to pass^k (every one of the k attempts passed).++ pass@k keeps its existing meaning. pass^k needs all k attempts to be+ observed, so it is ``None`` when ``stop_on_pass`` truncated the attempts.+ Unscored attempts never count as passes. The published ``cases`` block is+ capped for size, so the collector's ``_pairing_cases`` (every case and+ attempt) is read when present.+ """+ if not pass_summary:+ return None+ k = int(pass_summary.get("k") or 0)+ n_cases = int(pass_summary.get("total_cases") or 0)+ cases = pass_summary.get("_pairing_cases", pass_summary.get("cases"))+ pass_hat_k: float | None = None+ if k > 0 and n_cases > 0 and isinstance(cases, Mapping) and not (stop_on_pass and k > 1):+ all_passed = 0+ for case in cases.values():+ if not isinstance(case, Mapping) or case.get("extra_case"):+ continue+ attempts = [row for row in case.get("attempts") or [] if isinstance(row, Mapping)]+ if len(attempts) >= k and all(bool(row.get("passed")) for row in attempts[:k]):+ all_passed += 1+ pass_hat_k = round(all_passed / n_cases, 4)+ return {+ "pass_at_k": finite_number(pass_summary.get("rate")),+ "pass_hat_k": pass_hat_k,+ "k": k,+ "n_cases": n_cases,+ }+++def arm_not_applicable_metrics(trials: Sequence[TrialObservation]) -> list[str]:+ """Metrics that no trial of an arm scored, in metric-set order.++ In an arm that succeeded these are not applicable, for example the skill+ metrics of an arm without the skill. Each arm's pass@k uses only the+ metrics it scored, so arms with different lists are not a like-for-like+ pass@k comparison.+ """+ active: list[str] = []+ scored: set[str] = set()+ for trial in trials:+ reward = dict(trial.reward)+ _, metrics = metric_set_for_reward(reward)+ for metric in metrics:+ if metric not in active:+ active.append(metric)+ if metric_value(reward, metric) is not None:+ scored.add(metric)+ return [metric for metric in active if metric not in scored]+++# ---------------------------------------------------------------------------+# Tokens, cost and token efficiency+# ---------------------------------------------------------------------------+++def uncached_tokens(usage: Mapping[str, Any]) -> float | None:+ """Return uncached prompt plus completion tokens, or ``None`` if unknown.++ Prompt counters include cached tokens (ATIF ``total_prompt_tokens`` and+ Harbor ``n_input_tokens`` both do), so cached tokens are subtracted.+ """+ prompt = finite_number(usage.get("prompt_tokens"), non_negative=True)+ completion = finite_number(usage.get("completion_tokens"), non_negative=True)+ if prompt is None or completion is None:+ return None+ cached = finite_number(usage.get("cached_tokens"), non_negative=True) or 0.0+ return max(prompt - cached, 0.0) + completion+++def token_efficiency_score(tokens: float | None) -> float | None:+ """Soft-asymptote token score: ``1 / (1 + tokens / TOKEN_EFFICIENCY_HALF_LIFE)``."""+ if tokens is None or tokens <= 0:+ return None+ return 1.0 / (1.0 + tokens / TOKEN_EFFICIENCY_HALF_LIFE)+++def arm_token_efficiency(trials: Sequence[TrialObservation]) -> float | None:+ """Mean per-trial token efficiency; ``None`` unless every trial has token counts."""+ scores = [token_efficiency_score(uncached_tokens(trial.usage)) for trial in trials]+ if not scores or any(score is None for score in scores):+ return None+ return _round(_mean([score for score in scores if score is not None]))+++def arm_cost(trials: Sequence[TrialObservation], *, successes: int) -> dict[str, Any]:+ """Token and USD cost per passed case, counting every attempt.++ Totals are only reported when every attempt carries the counter, so a+ partial sum is never presented as the arm's cost. USD comes only from+ costs reported by Harbor or the agent trajectory; nothing is priced here.+ """+ token_counts = [uncached_tokens(trial.usage) for trial in trials]+ total_tokens = (+ round(math.fsum(value for value in token_counts if value is not None))+ if token_counts and all(value is not None for value in token_counts)+ else None+ )+ costs = [finite_number(trial.usage.get("cost_usd"), non_negative=True) for trial in trials]+ total_usd = (+ round(math.fsum(value for value in costs if value is not None), 6)+ if costs and all(value is not None for value in costs)+ else None+ )+ return {+ "tokens_per_success": round(total_tokens / successes, 2) if total_tokens is not None and successes else None,+ "usd_per_success": round(total_usd / successes, 6) if total_usd is not None and successes else None,+ "total_tokens": total_tokens,+ "total_usd": total_usd,+ "successes": successes,+ }+++# ---------------------------------------------------------------------------+# Measured always-on context cost+# ---------------------------------------------------------------------------+++#: Fewer paired cases than this give a delta with status ``insufficient``: shown, but not a measurement.+CONTEXT_COST_MIN_PAIRED_CASES = 3+++@dataclass+class _FirstTurns:+ """One arm's per-case mean first-turn prompt tokens, and the trials left out."""++ by_case: dict[str, float]+ used: int = 0+ hosted_search: int = 0+ missing: int = 0 # usage was read, but it has no first-turn count+ no_usage: int = 0 # no usage was collected for the trial at all+++def _case_first_turn_tokens(trials: Sequence[TrialObservation]) -> _FirstTurns:+ by_case: dict[str, list[float]] = {}+ turns = _FirstTurns({})+ for trial in trials:+ if _case_id(trial.case_id) is None:+ continue+ tokens = finite_number(trial.usage.get("first_turn_prompt_tokens"), non_negative=True)+ if tokens is not None:+ turns.used += 1+ by_case.setdefault(trial.case_id, []).append(tokens)+ elif finite_number(trial.usage.get("first_turn_hosted_search"), non_negative=True):+ turns.hosted_search += 1+ elif trial.usage:+ turns.missing += 1+ else:+ turns.no_usage += 1+ turns.by_case = {case_id: math.fsum(values) / len(values) for case_id, values in by_case.items()}+ return turns+++def _left_out(arm_name: str, turns: _FirstTurns) -> list[str]:+ parts = []+ if turns.no_usage:+ parts.append(f"no usage was collected for {turns.no_usage} trial(s) of the {arm_name}")+ if turns.missing:+ parts.append(f"{turns.missing} trial(s) of the {arm_name} have no first-turn prompt token count")+ if turns.hosted_search:+ parts.append(+ f"{turns.hosted_search} trial(s) of the {arm_name} ran a hosted web search inside the first model call, "+ "whose results count as that call's input"+ )+ return parts+++def context_cost_measured(+ with_arm: ArmObservations | None,+ without_arm: ArmObservations | None,+ *,+ expected_case_ids: Sequence[str] | None = None,+) -> dict[str, Any]:+ """Paired mean first-turn prompt-token delta (with minus without), by case.++ The first LLM turn carries the always-on context (system prompt, skill and+ rule preambles, tool schemas) before task work starts, so the paired delta+ measures what the plugin adds to every request. Every trial with a count+ is used, also one with no score or in an arm that failed elsewhere (a+ judge error does not change a token count). Trials without a count, and+ trials whose first call ran a hosted web search, are left out and the+ result says ``partial``.+ Fewer than :data:`CONTEXT_COST_MIN_PAIRED_CASES` pairs give ``insufficient``.+ """+ result: dict[str, Any] = {+ "method": CONTEXT_COST_METHOD,+ "delta_tokens_mean": None,+ "n_pairs": 0,+ "min_pairs": CONTEXT_COST_MIN_PAIRED_CASES,+ "status": "unavailable",+ "partial": False,+ "excluded": {"hosted_search": 0, "missing_tokens": 0},+ "reason": None,+ "missing_cases": [],+ }+ if with_arm is None or without_arm is None or without_arm.execution_status == "skipped":+ result["reason"] = "No baseline (without) arm was run."+ return result+ with_turns = _case_first_turn_tokens((*with_arm.trials, *with_arm.unscored))+ without_turns = _case_first_turn_tokens((*without_arm.trials, *without_arm.unscored))+ result["excluded"] = {+ "hosted_search": with_turns.hosted_search + without_turns.hosted_search,+ "missing_tokens": with_turns.missing + with_turns.no_usage + without_turns.missing + without_turns.no_usage,+ }+ left_out = [*_left_out("with-plugin arm", with_turns), *_left_out("baseline arm", without_turns)]+ for arm_name, turns in (("with-plugin arm", with_turns), ("baseline arm", without_turns)):+ if not turns.by_case:+ result["reason"] = f"The {arm_name} has no trial with a usable first-turn prompt token count" + (+ f" ({'; '.join(left_out)})." if left_out else "."+ )+ return result+ paired = sorted(set(with_turns.by_case) & set(without_turns.by_case))+ expected = {_case_id(case) for case in expected_case_ids or ()} - {None}+ if len(expected) > MAX_COMPARISON_CASES:+ raise ValueError("Comparison case limit exceeded")+ if expected:+ paired = [case for case in paired if case in expected]+ result["missing_cases"] = sorted(expected - set(paired))+ if not paired:+ result["reason"] = "No case has first-turn prompt tokens in both arms."+ return result+ deltas = [with_turns.by_case[case_id] - without_turns.by_case[case_id] for case_id in paired]+ unpaired = len(set(with_turns.by_case) ^ set(without_turns.by_case))+ incomplete_observations = not with_arm.observations_complete or not without_arm.observations_complete+ partial = bool(+ left_out+ or unpaired+ or result["missing_cases"]+ or not with_arm.succeeded+ or not without_arm.succeeded+ or incomplete_observations+ )+ status = "measured" if len(paired) >= CONTEXT_COST_MIN_PAIRED_CASES else "insufficient"+ reasons: list[str] = []+ if status == "insufficient":+ reasons.append(+ f"Only {len(paired)} case(s) have first-turn prompt tokens in both arms; at least "+ f"{CONTEXT_COST_MIN_PAIRED_CASES} are needed."+ )+ if partial:+ detail = [*left_out]+ if incomplete_observations:+ detail.append("executed-trial observation coverage is incomplete")+ if unpaired:+ detail.append(f"{unpaired} case(s) have a count in one arm only")+ if result["missing_cases"]:+ detail.append(f"{len(result['missing_cases'])} expected case(s) lack paired first-turn counts")+ if not detail:+ failed = [name for name, arm in (("with-plugin", with_arm), ("baseline", without_arm)) if not arm.succeeded]+ detail.append(f"the {' and '.join(failed)} arm did not complete, but every collected trial has its count")+ reasons.append("Partial: " + "; ".join(detail) + ".")+ result.update(+ {+ "delta_tokens_mean": round(math.fsum(deltas) / len(deltas), 2),+ "n_pairs": len(paired),+ "status": status,+ "partial": partial,+ "reason": " ".join(reasons) or None,+ }+ )+ return result+++# ---------------------------------------------------------------------------+# Per-case Integration completeness+# ---------------------------------------------------------------------------+++def _scored_pass_flags(trials: Sequence[TrialObservation], pass_threshold: float) -> dict[str, list[bool]]:+ flags: dict[str, list[bool]] = {}+ for trial in trials:+ if _case_id(trial.case_id) is None:+ continue+ score = (+ _logical_score(trial.logical_overall)+ if trial.logical_overall is not None and _reward_policy(trial.reward, logical=True) is not None+ else trial_quality_score(trial.reward)+ if trial.logical_overall is None+ else None+ )+ if score is None:+ continue+ flags.setdefault(trial.case_id, []).append(round(score, 4) >= pass_threshold)+ return flags+++def _required_attempts(flags: Sequence[bool], *, n_attempts: int, stop_on_pass: bool) -> int:+ if stop_on_pass:+ first_pass = next((index for index, passed in enumerate(flags, start=1) if passed), None)+ if first_pass is not None:+ return first_pass+ return n_attempts+++def integration_completeness(+ with_arm: ArmObservations | None,+ sum_of_parts_arm: ArmObservations | None,+ *,+ expected_case_ids: Sequence[str] | None,+ n_attempts: int,+ stop_on_pass: bool,+ pass_threshold: float,+ requested: bool = True,+ loose_parts_arm: ArmObservations | None = None,+) -> dict[str, Any]:+ """Check that the Integration arms cover the same cases with enough attempts.++ Coverage counts only scoreable attempts. ``complete`` requires every+ compared case to be present in both arms, no failed arm (including a failed+ sum-of-parts Harbor job), and no case below its required attempt count:+ ``n_attempts``, or the first passing attempt when ``stop_on_pass`` legitimately+ ended a case early. ``complete`` is ``None`` when no sum-of-parts arm was+ ``requested`` (effectiveness-only lift, ``--skip-baseline``): there is+ nothing to compare, so nothing is incomplete. A requested arm without+ observations is listed in ``failed_arms``.++ With a ``loose_parts_arm`` the result also holds ``loose_parts``: the same+ check of the loose-parts arm against the full plugin, which the+ Integration lift (glue) verdict reads. The full-versus-skills keys are+ unchanged by it.+ """+ pair = partial(+ _pair_completeness,+ with_arm,+ expected_case_ids=expected_case_ids,+ n_attempts=n_attempts,+ stop_on_pass=stop_on_pass,+ pass_threshold=pass_threshold,+ )+ if requested:+ result = pair(ARM_SUM_OF_PARTS, sum_of_parts_arm)+ else:+ result = {"complete": None, "missing_cases": [], "failed_arms": [], "attempt_shortfall": []}+ if loose_parts_arm is not None:+ result["loose_parts"] = pair(ARM_LOOSE_PARTS, loose_parts_arm)+ return result+++def _pair_completeness(+ with_arm: ArmObservations | None,+ control_name: str,+ control_arm: ArmObservations | None,+ *,+ expected_case_ids: Sequence[str] | None,+ n_attempts: int,+ stop_on_pass: bool,+ pass_threshold: float,+) -> dict[str, Any]:+ """The plugin arm against one control arm: missing cases, failed arms and attempt shortfall."""+ if with_arm is None or control_arm is None:+ missing_arms = [+ arm for arm, observed in ((ARM_WITH, with_arm), (control_name, control_arm)) if observed is None+ ]+ return {"complete": False, "missing_cases": [], "failed_arms": missing_arms, "attempt_shortfall": []}++ failed_arms: list[str] = []+ if not with_arm.succeeded or with_arm.job_failure or not with_arm.observations_complete:+ failed_arms.append(ARM_WITH)+ if not control_arm.succeeded or control_arm.job_failure or not control_arm.observations_complete:+ failed_arms.append(control_name)++ with_flags = _scored_pass_flags(with_arm.trials, pass_threshold)+ control_flags = _scored_pass_flags(control_arm.trials, pass_threshold)+ expected = list(dict.fromkeys(str(case_id) for case_id in (expected_case_ids or []) if str(case_id)))+ compared = expected or sorted(set(with_flags) | set(control_flags))++ missing_cases = [case_id for case_id in compared if not with_flags.get(case_id) or not control_flags.get(case_id)]+ attempt_shortfall: list[dict[str, Any]] = []+ for case_id in compared:+ for arm, flags_by_case in ((ARM_WITH, with_flags), (control_name, control_flags)):+ flags = flags_by_case.get(case_id, [])+ required = _required_attempts(flags, n_attempts=n_attempts, stop_on_pass=stop_on_pass)+ if len(flags) < required:+ attempt_shortfall.append({"case": case_id, "arm": arm, "expected": required, "observed": len(flags)})++ return {+ "complete": bool(compared) and not missing_cases and not failed_arms and not attempt_shortfall,+ "missing_cases": missing_cases,+ "failed_arms": failed_arms,+ "attempt_shortfall": attempt_shortfall,+ }+++# ---------------------------------------------------------------------------+# Assembly+# ---------------------------------------------------------------------------+++def build_agent_statistics(+ arms: Mapping[str, ArmObservations],+ *,+ expected_case_ids: Sequence[str] | None,+ n_attempts: int,+ stop_on_pass: bool,+ pass_threshold: float,+ sum_of_parts_requested: bool,+ baseline_is_sum_of_parts: bool = False,+ cross_component_case_ids: Collection[str] | None = None,+ load_failures_by_case: LoadFailures | None = None,+ never_activated_arms: Collection[str] = (),+ judge_mixed_cases: Mapping[str, Collection[str]] | None = None,+) -> dict[str, Any]:+ """Assemble every C4 statistics block for one agent.++ ``arms`` is keyed by ``with_skill``, ``without_skill``, ``sum_of_parts``+ and ``loose_parts``; absent keys mean the arm was not run. Only an agent+ with ``loose_parts`` observations gets a ``lift_ladder`` block+ (:func:`lift_ladder`, which drops the cases in ``load_failures_by_case``);+ otherwise the block is ``None`` and every other block is unchanged. A lift interval uses every case+ scored in both of its arms: one failed trial (a judge error or a timeout)+ marks it ``partial`` with ``n_cases`` of ``expected_cases`` pairs instead of+ removing it. Reliability requires successful, complete execution. Cost+ and token blocks also retain failed attempts when every observation is available.++ ``baseline_is_sum_of_parts`` marks a two-arm integration comparison,+ whose only baseline stages the member skills: its+ interval compares the plugin with its parts, so it is filed under+ ``integration`` and no effectiveness interval exists.++ With ``cross_component_case_ids`` (a plugin run whose staged cases were+ read), the integration interval also carries ``by_case_type``: the same+ lift on cross-component cases and on all other cases+ (:func:`lift_by_case_type`). ``None`` records no split.++ ``never_activated_arms`` names the skill-bearing arms whose positive trials+ never activated the skill. As for the Skill Lift, no interval is computed+ for a comparison that uses one of them, and no lift ladder (it splits the+ lift across every skill-bearing arm): the block is ``None``.+ """+ never_activated = set(never_activated_arms)+ index = _CaseIndex(+ {name: observed for name, observed in arms.items() if name != ARM_SUM_OF_PARTS or sum_of_parts_requested},+ judge_mixed_cases,+ )+ with_arm = arms.get(ARM_WITH)+ without_arm = arms.get(ARM_WITHOUT)+ sop_arm = arms.get(ARM_SUM_OF_PARTS) if sum_of_parts_requested else None+ loose_arm = arms.get(ARM_LOOSE_PARTS)+ ladder = None+ # A legacy two-arm run's only baseline holds the member skills, so it has no no-plugin rung to stand on.+ skill_arms = {ARM_WITH, ARM_SUM_OF_PARTS, ARM_LOOSE_PARTS}+ if loose_arm is not None and not baseline_is_sum_of_parts and not never_activated & skill_arms:+ ladder_arms = {+ ARM_WITHOUT: without_arm,+ ARM_SUM_OF_PARTS: sop_arm,+ ARM_LOOSE_PARTS: loose_arm,+ ARM_WITH: with_arm,+ }+ ladder = lift_ladder(+ {arm: observed for arm, observed in ladder_arms.items() if observed is not None},+ cross_component_case_ids,+ expected_case_ids=expected_case_ids,+ load_failures_by_case=load_failures_by_case,+ judge_mixed_cases=judge_mixed_cases,+ _index=index,+ )++ effectiveness = None+ if baseline_is_sum_of_parts:+ members_arm, members_name = without_arm, ARM_WITHOUT+ else:+ members_arm, members_name = sop_arm, ARM_SUM_OF_PARTS+ if ARM_WITH not in never_activated:+ effectiveness = paired_lift(+ with_arm,+ without_arm,+ treatment_arm=ARM_WITH,+ control_arm=ARM_WITHOUT,+ expected_case_ids=expected_case_ids,+ load_failures_by_case=load_failures_by_case,+ _index=index,+ )+ integration = None+ if not never_activated & {ARM_WITH, members_name}:+ integration = paired_lift(+ with_arm,+ members_arm,+ treatment_arm=ARM_WITH,+ control_arm=members_name,+ expected_case_ids=expected_case_ids,+ load_failures_by_case=load_failures_by_case,+ _index=index,+ )+ if integration is not None and cross_component_case_ids is not None:+ integration["by_case_type"] = lift_by_case_type(+ with_arm,+ members_arm,+ cross_component_case_ids,+ load_failures_by_case=load_failures_by_case,+ _index=index,+ _arms=(ARM_WITH, members_name),+ )++ reliability: dict[str, Any] = {}+ cost: dict[str, Any] = {}+ token_efficiency: dict[str, Any] = {}+ for name, arm in arms.items():+ if name == ARM_SUM_OF_PARTS and not sum_of_parts_requested:+ continue+ executed = (*arm.trials, *arm.unscored)+ summary = (+ reliability_summary(arm.pass_summary, stop_on_pass=stop_on_pass)+ if arm.succeeded and arm.observations_complete+ else None+ )+ if summary is not None:+ summary["not_applicable_metrics"] = arm_not_applicable_metrics(arm.trials)+ reliability[name] = summary+ successes = int(arm.pass_summary.get("passed_cases") or 0) if arm.pass_summary else 0+ if arm.succeeded or executed:+ cost[name] = arm_cost(executed if arm.observations_complete else (), successes=successes)+ cost[name]["observations_complete"] = arm.observations_complete+ token_efficiency[name] = arm_token_efficiency(executed) if arm.observations_complete else None++ completeness = integration_completeness(+ with_arm,+ sop_arm,+ expected_case_ids=expected_case_ids,+ n_attempts=n_attempts,+ stop_on_pass=stop_on_pass,+ pass_threshold=pass_threshold,+ requested=sum_of_parts_requested,+ loose_parts_arm=loose_arm,+ )+ if loose_arm is not None:+ all_pairs = {+ name: _pair_completeness(+ with_arm,+ name,+ observed,+ expected_case_ids=expected_case_ids,+ n_attempts=n_attempts,+ stop_on_pass=stop_on_pass,+ pass_threshold=pass_threshold,+ )+ for name, observed in (+ (ARM_WITHOUT, without_arm),+ (ARM_SUM_OF_PARTS, sop_arm),+ (ARM_LOOSE_PARTS, loose_arm),+ )+ }+ completeness["all_arms"] = {+ "complete": all(pair["complete"] for pair in all_pairs.values()),+ "comparisons": all_pairs,+ }+ return {+ "lift_uncertainty": {"effectiveness": effectiveness, "integration": integration},+ "reliability": reliability,+ "cost": cost,+ "token_efficiency": token_efficiency,+ "context_cost_measured": context_cost_measured(with_arm, without_arm, expected_case_ids=expected_case_ids),+ "lift_ladder": ladder,+ "integration_completeness": completeness,+ "comparison_eligibility": {+ "effectiveness": {"score_policy_mismatches": list(index.conflicts.get((ARM_WITH, ARM_WITHOUT), ()))},+ "integration": {"score_policy_mismatches": list(index.conflicts.get((ARM_WITH, members_name), ()))},+ },+ }+++def integration_claim(+ ladder: Mapping[str, Any] | None,+ content: Mapping[str, Any] | None,+ completeness: Mapping[str, Any] | None,+ *,+ agent: str,+ full_arm_load: str,+) -> dict[str, Any]:+ """Eligibility and advisory integration claim, never an evaluation gate.++ As in the source contract, an interval beginning at +.05 is real and one+ ending at -.05 is negative; cosmetic requires strict containment.+ """+ label = "Integration lift" if full_arm_load == "native" else "Wrapper effect (full wrapper arm minus loose parts)"+ result = {"eligible": False, "label": label, "verdict": None, "point_verdict": None, "reason": None}+ if agent != "claude-code" or full_arm_load != "native":+ result["reason"] = "Only native Claude Code measures plugin integration; wrapper/other loading is not measured."+ return result+ result["verdict"] = "inconclusive"+ ladder = ladder if isinstance(ladder, Mapping) else {}+ content = content if isinstance(content, Mapping) else {}+ completeness = completeness if isinstance(completeness, Mapping) else {}+ glue = ladder.get("glue")+ glue = glue if isinstance(glue, Mapping) else {}+ value = finite_number(glue.get("lift"))+ if value is not None:+ result["point_verdict"] = (+ "real_integration" if value >= 0.05 else "negative_integration" if value <= -0.05 else "cosmetic_bundling"+ )+ reason = None+ if (+ content.get("equal_across_arms") is not True+ or content.get("mismatches")+ or content.get("capability_differences")+ ):+ reason = "Capability equality is false or unverified."+ elif (+ content.get("content_basis") != "emitted Harbor tasks"+ or type(content.get("tasks_hashed")) is not int+ or content["tasks_hashed"] < 1+ ):+ reason = "Capability parity was not measured on emitted tasks."+ elif ladder.get("partial") is not False or glue.get("partial") is not False:+ reason = "The shared comparison is partial."+ elif not all(+ isinstance(item, Mapping) and item.get("complete") is True+ for item in (completeness, completeness.get("loose_parts"), completeness.get("all_arms"))+ ):+ reason = "Expected cases or attempts are incomplete."+ elif (+ type(glue.get("n")) is not int+ or not LIFT_CI_MIN_PAIRED_CASES <= glue["n"] <= MAX_COMPARISON_CASES+ or glue.get("precision") not in {PRECISION_ADEQUATE, PRECISION_LOW}+ ):+ reason = "Too few paired cross-component cases or unusable precision."+ low, high = finite_number(glue.get("ci_low")), finite_number(glue.get("ci_high"))+ if reason is None and (+ value is None or low is None or high is None or not -1 <= value <= 1 or not -1 <= low <= high <= 1+ ):+ reason = "No finite paired interval is available."+ if reason is None:+ if low >= 0.05:+ result["verdict"] = "real_integration"+ elif high <= -0.05:+ result["verdict"] = "negative_integration"+ elif low > -0.05 and high < 0.05:+ result["verdict"] = "cosmetic_bundling"+ else:+ reason = "The paired interval crosses an integration band edge."+ result["eligible"] = reason is None+ result["reason"] = reason+ return resultdiff --git a/src/skillevaluator/tier3/loose_parts.py b/src/skillevaluator/tier3/loose_parts.pynew file mode 100644index 00000000..3df64205--- /dev/null+++ b/src/skillevaluator/tier3/loose_parts.py@@ -0,0 +1,1688 @@+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+# SPDX-License-Identifier: Apache-2.0++"""The loose-parts arm of a Tier 3 plugin run, on native Claude Code (v1).++The Integration lift (glue) is *full plugin minus loose parts*. Loose parts are+the same skills, rules, MCP servers, subagents, commands and hooks the full+plugin arm stages, put where Claude Code loads them standalone, with no plugin+around them. Only what cannot load standalone is left out, and it is listed as+glue with the reason (``dropped_as_glue``).++Where each piece goes (``$CLAUDE_CONFIG_DIR`` is the run's Claude Code config+dir, which Harbor sets; ``/skilleval`` is the native bundle in the image):++* member skills: the task skills projection (Harbor copies them to+ ``$CLAUDE_CONFIG_DIR/skills/``), exactly as the skills-only arm;+* rules: user rules ``$CLAUDE_CONFIG_DIR/rules/skilleval-<plugin>/<rule>.md``,+ written by the plugin arm's own rule writer, so the bytes match;+* subagents and commands: ``$CLAUDE_CONFIG_DIR/agents/`` and ``commands/``,+ byte-identical copies;+* hooks: ``$CLAUDE_CONFIG_DIR/settings.json`` (``{"hooks": ...}``), the plugin+ arm's census-wrapped hooks with ``${CLAUDE_PLUGIN_ROOT}`` replaced by+ :data:`LOOSE_ROOT` (and ``${CLAUDE_PLUGIN_DATA}`` by :data:`LOOSE_DATA`);+* MCP servers: :data:`LOOSE_MCP_CONFIG`, passed with ``--mcp-config``: the+ plugin arm's ``.mcp.json`` entries (command, args, env) with the same+ substitution;+* plugin-root resources used by hooks or MCP servers: the same filtered tree+ as the full arm under :data:`LOOSE_ROOT`, preserving relative imports and+ data-file access without guessing a language-specific dependency graph.++Everything reaches the container through the native staging path+(:func:`~skillevaluator.tier3.harbor.native_staging.stage_native_bundle` and+``/skilleval/native/setup.sh``, run right before the agent starts by the+Harbor wrappers in :mod:`skillevaluator.tier3.harbor.loose_agents`).++Content hashes prove the two arms carry the same capability: every piece is+hashed as each arm stages it (staged into a private directory by the same+code the runner uses), after a documented normalization (the loose root+written back as ``${CLAUDE_PLUGIN_ROOT}``; a file the plugin arm rewrites only+to census-wrap its frontmatter hooks is hashed from its source).++Load is verified only by Claude Code's own ``system/init`` report+(:func:`loose_parts_load_evidence`), with one documented workaround: a+successful call of an MCP server's tool proves a server that init reported as+``pending`` (or not at all). Rules and hooks are not in that report.++This module is host-side and imports no Harbor code.+"""++from __future__ import annotations++import hashlib+import json+import os+import posixpath+import re+import shlex+import stat+import tempfile+from collections.abc import Iterable, Mapping, Sequence+from contextvars import ContextVar+from dataclasses import dataclass, field+from functools import cached_property, wraps+from pathlib import Path, PurePosixPath+from typing import Any, ClassVar++from skillevaluator.constants import (+ CONTENT_DEDUP_MAX_FILE_BYTES,+ CONTENT_DEDUP_MAX_FILES,+ CONTENT_DEDUP_MAX_TOTAL_BYTES,+)+from skillevaluator.plugin_paths import PluginRootReader+from skillevaluator.tier3.runtime_arms import ARM_LOOSE_PARTS, ARM_SUM_OF_PARTS, ARM_WITH, ARM_WITHOUT+from skillevaluator.utils.secure_fs import SecureRoot++MAX_PARITY_FILES = 16_384+MAX_PARITY_BYTES = 128 * 1024 * 1024+++class _ParityReads:+ def __init__(self) -> None:+ self.cache: dict[tuple[Any, ...], bytes] = {}+ self.total = 0++ def read(self, path: Path) -> bytes | None:+ try:+ metadata = path.lstat()+ key = (+ str(path.absolute()),+ metadata.st_dev,+ metadata.st_ino,+ metadata.st_size,+ metadata.st_mtime_ns,+ metadata.st_ctime_ns,+ metadata.st_mode,+ )+ if key not in self.cache:+ if len(self.cache) >= MAX_PARITY_FILES or self.total + metadata.st_size > MAX_PARITY_BYTES:+ raise ValueError("Parity content exceeds the bounded read budget")+ with SecureRoot(path.parent) as reader:+ data, _ = reader.read_bytes(Path(path.name), CONTENT_DEDUP_MAX_FILE_BYTES, expected=metadata)+ self.cache[key] = data+ self.total += len(data)+ return self.cache[key]+ except OSError:+ return None+++_PARITY_READS: ContextVar[_ParityReads | None] = ContextVar("private_comparison_parity_reads", default=None)+++def _scoped_parity_reads(function):+ @wraps(function)+ def bounded(*args, **kwargs):+ if _PARITY_READS.get() is not None:+ return function(*args, **kwargs)+ token = _PARITY_READS.set(_ParityReads())+ try:+ return function(*args, **kwargs)+ finally:+ _PARITY_READS.reset(token)++ return bounded+++from skillevaluator.tier3.plugin_native import (+ BUNDLE_DIRNAME,+ CONTAINER_ROOT,+ HOOK_CENSUS_SCRIPT,+ NATIVE_ROOT,+ CensusCheck,+ ClaudeCodeAdapter,+ HarnessAdapter,+ NativeBundle,+ NativePluginSource,+ PluginLoadError,+ _claude_rule_file_name,+ _claude_user_rule,+ _mcp_entry_claude,+ _redacted,+ _staged_handler_needle,+ _staged_hook_events,+ is_frontmatter_hook_source,+ native_refusal,+ plugin_slug,+ render_setup_script,+ safe_name,+ shell_path,+ wrap_hook_sources,+)++#: The arm key in results, lift JSON, and plugin signals (the statistics' own key), and its report label.+LOOSE_PARTS_ARM = ARM_LOOSE_PARTS+LOOSE_PARTS_LABEL = "Loose parts"+#: The one harness v1 measures.+LOOSE_AGENT = "claude-code"+LOOSE_ADAPTER_ID = "claude-code-loose-parts"+#: The census harness kind; the plugin-arm ``claude-code-init`` check must never read this arm.+LOOSE_HARNESS_KIND = "claude-code-loose-init"+#: Bundle-relative directory of the loose parts, and where it lands in the container.+LOOSE_BUNDLE_DIR = "native/claude-code/loose"+LOOSE_DIR = f"{CONTAINER_ROOT}/{LOOSE_BUNDLE_DIR}"+#: Stands in for ``${CLAUDE_PLUGIN_ROOT}``: referenced plugin files keep their relative path under it.+LOOSE_ROOT = f"{LOOSE_DIR}/root"+#: Stands in for ``${CLAUDE_PLUGIN_DATA}`` (created empty by setup, like a fresh plugin data dir).+LOOSE_DATA = "/tmp/skilleval-loose-data"+#: The plain MCP config the launch passes with ``--mcp-config``.+LOOSE_MCP_CONFIG = f"{LOOSE_DIR}/mcp.json"++#: Arm aliases accepted by the hashing and evidence functions -> the result arm key.+ARM_ALIASES: dict[str, str] = {+ "none": ARM_WITHOUT,+ ARM_WITHOUT: ARM_WITHOUT,+ "skills": ARM_SUM_OF_PARTS,+ ARM_SUM_OF_PARTS: ARM_SUM_OF_PARTS,+ "loose": LOOSE_PARTS_ARM,+ LOOSE_PARTS_ARM: LOOSE_PARTS_ARM,+ "full": ARM_WITH,+ ARM_WITH: ARM_WITH,+}+#: The staged arms whose content hashes are compared.+COMPARED_ARMS = (ARM_WITH, LOOSE_PARTS_ARM, ARM_SUM_OF_PARTS)++NOT_SUPPORTED_WRAPPER = (+ "not supported: use native loading. In wrapper mode the plugin arm is SkillEvaluator's own "+ "generated wrapper SKILL.md, so full plugin minus loose parts would measure that wrapper, not the plugin's glue"+)+NOTHING_LOOSE = (+ "loose parts equal the skills alone: the plugin has no rule, runnable MCP server, subagent, command, or hook "+ "that Claude Code can load standalone, so a loose-parts arm would repeat the skills-only arm"+)++# Init-verifiable piece kinds; rules, hooks, and scripts are not in Claude Code's startup report.+_INIT_KINDS = frozenset({"skill", "agent", "command", "mcp"})+#: The hook piece of every staged hook group without a command handler (and so without a hook id).+UNATTRIBUTED_HOOKS = "(groups without a command handler)"+_EVIDENCE_INIT = "system/init"+_EVIDENCE_CALL = "mcp-call"+_EVIDENCE_NONE = "unverifiable"+_PLUGIN_BUNDLE = f"{BUNDLE_DIRNAME}/{ClaudeCodeAdapter.bundle_dir}"+_LOOSE_BUNDLE = f"{BUNDLE_DIRNAME}/{LOOSE_BUNDLE_DIR}"+_RULES_BUNDLE = f"{BUNDLE_DIRNAME}/native/claude-code/rules"+_ROOT_VAR = "${CLAUDE_PLUGIN_ROOT}"+_DATA_VAR = "${CLAUDE_PLUGIN_DATA}"+# Only the braced form is replaced: Claude Code expands ``${VAR}`` in MCP fields, and a shell expands the bare+# ``$CLAUDE_PLUGIN_ROOT`` from the environment, which the loose launch sets (so both arms read it the same way).+_ROOT_VAR_RE = re.compile(r"\$\{CLAUDE_PLUGIN_ROOT\}")+_DATA_VAR_RE = re.compile(r"\$\{CLAUDE_PLUGIN_DATA\}")+# A root reference and the plugin path after it (``${CLAUDE_PLUGIN_ROOT}/x.sh``, ``"${CLAUDE_PLUGIN_ROOT}"/x.sh``).+# ``:`` and ``,`` end a path, so ``PYTHONPATH=${CLAUDE_PLUGIN_ROOT}/lib:${CLAUDE_PLUGIN_ROOT}/src`` names two.+_ROOT_PATH_RE = re.compile(+ r"(?:\$\{CLAUDE_PLUGIN_ROOT\}|\$CLAUDE_PLUGIN_ROOT(?![A-Za-z0-9_]))[\"']?(/[^\s\"'`;|&<>(){}$\\:,]*)?"+)+_HOOK_ID_SOURCE_RE = re.compile(r"^(?P<source>.+)#[^#]*\[\d+\]\.hooks\[\d+\]$")+#: Subagent frontmatter fields Claude Code ignores in a plugin agent but honours in a user agent+#: ("Plugin agent file ... sets <field>, which is ignored for plugin agents", Claude Code 2.1).+_PLUGIN_IGNORED_AGENT_FIELDS = {+ "permissionmode": "permissionMode",+ "hooks": "hooks",+ "mcpservers": "mcpServers",+}+PLUGIN_ROOT_IN_TEXT = (+ "Claude Code writes the plugin root into a plugin skill, subagent or command text for ${CLAUDE_PLUGIN_ROOT} "+ "(and ${CLAUDE_PLUGIN_DATA}); the loose copy keeps the variable, which only a shell expands (the loose "+ "launch exports it), so the model reads a different text in the two arms"+)+_SERVER_NAME_RE = re.compile(r"[^A-Za-z0-9_-]")++_GLUE_REASONS = {+ "output_style": (+ "output styles change the whole session's presentation; v1 counts a plugin output style as glue, "+ "not a standalone capability"+ ),+ "lsp": "LSP servers load only from a plugin's .lsp.json; Claude Code has no standalone user-level LSP config",+ "settings": "plugin settings.json keys apply only while the plugin is enabled (they configure the plugin session)",+ "bin": "bin/ executables join the Bash PATH only while the plugin is enabled",+ "manifest": (+ "the plugin manifest: the plugin identity, its <plugin>: namespacing of skills, agents, commands and MCP "+ "servers, and the plugin listing; loose parts have none of these by definition"+ ),+ "userConfig": "userConfig values are plugin install-time settings that exist only for an installed plugin",+}+++class LoosePartsUnsupported(PluginLoadError):+ """The loose-parts arm cannot run for this agent or load mode (the message starts with ``not supported:``)."""+++def not_supported_reason(agent: str) -> str:+ """Why an agent other than Claude Code has no loose-parts arm in v1."""+ return (+ f"not supported: the loose-parts arm runs on native Claude Code only in v1; {agent} is not measured "+ "(Codex, OpenCode and Hermes: not measured)"+ )+++def loose_parts_launch_env() -> dict[str, str]:+ """The plugin-root variables the loose launch exports, so scripts reading them find the loose root."""+ return {"CLAUDE_PLUGIN_ROOT": LOOSE_ROOT, "CLAUDE_PLUGIN_DATA": LOOSE_DATA}+++# --------------------------------------------------------------------------- #+# Public records #+# --------------------------------------------------------------------------- #+@dataclass(frozen=True)+class LoosePiece:+ """One loose part: ``kind:name`` id, the plugin-relative file it came from, and its loose-arm sha256."""++ id: str+ kind: str+ source: str | None+ sha256: str++ def to_dict(self) -> dict[str, Any]:+ return {"id": self.id, "kind": self.kind, "source": self.source, "sha256": self.sha256}+++@dataclass(frozen=True)+class GluePiece:+ """A plugin part that is not a loose part, with the reason."""++ id: str+ reason: str++ def to_dict(self) -> dict[str, str]:+ return {"id": self.id, "reason": self.reason}+++@dataclass(frozen=True)+class _Placement:+ """Where one piece is staged in one arm, relative to the task's ``environment/`` dir.++ ``key`` picks the piece out of a JSON file (an MCP server name or a hook+ source name). ``overrides`` maps a staged file to the source file hashed+ instead (a census rewrite), and ``exclude`` lists staged files a tree hash+ leaves out (files only the plugin arm generates). ``generated_dirs`` omits+ only mode records for generated parent directories absent from the source+ copy; their unexpected descendants still participate in the tree hash.+ """++ id: str+ kind: str+ path: str+ source: str | None = None+ key: str | None = None+ overrides: tuple[tuple[str, Path], ...] = ()+ override_digests: tuple[tuple[str, str], ...] = ()+ exclude: tuple[str, ...] = ()+ root_template: str | None = None+ generated_dirs: tuple[str, ...] = ()++ @property+ def name(self) -> str:+ return self.id.split(":", 1)[1]+++@dataclass(frozen=True)+class LooseParts:+ """The loose-parts arm for one plugin and agent (see the module docstring).++ ``bundle`` is the native bundle :func:`stage_native_bundle` consumes (``None``+ with ``error``). ``error`` starts with ``not supported:`` for a harness or+ load mode v1 does not measure, and is :data:`NOTHING_LOOSE` for a plugin+ whose loose parts equal its skills.+ """++ agent: str+ bundle: NativeBundle | None+ dropped_as_glue: tuple[GluePiece, ...] = ()+ not_staged: tuple[GluePiece, ...] = ()+ rules: tuple[str, ...] = ()+ mcp_servers: tuple[str, ...] = ()+ error: str | None = None+ source: NativePluginSource | None = field(default=None, repr=False, compare=False)+ _loose: tuple[_Placement, ...] = field(default=(), repr=False, compare=False)+ _full: tuple[_Placement, ...] = field(default=(), repr=False, compare=False)+ _normalized: tuple[tuple[str, str], ...] = field(default=(), repr=False, compare=False)+ #: ``(piece id, reason)`` for a piece staged byte for byte that Claude Code still loads differently in the two arms.+ _capability_differences: tuple[tuple[str, str], ...] = field(default=(), repr=False, compare=False)+ #: ``(piece id, reason)`` for a skill, subagent or command whose text names the plugin root (recorded, not compared).+ _root_text: tuple[tuple[str, str], ...] = field(default=(), repr=False, compare=False)++ # -- staging -------------------------------------------------------------- #+ def task_staging(self) -> Any:+ """The :class:`~skillevaluator.tier3.harbor.native_staging.NativeTaskStaging` for the loose arm's task."""+ from skillevaluator.tier3.harbor.native_staging import NativeTaskStaging++ if self.error is not None or self.bundle is None or self.source is None:+ raise _unsupported_or_load_error(self.error or NOTHING_LOOSE)+ adapter = LoosePartsClaudeCodeAdapter()+ refusal = native_refusal(adapter, self.source)+ if refusal is not None:+ raise PluginLoadError(refusal)+ return NativeTaskStaging(agent=self.agent, adapter=adapter, source=self.source, bundle=self.bundle)++ def stage_arm(self, arm: str, env_dir: Path, *, excluded_roots: Sequence[Path] = ()) -> None:+ """Stage one arm's plugin content into ``env_dir`` (a task's ``environment/``) with the runner's code.++ ``without_skill`` stages nothing, ``sum_of_parts`` the member skills+ (``skills/``), ``loose_parts`` the member skills and the loose bundle,+ and ``with_skill`` the native Claude Code plugin bundle.+ """+ from skillevaluator.tier3.harbor.native_staging import build_native_task_staging, stage_native_bundle++ arm = _arm_key(arm)+ source = self._require_source()+ excluded = (*excluded_roots, *source.excluded_paths)+ env_dir.mkdir(parents=True, exist_ok=True)+ if arm in {ARM_SUM_OF_PARTS, LOOSE_PARTS_ARM}:+ _stage_member_skills(env_dir / "skills", source.member_skills, excluded)+ if arm == LOOSE_PARTS_ARM:+ stage_native_bundle(env_dir, self.task_staging(), excluded_roots=excluded)+ elif arm == ARM_WITH:+ staging = build_native_task_staging(LOOSE_AGENT, ClaudeCodeAdapter(), source)+ stage_native_bundle(env_dir, staging, excluded_roots=excluded)++ # -- hashing -------------------------------------------------------------- #+ @_scoped_parity_reads+ def hash_staged_arm(self, arm: str, env_dir: Path) -> dict[str, str]:+ """``{piece id: sha256}`` of every piece one arm staged into ``env_dir`` (an absent piece is left out)."""+ cache: dict[str, Any] = {}+ hashes: dict[str, str] = {}+ for placement in self._placements(_arm_key(arm)):+ digest = _piece_digest(env_dir, placement, cache)+ if digest is not None:+ hashes[placement.id] = digest+ return dict(sorted(hashes.items()))++ def content_hashes_for_arm(self, arm: str, *, excluded_roots: Sequence[Path] = ()) -> dict[str, str]:+ """Stage ``arm`` (``none``/``skills``/``loose``/``full`` or the arm key) privately and hash its pieces."""+ if self.error is not None:+ return {}+ with tempfile.TemporaryDirectory(prefix="skilleval-loose-hash-") as scratch:+ env_dir = Path(scratch) / "environment"+ self.stage_arm(arm, env_dir, excluded_roots=excluded_roots)+ return self.hash_staged_arm(arm, env_dir)++ @_scoped_parity_reads+ def content_report(self, *, excluded_roots: Sequence[Path] = ()) -> dict[str, Any]:+ """``content_hashes`` of the full, loose, and skills-only arms, ``equal_across_arms`` and ``mismatches``."""+ if self.error is not None:+ return {"content_hashes": {}, "equal_across_arms": None, "mismatches": []}+ report = compare_content_hashes(+ {arm: self.content_hashes_for_arm(arm, excluded_roots=excluded_roots) for arm in COMPARED_ARMS}+ )+ if self._normalized:+ report["normalized"] = dict(self._normalized)+ return self._with_capability_differences(self._with_missing_pieces(report))++ @_scoped_parity_reads+ def content_report_for_tasks(self, task_dirs_by_arm: Mapping[str, Sequence[Path]]) -> dict[str, Any]:+ """The :meth:`content_report` of tasks the runner emitted: ``{arm: [task dir, ...]}`` per compared arm.++ Each task's ``environment/`` is hashed as that arm staged it. One arm's+ tasks must agree with each other (a piece that differs between two+ tasks of one arm is a mismatch too); the first task's hashes are then+ compared across arms. ``tasks_hashed`` counts the tasks read.+ """+ if self.error is not None:+ return {"content_hashes": {}, "equal_across_arms": None, "mismatches": [], "tasks_hashed": 0}+ per_arm: dict[str, dict[str, str]] = {}+ uneven: set[str] = set()+ counted = 0+ for arm, task_dirs in task_dirs_by_arm.items():+ arm_hashes = [self.hash_staged_arm(arm, Path(task) / "environment") for task in task_dirs]+ counted = max(counted, len(arm_hashes))+ if not arm_hashes:+ continue+ first = arm_hashes[0]+ for other in arm_hashes[1:]:+ uneven.update(piece for piece in first.keys() | other.keys() if first.get(piece) != other.get(piece))+ per_arm[_arm_key(arm)] = first+ report = compare_content_hashes(per_arm)+ report = self._with_missing_pieces(report)+ if uneven:+ report["mismatches"] = sorted(set(report["mismatches"]) | uneven)+ report["equal_across_arms"] = False+ report["tasks_hashed"] = counted+ if self._normalized:+ report["normalized"] = dict(self._normalized)+ if ARM_WITH in per_arm and LOOSE_PARTS_ARM in per_arm:+ report = self._with_capability_differences(report)+ return report++ def _with_missing_pieces(self, report: dict[str, Any]) -> dict[str, Any]:+ """Expected pieces cannot vanish from every arm and disappear from the comparison."""+ missing = {+ item.id+ for arm, hashes in report.get("content_hashes", {}).items()+ for item in self._placements(arm)+ if item.id not in hashes+ }+ if missing:+ report["mismatches"] = sorted(set(report.get("mismatches", ())) | missing)+ report["equal_across_arms"] = False+ return report++ def _with_capability_differences(self, report: dict[str, Any]) -> dict[str, Any]:+ """``report`` with every piece that loads differently in the two arms counted as a mismatch.++ Equal bytes are not equal capability when Claude Code reads the same+ file differently as a plugin part and as a user part, so the arms are+ never called equal then and the reason is kept per piece.+ """+ if not self._capability_differences:+ return report+ differences = dict(self._capability_differences)+ report["mismatches"] = sorted(set(report.get("mismatches", [])) | set(differences))+ report["equal_across_arms"] = False+ report["capability_differences"] = differences+ return report++ @cached_property+ def pieces(self) -> tuple[LoosePiece, ...]:+ """Every staged loose part with its loose-arm sha256 (staged privately once, then cached)."""+ if self.error is not None:+ return ()+ return self._pieces(self.content_hashes_for_arm(LOOSE_PARTS_ARM))++ def _pieces(self, hashes: Mapping[str, str]) -> tuple[LoosePiece, ...]:+ return tuple(+ LoosePiece(placement.id, placement.kind, placement.source, hashes.get(placement.id, ""))+ for placement in sorted(self._loose, key=lambda item: item.id)+ )++ # -- records -------------------------------------------------------------- #+ def census_plan(self) -> dict[str, Any]:+ """What the collector needs to read this arm's load census (``mode``, declared parts, hook ids, harness)."""+ if self.bundle is None:+ return {}+ return {"mode": "native", "arm": LOOSE_PARTS_ARM, **self.bundle.census_plan()}++ def info(+ self,+ *,+ content: Mapping[str, Any] | None = None,+ load_evidence: Mapping[str, Any] | None = None,+ ) -> dict[str, Any]:+ """The ``loose_parts`` JSON block: what the arm staged, the content hashes, glue, and load evidence.++ ``content`` defaults to :meth:`content_report` (it stages the three arms+ privately); ``load_evidence`` is the per-case/per-arm evidence the+ collector gathered (empty until it runs).+ """+ report = dict(content) if content is not None else self.content_report()+ loose_hashes = report.get("content_hashes", {}).get(LOOSE_PARTS_ARM)+ # The piece hashes come from the same staging as the report, so the block never disagrees with itself.+ pieces = self._pieces(loose_hashes or {})+ block: dict[str, Any] = {+ "arm": LOOSE_PARTS_ARM,+ "label": LOOSE_PARTS_LABEL,+ "agent": self.agent,+ "adapter": LOOSE_ADAPTER_ID,+ "error": self.error,+ "loose_dir": LOOSE_DIR,+ "loose_root": LOOSE_ROOT,+ "mcp_config": LOOSE_MCP_CONFIG,+ "pieces": [piece.to_dict() for piece in pieces],+ "rules": list(self.rules),+ "mcp_servers": list(self.mcp_servers),+ "content_hashes": report.get("content_hashes", {}),+ "equal_across_arms": report.get("equal_across_arms"),+ "mismatches": list(report.get("mismatches", [])),+ "dropped_as_glue": [piece.to_dict() for piece in self.dropped_as_glue],+ "not_staged": [piece.to_dict() for piece in self.not_staged],+ "capability_differences": dict(report.get("capability_differences") or {}),+ "plugin_root_in_text": dict(self._root_text),+ "load_evidence": dict(load_evidence or {}),+ }+ if report.get("normalized"):+ block["normalized"] = dict(report["normalized"])+ return block++ # -- internals ------------------------------------------------------------ #+ def _require_source(self) -> NativePluginSource:+ if self.source is None or self.error is not None:+ raise _unsupported_or_load_error(self.error or NOTHING_LOOSE)+ return self.source++ def _placements(self, arm: str) -> tuple[_Placement, ...]:+ if arm == LOOSE_PARTS_ARM:+ return self._loose+ if arm == ARM_WITH:+ return self._full+ if arm == ARM_SUM_OF_PARTS:+ return tuple(placement for placement in self._loose if placement.kind == "skill")+ return ()+++class LoosePartsClaudeCodeAdapter(HarnessAdapter):+ """Claude Code with the plugin's parts staged as plain user-level pieces: no plugin, no manifest."""++ agent = LOOSE_AGENT+ adapter_id = LOOSE_ADAPTER_ID+ description = (+ "stages the plugin's skills, rules, MCP servers, subagents, commands and hooks as plain Claude Code "+ "user-level pieces under $CLAUDE_CONFIG_DIR and --mcp-config; no plugin, no manifest"+ )+ components: ClassVar[Mapping[str, str]] = {+ "skill": "native",+ "rule": "native",+ "mcp": "native",+ "hook": "native",+ "agent": "native",+ "command": "native",+ }+ stage_wrapper_skill = False+ #: Staged into an arm without the target skill (scored like the skills-only arm, ``has_skill`` false).+ control_arm = True+ #: Member skills go through the task skills projection, exactly as in the skills-only arm.+ stage_member_skills = True+ plugin_mcp_via_task = False+ #: The plugin files that hooks and MCP servers reference are copied, so plugin-file servers start.+ copies_plugin_tree = True++ def build(self, source: NativePluginSource) -> NativeBundle:+ parts = build_loose_parts(source)+ if parts.error is not None or parts.bundle is None:+ raise _unsupported_or_load_error(parts.error or NOTHING_LOOSE)+ return parts.bundle+++# --------------------------------------------------------------------------- #+# Building the loose arm #+# --------------------------------------------------------------------------- #+def build_loose_parts(+ source: NativePluginSource | None,+ agent: str = LOOSE_AGENT,+ *,+ plugin_load: str = "native",+) -> LooseParts:+ """Plan the loose-parts arm for ``agent`` from the native plugin snapshot.++ Never raises for an unsupported agent or load mode, or a plugin with+ nothing loose: ``error`` says why. Raises ``ValueError`` for content that+ must never be staged (a referenced plugin file that is a link or special+ file, two parts with one destination, an inline literal secret).+ """+ if plugin_load == "wrapper" or source is None:+ return LooseParts(agent=agent, bundle=None, error=NOT_SUPPORTED_WRAPPER, source=source)+ if agent != LOOSE_AGENT:+ return LooseParts(agent=agent, bundle=None, error=not_supported_reason(agent), source=source)+ refusal = native_refusal(LoosePartsClaudeCodeAdapter(), source)+ if refusal is not None:+ return LooseParts(agent=agent, bundle=None, error=refusal, source=source)+ return _Builder(source).build()+++def loose_parts_task_staging(agent: str, source: NativePluginSource | None, *, plugin_load: str = "native") -> Any:+ """The runner's seam: the loose arm's native task staging for ``agent``.++ Returns the :class:`~skillevaluator.tier3.harbor.native_staging.NativeTaskStaging`+ that ``generate_harbor_tasks(native_plugin=...)`` and ``stage_native_bundle``+ consume. Raises :class:`LoosePartsUnsupported` (``not supported: ...``) for+ another harness or wrapper mode, and :class:`PluginLoadError` for a+ permission-bypass refusal or a plugin with nothing loose.+ """+ if plugin_load == "wrapper" or source is None:+ raise LoosePartsUnsupported(NOT_SUPPORTED_WRAPPER)+ if agent != LOOSE_AGENT:+ raise LoosePartsUnsupported(not_supported_reason(agent))+ refusal = native_refusal(LoosePartsClaudeCodeAdapter(), source)+ if refusal is not None:+ raise PluginLoadError(refusal)+ return build_loose_parts(source, agent, plugin_load=plugin_load).task_staging()+++_LOOSE_AGENTS = "skillevaluator.tier3.harbor.loose_agents"+#: (agent, base Harbor import path or ``None`` for the stock agent) -> loose-parts wrapper import path.+LOOSE_PARTS_AGENT_IMPORT_PATHS: dict[tuple[str, str | None], str] = {+ (LOOSE_AGENT, None): f"{_LOOSE_AGENTS}:LoosePartsClaudeCode",+ # The container wrapper the Docker-engine and GKE baselines run (Vertex AI routing, the Node/npm skip).+ (+ LOOSE_AGENT,+ "skillevaluator.tier3.harbor.local_agents:SkillEvaluatorClaudeCode",+ ): f"{_LOOSE_AGENTS}:LoosePartsContainerClaudeCode",+ (+ LOOSE_AGENT,+ "skillevaluator.tier3.harbor.local_agents:SkillEvaluatorNvidiaBuildClaudeCode",+ ): f"{_LOOSE_AGENTS}:LoosePartsNvidiaBuildClaudeCode",+}+++def loose_parts_agent_import_path(agent: str, base_import_path: str | None) -> str:+ """The Harbor wrapper that runs the loose arm's setup before ``agent`` starts, or raise ``LoosePartsUnsupported``."""+ if agent != LOOSE_AGENT:+ raise LoosePartsUnsupported(not_supported_reason(agent))+ if base_import_path in LOOSE_PARTS_AGENT_IMPORT_PATHS.values():+ return base_import_path+ # #264 resolves ordinary routes to setup-classifying private wrappers on+ # the coordinator. Recover their original backend rather than falling back+ # to a stock Claude agent and losing Build/container routing.+ from skillevaluator.tier3.plugin_native import NATIVE_AGENT_IMPORT_PATHS, PRIVATE_AGENT_IMPORT_PATHS++ for paths in (PRIVATE_AGENT_IMPORT_PATHS, NATIVE_AGENT_IMPORT_PATHS):+ match = next(+ (base for (name, base), wrapped in paths.items() if name == agent and wrapped == base_import_path), None+ )+ if (agent, base_import_path) not in LOOSE_PARTS_AGENT_IMPORT_PATHS and any(+ name == agent and wrapped == base_import_path for (name, _), wrapped in paths.items()+ ):+ base_import_path = match+ break+ try:+ return LOOSE_PARTS_AGENT_IMPORT_PATHS[(agent, base_import_path)]+ except KeyError:+ raise LoosePartsUnsupported(+ f"not supported: the loose-parts arm has no Harbor wrapper for {agent} over "+ f"{base_import_path or 'the stock agent'}"+ ) from None+++def _unsupported_or_load_error(message: str) -> PluginLoadError:+ return LoosePartsUnsupported(message) if message.startswith("not supported:") else PluginLoadError(message)+++def _arm_key(arm: str) -> str:+ try:+ return ARM_ALIASES[arm]+ except KeyError:+ raise ValueError(f"unknown arm for loose-parts hashing: {arm!r}") from None+++def _to_loose(value: Any) -> Any:+ """Replace ``${CLAUDE_PLUGIN_ROOT}`` and ``${CLAUDE_PLUGIN_DATA}`` with the loose directories, in every string."""+ if isinstance(value, str):+ value = _ROOT_VAR_RE.sub(lambda _match: LOOSE_ROOT, value)+ return _DATA_VAR_RE.sub(lambda _match: LOOSE_DATA, value)+ if isinstance(value, list):+ return [_to_loose(item) for item in value]+ if isinstance(value, dict):+ return {key: _to_loose(item) for key, item in value.items()}+ return value+++def _canonical_roots(value: Any, template: Any) -> Any:+ """Undo only the loose arm's recorded substitutions, at their original fields."""+ if isinstance(template, str):+ replaced = _to_loose(template)+ if replaced != template:+ if value != replaced:+ raise ValueError("The staged loose field does not contain its recorded root substitution")+ return template+ if isinstance(value, list):+ original = template if isinstance(template, list) else []+ return [+ _canonical_roots(item, original[index] if index < len(original) else None)+ for index, item in enumerate(value)+ ]+ if isinstance(value, dict):+ original = template if isinstance(template, dict) else {}+ return {key: _canonical_roots(item, original.get(key)) for key, item in value.items()}+ return value+++def _strings(value: Any) -> Iterable[str]:+ if isinstance(value, str):+ yield value+ elif isinstance(value, list):+ for item in value:+ yield from _strings(item)+ elif isinstance(value, dict):+ for item in value.values():+ yield from _strings(item)+++class _Builder:+ """Plans one loose-parts bundle; every write goes to ``bundle`` and every decision to a record."""++ def __init__(self, source: NativePluginSource) -> None:+ self.source = source+ self.adapter = ClaudeCodeAdapter()+ self.slug = self.adapter.plugin_name(source)+ self.reader = PluginRootReader(source.plugin_root)+ self.full_bundle = self.adapter.build(source)+ self.bundle = NativeBundle(+ agent=LOOSE_AGENT, adapter=LOOSE_ADAPTER_ID, components=LoosePartsClaudeCodeAdapter().component_modes()+ )+ self.checks: list[CensusCheck] = []+ self.not_loaded: list[tuple[str, str, str]] = []+ self.setup: list[str] = [': "${CLAUDE_CONFIG_DIR:=$HOME/.claude}"']+ self.loose: list[_Placement] = []+ self.full: list[_Placement] = []+ self.glue: list[GluePiece] = []+ self.not_staged: list[GluePiece] = []+ self.references: list[str] = []+ self.normalized: dict[str, str] = {}+ self.capability_differences: dict[str, str] = {}+ self.root_text: dict[str, str] = {}+ self.staged_skills = self.adapter.staged_skills(source)+ self.rewrites = self._frontmatter_rewrites()+ # Paths inside the plugin copy that are not plugin source files: the files the plugin arm generates and+ # the member skills it copies in from outside the plugin root. A tree hash of plugin files leaves them out.+ self.generated_in_plugin = tuple(+ sorted(+ rel.removeprefix(f"{ClaudeCodeAdapter.bundle_dir}/")+ for rel in (*self.full_bundle.generated, *(dest for _source, dest in self.full_bundle.trees))+ if rel.startswith(f"{ClaudeCodeAdapter.bundle_dir}/")+ )+ )++ # -- the build ------------------------------------------------------------ #+ def build(self) -> LooseParts:+ self._skills()+ rules = self._rules()+ servers = self._mcp()+ hooks = self._hooks()+ texts = self._texts()+ self._scripts()+ self._glue()+ if not (rules or servers or hooks or texts):+ return LooseParts(+ agent=LOOSE_AGENT,+ bundle=None,+ dropped_as_glue=tuple(sorted(self.glue, key=lambda piece: piece.id)),+ not_staged=tuple(sorted(self.not_staged, key=lambda piece: piece.id)),+ error=NOTHING_LOOSE,+ source=self.source,+ )+ # The launch always exports CLAUDE_PLUGIN_DATA, so its directory always exists (empty, as for a new plugin).+ self.setup.append(f"mkdir -p {LOOSE_DATA} 2>/dev/null || skilleval_note mkdir:{LOOSE_DATA}")+ self.bundle.declared = [check.declared() for check in self.checks]+ self.bundle.harness = {"kind": LOOSE_HARNESS_KIND, "plugin": self.slug}+ self.bundle.files.sort(key=lambda item: item[1])+ self.bundle.generated["native/setup.sh"] = render_setup_script(+ agent=LOOSE_AGENT, setup_lines=self.setup, checks=self.checks, not_loaded=self.not_loaded+ )+ return LooseParts(+ agent=LOOSE_AGENT,+ bundle=self.bundle,+ dropped_as_glue=tuple(sorted(self.glue, key=lambda piece: piece.id)),+ not_staged=tuple(sorted(self.not_staged, key=lambda piece: piece.id)),+ rules=tuple(sorted(rules)),+ mcp_servers=tuple(servers),+ source=self.source,+ _loose=tuple(sorted(self.loose, key=lambda item: item.id)),+ _full=tuple(sorted(self.full, key=lambda item: item.id)),+ _normalized=tuple(sorted(self.normalized.items())),+ _capability_differences=tuple(sorted(self.capability_differences.items())),+ _root_text=tuple(sorted(self.root_text.items())),+ )++ def _note_root_text(self, piece_id: str, text: str) -> None:+ if _ROOT_VAR_RE.search(text) or _DATA_VAR_RE.search(text):+ self.root_text[piece_id] = PLUGIN_ROOT_IN_TEXT++ def _place(+ self,+ piece_id: str,+ kind: str,+ source: str | None,+ *,+ full: str,+ loose: str,+ root_template: Any = None,+ **extra: Any,+ ) -> None:+ full_overrides = tuple(+ (staged, original)+ for staged, original in self.rewrites.items()+ if staged == full or staged.startswith(f"{full}/")+ )+ if full_overrides:+ self.normalized[piece_id] = (+ "the plugin arm census-wraps this file's frontmatter hooks; the source file is hashed for that arm"+ )+ expected = tuple(+ (staged, _sha256(self.full_bundle.generated[staged.removeprefix(f"{BUNDLE_DIRNAME}/")].encode("utf-8")))+ for staged, _original in full_overrides+ )+ self.full.append(+ _Placement(piece_id, kind, full, source, overrides=full_overrides, override_digests=expected, **extra)+ )+ self.loose.append(+ _Placement(+ piece_id,+ kind,+ loose,+ source,+ root_template=json.dumps(root_template) if root_template is not None else None,+ **extra,+ )+ )++ def _frontmatter_rewrites(self) -> dict[str, Path]:+ """Staged plugin-arm file -> its source, for each file the plugin arm rewrites to census-wrap frontmatter hooks.++ The rewrites are the plugin bundle's own generated files (``stage_frontmatter_hooks`` output), found at+ the path the plugin arm stages each hook's file (a relocated member skill keeps its new directory).+ """+ root = self.source.plugin_root.resolve()+ relocated = {+ copied.resolve().relative_to(root).as_posix(): staged+ for _name, staged, copied in self.staged_skills+ if copied is not None and copied.resolve().is_relative_to(root)+ }+ rewrites: dict[str, Path] = {}+ for hook in self.source.hooks:+ if not is_frontmatter_hook_source(hook) or hook.rel is None:+ continue+ rel = PurePosixPath(hook.rel)+ staged = (PurePosixPath(relocated.get(rel.parent.as_posix(), rel.parent.as_posix())) / rel.name).as_posix()+ if f"{ClaudeCodeAdapter.bundle_dir}/{staged}" in self.full_bundle.generated:+ rewrites[f"{_PLUGIN_BUNDLE}/{staged}"] = self.source.plugin_root / hook.rel+ return rewrites++ # -- pieces --------------------------------------------------------------- #+ def _skills(self) -> None:+ full_dirs = {PurePosixPath(rel).name: rel for _name, rel, _copied in self.staged_skills}+ loose_names = set()+ for skill in self.source.member_skills:+ name = skill.name+ loose_names.add(name)+ manifest_text = _skill_manifest_text(skill)+ self._note_references(manifest_text)+ self._note_root_text(f"skill:{name}", manifest_text)+ self.checks.append(+ CensusCheck(+ "skill", name, "file", f"$CLAUDE_CONFIG_DIR/skills/{name}/SKILL.md", label="user skills listing"+ )+ )+ rel = full_dirs.get(name)+ source_rel = _plugin_rel(self.source.plugin_root, skill)+ if rel is None:+ self.loose.append(_Placement(f"skill:{name}", "skill", f"skills/{name}", source_rel))+ continue+ self._place(f"skill:{name}", "skill", source_rel, full=f"{_PLUGIN_BUNDLE}/{rel}", loose=f"skills/{name}")+ for name, rel in full_dirs.items():+ if name not in loose_names:+ # The plugin arm loads a skill that is not a member skill: a mismatch the hashes report.+ self.full.append(_Placement(f"skill:{name}", "skill", f"{_PLUGIN_BUNDLE}/{rel}", rel))++ def _rules(self) -> list[str]:+ """Stage the rules with the plugin arm's own writer, so path and bytes are the same in both arms."""+ not_loaded: list[tuple[str, str, str]] = []+ lines = self.adapter._stage_rules(self.bundle, self.source, self.checks, not_loaded)+ # The rule writer starts with the same CLAUDE_CONFIG_DIR default line this setup already has.+ self.setup.extend(line for line in lines if line not in self.setup)+ self.not_loaded.extend(not_loaded)+ staged: list[str] = []+ for name, content in self.source.rules:+ text, reason = _claude_user_rule(name, content)+ if text is None:+ self.not_staged.append(GluePiece(f"rule:{name}", f"not staged in any arm: {reason}"))+ continue+ path = f"{_RULES_BUNDLE}/{_claude_rule_file_name(name)}"+ self._place(f"rule:{name}", "rule", _rule_source(self.source, name), full=path, loose=path)+ staged.append(name)+ return staged++ def _mcp(self) -> list[str]:+ """Write the plain MCP config: the plugin arm's ``.mcp.json`` entries with the root variables replaced."""+ entries = {+ _redacted(str(server["name"])): _mcp_entry_claude(server, self.source.mcp_declared.get(str(server["name"])))+ for server in (*self.source.mcp_servers, *self.source.plugin_file_mcp_servers)+ }+ for entry in entries.values():+ for key, value in entry.items():+ self._note_references(value, exact_operand=key in {"command", "args"})+ loose_entries = {name: _to_loose(entry) for name, entry in entries.items()}+ self.bundle.generated[f"{LOOSE_BUNDLE_DIR}/mcp.json"] = (+ json.dumps({"mcpServers": loose_entries}, indent=2) + "\n"+ )+ for name, entry in entries.items():+ self.checks.append(+ CensusCheck(+ "mcp", name, "contains", LOOSE_MCP_CONFIG, needle=json.dumps(name), label="--mcp-config listing"+ )+ )+ self._place(+ f"mcp:{name}",+ "mcp",+ ".mcp.json",+ full=f"{_PLUGIN_BUNDLE}/.mcp.json",+ loose=f"{_LOOSE_BUNDLE}/mcp.json",+ key=name,+ root_template=entry,+ )+ return list(entries)++ def _hooks(self) -> list[str]:+ """Write the user ``settings.json``: the plugin arm's census-wrapped hooks with the root variables replaced."""+ wrapped = wrap_hook_sources(self.source.hooks, plugin_root=self.source.plugin_root)+ staged: list[str] = []+ for hook in self.source.hooks:+ if is_frontmatter_hook_source(hook):+ # Stays inside its skill or command file, staged as written; the files it names through+ # the plugin root are copied, and the launch exports CLAUDE_PLUGIN_ROOT for its shell.+ self._note_references(hook.config)+ continue+ identifiers = [identifier for owner, _event, identifier in wrapped.ids if owner == hook.name]+ dropped = [event for owner, event in wrapped.dropped if owner == hook.name]+ for event in dropped:+ self.not_staged.append(+ GluePiece(+ f"hook:{hook.name}#{event}",+ f"not staged in any arm: the Cursor hook event '{event}' has no Claude Code event",+ )+ )+ needle = (+ json.dumps(identifiers[0])[1:-1] if identifiers else _staged_handler_needle(_staged_hook_events(hook))+ )+ if needle is None:+ self.not_staged.append(+ GluePiece(f"hook:{hook.name}", "not staged in any arm: no hook handler Claude Code loads")+ )+ continue+ self.checks.append(+ CensusCheck(+ "hook",+ hook.name,+ "contains",+ "$CLAUDE_CONFIG_DIR/settings.json",+ needle=_to_loose(needle),+ label="user settings hooks listing",+ )+ )+ staged.append(hook.name)+ if not identifiers:+ continue # no command handler, so no hook id: hashed with the other unattributed groups below+ self.bundle.hook_ids[hook.name] = identifiers+ self._place(+ f"hook:{hook.name}",+ "hook",+ hook.rel,+ full=f"{_PLUGIN_BUNDLE}/hooks/hooks.json",+ loose=f"{_LOOSE_BUNDLE}/settings.json",+ key=hook.name,+ root_template=_hook_groups(wrapped.config, hook.name),+ )+ if not staged:+ return staged+ events = wrapped.config["hooks"]+ if _hook_groups({"hooks": events}, ""):+ # Groups with only prompt, agent, or http handlers carry no hook id to name their source.+ self._place(+ f"hook:{UNATTRIBUTED_HOOKS}",+ "hook",+ None,+ full=f"{_PLUGIN_BUNDLE}/hooks/hooks.json",+ loose=f"{_LOOSE_BUNDLE}/settings.json",+ key="",+ root_template=_hook_groups({"hooks": events}, ""),+ )+ self._note_references(events)+ settings = {"hooks": _to_loose(events)}+ self.bundle.generated[f"{LOOSE_BUNDLE_DIR}/settings.json"] = json.dumps(settings, indent=2) + "\n"+ dest = shell_path("$CLAUDE_CONFIG_DIR/settings.json")+ source = shlex.quote(f"{NATIVE_ROOT}/claude-code/loose/settings.json")+ self.setup.append(+ f'if [ -e {dest} ] || [ -L {dest} ]; then skilleval_note "exists:$CLAUDE_CONFIG_DIR/settings.json"; '+ f"else skilleval_copy {source} {dest}; fi"+ )+ return staged++ def _texts(self) -> list[str]:+ """Copy the subagents and commands where Claude Code reads user agents and commands."""+ manifest_commands = self.adapter.manifest(self.source).get("commands")+ staged: list[str] = []+ destinations: dict[str, str] = {}+ for text in self.source.texts:+ if text.type == "output_style":+ self.glue.append(GluePiece(f"output_style:{text.name}", _GLUE_REASONS["output_style"]))+ continue+ if text.type not in {"agent", "command"}:+ continue+ if text.type == "agent":+ if text.rel is None:+ continue+ # Claude Code names a subagent by its frontmatter, so the file name only has to be safe.+ file_name = f"{safe_name(PurePosixPath(text.rel).stem)}.md"+ harness_name = text.name+ folder = "agents"+ full_rel = text.rel+ else:+ harness_name = (+ text.name+ if text.rel is None or isinstance(manifest_commands, dict)+ else PurePosixPath(text.rel).stem+ )+ file_name = f"{safe_name(harness_name)}.md"+ folder = "commands"+ full_rel = text.rel if text.rel is not None else f"commands/{safe_name(text.name)}.md"+ dest_rel = f"{folder}/{file_name}"+ if dest_rel in destinations:+ raise ValueError(+ f"Refusing to stage loose parts: '{destinations[dest_rel]}' and '{text.name}' would both be "+ f"staged as $CLAUDE_CONFIG_DIR/{dest_rel}"+ )+ destinations[dest_rel] = text.name+ bundle_rel = f"{LOOSE_BUNDLE_DIR}/{dest_rel}"+ # A body can run a plugin file (``!`${CLAUDE_PLUGIN_ROOT}/scripts/x.sh` ``): copy what it names.+ self._note_references(text.text)+ piece_id = f"{text.type}:{harness_name}"+ self._note_root_text(piece_id, text.text)+ if text.type == "agent" and (ignored := _plugin_ignored_agent_fields(text.text)):+ self.capability_differences[piece_id] = (+ f"the subagent sets {', '.join(ignored)}: Claude Code says this is ignored for plugin agents, "+ "but a user agent honours it, so the same file is a different capability in the two arms"+ )+ if text.rel is None:+ self.bundle.generated[bundle_rel] = text.text+ else:+ self.bundle.files.append((self._plugin_file(text.rel, text.type), bundle_rel))+ target = f"$CLAUDE_CONFIG_DIR/{dest_rel}"+ self.setup.append(+ f"skilleval_copy {shlex.quote(f'{NATIVE_ROOT}/claude-code/loose/{dest_rel}')} {shell_path(target)}"+ )+ self.checks.append(+ CensusCheck(+ text.type,+ text.name,+ "file",+ target,+ label=f"user {folder} listing",+ harness_name=harness_name if text.type == "command" else "",+ )+ )+ self._place(+ piece_id,+ text.type,+ text.rel,+ full=f"{_PLUGIN_BUNDLE}/{full_rel}",+ loose=f"{BUNDLE_DIRNAME}/{bundle_rel}",+ )+ staged.append(text.name)+ return staged++ def _plugin_file(self, rel: str, what: str) -> Path:+ kind = self.reader.kind(PurePosixPath(rel))+ if kind != "file":+ raise ValueError(+ f"Refusing to stage plugin {what} '{rel}' for the loose-parts arm: it is not a single-link regular "+ f"file inside the plugin root ({kind})"+ )+ return self.source.plugin_root / rel++ def _note_references(self, value: Any, *, exact_operand: bool = False) -> None:+ for text in _strings(value):+ if exact_operand:+ prefix = next(+ (+ root+ for root in (_ROOT_VAR, "$CLAUDE_PLUGIN_ROOT")+ if text == root or text.startswith(root + "/")+ ),+ None,+ )+ if prefix is not None:+ raw = text[len(prefix) :]+ rel = posixpath.normpath(raw.strip("/")) if raw.strip("/") else "."+ # MCP command/args values are complete operands, so spaces,+ # colons and commas can belong to an authored filename.+ # A dynamic or shell expression needs the full root instead.+ match = _ROOT_PATH_RE.match(text)+ ambiguous = "$" in raw or (+ rel != ".."+ and not rel.startswith("../")+ and self.reader.kind(PurePosixPath(rel)) == "missing"+ and match is not None+ and (match.group(1) or "") != raw+ )+ self.references.append("" if ambiguous else raw)+ continue+ for match in _ROOT_PATH_RE.finditer(text):+ suffix = text[match.end() :]+ ambiguous = bool(suffix) and (+ suffix[0].isspace()+ or (+ suffix[0] in ":,"+ and not any(suffix[1:].startswith(root) for root in (_ROOT_VAR, "$CLAUDE_PLUGIN_ROOT"))+ )+ )+ # Quoted shell paths may contain spaces after the regex's+ # prefix. Copying the root keeps those files available too.+ self.references.append("" if ambiguous else match.group(1) or "")++ def _scripts(self) -> None:+ """Preserve referenced resources and their transitive dependencies under the same root."""+ rels: set[str] = set()+ for raw in self.references:+ rel = posixpath.normpath(raw.strip("/")) if raw.strip("/") else "."+ rels.add(rel)+ copy_root = False+ for rel in sorted(rels):+ if rel == ".":+ copy_root = True+ continue+ if rel == ".." or rel.startswith("../"):+ self.not_staged.append(+ GluePiece(+ f"script:{rel}",+ "points outside the plugin root; neither arm stages it (the plugin arm's copy holds only the "+ "plugin root)",+ )+ )+ continue+ if not self._kept_by_plugin_copy(rel):+ self.not_staged.append(+ GluePiece(f"script:{rel}", "the plugin arm's copy leaves this path out, so the loose arm does too")+ )+ continue+ kind = self.reader.kind(PurePosixPath(rel))+ if kind == "missing":+ self.not_staged.append(+ GluePiece(f"script:{rel}", "the referenced path does not exist in the plugin; neither arm has it")+ )+ continue+ if kind not in {"file", "dir"}:+ raise ValueError(+ f"Refusing to stage referenced plugin file '{rel}' for the loose-parts arm: it is a link or "+ f"special file ({kind})"+ )+ copy_root = True+ exclude = tuple(path for path in self.generated_in_plugin if path == rel or path.startswith(f"{rel}/"))+ self._place(+ f"script:{rel}",+ "script",+ rel,+ full=f"{_PLUGIN_BUNDLE}/{rel}",+ loose=f"{_LOOSE_BUNDLE}/root/{rel}",+ exclude=tuple(path.removeprefix(rel).lstrip("/") for path in exclude),+ )+ if copy_root:+ # An entrypoint can import siblings, ancestor-relative modules or+ # data at runtime. Reuse the full arm's filtered resource copy and+ # hash its entire tree so missing/changed support files break parity.+ self._whole_root()++ def _whole_root(self) -> None:+ """Copy and compare the full arm's resource tree without activating a plugin loader."""+ self.bundle.plugin_tree = f"{LOOSE_BUNDLE_DIR}/root"+ parents = {+ parent.as_posix()+ for rel in self.generated_in_plugin+ for parent in PurePosixPath(rel).parents+ if parent.parts+ }+ self._place(+ "script:.",+ "script",+ ".",+ full=_PLUGIN_BUNDLE,+ loose=f"{_LOOSE_BUNDLE}/root",+ exclude=self.generated_in_plugin,+ generated_dirs=tuple(+ sorted(+ rel+ for rel in parents+ if not self._kept_by_plugin_copy(rel) or self.reader.kind(PurePosixPath(rel)) != "dir"+ )+ ),+ )++ def _kept_by_plugin_copy(self, rel: str) -> bool:+ """Whether the plugin arm's whole-plugin copy keeps ``rel`` (same ignore rules as ``stage_native_bundle``)."""+ from skillevaluator.tier3.harbor.native_staging import _plugin_tree_ignore++ ignore = _plugin_tree_ignore(self.source)+ directory = self.source.plugin_root+ for part in PurePosixPath(rel).parts:+ if part in ignore(str(directory), [part]):+ return False+ directory = directory / part+ return True++ def _glue(self) -> None:+ for name in sorted(self.source.lsp_servers):+ self.glue.append(GluePiece(f"lsp:{name}", _GLUE_REASONS["lsp"]))+ for key in sorted(self.source.settings):+ self.glue.append(GluePiece(f"settings:{key}", _GLUE_REASONS["settings"]))+ for component_type, name in self.source.other:+ if component_type == "bin":+ self.glue.append(GluePiece("bin:bin/", _GLUE_REASONS["bin"]))+ elif (component_type == "lsp" and name in self.source.lsp_servers) or (+ component_type == "settings" and self.source.settings+ ):+ continue+ else:+ self.not_staged.append(+ GluePiece(+ f"{component_type}:{name}",+ "not staged in any arm: the Claude Code plugin arm does not load it either",+ )+ )+ self.glue.append(+ GluePiece(f"manifest:{self.source.manifest_rel or '.claude-plugin/plugin.json'}", _GLUE_REASONS["manifest"])+ )+ user_config = self.source.manifest.get("userConfig") if self.source.contained else None+ if isinstance(user_config, Mapping):+ for key in sorted(str(item) for item in user_config):+ self.glue.append(GluePiece(f"userConfig:{key}", _GLUE_REASONS["userConfig"]))+++def _plugin_ignored_agent_fields(text: str) -> list[str]:+ """The frontmatter fields of a subagent that Claude Code ignores in a plugin agent (in declared spelling)."""+ from skillevaluator.plugin_markdown import parse_markdown++ frontmatter = parse_markdown(text).frontmatter+ found: list[str] = []+ for key in frontmatter:+ spelled = _PLUGIN_IGNORED_AGENT_FIELDS.get(str(key).replace("-", "").replace("_", "").casefold())+ if spelled is not None and spelled not in found:+ found.append(spelled)+ return sorted(found)+++def _skill_manifest_text(skill_dir: Path) -> str:+ """A member skill's ``SKILL.md`` (bounded, no-follow read), or ``""`` when it has none."""+ reader = PluginRootReader(skill_dir)+ for name in ("SKILL.md", "skill.md"):+ if reader.kind(PurePosixPath(name)) == "file":+ return reader.read_text(PurePosixPath(name), CONTENT_DEDUP_MAX_FILE_BYTES)+ return ""+++def _plugin_rel(plugin_root: Path, path: Path) -> str | None:+ try:+ return path.resolve().relative_to(plugin_root.resolve()).as_posix()+ except ValueError:+ return None+++def _rule_source(source: NativePluginSource, name: str) -> str | None:+ """The plugin-relative rule file, when the rule name is one (``rules/<name>``)."""+ candidate = PurePosixPath("rules") / name+ reader = PluginRootReader(source.plugin_root)+ return candidate.as_posix() if reader.kind(candidate) == "file" else None+++def _stage_member_skills(skills_dir: Path, member_skills: Sequence[Path], excluded_roots: Sequence[Path]) -> None:+ """The task skills projection of the member skills, with the emitter's own copy rules."""+ from skillevaluator.tier3.harbor.adapter import _runtime_skill_copy_ignore, _sanitize_staged_runtime_skills+ from skillevaluator.tier3.harbor.secure_copy import copytree_secure++ skills_dir.mkdir(parents=True, exist_ok=True)+ for skill in member_skills:+ if not skill.is_dir():+ continue+ destination = skills_dir / skill.name+ if destination.exists():+ continue+ copytree_secure(+ skill, destination, dirs_exist_ok=True, ignore=_runtime_skill_copy_ignore(skill, excluded_roots)+ )+ _sanitize_staged_runtime_skills(skills_dir)+++# --------------------------------------------------------------------------- #+# Hashing #+# --------------------------------------------------------------------------- #+def _sha256(data: bytes) -> str:+ return hashlib.sha256(data).hexdigest()+++def _canonical_json(value: Any, root_template: str | None = None) -> bytes:+ if root_template is not None:+ value = _canonical_roots(value, json.loads(root_template))+ return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode()+++def _read_regular(path: Path) -> bytes | None:+ return (_PARITY_READS.get() or _ParityReads()).read(path)+++def _tree_digest(+ root: Path, overrides: Mapping[str, Path], exclude: Sequence[str], generated_dirs: Sequence[str] = ()+) -> str | None:+ """Hash relative file content and file/directory rwx modes without following links."""+ try:+ if not stat.S_ISDIR(root.lstat().st_mode):+ return None+ except OSError:+ return None+ lines: list[str] = []+ total_bytes = 0+ file_count = 0+ directory_count = 0+ generated_parents = frozenset(generated_dirs)+ for directory, dirnames, filenames in os.walk(root, followlinks=False):+ relative_directory = Path(directory).relative_to(root).as_posix()+ if any(relative_directory == skipped or relative_directory.startswith(f"{skipped}/") for skipped in exclude):+ dirnames[:] = []+ continue+ metadata = Path(directory).lstat()+ if not stat.S_ISDIR(metadata.st_mode):+ return None+ directory_count += 1+ if directory_count > MAX_PARITY_FILES:+ raise ValueError("Parity content exceeds the bounded directory budget")+ # The full adapter creates manifest/census configuration directories.+ # Normalize only a leaf containing exclusively explicitly excluded+ # generated files; unexpected children remain part of the digest.+ generated_only_leaf = (+ bool(filenames)+ and not dirnames+ and all((PurePosixPath(relative_directory) / name).as_posix() in exclude for name in filenames)+ )+ if not generated_only_leaf and relative_directory not in generated_parents:+ lines.append(f"{relative_directory}/\tmode:{stat.S_IMODE(metadata.st_mode) & 0o777:o}\n")+ dirnames.sort()+ for name in sorted(filenames) + [item for item in dirnames if (Path(directory) / item).is_symlink()]:+ path = Path(directory) / name+ rel = path.relative_to(root).as_posix()+ if any(rel == skipped or rel.startswith(f"{skipped}/") for skipped in exclude):+ continue+ metadata = path.lstat()+ if not stat.S_ISREG(metadata.st_mode):+ return None+ original = overrides.get(path.as_posix())+ data = _read_regular(original if original is not None else path)+ if data is None:+ return None+ total_bytes += len(data)+ file_count += 1+ if file_count > CONTENT_DEDUP_MAX_FILES or total_bytes > CONTENT_DEDUP_MAX_TOTAL_BYTES:+ raise ValueError("Parity content exceeds the bounded file budget")+ lines.append(f"{rel}\t{_sha256(data)}\t{stat.S_IMODE(metadata.st_mode) & 0o777:o}\n")+ return _sha256("".join(sorted(lines)).encode())+++def _load_json(env_dir: Path, rel: str, cache: dict[str, Any]) -> Any:+ if rel not in cache:+ data = _read_regular(env_dir / rel)+ try:+ cache[rel] = json.loads(data.decode("utf-8")) if data is not None else None+ except (UnicodeDecodeError, ValueError):+ cache[rel] = None+ return cache[rel]+++def _hook_owner(handler: Any) -> str | None:+ """The hook source a census-wrapped command handler belongs to (from its hook id)."""+ if not isinstance(handler, Mapping):+ return None+ identifier: str | None = None+ command = handler.get("command")+ args = handler.get("args")+ if command == "/bin/sh" and isinstance(args, list) and len(args) > 1 and args[0] == HOOK_CENSUS_SCRIPT:+ identifier = str(args[1])+ elif isinstance(command, str) and command.startswith(f"/bin/sh {HOOK_CENSUS_SCRIPT} "):+ try:+ words = shlex.split(command)+ except ValueError:+ return None+ identifier = words[2] if len(words) > 2 else None+ match = _HOOK_ID_SOURCE_RE.match(identifier or "")+ return match.group("source") if match else None+++def _hook_groups(config: Any, source_name: str) -> list[Any] | None:+ """``[event, group]`` pairs of one hook source, in staged order (a group belongs to its first id's source)."""+ events = config.get("hooks") if isinstance(config, Mapping) else None+ if not isinstance(events, Mapping):+ return None+ owned: list[Any] = []+ for event, groups in events.items():+ for group in groups if isinstance(groups, list) else ():+ handlers = group.get("hooks") if isinstance(group, Mapping) else None+ owners = [owner for handler in handlers or () if (owner := _hook_owner(handler)) is not None]+ if (owners[0] if owners else "") == source_name:+ owned.append([event, group])+ return owned or None+++def _piece_digest(env_dir: Path, placement: _Placement, cache: dict[str, Any]) -> str | None:+ """One piece's sha256 in one staged arm, or ``None`` when the arm does not have it."""+ if placement.kind == "mcp":+ config = _load_json(env_dir, placement.path, cache)+ servers = config.get("mcpServers") if isinstance(config, Mapping) else None+ entry = servers.get(placement.key) if isinstance(servers, Mapping) else None+ return (+ _structured_piece_digest(env_dir / placement.path, entry, placement.root_template)+ if isinstance(entry, Mapping)+ else None+ )+ if placement.kind == "hook":+ groups = _hook_groups(_load_json(env_dir, placement.path, cache), placement.key or "")+ return (+ _structured_piece_digest(env_dir / placement.path, groups, placement.root_template)+ if groups is not None+ else None+ )+ target = env_dir / placement.path+ for staged, expected in placement.override_digests:+ emitted = _read_regular(env_dir / staged)+ if emitted is None or _sha256(emitted) != expected:+ return None+ overrides = {(env_dir / staged).as_posix(): original for staged, original in placement.overrides}+ try:+ metadata = target.lstat()+ is_dir = stat.S_ISDIR(metadata.st_mode)+ except OSError:+ return None+ if is_dir:+ if placement.kind == "skill" and _read_regular(target / "SKILL.md") is None:+ return None+ return _tree_digest(target, overrides, placement.exclude, placement.generated_dirs)+ if not stat.S_ISREG(metadata.st_mode) or metadata.st_nlink != 1:+ return None+ original = overrides.get(target.as_posix())+ data = _read_regular(original if original is not None else target)+ latest = target.lstat()+ if (metadata.st_ino, metadata.st_size, metadata.st_mtime_ns, metadata.st_ctime_ns) != (+ latest.st_ino,+ latest.st_size,+ latest.st_mtime_ns,+ latest.st_ctime_ns,+ ):+ return None+ mode = stat.S_IMODE(latest.st_mode) & 0o777+ return _sha256(data + f"\nmode:{mode:o}".encode()) if data is not None else None+++def _structured_piece_digest(path: Path, payload: Any, root_template: str | None = None) -> str | None:+ """Config layout may differ across arms; its effective payload and rwx bits must match."""+ try:+ metadata = path.lstat()+ except OSError:+ return None+ if not stat.S_ISREG(metadata.st_mode) or metadata.st_nlink != 1:+ return None+ try:+ canonical = _canonical_json(payload, root_template)+ except ValueError:+ return None+ return _sha256(canonical + f"\nmode:{stat.S_IMODE(metadata.st_mode) & 0o777:o}".encode())+++def compare_content_hashes(hashes_by_arm: Mapping[str, Mapping[str, str]]) -> dict[str, Any]:+ """``content_hashes``, ``equal_across_arms``, and ``mismatches`` for the staged arms' piece hashes.++ A piece is a mismatch when the full and loose arms disagree on it (or only+ one has it), or when the skills-only arm's copy of a skill differs from the+ others, misses a skill they have, or holds a non-skill piece. Glue is never+ hashed, and names are bare ids, so namespacing and packaging never count.+ """+ hashes = {_arm_key(arm): dict(sorted(values.items())) for arm, values in hashes_by_arm.items()}+ full, loose, skills = hashes.get(ARM_WITH), hashes.get(LOOSE_PARTS_ARM), hashes.get(ARM_SUM_OF_PARTS)+ mismatches: set[str] = set()+ if full is not None and loose is not None:+ mismatches.update(piece for piece in full.keys() | loose.keys() if full.get(piece) != loose.get(piece))+ if skills is not None:+ for piece, digest in skills.items():+ if not piece.startswith("skill:"):+ mismatches.add(piece)+ for other in (full, loose):+ if other is not None and other.get(piece) != digest:+ mismatches.add(piece)+ reference = loose if loose is not None else full+ mismatches.update(piece for piece in reference or () if piece.startswith("skill:") and piece not in skills)+ return {+ "content_hashes": dict(sorted(hashes.items())),+ "equal_across_arms": not mismatches if all(arm in hashes and hashes[arm] for arm in COMPARED_ARMS) else None,+ "mismatches": sorted(mismatches),+ }+++# --------------------------------------------------------------------------- #+# Load evidence (Claude Code system/init, MCP call evidence) #+# --------------------------------------------------------------------------- #+def _init_names(events: Sequence[Mapping[str, Any]], key: str) -> set[str]:+ return {item for event in events for item in event.get(key) or () if isinstance(item, str)}+++def _mcp_statuses(events: Sequence[Mapping[str, Any]]) -> dict[str, str]:+ """Server name -> the best status any init event reported (``connected`` wins)."""+ statuses: dict[str, str] = {}+ for event in events:+ for entry in event.get("mcp_servers") or ():+ if isinstance(entry, Mapping) and isinstance(entry.get("name"), str):+ status = str(entry.get("status") or "unknown")[:64]+ if statuses.get(entry["name"]) != "connected":+ statuses[entry["name"]] = status+ return statuses+++def _successful_mcp_calls(trajectory: Any, servers: Sequence[str], plugin: str) -> dict[str, dict[str, Any]]:+ """Per declared server, the MCP call outcome counts the plugin signals read from the trajectory."""+ if not isinstance(trajectory, Mapping) or not servers:+ return {}+ from skillevaluator.tier3.eval_core.plugin_signals import compute_plugin_signals++ signals = compute_plugin_signals(trajectory, None, declared={"mcp": list(servers), "plugin": [plugin]})+ by_server = (signals or {}).get("mcp_calls", {}).get("by_server", {})+ return {str(name): dict(counts) for name, counts in by_server.items() if isinstance(counts, Mapping)}+++def loose_parts_load_evidence(+ system_init_events: Sequence[Mapping[str, Any] | None],+ trajectory: Mapping[str, Any] | None,+ loose_parts: LooseParts,+ *,+ arm: str = LOOSE_PARTS_ARM,+ mcp_calls: Mapping[str, Mapping[str, Any]] | None = None,+) -> dict[str, dict[str, Any]]:+ """``{piece id: {loaded, source, evidence, ...}}`` for one trial of ``arm``.++ Only Claude Code's own ``system/init`` report marks a piece loaded+ (``source: system/init``): a skill in ``skills``/``slash_commands``, a+ subagent in ``agents``, a command in ``slash_commands``, an MCP server in+ ``mcp_servers`` with status ``connected``. The loose arm and the+ skills-only arm use bare names; the full plugin arm the plugin's+ namespaced names (``<plugin>:<name>``, ``plugin:<plugin>:<server>``) and the+ plugin must be listed. Workaround for servers init reports ``pending``+ (slow start) or not at all: a successful call of one of the server's tools+ in the trial's ATIF trajectory marks it loaded (``source: mcp-call``).+ Rules, hooks, and referenced scripts are not in the startup report:+ ``loaded`` is ``None`` (``source: unverifiable``), never a load failure.+ """+ arm = _arm_key(arm)+ events = [event for event in system_init_events if isinstance(event, Mapping)]+ placements = loose_parts._placements(arm)+ source = loose_parts.source+ plugin = plugin_slug(source.plugin_name) if source is not None else ""+ full = arm == ARM_WITH+ names = {+ "skill": _init_names(events, "skills") | _init_names(events, "slash_commands"),+ "agent": _init_names(events, "agents"),+ "command": _init_names(events, "slash_commands"),+ }+ statuses = _mcp_statuses(events)+ tools = _init_names(events, "tools")+ plugin_listed = any(+ isinstance(entry, Mapping) and entry.get("name") == plugin+ for event in events+ for entry in event.get("plugins") or ()+ )+ servers = [placement.name for placement in placements if placement.kind == "mcp"]+ calls = dict(mcp_calls) if mcp_calls is not None else _successful_mcp_calls(trajectory, servers, plugin)+ evidence: dict[str, dict[str, Any]] = {}+ for placement in placements:+ kind, name = placement.kind, placement.name+ if kind not in _INIT_KINDS:+ evidence[placement.id] = {+ "loaded": None,+ "source": _EVIDENCE_NONE,+ "evidence": f"Claude Code's system/init event does not report {kind}s; staged file listing only",+ }+ continue+ if not events:+ evidence[placement.id] = {+ "loaded": False,+ "source": _EVIDENCE_NONE,+ "evidence": "no system/init event in the Claude Code log (the agent did not start)",+ }+ continue+ if full and not plugin_listed:+ evidence[placement.id] = {+ "loaded": False,+ "source": _EVIDENCE_INIT,+ "evidence": f"claude-code did not load plugin {plugin} (not listed in its init event)",+ }+ continue+ if kind == "mcp":+ evidence[placement.id] = _mcp_evidence(name, plugin if full else "", statuses, tools, calls)+ continue+ reported = f"{plugin}:{name}" if full else name+ if reported in names[kind]:+ evidence[placement.id] = {"loaded": True, "source": _EVIDENCE_INIT, "evidence": f"init lists {reported}"}+ else:+ evidence[placement.id] = {+ "loaded": False,+ "source": _EVIDENCE_INIT,+ "evidence": f"not reported by claude-code init (no {reported})",+ }+ return evidence+++def _mcp_evidence(+ server: str,+ plugin: str,+ statuses: Mapping[str, str],+ tools: set[str],+ calls: Mapping[str, Mapping[str, Any]],+) -> dict[str, Any]:+ reported = f"plugin:{plugin}:{server}" if plugin else server+ spelled = _SERVER_NAME_RE.sub("_", f"plugin_{plugin}_{server}" if plugin else server)+ prefix = f"mcp__{spelled}__"+ listed_tools = sorted(tool[len(prefix) :] for tool in tools if tool.startswith(prefix))+ status = statuses.get(reported)+ row: dict[str, Any] = {"status": status, "tools": listed_tools}+ if status == "connected":+ return {"loaded": True, "source": _EVIDENCE_INIT, "evidence": f"init reports {reported} connected", **row}+ said = f"init reported {reported} {status}" if status else f"init did not report {reported}"+ counts = calls.get(server) or {}+ succeeded = counts.get("succeeded") if isinstance(counts.get("succeeded"), int) else 0+ if succeeded:+ called = sorted(+ label for label, outcome in (counts.get("by_tool") or {}).items() if (outcome or {}).get("succeeded")+ )+ return {+ "loaded": True,+ "source": _EVIDENCE_CALL,+ "evidence": f"{said}; {succeeded} successful call(s) of {', '.join(called) or server} in the trajectory",+ **row,+ }+ attempted = counts.get("total") if isinstance(counts.get("total"), int) else 0+ note = f"; {attempted} call(s), none succeeded" if attempted else "; no successful call of its tools"+ return {"loaded": False, "source": _EVIDENCE_INIT, "evidence": f"{said}{note}", **row}+++def load_failures_by_case(+ evidence_by_arm: Mapping[str, Mapping[str, Sequence[Mapping[str, Mapping[str, Any]]]]],+) -> dict[str, list[dict[str, Any]]]:+ """``{case id: [{arm, attempt, piece, reason}]}`` for every case where a staged piece did not load.++ ``evidence_by_arm`` maps an arm to ``{case id: [evidence per attempt]}``+ (each from :func:`loose_parts_load_evidence`). A piece with ``loaded`` False+ in any attempt of any arm makes the case a load failure; the stats drop that+ case from every arm, so the arms stay paired. ``None`` (not verifiable) is+ never a failure.+ """+ failures: dict[str, list[dict[str, Any]]] = {}+ for arm, cases in sorted(evidence_by_arm.items()):+ for case_id, attempts in sorted(cases.items()):+ for attempt, evidence in enumerate(attempts, start=1):+ for piece, row in sorted(evidence.items()):+ if isinstance(row, Mapping) and row.get("loaded") is False:+ failures.setdefault(str(case_id), []).append(+ {+ "arm": _arm_key(arm),+ "attempt": attempt,+ "piece": piece,+ "reason": str(row.get("evidence") or "not loaded"),+ }+ )+ return failuresdiff --git a/src/skillevaluator/tier3/mcp_proof.py b/src/skillevaluator/tier3/mcp_proof.pynew file mode 100644index 00000000..197c7ee2--- /dev/null+++ b/src/skillevaluator/tier3/mcp_proof.py@@ -0,0 +1,985 @@+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+# SPDX-License-Identifier: Apache-2.0++"""Private opt-in MCP proof for author-supplied URL servers.++Before a plugin run, :func:`probe_mcp_servers` performs one bounded host probe+per URL server: MCP ``initialize`` followed by ``tools/list`` over streamable+HTTP or SSE, using the ``mcp`` client library with a constrained ``httpx``+client (no redirects, no proxy or ``.netrc`` credentials, strict timeouts, and+a response-size cap). The endpoint policy runs first: cloud metadata endpoints+are never probed, and private, loopback, or link-local endpoints (including+host names that resolve to them) are probed only when+``mcp.allowed_private_hosts`` allowlists them, by address, network, or host+name. Every connection is pinned to the addresses the policy checked, so a+host name that re-resolves elsewhere (DNS rebinding) is never reached. The+only headers sent are the server's declared literal header values.+The plugin chooses both the URL and the header template, so a ``${VAR}`` or+``$VAR`` reference is expanded from the host environment only when the user+grants that variable through ``EvidenceOptions.env_grants`` (``NAME`` for every server,+``NAME=HOST`` or ``NAME@SERVER`` for one); a header that references any other+variable is not sent, and the variable is never read. Each server's detail+records which variables were sent to which host. The whole probe, DNS lookup+included, runs under one deadline, and a transport error that the ``mcp``+client only logs (an oversized body, an off-origin SSE endpoint) ends the+probe at once.++After the run, :func:`apply_in_agent_mcp_proof` updates each server from the+with-plugin arm's ``plugin_signals_summary.mcp_calls.by_server`` and each+agent's ``plugin_load_census``: ``used-successfully`` when at least one call+succeeded; otherwise ``not-loaded-in-agent`` when the harness reported that the+server failed to load in each observed trial (the observer's+per-trial ``mcp_load`` tally tells a flaky load from a server that never+loaded), and ``called-no-success`` when the agent called it but no call+succeeded. Only a successful call or a successful+host ``initialize`` (``reachable-host``, not contradicted in the agent) proves+a server reachable; a failed call never turns ``unreachable`` into anything+else.++The result is ``provenance["mcp_proof"] = {"<server>": {"status", "tools",+"detail"}}``. It is advisory: a runnable URL server that stays unproven never+changes the ``INCOMPLETE`` rule or any score.+"""++from __future__ import annotations++import ipaddress+import json+import logging+import os+import re+import socket+import threading+import time+from collections.abc import Callable, Iterable, Iterator, Mapping+from contextlib import contextmanager+from dataclasses import dataclass+from itertools import islice+from typing import Any+from urllib.parse import urlsplit++from skillevaluator.tier3.eval_core.plugin_signals import declared_mcp_server_matcher, sanitize_input_schema+from skillevaluator.utils.redaction import redact_printable_text+from skillevaluator.validators.mcp_static import HostAllowlist, classify_endpoint_host, host_name_is_allowlisted++STATUS_DECLARED = "declared"+STATUS_REACHABLE_HOST = "reachable-host"+STATUS_UNREACHABLE = "unreachable"+STATUS_UNSUPPORTED = "unsupported"+STATUS_CALLED_NO_SUCCESS = "called-no-success"+STATUS_NOT_LOADED_IN_AGENT = "not-loaded-in-agent"+STATUS_USED_SUCCESSFULLY = "used-successfully"+#: Written by older releases for "called, nothing succeeded"; read as ``called-no-success``.+STATUS_REACHABLE_IN_AGENT = "reachable-in-agent"+MCP_PROOF_STATUSES = (+ STATUS_DECLARED,+ STATUS_REACHABLE_HOST,+ STATUS_UNREACHABLE,+ STATUS_UNSUPPORTED,+ STATUS_CALLED_NO_SUCCESS,+ STATUS_NOT_LOADED_IN_AGENT,+ STATUS_USED_SUCCESSFULLY,+)+#: Statuses that prove a server reachable: a successful host initialize or a successful agent call.+PROVEN_STATUSES = frozenset({STATUS_REACHABLE_HOST, STATUS_USED_SUCCESSFULLY})+# Statuses the agent's own evidence may replace when no call succeeded. A host+# failure (unreachable, unsupported) stays: a failed call proves nothing more.+_AGENT_REPLACEABLE = frozenset(+ {STATUS_DECLARED, STATUS_REACHABLE_HOST, STATUS_CALLED_NO_SUCCESS, STATUS_REACHABLE_IN_AGENT}+)+_COUNT_KEYS = ("total", "succeeded", "failed", "unknown")++MAX_PROBE_TOOLS = 50+MAX_PROBE_SERVERS = 32+MAX_RESOLVED_ADDRESSES = 32+MAX_RESPONSE_BYTES = 1024 * 1024+MAX_DETAIL_CHARS = 300+MAX_TOOL_NAME_CHARS = 128+#: Largest kept tool ``inputSchema`` (serialized), and the budget for all servers' schemas in one run.+MAX_INPUT_SCHEMA_CHARS = 2048+MAX_INPUT_SCHEMAS_CHARS = 128 * 1024+CONNECT_TIMEOUT_S = 5.0+READ_TIMEOUT_S = 10.0+TOTAL_TIMEOUT_S = 20.0++NOT_REQUESTED_DETAIL = "host probe not requested (private EvidenceOptions.probe_mcp is false)"+_HEADER_REF_RE = re.compile(r"\$\{([A-Za-z_][A-Za-z0-9_]*)\}|\$([A-Za-z_][A-Za-z0-9_]*)")+# A private environment grant: ``NAME``, ``NAME=HOST`` (only servers whose URL host is HOST) or+# ``NAME@SERVER`` (only that server). NAME has the shape of a header's ``${VAR}`` reference.+_ENV_GRANT_RE = re.compile(r"([A-Za-z_][A-Za-z0-9_]*)(?:=(.+)|@(.+))?")+_GRANT_HOST_RE = re.compile(r"[a-z0-9._:-]{1,253}")+_CONTROL_RE = re.compile(r"[\x00-\x1f\x7f-\x9f]")+# The ``mcp`` client logs and swallows some transport errors; the probe listens to these loggers.+_MCP_LOGGER_NAMES = ("mcp",)+_STREAMABLE_TRANSPORTS = frozenset({"", "http", "streamable-http", "streamable_http", "streamablehttp"})+_SSE_TRANSPORTS = frozenset({"sse"})+++class _ProbeRefused(Exception):+ """The endpoint policy or transport rules forbid a probe."""++ def __init__(self, status: str, detail: str) -> None:+ super().__init__(detail)+ self.status = status+ self.detail = detail+++class _ProbeFailed(Exception):+ """A fatal transport error the ``mcp`` client would otherwise only log (the probe stops at once)."""+++@dataclass(frozen=True)+class EnvGrant:+ """One private environment grant: ``NAME`` (any server), ``NAME=HOST`` or ``NAME@SERVER``."""++ name: str+ host: str = ""+ server: str = ""++ def applies_to(self, server: str, host: str) -> bool:+ if self.host:+ return self.host == host+ if self.server:+ return self.server == server+ return True+++def parse_env_grant(value: str | EnvGrant) -> EnvGrant:+ """Parse ``NAME``, ``NAME=HOST`` or ``NAME@SERVER``; raise ``ValueError`` when malformed."""+ if isinstance(value, EnvGrant):+ return value+ match = _ENV_GRANT_RE.fullmatch(str(value))+ if match is None:+ raise ValueError(f"{value!r} is not an environment variable name (NAME, NAME=HOST or NAME@SERVER)")+ name, host, server = match.group(1), match.group(2), match.group(3)+ if host is not None:+ host = host.strip("[]").lower()+ if not _GRANT_HOST_RE.fullmatch(host):+ raise ValueError(f"{value!r}: {host!r} is not a host name or address")+ if ":" in host:+ try:+ ipaddress.IPv6Address(host)+ except ValueError as exc:+ raise ValueError(f"{value!r}: give the host without a port") from exc+ return EnvGrant(name, host=host)+ if server is not None:+ if _CONTROL_RE.search(server) or len(server) > 256:+ raise ValueError(f"{value!r}: the server name is not printable")+ return EnvGrant(name, server=server)+ return EnvGrant(name)+++def _target_host(url: str) -> str:+ try:+ return (urlsplit(url).hostname or "").lower()+ except ValueError:+ return ""+++def _allowed_env(target: Mapping[str, Any], grants: Iterable[EnvGrant]) -> frozenset[str]:+ """Variable names the user opted in for this server (by name, by its URL host, or for every server)."""+ server = str(target.get("name") or "").strip()+ host = _target_host(str(target.get("url") or ""))+ return frozenset(grant.name for grant in grants if grant.applies_to(server, host))+++def planned_env_sends(+ targets: Iterable[Mapping[str, Any]], expand_env: Iterable[str | EnvGrant] = ()+) -> list[tuple[str, str, list[str]]]:+ """``(server, host, variables)`` for each server whose declared headers would carry opted-in variables.++ Only names that the user opted in for that server and that its headers+ reference are listed; the values are never read here.+ """+ grants = [parse_env_grant(value) for value in expand_env]+ plan: list[tuple[str, str, list[str]]] = []+ for target in islice(targets, MAX_PROBE_SERVERS):+ name = str(target.get("name") or "").strip()+ headers = target.get("headers")+ if not name or not grants or not isinstance(headers, Mapping):+ continue+ allowed = _allowed_env(target, grants)+ referenced: dict[str, None] = {}+ for value in headers.values():+ if isinstance(value, str):+ refs = [first or second for first, second in _HEADER_REF_RE.findall(value)]+ if refs and all(ref in allowed for ref in refs):+ referenced.update(dict.fromkeys(refs))+ if referenced:+ plan.append((name, _target_host(str(target.get("url") or "")), sorted(referenced)))+ return plan+++def _detail(text: Any) -> str:+ """Bounded, printable, redacted text: terminal controls and C0/C1 characters are removed."""+ text = redact_printable_text(str(text), max_input_chars=MAX_DETAIL_CHARS * 8)+ return " ".join(text.split())[:MAX_DETAIL_CHARS]+++def _entry(status: str, detail: str, tools: Iterable[str] = ()) -> dict[str, Any]:+ return {"status": status, "tools": list(islice(tools, MAX_PROBE_TOOLS)), "detail": _detail(detail)}+++def _input_schema(raw: Any) -> dict[str, Any] | None:+ """A tool's ``inputSchema`` cut to the subset argument checks run, or ``None`` when nothing is left to check.++ A schema that only says the arguments are an object checks nothing and is+ dropped, and so is one whose kept part serializes to more than+ ``MAX_INPUT_SCHEMA_CHARS``.+ """+ schema = sanitize_input_schema(raw)+ if not schema or set(schema) == {"type"}:+ return None+ try:+ size = len(json.dumps(schema, sort_keys=True, allow_nan=False))+ except (TypeError, ValueError, RecursionError):+ return None+ return schema if size <= MAX_INPUT_SCHEMA_CHARS else None+++def declared_mcp_proof(targets: Iterable[Mapping[str, Any]], detail: str = NOT_REQUESTED_DETAIL) -> dict[str, Any]:+ """Every URL server as ``declared`` (no host probe was run)."""+ proof: dict[str, Any] = {}+ for target in islice(targets, MAX_PROBE_SERVERS):+ name = str(target.get("name") or "").strip()+ if name:+ proof[name] = _entry(STATUS_DECLARED, detail)+ return proof+++def _resolve_headers(+ headers: Mapping[str, Any], environ: Mapping[str, str], expand_env: Iterable[str] = ()+) -> tuple[dict[str, str], list[str], list[str], list[str]]:+ """Return ``(headers to send, unset variables, withheld variables, sent variables)``.++ A header whose ``${VAR}``/``$VAR`` references all name variables in+ ``expand_env`` (the names opted in for this server) is sent with them+ resolved from ``environ``; one that references an opted-in but unset+ variable is dropped (``unset``). A header that references any variable not+ opted in for this server is dropped without reading that variable+ (``withheld``). Literal headers are sent as declared.+ """+ allowed = frozenset(expand_env)+ resolved: dict[str, str] = {}+ missing: list[str] = []+ withheld: list[str] = []+ sent: list[str] = []+ for key, value in headers.items():+ if not isinstance(key, str) or not isinstance(value, str):+ continue+ names = [first or second for first, second in _HEADER_REF_RE.findall(value)]+ blocked = [name for name in names if name not in allowed]+ if blocked:+ withheld.extend(blocked)+ continue+ unset = [name for name in names if not environ.get(name)]+ if unset:+ missing.extend(unset)+ continue+ resolved[key] = _HEADER_REF_RE.sub(lambda match: environ[match.group(1) or match.group(2)], value)+ sent.extend(names)+ return resolved, sorted(set(missing)), sorted(set(withheld)), sorted(set(sent))+++def _header_note(missing: list[str], withheld: list[str], sent: list[str] | None = None, host: str = "") -> str:+ note = ""+ if sent:+ note += f"; sent {', '.join(sent)} to {host or 'the server'}"+ if withheld:+ note += (+ f"; headers not sent because they reference host variables without a private environment grant "+ f"for this server: {', '.join(withheld)}"+ )+ if missing:+ note += f"; headers skipped because unset: {', '.join(missing)}"+ return note+++def _address_policy(address: str, allowlist: HostAllowlist, *, name_allowed: bool, label: str) -> bool:+ """Return True for an allowed private address; raise for a refused one; False when public.++ ``name_allowed`` is True when the URL's host name itself is allowlisted, which+ accepts any private address it resolves to. Metadata addresses are always refused.+ """+ endpoint = classify_endpoint_host(address)+ if endpoint is None:+ return False+ if endpoint.kind == "metadata":+ raise _ProbeRefused(STATUS_DECLARED, f"not probed: {label} is a cloud metadata endpoint (never probed)")+ if name_allowed or allowlist.allows(endpoint):+ return True+ raise _ProbeRefused(+ STATUS_DECLARED,+ f"not probed: {label} is a {endpoint.reason} address; allowlist it with mcp.allowed_private_hosts to probe it",+ )+++def _resolve_within(+ resolver: Callable[[str, int], Iterable[str]], host: str, port: int, timeout: float | None+) -> list[str]:+ """Run ``resolver`` in a daemon worker thread and wait at most ``timeout`` seconds.++ ``getaddrinfo`` cannot be interrupted, so a black-holed name would otherwise+ block for the OS resolver timeout. On timeout the lookup is abandoned (the+ daemon thread never holds up interpreter exit) and ``TimeoutError`` is raised.+ """+ if timeout is None:+ return list(resolver(host, port))+ box: dict[str, Any] = {}+ done = threading.Event()++ def run() -> None:+ try:+ box["addresses"] = list(resolver(host, port))+ except BaseException as exc: # handed to the caller below+ box["error"] = exc+ finally:+ done.set()++ threading.Thread(target=run, name="skilleval-mcp-probe-dns", daemon=True).start()+ if not done.wait(max(timeout, 0.0)):+ raise TimeoutError(f"DNS resolution timed out after {timeout:g}s")+ if "error" in box:+ raise box["error"]+ return box["addresses"]+++def _check_endpoint(+ url: str,+ transport: str,+ allowed_private_hosts: HostAllowlist | Iterable[str],+ resolver: Callable[[str, int], Iterable[str]],+ resolve_timeout: float | None = None,+) -> tuple[str, tuple[str, ...]]:+ """Apply the transport and endpoint policy; return ``(transport kind, checked addresses)``.++ A host name is judged by the addresses it resolves to, unless the name itself+ is allowlisted (exact or ``*.suffix``), which accepts its private addresses.+ Plaintext ``http`` is probed only when every address is an allowed private one.+ The probe connects only to the returned addresses and never resolves the host+ name again, so a name that later resolves to a private or metadata address+ (DNS rebinding) cannot redirect the probe or its headers.+ """+ try:+ parts = urlsplit(url)+ host = parts.hostname or ""+ port = parts.port+ except ValueError as exc:+ raise _ProbeRefused(STATUS_UNSUPPORTED, f"malformed URL: {exc}") from exc+ scheme = (parts.scheme or "").lower()+ kind_raw = (transport or "").strip().lower()+ if kind_raw in _SSE_TRANSPORTS:+ kind = "sse"+ elif kind_raw in _STREAMABLE_TRANSPORTS:+ kind = "streamable-http"+ else:+ raise _ProbeRefused(STATUS_UNSUPPORTED, f"transport {kind_raw!r} is not probed")+ if scheme in {"ws", "wss"}:+ raise _ProbeRefused(STATUS_UNSUPPORTED, "WebSocket MCP transport is not probed")+ if scheme not in {"https", "http"} or not host:+ raise _ProbeRefused(STATUS_UNSUPPORTED, f"scheme {scheme or '(none)'!r} is not probed")+ try:+ ipaddress.ip_address(host.strip("[]"))+ literal = True+ except ValueError:+ literal = False+ allowlist = HostAllowlist.of(allowed_private_hosts)+ label = f"host {host!r}"+ if literal:+ checked = [(host.strip("[]"), label)]+ name_allowed = False+ else:+ endpoint = classify_endpoint_host(host)+ if endpoint is not None and endpoint.kind == "metadata":+ raise _ProbeRefused(STATUS_DECLARED, f"not probed: {label} is a cloud metadata endpoint (never probed)")+ name_allowed = host_name_is_allowlisted(host, allowlist)+ try:+ addresses = _resolve_within(resolver, host, port or (443 if scheme == "https" else 80), resolve_timeout)+ addresses = addresses[:MAX_RESOLVED_ADDRESSES]+ except TimeoutError as exc:+ raise _ProbeRefused(STATUS_UNREACHABLE, str(exc) or "DNS resolution timed out") from exc+ except OSError as exc:+ raise _ProbeRefused(STATUS_UNREACHABLE, f"DNS resolution failed: {type(exc).__name__}") from exc+ if not addresses:+ raise _ProbeRefused(STATUS_UNREACHABLE, "DNS resolution returned no addresses")+ checked = [(str(address), f"{label} (resolved address {address})") for address in addresses]+ private = [+ _address_policy(address, allowlist, name_allowed=name_allowed, label=where) for address, where in checked+ ]+ if scheme == "http" and not all(private):+ raise _ProbeRefused(STATUS_DECLARED, "not probed: plaintext http is probed only for allowlisted private hosts")+ return kind, tuple(address for address, _where in checked)+++def _default_resolver(host: str, port: int) -> list[str]:+ infos = socket.getaddrinfo(host, port, type=socket.SOCK_STREAM)+ return sorted({str(info[4][0]) for info in infos})+++def _capped_client(+ headers: dict[str, str],+ *,+ url: str,+ addresses: tuple[str, ...],+ max_bytes: int,+ connect_timeout: float,+ read_timeout: float,+ on_failure: Callable[[str], None] | None = None,+):+ """An httpx client that never follows redirects, ignores proxy/.netrc settings, and caps bodies.++ Its TCP connections go only to ``addresses`` (the endpoint policy's checked+ addresses) and only for ``url``'s host and port. TLS still verifies the+ certificate against the host name, and requests keep their ``Host`` header.+ Responses must use identity encoding so HTTPX cannot inflate a compressed+ body above the byte cap. An unsafe response raises, and ``on_failure`` is+ told first: the ``mcp`` client catches transport errors and would otherwise+ wait for a reply until the total timeout.+ """+ import httpcore+ import httpx++ target = httpx.URL(url)+ pinned_host = target.raw_host.decode("ascii").lower()+ pinned_port = target.port or (443 if target.scheme == "https" else 80)++ class _PinnedBackend(httpcore.AsyncNetworkBackend):+ """Connect only to the checked addresses; the host name is never resolved again."""++ def __init__(self) -> None:+ self._inner = httpcore.AnyIOBackend()++ async def connect_tcp(+ self,+ host: str,+ port: int,+ timeout: float | None = None,+ local_address: str | None = None,+ socket_options: Any = None,+ ) -> Any:+ if host.lower() != pinned_host or port != pinned_port:+ raise httpcore.ConnectError(f"refused: {host}:{port} is not the policy-checked endpoint")+ failure: Exception = httpcore.ConnectError("no policy-checked address")+ for address in addresses:+ try:+ return await self._inner.connect_tcp(+ address, port, timeout=timeout, local_address=local_address, socket_options=socket_options+ )+ except (httpcore.ConnectError, httpcore.ConnectTimeout) as exc:+ failure = exc+ raise failure++ async def sleep(self, seconds: float) -> None:+ await self._inner.sleep(seconds)++ class _ResponseRefused(httpx.HTTPError):+ pass++ class _CappedStream(httpx.AsyncByteStream):+ def __init__(self, inner: Any) -> None:+ self._inner = inner++ async def __aiter__(self):+ total = 0+ async for chunk in self._inner:+ total += len(chunk)+ if total > max_bytes:+ message = f"response exceeded {max_bytes} bytes"+ if on_failure is not None:+ on_failure(message)+ raise _ResponseRefused(message)+ yield chunk++ async def aclose(self) -> None:+ await self._inner.aclose()++ class _CappedTransport(httpx.AsyncBaseTransport):+ def __init__(self) -> None:+ self._inner = httpx.AsyncHTTPTransport(retries=0)+ # httpx exposes no network-backend hook: swap its connection pool for+ # a pinned one, and refuse to run (fail closed) if the layout changed.+ if not isinstance(getattr(self._inner, "_pool", None), httpcore.AsyncConnectionPool):+ raise RuntimeError("cannot pin the probe connection with this httpx version")+ self._inner._pool = httpcore.AsyncConnectionPool(+ ssl_context=httpx.create_ssl_context(),+ retries=0,+ network_backend=_PinnedBackend(),+ )++ async def handle_async_request(self, request: httpx.Request) -> httpx.Response:+ request.headers["Accept-Encoding"] = "identity"+ response = await self._inner.handle_async_request(request)+ encoding = response.headers.get("content-encoding", "identity").strip().casefold()+ if encoding != "identity":+ # Inspect headers before exposing the stream to HTTPX's decoder.+ await response.aclose()+ message = "response content encoding must be identity"+ if on_failure is not None:+ on_failure(message)+ raise _ResponseRefused(message)+ return httpx.Response(+ status_code=response.status_code,+ headers=response.headers,+ stream=_CappedStream(response.stream),+ extensions=response.extensions,+ )++ async def aclose(self) -> None:+ await self._inner.aclose()++ client_headers = httpx.Headers(headers)+ client_headers["Accept-Encoding"] = "identity"+ return httpx.AsyncClient(+ headers=client_headers,+ timeout=httpx.Timeout(read_timeout, connect=connect_timeout),+ follow_redirects=False,+ trust_env=False,+ transport=_CappedTransport(),+ )+++def _root_cause(exc: BaseException) -> BaseException:+ while isinstance(exc, BaseExceptionGroup) and exc.exceptions:+ exc = exc.exceptions[0]+ return exc+++class _ProbeAbort:+ """The first fatal error of one probe; recording it cancels the probe's scope."""++ def __init__(self) -> None:+ self.reason = ""+ self.scope: Any = None++ def fail(self, reason: str) -> None:+ if not self.reason:+ self.reason = reason or "transport error"+ if self.scope is not None:+ self.scope.cancel()+++class _McpErrorCapture(logging.Handler):+ """Turns an error the ``mcp`` client logs (and then swallows) on the probe's thread into a probe failure."""++ def __init__(self, abort: _ProbeAbort) -> None:+ super().__init__(logging.ERROR)+ self._abort = abort+ self._thread = threading.get_ident()++ def emit(self, record: logging.LogRecord) -> None:+ if record.thread != self._thread:+ return+ exc = record.exc_info[1] if record.exc_info else None+ self._abort.fail(f"{type(exc).__name__}: {exc}" if exc is not None else record.getMessage())+++_MCP_LOG_LOCK = threading.Lock()+_MCP_LOG_STATES: dict[logging.Logger, tuple[int, bool, tuple[logging.Handler, ...], int]] = {}+++@contextmanager+def _quiet_mcp_logs(abort: _ProbeAbort) -> Iterator[None]:+ """Keep ``mcp`` log records (tracebacks included) off the console during a probe; errors fail it."""+ handler = _McpErrorCapture(abort)+ loggers = tuple(logging.getLogger(name) for name in _MCP_LOGGER_NAMES)+ with _MCP_LOG_LOCK:+ for logger in loggers:+ if logger not in _MCP_LOG_STATES:+ _MCP_LOG_STATES[logger] = (logger.level, logger.propagate, tuple(logger.handlers), 0)+ for existing in tuple(logger.handlers):+ logger.removeHandler(existing)+ logger.setLevel(logging.ERROR)+ logger.propagate = False+ level, propagate, handlers, count = _MCP_LOG_STATES[logger]+ _MCP_LOG_STATES[logger] = level, propagate, handlers, count + 1+ logger.addHandler(handler)+ try:+ yield+ finally:+ with _MCP_LOG_LOCK:+ for logger in loggers:+ logger.removeHandler(handler)+ level, propagate, handlers, count = _MCP_LOG_STATES[logger]+ if count > 1:+ _MCP_LOG_STATES[logger] = level, propagate, handlers, count - 1+ else:+ del _MCP_LOG_STATES[logger]+ for existing in handlers:+ logger.addHandler(existing)+ logger.setLevel(level)+ logger.propagate = propagate+++async def _probe_session(+ url: str, kind: str, headers: dict[str, str], *, addresses: tuple[str, ...], total_timeout: float+) -> tuple[list[str], dict[str, dict[str, Any]]]:+ import anyio+ from mcp import ClientSession+ from mcp.client.sse import sse_client+ from mcp.client.streamable_http import streamable_http_client++ abort = _ProbeAbort()++ def factory(headers=None, timeout=None, auth=None):+ return _capped_client(+ dict(headers or {}),+ url=url,+ addresses=addresses,+ max_bytes=MAX_RESPONSE_BYTES,+ connect_timeout=CONNECT_TIMEOUT_S,+ read_timeout=READ_TIMEOUT_S,+ on_failure=abort.fail,+ )++ listed: Any = None+ with _quiet_mcp_logs(abort), anyio.fail_after(total_timeout):+ try:+ with anyio.CancelScope() as scope:+ abort.scope = scope+ if kind == "sse":+ context = sse_client(+ url,+ headers=headers,+ timeout=READ_TIMEOUT_S,+ sse_read_timeout=READ_TIMEOUT_S,+ httpx_client_factory=factory,+ )+ async with context as (read, write), ClientSession(read, write) as session:+ await session.initialize()+ listed = await session.list_tools()+ else:+ async with (+ factory(headers) as client,+ streamable_http_client(url, http_client=client) as (read, write, _session_id),+ ClientSession(read, write) as session,+ ):+ await session.initialize()+ listed = await session.list_tools()+ except BaseException as exc:+ if abort.reason and not isinstance(exc, KeyboardInterrupt | SystemExit):+ raise _ProbeFailed(abort.reason) from None+ raise+ if abort.reason and (scope.cancelled_caught or listed is None):+ raise _ProbeFailed(abort.reason)+ names: list[str] = []+ schemas: dict[str, dict[str, Any]] = {}+ for tool in getattr(listed, "tools", None) or []:+ name = getattr(tool, "name", None)+ if isinstance(name, str) and name and len(names) < MAX_PROBE_TOOLS:+ safe = _detail(name)[:MAX_TOOL_NAME_CHARS]+ names.append(safe)+ schema = _input_schema(getattr(tool, "inputSchema", None))+ if schema is not None and safe == name:+ schemas[name] = schema+ return names, schemas+++def probe_mcp_server(+ target: Mapping[str, Any],+ *,+ allowed_private_hosts: HostAllowlist | Iterable[str] = (),+ expand_env: Iterable[str | EnvGrant] = (),+ environ: Mapping[str, str] | None = None,+ resolver: Callable[[str, int], Iterable[str]] | None = None,+ total_timeout: float = TOTAL_TIMEOUT_S,+) -> dict[str, Any]:+ """Probe one URL MCP server; return ``{"status", "tools", "detail"}``.++ ``expand_env`` holds the private environment grants: ``NAME`` (any+ server), ``NAME=HOST`` (servers whose URL host is HOST) or ``NAME@SERVER``+ (that server). Only names opted in for this server are read from+ ``environ`` (the process environment when ``None``) into its declared+ headers, and the detail records which names were sent to which host.+ ``total_timeout`` bounds the whole probe, DNS lookup included.+ """+ url = str(target.get("url") or "")+ started = time.monotonic()+ try:+ kind, addresses = _check_endpoint(+ url,+ str(target.get("transport") or ""),+ allowed_private_hosts,+ resolver or _default_resolver,+ resolve_timeout=total_timeout,+ )+ except _ProbeRefused as refused:+ return _entry(refused.status, refused.detail)+ raw_headers = target.get("headers")+ headers, missing, withheld, sent = _resolve_headers(+ raw_headers if isinstance(raw_headers, Mapping) else {},+ os.environ if environ is None else environ,+ _allowed_env(target, [parse_env_grant(value) for value in expand_env]),+ )+ note = _header_note(missing, withheld, sent, _target_host(url))+ try:+ import anyio+ import httpx # noqa: F401 -- availability check+ import mcp # noqa: F401 -- availability check+ except ImportError:+ return _entry(+ STATUS_DECLARED, "not probed: the mcp client library is not installed (install skillevaluator[tier3])"+ )+ remaining = total_timeout - (time.monotonic() - started)+ if remaining <= 0:+ return _entry(STATUS_UNREACHABLE, f"{kind} probe failed: timed out after {total_timeout:g}s")+ try:+ tools, schemas = anyio.run(+ lambda: _probe_session(url, kind, headers, addresses=addresses, total_timeout=remaining)+ )+ except BaseException as exc:+ if isinstance(exc, KeyboardInterrupt | SystemExit):+ raise+ cause = _root_cause(exc)+ message = str(cause) or type(cause).__name__+ if isinstance(cause, _ProbeFailed):+ return _entry(STATUS_UNREACHABLE, f"{kind} probe failed: {message}{note}")+ if isinstance(cause, TimeoutError):+ message = f"timed out after {total_timeout:g}s"+ return _entry(STATUS_UNREACHABLE, f"{kind} probe failed: {type(cause).__name__}: {message}{note}")+ entry = _entry(STATUS_REACHABLE_HOST, f"initialize and tools/list succeeded over {kind}{note}", tools)+ if schemas:+ entry["input_schemas"] = schemas+ return entry+++def probe_mcp_servers(+ targets: Iterable[Mapping[str, Any]],+ *,+ allowed_private_hosts: HostAllowlist | Iterable[str] = (),+ expand_env: Iterable[str | EnvGrant] = (),+ environ: Mapping[str, str] | None = None,+ resolver: Callable[[str, int], Iterable[str]] | None = None,+ total_timeout: float = TOTAL_TIMEOUT_S,+) -> dict[str, Any]:+ """Probe every author-supplied URL MCP server (bounded to 32 servers)."""+ allowed = HostAllowlist.of(allowed_private_hosts)+ opted_in = tuple(parse_env_grant(value) for value in expand_env)+ proof: dict[str, Any] = {}+ for target in islice(targets, MAX_PROBE_SERVERS):+ name = str(target.get("name") or "").strip()+ if not name:+ continue+ proof[name] = probe_mcp_server(+ target,+ allowed_private_hosts=allowed,+ expand_env=opted_in,+ environ=environ,+ resolver=resolver,+ total_timeout=total_timeout,+ )+ return proof+++def _with_plugin_mcp_servers(+ engine_result: Mapping[str, Any] | None, declared: Iterable[str]+) -> dict[str, dict[str, int]]:+ """Per-declared-server call counts summed over every agent's with-plugin arm.++ Each ``by_server`` key maps to at most one declared server through the+ plugin-signals name rules (exact name first; a harness spelling only when it+ names exactly one declared server). Keys that name no declared server are+ dropped, so one server's calls never credit another.+ """+ totals: dict[str, dict[str, int]] = {}+ agents = engine_result.get("agents") if isinstance(engine_result, Mapping) else None+ if not isinstance(agents, Mapping):+ return totals+ # Index the declared servers once for every agent's lookups.+ match_server = declared_mcp_server_matcher(str(name) for name in declared)+ for agent in agents.values():+ summaries = agent.get("plugin_signals_summary") if isinstance(agent, Mapping) else None+ arm = summaries.get("with_skill") if isinstance(summaries, Mapping) else None+ mcp_calls = arm.get("mcp_calls") if isinstance(arm, Mapping) else None+ by_server = mcp_calls.get("by_server") if isinstance(mcp_calls, Mapping) else None+ if not isinstance(by_server, Mapping):+ continue+ for server, counts in by_server.items():+ if not isinstance(counts, Mapping):+ continue+ target = match_server(str(server))+ if target is None:+ continue+ bucket = totals.setdefault(target, dict.fromkeys(_COUNT_KEYS, 0))+ for key in bucket:+ value = counts.get(key)+ if isinstance(value, int) and not isinstance(value, bool):+ bucket[key] += value+ return totals+++#: Key of the with-plugin arm's per-server load tally (``{server: {"loaded", "trials"}}``) in its+#: ``plugin_signals_summary``, written by the private observer from each trial's confirmed load census.+MCP_LOAD_KEY = "mcp_load"+++def _census_load(+ engine_result: Mapping[str, Any] | None,+) -> tuple[set[str], dict[str, str], dict[str, tuple[int, int, int]]]:+ """What the load census says about each MCP server, over every agent.++ Returns ``(servers some agent's harness loaded in every trial, server ->+ why an agent did not load it in some trial, server -> (trials that loaded+ it, trials whose census said either way, trials with unknown load state))``. The census summary+ keeps a component's weakest trial, so one flaky trial puts a server in+ ``not_loaded``; the per-trial counts (the with-plugin arm's ``mcp_load``+ tally) tell a flaky load from a server that never loaded.+ """+ loaded: set[str] = set()+ not_loaded: dict[str, str] = {}+ trials: dict[str, tuple[int, int, int]] = {}+ agents = engine_result.get("agents") if isinstance(engine_result, Mapping) else None+ for agent_name, agent in agents.items() if isinstance(agents, Mapping) else ():+ if not isinstance(agent, Mapping):+ continue+ census = agent.get("plugin_load_census")+ if isinstance(census, Mapping):+ for entry in census.get("loaded") or ():+ if isinstance(entry, Mapping) and entry.get("type") == "mcp" and isinstance(entry.get("name"), str):+ loaded.add(entry["name"])+ for entry in census.get("not_loaded") or ():+ if isinstance(entry, Mapping) and entry.get("type") == "mcp" and isinstance(entry.get("name"), str):+ reason = str(entry.get("reason") or "not loaded")+ not_loaded.setdefault(entry["name"], f"{agent_name}: {reason}")+ summaries = agent.get("plugin_signals_summary")+ arm = summaries.get("with_skill") if isinstance(summaries, Mapping) else None+ tally = arm.get(MCP_LOAD_KEY) if isinstance(arm, Mapping) else None+ for server, counts in tally.items() if isinstance(tally, Mapping) else ():+ if not isinstance(server, str) or not isinstance(counts, Mapping):+ continue+ # Historical tallies omitted unknown and described only known load states.+ found, seen, unknown = counts.get("loaded"), counts.get("trials"), counts.get("unknown", 0)+ if not all(isinstance(value, int) and not isinstance(value, bool) for value in (found, seen, unknown)):+ continue+ if 0 <= found <= seen and unknown >= 0:+ before = trials.get(server, (0, 0, 0))+ trials[server] = (before[0] + found, before[1] + seen, before[2] + unknown)+ return loaded, not_loaded, trials+++def _calls_detail(counts: Mapping[str, int]) -> str:+ return (+ f"agent made {counts['total']} call(s): {counts['succeeded']} succeeded, {counts['failed']} failed, "+ f"{counts['unknown']} unknown, in the with-plugin arm"+ )+++_HOST_DETAIL_MARK = "; host probe"+++def _with_host_detail(agent_detail: str, entry: Mapping[str, Any]) -> str:+ """``<agent detail>; host probe: <pre-run detail>`` (the pre-run part kept once on a re-run)."""+ host = str(entry.get("detail") or "")+ if _HOST_DETAIL_MARK in host:+ host = host.split(_HOST_DETAIL_MARK, 1)[1]+ host = host[2:] if host.startswith(": ") else f"host probe{host}"+ if not host:+ return _detail(agent_detail)+ separator = "; " if host.startswith("host probe") else "; host probe: "+ return _detail(f"{agent_detail}{separator}{host}")+++def apply_in_agent_mcp_proof(mcp_proof: Mapping[str, Any], engine_result: Mapping[str, Any] | None) -> dict[str, Any]:+ """Update each server's status from the with-plugin arm's MCP calls and load census.++ * At least one successful call: ``used-successfully``, whatever the host probe said.+ * Otherwise, when an agent's load census says the harness did not load the+ server (its init reported it failed, or never listed it), no agent's+ census confirmed it, and no scored trial loaded or listed it:+ ``not-loaded-in-agent``. A server that loaded in some trials keeps its+ status (or becomes ``called-no-success``) with a "not loaded in N of M+ trials" note.+ * Otherwise, when the agent called it but nothing succeeded:+ ``called-no-success``.++ Only ``declared`` and ``reachable-host`` (and an earlier in-agent status)+ are replaced without a successful call; ``unreachable`` and ``unsupported``+ stay, because a failed call proves no more than the host probe did.+ """+ calls = _with_plugin_mcp_servers(engine_result, mcp_proof)+ loaded, not_loaded, load_trials = _census_load(engine_result)+ upgraded: dict[str, Any] = {}+ for name, raw in mcp_proof.items():+ entry = dict(raw) if isinstance(raw, Mapping) else _entry(STATUS_DECLARED, "")+ server = str(name)+ counts = calls.get(server) or dict.fromkeys(_COUNT_KEYS, 0)+ status = entry.get("status")+ found, seen, unknown = load_trials.get(server, (0, 0, 0))+ missed_some = status in _AGENT_REPLACEABLE and server in not_loaded and server not in loaded+ if counts["succeeded"] > 0:+ if status != STATUS_USED_SUCCESSFULLY:+ entry["status"] = STATUS_USED_SUCCESSFULLY+ entry["detail"] = _with_host_detail(_calls_detail(counts), entry)+ elif missed_some and found == 0 and unknown == 0:+ agent_detail = f"not loaded in the with-plugin arm ({not_loaded[server]})"+ if counts["total"]:+ agent_detail = f"{agent_detail}; {_calls_detail(counts)}"+ entry["status"] = STATUS_NOT_LOADED_IN_AGENT+ entry["detail"] = _with_host_detail(agent_detail, entry)+ elif missed_some:+ # Loaded in some trials: a flaky load, not a server the agent never had.+ missed = seen - found+ note = (+ f"not loaded in {missed} of {seen} with-plugin trial(s) ({not_loaded[server]})"+ if missed+ else f"not loaded in a with-plugin trial that was not scored ({not_loaded[server]})"+ )+ if unknown:+ note += f"; {unknown} with-plugin trial(s) have unknown load state"+ if counts["total"] > 0:+ entry["status"] = STATUS_CALLED_NO_SUCCESS+ note = f"{_calls_detail(counts)}; {note}"+ entry["detail"] = _with_host_detail(note, entry)+ elif status in _AGENT_REPLACEABLE and counts["total"] > 0:+ entry["status"] = STATUS_CALLED_NO_SUCCESS+ entry["detail"] = _with_host_detail(_calls_detail(counts), entry)+ upgraded[server] = entry+ return upgraded+++def _compact_json(value: Any) -> str:+ """Return compact ASCII JSON for the shared schema budget (one char per byte)."""+ return json.dumps(value, sort_keys=True, separators=(",", ":"))+++def mcp_input_schemas(+ mcp_proof: Mapping[str, Any] | None, *, budget: int = MAX_INPUT_SCHEMAS_CHARS+) -> dict[str, dict[str, dict[str, Any]]]:+ """``{server: {tool: schema}}`` from a proof's host probes, at most ``budget`` bytes of compact JSON in all.++ Each kept tool is counted as written: its schema, its quoted name and the+ separators, plus each kept server's quoted name and braces.+ """+ schemas: dict[str, dict[str, dict[str, Any]]] = {}+ budget = min(budget, MAX_INPUT_SCHEMAS_CHARS) - 2+ for server, entry in islice((mcp_proof or {}).items(), MAX_PROBE_SERVERS):+ tools = entry.get("input_schemas") if isinstance(entry, Mapping) else None+ if not isinstance(server, str) or not isinstance(tools, Mapping):+ continue+ for tool, raw in islice(tools.items(), MAX_PROBE_TOOLS):+ schema = _input_schema(raw) if isinstance(tool, str) and tool else None+ if schema is None:+ continue+ size = len(_compact_json(tool)) + len(_compact_json(schema)) + 2+ if server not in schemas:+ size += len(_compact_json(server)) + 4+ if size > budget:+ return schemas+ budget -= size+ schemas.setdefault(server, {})[tool] = schema+ return schemas+++__all__ = [+ "MCP_LOAD_KEY",+ "MCP_PROOF_STATUSES",+ "NOT_REQUESTED_DETAIL",+ "PROVEN_STATUSES",+ "EnvGrant",+ "apply_in_agent_mcp_proof",+ "declared_mcp_proof",+ "mcp_input_schemas",+ "parse_env_grant",+ "planned_env_sends",+ "probe_mcp_server",+ "probe_mcp_servers",+]diff --git a/src/skillevaluator/tier3/plugin_comparisons.py b/src/skillevaluator/tier3/plugin_comparisons.pynew file mode 100644index 00000000..f94e12a7--- /dev/null+++ b/src/skillevaluator/tier3/plugin_comparisons.py@@ -0,0 +1,490 @@+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+# SPDX-License-Identifier: Apache-2.0++"""Private comparison planning over the existing runtime and its cached evidence."""++from __future__ import annotations++from collections.abc import Callable, Mapping, Sequence+from dataclasses import dataclass+from pathlib import Path+from typing import Any++from skillevaluator.tier3.harbor import collector+from skillevaluator.tier3.harbor.loose_parts_arm import (+ LOAD_EVIDENCE_FILENAME,+ LoosePartsPlan,+ compact_load_evidence,+ plan_loose_parts,+)+from skillevaluator.tier3.harbor.metrics import finite_number+from skillevaluator.tier3.harbor.stats import (+ ArmObservations,+ TrialObservation,+ build_agent_statistics,+ integration_claim,+)+from skillevaluator.tier3.plugin_prepare import PreparationError, verify_source_identity+from skillevaluator.tier3.runtime_arms import ARM_LOOSE_PARTS, ARM_SUM_OF_PARTS, ARM_WITH, ARM_WITHOUT++VARIANTS = {ARM_WITH: "with", ARM_WITHOUT: "without", ARM_SUM_OF_PARTS: "sumofparts", ARM_LOOSE_PARTS: "looseparts"}+COMPARISON_FILENAME = "private_comparisons.json"+++@dataclass(frozen=True)+class ComparisonOptions:+ """Explicit private controls; public options and the ordinary two-arm path stay separate."""++ member_skills: bool = True+ loose_parts: bool = False++ def __post_init__(self) -> None:+ if type(self.member_skills) is not bool or type(self.loose_parts) is not bool:+ raise ValueError("comparison options must be boolean")+ if self.loose_parts and not self.member_skills:+ raise ValueError("loose-parts comparisons require the member-skills arm")+++def observation_usage(record: Any) -> dict[str, float]:+ """Normalize observed counters once; absent counters and actual zero remain distinct."""+ raw, first = record.usage or {}, record.first_turn_usage or {}+ aliases = {+ "prompt_tokens": ("prompt_tokens", "total_prompt_tokens"),+ "completion_tokens": ("completion_tokens", "total_completion_tokens"),+ "cached_tokens": ("cached_tokens", "total_cached_tokens"),+ "cost_usd": ("cost_usd", "total_cost_usd"),+ }+ usage = {}+ for name, candidates in aliases.items():+ for candidate in candidates:+ value = finite_number(raw.get(candidate), non_negative=True)+ if value is not None:+ usage[name] = value+ break+ excluded = getattr(record, "first_turn_excluded_reason", None)+ if first.get("first_turn_hosted_search") or excluded == "hosted_search":+ usage["first_turn_hosted_search"] = 1+ elif not excluded:+ value = finite_number(first.get("prompt_tokens", first.get("total_prompt_tokens")), non_negative=True)+ if value is not None:+ usage["first_turn_prompt_tokens"] = value+ return usage+++def arm_observations(+ records: Sequence[Any],+ *,+ expected_case_ids: Sequence[str],+ n_attempts: int,+ stop_on_pass: bool,+ pass_threshold: float,+ job_failure: str = "",+ recovered_attempts: Sequence[dict[str, Any]] = (),+) -> ArmObservations:+ """Use the collector's existing score and attempt rules on cached logical records."""+ expected = set(expected_case_ids)+ rewards, scored, unscored = [], [], []+ attempts: dict[str, list[tuple[int | None, float | None]]] = {case: [] for case in expected_case_ids}+ identities: set[str] = set()+ complete = bool(expected)+ for record in records:+ case = record.case_id if record.case_id in expected else ""+ reward = dict(record.reward) if isinstance(record.reward, Mapping) else {}+ reward["entry_id"] = case+ identity = getattr(record, "trial_id", None)+ if identity:+ if identity in identities:+ complete = False+ identities.add(identity)+ reward.setdefault("_trial_root_name", identity)+ usage = observation_usage(record)+ if not case:+ usage.pop("first_turn_prompt_tokens", None)+ score = collector._overall_score(reward) if case else None+ if case:+ ordinal = collector._attempt_ordinal(reward) or collector._positive_attempt_ordinal(+ getattr(record, "attempt_ordinal", None)+ )+ attempts[case].append((ordinal, score))+ else:+ complete = False+ observation = TrialObservation(+ case,+ reward,+ usage,+ score if reward.get("_logical_mixed_contracts") else None,+ )+ if score is None:+ unscored.append(observation)+ else:+ rewards.append(reward)+ scored.append(observation)+ common = {+ "expected_case_ids": list(expected_case_ids),+ "n_attempts": n_attempts,+ "stop_on_pass": stop_on_pass,+ "pass_threshold": pass_threshold,+ "recovered_attempts": list(recovered_attempts),+ }+ summary = collector._pass_summary(rewards, expected_cases=len(expected_case_ids), **common)+ execution = collector._condition_execution_summary(+ rewards,+ expected_cases=len(expected_case_ids),+ job_failure=job_failure,+ **common,+ )+ for case_attempts in attempts.values():+ ordinals = [ordinal for ordinal, _score in case_attempts]+ explicit = [ordinal for ordinal in ordinals if ordinal is not None]+ if explicit and len(explicit) != len(ordinals):+ complete = False+ ordered = sorted(case_attempts, key=lambda item: item[0] or 0) if explicit else case_attempts+ required = n_attempts+ if stop_on_pass:+ first_pass = next(+ (+ (index, ordinal)+ for index, (ordinal, score) in enumerate(ordered, 1)+ if score is not None and score >= pass_threshold+ ),+ None,+ )+ if first_pass:+ required = first_pass[1] or first_pass[0]+ if required > n_attempts or len(case_attempts) != required:+ complete = False+ if explicit and set(explicit) != set(range(1, required + 1)):+ complete = False+ return ArmObservations(+ execution["execution_status"], tuple(scored), summary, job_failure, tuple(unscored), complete+ )+++class ComparisonRuntime:+ """Delegate native execution, adding private arms and advisory comparison artifacts."""++ def __init__(self, native: Any, package: Any, options: ComparisonOptions, agents: Sequence[str], *, baseline: bool):+ self.native, self.package, self.options = native, package, options+ self.agents, self.baseline = tuple(agents), baseline+ self.comparison_plan = self+ self.task_sets: dict[str, dict[str, tuple[Path, ...]]] = {}+ self.jobs: dict[str, dict[str, Path]] = {}+ self.launch_errors: dict[str, dict[str, str]] = {agent: {} for agent in agents}+ self.expected_case_ids: tuple[str, ...] = ()+ self.n_attempts, self.stop_on_pass, self.pass_threshold = 1, False, 0.5+ self.data: dict[str, Any] = {"status": "not_collected", "agents": {}, "errors": []}+ self.members = package.native_source.member_skills if options.member_skills else ()+ self.loose_plan = LoosePartsPlan(requested=options.loose_parts, order=self.agents)++ def begin_attempt(self) -> None:+ """Start a fresh comparison attempt alongside the native runtime's reset."""+ begin = getattr(self.native, "begin_attempt", None)+ if begin is not None:+ begin()+ self.task_sets, self.jobs = {}, {}+ self.launch_errors = {agent: {} for agent in self.agents}+ self.expected_case_ids = ()+ self.data = {"status": "not_collected", "agents": {}, "errors": []}+ self.loose_plan = LoosePartsPlan(requested=self.options.loose_parts, order=self.agents)++ def configure_agent_routes(self, paths: Mapping[str, str]) -> None:+ """Bind the coordinator-resolved routes once for this engine attempt."""+ self.task_sets = {}+ self.launch_errors = {agent: {} for agent in self.agents}+ self.loose_plan = plan_loose_parts(+ requested=self.options.loose_parts,+ agents=self.agents,+ decisions=self.native.decisions,+ source=self.package.native_source,+ agent_import_paths=paths,+ unavailable=None if self.baseline and self.members else "the ladder requires a baseline and member skills",+ )++ def with_agent_import_path(self, agent: str, base_import_path: str | None) -> str | None:+ return self.native.with_agent_import_path(agent, base_import_path)++ def baseline_agent_import_path(self, agent: str, base_import_path: str | None) -> str | None:+ return self.native.baseline_agent_import_path(agent, base_import_path)++ def stage_with_arm(self, agent: str, task_dirs: Sequence[Path]) -> None:+ self.native.stage_with_arm(agent, task_dirs)++ def observe_collected_trials(self, *args: Any, **kwargs: Any) -> None:+ observe = getattr(self.native, "observe_collected_trials", None)+ if observe is not None:+ observe(*args, **kwargs)++ def configure_run(self, **kwargs: Any) -> None:+ self.data = {"status": "not_collected", "agents": {}, "errors": []}+ self.jobs = kwargs.get("arm_jobs") or {}+ configure = getattr(self.native, "configure_run", None)+ if configure is not None:+ configure(**kwargs)++ def run_config(self) -> dict[str, Any]:+ return {"member_skills": self.options.member_skills, "loose_parts": self.loose_plan.run_config()}++ def stage_additional(+ self,+ agent: str,+ emit_control: Callable[[str, Sequence[Path]], Sequence[Path]],+ task_sets: Mapping[str, Sequence[Path]],+ base_import_path: str | None,+ ) -> tuple[tuple[str, Path, str | None], ...]:+ verify_source_identity(self.package)+ staged = {arm: tuple(paths) for arm, paths in task_sets.items()}+ if ARM_WITH not in staged:+ self.task_sets[agent] = staged+ self.loose_plan.arms.pop(agent, None)+ self.loose_plan.not_measured[agent] = "with-skill arm disabled; private comparison arms require it"+ return ()+ jobs = []+ if self.members:+ paths = tuple(emit_control(VARIANTS[ARM_SUM_OF_PARTS], self.members))+ if not paths:+ raise PreparationError("member-skills arm has no emitted tasks")+ staged[ARM_SUM_OF_PARTS] = paths+ jobs.append((VARIANTS[ARM_SUM_OF_PARTS], paths[0].parent, base_import_path))+ loose = self.loose_plan.arm_for(agent)+ if loose is not None:+ from skillevaluator.tier3.harbor.native_staging import stage_native_bundle++ paths = tuple(emit_control(VARIANTS[ARM_LOOSE_PARTS], self.members))+ if not paths:+ raise PreparationError("loose-parts arm has no emitted tasks")+ for path in paths:+ environment = path / "environment"+ dockerfile = environment / "Dockerfile"+ if dockerfile.is_symlink() or not dockerfile.is_file():+ raise PreparationError("loose-parts staging needs an owned task Dockerfile")+ lines = stage_native_bundle(+ environment, loose.staging, excluded_roots=self.package.native_source.excluded_paths+ )+ text = dockerfile.read_text(encoding="utf-8")+ dockerfile.write_text(text.rstrip("\n") + "\n" + "\n".join(lines) + "\n", encoding="utf-8")+ staged[ARM_LOOSE_PARTS] = paths+ jobs.append((VARIANTS[ARM_LOOSE_PARTS], paths[0].parent, loose.import_path))+ self.task_sets[agent] = staged+ verify_source_identity(self.package)+ return tuple(jobs)++ def bind_execution(+ self, case_ids: Sequence[str], *, n_attempts: int, stop_on_pass: bool, pass_threshold: float+ ) -> None:+ from skillevaluator.tier3.harbor.stats import MAX_COMPARISON_CASES++ if len(case_ids) > MAX_COMPARISON_CASES:+ raise ValueError("private comparison case limit exceeded")+ self.expected_case_ids = tuple(case_ids)+ self.n_attempts, self.stop_on_pass, self.pass_threshold = n_attempts, stop_on_pass, pass_threshold+ for agent, arms in self.task_sets.items():+ selected = {arm: tuple(path for path in paths if path.is_dir()) for arm, paths in arms.items()}+ self.task_sets[agent] = selected+ compared = {arm: selected.get(arm, ()) for arm in (ARM_WITH, ARM_SUM_OF_PARTS, ARM_LOOSE_PARTS)}+ self.loose_plan.hash_tasks(agent, compared)+ loose = self.loose_plan.arm_for(agent)+ if loose is not None and any(len(paths) != len(case_ids) for paths in compared.values()):+ loose.content.update(equal_across_arms=None, error="Expected emitted task coverage is incomplete")++ def _cache_init_events(self) -> None:+ """Read loose-arm startup evidence once, alongside the observer's artifact cache."""+ from skillevaluator.tier3 import plugin_native+ from skillevaluator.tier3.plugin_evidence import _trial_roots++ cache = getattr(self.native, "trial_init_events", None)+ if cache is None:+ return+ for agent in self.agents:+ if self.loose_plan.arm_for(agent) is None:+ continue+ for arm in (ARM_SUM_OF_PARTS, ARM_LOOSE_PARTS):+ job = self.jobs.get(agent, {}).get(arm)+ if job is None or not job.is_dir():+ continue+ roots, _limited = _trial_roots(job)+ for root in roots:+ if root in cache:+ continue+ text = collector._read_bounded_text(root / "agent" / "claude-code.txt", max_bytes=8 * 1024 * 1024)+ cache[root] = plugin_native.claude_init_event(text) if text else None++ def collect_evidence(self, jobs_dir: Path, result: dict[str, Any] | None) -> None:+ try:+ self._cache_init_events()+ except Exception as exc:+ self._record_error(exc)+ self.native.collect_evidence(jobs_dir, result)+ if result is None:+ return+ try:+ self._collect_statistics(result, jobs_dir)+ except Exception as exc:+ self._record_error(exc)+ self.data["status"] = "incomplete" if self.data["errors"] else "recorded"+ self._write_artifact(jobs_dir.parent / COMPARISON_FILENAME, jobs_dir.parent, self.provenance)+ result["private_comparisons"] = self.provenance()++ def _record_error(self, exc: Exception, *, artifact: str = "") -> None:+ from skillevaluator.tier3.harbor.runner import _bounded_execution_error++ context = f"{artifact}: " if artifact else ""+ self.data["errors"].append(_bounded_execution_error(f"{context}{type(exc).__name__}: {exc}"))+ self.data["status"] = "incomplete"++ def _write_artifact(self, path: Path, output: Path, payload: Callable[[], Any]) -> None:+ """An advisory artifact cannot prevent the others from being collected."""+ try:+ if path.parent != output:+ collector._assert_safe_generated_output_path(path.parent, output, follow_target=False)+ path.parent.mkdir(exist_ok=True)+ collector._write_generated_root_json(path, output, payload())+ except Exception as exc:+ self._record_error(exc, artifact=path.relative_to(output).as_posix())++ def _collect_statistics(self, result: dict[str, Any], jobs_dir: Path) -> None:+ from skillevaluator.tier3.harbor.activation import summarize_skill_activation+ from skillevaluator.tier3.loose_parts import load_failures_by_case, loose_parts_load_evidence+ from skillevaluator.tier3.plugin_evidence import _trial_roots++ observer = getattr(self.native, "observer", None)+ if observer is None:+ raise ValueError("private comparisons have no runtime evidence observer")+ for agent in self.agents:+ arm_rows = observer.logical_trials.get(agent, {})+ arms, conditions, failure_details, evidence = {}, {}, {}, {}+ for arm, job in self.jobs.get(agent, {}).items():+ records = arm_rows.get(arm, ())+ rewards = [dict(record.reward) for record in records if isinstance(record.reward, Mapping)]+ recovered = (+ collector._recovered_timeout_attempts(job, rewards, n_attempts=self.n_attempts)+ if self.stop_on_pass and job.is_dir()+ else []+ )+ ok, reason = collector.validate_harbor_job_result(+ job / "result.json",+ expected_trials=None if self.stop_on_pass else len(self.expected_case_ids) * self.n_attempts,+ )+ trial_failures = collector._extract_trial_failures(job) if recovered else []+ job_recovered = collector._recovered_job_failure(job, trial_failures, recovered) if recovered else False+ failure = self.launch_errors[agent].get(VARIANTS[arm], "") or (+ "" if ok or job_recovered else collector._published_job_failure(reason)+ )+ observed = arm_observations(+ records,+ expected_case_ids=self.expected_case_ids,+ n_attempts=self.n_attempts,+ stop_on_pass=self.stop_on_pass,+ pass_threshold=self.pass_threshold,+ job_failure=failure,+ recovered_attempts=recovered,+ )+ arms[arm] = observed+ conditions[arm] = collector._condition_execution_summary(+ [dict(trial.reward) for trial in observed.trials],+ expected_case_ids=list(self.expected_case_ids),+ expected_cases=len(self.expected_case_ids),+ n_attempts=self.n_attempts,+ stop_on_pass=self.stop_on_pass,+ pass_threshold=self.pass_threshold,+ job_failure=failure,+ recovered_attempts=recovered,+ )+ if arm in {ARM_SUM_OF_PARTS, ARM_LOOSE_PARTS}:+ roots, limited = _trial_roots(job) if job.is_dir() else ([], False)+ recovered_names = {row["trial"] for row in recovered}+ failures = [+ {"trial": collector._published_trial_label(root.name), "reason": reason}+ for root in roots+ if collector._published_trial_label(root.name) not in recovered_names+ if (reason := collector._trial_failure_reason(root))+ ]+ metadata = collector._failure_list_metadata("details", failures)+ if limited:+ metadata["details_truncated"] = True+ failure_details[arm] = (collector._public_failure_list(failures), metadata)+ loose = self.loose_plan.arm_for(agent)+ if loose is not None and arm in {ARM_WITH, ARM_SUM_OF_PARTS, ARM_LOOSE_PARTS}:+ by_case = evidence.setdefault(arm, {})+ for record in records:+ if record.case_id not in self.expected_case_ids:+ continue+ signals = record.signals or {}+ init = getattr(record, "init_event", None)+ row = loose_parts_load_evidence(+ [init] if init else [],+ None,+ loose.parts,+ arm=arm,+ mcp_calls=(signals.get("mcp_calls") or {}).get("by_server", {}),+ )+ by_case.setdefault(record.case_id, []).append(row)+ failures = load_failures_by_case(evidence)+ cross_cases = {+ case for case in self.expected_case_ids if observer.context.case_spec(case).get("cross_component")+ }+ stats = build_agent_statistics(+ arms,+ expected_case_ids=self.expected_case_ids,+ n_attempts=self.n_attempts,+ stop_on_pass=self.stop_on_pass,+ pass_threshold=self.pass_threshold,+ sum_of_parts_requested=bool(self.members),+ cross_component_case_ids=cross_cases,+ load_failures_by_case=failures,+ never_activated_arms={+ arm+ for arm, observed in arms.items()+ if summarize_skill_activation(trial.reward for trial in observed.trials)["status"]+ == "never_activated"+ },+ )+ loose = self.loose_plan.arm_for(agent)+ if loose is not None:+ decision = self.native.decisions.get(agent)+ stats["integration_claim"] = integration_claim(+ stats["lift_ladder"],+ loose.content,+ stats["integration_completeness"],+ agent=agent,+ full_arm_load="native" if decision and decision.native else "wrapper",+ )+ info = result.get("agents", {}).get(agent)+ if isinstance(info, dict):+ info.setdefault("conditions", {}).update(+ {arm: state for arm, state in conditions.items() if arm in {ARM_SUM_OF_PARTS, ARM_LOOSE_PARTS}}+ )+ for arm, (details, metadata) in failure_details.items():+ info.setdefault("trial_failures", {})[arm] = details+ info.setdefault("failure_detail_metadata", {})[f"{arm}_trials"] = metadata+ info.update(stats)+ self.data["agents"][agent] = {+ **stats,+ "conditions": conditions,+ "loose_parts": loose.info(evidence)+ if loose is not None+ else {+ "measured": False,+ "not_measured_reason": self.loose_plan.not_measured.get(agent)+ or (+ "not requested" if not self.options.loose_parts else "No supported loose-parts arm was prepared"+ ),+ },+ "launch_errors": self.launch_errors[agent],+ }+ output = jobs_dir.parent+ directory = output / agent+ self._write_artifact(directory / "statistics.json", output, lambda stats=stats: stats)+ if loose is not None:+ self._write_artifact(+ directory / LOAD_EVIDENCE_FILENAME,+ output,+ lambda evidence=evidence: compact_load_evidence(evidence),+ )++ def provenance(self) -> dict[str, Any]:+ return {+ "requested": {"member_skills": self.options.member_skills, "loose_parts": self.options.loose_parts},+ "source": vars(self.package.identity),+ "expected_case_ids": list(self.expected_case_ids),+ **self.data,+ }diff --git a/src/skillevaluator/tier3/plugin_evidence.py b/src/skillevaluator/tier3/plugin_evidence.pynew file mode 100644index 00000000..a7cc2e2c--- /dev/null+++ b/src/skillevaluator/tier3/plugin_evidence.py@@ -0,0 +1,624 @@+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+# SPDX-License-Identifier: Apache-2.0+"""Private advisory evidence collected before the existing runtime releases its artifacts."""++from __future__ import annotations++import json+import os+import stat+from collections.abc import Mapping+from dataclasses import dataclass, replace+from itertools import islice+from pathlib import Path+from typing import TYPE_CHECKING, Any++from skillevaluator.tier3.eval_core.plugin_signals import (+ MCP_INPUT_SCHEMAS_KEY,+ PLUGIN_PART_ARMS,+ build_plugin_signals_context,+ compute_plugin_signals,+ summarize_plugin_signals,+ top_argument_failures,+ validate_plugin_case_fields,+)+from skillevaluator.tier3.harbor import collector+from skillevaluator.tier3.harbor.metrics import finite_number+from skillevaluator.tier3.mcp_proof import (+ MCP_LOAD_KEY,+ TOTAL_TIMEOUT_S,+ apply_in_agent_mcp_proof,+ declared_mcp_proof,+ mcp_input_schemas,+ parse_env_grant,+ probe_mcp_servers,+)+from skillevaluator.tier3.plugin_prepare import verify_source_identity+from skillevaluator.tier3.plugin_trace_artifacts import TrialTraceArtifacts+from skillevaluator.tier3.runtime_arms import ARM_LOOSE_PARTS, ARM_SUM_OF_PARTS, ARM_WITH, ARM_WITHOUT+from skillevaluator.utils.redaction import redact_printable_text+from skillevaluator.utils.secure_fs import SecureRoot, stat_is_link_or_reparse++if TYPE_CHECKING:+ from skillevaluator.tier3.plugin_execution import NativeRuntimeHooks+ from skillevaluator.tier3.plugin_prepare import PreparedRuntimePackage, WrapperSkill++RUNTIME_EVIDENCE_FILENAME = "plugin_runtime_evidence.json"+MAX_TRIALS_PER_ARM = 512+MAX_TRIAL_ENTRIES = 4096+MAX_EVIDENCE_BYTES = 2 * 1024 * 1024+ARM_SUFFIXES = {+ ARM_WITH: "with",+ ARM_WITHOUT: "without",+ ARM_SUM_OF_PARTS: "sumofparts",+ ARM_LOOSE_PARTS: "looseparts",+}+++def _safe_text(value: Any, limit: int = 300) -> str:+ return redact_printable_text(str(value or ""), max_len=limit, max_input_chars=limit * 4 + 256)+++@dataclass(frozen=True)+class EvidenceOptions:+ """Private opt-ins; ordinary calls perform no host probe or environment expansion."""++ probe_mcp: bool = False+ env_grants: tuple[str, ...] = ()+ allowed_private_hosts: tuple[str, ...] = ()+ timeout: float = TOTAL_TIMEOUT_S++ def __post_init__(self) -> None:+ if type(self.probe_mcp) is not bool:+ raise ValueError("probe_mcp must be a boolean")+ if finite_number(self.timeout) is None or not 0 < self.timeout <= TOTAL_TIMEOUT_S:+ raise ValueError(f"probe timeout must be positive and at most {TOTAL_TIMEOUT_S:g}s")+ if len(self.env_grants) > 64 or len(self.allowed_private_hosts) > 64:+ raise ValueError("private evidence policy exceeds its collection bound")+ for grant in self.env_grants:+ parse_env_grant(grant)+++@dataclass(frozen=True)+class LogicalTrialEvidence:+ """Cached logical observations for later private comparisons; raw rewards stay in memory."""++ trial_id: str+ case_id: str | None+ reward: Mapping[str, Any] | None+ signals: Mapping[str, Any] | None+ native_census: Mapping[str, Any] | None+ usage: Mapping[str, float | None] | None+ first_turn_usage: Mapping[str, float | None] | None+ evidence_status: str+ init_event: Mapping[str, Any] | None = None+ first_turn_excluded_reason: str | None = None+ attempt_ordinal: int | None = None++ def persisted(self) -> dict[str, Any]:+ return {+ "trial_id": self.trial_id,+ "case_id": self.case_id,+ "signals": self.signals,+ "evidence_status": self.evidence_status,+ }+++def _usage(raw: Any) -> dict[str, float | None] | None:+ if not isinstance(raw, Mapping):+ return None+ fields = (+ "total_prompt_tokens",+ "total_completion_tokens",+ "total_cached_tokens",+ "total_cost_usd",+ "prompt_tokens",+ "completion_tokens",+ "cached_tokens",+ "cost_usd",+ )+ kept = {key: finite_number(raw[key], non_negative=True) for key in fields if key in raw}+ return kept or None+++def _harbor_usage(result: Any) -> dict[str, float]:+ """Finite recorded counters, summing complete native steps without a root mirror."""+ if not isinstance(result, dict) or not collector._is_serialized_harbor_trial_result(result):+ return {}+ steps = result.get("step_results")+ if steps is not None:+ if collector._valid_step_result_names(steps, max_count=collector._MAX_TRAJECTORY_STEP_DIRECTORIES) is None:+ return {}+ constituents = steps+ else:+ constituents = [result]+ counters = {}+ for name, native in (+ ("prompt_tokens", "n_input_tokens"),+ ("completion_tokens", "n_output_tokens"),+ ("cached_tokens", "n_cache_tokens"),+ ("cost_usd", "cost_usd"),+ ):+ values = [+ finite_number(row.get("agent_result", {}).get(native), non_negative=True)+ if isinstance(row.get("agent_result"), Mapping)+ else None+ for row in constituents+ ]+ if all(value is not None for value in values):+ total = finite_number(sum(values), non_negative=True)+ if total is not None:+ counters[name] = total+ return counters+++def _first_usage(trajectory: Mapping[str, Any] | None) -> tuple[Mapping[str, float | None] | None, str | None]:+ steps = trajectory.get("steps") if isinstance(trajectory, Mapping) else None+ for step in steps[:10_000] if isinstance(steps, list) else ():+ if isinstance(step, Mapping) and step.get("source") == "agent" and step.get("llm_call_count") != 0:+ calls = step.get("tool_calls")+ if isinstance(calls, list) and any(+ isinstance(call, Mapping) and call.get("function_name") == "web_search_call" for call in calls[:4000]+ ):+ return None, "hosted_search"+ usage = _usage(step.get("metrics"))+ return usage, None if usage and usage.get("prompt_tokens") is not None else "missing_tokens"+ return None, "missing_tokens"+++def _trial_roots(job: Path) -> tuple[list[Path], bool]:+ """Bound directory enumeration and refuse links before opening any trial artifacts."""+ with SecureRoot(job):+ with os.scandir(job) as entries:+ found = list(islice(entries, MAX_TRIAL_ENTRIES + 1))+ limited = len(found) > MAX_TRIAL_ENTRIES+ roots = []+ for entry in sorted(found[:MAX_TRIAL_ENTRIES], key=lambda item: item.name):+ metadata = entry.stat(follow_symlinks=False)+ if stat.S_ISDIR(metadata.st_mode) and not stat_is_link_or_reparse(metadata):+ roots.append(Path(entry.path))+ return roots[:MAX_TRIALS_PER_ARM], limited or len(roots) > MAX_TRIALS_PER_ARM+++def _load_tally(records: tuple[LogicalTrialEvidence, ...]) -> dict[str, dict[str, int]]:+ found = []+ for record in records:+ census = record.native_census+ if not isinstance(census, Mapping) or census.get("fallback"):+ found.append({})+ continue+ servers: dict[str, bool | None] = {}+ for field in ("loaded", "listed", "staged", "not_loaded"):+ rows = census.get(field)+ for row in rows[:256] if isinstance(rows, list) else ():+ if isinstance(row, Mapping) and row.get("type") == "mcp" and isinstance(row.get("name"), str):+ previous = servers.get(row["name"])+ known = field == "loaded" or (field == "not_loaded" and record.init_event is not None)+ servers[row["name"]] = (+ True if field == "loaded" or previous is True else False if known else previous+ )+ found.append(servers)+ names = sorted({name for row in found for name in row})[:64]+ return {+ name: {+ "loaded": sum(row.get(name) is True for row in found),+ "trials": sum(row.get(name) is not None for row in found),+ "unknown": sum(row.get(name) is None for row in found),+ }+ for name in names+ }+++def _proof_census(records: tuple[LogicalTrialEvidence, ...]) -> dict[str, Any]:+ """Only confirmed harness states can contradict the result of a host probe."""+ tally = _load_tally(records)+ loaded = [+ {"type": "mcp", "name": name}+ for name, counts in tally.items()+ if counts["trials"] > 0 and counts["loaded"] == counts["trials"] and not counts["unknown"]+ ]+ not_loaded = {}+ for record in records:+ if record.init_event is None or not isinstance(record.native_census, Mapping):+ continue+ for row in record.native_census.get("not_loaded", ())[:256]:+ if isinstance(row, Mapping) and row.get("type") == "mcp" and isinstance(row.get("name"), str):+ not_loaded[row["name"]] = {+ "type": "mcp",+ "name": row["name"],+ "reason": "harness reported the server unavailable",+ }+ return {"loaded": loaded, "not_loaded": list(not_loaded.values())}+++class RuntimeEvidenceObserver:+ """One prepared declaration context and one artifact cache for a private run."""++ def __init__(+ self,+ package: PreparedRuntimePackage,+ wrapper: WrapperSkill,+ hooks: NativeRuntimeHooks,+ options: EvidenceOptions,+ ) -> None:+ self.package, self.wrapper, self.hooks, self.options = package, wrapper, hooks, options+ self.logical_trials: dict[str, dict[str, tuple[LogicalTrialEvidence, ...]]] = {}+ self.summaries: dict[str, dict[str, dict[str, Any]]] = {}+ self.errors: list[str] = []+ self.artifacts = TrialTraceArtifacts()+ self.arm_jobs: dict[str, dict[str, Path]] = {}+ self.selector_map: Mapping[str, str] | None = None+ self._rewards: dict[Path, tuple[list[dict[str, Any]], list[dict[str, Any]]]] = {}+ self._collected: set[Path] = set()+ self._jobs_dir: Path | None = None+ self._sidecar: Path | None = None+ verify_source_identity(package)++ entries = collector._read_json(wrapper.path / "evals" / "evals.json")+ if not isinstance(entries, list):+ raise ValueError("prepared runtime dataset is unreadable")+ for entry in entries:+ self.errors.extend(_safe_text(error) for error in validate_plugin_case_fields(entry))+ self.context = build_plugin_signals_context(+ member_skills=[Path(str(member.path)).name for member in package.skills],+ mcp_servers=[server["name"] for server in package.native_source.mcp_servers],+ wrapper_skills=[wrapper.name],+ entries=entries,+ plugin_name=package.plugin_name,+ subagents=[text.name for text in package.native_source.texts if text.type == "agent"],+ commands=[text.name for text in package.native_source.texts if text.type == "command"],+ )+ portable = {server["name"] for server in package.native_source.mcp_servers}+ self.context = replace(+ self.context,+ unstaged_mcp_servers=tuple(+ server["name"] for server in package.mcp_servers if server["name"] not in portable+ ),+ )+ targets = [+ {**server, "transport": server.get("transport") or server.get("type") or "http"}+ for server in package.mcp_servers+ if server.get("url")+ ]+ self.proof = (+ probe_mcp_servers(+ targets,+ allowed_private_hosts=options.allowed_private_hosts,+ expand_env=options.env_grants,+ total_timeout=options.timeout,+ )+ if options.probe_mcp+ else declared_mcp_proof(targets)+ )+ self.context = replace(self.context, mcp_input_schemas=mcp_input_schemas(self.proof))+ self._host_proof = self.proof+ self._declaration_errors = tuple(self.errors)+ verify_source_identity(package)++ def _reset_attempt(self) -> None:+ self.logical_trials = {}+ self.summaries = {}+ self.errors = list(self._declaration_errors)+ self.proof = self._host_proof+ self.artifacts = TrialTraceArtifacts()+ self._rewards = {}+ self._collected = set()++ def begin_attempt(self) -> None:+ """Release prior observations even when the next attempt creates no artifacts."""+ self._reset_attempt()+ self._jobs_dir = None+ self._sidecar = None+ self.selector_map = None+ self.arm_jobs = {}+ self.hooks.trial_censuses = {}+ self.hooks.trial_init_events = {}++ def _begin_attempt(self, jobs_dir: Path) -> None:+ """Scope a handoff while retaining coordinator data registered for its jobs."""+ if self._jobs_dir == jobs_dir:+ return+ self._jobs_dir = jobs_dir+ self._reset_attempt()+ self.arm_jobs = {+ agent: {arm: job for arm, job in arms.items() if job.parent == jobs_dir}+ for agent, arms in self.arm_jobs.items()+ }+ self.hooks.trial_censuses = {+ root: row for root, row in self.hooks.trial_censuses.items() if root.parent.parent == jobs_dir+ }+ self.hooks.trial_init_events = {+ root: row for root, row in self.hooks.trial_init_events.items() if root.parent.parent == jobs_dir+ }++ def configure_run(+ self,+ *,+ case_id_by_task_selector: Mapping[str, str] | None = None,+ arm_jobs: Mapping[str, Mapping[str, Path]] | None = None,+ ) -> None:+ if case_id_by_task_selector is not None:+ selected = collector._validated_case_id_by_task_selector(+ dict(case_id_by_task_selector), list(case_id_by_task_selector.values())+ )+ if not set(selected.values()).issubset(self.wrapper.case_sources):+ raise ValueError("Task selector mapping contains unknown logical case identities")+ self.selector_map = selected+ if arm_jobs is not None:+ self.arm_jobs = {agent: dict(arms) for agent, arms in arm_jobs.items()}++ def observe_collected_trials(+ self,+ agent: str,+ arm: str,+ job_dir: Path,+ collected: list[dict[str, Any]],+ scoreable: list[dict[str, Any]],+ selector_map: Mapping[str, str] | None,+ ) -> None:+ self._begin_attempt(job_dir.parent)+ self.configure_run(case_id_by_task_selector=selector_map)+ self.arm_jobs.setdefault(agent, {})[arm] = job_dir+ self._rewards[job_dir] = (collected, scoreable)++ def _case_id(self, root: Path, row: Mapping[str, Any] | None, result: Any) -> str | None:+ selector = row.get("_trusted_task_selector") if isinstance(row, Mapping) else None+ if self.selector_map is not None:+ if not selector:+ selector = collector._task_selector_from_harbor_result(result) if isinstance(result, dict) else None+ return self.selector_map.get(selector) if selector else None+ # Generated trial selectors are coordinator-owned. Reward entry_id is+ # deliberately never an identity source, even if it names a valid case.+ selector = root.name.split("__", 1)[0]+ for suffix in ("-with-skill", "-without-skill", "-sum-of-parts", "-loose-parts"):+ selector = selector.removesuffix(suffix)+ return selector if selector in self.wrapper.case_sources else None++ def _context_for_staging(self) -> None:+ extra_servers, unavailable, aliases, lsp = {}, {}, {}, {}+ for agent, decision in self.hooks.decisions.items():+ staging = self.hooks.stagings.get(agent)+ if staging is None or not decision.native:+ unavailable[agent] = ("subagent", "command", "lsp")+ continue+ declared = staging.bundle.declared+ kinds = {"subagent" if row.get("type") == "agent" else row.get("type") for row in declared}+ unavailable[agent] = tuple(kind for kind in ("subagent", "command", "lsp") if kind not in kinds)+ extra_servers[agent] = tuple(row["name"] for row in declared if row.get("type") == "mcp")+ adapter = staging.adapter+ if hasattr(adapter, "staged_name"):+ aliases.update(+ {+ adapter.staged_name(self.package.native_source, text).casefold(): text.name+ for text in self.package.native_source.texts+ if text.type == "agent"+ }+ )+ if "lsp" in kinds:+ lsp[agent] = {+ name: tuple(config.get("extensionToLanguage", {}))+ for name, config in self.package.native_source.lsp_servers.items()+ if isinstance(config, Mapping)+ }+ self.context = replace(+ self.context,+ agent_mcp_servers=extra_servers,+ agent_unstaged=unavailable,+ subagent_aliases=aliases,+ agent_lsp_servers=lsp,+ )++ def collect_arm(self, agent: str, arm: str, job: Path) -> tuple[LogicalTrialEvidence, ...]:+ """Cache one arm once; callers may register later private arms through ``arm_jobs``."""+ self._begin_attempt(job.parent)+ previous = self.logical_trials.get(agent, {}).get(arm)+ if previous is not None:+ return previous+ roots, limited = _trial_roots(job)+ cached = self._rewards.get(job)+ if cached is None:+ rewards = collector._extract_rewards(job, self.selector_map, selected_trial_roots=roots)+ scoreable, _failures = collector._partition_scoreable_rewards(rewards)+ else:+ rewards, scoreable = cached+ groups: dict[str, list[dict[str, Any]]] = {}+ raw_by_root: dict[str, Mapping[str, Any]] = {}+ for reward in rewards:+ root = collector._safe_trial_path_component(reward.get("_trial_root_name"))+ if root:+ raw_by_root.setdefault(root, reward)+ for reward in scoreable:+ root = collector._safe_trial_path_component(reward.get("_trial_root_name"))+ if root:+ groups.setdefault(root, []).append(reward)+ labels = collector._published_trial_labels_by_root([root.name for root in roots])+ records = []+ for root in roots:+ result_payload = collector._read_json(root / "result.json")+ rows = groups.get(root.name, [])+ logical = collector._logical_attempt_rewards(rows)+ reward = logical[0] if len(logical) == 1 else None+ case_id = self._case_id(root, raw_by_root.get(root.name), result_payload)+ if reward is not None:+ reward = {**reward, "entry_id": case_id, "_authoritative_entry_id": True}+ trace = self.artifacts.trace(root)+ spec = self.context.case_spec(case_id or "")+ if arm in PLUGIN_PART_ARMS and self.context.mcp_input_schemas:+ spec = {**spec, MCP_INPUT_SCHEMAS_KEY: self.context.mcp_input_schemas}+ signals = (+ compute_plugin_signals(+ trace.trajectory,+ spec,+ declared=self.context.declared_for(arm, agent),+ wrapper_skills=self.context.wrapper_skills,+ mcp_call_servers=trace.call_servers,+ subagent_aliases=self.context.aliases_for(arm),+ lsp_servers=self.context.lsp_servers_for(arm, agent),+ )+ if self.context.arm_enabled(arm)+ else None+ )+ status = (+ ("recorded" if signals.get("evidence_complete") is not False else "incomplete")+ if signals is not None+ else "missing"+ if trace.trajectory is None+ else "not_applicable"+ )+ usage = _usage((trace.trajectory or {}).get("final_metrics"))+ for name, value in _harbor_usage(result_payload).items():+ if not any((usage or {}).get(alias) is not None for alias in (name, f"total_{name}")):+ usage = {**(usage or {}), name: value}+ first_usage, excluded = _first_usage(trace.trajectory)+ if excluded == "hosted_search":+ usage = {**(usage or {}), "first_turn_hosted_search": 1.0}+ elif first_usage and first_usage.get("prompt_tokens") is not None:+ usage = {**(usage or {}), "first_turn_prompt_tokens": first_usage["prompt_tokens"]}+ records.append(+ LogicalTrialEvidence(+ labels[root.name],+ case_id,+ reward,+ signals,+ self.hooks.trial_censuses.get(root),+ usage,+ first_usage,+ status,+ self.hooks.trial_init_events.get(root),+ excluded,+ collector._structural_attempt_ordinal(+ root.name,+ collector._task_selector_from_harbor_result(result_payload)+ if isinstance(result_payload, dict)+ else case_id or "",+ ),+ )+ )+ result = tuple(records)+ self.logical_trials.setdefault(agent, {})[arm] = result+ summary = summarize_plugin_signals([record.signals for record in result])+ summary.update(+ evidence_complete=bool(result)+ and not limited+ and all(r.evidence_status not in {"missing", "incomplete"} and r.case_id is not None for r in result),+ trials_truncated=limited,+ n_missing_trials=0,+ )+ summary["n_missing_trajectory"] = sum(record.evidence_status == "missing" for record in result)+ tally = _load_tally(result)+ if tally:+ summary[MCP_LOAD_KEY] = tally+ failures = top_argument_failures(record.signals for record in result)+ if failures:+ summary["arguments"]["top_failures"] = failures+ self.summaries.setdefault(agent, {})[arm] = summary+ return result++ def collect(self, jobs_dir: Path, result: dict[str, Any] | None) -> None:+ self._begin_attempt(jobs_dir)+ if jobs_dir in self._collected:+ return+ self._collected.add(jobs_dir)+ self._sidecar = jobs_dir.parent / RUNTIME_EVIDENCE_FILENAME+ self._context_for_staging()+ agents = (result or {}).get("agents", {})+ names = sorted(set(self.hooks.decisions) | (set(agents) if isinstance(agents, Mapping) else set()))[:32]+ for agent in names:+ conditions = agents.get(agent, {}).get("conditions", {}) if isinstance(agents, Mapping) else {}+ requested = {+ arm+ for arm, condition in conditions.items()+ if arm in ARM_SUFFIXES+ and isinstance(condition, Mapping)+ and condition.get("execution_status") != "skipped"+ }+ if not conditions:+ requested = {ARM_WITH}+ registered = self.arm_jobs.get(agent, {})+ requested.update(registered)+ jobs = {+ arm: registered.get(arm, jobs_dir / f"{self.wrapper.name}-{agent}-{ARM_SUFFIXES[arm]}")+ for arm in requested+ if arm in ARM_SUFFIXES+ }+ for arm, job in jobs.items():+ condition = conditions.get(arm, {})+ expected = condition.get("expected_attempts")+ expected = expected if type(expected) is int and expected >= 0 else 1+ try:+ if job.exists():+ records = self.collect_arm(agent, arm, job)+ else:+ records = ()+ self.logical_trials.setdefault(agent, {})[arm] = records+ self.summaries.setdefault(agent, {})[arm] = summarize_plugin_signals([])+ missing = max(0, expected - len(records))+ summary = self.summaries[agent][arm]+ summary["n_missing_trials"] = missing+ summary["evidence_complete"] = (+ bool(records) and not missing and summary.get("evidence_complete", False)+ )+ except Exception as exc:+ self.errors.append(_safe_text(f"{agent}/{arm}: {type(exc).__name__}: {exc}"))+ self.summaries.setdefault(agent, {})[arm] = {+ **summarize_plugin_signals([]),+ "evidence_complete": False,+ "n_missing_trials": expected,+ }+ if result is not None and isinstance(agents, dict) and isinstance(agents.get(agent), dict):+ agents[agent]["plugin_signals_summary"] = self.summaries.get(agent, {})+ proof_agents = {+ agent: {+ "plugin_signals_summary": self.summaries.get(agent, {}),+ "plugin_load_census": _proof_census(self.logical_trials.get(agent, {}).get(ARM_WITH, ())),+ }+ for agent in names+ }+ self.proof = apply_in_agent_mcp_proof(self._host_proof, {"agents": proof_agents})+ if result is not None:+ result["runtime_evidence"] = self.provenance()+ self.persist()++ def record_failure(self, exc: Exception) -> None:+ """Keep observer failures advisory and redact diagnostics before publication."""+ self.errors.append(_safe_text(f"{type(exc).__name__}: {exc}"))++ def provenance(self) -> dict[str, Any]:+ return {+ "source": vars(self.package.identity),+ "mcp_proof": {+ name: {key: value for key, value in row.items() if key in {"status", "tools", "detail"}}+ for name, row in self.proof.items()+ },+ "errors": self.errors[:64],+ "evidence_complete": bool(self.summaries)+ and not self.errors+ and all(summary.get("evidence_complete") for arms in self.summaries.values() for summary in arms.values()),+ }++ def persist(self) -> None:+ if self._sidecar is None:+ return+ data = {+ **self.provenance(),+ "agents": {+ agent: {+ arm: {+ "summary": self.summaries[agent][arm],+ "trials": [row.persisted() for row in rows],+ "trial_details_total": len(rows),+ }+ for arm, rows in arms.items()+ }+ for agent, arms in self.logical_trials.items()+ },+ }+ while len(json.dumps(data, ensure_ascii=True, allow_nan=False)) > MAX_EVIDENCE_BYTES:+ groups = [arm for arms in data["agents"].values() for arm in arms.values() if arm["trials"]]+ if not groups:+ raise ValueError("runtime evidence summary exceeds its publication bound")+ for arm in groups:+ arm["trials"] = arm["trials"][: len(arm["trials"]) // 2]+ arm["trial_details_truncated"] = True+ data["evidence_complete"] = False+ collector._write_generated_root_json(self._sidecar, self._sidecar.parent, data)diff --git a/src/skillevaluator/tier3/plugin_execution.py b/src/skillevaluator/tier3/plugin_execution.pyindex 6f0baf9f..d49520fc 100644--- a/src/skillevaluator/tier3/plugin_execution.py+++ b/src/skillevaluator/tier3/plugin_execution.py@@ -27,6 +27,8 @@ from skillevaluator.tier3.plugin_native import (if TYPE_CHECKING:from skillevaluator.evaluation.options import EvaluationOptions+ from skillevaluator.tier3.plugin_comparisons import ComparisonOptions+ from skillevaluator.tier3.plugin_evidence import EvidenceOptions, RuntimeEvidenceObserverfrom skillevaluator.tier3.plugin_prepare import PreparedRuntimePackage@@ -83,6 +85,32 @@ class NativeRuntimeHooks:evidence: dict[str, Any] = field(default_factory=dict)evidence_error: str | None = Nonecollected_jobs: set[Path] = field(default_factory=set)+ trial_censuses: dict[Path, Mapping[str, Any]] = field(default_factory=dict)+ trial_init_events: dict[Path, Mapping[str, Any]] = field(default_factory=dict)+ observer: RuntimeEvidenceObserver | None = None++ def configure_run(+ self,+ *,+ case_id_by_task_selector: Mapping[str, str] | None = None,+ arm_jobs: Mapping[str, Mapping[str, Path]] | None = None,+ ) -> None:+ """Supply coordinator-owned identities and additional private arm locations."""+ if self.observer is not None:+ self.observer.configure_run(case_id_by_task_selector=case_id_by_task_selector, arm_jobs=arm_jobs)++ def observe_collected_trials(+ self,+ agent: str,+ arm: str,+ job_dir: Path,+ collected: list[dict[str, Any]],+ scoreable: list[dict[str, Any]],+ selector_map: Mapping[str, str] | None,+ ) -> None:+ """Reuse the engine's reward reads; behavior evidence never participates in scoring."""+ if self.observer is not None:+ self.observer.observe_collected_trials(agent, arm, job_dir, collected, scoreable, selector_map)def begin_attempt(self) -> None:# Replace dictionaries: earlier results retain their own evidence even@@ -91,6 +119,8 @@ class NativeRuntimeHooks:self.evidence = {}self.evidence_error = Noneself.collected_jobs = set()+ if self.observer is not None:+ self.observer.begin_attempt()def with_agent_import_path(self, agent: str, base_import_path: str | None) -> str | None:decision = self.decisions.get(agent)@@ -166,6 +196,13 @@ class NativeRuntimeHooks:result["native_evidence"] = self.evidenceif self.evidence_error:result["native_evidence_error"] = self.evidence_error+ if self.observer is not None:+ try:+ self.observer.collect(jobs_dir, result)+ except Exception as exc:+ self.observer.record_failure(exc)+ if result is not None:+ result["runtime_evidence"] = self.observer.provenance()def _stage_native_skills(task_dir: Path, staging: Any) -> None:@@ -364,6 +401,9 @@ def evaluate_prepared_plugin(evals_source: Path | None = None,evaluate: Callable[[EvaluationOptions], dict[str, Any]] | None = None,publish: Callable[[PrivateRunOutcome], None] | None = None,+ evidence_options: EvidenceOptions | None = None,+ observer_factory: Callable[..., RuntimeEvidenceObserver] | None = None,+ comparison_options: ComparisonOptions | None = None,) -> PrivateRunOutcome:"""Evaluate a prepared plugin through the existing engine and remove the operation-owned staging.@@ -377,6 +417,7 @@ def evaluate_prepared_plugin(from dataclasses import replacefrom skillevaluator.tier3.commands import parse_agents+ from skillevaluator.tier3.plugin_evidence import EvidenceOptions, RuntimeEvidenceObserverfrom skillevaluator.tier3.plugin_native import apply_native_refusals, resolve_plugin_loadfrom skillevaluator.tier3.plugin_prepare import bind_runtime_results, stage_wrapper_skill, wrapper_skill_name@@ -385,6 +426,13 @@ def evaluate_prepared_plugin(agents = parse_agents(options.agents)if load != "wrapper" and not agents:raise PluginLoadError("native plugin loading needs explicit agents")+ if comparison_options is not None:+ from skillevaluator.tier3.plugin_comparisons import ComparisonOptions, ComparisonRuntime++ if not isinstance(comparison_options, ComparisonOptions):+ raise ValueError("comparison_options must be private ComparisonOptions")+ if not agents:+ raise PluginLoadError("private comparisons need explicit agents")decisions = apply_native_refusals(load, resolve_plugin_load(load, agents, env_mode=options.env_mode), package.native_source)@@ -399,6 +447,18 @@ def evaluate_prepared_plugin(shutil.rmtree(temp, ignore_errors=True)raisehooks = NativeRuntimeHooks(package.native_source, decisions, package=package)+ comparison_runtime = None+ try:+ hooks.observer = (observer_factory or RuntimeEvidenceObserver)(+ package, wrapper, hooks, evidence_options or EvidenceOptions()+ )+ if comparison_options is not None:+ comparison_runtime = ComparisonRuntime(+ hooks, package, comparison_options, agents, baseline=not options.skip_baseline+ )+ except BaseException:+ shutil.rmtree(temp, ignore_errors=True)+ raiseprovenance = {"plugin": package.plugin_name,"source": {@@ -424,6 +484,11 @@ def evaluate_prepared_plugin(outcome.provenance["native_evidence"] = hooks.evidenceif hooks.evidence_error:outcome.provenance["native_evidence_error"] = hooks.evidence_error+ if hooks.observer is not None:+ observed = hooks.observer.provenance()+ outcome.provenance.update(runtime_evidence=observed, mcp_proof=observed["mcp_proof"])+ if comparison_runtime is not None:+ outcome.provenance["private_comparisons"] = comparison_runtime.provenance()if publish is not None:publish(outcome)@@ -432,7 +497,7 @@ def evaluate_prepared_plugin(return run_private_evaluation(replace(options, skill_path=wrapper.path, results_dir=results_dir),- hooks=hooks,+ hooks=comparison_runtime or hooks,provenance=provenance,evaluate=evaluate,publish=publish_with_evidence,@@ -458,13 +523,25 @@ def collect_native_evidence(hooks: NativeRuntimeHooks, jobs_dir: Path, skill_namdeclared = plan["declared"]job = jobs_dir / f"{skill_name}-{agent}-with"trials = sorted(p for p in job.iterdir() if (p / "agent").is_dir()) if job.is_dir() else []- censuses = [_trial_census(native, agent, trial / "agent", declared, plan) for trial in trials]+ censuses = []+ for trial in trials:+ census = _trial_census(native, agent, trial / "agent", declared, plan, init_events=hooks.trial_init_events)+ hooks.trial_censuses[trial] = census+ censuses.append(census)summary = native.summarize_censuses(agent, "native", censuses, staged_hook_ids=plan["hook_ids"])evidence[agent] = {**summary, "trials": len(censuses), "job": job.name}return evidence-def _trial_census(native: Any, agent: str, logs: Path, declared: Any, plan: Mapping[str, Any]) -> dict[str, Any]:+def _trial_census(+ native: Any,+ agent: str,+ logs: Path,+ declared: Any,+ plan: Mapping[str, Any],+ *,+ init_events: dict[Path, Mapping[str, Any]] | None = None,+) -> dict[str, Any]:raw = native.read_census_file(logs / native.LOAD_CENSUS_FILENAME)if raw is None:return {**native.fallback_census(agent, "native", declared), "fallback": True}@@ -475,5 +552,7 @@ def _trial_census(native: Any, agent: str, logs: Path, declared: Any, plan: Mappwith log.open("rb") as handle:text = handle.read(_MAX_AGENT_LOG_BYTES).decode("utf-8", errors="replace")if (init := native.claude_init_event(text)) is not None:+ if init_events is not None:+ init_events[logs.parent] = initcensus = native.apply_claude_init_evidence(census, declared, init, plugin=harness["plugin"])return censusdiff --git a/src/skillevaluator/tier3/plugin_trace_artifacts.py b/src/skillevaluator/tier3/plugin_trace_artifacts.pynew file mode 100644index 00000000..2519ca15--- /dev/null+++ b/src/skillevaluator/tier3/plugin_trace_artifacts.py@@ -0,0 +1,534 @@+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+# SPDX-License-Identifier: Apache-2.0+"""Bounded harness evidence readers for the private runtime observer."""++from __future__ import annotations++import json+import os+import re+import stat+from collections.abc import Mapping+from dataclasses import dataclass+from itertools import islice+from pathlib import Path+from typing import Any++from skillevaluator.tier3.harbor.collector import (+ _materialized_trial_trajectory,+ _read_bounded_text,+ _safe_trial_path_component,+)+from skillevaluator.utils.secure_fs import stat_is_link_or_reparse+++def _plugin_signal_trajectory(trial_root: Path) -> dict[str, Any] | None:+ """Reuse the collector's bounded, topology-authorized continuation/child graph."""+ trajectory, _reason = _materialized_trial_trajectory(trial_root, None)+ return trajectory+++_CODEX_LOG_MAX_BYTES = 16 * 1024 * 1024+_CODEX_SESSION_FILES = 8+_CODEX_SESSION_WALK_ENTRIES = 4_096+_CODEX_MCP_CALLS = 4_000+_CODEX_SERVER_NAME_RE = re.compile(r"[^\x00-\x1f\x7f-\x9f]{1,128}")+++def _codex_server(value: Any) -> str | None:+ if not isinstance(value, str):+ return None+ server = value.removeprefix("mcp__").removesuffix("__")+ return server if _CODEX_SERVER_NAME_RE.fullmatch(server) else None+++def _codex_json_lines(path: Path) -> list[Any]:+ """Parsed JSON lines that mention MCP (bounded, no-follow read; unparsable lines are skipped)."""+ text = _read_bounded_text(path, max_bytes=_CODEX_LOG_MAX_BYTES)+ if not text or "mcp" not in text.casefold():+ return []+ events: list[Any] = []+ for line in text.splitlines():+ if "mcp" not in line.casefold():+ continue+ try:+ events.append(json.loads(line))+ except (ValueError, RecursionError):+ continue+ return events+++def _is_real_directory(path: Path) -> bool:+ try:+ metadata = path.lstat()+ except OSError:+ return False+ return not stat_is_link_or_reparse(metadata) and stat.S_ISDIR(metadata.st_mode)+++def _codex_session_files(sessions: Path) -> list[Path]:+ """Return the first Codex session logs (``*.jsonl``) under ``<trial>/agent/sessions``.++ The tree is agent-writable, so the walk never follows a link or reparse+ point and lists at most ``_CODEX_SESSION_WALK_ENTRIES`` entries. It visits+ names in sorted order, a directory's files before its subdirectories. A+ directory with more entries than the budget has left ends the walk, so the+ logs found never depend on the order the filesystem lists names in.+ """+ if not (_is_real_directory(sessions.parent) and _is_real_directory(sessions)):+ return []+ found: list[Path] = []+ budget = _CODEX_SESSION_WALK_ENTRIES+ pending = [sessions]+ while pending and budget > 0 and len(found) < _CODEX_SESSION_FILES:+ directory = pending.pop()+ try:+ with os.scandir(directory) as iterator:+ # One entry past the budget shows that the directory does not fit in it.+ entries = list(islice(iterator, budget + 1))+ except OSError:+ continue+ if len(entries) > budget:+ break+ budget -= len(entries)+ subdirectories: list[Path] = []+ for entry in sorted(entries, key=lambda entry: entry.name):+ try:+ metadata = entry.stat(follow_symlinks=False)+ except OSError:+ continue+ if stat_is_link_or_reparse(metadata):+ continue+ if stat.S_ISDIR(metadata.st_mode):+ subdirectories.append(Path(entry.path))+ elif stat.S_ISREG(metadata.st_mode) and entry.name.endswith(".jsonl"):+ found.append(Path(entry.path))+ pending.extend(reversed(subdirectories))+ return found[:_CODEX_SESSION_FILES]+++_CODEX_ARGS_KEY_CHARS = 4_096+++def _codex_args_key(value: Any) -> str | None:+ """Canonical JSON text of a Codex call's arguments (a JSON string or an object), bounded."""+ if isinstance(value, str):+ try:+ value = json.loads(value)+ except (ValueError, RecursionError):+ return value if len(value) <= _CODEX_ARGS_KEY_CHARS else None+ try:+ text = json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False)+ except (TypeError, ValueError, RecursionError):+ return None+ return text if len(text) <= _CODEX_ARGS_KEY_CHARS else None+++def _codex_item_status(item: Mapping[str, Any]) -> str | None:+ """``failed`` or ``completed`` for a finished ``mcp_tool_call`` item, else ``None``."""+ error = item.get("error")+ result = item.get("result")+ flagged = isinstance(result, Mapping) and any(result.get(key) is True for key in ("isError", "is_error"))+ if error not in (None, "", {}, []) or flagged:+ return "failed"+ status = item.get("status")+ if isinstance(status, str) and status.strip().casefold() in {"failed", "completed"}:+ return status.strip().casefold()+ return None+++def _codex_txt_items(agent_dir: Path) -> list[dict[str, Any]]:+ """``mcp_tool_call`` items of ``codex.txt`` in start order: server, tool, arguments key, status.++ ``codex.txt`` item ids (``item_N``) are not tool call ids, so callers pair+ an item with a call by server, tool, and arguments.+ """+ items: dict[str, dict[str, Any]] = {}+ for event in _codex_json_lines(agent_dir / "codex.txt"):+ event_type = event.get("type") if isinstance(event, Mapping) else None+ if not isinstance(event_type, str) or event_type not in {"item.started", "item.completed"}:+ continue+ item = event.get("item")+ if not isinstance(item, Mapping) or item.get("type") != "mcp_tool_call":+ continue+ server, tool, item_id = _codex_server(item.get("server")), item.get("tool"), item.get("id")+ if not server or not isinstance(tool, str) or not tool or not isinstance(item_id, str):+ continue+ entry = items.get(item_id)+ if entry is None:+ if len(items) >= _CODEX_MCP_CALLS:+ continue+ entry = items[item_id] = {+ "server": server,+ "tool": tool,+ "args": _codex_args_key(item.get("arguments")),+ "status": None,+ }+ if event.get("type") == "item.completed":+ entry["status"] = _codex_item_status(item)+ entry["text"] = _codex_item_text(item)+ return list(items.values())+++_CODEX_TEXT_CHARS = 512+++def _codex_item_text(item: Mapping[str, Any]) -> str:+ """The start of a finished item's first text block (the tool's own answer), or ``""``."""+ result = item.get("result")+ content = result.get("content") if isinstance(result, Mapping) else None+ for block in content[:8] if isinstance(content, list) else ():+ if isinstance(block, Mapping) and isinstance(block.get("text"), str) and block["text"].strip():+ return block["text"][:_CODEX_TEXT_CHARS]+ return ""+++def _codex_call_outputs(trajectory: Mapping[str, Any] | None) -> dict[str, str]:+ """Tool call id -> the start of its result text in the trajectory (bounded)."""+ outputs: dict[str, str] = {}+ steps = trajectory.get("steps") if isinstance(trajectory, Mapping) else None+ for step in steps[:_CODEX_MCP_CALLS] if isinstance(steps, list) else ():+ observation = step.get("observation") if isinstance(step, Mapping) else None+ results = observation.get("results") if isinstance(observation, Mapping) else None+ for result in results[:64] if isinstance(results, list) else ():+ if not isinstance(result, Mapping) or not isinstance(result.get("source_call_id"), str):+ continue+ content = result.get("content")+ if isinstance(content, list):+ content = "\n".join(+ block.get("text", "") if isinstance(block, Mapping) else str(block)+ for block in content[:8]+ if isinstance(block, Mapping | str)+ )+ if isinstance(content, str):+ outputs.setdefault(result["source_call_id"], content[: _CODEX_TEXT_CHARS * 16])+ return outputs+++def _codex_server_key(server: str) -> str:+ # The session log spells a server the way Codex sanitizes it (``deepwiki_bad``);+ # codex.txt keeps the configured name (``deepwiki-bad``).+ return re.sub(r"[^a-z0-9]", "_", server.casefold())+++def _text_in_output(text: str, output: str) -> bool:+ # The trajectory keeps the tool text raw, or JSON-escaped inside Codex's content wrapper.+ return bool(text) and (text in output or json.dumps(text, ensure_ascii=False)[1:-1] in output)+++def _take_codex_item(+ items: list[dict[str, Any]], *, tool: str, args: str | None, server: str | None, output: str = ""+) -> Any:+ """Pop the unpaired item with this tool and arguments (and server, when known).++ An answer can disambiguate candidates. Conflicting identities/outcomes+ remain unknown rather than inheriting the first log item's success.+ """+ key = _codex_server_key(server) if server is not None else None+ fits = [+ index+ for index, item in enumerate(items)+ if item["tool"] == tool and item["args"] == args and (key is None or _codex_server_key(item["server"]) == key)+ ]+ if not fits or args is None:+ return None+ matching_output = [index for index in fits if _text_in_output(items[index].get("text") or "", output)]+ candidates = matching_output or fits+ outcomes = {(_codex_server_key(items[index]["server"]), items[index].get("status")) for index in candidates}+ if len(outcomes) != 1:+ return None+ return items.pop(candidates[0])+++def _codex_mcp_calls(trial_root: Path, trajectory: Mapping[str, Any] | None) -> tuple[dict[str, str], dict[str, str]]:+ """Codex MCP calls read from the raw Codex logs: ``(call id -> server, call id -> status)``.++ Harbor's Codex trajectory keeps only the bare MCP tool name (for example+ ``lookup``) and no error flag. Codex's own session log records the server+ (``namespace: "mcp__<server>"`` on the call, and an ``McpToolCall`` item+ with the call id); the ``codex.txt`` JSON stream records each call's+ outcome (``status: "failed"`` or ``"completed"``) but under its own item+ id. So each session-log call is paired with the first unpaired+ ``codex.txt`` item of the same server, tool, and arguments. Without a+ readable session log, the trajectory's calls are paired with ``codex.txt``+ items the same way (tool and arguments), never by name and order alone; a+ tool name served by one server only still maps without arguments. When+ several items fit, the one whose answer is in the call's trajectory result+ wins. Empty for other harnesses.+ """+ servers: dict[str, str] = {}+ session_calls: list[tuple[str, str, str, str | None]] = []+ agent_dir = trial_root / "agent"+ for path in _codex_session_files(agent_dir / "sessions"):+ for event in _codex_json_lines(path):+ payload = event.get("payload") if isinstance(event, Mapping) else None+ if not isinstance(payload, Mapping):+ continue+ item = payload.get("item")+ tool: Any = None+ args: str | None = None+ item_type = item.get("type") if isinstance(item, Mapping) else None+ payload_type = payload.get("type")+ if isinstance(item_type, str) and item_type in {"McpToolCall", "mcp_tool_call"}:+ call_id, server = item.get("id"), _codex_server(item.get("server"))+ tool, args = item.get("tool"), _codex_args_key(item.get("arguments"))+ elif isinstance(payload_type, str) and payload_type in {"function_call", "custom_tool_call"}:+ namespace = payload.get("namespace")+ call_id = payload.get("call_id")+ server = _codex_server(namespace) if isinstance(namespace, str) and namespace[:5] == "mcp__" else None+ tool = payload.get("name")+ args = _codex_args_key(payload.get("arguments", payload.get("input")))+ else:+ continue+ if isinstance(call_id, str) and call_id and server and len(servers) < _CODEX_MCP_CALLS:+ if call_id not in servers and isinstance(tool, str) and tool:+ session_calls.append((call_id, server, tool, args))+ servers.setdefault(call_id, server)+ items = _codex_txt_items(agent_dir)+ outputs = _codex_call_outputs(trajectory) if items else {}+ statuses: dict[str, str] = {}+ if servers:+ for call_id, server, tool, args in session_calls:+ item = _take_codex_item(items, tool=tool, args=args, server=server, output=outputs.get(call_id, ""))+ if item is not None and item["status"]:+ statuses[call_id] = item["status"]+ elif item is None and any(+ entry["tool"] == tool and _codex_server_key(entry["server"]) == _codex_server_key(server)+ for entry in items+ ):+ statuses[call_id] = "unknown"+ return servers, statuses+ if not items or not isinstance(trajectory, Mapping):+ return servers, statuses+ servers_by_tool: dict[str, set[str]] = {}+ for item in items:+ servers_by_tool.setdefault(item["tool"], set()).add(item["server"])+ for step in (trajectory.get("steps") or ())[:_CODEX_MCP_CALLS]:+ for call in (step.get("tool_calls") if isinstance(step, Mapping) else None) or ():+ if not isinstance(call, Mapping):+ continue+ name, call_id = call.get("function_name"), call.get("tool_call_id")+ if not isinstance(name, str) or not isinstance(call_id, str) or not call_id or call_id in servers:+ continue+ item = _take_codex_item(+ items,+ tool=name,+ args=_codex_args_key(call.get("arguments")),+ server=None,+ output=outputs.get(call_id, ""),+ )+ if item is not None:+ servers[call_id] = item["server"]+ if item["status"]:+ statuses[call_id] = item["status"]+ elif len(servers_by_tool.get(name, ())) == 1:+ servers[call_id] = next(iter(servers_by_tool[name]))+ statuses[call_id] = "unknown"+ if len(servers) >= _CODEX_MCP_CALLS:+ return servers, statuses+ return servers, statuses+++_OPENCODE_LOG_MAX_BYTES = 16 * 1024 * 1024+_OPENCODE_FAILED_CALLS = 4_000+_OPENCODE_ERROR_CHARS = 2_048+++def _opencode_call_errors(trial_root: Path) -> dict[str, str]:+ """OpenCode tool calls that failed, read from the raw ``opencode.txt``: ``call id -> error text``.++ OpenCode's JSON stream ends each tool part with ``state.status``:+ ``completed`` with an ``output``, or ``error`` with an ``error``. Harbor's+ OpenCode trajectory keeps a call's result only when it has an output, so a+ failed call (a ``task`` to an agent that does not exist, an MCP call whose+ connection closed) is left with no result and no outcome. The part's+ ``callID`` is the trajectory's tool call id. Empty for other harnesses.+ """+ from skillevaluator.tier3.eval_core.log_converters import _opencode_error_text++ text = _read_bounded_text(trial_root / "agent" / "opencode.txt", max_bytes=_OPENCODE_LOG_MAX_BYTES)+ errors: dict[str, str] = {}+ for line in text.splitlines() if text and "tool_use" in text else ():+ if "tool_use" not in line or "error" not in line:+ continue+ try:+ event = json.loads(line)+ except (ValueError, RecursionError):+ continue+ part = event.get("part") if isinstance(event, Mapping) and event.get("type") == "tool_use" else None+ state = part.get("state") if isinstance(part, Mapping) else None+ if not isinstance(state, Mapping) or state.get("status") != "error":+ continue+ call_id = part.get("callID", part.get("id"))+ if isinstance(call_id, str) and call_id and call_id not in errors:+ errors[call_id] = _opencode_error_text(dict(state))[:_OPENCODE_ERROR_CHARS]+ if len(errors) >= _OPENCODE_FAILED_CALLS:+ break+ return errors+++def _call_status(result: Any, statuses: Mapping[str, str]) -> str | None:+ call_id = result.get("source_call_id") if isinstance(result, Mapping) else None+ return statuses.get(call_id) if isinstance(call_id, str) else None+++def _step_results(step: Any) -> list[Any]:+ observation = step.get("observation") if isinstance(step, Mapping) else None+ results = observation.get("results") if isinstance(observation, Mapping) else None+ return results if isinstance(results, list) else []+++def _with_harness_statuses(+ trajectory: dict[str, Any] | None, statuses: Mapping[str, str], errors: Mapping[str, str] | None = None+) -> dict[str, Any] | None:+ """A copy of ``trajectory`` whose results carry the harness's own per-call status.++ The status goes to ``extra.harness_status`` of every result whose+ ``source_call_id`` names the call, where plugin signals read it as the+ call's structured outcome. A call in ``errors`` (call id -> error text)+ that no result names gets one in its step: the error text, with an+ ``error`` status. The trajectory read from disk is not changed.+ """+ from skillevaluator.tier3.eval_core.plugin_signals import HARNESS_STATUS_KEY++ steps = trajectory.get("steps") if isinstance(trajectory, dict) else None+ if not (statuses or errors) or not isinstance(steps, list):+ return trajectory+ errors = errors or {}+ # Only a call that no result names had its result dropped.+ answered = {+ result["source_call_id"]+ for step in steps+ for result in _step_results(step)+ if errors and isinstance(result, Mapping) and isinstance(result.get("source_call_id"), str)+ }+ updated: list[Any] = []+ for step in steps:+ results = _step_results(step)+ calls = step.get("tool_calls") if errors and isinstance(step, Mapping) else None+ dropped = {+ call_id: None+ for call in (calls if isinstance(calls, list) else ())+ if isinstance(call, Mapping)+ and isinstance(call_id := call.get("tool_call_id"), str)+ and call_id in errors+ and call_id not in answered+ }+ if not dropped and not any(_call_status(result, statuses) for result in results):+ updated.append(step)+ continue+ marked = []+ for result in results:+ status = _call_status(result, statuses)+ if status is None:+ marked.append(result)+ continue+ extra = result.get("extra") if isinstance(result.get("extra"), Mapping) else {}+ marked.append({**result, "extra": {**extra, HARNESS_STATUS_KEY: status}})+ # The harness dropped these results (OpenCode keeps none for a failed call).+ marked.extend(+ {"source_call_id": call_id, "content": errors[call_id], "extra": {HARNESS_STATUS_KEY: "error"}}+ for call_id in dropped+ )+ answered.update(dropped)+ observation = step.get("observation") if isinstance(step.get("observation"), Mapping) else {}+ updated.append({**step, "observation": {**observation, "results": marked}})+ return {**trajectory, "steps": updated}+++@dataclass(frozen=True)+class TrialTrace:+ trajectory: dict[str, Any] | None+ call_servers: Mapping[str, str]+++def _harness_sources(trial_root: Path, trajectory: dict[str, Any] | None) -> list[tuple[Path, dict[str, Any] | None]]:+ """Partition a materialized native multi-step trajectory by its authorized sources.++ A physical steps layout makes the materializer require authoritative+ ``step_results`` and reject a root trajectory. Only after that succeeds+ are its generated step names trusted as sibling harness-log locations.+ Single-step trajectories cannot select log paths through authored extras.+ """+ if trajectory is None or not _is_real_directory(trial_root / "steps"):+ return [(trial_root, trajectory)]+ extra = trajectory.get("extra")+ multi = extra.get("harbor_multi_step") if isinstance(extra, Mapping) else None+ names = multi.get("step_names") if isinstance(multi, Mapping) else None+ if not isinstance(names, list) or any(_safe_trial_path_component(name) != name for name in names):+ return [(trial_root, None)]+ groups: dict[str, list[Any]] = {name: [] for name in names}+ for step in trajectory.get("steps", []):+ extra = step.get("extra") if isinstance(step, Mapping) else None+ name = extra.get("harbor_step_name") if isinstance(extra, Mapping) else None+ if name not in groups:+ return [(trial_root, None)]+ groups[name].append(step)+ return [(trial_root / "steps" / name, {**trajectory, "steps": groups[name]}) for name in names]+++def _scope_harness_call_ids(trajectory: dict[str, Any], prefix: str) -> dict[str, Any]:+ """Keep independent step sessions' reused call IDs distinct in the merged document."""+ updated = []+ for step in trajectory["steps"]:+ if not isinstance(step, Mapping):+ updated.append(step)+ continue+ scoped = dict(step)+ calls = step.get("tool_calls")+ if isinstance(calls, list):+ scoped["tool_calls"] = [+ {**call, "tool_call_id": prefix + call["tool_call_id"]}+ if isinstance(call, Mapping) and isinstance(call.get("tool_call_id"), str)+ else call+ for call in calls+ ]+ observation = step.get("observation")+ results = observation.get("results") if isinstance(observation, Mapping) else None+ if isinstance(results, list):+ scoped["observation"] = {+ **observation,+ "results": [+ {**result, "source_call_id": prefix + result["source_call_id"]}+ if isinstance(result, Mapping) and isinstance(result.get("source_call_id"), str)+ else result+ for result in results+ ],+ }+ updated.append(scoped)+ return {**trajectory, "steps": updated}+++class TrialTraceArtifacts:+ """Read one whole-trial trajectory and harness logs once per logical root."""++ def __init__(self) -> None:+ self._traces: dict[Path, TrialTrace] = {}++ def trace(self, trial_root: Path) -> TrialTrace:+ if trial_root not in self._traces:+ trajectory = _plugin_signal_trajectory(trial_root)+ sources = _harness_sources(trial_root, trajectory)+ servers, steps = {}, []+ for index, (source_root, source) in enumerate(sources):+ if source is None:+ trajectory = None+ break+ try:+ local_servers, statuses = _codex_mcp_calls(source_root, source)+ errors = _opencode_call_errors(source_root)+ statuses.update(dict.fromkeys(errors, "error"))+ marked = _with_harness_statuses(source, statuses, errors)+ except Exception:+ # Advisory harness logs cannot discard an authorized trajectory.+ local_servers, marked = {}, source+ if source_root != trial_root:+ prefix = f"harbor-step:{index}:"+ marked = _scope_harness_call_ids(marked, prefix)+ local_servers = {prefix + call_id: server for call_id, server in local_servers.items()}+ servers.update(local_servers)+ steps.extend(marked["steps"])+ if trajectory is not None:+ trajectory = {**trajectory, "steps": steps}+ self._traces[trial_root] = TrialTrace(trajectory, servers)+ return self._traces[trial_root]diff --git a/src/skillevaluator/tier3/runtime_arms.py b/src/skillevaluator/tier3/runtime_arms.pynew file mode 100644index 00000000..a91f0b33--- /dev/null+++ b/src/skillevaluator/tier3/runtime_arms.py@@ -0,0 +1,8 @@+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+# SPDX-License-Identifier: Apache-2.0+"""Private arm identifiers shared by runtime evidence and comparisons."""++ARM_WITH = "with_skill"+ARM_WITHOUT = "without_skill"+ARM_SUM_OF_PARTS = "sum_of_parts"+ARM_LOOSE_PARTS = "loose_parts"diff --git a/src/skillevaluator/utils/redaction.py b/src/skillevaluator/utils/redaction.pyindex bf20cb2d..82718a1a 100644--- a/src/skillevaluator/utils/redaction.py+++ b/src/skillevaluator/utils/redaction.py@@ -11,6 +11,8 @@ from collections.abc import Iterable, Mappingfrom typing import Anyfrom urllib.parse import unquote+from skillevaluator.utils.rich_markup import strip_terminal_controls+_SECRET_KEY_PARTS = {"auth","authorization",@@ -92,6 +94,9 @@ _CREDENTIAL_URI_USERINFO_RE = re.compile(r"(?i)(?P<scheme>[a-z][a-z0-9+.-]{0,31}_SENSITIVE_QUOTED_ASSIGNMENT_RE = re.compile(rf"(?i){_SENSITIVE_KEY_PATTERN}\s*+(?P<sep>[:=])\s*+(?:\"[^\"\r\n]*+\"|'[^'\r\n]*+')")+_SENSITIVE_OPEN_QUOTED_ASSIGNMENT_RE = re.compile(+ rf"(?im){_SENSITIVE_KEY_PATTERN}\s*+(?P<sep>[:=])\s*+(?:\"[^\"\r\n]*+$|'[^'\r\n]*+$)"+)# The whitespace after ":" must stay backtrackable: in "token: ," the value is the last blank._SENSITIVE_COLON_ASSIGNMENT_RE = re.compile(rf"(?im){_SENSITIVE_KEY_PATTERN}\s*+(?P<sep>:)\s*[^\r\n,;]+")_SENSITIVE_EQUALS_ASSIGNMENT_RE = re.compile(rf"(?i){_SENSITIVE_KEY_PATTERN}\s*+(?P<sep>=)\s*+[^\s\"',;]+")@@ -249,8 +254,14 @@ def _redact_unterminated_private_key(text: str) -> str:return text-def redact_sensitive_text(value: str, *, max_len: int | None = None) -> str:+def redact_sensitive_text(value: str, *, max_len: int | None = None, max_input_chars: int | None = None) -> str:"""Best-effort masking for credentials before writing logs or artifacts."""+ if max_input_chars is not None and len(value) > max_input_chars:+ window = value[:max_input_chars]+ # A cut credential may no longer match its redaction pattern. Drop+ # the final token before redaction can shorten earlier content.+ boundary = next((index for index in range(len(window) - 1, -1, -1) if window[index].isspace()), -1)+ value = window[: boundary + 1]out = value# Remove multiline private-key material before any single-line assignment# or header rule can consume only its BEGIN delimiter and orphan the body.@@ -261,6 +272,7 @@ def redact_sensitive_text(value: str, *, max_len: int | None = None) -> str:out = _CREDENTIAL_URI_USERINFO_RE.sub(r"\g<scheme><redacted>@", out)out = _AUTH_HEADER_RE.sub(_redact_auth_header, out)out = _SENSITIVE_QUOTED_ASSIGNMENT_RE.sub(_redact_sensitive_assignment, out)+ out = _SENSITIVE_OPEN_QUOTED_ASSIGNMENT_RE.sub(_redact_sensitive_assignment, out)out = _SENSITIVE_COLON_ASSIGNMENT_RE.sub(_redact_sensitive_assignment, out)out = _SENSITIVE_EQUALS_ASSIGNMENT_RE.sub(_redact_sensitive_assignment, out)for pattern, replacement in _REDACTIONS:@@ -272,6 +284,11 @@ def redact_sensitive_text(value: str, *, max_len: int | None = None) -> str:return out+def redact_printable_text(value: str, *, max_len: int | None = None, max_input_chars: int | None = None) -> str:+ """Strip terminal controls before masking credentials they may have split."""+ return redact_sensitive_text(strip_terminal_controls(value), max_len=max_len, max_input_chars=max_input_chars)++def contains_credential_value(value: object) -> bool:"""Return whether text contains credential material, including embedded tokens."""if value is None:diff --git a/src/skillevaluator/validators/private_plugin_schema.py b/src/skillevaluator/validators/private_plugin_schema.pyindex fe27c260..276288c3 100644--- a/src/skillevaluator/validators/private_plugin_schema.py+++ b/src/skillevaluator/validators/private_plugin_schema.py@@ -163,12 +163,10 @@ def validate_inventory_schema(inventory: PluginInventory) -> ValidationResult:if not inventory.complete:result.mark_scan_incomplete("plugin-inventory")result.metadata["plugin"] = {+ **inventory.metadata(),"root": str(inventory.root),"manifest_sha256": inventory.manifest_sha256,"manifest_declarations": declarations,- "component_inventory": inventory.component_inventory(),- "mcp": inventory.mcp_summary(),- "context_cost": inventory.context_cost(),}inventory_metadata = result.metadata["plugin"]dependencies = dependency_records(inventory)diff --git a/tests/reporting/test_private_static_reports.py b/tests/reporting/test_private_static_reports.pynew file mode 100644index 00000000..310734a4--- /dev/null+++ b/tests/reporting/test_private_static_reports.py@@ -0,0 +1,679 @@+# SPDX-License-Identifier: Apache-2.0+"""Private static views share facts without enabling public plugin workflows."""++from __future__ import annotations++import html+import importlib+import json+from copy import deepcopy++import pytest++from skillevaluator.models.result import Finding, Severity, ValidationResult+from skillevaluator.reporting import (+ BenchmarkReporter,+ CLIReporter,+ HTMLReporter,+ JSONReporter,+ MarkdownReporter,+ SARIFReporter,+)+++def api():+ return importlib.import_module("skillevaluator.reporting.plugin_static")+++def renderers():+ return importlib.import_module("skillevaluator.reporting.plugin_static_render")+++def reporters():+ return [+ CLIReporter(),+ JSONReporter(include_timestamp=False),+ MarkdownReporter(include_timestamp=False),+ HTMLReporter(include_timestamp=False),+ SARIFReporter(include_timestamp=False),+ BenchmarkReporter(include_timestamp=False),+ ]+++def bundle():+ schema = ValidationResult(validator_name="Private Manifest and Component Checks")+ schema.metadata["plugin"] = {+ "root": "/work/bundle",+ "manifest_sha256": "a" * 64,+ "manifest_declarations": [+ {+ "selected": True,+ "manifest_filename": ".claude-plugin/plugin.json",+ "manifest_type": "claude_plugin_json",+ "name": "report-demo",+ "version": "1.0",+ }+ ],+ "component_inventory": {+ "complete": True,+ "components": [+ {+ "type": "skill",+ "name": "loader",+ "path": "skills/loader",+ "origin": "packaged",+ "support": "evaluated",+ "findings": 1,+ },+ {+ "type": "app",+ "name": "calendar-app",+ "path": "apps.json",+ "support": "unsupported",+ "loaded_by": "Codex",+ "effective": False,+ },+ ],+ "counts": {"skill": 1, "app": 1},+ },+ "dependency_resolution": {+ "skills": [+ {+ "ref": "external-ref",+ "state": "external",+ "declared_by": "agent_plugin.yaml",+ "reason": "not fetched",+ }+ ]+ },+ "mcp": {+ "servers": [+ {+ "name": "search-server",+ "pinned": False,+ "kind": "command",+ "transport": "stdio",+ "pin_detail": "floating runner",+ }+ ],+ "pinning": {"total": 1, "pinned": 0, "unpinned": 1},+ },+ "context_cost": {+ "method": "static_estimate",+ "estimator": "chars_div_4_cjk",+ "harness": "claude-code",+ "load_mode": "native",+ "always_on_tokens": -20,+ "on_demand_tokens": 250,+ "lower_bound": True,+ "not_counted": ["MCP schemas"],+ "by_component": [+ {+ "type": "skill",+ "name": "loader",+ "always_on_tokens": 20,+ "on_demand_tokens": 250,+ "basis": "frontmatter",+ }+ ],+ "by_harness": [+ {"harness": "codex", "load_mode": "wrapper", "always_on_tokens": 0, "on_demand_tokens": 270}+ ],+ },+ "hook_risk": {+ "hooks": [+ {+ "id": "startup-hook",+ "event": "SessionStart",+ "target": "echo harmless",+ "risk_flags": ["context_injection"],+ }+ ]+ },+ "privileges": {+ "components": [+ {+ "type": "agent",+ "name": "reviewer-agent",+ "path": "agents/reviewer.md",+ "inherits_all_tools": True,+ "flags": ["inherits_all_tools"],+ }+ ]+ },+ }+ schema.add_finding(Finding("TEST", Severity.MEDIUM, "example", "Example", "/work/bundle/skills/loader/SKILL.md"))+ audit = ValidationResult(validator_name="Private audit")+ audit.metadata["plugin"] = {+ "cve_summary": {+ "ecosystems": {+ "python": {+ "status": "incomplete",+ "sources": 2,+ "audited": 1,+ "vulnerabilities": 3,+ "findings_truncated": True,+ "errors": ["audit capped"],+ }+ }+ },+ "validator_parity": {+ "status": "compared",+ "claude_verdict": "passed",+ "strict_verdict": "failed",+ "skillevaluator_verdict": "passed",+ "agree": True,+ "warnings": ["parity-warning"],+ },+ }+ audit.mark_scan_incomplete("pip-audit")+ audit.add_finding(+ Finding(+ "CVE",+ Severity.HIGH,+ "python-vulnerability",+ "CVE-fixture",+ "requirements.txt",+ metadata={"ecosystem": "python"},+ )+ )+ endpoint = ValidationResult(validator_name="Private Endpoint Checks")+ endpoint.metadata["endpoint_resolution"] = {+ "endpoints": [+ {+ "kind": "mcp",+ "name": "remote-endpoint",+ "url": "https://service.example/path?token=hidden-value",+ "status": "private",+ "head": {"skipped": "non-public"},+ "redirect": {"classification": "metadata"},+ }+ ]+ }+ catalog = ValidationResult(validator_name="Inter-Plugin Deduplication")+ catalog.metadata.update(+ advisory_tier2=True,+ plugin={+ "catalog_skill_similarity": {+ "status": "compared",+ "catalog_entries": 4,+ "matches": [{"skill": "loader", "match": "catalog-loader", "similarity": 0.95}],+ },+ "inter_plugin_similarity": {"status": "skipped", "reason": "catalog unavailable", "matches": []},+ },+ )+ return [schema, audit, endpoint, catalog]+++def test_projection_preserves_real_private_shapes_and_does_not_mutate():+ results = bundle()+ before = deepcopy(results)+ view = api().project_plugin_static(results)+ assert view is not None+ payload = view.payload()+ assert payload["name"] == "report-demo" and payload["status"] == "failed"+ sections = {section["key"]: section for section in payload["sections"]}+ assert {+ "manifests",+ "inventory",+ "dependencies",+ "mcp",+ "context",+ "context_harnesses",+ "hooks",+ "privileges",+ "audits",+ "parity",+ "endpoints",+ "catalog_skills",+ "catalog_plugins",+ } <= sections.keys()+ assert "3" in json.dumps(sections["audits"])+ assert "unknown" in json.dumps(sections["audits"]).lower() or "partial" in json.dumps(sections["audits"]).lower()+ assert "-20" in json.dumps(sections["context"])+ assert "hidden-value" not in json.dumps(payload)+ assert results == before+++def test_all_six_private_surfaces_carry_the_same_static_facts():+ output = renderers().render_plugin_static_reports(bundle(), reporters=reporters())+ assert set(output) == {"cli", "json", "markdown", "html", "sarif", "benchmark"}+ for name, document in output.items():+ for marker in (+ "report-demo",+ "calendar-app",+ "external-ref",+ "search-server",+ "startup-hook",+ "reviewer-agent",+ "remote-endpoint",+ "catalog-loader",+ "catalog unavailable",+ "chars_div_4_cjk",+ "audit capped",+ "parity-warning",+ ):+ assert marker in html.unescape(document).replace("\\_", "_"), (name, marker)+ assert "hidden-value" not in document+ expected = json.loads(output["json"])["plugin_static"]+ assert json.loads(output["sarif"])["runs"][0]["properties"]["pluginStatic"] == expected+ baseline_html = HTMLReporter(include_timestamp=False).render_all(bundle())+ assert output["html"].split("</head>", 1)[0] == baseline_html.split("</head>", 1)[0]+ assert 'id="report-data"' in output["html"]+ assert 'id="plugin-static-evidence"' in output["html"]+ assert "Recommended for publication" not in output["benchmark"]+++def test_absent_empty_and_malformed_blocks_leave_generic_reports_unchanged():+ for metadata in ({}, {"plugin": {}}, {"plugin": "wrong"}):+ result = ValidationResult(validator_name="SCHEMA", metadata=metadata)+ report_list = reporters()+ before = {reporter.name: reporter.render_all([result]) for reporter in report_list}+ assert renderers().render_plugin_static_reports([result], reporters=report_list) == before+++def test_projection_reduction_happens_once_for_six_adapters(monkeypatch):+ module = renderers()+ project = module.project_plugin_static+ calls = []++ def counting(results):+ calls.append(len(results))+ return project(results)++ monkeypatch.setattr(module, "project_plugin_static", counting)+ module.render_plugin_static_reports(bundle(), reporters=reporters())+ assert calls == [4]++[email protected]("force_color", [False, True])+def test_huge_cyclic_and_hostile_static_metadata_is_bounded_and_sanitized(monkeypatch, force_color):+ monkeypatch.setenv("TERM", "xterm" if force_color else "dumb")+ monkeypatch.delenv("NO_COLOR", raising=False)+ if force_color:+ monkeypatch.setenv("FORCE_COLOR", "1")+ else:+ monkeypatch.delenv("FORCE_COLOR", raising=False)+ results = bundle()+ hostile = (+ '<img src=x onerror=alert(1)> [bold]oops[/bold] API_KEY="'+ "private-test-secret"+ '"\x1b]8;;https://evil.example\x07click\x1b]8;;\x07\u202e\ud800'+ )+ components = results[0].metadata["plugin"]["component_inventory"]["components"]+ components[:] = [+ {"type": "skill", "name": hostile, "path": f"skills/{index}", "support": "evaluated"} for index in range(800)+ ]+ results[0].metadata["plugin"]["context_cost"]["notes"] = ["x" * 2_000_000]+ results[0].metadata["plugin"]["cycle"] = results[0].metadata+ projection = api().project_plugin_static(results)+ payload = projection.payload()+ assert payload["truncated"] is True+ serialized = json.dumps(payload, ensure_ascii=False)+ assert "\\u001b" not in serialized and "private-test-secret" not in serialized and "\ud800" not in serialized+ assert len(serialized.encode("utf-8")) <= api().MAX_OUTPUT_BYTES+ document = renderers().render_plugin_static_reports(results, reporters=reporters())+ for name, text in document.items():+ assert "private-test-secret" not in text and "\ud800" not in text, name+ if name == "cli":+ original, fragment = text.split("\n\nPlugin static evidence\n", 1)+ assert ("\x1b[" in original) is force_color+ else:+ fragment = text+ assert "\x1b" not in fragment, name+ if name in {"html", "markdown", "benchmark"}:+ assert "<img src=x onerror=alert(1)>" not in text+++def test_sarif_component_attribution_is_lexical_and_unambiguous():+ results = bundle()+ view = api().project_plugin_static(results)+ assert view.component_for("[loader] SKILL.md")["path"] == "skills/loader"+ assert view.component_for("/outside/skills/loader/SKILL.md") is None+ assert view.component_for("skills/loader/../../escape") is None+ result = renderers().render_plugin_static_reports(results, reporters=[SARIFReporter(include_timestamp=False)])+ finding = json.loads(result["sarif"])["runs"][0]["results"][0]+ assert finding["properties"]["pluginComponent"]["name"] == "loader"+++def test_private_schema_bridges_existing_risk_records(tmp_path):+ from skillevaluator.plugin_inventory import plugin_inventory_for_root+ from skillevaluator.validators.private_plugin_schema import validate_inventory_schema++ manifest = tmp_path / ".claude-plugin/plugin.json"+ manifest.parent.mkdir()+ manifest.write_text(+ json.dumps(+ {+ "name": "real-private",+ "hooks": {"SessionStart": [{"hooks": [{"type": "command", "command": "echo fixture"}]}]},+ }+ )+ )+ inventory = plugin_inventory_for_root(tmp_path)+ result = validate_inventory_schema(inventory)+ assert result.metadata["plugin"]["hook_risk"]["hooks"]+ assert api().project_plugin_static([result]).payload()["name"] == "real-private"+++def test_report_bounds_reserve_identity_and_later_risk_evidence():+ results = bundle()+ results[0].metadata["plugin"]["component_inventory"]["components"] = [+ {"type": "skill", "name": "long-name-" + "x" * 200, "path": f"skills/{index}", "support": "evaluated"}+ for index in range(800)+ ]+ payload = api().project_plugin_static(results).payload()+ assert payload["name"] == "report-demo"+ hooks = next(section for section in payload["sections"] if section["key"] == "hooks")+ assert "startup-hook" in json.dumps(hooks)+ assert "context_injection" in json.dumps(hooks)++[email protected]("long_cells", [False, True], ids=["row-limit", "text-limit"])+def test_crowded_reports_retain_risk_evidence_across_all_six_formats(long_cells):+ results = bundle()+ block = results[0].metadata["plugin"]+ count = 64 if long_cells else 100+ text = "x" * 300 if long_cells else "fixture"+ block["component_inventory"]["components"] = [+ dict.fromkeys(("type", "name", "origin", "path", "support", "problem", "declared_by", "loaded_by"), text)+ for _ in range(count)+ ]+ block["dependency_resolution"] = {+ "skills": [dict.fromkeys(("ref", "state", "path", "declared_by", "reason"), text) for _ in range(count)]+ }+ other_count = 1 if long_cells else 100+ block["mcp"]["servers"] = [{"name": f"server-{index}"} for index in range(other_count)]+ block["context_cost"]["by_component"] = [{"name": f"context-{index}"} for index in range(other_count)]+ block["hook_risk"]["hooks"] = [+ {"id": f"flagged-hook-{index}", "event": "SessionStart", "risk_flags": ["context_injection"]}+ for index in range(5)+ ]+ block["privileges"]["components"][0].update(permission_mode="bypass_permissions", flags=["bypass_permissions"])+ before = deepcopy(results)++ output = renderers().render_plugin_static_reports(results, reporters=reporters())++ payload = json.loads(output["json"])["plugin_static"]+ sections = {section["key"]: section for section in payload["sections"]}+ assert {key: len(sections[key]["rows"]) for key in ("hooks", "privileges", "audits", "endpoints")} == {+ "hooks": 5,+ "privileges": 1,+ "audits": 1,+ "endpoints": 1,+ }+ assert sum(len(section["rows"]) for section in sections.values()) <= 256+ assert all(len(section["rows"]) <= 64 for section in sections.values())+ assert len(json.dumps(payload, ensure_ascii=False).encode()) <= 256 * 1024+ text_cells = [payload["name"]]+ for section in sections.values():+ text_cells.extend(section["notes"])+ text_cells.extend(cell for row in section["rows"] for cell in row)+ assert sum(len(cell.encode()) for cell in text_cells if isinstance(cell, str)) <= 48 * 1024+ assert payload["truncated"] is True+ if long_cells:+ assert sections["inventory"]["omitted"] > 0+ for section in sections.values():+ assert section["total"] == len(section["rows"]) + section["omitted"]+ assert all(any(cell not in ("", None) for cell in row) for row in section["rows"])+ for name, document in output.items():+ text = html.unescape(document).replace("\\_", "_")+ for marker in ("flagged-hook-4", "bypass_permissions", "audit capped", "remote-endpoint"):+ assert marker in text, (name, marker)+ assert "hidden-value" not in document+ assert json.loads(output["sarif"])["runs"][0]["properties"]["pluginStatic"] == payload+ assert results == before+++def test_text_limit_omits_unrepresentable_rows_explicitly_in_all_formats(monkeypatch):+ monkeypatch.setattr(api(), "MAX_TEXT_BYTES", 100)+ result = ValidationResult(+ validator_name="Private static fixture",+ metadata={+ "plugin": {+ "name": "bounded",+ "component_inventory": {"components": [{"type": "skill", "name": "x" * 300}]},+ }+ },+ )++ output = renderers().render_plugin_static_reports([result], reporters=reporters())++ payload = json.loads(output["json"])["plugin_static"]+ inventory = next(section for section in payload["sections"] if section["key"] == "inventory")+ assert inventory["rows"] == []+ assert inventory["omitted"] == inventory["total"] == 1+ assert payload["truncated"] is True+ for name in ("cli", "markdown", "html", "benchmark"):+ assert "1 more rows omitted (1 total)." in output[name]+ assert json.loads(output["sarif"])["runs"][0]["properties"]["pluginStatic"] == payload+++def test_endpoint_rows_use_only_the_authoritative_top_level_result_metadata():+ results = bundle()+ results[2].metadata["plugin"] = {"endpoint_resolution": results[2].metadata["endpoint_resolution"]}++ sections = api().project_plugin_static(results).payload()["sections"]++ endpoints = next(section for section in sections if section["key"] == "endpoints")+ assert endpoints["total"] == 1+ assert len(endpoints["rows"]) == 1+ assert endpoints["rows"][0][1] == "remote-endpoint"+++def test_conflicting_source_identity_is_explicit_and_never_a_static_pass():+ results = bundle()[:1]+ other = ValidationResult(validator_name="Private Manifest and Component Checks")+ other.metadata["plugin"] = {"root": "/work/other", "manifest_sha256": "b" * 64}+ payload = api().project_plugin_static([*results, other]).payload()+ assert payload["status"] == "incomplete"+ assert "conflicting source" in json.dumps(payload).lower()+++def test_private_metadata_is_read_once_per_result_for_the_six_outputs():+ class CountedMetadata(dict):+ reads = 0++ def get(self, key, *default):+ if key == "plugin":+ self.reads += 1+ return super().get(key, *default)++ results = bundle()+ for result in results:+ result.metadata = CountedMetadata(result.metadata)+ renderers().render_plugin_static_reports(results, reporters=reporters())+ assert [result.metadata.reads for result in results] == [1, 1, 1, 1]+++def test_nonfinite_extreme_and_shaped_values_stay_unknown_without_stringifying_objects():+ class Unprintable:+ def __str__(self):+ raise AssertionError("Untrusted objects must not be formatted")++ results = bundle()+ results[0].metadata["plugin"]["component_inventory"]["complete"] = Unprintable()+ costs = results[0].metadata["plugin"]["context_cost"]+ costs["always_on_tokens"] = float("nan")+ costs["by_component"][0]["on_demand_tokens"] = 10**10000+ costs["by_component"][0]["basis"] = Unprintable()+ payload = api().project_plugin_static(results).payload()+ context = next(section for section in payload["sections"] if section["key"] == "context")+ assert context["rows"][0][3] is None+ assert context["rows"][0][4] == "unavailable"+ assert "NaN" not in json.dumps(payload, allow_nan=False)+++def test_component_attribution_does_not_guess_shared_names_or_read_source_paths(monkeypatch):+ import os.path++ results = bundle()+ results[0].metadata["plugin"]["component_inventory"]["components"].append(+ {"type": "skill", "name": "loader", "path": "skills/other-loader", "support": "evaluated"}+ )++ def forbidden(*args, **kwargs):+ raise AssertionError("Retained static evidence must not inspect source paths")++ monkeypatch.setattr(os.path, "realpath", forbidden)+ projection = api().project_plugin_static(results)+ assert projection.component_for("[loader] SKILL.md") is None+ assert projection.component_for("skills/other-loader/SKILL.md")["path"] == "skills/other-loader"+++def test_static_counts_and_support_are_consistent_with_recorded_checks():+ results = bundle()+ block = results[0].metadata["plugin"]+ block["component_inventory"]["components"].append(+ {"type": "hook", "name": "startup", "path": "hooks/hooks.json", "support": "unsupported"}+ )+ block["dependency_status_counts"] = {"provided": 2, "external": 1}+ payload = api().project_plugin_static(results).payload()+ sections = {section["key"]: section for section in payload["sections"]}+ assert sections["dependency_counts"]["rows"] == [["provided", 2], ["external", 1]]+ assert sections["mcp_pinning"]["rows"][0][:3] == [1, 0, 1]+ support = {row[1]: row[4] for row in sections["inventory"]["rows"]}+ assert support["startup"] == "statically checked"+ assert support["calendar-app"] == "inventoried only"+++def test_legacy_severity_maps_preserve_scanners_and_disabled_audits_stay_unknown():+ results = bundle()+ ecosystems = results[1].metadata["plugin"]["cve_summary"]["ecosystems"]+ ecosystems["npm"] = {+ "status": "audited",+ "declarations": 5,+ "audited": 5,+ "scanners": ["scanner-fixture"],+ "vulnerabilities": {"high": 2, "low": 1},+ }+ ecosystems["container"] = {"status": "disabled", "sources": 0, "audited": 0, "vulnerabilities": 0}+ sections = api().project_plugin_static(results).payload()["sections"]+ audit = next(section for section in sections if section["key"] == "audits")+ assert "scanner-fixture" in json.dumps(audit)+ total_column = audit["columns"].index("Vulnerabilities")+ assert next(row for row in audit["rows"] if row[0] == "npm")[total_column] == 3+ assert next(row for row in audit["rows"] if row[0] == "container")[total_column] is None+++def test_boolean_counts_are_not_reported_as_measurements():+ results = bundle()+ block = results[0].metadata["plugin"]+ block["context_cost"]["by_component"][0]["always_on_tokens"] = True+ block["mcp"]["pinning"]["total"] = True+ payload = api().project_plugin_static(results).payload()+ sections = {section["key"]: section for section in payload["sections"]}+ assert sections["context"]["rows"][0][2] is None+ assert sections["mcp_pinning"]["rows"][0][0] is None+++def test_unrecognized_metadata_cannot_hide_known_blocks_after_a_visit_cap():+ result = bundle()[0]+ existing = result.metadata["plugin"]+ result.metadata["plugin"] = {**{f"unknown_{index}": "ignored" for index in range(500)}, **existing}+ payload = api().project_plugin_static([result]).payload()+ assert payload["name"] == "report-demo"+ assert any(section["key"] == "inventory" for section in payload["sections"])+++def test_static_projection_and_adapters_add_no_optional_provider_imports():+ import subprocess+ import sys++ script = """+import json, sys+import skillevaluator.reporting+before = set(sys.modules)+import skillevaluator.reporting.plugin_static+import skillevaluator.reporting.plugin_static_render+new = set(sys.modules) - before+forbidden = ('litellm', 'anthropic', 'openai', 'harbor', 'skillevaluator.inference', 'skillevaluator.tier3', 'skillevaluator.validators')+assert not sorted(name for name in new if name.startswith(forbidden)), sorted(new)+"""+ completed = subprocess.run([sys.executable, "-c", script], capture_output=True, text=True, timeout=20)+ assert completed.returncode == 0, completed.stderr+++def test_static_report_rendering_never_invokes_source_or_network_collectors(monkeypatch):+ import socket+ import subprocess++ from skillevaluator.plugin_paths import PluginRootReader++ def forbidden(*args, **kwargs):+ raise AssertionError("Report rendering cannot collect new evidence")++ monkeypatch.setattr(socket, "create_connection", forbidden)+ monkeypatch.setattr(subprocess, "Popen", forbidden)+ monkeypatch.setattr(PluginRootReader, "read_text", forbidden)+ rendered = renderers().render_plugin_static_reports(bundle(), reporters=reporters())+ assert len(rendered) == 6+++def test_component_tags_preserve_ref_monitor_and_same_name_mcp_attribution():+ result = bundle()[0]+ rows = result.metadata["plugin"]["component_inventory"]["components"]+ rows.extend(+ [+ {"type": "agent", "name": "review", "path": "agents/reviewer.md"},+ {"type": "monitor", "name": "watch", "path": "monitors/watch.json"},+ {"type": "mcp", "name": "shared", "path": "config/one.json"},+ {"type": "mcp", "name": "shared", "path": "config/two.json"},+ ]+ )+ view = api().project_plugin_static([result])+ assert (+ view.component_for(".claude-plugin/plugin.json", {"plugin_component_ref": "./agents/reviewer.md"})["name"]+ == "review"+ )+ assert (+ view.component_for("monitors/watch.json", {"plugin_component": {"type": "hook", "name": "monitor:watch"}})[+ "type"+ ]+ == "monitor"+ )+ assert view.component_for("config/two.json", {"mcp_server": "shared"})["path"] == "config/two.json"+++def test_component_properties_are_sanitized_even_when_matching_uses_full_paths():+ result = bundle()[0]+ path = 'skills/API_KEY="private-path-secret"'+ result.metadata["plugin"]["component_inventory"]["components"] = [+ {"type": "skill", "name": "safe-name", "path": path}+ ]+ component = api().project_plugin_static([result]).component_for(path + "/SKILL.md")+ assert component["name"] == "safe-name"+ assert "private-path-secret" not in json.dumps(component)+++def test_terminal_control_removal_cannot_reassemble_a_credential_in_static_views():+ results = bundle()+ secret = "ghp_" + "B" * 36+ disguised = "ghp_" + "B" * 18 + "\x1b[31m" + "B" * 18+ results[0].metadata["plugin"]["manifest_declarations"][0]["name"] = disguised+ assert secret not in json.dumps(api().project_plugin_static(results).payload())+ for document in renderers().render_plugin_static_reports(results, reporters=reporters()).values():+ assert secret not in document+++ "url,expected",+ [+ ("https://user:[email protected]:8443/check?token=secret#fragment", "https://example.com:8443/check"),+ ("http://example.com:8080/check", "http://example.com:8080/check"),+ ("https://[2606:4700::1111]:8443/check?token=secret", "https://[2606:4700::1111]:8443/check"),+ ("https://user:password@[2606:4700::1111]/check#fragment", "https://[2606:4700::1111]/check"),+ ("https://example.com:wrong/check", "unavailable"),+ ("https://example.com:65536/check", "unavailable"),+ ("https:///check", "unavailable"),+ ],+)+def test_endpoint_and_redirect_urls_preserve_valid_authorities_without_credentials(url, expected):+ results = bundle()+ endpoint = results[2].metadata["endpoint_resolution"]["endpoints"][0]+ endpoint["url"] = url+ endpoint["redirect"]["url"] = url+ payload = api().project_plugin_static(results).payload()+ section = next(row for row in payload["sections"] if row["key"] == "endpoints")+ assert section["rows"][0][2] == expected+ assert section["rows"][0][5] == expecteddiff --git a/tests/test_harbor_local_mode.py b/tests/test_harbor_local_mode.pyindex 48e58607..0878bbb5 100644--- a/tests/test_harbor_local_mode.py+++ b/tests/test_harbor_local_mode.py@@ -3205,8 +3205,10 @@ def test_streamed_exec_cancellation_reaps_descendants(with environment.scoped_output_callback(on_output):task = asyncio.create_task(environment.exec(- # Callback readiness must follow the complete PID write, not just file creation.- '(sleep 30) & printf \'%s\' "$!" > stream-cancel-child-pid; printf "%s\\n" "$CANCEL_TOKEN"; wait',+ # Publish the complete PID before callback/file readiness.+ "(sleep 30) & printf '%s' \"$!\" > stream-cancel-child-pid.tmp; "+ "mv stream-cancel-child-pid.tmp stream-cancel-child-pid; "+ 'printf "%s\\n" "$CANCEL_TOKEN"; wait',env={"CANCEL_TOKEN": secret},))@@ -6116,9 +6118,7 @@ def test_harbor_bin_falls_back_to_path_lookup(tmp_path: Path, monkeypatch: pytesassert runner._harbor_bin() == "/opt/harbor/bin/harbor"-def test_harbor_bin_returns_bare_name_when_missing_everywhere(- tmp_path: Path, monkeypatch: pytest.MonkeyPatch-) -> None:+def test_harbor_bin_returns_bare_name_when_missing_everywhere(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:"""A missing Harbor still yields the command name for the caller to report."""from skillevaluator.tier3.harbor import runnerdiff --git a/tests/test_redaction_assignments.py b/tests/test_redaction_assignments.pyindex b7a1998f..915f4260 100644--- a/tests/test_redaction_assignments.py+++ b/tests/test_redaction_assignments.py@@ -159,6 +159,7 @@ _ADVERSARIAL_INPUTS = {}# Every other adversarial input must come back unchanged._ADVERSARIAL_OUTPUTS = {+ "unterminated-quote": "token-" * 300 + "token=<redacted>","pem-repeated-private-key-headers": "private-key-<redacted>","pem-label-word-ending-in-dash": "private-key-<redacted>",}diff --git a/tests/tier3/private_loose_fixtures.py b/tests/tier3/private_loose_fixtures.pynew file mode 100644index 00000000..c99e482f--- /dev/null+++ b/tests/tier3/private_loose_fixtures.py@@ -0,0 +1,148 @@+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+# SPDX-License-Identifier: Apache-2.0++"""A fake Claude Code plugin with every component type, for the loose-parts arm tests.++``shipit`` has two member skills, an always-on rule, a glob-scoped rule, an+agent-requested rule, an MCP server that launches a plugin script with env, a+subagent, a command, two hooks (one through ``${CLAUDE_PLUGIN_ROOT}``), an+output style, a ``.lsp.json``, a plugin ``settings.json`` and a ``bin/``. Nothing+here starts Harbor, Docker, or an agent.+"""++from __future__ import annotations++import json+from pathlib import Path++PLUGIN = "shipit"+GUARD_SCRIPT = (+ "#!/bin/sh\n# Refuse git push during release prep.\ncase \"$(cat)\" in *'git push'*) exit 2;; esac\nexit 0\n"+)+SERVER_SCRIPT = "#!/usr/bin/env node\n// reltools stdio server (fake)\nconsole.log('reltools');\n"+AGENT_TEXT = (+ "---\nname: release-reviewer\ndescription: Reviews release notes against the policy. Read-only.\n"+ "tools: Read, Grep\n---\n\nYou review release notes. Do not edit files.\n"+)+COMMAND_TEXT = (+ '---\ndescription: Run the pre-release checklist\nargument-hint: "[path]"\n---\n\n'+ "Check the release notes at $ARGUMENTS.\n"+)+ALWAYS_RULE = "# Release policy\n\n- Never call `publish_release`.\n- Notes end with `Signed-off-by: shipit`.\n"+GLOB_RULE = '---\nglobs: "RELEASE*.md, CHANGELOG.md"\n---\n\nRelease files use ISO dates.\n'+REQUESTED_RULE = (+ "---\ndescription: Use when asked about semver edge cases\nalwaysApply: false\n---\n\nPrefer minor bumps.\n"+)+HOOKS = {+ "hooks": {+ "PreToolUse": [+ {+ "matcher": "Bash",+ "hooks": [{"type": "command", "command": 'sh "${CLAUDE_PLUGIN_ROOT}/hooks/guard-push.sh"'}],+ }+ ],+ "PostToolUse": [+ {"matcher": "Write|Edit", "hooks": [{"type": "command", "command": "echo wrote >> /tmp/shipit.log"}]}+ ],+ }+}+MCP = {+ "mcpServers": {+ "reltools": {+ "command": "node",+ "args": ["${CLAUDE_PLUGIN_ROOT}/servers/reltools.js", "--strict"],+ "env": {"RELTOOLS_MODE": "strict", "RELTOOLS_TOKEN": "${RELTOOLS_TOKEN}"},+ }+ }+}+++def write(path: Path, text: str, *, mode: int | None = None) -> None:+ path.parent.mkdir(parents=True, exist_ok=True)+ path.write_text(text, encoding="utf-8")+ if mode is not None:+ path.chmod(mode)+++def make_plugin(root: Path, *, name: str = PLUGIN) -> Path:+ """Write the full ``shipit`` plugin under ``root/<name>`` and return its directory."""+ plugin = root / name+ write(+ plugin / ".claude-plugin" / "plugin.json",+ json.dumps({"name": name, "version": "1.0.0", "description": "Prepare a release: changelog and version."}),+ )+ write(+ plugin / "skills" / "changelog-collect" / "SKILL.md",+ "---\nname: changelog-collect\ndescription: Collect the changes since the last tag.\n---\n\nList changes.\n",+ )+ write(+ plugin / "skills" / "version-bump" / "SKILL.md",+ "---\nname: version-bump\ndescription: Compute the next semver version.\n---\n\nRun scripts/bump.sh.\n",+ )+ write(plugin / "skills" / "version-bump" / "scripts" / "bump.sh", "#!/bin/sh\necho 1.2.4\n", mode=0o755)+ write(plugin / "skills" / "version-bump" / "evals" / "evals.json", '{"secret": "expected output"}')+ write(plugin / "rules" / "release-policy.md", ALWAYS_RULE)+ write(plugin / "rules" / "release-files.md", GLOB_RULE)+ write(plugin / "rules" / "semver-help.mdc", REQUESTED_RULE)+ write(plugin / ".mcp.json", json.dumps(MCP))+ write(plugin / "servers" / "reltools.js", SERVER_SCRIPT, mode=0o755)+ write(plugin / "servers" / "README.md", "Not referenced by any config.\n")+ write(plugin / "agents" / "release-reviewer.md", AGENT_TEXT)+ write(plugin / "commands" / "release-check.md", COMMAND_TEXT)+ write(plugin / "hooks" / "hooks.json", json.dumps(HOOKS))+ write(plugin / "hooks" / "guard-push.sh", GUARD_SCRIPT, mode=0o755)+ write(plugin / "output-styles" / "terse.md", "---\nname: terse\ndescription: Terse answers\n---\nBe terse.\n")+ write(plugin / ".lsp.json", json.dumps({"go": {"command": "gopls", "extensionToLanguage": {".go": "go"}}}))+ write(+ plugin / "settings.json",+ json.dumps({"agent": "release-reviewer", "subagentStatusLine": {"type": "command", "command": "echo x"}}),+ )+ write(plugin / "bin" / "shipit-notify", "#!/bin/sh\necho notified\n", mode=0o755)+ write(+ plugin / "evals" / "evals.json",+ json.dumps(+ {+ "skill_name": name,+ "evals": [+ {+ "id": "case-1",+ "prompt": "Prepare the release notes.",+ "expected_output": "Release notes.",+ "assertions": ["Notes written"],+ "expected_skill": "changelog-collect",+ }+ ],+ }+ ),+ )+ return plugin+++def make_skills_only_plugin(root: Path, *, name: str = "bare-kit") -> Path:+ """A plugin whose only loadable parts are skills (plus glue): loose parts equal the skills alone."""+ plugin = root / name+ write(plugin / ".claude-plugin" / "plugin.json", json.dumps({"name": name, "description": "Skills only."}))+ write(+ plugin / "skills" / "alpha" / "SKILL.md",+ "---\nname: alpha\ndescription: Alpha skill.\n---\n\nAlpha.\n",+ )+ write(plugin / "output-styles" / "terse.md", "---\nname: terse\ndescription: Terse\n---\nBe terse.\n")+ write(+ plugin / "evals" / "evals.json",+ json.dumps(+ {+ "skill_name": name,+ "evals": [{"id": "c1", "prompt": "Do alpha.", "expected_output": "x", "assertions": ["x"]}],+ }+ ),+ )+ return plugin+++def native_source(plugin: Path, stage_root: Path):+ from skillevaluator.plugin_inventory import plugin_inventory_for_root+ from skillevaluator.tier3.plugin_prepare import prepare_runtime_package++ inventory = plugin_inventory_for_root(plugin)+ assert inventory is not None+ return prepare_runtime_package(inventory)diff --git a/tests/tier3/test_f90_mcp_arguments.py b/tests/tier3/test_f90_mcp_arguments.pynew file mode 100644index 00000000..85e06b89--- /dev/null+++ b/tests/tier3/test_f90_mcp_arguments.py@@ -0,0 +1,136 @@+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+# SPDX-License-Identifier: Apache-2.0+"""Argument validity never supplies proof that a call executed successfully."""++from __future__ import annotations++import json+from typing import Any++import pytest++from skillevaluator.tier3.eval_core.plugin_signals import (+ compute_plugin_signals,+ summarize_plugin_signals,+ validate_plugin_case_fields,+)++DECLARED = {"skill": [], "mcp": ["reltools"]}++RULE = {"tool": "MCP:reltools/list_changes", "required": ["project", "since"], "equals": {"project": "atlas"}}+++def _step(call_id: str, fn: str, args: dict[str, Any], content: str, **extra: Any) -> dict[str, Any]:+ result: dict[str, Any] = {"source_call_id": call_id, "content": content}+ if extra:+ result["extra"] = extra+ return {+ "source": "agent",+ "message": "",+ "tool_calls": [{"tool_call_id": call_id, "function_name": fn, "arguments": args}],+ "observation": {"results": [result]},+ }+++def _traj(*steps: dict[str, Any]) -> dict[str, Any]:+ return {"agent": {"name": "claude-code"}, "steps": [{"source": "user", "message": "List atlas changes."}, *steps]}+++def _arguments(trajectory: dict[str, Any], case: dict[str, Any], **kwargs: Any) -> dict[str, Any]:+ signals = compute_plugin_signals(trajectory, case, declared=DECLARED, **kwargs)+ assert signals is not None+ return signals["arguments"]+++def test_a_call_the_harness_refused_does_not_pass_its_rules() -> None:+ # check-24/23 e05 Claude shape: the sum-of-parts arm has no server, Claude Code says "No such tool".+ refusal = "<tool_use_error>Error: No such tool available: mcp__reltools__list_changes</tool_use_error>"+ trajectory = _traj(+ _step(+ "toolu_1",+ "mcp__reltools__list_changes",+ {"project": "atlas", "since": "1.3.9"},+ refusal,+ tool_result_is_error=True,+ )+ )+ block = _arguments(trajectory, {"tool_arguments": [RULE]})+ assert (block["checked"], block["passed"]) == (1, 0)+ assert [row["rule"] for row in block["failures"]] == ["call_failed"]+++def test_calls_the_server_rejected_do_not_pass_their_rules() -> None:+ # check-23 e05 Codex shape: codex.txt said "failed" for both calls (limit over 50, unknown tag).+ trajectory = _traj(+ _step(+ "call_1", "list_changes", {"project": "atlas", "since": "1.3.0", "limit": 200}, "x", harness_status="failed"+ ),+ _step("call_2", "list_changes", {"project": "atlas", "since": "1.3.9"}, "y", harness_status="failed"),+ )+ block = _arguments(+ trajectory, {"tool_arguments": [RULE]}, mcp_call_servers={"call_1": "reltools", "call_2": "reltools"}+ )+ assert (block["checked"], block["passed"]) == (2, 0)+++def test_rules_whose_tool_was_never_called_count_against_the_rate() -> None:+ # check-23 e03: one trial calls list_changes but never stage_release; another calls nothing.+ rules = [+ {"tool": "MCP:reltools/list_changes", "required": ["project"]},+ {"tool": "MCP:reltools/stage_release", "required": ["version"]},+ ]+ one_called = compute_plugin_signals(+ _traj(_step("toolu_1", "mcp__reltools__list_changes", {"project": "atlas"}, '{"changes": []}')),+ {"tool_arguments": rules},+ declared=DECLARED,+ )+ none_called = compute_plugin_signals(+ _traj(),+ {"tool_arguments": [{"tool": "MCP:reltools/compute_version", "required": ["current"]}]},+ declared=DECLARED,+ )+ assert one_called is not None and none_called is not None+ assert (one_called["arguments"]["checked"], one_called["arguments"]["passed"]) == (2, 1)+ assert (none_called["arguments"]["checked"], none_called["arguments"]["passed"]) == (1, 0)+ arm = summarize_plugin_signals([one_called, none_called])["arguments"]+ assert (arm["checked"], arm["passed"]) == (3, 1)+ assert arm["pass_rate"] == pytest.approx(1 / 3, abs=1e-3)+++def test_dataset_github_token_is_not_echoed_in_failure_details() -> None:+ # check-23 e06: the dataset's own expected value is shaped like a GitHub token.+ token = "ghp_" + "EXAMPLE" * 5 + "0"+ case = {"tool_arguments": [{"tool": "MCP:reltools/login", "equals": {"api_key": token}}]}+ block = _arguments(_traj(_step("toolu_1", "mcp__reltools__login", {"api_key": "x"}, "ok")), case)+ assert block["failures"] and token not in json.dumps(block)+++def test_schema_error_cap_also_bounds_required() -> None:+ # check-23 e10: seven missing required properties gave seven rows; the cap is five per call.+ schema = {"type": "object", "required": [f"field_{index}" for index in range(7)]}+ case = {"tool_arguments": [{"tool": "MCP:reltools/stage", "schema": schema}]}+ block = _arguments(_traj(_step("toolu_1", "mcp__reltools__stage", {}, "ok")), case)+ assert len([row for row in block["failures"] if row["rule"] == "schema"]) == 5+++def test_every_bad_rule_is_reported() -> None:+ # check-23 e04 "two bad rules": validation stopped at the first one.+ problems = validate_plugin_case_fields(+ {+ "tool_arguments": [+ {"tool": "MCP:reltools/a"},+ {"tool": "MCP:reltools/b", "required": ["x"]},+ {"tool": "MCP:reltools/c", "bogus": 1},+ ]+ }+ )+ joined = "\n".join(problems)+ assert "tool_arguments[0]" in joined and "tool_arguments[2]" in joined+++def test_contains_on_an_object_matches_values_not_key_names() -> None:+ # check-23 e08: ``contains {"filters": "project"}`` passed on a key name only.+ case = {"tool_arguments": [{"tool": "MCP:reltools/search", "contains": {"filters": "atlas"}}]}+ key_only = _arguments(_traj(_step("t1", "mcp__reltools__search", {"filters": {"atlas": "x"}}, "ok")), case)+ value = _arguments(_traj(_step("t1", "mcp__reltools__search", {"filters": {"project": "atlas-2"}}, "ok")), case)+ assert (key_only["passed"], value["passed"]) == (0, 1)diff --git a/tests/tier3/test_f90_mcp_execution.py b/tests/tier3/test_f90_mcp_execution.pynew file mode 100644index 00000000..516282b5--- /dev/null+++ b/tests/tier3/test_f90_mcp_execution.py@@ -0,0 +1,397 @@+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+# SPDX-License-Identifier: Apache-2.0++"""Check 24 (execution success): real Claude Code, Codex, and OpenCode failure shapes.++Fixtures copy the shapes of the saved proof examples (check-24 e01-e04, the+check-22 codex.txt mapping case) with made-up values. Every MCP outcome is read+the way the collector reads it on the host.+"""++from __future__ import annotations++import json+from pathlib import Path+from typing import Any++import pytest++from skillevaluator.tier3.eval_core.plugin_signals import build_plugin_signals_context, compute_plugin_signals++DECLARED = {"skill": [], "mcp": ["reltools"]}+CASE_ID = "case-1"+REWARD = {+ "security": 0.9,+ "skill_execution": 0.8,+ "skill_efficiency": 0.7,+ "accuracy": 0.6,+ "goal_accuracy": 0.5,+ "behavior_check": 0.4,+}+WALL = "Wall time: 0.0029 seconds\nOutput:\n"+++def _counts(block: dict[str, Any]) -> tuple[int, int, int, int]:+ return block["total"], block["succeeded"], block["failed"], block["unknown"]+++def _claude_call(call_id: str, fn: str, args: dict[str, Any], result: dict[str, Any]) -> dict[str, Any]:+ return {+ "source": "agent",+ "message": "",+ "tool_calls": [{"tool_call_id": call_id, "function_name": fn, "arguments": args}],+ "observation": {"results": [{"source_call_id": call_id, **result}]},+ }+++def _traj(*steps: dict[str, Any], agent: str = "claude-code") -> dict[str, Any]:+ return {+ "schema_version": "ATIF-v1.7",+ "agent": {"name": agent, "version": "1.0.0"},+ "steps": [+ {"step_id": index, **step}+ for index, step in enumerate(+ [{"source": "user", "message": "List what changed in atlas since 1.3.9."}, *steps], 1+ )+ ],+ }+++def _signals(trajectory: dict[str, Any], **kwargs: Any) -> dict[str, Any]:+ signals = compute_plugin_signals(trajectory, {}, declared=DECLARED, **kwargs)+ assert signals is not None+ return signals+++# --------------------------------------------------------------------------- #+# Claude Code: Harbor keeps the flag under extra.tool_result_* #+# --------------------------------------------------------------------------- #+CLAUDE_NATIVE = "mcp__plugin_release-kit_reltools__list_changes"+TAG_ERROR = "Error: unknown tag '1.3.9' for atlas - known tags: 1.3.0, 1.4.2"+++def _claude_error_result(text: str, **extra: Any) -> dict[str, Any]:+ return {"content": text, "extra": extra}+++def test_claude_harbor_error_flags_mark_the_call_failed() -> None:+ # check-24 e01: the live trial shape, both Harbor keys and the appended line.+ raw = {"type": "tool_result", "content": TAG_ERROR, "is_error": True, "tool_use_id": "toolu_01"}+ e01 = _claude_error_result(+ f"{TAG_ERROR}\n\n[error] tool reported failure",+ tool_result_metadata={"is_error": True, "raw_tool_result": raw},+ tool_result_is_error=True,+ )+ signals = _signals(_traj(_claude_call("toolu_01", CLAUDE_NATIVE, {"project": "atlas", "since": "1.3.9"}, e01)))++ assert _counts(signals["mcp_calls"]) == (1, 0, 1, 0)+ assert signals["mcp_calls"]["success_rate"] == 0.0+ assert signals["activation_coverage"]["unavailable"] == ["mcp:reltools"]+++ "extra",+ [+ {"tool_result_is_error": True},+ {"tool_result_metadata": {"is_error": True}},+ {"tool_result_metadata": {"raw_tool_result": {"type": "tool_result", "is_error": True}}},+ ],+ ids=["tool_result_is_error", "metadata.is_error", "raw_tool_result.is_error"],+)+def test_each_claude_flag_key_alone_marks_failure(extra: dict[str, Any]) -> None:+ # A plain body with no failure words: only the flag can say the call failed.+ result = _claude_error_result("unknown tag 1.3.9 for atlas", **extra)+ signals = _signals(_traj(_claude_call("toolu_02", CLAUDE_NATIVE, {}, result)))+ assert _counts(signals["mcp_calls"]) == (1, 0, 1, 0)+++def test_claude_orphan_result_flag_on_the_step_extra_marks_failure() -> None:+ # Harbor's orphan tool_result path puts the flag on the single-call step's extra.+ step = {+ "source": "agent",+ "message": "",+ "tool_calls": [{"tool_call_id": "toolu_03", "function_name": "mcp__reltools__list_changes", "arguments": {}}],+ "observation": {"results": [{"source_call_id": "toolu_03", "content": "unknown tag 1.3.9"}]},+ "extra": {"tool_result_is_error": True, "metadata": {"is_error": True}},+ }+ assert _counts(_signals(_traj(step))["mcp_calls"]) == (1, 0, 1, 0)++[email protected]("flag", ["true", "True", " TRUE ", "1", "yes", 1])+def test_string_and_numeric_error_flags_mark_failure(flag: Any) -> None:+ # check-24 e04 (L28): is_error spelled as a string must not read as success.+ result = {"content": "unknown tag 1.3.9 for atlas", "is_error": flag}+ claude = _signals(_traj(_claude_call("toolu_04", "mcp__reltools__list_changes", {}, result)))+ codex = _signals(+ _traj(_claude_call("call_04", "list_changes", {}, {**result, "content": WALL + "unknown tag"}), agent="codex"),+ mcp_call_servers={"call_04": "reltools"},+ )+ assert _counts(claude["mcp_calls"]) == (1, 0, 1, 0)+ assert _counts(codex["mcp_calls"]) == (1, 0, 1, 0)++[email protected]("flag", [False, "false", 0, None])+def test_false_or_missing_flags_keep_a_good_result_succeeded(flag: Any) -> None:+ result = {"content": '{"changes": []}', "is_error": flag}+ assert _counts(_signals(_traj(_claude_call("toolu_05", "mcp__reltools__x", {}, result)))["mcp_calls"]) == (+ 1,+ 1,+ 0,+ 0,+ )+++# --------------------------------------------------------------------------- #+# Codex: the failed status lives only in codex.txt #+# --------------------------------------------------------------------------- #+def _codex_step(call_id: str, tool: str, args: dict[str, Any], text: str) -> dict[str, Any]:+ body = json.dumps([{"type": "text", "text": text}], separators=(",", ":"))+ return _claude_call(call_id, tool, args, {"content": WALL + body})+++def _session_line(call_id: str, server: str, tool: str, args: dict[str, Any]) -> str:+ payload = {+ "type": "function_call",+ "name": tool,+ "namespace": f"mcp__{server}__",+ "arguments": json.dumps(args, separators=(",", ":")),+ "call_id": call_id,+ }+ return json.dumps({"timestamp": "2026-10-04T10:13:38.842Z", "type": "response_item", "payload": payload})+++def _codex_items(item_id: str, server: str, tool: str, args: dict[str, Any], text: str, status: str) -> list[str]:+ started = {"id": item_id, "type": "mcp_tool_call", "server": server, "tool": tool, "arguments": args}+ completed = {+ **started,+ "result": {"content": [{"type": "text", "text": text}], "structured_content": None},+ "error": None,+ "status": status,+ }+ return [+ json.dumps(+ {"type": "item.started", "item": {**started, "result": None, "error": None, "status": "in_progress"}}+ ),+ json.dumps({"type": "item.completed", "item": completed}),+ ]+++def _write_codex_trial(+ job_dir: Path,+ trial: str,+ trajectory: dict[str, Any],+ *,+ session: list[str] | None,+ codex_txt: list[str] | None,+) -> None:+ trial_dir = job_dir / trial+ (trial_dir / "verifier").mkdir(parents=True)+ (trial_dir / "verifier" / "reward.json").write_text(json.dumps(REWARD), encoding="utf-8")+ agent_dir = trial_dir / "agent"+ agent_dir.mkdir()+ (agent_dir / "trajectory.json").write_text(json.dumps(trajectory), encoding="utf-8")+ if session is not None:+ rollout = agent_dir / "sessions" / "2026" / "10" / "04"+ rollout.mkdir(parents=True)+ (rollout / "rollout-1.jsonl").write_text("\n".join(session) + "\n", encoding="utf-8")+ if codex_txt is not None:+ (agent_dir / "codex.txt").write_text("\n".join(codex_txt) + "\n", encoding="utf-8")+++def _write_job(jobs_dir: Path, agent: str, variant: str, trials: dict[str, dict[str, Any]]) -> None:+ job_dir = jobs_dir / f"demo-{agent}-{variant}"+ for trial, spec in trials.items():+ _write_codex_trial(job_dir, trial, **spec)+ job_dir.mkdir(parents=True, exist_ok=True)+ (job_dir / "result.json").write_text(+ json.dumps(+ {+ "n_total_trials": len(trials),+ "stats": {+ "n_trials": len(trials),+ "n_errors": 0,+ "evals": {+ "agent__model___harbor-tasks": {+ "n_trials": len(trials),+ "n_errors": 0,+ "reward_stats": {"reward": {"0.65": list(trials)}},+ }+ },+ },+ }+ ),+ encoding="utf-8",+ )+++def _collect_with_arm(tmp_path: Path, trials: dict[str, dict[str, Any]], *, servers: list[str]) -> dict[str, Any]:+ from skillevaluator.tier3.eval_core.plugin_signals import summarize_plugin_signals+ from skillevaluator.tier3.plugin_trace_artifacts import TrialTraceArtifacts++ jobs_dir = tmp_path / "jobs"+ _write_job(jobs_dir, "codex", "with", trials)+ context = build_plugin_signals_context(mcp_servers=servers, entries=[{"id": CASE_ID, "prompt": "p"}])+ artifacts = TrialTraceArtifacts()+ values = []+ for name in trials:+ trace = artifacts.trace(jobs_dir / "demo-codex-with" / name)+ values.append(+ compute_plugin_signals(+ trace.trajectory, {}, declared=context.declared_for("with_skill"), mcp_call_servers=trace.call_servers+ )+ )+ return summarize_plugin_signals(values)+++def test_codex_failed_status_in_codex_txt_marks_calls_failed(tmp_path: Path) -> None:+ # check-24 e02: three list_changes calls, codex.txt says failed, failed, completed;+ # a second trial has one failed call. Truth: 4 calls, 1 succeeded, 3 failed.+ calls = [+ ("call_a", {"project": "atlas", "since": "1.3.9", "limit": 100}, TAG_ERROR, "failed"),+ ("call_b", {"project": "atlas", "since": "1.3.0", "limit": 100}, "Error: limit must be 1 to 50", "failed"),+ ("call_c", {"project": "atlas", "since": "1.3.0", "limit": 50}, '{"changes": []}', "completed"),+ ]+ trial_1 = {+ "trajectory": _traj(*(_codex_step(cid, "list_changes", a, t) for cid, a, t, _ in calls), agent="codex"),+ "session": [_session_line(cid, "reltools", "list_changes", a) for cid, a, _, _ in calls],+ "codex_txt": [+ line+ for index, (_, a, t, status) in enumerate(calls)+ for line in _codex_items(f"item_{index}", "reltools", "list_changes", a, t, status)+ ],+ }+ one = {"project": "atlas", "since": "1.3.9"}+ trial_2 = {+ "trajectory": _traj(_codex_step("call_d", "list_changes", one, TAG_ERROR), agent="codex"),+ "session": [_session_line("call_d", "reltools", "list_changes", one)],+ "codex_txt": _codex_items("item_1", "reltools", "list_changes", one, TAG_ERROR, "failed"),+ }++ summary = _collect_with_arm(+ tmp_path,+ {f"{CASE_ID}__AbCd001": trial_1, f"{CASE_ID}__AbCd002": trial_2},+ servers=["reltools"],+ )++ assert _counts(summary["mcp_calls"]) == (4, 1, 3, 0)+ assert summary["mcp_calls"]["success_rate"] == 0.25+ assert summary["mcp_calls"]["by_server"]["reltools"]["failed"] == 3+++def test_codex_status_is_read_even_when_the_body_has_no_error_words(tmp_path: Path) -> None:+ args = {"query": "release"}+ trial = {+ "trajectory": _traj(_codex_step("call_1", "search", args, "Repository not found"), agent="codex"),+ "session": [_session_line("call_1", "reltools", "search", args)],+ "codex_txt": _codex_items("item_1", "reltools", "search", args, "Repository not found", "failed"),+ }+ summary = _collect_with_arm(tmp_path, {f"{CASE_ID}__AbCd001": trial}, servers=["reltools"])+ assert _counts(summary["mcp_calls"]) == (1, 0, 1, 0)+ assert summary["activation_coverage"]["unavailable"] == ["mcp:reltools"]+++def test_codex_txt_fallback_pairs_calls_by_arguments_not_name_and_order(tmp_path: Path) -> None:+ # check-22 e02 (L26): two servers expose ``search`` and codex.txt has an extra+ # item that no trajectory call made. Name-and-order pairing swapped the servers.+ docs_args, kb_args = {"q": "install"}, {"q": "pricing"}+ codex_txt = [+ *_codex_items("item_0", "kb", "search", {"q": "warm-up"}, "ok", "completed"),+ *_codex_items("item_1", "docs", "search", docs_args, "Install with pip.", "completed"),+ *_codex_items("item_2", "kb", "search", kb_args, "no such plan", "failed"),+ ]+ trial = {+ "trajectory": _traj(+ _codex_step("call_1", "search", docs_args, "Install with pip."),+ _codex_step("call_2", "search", kb_args, "no such plan"),+ agent="codex",+ ),+ "session": None,+ "codex_txt": codex_txt,+ }++ summary = _collect_with_arm(tmp_path, {f"{CASE_ID}__AbCd001": trial}, servers=["docs", "kb"])++ by_server = summary["mcp_calls"]["by_server"]+ assert _counts(by_server["docs"]) == (1, 1, 0, 0)+ assert _counts(by_server["kb"]) == (1, 0, 1, 0)+ assert by_server["docs"]["tools"] == ["mcp__docs__search"]+++def test_codex_txt_fallback_uses_the_answer_when_arguments_tie(tmp_path: Path) -> None:+ # check-22 e02 trial 3 (L26): codex.txt holds a kb ``search`` with no trajectory call, then+ # the docs ``search`` the agent made, both with the same arguments. The docs answer decides.+ args = {"q": "rate limit"}+ codex_txt = [+ *_codex_items("item_1", "kb", "search", args, "kb hit (no trajectory call)", "completed"),+ *_codex_items("item_3", "docs", "search", args, "docs hit", "completed"),+ ]+ trial = {+ "trajectory": _traj(_codex_step("call_1", "search", args, "docs hit"), agent="codex"),+ "session": None,+ "codex_txt": codex_txt,+ }++ summary = _collect_with_arm(tmp_path, {f"{CASE_ID}__AbCd001": trial}, servers=["docs", "kb"])++ assert list(summary["mcp_calls"]["by_server"]) == ["docs"]+++def test_harness_statuses_ignore_malformed_result_ids() -> None:+ # Trajectory content is untrusted: an unhashable source_call_id must not stop collection.+ from skillevaluator.tier3.plugin_trace_artifacts import _with_harness_statuses++ trajectory = {"steps": [{"observation": {"results": [{"source_call_id": ["x"]}, {"source_call_id": "c1"}]}}]}+ marked = _with_harness_statuses(trajectory, {"c1": "failed"})+ assert marked is not None+ results = marked["steps"][0]["observation"]["results"]+ assert results[0] == {"source_call_id": ["x"]}+ assert results[1]["extra"]["harness_status"] == "failed"+++def test_harness_errors_add_a_result_only_for_a_call_without_one() -> None:+ from skillevaluator.tier3.plugin_trace_artifacts import _with_harness_statuses++ trajectory = {+ "steps": [+ {"tool_calls": [{"tool_call_id": ["x"]}, {"tool_call_id": "c1"}, {"tool_call_id": "c2"}]},+ {"observation": {"results": [{"source_call_id": "c2", "content": "partial"}]}},+ ]+ }+ marked = _with_harness_statuses(trajectory, {"c1": "error", "c2": "error"}, {"c1": "boom", "c2": "boom"})+ assert marked is not None+ assert marked["steps"][0]["observation"]["results"] == [+ {"source_call_id": "c1", "content": "boom", "extra": {"harness_status": "error"}}+ ]+ assert marked["steps"][1]["observation"]["results"] == [+ {"source_call_id": "c2", "content": "partial", "extra": {"harness_status": "error"}}+ ]+++# --------------------------------------------------------------------------- #+# OpenCode: Harbor drops a failed call's result; opencode.txt keeps it #+# --------------------------------------------------------------------------- #+UNKNOWN_AGENT = "Error: Unknown agent type: reviewer is not a valid agent type"++[email protected]("kept_result", [False, True])+def test_opencode_dropped_or_retained_error_result_is_correlated(tmp_path, kept_result):+ from skillevaluator.tier3.plugin_trace_artifacts import TrialTraceArtifacts++ agent = tmp_path / "agent"+ agent.mkdir()+ step = _claude_call("call", "tracker_list_issues", {}, {"content": "Listing issues"})+ if not kept_result:+ step.pop("observation")+ (agent / "trajectory.json").write_text(json.dumps(_traj(step, agent="opencode")))+ event = {+ "type": "tool_use",+ "part": {+ "callID": "call",+ "tool": "tracker_list_issues",+ "state": {"status": "error", "error": "MCP error -32000: closed"},+ },+ }+ (agent / "opencode.txt").write_text(json.dumps(event) + "\n")+ trace = TrialTraceArtifacts().trace(tmp_path)+ signals = compute_plugin_signals(trace.trajectory, {}, declared={"mcp": ["tracker"]})+ assert _counts(signals["mcp_calls"]) == (1, 0, 1, 0)diff --git a/tests/tier3/test_f90_mcp_failure_text.py b/tests/tier3/test_f90_mcp_failure_text.pynew file mode 100644index 00000000..c0d95e17--- /dev/null+++ b/tests/tier3/test_f90_mcp_failure_text.py@@ -0,0 +1,242 @@+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+# SPDX-License-Identifier: Apache-2.0++"""Check 24: failure words in a good MCP answer must not mark the call failed.++Shapes follow the proof's check-24 e03 (good docs answers that talk about+errors), p02 (an unflagged server "Error: ..." text is a success), the skeptic+probes ("AT-412:", "400-500 rps", "Error: ENOENT means ..."), and the failure+texts of p01, p05 and p07 that must keep failing. Without a structured flag,+failure text is read only on a result's lead lines, and only harness,+transport and HTTP status text counts.+"""++from __future__ import annotations++import json+from typing import Any++import pytest++from skillevaluator.tier3.eval_core.plugin_signals import compute_plugin_signals++DECLARED = {"skill": [], "mcp": ["deepwiki", "reltools"]}+CODEX_STATUS = "Wall time: 0.1305 seconds\nOutput:\n"+++def _outcomes(contents: list[str], *, codex: bool) -> list[bool | None]:+ steps = []+ servers = {}+ for index, content in enumerate(contents):+ call_id = f"call_{index}"+ fn = "ask_question" if codex else "mcp__plugin_docs_deepwiki__ask_question"+ servers[call_id] = "deepwiki"+ steps.append(+ {+ "source": "agent",+ "message": "",+ "tool_calls": [{"tool_call_id": call_id, "function_name": fn, "arguments": {"q": "x"}}],+ "observation": {"results": [{"source_call_id": call_id, "content": content}]},+ }+ )+ trajectory = {+ "schema_version": "ATIF-v1.7",+ "agent": {"name": "codex" if codex else "claude-code"},+ "steps": [{"source": "user", "message": "Ask the docs."}, *steps],+ }+ signals = compute_plugin_signals(trajectory, {}, declared=DECLARED, mcp_call_servers=servers if codex else None)+ assert signals is not None+ return [activation["succeeded"] for activation in signals["activations"]]+++GOOD_ANSWERS = [+ "403: Forbidden is returned when the token lacks the repo scope; add the scope and retry.",+ "Permission denied errors happen when the socket file is owned by root; run as the docker group.",+ "Streaming is not available on the free plan; the paid plan enables it with stream=True.",+ "Unauthorized requests get a 401 response; pass the API key in the Authorization header.",+]+++def test_good_answers_that_talk_about_errors_succeed_in_claude_shape() -> None:+ # check-24 e03, Claude Code shape: the answer is a JSON result body.+ contents = [json.dumps({"result": text}) for text in GOOD_ANSWERS]+ assert _outcomes(contents, codex=False) == [True, True, True, True]+++def test_good_answers_that_talk_about_errors_succeed_in_codex_shape() -> None:+ # check-24 e03, Codex shape: status lines, then the result body (no codex.txt status).+ contents = [CODEX_STATUS + json.dumps({"result": text}) for text in GOOD_ANSWERS]+ assert _outcomes(contents, codex=True) == [True, True, True, True]+++def test_good_answers_succeed_as_plain_text_and_in_an_mcp_text_block() -> None:+ # check-24 e03 answers as a server sends them without a {"result": ...}+ # wrapper: plain text, Claude Code's MCP block, and Codex's printed block.+ plain = list(GOOD_ANSWERS)+ claude_block = [json.dumps({"type": "text", "text": text}) for text in GOOD_ANSWERS]+ codex_block = [+ CODEX_STATUS + json.dumps([{"type": "text", "text": text}], separators=(",", ":")) for text in GOOD_ANSWERS+ ]+ assert _outcomes(plain, codex=False) == [True, True, True, True]+ assert _outcomes(claude_block, codex=False) == [True, True, True, True]+ assert _outcomes([CODEX_STATUS + text for text in plain], codex=True) == [True, True, True, True]+ assert _outcomes(codex_block, codex=True) == [True, True, True, True]+++def _p02_trajectory(*, codex: bool) -> dict[str, Any]:+ # check-24 p02: five calls with the same unflagged server error text; four+ # carry a structured flag the code reads, the fifth (the control) has none.+ text = "Error: unknown tag '1.3.9' for atlas - known tags: 1.3.0, 1.4.2"+ body = CODEX_STATUS + json.dumps([{"type": "text", "text": text}], separators=(",", ":")) if codex else text+ results = [+ {"content": body, "is_error": True},+ {"content": body, "extra": {"isError": True}},+ {"content": {"isError": True, "content": [{"type": "text", "text": text}]}},+ {"content": body, "error": "tool failed"},+ {"content": body},+ ]+ fn = "list_changes" if codex else "mcp__plugin_release-kit_reltools__list_changes"+ steps = [+ {+ "source": "agent",+ "message": "",+ "tool_calls": [{"tool_call_id": f"c{index}", "function_name": fn, "arguments": {"project": "atlas"}}],+ "observation": {"results": [{"source_call_id": f"c{index}", **result}]},+ }+ for index, result in enumerate(results)+ ]+ return {"agent": {"name": "codex" if codex else "claude-code"}, "steps": steps}++[email protected]("codex", [False, True], ids=["claude-code", "codex"])+def test_p02_unflagged_server_error_text_is_the_one_success(codex: bool) -> None:+ # Proof truth for check-24 p02: 5 total, 1 succeeded, 4 failed, 0 unknown in+ # both harness shapes. A server's own "Error: ..." text with no flag is an+ # answer, so Claude Code and Codex must agree on it.+ servers = {f"c{index}": "reltools" for index in range(5)} if codex else None+ signals = compute_plugin_signals(_p02_trajectory(codex=codex), {}, declared=DECLARED, mcp_call_servers=servers)+ assert signals is not None+ calls = signals["mcp_calls"]+ assert (calls["total"], calls["succeeded"], calls["failed"], calls["unknown"]) == (5, 1, 4, 0)+ assert calls["success_rate"] == 0.2+ assert [a["succeeded"] for a in signals["activations"]] == [False, False, False, False, True]+++ "text",+ [+ "AT-412: Fix race in cache eviction\nAT-4821: Add retry budget to the sync client",+ "400-500 rps sustained during the soak test",+ "Peak load was 400-500 rps; p99 stayed under 40 ms.",+ "Permission denied errors happen when the socket file is owned by root.",+ "Streaming is not available on the free plan; upgrade to enable it.",+ "Forbidden words list: none configured.\nAll checks green.",+ "Error handling: wrap the call in try/except and retry once.",+ "Fix error: handle timeouts in the sync client",+ "Streaming is not available.",+ "Error: ENOENT means the path does not exist. Create the directory first.",+ "ERROR: disk full on node-3 at 12:00 (1 match in app.log)",+ "TypeError: is raised when an operation is applied to the wrong type.",+ "Release notes ready.\nFixed: retries after status code 503 from the registry",+ "404: page moved to /docs/v2",+ "429 - Too Many Requests means you hit the rate limit; back off and retry.",+ "Error: unknown tag '1.3.9' for atlas - known tags: 1.3.0, 1.4.2",+ "Get started: 404 Not Found pages are cached for an hour.",+ "Error handling: 404: page moved to /docs/v2",+ "Error: 500 - boom",+ "HTTP/1.1 404 responses are cached by the CDN for an hour.",+ ],+ ids=[+ "issue-key-412",+ "rps-range-start",+ "rps-range-mid",+ "perm-sentence",+ "not-available-mid",+ "forbidden-word",+ "error-handling-heading",+ "changelog-fix-error",+ "feature-not-available",+ "error-prefix-doc",+ "log-search-hit",+ "exception-doc",+ "status-code-in-last-line",+ "status-number-colon",+ "status-reason-in-sentence",+ "p02-unflagged-server-error",+ "request-word-in-a-heading",+ "error-heading-then-status-number",+ "error-prefix-status-without-reason",+ "http-version-status-in-prose",+ ],+)+def test_skeptic_probes_are_not_failures(text: str) -> None:+ wrapped = json.dumps([{"type": "text", "text": text}], separators=(",", ":"))+ assert _outcomes([text], codex=False) == [True]+ assert _outcomes([json.dumps({"type": "text", "text": text})], codex=False) == [True]+ assert _outcomes([CODEX_STATUS + wrapped], codex=True) == [True]+++ "text",+ [+ "MCP error -32603: Internal error while reading the tag index",+ "401 - Unauthorized: missing release token",+ "Permission denied: notes_path must be inside /workspace",+ "MCP error -32000: Connection closed",+ "<tool_use_error>Error: No such tool available: mcp__plugin_docs_deepwiki__ask_question</tool_use_error>",+ "tool call error: deepwiki: connection refused",+ "error: failed to connect to MCP server 'deepwiki'",+ "HTTP/1.1 503 Service Unavailable",+ "Request failed with status code 404",+ "404 Not Found",+ "HTTP Error 404: Not Found",+ "AxiosError: Request failed with status code 401",+ "403: Forbidden - release token lacks scope",+ "Tool 'ask_question' is not available.",+ # A status line after a request line or an error clause (#180's cases).+ "GET /repos/x: 404 - Not Found",+ "Error calling tool: 404: not found",+ "Failed to fetch /v1/notes: 502 Bad Gateway",+ # An HTTP/x.y response status line needs no reason phrase.+ "HTTP/1.1 404 - x",+ "HTTP/2 500",+ "HTTP/1.1 502: upstream closed the connection",+ ],+)+def test_real_failure_texts_still_fail(text: str) -> None:+ wrapped = json.dumps([{"type": "text", "text": text}], separators=(",", ":"))+ assert _outcomes([text], codex=False) == [False]+ assert _outcomes([json.dumps({"type": "text", "text": text})], codex=False) == [False]+ assert _outcomes([CODEX_STATUS + wrapped], codex=True) == [False]+++def test_an_error_line_appended_after_a_long_answer_still_fails() -> None:+ body = "x" * 5000 + "\n\n[error] tool reported failure"+ assert _outcomes([body], codex=False) == [False]+++def test_harness_success_status_wins_over_error_words() -> None:+ # A Codex ``completed`` status (from codex.txt) is the harness's own outcome.+ trajectory = {+ "agent": {"name": "codex"},+ "steps": [+ {+ "source": "agent",+ "tool_calls": [{"tool_call_id": "c1", "function_name": "ask_question", "arguments": {}}],+ "observation": {+ "results": [+ {+ "source_call_id": "c1",+ "content": CODEX_STATUS + "Permission denied: see the access guide",+ "extra": {"harness_status": "completed"},+ }+ ]+ },+ }+ ],+ }+ signals: dict[str, Any] | None = compute_plugin_signals(+ trajectory, {}, declared=DECLARED, mcp_call_servers={"c1": "deepwiki"}+ )+ assert signals is not None+ assert signals["mcp_calls"]["succeeded"] == 1diff --git a/tests/tier3/test_f90_mcp_proof_status.py b/tests/tier3/test_f90_mcp_proof_status.pynew file mode 100644index 00000000..3d956077--- /dev/null+++ b/tests/tier3/test_f90_mcp_proof_status.py@@ -0,0 +1,158 @@+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+# SPDX-License-Identifier: Apache-2.0+"""Reachability proof preserves failed, unknown, flaky and successful observations."""++from __future__ import annotations++from typing import Any++import pytest++from skillevaluator.tier3.eval_core.plugin_signals import compute_plugin_signals, summarize_plugin_signals+from skillevaluator.tier3.mcp_proof import NOT_REQUESTED_DETAIL, apply_in_agent_mcp_proof++HOST_OK = "initialize and tools/list succeeded over streamable-http"+++def _probe(status: str, detail: str = HOST_OK, tools: tuple[str, ...] = ()) -> dict[str, Any]:+ return {"status": status, "tools": list(tools), "detail": detail}+++def _engine(by_server: dict[str, dict[str, int]], census: dict[str, Any] | None = None) -> dict[str, Any]:+ agent: dict[str, Any] = {"plugin_signals_summary": {"with_skill": {"mcp_calls": {"by_server": by_server}}}}+ if census is not None:+ agent["plugin_load_census"] = census+ return {"agents": {"claude-code": agent}}+++def _census(loaded: list[str], not_loaded: dict[str, str]) -> dict[str, Any]:+ return {+ "agent": "claude-code",+ "mode": "native",+ "trials": 12,+ "harness": "claude-code system/init event",+ "loaded": [+ {"type": "mcp", "name": name, "evidence": f"claude-code system/init event: plugin:docs:{name} connected"}+ for name in loaded+ ],+ "not_loaded": [{"type": "mcp", "name": name, "reason": reason} for (name, reason) in not_loaded.items()],+ }+++def _statuses(proof: dict[str, Any]) -> dict[str, str]:+ return {name: entry["status"] for (name, entry) in proof.items()}+++def test_server_whose_init_failed_in_the_agent_is_not_proven() -> None:+ proof = {"deepwiki": _probe("reachable-host"), "cfdocs": _probe("reachable-host")}+ census = _census(+ ["deepwiki"], {"cfdocs": "claude-code init reported MCP server plugin:docs:cfdocs with status failed"}+ )+ final = apply_in_agent_mcp_proof(proof, _engine({"deepwiki": {"total": 10, "succeeded": 10}}, census))+ assert _statuses(final) == {"deepwiki": "used-successfully", "cfdocs": "not-loaded-in-agent"}+ assert "status failed" in final["cfdocs"]["detail"]+ assert "host probe: " + HOST_OK in final["cfdocs"]["detail"]+++def test_server_the_agent_never_loaded_is_not_proven() -> None:+ proof = {+ "deepwiki": _probe("reachable-host"),+ "deepwiki-bad": _probe("reachable-host"),+ "gone": _probe("unreachable", "DNS resolution failed: gaierror"),+ }+ census = _census(+ ["deepwiki"],+ {+ "deepwiki-bad": "not reported by claude-code init (no plugin:docs:deepwiki-bad)",+ "gone": "claude-code init reported MCP server plugin:docs:gone with status failed",+ },+ )+ final = apply_in_agent_mcp_proof(proof, _engine({"deepwiki": {"total": 1, "succeeded": 1}}, census))+ assert _statuses(final) == {+ "deepwiki": "used-successfully",+ "deepwiki-bad": "not-loaded-in-agent",+ "gone": "unreachable",+ }+++def test_called_but_nothing_succeeded_is_not_green_and_not_counted() -> None:+ proof = {+ "deepwiki": _probe("declared", NOT_REQUESTED_DETAIL),+ "deepwiki-bad": _probe("declared", NOT_REQUESTED_DETAIL),+ "gone": _probe("declared", NOT_REQUESTED_DETAIL),+ }+ final = apply_in_agent_mcp_proof(+ proof, _engine({"deepwiki-bad": {"total": 2, "succeeded": 0, "failed": 2, "unknown": 0}})+ )+ entry = final["deepwiki-bad"]+ assert entry["status"] == "called-no-success"+ assert entry["detail"].startswith("agent made 2 call(s): 0 succeeded, 2 failed, 0 unknown")+++def test_a_failed_call_never_turns_an_unreachable_server_reachable() -> None:+ proof = {"gone": _probe("unreachable", "DNS resolution failed: gaierror")}+ final = apply_in_agent_mcp_proof(proof, _engine({"gone": {"total": 1, "succeeded": 0, "failed": 1}}))+ assert final["gone"]["status"] == "unreachable"+++def test_one_successful_call_still_proves_the_server() -> None:+ proof = {"gone": _probe("unreachable", "DNS resolution failed: gaierror")}+ census = _census([], {"gone": "claude-code init reported MCP server plugin:docs:gone with status failed"})+ final = apply_in_agent_mcp_proof(proof, _engine({"gone": {"total": 2, "succeeded": 1, "failed": 1}}, census))+ assert final["gone"]["status"] == "used-successfully"+++def test_failed_claude_calls_end_to_end_do_not_prove_the_server() -> None:+ trajectory = {+ "agent": {"name": "claude-code"},+ "steps": [+ {+ "source": "agent",+ "tool_calls": [+ {+ "tool_call_id": "toolu_01",+ "function_name": "mcp__plugin_release-kit_reltools__list_changes",+ "arguments": {"project": "atlas"},+ }+ ],+ "observation": {+ "results": [+ {+ "source_call_id": "toolu_01",+ "content": "unknown tag 1.3.9\n\n[error] tool reported failure",+ "extra": {"tool_result_is_error": True},+ }+ ]+ },+ }+ ],+ }+ signals = compute_plugin_signals(trajectory, {}, declared={"skill": [], "mcp": ["reltools"]})+ summary = summarize_plugin_signals([signals])+ engine = {"agents": {"claude-code": {"plugin_signals_summary": {"with_skill": summary}}}}+ final = apply_in_agent_mcp_proof({"reltools": _probe("declared", NOT_REQUESTED_DETAIL)}, engine)+ assert final["reltools"]["status"] == "called-no-success"+++def test_applying_twice_is_stable() -> None:+ proof = {"docs": _probe("reachable-host"), "kb": _probe("declared", NOT_REQUESTED_DETAIL)}+ engine = _engine({"kb": {"total": 1, "succeeded": 0, "unknown": 1}})+ once = apply_in_agent_mcp_proof(proof, engine)+ assert apply_in_agent_mcp_proof(once, engine) == once+ assert apply_in_agent_mcp_proof(once, None) == once+++ "loaded,failed_calls,expected",+ [(11, 0, "reachable-host"), (11, 2, "called-no-success"), (0, 2, "not-loaded-in-agent")],+)+def test_flaky_and_never_loaded_remain_distinct(loaded, failed_calls, expected):+ census = _census([], {"docs": "init reported unavailable"})+ engine = _engine({"docs": {"total": failed_calls, "succeeded": 0, "failed": failed_calls, "unknown": 0}}, census)+ engine["agents"]["claude-code"]["plugin_signals_summary"]["with_skill"]["mcp_load"] = {+ "docs": {"loaded": loaded, "trials": 12}+ }+ final = apply_in_agent_mcp_proof({"docs": _probe("reachable-host")}, engine)+ assert final["docs"]["status"] == expected+ if loaded:+ assert "not loaded in 1 of 12" in final["docs"]["detail"]diff --git a/tests/tier3/test_f90_signals_conflict.py b/tests/tier3/test_f90_signals_conflict.pynew file mode 100644index 00000000..eadb1c15--- /dev/null+++ b/tests/tier3/test_f90_signals_conflict.py@@ -0,0 +1,326 @@+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+# SPDX-License-Identifier: Apache-2.0++"""Check 19 conflict probes: failed calls, probe ids, side channels, refs, arms, reports (proof M29, L23).++Shapes follow the check-19 replay examples: Claude Code native and wrapper+calls, and Codex ``exec_command``/bare MCP names.+"""++from __future__ import annotations++from typing import Any++import pytest++from skillevaluator.tier3.eval_core.plugin_signals import (+ ARM_SUM_OF_PARTS,+ ARM_WITH_SKILL,+ build_plugin_signals_context,+ compute_plugin_signals,+ plugin_case_spec,+ validate_plugin_case_fields,+)++CONTEXT = build_plugin_signals_context(+ member_skills=["changelog-collect", "version-bump", "release-notes"],+ mcp_servers=["reltools"],+ wrapper_skills=["release-kit-plugin-eval", "release-kit"],+)+DOCS = build_plugin_signals_context(+ member_skills=["repo-docs", "cloudflare-docs"],+ mcp_servers=["deepwiki", "cfdocs"],+ wrapper_skills=["remote-docs-plugin-eval", "remote-docs"],+)+SKILLS = "/tmp/agent-home/.agents/skills"+STAGE_NOT_PUBLISH = {+ "id": "stage-not-publish",+ "must_use": "MCP:reltools/stage_release",+ "must_not_use": "MCP:reltools/publish_release",+}+JSONRPC = (+ "printf '%s\\n%s\\n' "+ '\'{"jsonrpc":"2.0","id":1,"method":"initialize","params":{"protocolVersion":"2025-03-26"}}\' '+ '\'{"jsonrpc":"2.0","id":2,"method":"tools/call","params":{"name":"publish_release",'+ '"arguments":{"project":"atlas","version":"1.5.0"}}}\' | /tmp/release-kit/reltools'+)+++def _step(call_id: str, fn: str, args: dict[str, Any], content: str = "ok", **result: Any) -> dict[str, Any]:+ return {+ "source": "agent",+ "tool_calls": [{"tool_call_id": call_id, "function_name": fn, "arguments": args}],+ "observation": {"results": [{"source_call_id": call_id, "content": content, **result}]},+ }+++def _trajectory(agent: str, *steps: dict[str, Any]) -> dict[str, Any]:+ return {"agent": {"name": agent}, "steps": [{"source": "user", "message": "Prepare the release."}, *steps]}+++def _exec(call_id: str, cmd: str, output: str = "") -> dict[str, Any]:+ content = f"Chunk ID: c0\nWall time: 0.0000 seconds\nProcess exited with code 0\nOutput:\n{output}"+ return _step(call_id, "exec_command", {"cmd": cmd, "workdir": "/workspace"}, content)+++def _conflict(+ trajectory: dict[str, Any], *probes: dict[str, Any], context: Any = CONTEXT, arm: str = ARM_WITH_SKILL+) -> dict[str, Any]:+ signals = compute_plugin_signals(+ trajectory,+ plugin_case_spec({"id": "c", "conflict_probes": list(probes)}),+ declared=context.declared_for(arm),+ wrapper_skills=context.wrapper_skills,+ )+ assert signals is not None+ return signals["conflict"]+++# ---------------------------------------------------------------------------+# A failed call does not satisfy must_use (M29; check-19 e01)+# ---------------------------------------------------------------------------+++ ("agent", "fn", "result"),+ [+ (+ "claude-code",+ "mcp__reltools__stage_release",+ {"content": "<tool_use_error>Error: No such tool available: mcp__reltools__stage_release</tool_use_error>"},+ ),+ ("claude-code", "mcp__plugin_release-kit_reltools__stage_release", {"content": "MCP error -32000: closed"}),+ ("codex", "mcp__reltools__stage_release", {"content": "Output:\nfailed", "is_error": True}),+ ],+ ids=["claude-wrapper-missing-tool", "claude-native-mcp-error", "codex-flagged-error"],+)+def test_a_failed_call_does_not_satisfy_must_use(agent: str, fn: str, result: dict[str, Any]) -> None:+ trajectory = _trajectory(agent, _step("x1", fn, {"project": "atlas"}, **result))++ block = _conflict(trajectory, STAGE_NOT_PUBLISH)++ assert (block["checked"], block["passed"]) == (1, 0)+ assert block["failures"][0]["detail"] == "must_use MCP:reltools/stage_release was not activated"+++def test_a_failed_forbidden_call_is_still_an_attempt() -> None:+ trajectory = _trajectory(+ "claude-code",+ _step("t1", "mcp__reltools__stage_release", {"project": "atlas"}, '{"staged": true}'),+ _step("t2", "mcp__reltools__publish_release", {"project": "atlas"}, "MCP error -32000: closed"),+ )+ block = _conflict(trajectory, STAGE_NOT_PUBLISH)+ assert block["failures"][0]["detail"] == "must_not_use MCP:reltools/publish_release was activated"+++# ---------------------------------------------------------------------------+# Probe ids and contradictions (M29, L23; check-19 e08)+# ---------------------------------------------------------------------------+++def test_probe_ids_that_differ_only_by_spaces_are_rejected_not_silently_dropped() -> None:+ entry = {+ "id": "c",+ "conflict_probes": [+ {"id": "dup", "must_use": "Skill:release-notes", "must_not_use": "Skill:version-bump"},+ {"id": " dup", "must_use": "Skill:release-notes", "must_not_use": "Skill:changelog-collect"},+ ],+ }+ problems = validate_plugin_case_fields(entry)+ assert problems == ["conflict_probes[1].id: must be unique (ids are compared without surrounding spaces)"]++ entry["conflict_probes"][1]["id"] = " other "+ assert validate_plugin_case_fields(entry) == []+ spec = plugin_case_spec(entry)+ trajectory = _trajectory("claude-code", _step("t1", "Skill", {"skill": "release-notes"}, "Launching skill"))+ signals = compute_plugin_signals(trajectory, spec, declared=CONTEXT.declared_for(ARM_WITH_SKILL))+ assert signals is not None+ assert (signals["conflict"]["checked"], signals["conflict"]["passed"]) == (2, 2)+++ ("must_use", "must_not_use"),+ [+ ("MCP:reltools/stage_release", "MCP:reltools/stage_release"),+ ("MCP:reltools/stage_release", "MCP:reltools"),+ (["Skill:release-notes", "Skill:version-bump"], "Skill:*"),+ ],+)+def test_a_self_contradicting_probe_is_rejected(must_use: Any, must_not_use: Any) -> None:+ problems = validate_plugin_case_fields(+ {"id": "c", "conflict_probes": [{"id": "p", "must_use": must_use, "must_not_use": must_not_use}]}+ )+ assert problems == ["conflict_probes[0]: can never pass: every must_use ref is also matched by must_not_use"]+++def test_a_probe_with_one_safe_alternative_is_accepted() -> None:+ probe = {+ "id": "p",+ "must_use": ["MCP:reltools/stage_release", "Skill:release-notes"],+ "must_not_use": "MCP:reltools",+ }+ assert validate_plugin_case_fields({"id": "c", "conflict_probes": [probe]}) == []++[email protected]("ref", ["rule:release-policy", "hook:PreToolUse", "Hooks:pre-commit"])+def test_rule_and_hook_refs_are_rejected_with_a_clear_error(ref: str) -> None:+ """check-19 e03: these refs could never match a tool call."""+ problems = validate_plugin_case_fields(+ {"id": "c", "conflict_probes": [{"id": "p", "must_use": "Skill:release-notes", "must_not_use": ref}]}+ )+ assert len(problems) == 1+ assert "refs are not supported: rules and hooks are not tool calls" in problems[0]+++# ---------------------------------------------------------------------------+# Side channels and batched reads (M29, M7; check-19 e06, e02)+# ---------------------------------------------------------------------------+++def test_an_mcp_tool_called_through_the_shell_counts() -> None:+ """check-19 e06: publish_release sent as JSON-RPC to the server program through Bash/exec_command."""+ published = '{"jsonrpc":"2.0","id":2,"result":{"content":[{"type":"text","text":"{\\"published\\": true}"}]}}'+ claude = _trajectory(+ "claude-code",+ _step("t1", "Skill", {"skill": "release-notes"}, "Launching skill: release-notes"),+ _step("t2", "Bash", {"command": JSONRPC}, published),+ )+ codex = _trajectory(+ "codex",+ _exec("c1", f"sed -n '1,220p' {SKILLS}/release-notes/SKILL.md", "---\nname: release-notes\n---\n"),+ _exec("c2", JSONRPC, published),+ )+ probe = {+ "id": "notes-not-publish",+ "must_use": "Skill:release-notes",+ "must_not_use": "MCP:reltools/publish_release",+ }+ for trajectory in (claude, codex):+ block = _conflict(trajectory, probe)+ assert block["passed"] == 0, trajectory["agent"]+ assert "must_not_use MCP:reltools/publish_release was activated" in block["failures"][0]["detail"]+ staged = _trajectory(+ "claude-code", _step("t1", "Bash", {"command": JSONRPC.replace("publish_release", "stage_release")})+ )+ assert _conflict(staged, STAGE_NOT_PUBLISH)["passed"] == 1+++def test_running_a_member_skills_script_uses_that_skill() -> None:+ trajectory = _trajectory("codex", _exec("c1", f"python3 {SKILLS}/version-bump/scripts/bump.py --minor", "1.5.0"))+ probe = {"id": "no-bump", "must_use": "MCP:reltools", "must_not_use": "Skill:version-bump"}+ block = _conflict(trajectory, probe)+ assert "must_not_use Skill:version-bump was activated" in block["failures"][0]["detail"]+++def test_one_read_of_several_skill_files_is_not_choosing_one_of_them() -> None:+ """check-19 e02 (Codex): the agent read three SKILL.md files at once and never computed a version."""+ paths = " ".join(f"{SKILLS}/{name}/SKILL.md" for name in ("changelog-collect", "release-notes", "version-bump"))+ trajectory = _trajectory(+ "codex",+ _exec("c1", f"cat {paths}", "---\nname: changelog-collect\n---\n---\nname: release-notes\n---\n"),+ _step("c2", "mcp__reltools__stage_release", {"project": "atlas"}, '{"staged": true}'),+ )+ probe = {+ "id": "given-tag-not-recomputed",+ "must_use": "Skill:release-notes",+ "must_not_use": ["MCP:reltools/compute_version", "Skill:version-bump"],+ }+ assert _conflict(trajectory, probe)["passed"] == 1+ # The `cp` shape of the same example is not a read at all.+ copied = _trajectory("codex", _exec("c1", f"cp {SKILLS}/version-bump/SKILL.md /tmp/vb.md"))+ copied["steps"].append(_exec("c2", f"cat {SKILLS}/release-notes/SKILL.md", "---\nname: release-notes\n---\n"))+ assert _conflict(copied, probe)["passed"] == 1+++# ---------------------------------------------------------------------------+# Refs and arms (L23; check-19 e05)+# ---------------------------------------------------------------------------+++def test_unprefixed_refs_mean_the_same_tool_on_both_harnesses() -> None:+ """check-19 e05: Codex web_search_call and exec_command answer to WebFetch and Bash."""+ claude = _trajectory(+ "claude-code",+ _step("t1", "Skill", {"skill": "remote-docs:cloudflare-docs"}, "Launching skill"),+ _step("t2", "mcp__plugin_remote-docs_deepwiki__ask_wiki_question", {"question": "alarms"}),+ _step("t3", "WebFetch", {"url": "https://example.com/alarms", "prompt": "alarms"}),+ _step("t4", "Bash", {"command": "ls /workspace"}),+ )+ codex = _trajectory(+ "codex",+ _exec("c1", f"sed -n '1,220p' {SKILLS}/cloudflare-docs/SKILL.md", "---\nname: cloudflare-docs\n---\n"),+ _step("c2", "mcp__deepwiki__ask_wiki_question", {"question": "alarms"}),+ {+ "source": "agent",+ "tool_calls": [+ {+ "tool_call_id": "",+ "function_name": "web_search_call",+ "arguments": {"action_type": "open_page", "url": "https://example.com/alarms"},+ }+ ],+ },+ _exec("c4", "ls /workspace"),+ )+ probes = (+ {"id": "no-web", "must_use": "MCP:deepwiki", "must_not_use": "WebFetch"},+ {"id": "no-shell", "must_use": "Skill:cloudflare-docs", "must_not_use": "Bash"},+ )+ for trajectory in (claude, codex):+ block = _conflict(trajectory, *probes, context=DOCS)+ assert (block["checked"], block["passed"]) == (2, 0), trajectory["agent"]+++NOTES = "# cobalt 0.7.4\n\n## Bug Fixes\n\n- CB-0377: Handle empty config file\n"+STAGE_NO_EDIT = {"id": "stage-no-edit", "must_use": "MCP:reltools/stage_release", "must_not_use": ["Write", "Edit"]}+++ "codex_write",+ [+ f"mkdir -p /workspace/out && cat > /workspace/out/RELEASE_NOTES.md <<'EOF'\n{NOTES}EOF",+ f"python3 - <<'PY'\nfrom pathlib import Path\nPath('out/RELEASE_NOTES.md').write_text({NOTES!r})\nPY",+ "sed -i 's/0.7.3/0.7.4/' RELEASE_NOTES.md",+ ],+ ids=["heredoc-redirect", "heredoc-python", "sed-in-place"],+)+def test_write_and_edit_refs_see_a_shell_write_on_both_harnesses(codex_write: str) -> None:+ """Live sk19 stage-no-edit: Claude wrote with ``Write``, Codex with ``exec_command``; both must be flagged."""+ claude = _trajectory(+ "claude-code",+ _step("t1", "mcp__plugin_release-kit_reltools__stage_release", {"project": "cobalt"}),+ _step("t2", "Write", {"file_path": "/workspace/RELEASE_NOTES.md", "content": NOTES}),+ )+ codex = _trajectory(+ "codex",+ _step("c1", "mcp__reltools__stage_release", {"project": "cobalt"}),+ _exec("c2", codex_write),+ )+ for trajectory in (claude, codex):+ block = _conflict(trajectory, STAGE_NO_EDIT)+ assert (block["checked"], block["passed"]) == (1, 0), trajectory["agent"]+ assert block["failures"][0]["detail"] == "must_not_use Write | Edit was activated"+++def test_a_shell_call_that_writes_no_file_is_not_a_write() -> None:+ """Only real files count: ``2>/dev/null`` and a plain read are not writes."""+ codex = _trajectory(+ "codex",+ _step("c1", "mcp__reltools__stage_release", {"project": "cobalt"}),+ _exec("c2", "rg --files -g 'RELEASE_NOTES.md' /workspace 2>/dev/null"),+ _exec("c3", "cat RELEASE_NOTES.md > /dev/null && cat <<'EOF'\nnot a file\nEOF"),+ )+ block = _conflict(codex, STAGE_NO_EDIT)+ assert (block["checked"], block["passed"]) == (1, 1), block+++def test_probes_the_arm_cannot_satisfy_are_skipped() -> None:+ trajectory = _trajectory("claude-code", _step("t1", "Skill", {"skill": "release-notes"}, "Launching skill"))+ block = _conflict(trajectory, STAGE_NOT_PUBLISH, arm=ARM_SUM_OF_PARTS)+ assert block["skipped"] == ["stage-not-publish"]+ assert (block["checked"], block["status"]) == (0, "not_applicable")+++# ---------------------------------------------------------------------------+# Reports name the failed probes (L23 pooled rate)+# ---------------------------------------------------------------------------diff --git a/tests/tier3/test_f90_signals_handoff.py b/tests/tier3/test_f90_signals_handoff.pynew file mode 100644index 00000000..e47b39e0--- /dev/null+++ b/tests/tier3/test_f90_signals_handoff.py@@ -0,0 +1,490 @@+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+# SPDX-License-Identifier: Apache-2.0++"""Check 18 handoffs: value matching, windows, paths, shell forms, and validation (proof M28, L22).++The trajectories follow the check-18 replay examples, in the Harbor ATIF shapes+of Claude Code (``Skill`` loads, ``Bash``/``Write``/``Read``, sidechain steps+for subagents) and Codex (``exec_command`` with ``workdir``, bare MCP names).+"""++from __future__ import annotations++import json+from typing import Any++import pytest++from skillevaluator.tier3.eval_core.plugin_signals import (+ ARM_SUM_OF_PARTS,+ ARM_WITH_SKILL,+ build_plugin_signals_context,+ compute_plugin_signals,+ plugin_case_spec,+ validate_plugin_case_fields,+)++CONTEXT = build_plugin_signals_context(+ member_skills=["changelog-collect", "version-bump", "release-notes"],+ mcp_servers=["reltools", "fsx"],+ wrapper_skills=["release-kit-plugin-eval", "release-kit"],+ subagents=["release-reviewer"],+)+SKILLS = "/tmp/agent-home/.agents/skills"+CC, RN = "Skill:changelog-collect", "Skill:release-notes"+++def _step(call_id: str, fn: str, args: dict[str, Any], content: str = "ok", **extra: Any) -> dict[str, Any]:+ return {+ "source": "agent",+ "tool_calls": [{"tool_call_id": call_id, "function_name": fn, "arguments": args}],+ "observation": {"results": [{"source_call_id": call_id, "content": content}]},+ "extra": {"cwd": "/workspace", "is_sidechain": False, **extra},+ }+++def _claude(*steps: dict[str, Any]) -> dict[str, Any]:+ return {+ "agent": {"name": "claude-code", "extra": {"cwds": ["/workspace"]}},+ "steps": [{"source": "user", "message": "Collect the changes and write notes."}, *steps],+ }+++def _codex(*steps: dict[str, Any]) -> dict[str, Any]:+ return {+ "agent": {"name": "codex", "extra": {"cwd": "/workspace"}},+ "steps": [{"source": "user", "message": "Collect the changes and write notes."}, *steps],+ }+++def _launch(call_id: str, name: str, **extra: Any) -> dict[str, Any]:+ return _step(call_id, "Skill", {"skill": f"release-kit:{name}"}, f"Launching skill: release-kit:{name}", **extra)+++def _bash(call_id: str, command: str, output: str = "", **extra: Any) -> dict[str, Any]:+ return _step(call_id, "Bash", {"command": command}, output or "(Bash completed with no output)", **extra)+++def _exec(call_id: str, cmd: str, output: str = "", *, code: int = 0) -> dict[str, Any]:+ content = f"Chunk ID: c0\nWall time: 0.0000 seconds\nProcess exited with code {code}\nOutput:\n{output}"+ return _step(call_id, "exec_command", {"cmd": cmd, "workdir": "/workspace"}, content)+++def _codex_read(call_id: str, *members: str) -> dict[str, Any]:+ cmd = " && ".join(f"sed -n '1,220p' {SKILLS}/{member}/SKILL.md" for member in members)+ return _exec(call_id, cmd, "".join(f"---\nname: {member}\n---\n" for member in members))+++def _handoff(trajectory: dict[str, Any], *handoffs: dict[str, Any], arm: str = ARM_WITH_SKILL) -> dict[str, Any]:+ signals = compute_plugin_signals(+ trajectory,+ plugin_case_spec({"id": "c", "handoffs": list(handoffs)}),+ declared=CONTEXT.declared_for(arm),+ wrapper_skills=CONTEXT.wrapper_skills,+ )+ assert signals is not None+ return signals["handoff"]+++def _passes(trajectory: dict[str, Any], **handoff: Any) -> bool:+ return _handoff(trajectory, {"producer": CC, "consumer": RN, **handoff})["passed"] == 1+++# ---------------------------------------------------------------------------+# Values are compared as values (M28; check-18 e08)+# ---------------------------------------------------------------------------++CHANGES = 'AT-4821 say "hi"\npath C:\\rk\\out\nowner café-7\n'+MCP_CHANGES = json.dumps({"changes": [{"id": "AT-4835", "note": 'ok "go"'}]})+NOTES = '# notes\n- AT-4821: say "hi"\n- path C:\\rk\\out\n- owner café-7\n- AT-4835: ok "go"\n'+++def _e08_claude() -> dict[str, Any]:+ return _claude(+ _launch("t1", "changelog-collect"),+ _bash("t2", "cat input/changes.txt", CHANGES),+ _step(+ "t3",+ "mcp__plugin_release-kit_reltools__list_changes",+ {"project": "atlas"},+ json.dumps({"type": "text", "text": MCP_CHANGES}),+ ),+ _launch("t4", "release-notes"),+ _step("t5", "Write", {"file_path": "/workspace/RELEASE_NOTES.md", "content": NOTES}),+ )+++def _e08_codex() -> dict[str, Any]:+ mcp_output = "Wall time: 0.0010 seconds\nOutput:\n" + json.dumps([{"type": "text", "text": MCP_CHANGES}])+ return _codex(+ _codex_read("c1", "changelog-collect"),+ _exec("c2", "cat input/changes.txt", CHANGES),+ _step("c3", "mcp__reltools__list_changes", {"project": "atlas"}, mcp_output),+ _codex_read("c4", "release-notes"),+ _exec("c5", f"cat > RELEASE_NOTES.md <<'MD'\n{NOTES}MD"),+ )++[email protected]("trajectory", [_e08_claude(), _e08_codex()], ids=["claude-code", "codex"])+ ("value", "passes"),+ [('say "hi"', True), ("C:\\rk\\out", True), ("café-7", True), ('ok "go"', True), ("AT-48", False)],+)+def test_values_with_quotes_and_backslashes_pass_and_a_prefix_of_a_longer_id_does_not(+ trajectory: dict[str, Any], value: str, passes: bool+) -> None:+ assert _passes(trajectory, value=value) is passes+++def test_one_read_of_several_skill_files_does_not_open_a_shared_window() -> None:+ """check-18 e02: release-notes never does anything after a batched read."""+ tag = "1.5.0+rk.23b59e"+ compute = json.dumps({"build_tag": tag})+ claude = _claude(+ _bash("t1", f"cat {SKILLS}/version-bump/SKILL.md {SKILLS}/release-notes/SKILL.md", "# vb\n# rn\n"),+ _step("t2", "mcp__plugin_release-kit_reltools__compute_version", {"current": "1.4.2"}, compute),+ _bash("t3", f"echo '{tag}' > out/version.txt"),+ )+ codex = _codex(+ _codex_read("c1", "version-bump", "release-notes"),+ _step("c2", "mcp__reltools__compute_version", {"current": "1.4.2"}, compute),+ _exec("c3", f"echo '{tag}' > out/version.txt"),+ )+ for trajectory in (claude, codex):+ block = _handoff(trajectory, {"producer": "Skill:version-bump", "consumer": RN, "value": tag})+ assert block["passed"] == 0, trajectory["agent"]+ assert "read together with other skills" in block["failures"][0]["detail"]+++# ---------------------------------------------------------------------------+# Windows (L22; check-18 e03, e04)+# ---------------------------------------------------------------------------+++def test_another_skill_load_does_not_take_the_producers_write() -> None:+ """check-18 e03: a non-member skill load (Claude) or a peek at the consumer (Codex) before the save."""+ changes = json.dumps([{"id": "AT-4821"}])+ claude = _claude(+ _launch("t1", "changelog-collect"),+ _step("t2", "mcp__plugin_release-kit_reltools__list_changes", {"project": "atlas"}, changes),+ _launch("t3", "mcp_tool"),+ _step("t4", "Write", {"file_path": "/workspace/out/changes.json", "content": changes}),+ _launch("t5", "release-notes"),+ _step("t6", "Read", {"file_path": "/workspace/out/changes.json"}, f"1\t{changes}"),+ )+ codex = _codex(+ _codex_read("c1", "changelog-collect"),+ _step("c2", "mcp__reltools__list_changes", {"project": "atlas"}, changes),+ _codex_read("c3", "release-notes"),+ _exec("c4", f"cat > out/changes.json <<'EOF'\n{changes}\nEOF"),+ _exec("c5", "cat out/changes.json", changes),+ )+ for trajectory in (claude, codex):+ assert _passes(trajectory, artifact="out/changes.json"), trajectory["agent"]+++def test_a_third_component_still_closes_the_producer_window() -> None:+ trajectory = _claude(+ _launch("t1", "changelog-collect"),+ _launch("t2", "version-bump"),+ _step("t3", "Write", {"file_path": "/workspace/out/changes.json", "content": "[]"}),+ _launch("t4", "release-notes"),+ _step("t5", "Read", {"file_path": "/workspace/out/changes.json"}, "1\t[]"),+ )+ assert not _passes(trajectory, artifact="out/changes.json")+++def test_a_subagent_keeps_its_own_window_and_works_for_the_skill_that_started_it() -> None:+ """Harbor 0.20+ shape: sidechain steps are the subagent's own calls (check-18 e04)."""+ trajectory = _claude(+ _launch("t1", "changelog-collect"),+ _step("t2", "Agent", {"subagent_type": "general-purpose", "prompt": "Save the changes."}, "Saved."),+ {"source": "user", "message": "Save the changes.", "extra": {"is_sidechain": True}},+ _step("t3", "Write", {"file_path": "/workspace/out/changes.json", "content": "[]"}, is_sidechain=True),+ _launch("t4", "version-bump", is_sidechain=True),+ _step("t5", "Write", {"file_path": "/workspace/out/version.json", "content": "{}"}, is_sidechain=True),+ _step("t6", "Write", {"file_path": "/workspace/out/more.json", "content": "[]"}),+ _launch("t7", "release-notes"),+ _step("t8", "Read", {"file_path": "/workspace/out/changes.json"}, "1\t[]"),+ _step("t9", "Read", {"file_path": "/workspace/out/version.json"}, "1\t{}"),+ _step("t10", "Read", {"file_path": "/workspace/out/more.json"}, "1\t[]"),+ )+ # The subagent starts in the parent's changelog-collect window, so its first write is the producer's.+ assert _passes(trajectory, artifact="out/changes.json")+ # After the subagent switched to version-bump, its writes are version-bump's.+ assert not _passes(trajectory, artifact="out/version.json")+ # The subagent's switch does not move the parent out of changelog-collect's window.+ assert _passes(trajectory, artifact="out/more.json")+++def test_concurrent_subagents_keep_separate_windows() -> None:+ """Harbor 0.24 interleaves concurrent subagents by time and tags each step with ``agent_id``."""+ trajectory = _claude(+ _launch("t1", "changelog-collect"),+ _step("t2", "Agent", {"subagent_type": "general-purpose", "prompt": "Bump."}, "Done."),+ _step("t3", "Agent", {"subagent_type": "general-purpose", "prompt": "Save."}, "Done."),+ _launch("a1", "version-bump", is_sidechain=True, agent_id="a-1"),+ _step(+ "b1",+ "Write",+ {"file_path": "/workspace/out/changes.json", "content": "[]"},+ is_sidechain=True,+ agent_id="b-2",+ ),+ _launch("t4", "release-notes"),+ _step("t5", "Read", {"file_path": "/workspace/out/changes.json"}, "1\t[]"),+ )+ # Subagent b-2 still works in the parent's changelog-collect window; a-1's switch is its own.+ assert _passes(trajectory, artifact="out/changes.json")+++def test_a_subagent_producer_owns_the_writes_of_its_sidechain() -> None:+ trajectory = _claude(+ _step("t1", "Agent", {"subagent_type": "release-kit:release-reviewer", "prompt": "Review."}, "Done."),+ {"source": "user", "message": "Review.", "extra": {"is_sidechain": True}},+ _step("t2", "Write", {"file_path": "/workspace/out/review.json", "content": "{}"}, is_sidechain=True),+ _launch("t3", "release-notes"),+ _step("t4", "Read", {"file_path": "/workspace/out/review.json"}, "1\t{}"),+ )+ block = _handoff(trajectory, {"producer": "Agent:release-reviewer", "consumer": RN, "artifact": "out/review.json"})+ assert block["passed"] == 1, block["failures"]+++def _agent_result(call_id: str, agent_type: str, agent_id: str, *, structured: bool) -> dict[str, Any]:+ """A parent ``Agent`` result as Harbor 0.24 records it for Claude Code (saved check-19 live run)."""+ use = {"status": "completed", "prompt": "Go.", "agentId": agent_id, "agentType": agent_type, "content": []}+ text = f"Done.\n\nagentId: {agent_id} (use SendMessage with to: '{agent_id}' to continue this agent)"+ result: dict[str, Any] = {"source_call_id": call_id, "content": f"{text}\n\n[metadata] {json.dumps(use)}"}+ if structured:+ result["extra"] = {"tool_result_metadata": {"tool_use_result": use}}+ return result++[email protected]("structured", [True, False], ids=["result-metadata", "metadata-text"])[email protected]("together", [True, False], ids=["one-step", "back-to-back"])+def test_parallel_subagents_are_credited_by_their_agent_id(structured: bool, together: bool) -> None:+ """The plugin agent and a general-purpose agent start together; the plugin agent writes the artifact."""+ calls = [+ ("p1", {"subagent_type": "release-kit:release-reviewer", "prompt": "Review."}, "release-kit:release-reviewer"),+ ("p2", {"subagent_type": "general-purpose", "prompt": "Search."}, "general-purpose"),+ ]+ agent_ids = {"p1": "rev-1", "p2": "gp-2"}+ spawns: list[dict[str, Any]] = []+ for group in [calls] if together else [[call] for call in calls]:+ spawns.append(+ {+ "source": "agent",+ "tool_calls": [+ {"tool_call_id": call_id, "function_name": "Agent", "arguments": args} for call_id, args, _ in group+ ],+ "observation": {+ "results": [+ _agent_result(call_id, agent_type, agent_ids[call_id], structured=structured)+ for call_id, _, agent_type in group+ ]+ },+ "extra": {"cwd": "/workspace", "is_sidechain": False},+ }+ )+ trajectory = _claude(+ *spawns,+ _step(+ "r1",+ "Write",+ {"file_path": "/workspace/out/review.json", "content": "{}"},+ is_sidechain=True,+ agent_id="rev-1",+ ),+ _step("g1", "Bash", {"command": "ls"}, "x", is_sidechain=True, agent_id="gp-2"),+ _launch("t3", "release-notes"),+ _step("t4", "Read", {"file_path": "/workspace/out/review.json"}, "1\t{}"),+ )++ block = _handoff(trajectory, {"producer": "Agent:release-reviewer", "consumer": RN, "artifact": "out/review.json"})+ assert block["passed"] == 1, block["failures"]+ # The general-purpose agent did not write it.+ other = _handoff(trajectory, {"producer": "Agent:general-purpose", "consumer": RN, "artifact": "out/review.json"})+ assert other["passed"] == 0+++# ---------------------------------------------------------------------------+# Paths and shell forms (L22; check-18 e05, e06, e07, e10)+# ---------------------------------------------------------------------------+++def test_paths_respect_the_working_directory() -> None:+ """check-18 e05: absolute vs relative, and the same name in another folder."""+ claude = _claude(+ _launch("t1", "changelog-collect"),+ _bash("t2", "cat > out/changes.json <<'EOF'\n[]\nEOF"),+ _step("t3", "Write", {"file_path": "/tmp/scratch/plan.json", "content": "{}"}),+ _launch("t4", "release-notes"),+ _bash("t5", "cat out/changes.json", "[]"),+ _step("t6", "Read", {"file_path": "/workspace/plan.json"}, '1\t{"old": true}'),+ )+ codex = _codex(+ _codex_read("c1", "changelog-collect"),+ _exec("c2", "cat > out/changes.json <<'EOF'\n[]\nEOF"),+ _exec("c3", "echo '{}' > /tmp/scratch/plan.json"),+ _codex_read("c4", "release-notes"),+ _exec("c5", "cat out/changes.json", "[]"),+ _exec("c6", "cat /workspace/plan.json", '{"old": true}'),+ )+ for trajectory in (claude, codex):+ assert _passes(trajectory, artifact="/workspace/out/changes.json"), trajectory["agent"]+ assert not _passes(trajectory, artifact="plan.json"), trajectory["agent"]+++ ("write", "read"),+ [+ ("""python3 -c "open('out/a.json','w').write('[]')\"""", "cat out/a.json"),+ ("echo '[]' > out/b.json", """python3 -c "import json; print(json.load(open('out/b.json')))\""""),+ ("echo '[]' > out/c.json", "cd out && cat c.json"),+ ("sed -i 's/old/new/' out/d.json", "cat out/d.json"),+ ("echo '[]' > out/e.json", "cat out/*.json"),+ # Interpreter code fed as a here-document (the most common form in the saved live runs).+ ("python3 - <<'PY'\nimport json\nopen('out/f.json','w').write('[]')\nPY", "cat out/f.json"),+ (+ "python3 <<PY\nfrom pathlib import Path\nPath('/workspace/out/g.json').write_text('[]')\nPY",+ "cat out/g.json",+ ),+ ("echo '[]' > out/h.json", "python3 - <<'PY'\nimport json\nprint(json.load(open('out/h.json')))\nPY"),+ (+ "python3 - <<'PY'\nfrom pathlib import Path\nout = Path('/workspace/out/i.json')\nout.write_text('[]')\nPY",+ "python - <<'PY'\nfrom pathlib import Path\np=Path('/workspace/out/i.json')\nwith p.open(encoding='utf-8') as f:\n"+ " print(f.read())\nPY",+ ),+ ("node - <<'JS'\nrequire('fs').writeFileSync('out/j.json', '[]')\nJS", "cat out/j.json"),+ ("cat <<'EOF' > out/k.json\n[]\nEOF", "wc -c out/k.json"),+ ],+ ids=[+ "python-write",+ "python-read",+ "cd",+ "sed-in-place",+ "glob",+ "heredoc-python-write",+ "heredoc-path-write",+ "heredoc-python-read",+ "heredoc-named-path",+ "heredoc-node-write",+ "heredoc-then-redirect",+ ],+)+def test_shell_writer_and_reader_forms_are_seen(write: str, read: str) -> None:+ """check-18 e06, on both harness shapes."""+ artifact = "out/" + next(name for name in "abcdefghijk" if f"out/{name}.json" in write + read) + ".json"+ claude = _claude(+ _launch("t1", "changelog-collect"), _bash("t2", write), _launch("t3", "release-notes"), _bash("t4", read, "[]")+ )+ codex = _codex(+ _codex_read("c1", "changelog-collect"),+ _exec("c2", write),+ _codex_read("c3", "release-notes"),+ _exec("c4", read, "[]"),+ )+ for trajectory in (claude, codex):+ assert _passes(trajectory, artifact=artifact), (trajectory["agent"], write, read)+++def test_a_call_that_only_names_the_path_is_not_a_read() -> None:+ """check-18 e07: an MCP overwrite, ``rm``, and a subagent asked to delete the file."""+ trajectory = _claude(+ _launch("t1", "changelog-collect"),+ _step("t2", "Write", {"file_path": "/workspace/out/changes.json", "content": "[]"}),+ _step("t3", "Write", {"file_path": "/workspace/out/other.json", "content": "{}"}),+ _step("t4", "Write", {"file_path": "/workspace/out/third.json", "content": "{}"}),+ _launch("t5", "release-notes"),+ _step("t6", "mcp__plugin_release-kit_fsx__write_file", {"path": "out/changes.json", "content": "[]"}),+ _bash("t7", "rm out/other.json"),+ _step("t8", "Agent", {"subagent_type": "general-purpose", "prompt": "Delete /workspace/out/third.json now"}),+ )+ for artifact in ("out/changes.json", "out/other.json", "out/third.json"):+ assert not _passes(trajectory, artifact=artifact), artifact+ reader = _claude(+ _launch("t1", "changelog-collect"),+ _step("t2", "Write", {"file_path": "/workspace/out/changes.json", "content": "[]"}),+ _launch("t3", "release-notes"),+ _step("t4", "mcp__plugin_release-kit_fsx__read_file", {"path": "/workspace/out/changes.json"}, "[]"),+ )+ assert _passes(reader, artifact="out/changes.json")+++def test_a_read_in_the_same_step_or_a_failed_read_does_not_count() -> None:+ """check-18 e10: a parallel write and read, and a read that says "No such file"."""+ same_step = _claude(+ _launch("t1", "changelog-collect"),+ {+ "source": "agent",+ "tool_calls": [+ {+ "tool_call_id": "w",+ "function_name": "Write",+ "arguments": {"file_path": "/workspace/out/x.json", "content": "[]"},+ },+ {+ "tool_call_id": "r",+ "function_name": "mcp__plugin_release-kit_fsx__read_file",+ "arguments": {"path": "out/x.json"},+ },+ ],+ "observation": {+ "results": [+ {"source_call_id": "w", "content": "File created"},+ {"source_call_id": "r", "content": "[]"},+ ]+ },+ },+ _launch("t2", "release-notes"),+ )+ assert not _handoff(same_step, {"producer": CC, "consumer": "MCP:fsx", "artifact": "out/x.json"})["passed"]++ failed_read = _claude(+ _launch("t1", "changelog-collect"),+ _step("t2", "Write", {"file_path": "/workspace/out/x.json", "content": "[]"}),+ _launch("t3", "release-notes"),+ _step("t4", "Read", {"file_path": "/workspace/out/y.json"}, "File does not exist."),+ _bash("t5", "cat out/x.json", "cat: out/x.json: No such file or directory"),+ )+ assert not _passes(failed_read, artifact="out/x.json")+++# ---------------------------------------------------------------------------+# Components the arm cannot carry+# ---------------------------------------------------------------------------+++def test_handoffs_whose_producer_or_consumer_the_arm_cannot_carry_are_skipped() -> None:+ """Codex loads no plugin subagents, and the member-skills arm stages no MCP servers."""+ skip = "this arm cannot carry that component type"+ typed = {"producer": "Agent:release-reviewer", "consumer": RN, "value": "AT-4821"}+ untyped = {"producer": RN, "consumer": "release-reviewer", "value": "AT-4821"}+ block = _handoff(_codex(_codex_read("c1", "release-notes")), typed, untyped)+ assert block["skipped"] == [+ {"producer": "Agent:release-reviewer", "consumer": RN, "reason": skip},+ {"producer": RN, "consumer": "release-reviewer", "reason": skip},+ ]+ assert (block["checked"], block["failures"], block["status"]) == (0, [], "not_applicable")++ mcp = {"producer": "MCP:reltools/list_changes", "consumer": RN, "value": "AT-4821"}+ members = {"producer": CC, "consumer": RN, "value": "AT-4821"}+ block = _handoff(_claude(_launch("t1", "release-notes")), mcp, members, arm=ARM_SUM_OF_PARTS)+ assert [item["producer"] for item in block["skipped"]] == ["MCP:reltools/list_changes"]+ assert (block["checked"], block["passed"], block["status"]) == (1, 0, "scored")+++# ---------------------------------------------------------------------------+# Validation (L22; check-18 e09)+# ---------------------------------------------------------------------------+++def test_self_handoff_is_rejected_and_control_characters_get_a_clear_message() -> None:+ problems = validate_plugin_case_fields({"id": "c", "handoffs": [{"producer": CC, "consumer": CC, "value": "v"}]})+ assert problems == [+ "handoffs[0]: producer and consumer must name different components (a self-handoff always passes)"+ ]+ problems = validate_plugin_case_fields(+ {"id": "c", "handoffs": [{"producer": CC, "consumer": RN, "artifact": "out/a\x07.json"}]}+ )+ assert problems == ["handoffs[0].artifact: must not contain control characters"]diff --git a/tests/tier3/test_f90_signals_handoff_prompt.py b/tests/tier3/test_f90_signals_handoff_prompt.pynew file mode 100644index 00000000..08c6826f--- /dev/null+++ b/tests/tier3/test_f90_signals_handoff_prompt.py@@ -0,0 +1,160 @@+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+# SPDX-License-Identifier: Apache-2.0++"""Value handoffs: only the real task prompt counts as "prompt" (proof H15, L22 prompt rule).++The trajectories copy the Harbor ATIF shapes of Claude Code and Codex: after a+``Skill`` call Claude Code injects a ``user`` step that holds the skill body and+an ``ARGUMENTS:`` line with the agent's own Skill arguments, and a Claude+subagent's prompt is a sidechain ``user`` step written by the parent agent.+Neither is the task prompt.+"""++from __future__ import annotations++import json+from typing import Any++from skillevaluator.tier3.eval_core.plugin_signals import compute_plugin_signals, plugin_case_spec++TAG = "1.5.0+rk.23b59e"+DECLARED = {+ "skill": ["changelog-collect", "version-bump", "release-notes"],+ "mcp": ["reltools"],+ "plugin": ["release-kit"],+}+CASE = plugin_case_spec(+ {+ "id": "full-release",+ "handoffs": [{"producer": "Skill:version-bump", "consumer": "Skill:release-notes", "value": TAG}],+ }+)+++def _agent(call_id: str, fn: str, args: dict[str, Any], content: str, **extra: Any) -> dict[str, Any]:+ return {+ "source": "agent",+ "message": "",+ "tool_calls": [{"tool_call_id": call_id, "function_name": fn, "arguments": args}],+ "observation": {"results": [{"source_call_id": call_id, "content": content}]},+ "extra": {"is_sidechain": False, **extra},+ }+++def _user(text: str, *, sidechain: bool = False) -> dict[str, Any]:+ return {"source": "user", "message": text, "extra": {"is_sidechain": sidechain}}+++def _skill_load(name: str, arguments: str = "") -> dict[str, Any]:+ body = f"Base directory for this skill: /tmp/skilleval/native/claude-code/plugin/skills/{name}\n\n# {name}\n"+ return _user(body + (f"\n\nARGUMENTS: {arguments}" if arguments else ""))+++def _claude_echo_trajectory(prompt: str = "Prepare the next atlas release. The last tag is 1.4.2.") -> dict:+ """check-18 e01: Claude passes the computed tag to release-notes as Skill arguments."""+ compute = json.dumps({"build_tag": TAG, "bump": "minor"})+ return {+ "agent": {"name": "claude-code"},+ "steps": [+ _user(prompt),+ _agent("t1", "Skill", {"skill": "release-kit:version-bump"}, "Launching skill: release-kit:version-bump"),+ _skill_load("version-bump"),+ _agent(+ "t2",+ "mcp__plugin_release-kit_reltools__compute_version",+ {"current": "1.4.2", "bump": "minor"},+ compute,+ ),+ _agent(+ "t3",+ "Skill",+ {"skill": "release-kit:release-notes", "args": f"project=atlas build_tag={TAG}"},+ "Launching skill: release-kit:release-notes",+ ),+ _skill_load("release-notes", f"project=atlas build_tag={TAG}"),+ _agent(+ "t4",+ "mcp__plugin_release-kit_reltools__stage_release",+ {"project": "atlas", "version": TAG, "notes_path": "RELEASE_NOTES.md"},+ '{"staged": true}',+ ),+ {"source": "agent", "message": "Staged.", "extra": {"is_sidechain": False}},+ ],+ }+++def _handoff(trajectory: dict[str, Any], declared: dict[str, Any] | None = None) -> dict[str, Any]:+ signals = compute_plugin_signals(trajectory, CASE, declared=declared or DECLARED)+ assert signals is not None+ return signals["handoff"]+++def test_claude_skill_argument_echo_is_not_the_task_prompt() -> None:+ handoff = _handoff(_claude_echo_trajectory())++ assert (handoff["checked"], handoff["passed"]) == (1, 1), handoff["failures"]+++def test_value_in_the_real_task_prompt_still_fails() -> None:+ handoff = _handoff(_claude_echo_trajectory(prompt=f"Release atlas as {TAG}."))++ assert handoff["passed"] == 0+ assert "task prompt" in handoff["failures"][0]["detail"]+++def test_codex_leading_system_and_user_steps_are_the_prompt() -> None:+ """Codex puts AGENTS.md in a leading system/user step: a value there is still prompt text."""+ exec_read = "Process exited with code 0\nOutput:\n---\nname: {0}\n---\n# {0}\n"+ trajectory = {+ "agent": {"name": "codex"},+ "steps": [+ {"source": "system", "message": f"<permissions instructions> pinned tag {TAG}"},+ {"source": "user", "message": "Prepare the next atlas release."},+ _agent(+ "c1",+ "exec_command",+ {"cmd": "sed -n '1,220p' /tmp/agent-home/.agents/skills/version-bump/SKILL.md"},+ exec_read.format("version-bump"),+ ),+ _agent("c2", "mcp__reltools__compute_version", {"current": "1.4.2"}, f'{{"build_tag": "{TAG}"}}'),+ _agent(+ "c3",+ "exec_command",+ {"cmd": "sed -n '1,220p' /tmp/agent-home/.agents/skills/release-notes/SKILL.md"},+ exec_read.format("release-notes"),+ ),+ _agent("c4", "mcp__reltools__stage_release", {"version": TAG}, '{"staged": true}'),+ ],+ }++ handoff = _handoff(trajectory)++ assert handoff["passed"] == 0+ assert "task prompt" in handoff["failures"][0]["detail"]++ trajectory["steps"][0]["message"] = "<permissions instructions>"+ assert _handoff(trajectory)["passed"] == 1+++def test_subagent_prompt_written_by_the_parent_is_not_the_task_prompt() -> None:+ """check-18 e04 (Harbor 0.20+ shape): a sidechain user step is the parent's own words."""+ trajectory = _claude_echo_trajectory()+ steps = trajectory["steps"]+ # Replace the echoed Skill arguments with a subagent that receives the tag in its prompt.+ steps[4] = _agent(+ "t3", "Skill", {"skill": "release-kit:release-notes"}, "Launching skill: release-kit:release-notes"+ )+ steps[5] = _skill_load("release-notes")+ steps[6:6] = [+ _agent(+ "t5",+ "Agent",+ {"subagent_type": "general-purpose", "prompt": f"Write the notes for {TAG}."},+ "Wrote the notes.",+ ),+ _user(f"Write the notes for {TAG}.", sidechain=True),+ ]++ handoff = _handoff(trajectory)++ assert (handoff["checked"], handoff["passed"]) == (1, 1), handoff["failures"]diff --git a/tests/tier3/test_mcp_proof.py b/tests/tier3/test_mcp_proof.pynew file mode 100644index 00000000..f6616a8f--- /dev/null+++ b/tests/tier3/test_mcp_proof.py@@ -0,0 +1,687 @@+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+# SPDX-License-Identifier: Apache-2.0++"""Private MCP proof against a local fake HTTP MCP server."""++from __future__ import annotations++import json+import logging+import queue+import socket+import threading+import time+from collections.abc import Iterator+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer+from typing import Any++import pytest++from skillevaluator.tier3.mcp_proof import (+ NOT_REQUESTED_DETAIL,+ _check_endpoint,+ _ProbeRefused,+ apply_in_agent_mcp_proof,+ declared_mcp_proof,+ parse_env_grant,+ probe_mcp_server,+ probe_mcp_servers,+)++pytest.importorskip("mcp")++TOOLS = [{"name": "search", "inputSchema": {"type": "object"}}, {"name": "get_page", "inputSchema": {"type": "object"}}]+LOCAL = ("127.0.0.1",)+++def test_declared_proof_does_not_consume_targets_past_its_bound():+ def targets():+ for index in range(32):+ yield {"name": f"s{index}", "url": "https://example.invalid/mcp"}+ raise AssertionError("consumed past the target bound")++ assert len(declared_mcp_proof(targets())) == 32+++def test_detail_redacts_credentials_crossing_its_input_window():+ from skillevaluator.tier3.mcp_proof import MAX_DETAIL_CHARS, _detail++ # The long assignment shrinks during redaction, bringing the final token+ # into the visible output after its last character was cut off.+ secret = "ghp_" + "B" * 36+ prefix = "token=" + "a" * (MAX_DETAIL_CHARS * 8 - 39 - 7) + " "+ rendered = _detail(prefix + secret)+ assert "B" * 12 not in rendered+ assert "ghp_" not in rendered+ assert len(rendered) <= MAX_DETAIL_CHARS++[email protected]("separator", ["=", ":"])[email protected]("quote", ["'", '"'])+def test_detail_does_not_expose_a_quoted_secret_cut_at_whitespace(separator, quote):+ from skillevaluator.tier3.mcp_proof import _detail++ rendered = _detail(f"password{separator}{quote}secret-one " + "x" * 3000 + quote)+ assert "secret-one" not in rendered+++def test_overlapping_probe_log_contexts_restore_only_after_the_last_exit():+ from skillevaluator.tier3.mcp_proof import _ProbeAbort, _quiet_mcp_logs++ logger = logging.getLogger("mcp")+ original = (logger.level, logger.propagate, list(logger.handlers))+ records = []++ class RecordHandler(logging.Handler):+ def emit(self, record):+ records.append(record.getMessage())++ sink = RecordHandler()+ logger.setLevel(logging.INFO)+ logger.propagate = True+ logger.addHandler(sink)+ first, second = _quiet_mcp_logs(_ProbeAbort()), _quiet_mcp_logs(_ProbeAbort())+ first_open = second_open = False+ try:+ first.__enter__()+ first_open = True+ second.__enter__()+ second_open = True+ first.__exit__(None, None, None)+ first_open = False+ logger.error("fixture private SDK error")+ assert not records+ assert logger.propagate is False+ second.__exit__(None, None, None)+ second_open = False+ assert logger.level == logging.INFO and logger.propagate is True+ assert logger.handlers == [*original[2], sink]+ finally:+ if first_open:+ first.__exit__(None, None, None)+ if second_open:+ second.__exit__(None, None, None)+ logger.handlers[:] = original[2]+ logger.setLevel(original[0])+ logger.propagate = original[1]+++def _result(method: str, params: dict[str, Any]) -> dict[str, Any]:+ if method == "initialize":+ return {+ "protocolVersion": params.get("protocolVersion", "2025-06-18"),+ "capabilities": {"tools": {}},+ "serverInfo": {"name": "fake", "version": "1.0"},+ }+ if method == "tools/list":+ return {"tools": TOOLS}+ return {}+++class _Handler(BaseHTTPRequestHandler):+ server: _FakeServer++ def log_message(self, *_args: Any) -> None:+ return++ def _empty(self, status: int) -> None:+ self.send_response(status)+ self.send_header("Content-Length", "0")+ self.end_headers()++ def do_DELETE(self) -> None:+ self._empty(200)++ def do_GET(self) -> None:+ if self.path == "/redirect":+ self.send_response(302)+ self.send_header("Location", "http://169.254.169.254/latest/meta-data")+ self.send_header("Content-Length", "0")+ self.end_headers()+ return+ if self.path == "/sse-offorigin":+ # The endpoint event points at another origin; the client must refuse it at once.+ self.send_response(200)+ self.send_header("Content-Type", "text/event-stream")+ self.end_headers()+ self.wfile.write(b"event: endpoint\ndata: http://127.0.0.1:9/messages?session_id=s1\n\n")+ self.wfile.flush()+ time.sleep(3)+ return+ if self.path != "/sse":+ self._empty(405)+ return+ self.send_response(200)+ self.send_header("Content-Type", "text/event-stream")+ self.send_header("Cache-Control", "no-cache")+ self.end_headers()+ self.wfile.write(b"event: endpoint\ndata: /messages?session_id=s1\n\n")+ self.wfile.flush()+ while True:+ try:+ message = self.server.sse_queue.get(timeout=10)+ except queue.Empty:+ return+ if message is None:+ return+ self.wfile.write(b"event: message\ndata: " + json.dumps(message).encode() + b"\n\n")+ self.wfile.flush()++ def do_POST(self) -> None:+ self.server.seen_headers.append(dict(self.headers))+ body = json.loads(self.rfile.read(int(self.headers.get("Content-Length", "0"))))+ if self.path == "/huge":+ data = b'{"jsonrpc":"2.0","id":0,"result":{"pad":"' + b"x" * (2 * 1024 * 1024) + b'"}}'+ self.send_response(200)+ self.send_header("Content-Type", "application/json")+ self.send_header("Content-Length", str(len(data)))+ self.end_headers()+ self.wfile.write(data)+ return+ if self.path == "/redirect":+ self.send_response(307)+ self.send_header("Location", "http://169.254.169.254/latest/meta-data")+ self.send_header("Content-Length", "0")+ self.end_headers()+ return+ if "id" not in body:+ self._empty(202)+ return+ reply = {"jsonrpc": "2.0", "id": body["id"], "result": _result(body["method"], body.get("params") or {})}+ if self.path.startswith("/messages"):+ self.server.sse_queue.put(reply)+ self._empty(202)+ return+ data = json.dumps(reply).encode()+ self.send_response(200)+ self.send_header("Content-Type", "application/json")+ self.send_header("Mcp-Session-Id", "session-1")+ self.send_header("Content-Length", str(len(data)))+ self.end_headers()+ self.wfile.write(data)+++class _FakeServer(ThreadingHTTPServer):+ daemon_threads = True++ def __init__(self) -> None:+ super().__init__(("127.0.0.1", 0), _Handler)+ self.seen_headers: list[dict[str, str]] = []+ self.sse_queue: queue.Queue[dict[str, Any] | None] = queue.Queue()+++def fake_mcp() -> Iterator[str]:+ server = _FakeServer()+ thread = threading.Thread(target=server.serve_forever, daemon=True)+ thread.start()+ try:+ yield f"http://127.0.0.1:{server.server_address[1]}"+ finally:+ server.sse_queue.put(None)+ server.shutdown()+ server.server_close()+++def test_streamable_http_probe_lists_tools_with_opted_in_header_refs(fake_mcp: str) -> None:+ target = {+ "name": "docs",+ "url": f"{fake_mcp}/mcp",+ "transport": "http",+ "headers": {"Authorization": "Bearer ${DOCS_TOKEN}", "X-Missing": "$UNSET_VAR"},+ }++ entry = probe_mcp_server(+ target,+ allowed_private_hosts=LOCAL,+ expand_env=("DOCS_TOKEN", "UNSET_VAR"),+ environ={"DOCS_TOKEN": "tok-123"},+ )++ assert entry["status"] == "reachable-host"+ assert entry["tools"] == ["search", "get_page"]+ assert "streamable-http" in entry["detail"]+ assert "UNSET_VAR" in entry["detail"]+ assert "tok-123" not in json.dumps(entry)+++def _spy_headers(monkeypatch: pytest.MonkeyPatch) -> list[dict[str, str]]:+ import skillevaluator.tier3.mcp_proof as proof++ seen: list[dict[str, str]] = []+ original = proof._capped_client++ def spy(headers: dict[str, str], **kwargs: Any):+ seen.append(dict(headers))+ return original(headers, **kwargs)++ monkeypatch.setattr(proof, "_capped_client", spy)+ return seen+++def test_probe_expands_only_opted_in_variables(fake_mcp: str, monkeypatch: pytest.MonkeyPatch) -> None:+ seen = _spy_headers(monkeypatch)+ target = {+ "name": "docs",+ "url": f"{fake_mcp}/mcp",+ "headers": {"Authorization": "Bearer ${DOCS_TOKEN}", "X-Trace": "$OTHER", "X-Client": "skilleval"},+ }++ probe_mcp_server(+ target, allowed_private_hosts=LOCAL, expand_env=("DOCS_TOKEN",), environ={"DOCS_TOKEN": "tok-123", "OTHER": "x"}+ )++ assert seen == [{"Authorization": "Bearer tok-123", "X-Client": "skilleval"}]+++def test_probe_never_sends_host_variables_by_default(fake_mcp: str, monkeypatch: pytest.MonkeyPatch) -> None:+ """A plugin picks both the URL and the header template; host secrets must not follow them."""+ seen = _spy_headers(monkeypatch)+ monkeypatch.setenv("ANTHROPIC_API_KEY", "sk-ant-host-secret-123")+ monkeypatch.setenv("GITHUB_TOKEN", "ghp_hostsecret456")+ target = {+ "name": "docs",+ "url": f"{fake_mcp}/mcp",+ "headers": {"X-A": "${ANTHROPIC_API_KEY}", "X-B": "$GITHUB_TOKEN", "X-Client": "skilleval"},+ }++ [entry] = probe_mcp_servers([target], allowed_private_hosts=LOCAL).values()++ assert seen == [{"X-Client": "skilleval"}]+ assert entry["status"] == "reachable-host"+ assert "ANTHROPIC_API_KEY" in entry["detail"] and "GITHUB_TOKEN" in entry["detail"]+ assert "private environment grant" in entry["detail"]+ assert "sk-ant-host-secret-123" not in json.dumps(entry)+++def test_an_explicit_empty_environment_is_not_the_host_environment(+ fake_mcp: str, monkeypatch: pytest.MonkeyPatch+) -> None:+ seen = _spy_headers(monkeypatch)+ monkeypatch.setenv("DOCS_TOKEN", "from-the-host")+ target = {"name": "docs", "url": f"{fake_mcp}/mcp", "headers": {"Authorization": "Bearer ${DOCS_TOKEN}"}}++ entry = probe_mcp_server(target, allowed_private_hosts=LOCAL, expand_env=("DOCS_TOKEN",), environ={})++ assert seen == [{}]+ assert "headers skipped because unset: DOCS_TOKEN" in entry["detail"]+++def test_probe_connects_only_to_policy_checked_addresses(monkeypatch: pytest.MonkeyPatch) -> None:+ """DNS rebinding: a name that re-resolves to a private address at connect time is never reached."""+ import anyio++ listener = socket.socket()+ listener.bind(("127.0.0.1", 0))+ listener.listen()+ port = listener.getsockname()[1]+ real_getaddrinfo = socket.getaddrinfo++ def rebinding_getaddrinfo(host: Any, *args: Any, **kwargs: Any) -> Any:+ if host in {"rebind.example", b"rebind.example"}:+ host = "127.0.0.1"+ return real_getaddrinfo(host, *args, **kwargs)++ real_connect = anyio.connect_tcp+ attempted: list[str] = []++ async def guarded_connect(remote_host: str, remote_port: int, **kwargs: Any) -> Any:+ attempted.append(str(remote_host))+ if remote_host == "93.184.216.34":+ raise OSError("public test address is unreachable")+ return await real_connect(remote_host, remote_port, **kwargs)++ monkeypatch.setattr(socket, "getaddrinfo", rebinding_getaddrinfo)+ monkeypatch.setattr(anyio, "connect_tcp", guarded_connect)+ target = {"name": "rebind", "url": f"https://rebind.example:{port}/mcp", "headers": {"X-Token": "${DOCS_TOKEN}"}}+ try:+ entry = probe_mcp_server(+ target,+ expand_env=("DOCS_TOKEN",),+ environ={"DOCS_TOKEN": "tok-123"},+ resolver=lambda _h, _p: ["93.184.216.34"],+ total_timeout=3,+ )+ listener.setblocking(False)+ with pytest.raises(BlockingIOError):+ listener.accept()+ finally:+ listener.close()++ assert attempted == ["93.184.216.34"]+ assert entry["status"] == "unreachable"+++def test_pinned_client_refuses_other_hosts_and_ports(monkeypatch: pytest.MonkeyPatch) -> None:+ import anyio+ import httpx++ from skillevaluator.tier3.mcp_proof import _capped_client++ attempted: list[str] = []++ async def recording_connect(remote_host: str, remote_port: int, **_kwargs: Any) -> Any:+ attempted.append(f"{remote_host}:{remote_port}")+ raise OSError("no network in tests")++ monkeypatch.setattr(anyio, "connect_tcp", recording_connect)++ async def fetch(url: str) -> None:+ client = _capped_client(+ {},+ url="https://mcp.example.com/mcp",+ addresses=("93.184.216.34", "93.184.216.35"),+ max_bytes=1024,+ connect_timeout=1,+ read_timeout=1,+ )+ async with client:+ await client.get(url)++ for url in ("https://metadata.example/mcp", "https://mcp.example.com:8443/mcp", "https://mcp.example.com/mcp"):+ with pytest.raises(httpx.ConnectError):+ anyio.run(fetch, url)++ assert attempted == ["93.184.216.34:443", "93.184.216.35:443"]+++def test_sse_probe(fake_mcp: str) -> None:+ entry = probe_mcp_server(+ {"name": "legacy", "url": f"{fake_mcp}/sse", "transport": "sse"}, allowed_private_hosts=LOCAL+ )++ assert entry["status"] == "reachable-host", entry+ assert entry["tools"] == ["search", "get_page"]+++def test_private_endpoints_are_not_probed_without_an_allowlist(fake_mcp: str) -> None:+ entry = probe_mcp_server({"name": "docs", "url": f"{fake_mcp}/mcp"})++ assert entry["status"] == "declared"+ assert "allowed_private_hosts" in entry["detail"]+++ "url",+ ["https://169.254.169.254/mcp", "https://metadata.google.internal/mcp", "https://[fd00:ec2::254]/mcp"],+)+def test_metadata_endpoints_are_never_probed_even_when_allowlisted(url: str) -> None:+ entry = probe_mcp_server({"name": "meta", "url": url}, allowed_private_hosts=("0.0.0.0/0", "::/0", "*.internal"))++ assert entry["status"] == "declared"+ assert "metadata" in entry["detail"]+++def test_public_names_resolving_to_private_addresses_are_refused() -> None:+ entry = probe_mcp_server(+ {"name": "pub", "url": "https://mcp.example.com/mcp"}, resolver=lambda _h, _p: ["10.0.0.7"]+ )++ assert entry["status"] == "declared"+ assert "10.0.0.7" in entry["detail"]+++def test_plaintext_http_to_a_public_host_is_refused() -> None:+ entry = probe_mcp_server({"name": "pub", "url": "http://mcp.example.com/mcp"}, resolver=lambda _h, _p: ["1.1.1.1"])++ assert entry["status"] == "declared"+ assert "plaintext" in entry["detail"]+++ ("url", "allowed", "addresses"),+ [+ ("https://mcp.corp.internal/mcp", ("mcp.corp.internal",), ["10.1.2.3"]),+ ("https://mcp.corp.internal/mcp", ("*.corp.internal",), ["10.1.2.3"]),+ ("https://mcp.corp.internal/mcp", ("10.0.0.0/8",), ["10.1.2.3"]),+ ("http://mcp.corp.internal/mcp", ("mcp.corp.internal",), ["10.1.2.3"]),+ ("http://mcp.corp.internal/mcp", ("10.0.0.0/8",), ["10.1.2.3"]),+ ("http://localhost:8080/mcp", ("localhost",), ["127.0.0.1", "::1"]),+ ("http://localhost:8080/mcp", ("127.0.0.1", "::1"), ["127.0.0.1", "::1"]),+ ],+)+def test_private_hosts_can_be_allowlisted_by_name_or_address(+ url: str, allowed: tuple[str, ...], addresses: list[str]+) -> None:+ kind, checked = _check_endpoint(url, "http", allowed, lambda _h, _p: addresses)++ assert (kind, checked) == ("streamable-http", tuple(addresses))+++ ("url", "allowed", "addresses", "detail"),+ [+ # A name allowlist never admits a metadata address, and naming a metadata host never allowlists it.+ ("https://mcp.corp.internal/mcp", ("mcp.corp.internal",), ["169.254.169.254"], "metadata"),+ ("https://metadata.google.internal/mcp", ("metadata.google.internal",), ["10.0.0.1"], "metadata"),+ # Only the resolved ::1 is allowlisted, so the other loopback address is refused.+ ("http://localhost:8080/mcp", ("::1",), ["127.0.0.1", "::1"], "127.0.0.1"),+ # A name allowlist does not make plaintext http to a public address acceptable.+ ("http://mcp.example.com/mcp", ("mcp.example.com",), ["1.1.1.1"], "plaintext"),+ ("https://mcp.corp.internal/mcp", ("other.corp.internal",), ["10.1.2.3"], "allowed_private_hosts"),+ ],+)+def test_name_allowlisting_keeps_the_endpoint_policy(+ url: str, allowed: tuple[str, ...], addresses: list[str], detail: str+) -> None:+ with pytest.raises(_ProbeRefused) as refused:+ _check_endpoint(url, "http", allowed, lambda _h, _p: addresses)++ assert refused.value.status == "declared"+ assert detail in refused.value.detail+++ ("target", "status"),+ [+ ({"name": "ws", "url": "wss://mcp.example.com/mcp"}, "unsupported"),+ ({"name": "grpc", "url": "https://mcp.example.com/mcp", "transport": "grpc"}, "unsupported"),+ ({"name": "ftp", "url": "ftp://mcp.example.com/mcp"}, "unsupported"),+ ],+)+def test_unsupported_transports(target: dict[str, Any], status: str) -> None:+ assert probe_mcp_server(target, resolver=lambda _h, _p: ["1.1.1.1"])["status"] == status+++def test_unreachable_server() -> None:+ entry = probe_mcp_server({"name": "dead", "url": "http://127.0.0.1:1/mcp"}, allowed_private_hosts=LOCAL)++ assert entry["status"] == "unreachable"+++def test_redirects_are_not_followed(fake_mcp: str) -> None:+ entry = probe_mcp_server({"name": "r", "url": f"{fake_mcp}/redirect"}, allowed_private_hosts=LOCAL)++ assert entry["status"] == "unreachable"+++def test_oversized_responses_are_capped(fake_mcp: str) -> None:+ entry = probe_mcp_server({"name": "big", "url": f"{fake_mcp}/huge"}, allowed_private_hosts=LOCAL)++ assert entry["status"] == "unreachable"+++def test_probe_servers_is_bounded_and_skips_nameless(fake_mcp: str) -> None:+ targets = [{"name": "", "url": f"{fake_mcp}/mcp"}, {"name": "docs", "url": f"{fake_mcp}/mcp"}]++ proof = probe_mcp_servers(targets, allowed_private_hosts=LOCAL)++ assert list(proof) == ["docs"]+ assert proof["docs"]["status"] == "reachable-host"+++def _engine(by_server: dict[str, dict[str, int]]) -> dict[str, Any]:+ return {"agents": {"codex": {"plugin_signals_summary": {"with_skill": {"mcp_calls": {"by_server": by_server}}}}}}+++def test_in_agent_evidence_upgrades_and_never_downgrades() -> None:+ proof = {+ "docs": {"status": "reachable-host", "tools": ["search"], "detail": "ok"},+ "tracker": {"status": "declared", "tools": [], "detail": NOT_REQUESTED_DETAIL},+ "idle": {"status": "unreachable", "tools": [], "detail": "down"},+ }++ upgraded = apply_in_agent_mcp_proof(+ proof, _engine({"docs": {"total": 3, "succeeded": 2}, "tracker": {"total": 1, "succeeded": 0}})+ )++ assert upgraded["docs"]["status"] == "used-successfully"+ assert upgraded["docs"]["tools"] == ["search"]+ assert "host probe: ok" in upgraded["docs"]["detail"]+ # Called, but nothing succeeded: not proof that the server is reachable (proof M30).+ assert upgraded["tracker"]["status"] == "called-no-success"+ assert upgraded["idle"] == proof["idle"]+ assert apply_in_agent_mcp_proof(upgraded, None) == upgraded+++def test_in_agent_evidence_indexes_the_declared_servers_once(monkeypatch: pytest.MonkeyPatch) -> None:+ from skillevaluator.tier3.eval_core import plugin_signals++ builds: list[object] = []+ build = plugin_signals._McpNames.__init__++ def counting_build(self: Any, declared: Any) -> None:+ builds.append(self)+ build(self, declared)++ monkeypatch.setattr(plugin_signals._McpNames, "__init__", counting_build)+ by_server = {+ "docs": {"total": 1, "succeeded": 1},+ "plugin_demo_tracker": {"total": 2, "succeeded": 0},+ "slack": {"total": 1, "succeeded": 1},+ }+ engine = {+ "agents": {+ agent: {"plugin_signals_summary": {"with_skill": {"mcp_calls": {"by_server": by_server}}}}+ for agent in ("claude-code", "codex", "opencode")+ }+ }+ proof = {name: {"status": "declared", "tools": [], "detail": NOT_REQUESTED_DETAIL} for name in ("docs", "tracker")}++ upgraded = apply_in_agent_mcp_proof(proof, engine)++ assert len(builds) == 1+ assert upgraded["docs"]["status"] == "used-successfully"+ # Every agent's plugin_demo_tracker calls count for the declared tracker; none succeeded, so it is+ # called-no-success, not reachable (proof M30).+ assert upgraded["tracker"]["status"] == "called-no-success"+ assert upgraded["tracker"]["detail"].startswith("agent made 6 call(s): 0 succeeded")+++def test_declared_default_without_probe() -> None:+ proof = declared_mcp_proof([{"name": "docs", "url": "https://docs.example.com/mcp"}])++ assert proof == {"docs": {"status": "declared", "tools": [], "detail": NOT_REQUESTED_DETAIL}}+++# --------------------------------------------------------------------------- #+# Probe deadlines, fatal transport errors, output hygiene, env scoping #+# --------------------------------------------------------------------------- #+def test_an_oversized_response_ends_the_probe_at_once_without_mcp_tracebacks(+ fake_mcp: str, caplog: pytest.LogCaptureFixture+) -> None:+ caplog.set_level(logging.DEBUG)+ started = time.monotonic()++ entry = probe_mcp_server({"name": "big", "url": f"{fake_mcp}/huge"}, allowed_private_hosts=LOCAL, total_timeout=10)++ assert time.monotonic() - started < 5+ assert entry["status"] == "unreachable"+ assert "response exceeded 1048576 bytes" in entry["detail"]+ assert not [record for record in caplog.records if record.name.startswith("mcp")]+++def test_an_off_origin_sse_endpoint_fails_fast(fake_mcp: str) -> None:+ started = time.monotonic()++ entry = probe_mcp_server(+ {"name": "sse", "url": f"{fake_mcp}/sse-offorigin", "transport": "sse"},+ allowed_private_hosts=LOCAL,+ total_timeout=10,+ )++ assert time.monotonic() - started < 5+ assert entry["status"] == "unreachable"+ assert "origin" in entry["detail"]+++def test_a_slow_dns_lookup_counts_against_the_probe_deadline() -> None:+ def slow_resolver(_host: str, _port: int) -> list[str]:+ time.sleep(5)+ return ["93.184.216.34"]++ started = time.monotonic()+ entry = probe_mcp_server(+ {"name": "docs", "url": "https://docs.example.com/mcp"}, resolver=slow_resolver, total_timeout=0.5+ )++ assert time.monotonic() - started < 2+ assert entry["status"] == "unreachable"+ assert "DNS resolution timed out" in entry["detail"]+++def test_probed_tool_names_and_details_are_printable(fake_mcp: str, monkeypatch: pytest.MonkeyPatch) -> None:+ hostile = [+ {"name": "\x1b[31mred\x1b[0m", "inputSchema": {"type": [["x"]]}},+ {"name": "\x1b]0;pwned\x07title", "inputSchema": {"type": [{"x": "string"}]}},+ {+ "name": "bell\x07\x9bcsi",+ "inputSchema": {"type": "object", "properties": {"nested": {"type": [["x"]], "minimum": 1}}},+ },+ {"name": "safe", "inputSchema": {"type": "object", "properties": {"nested": {"type": [{}], "minimum": 1}}}},+ ]+ monkeypatch.setitem(globals(), "TOOLS", hostile)++ entry = probe_mcp_server({"name": "docs", "url": f"{fake_mcp}/mcp"}, allowed_private_hosts=LOCAL)++ assert entry["status"] == "reachable-host"+ assert entry["tools"] == ["red", "title", "bellcsi", "safe"]+ assert entry["input_schemas"] == {"safe": {"type": "object", "properties": {"nested": {"minimum": 1}}}}+ assert not any(ord(char) < 0x20 or 0x7F <= ord(char) <= 0x9F for char in json.dumps(entry, ensure_ascii=False))+++def _scoped_targets(fake_mcp: str) -> list[dict[str, Any]]:+ port = fake_mcp.rsplit(":", 1)[1]+ headers = {"Authorization": "Bearer ${DOCS_TOKEN}"}+ return [+ {"name": "docs", "url": f"http://docs.test:{port}/mcp", "headers": headers},+ {"name": "collector", "url": f"http://collector.test:{port}/mcp", "headers": headers},+ ]+++ ("grant", "expected"),+ [+ ("DOCS_TOKEN=docs.test", [{"Authorization": "Bearer tok-123"}, {}]),+ ("DOCS_TOKEN@collector", [{}, {"Authorization": "Bearer tok-123"}]),+ ("DOCS_TOKEN", [{"Authorization": "Bearer tok-123"}, {"Authorization": "Bearer tok-123"}]),+ ],+)+def test_probe_mcp_env_can_be_scoped_to_one_host_or_server(+ fake_mcp: str, monkeypatch: pytest.MonkeyPatch, grant: str, expected: list[dict[str, str]]+) -> None:+ seen = _spy_headers(monkeypatch)++ proof = probe_mcp_servers(+ _scoped_targets(fake_mcp),+ allowed_private_hosts=LOCAL,+ expand_env=(grant,),+ environ={"DOCS_TOKEN": "tok-123"},+ resolver=lambda _host, _port: ["127.0.0.1"],+ )++ assert seen == expected+ for (name, entry), headers in zip(proof.items(), expected, strict=True):+ host = f"{name}.test"+ assert entry["status"] == "reachable-host"+ assert (f"sent DOCS_TOKEN to {host}" in entry["detail"]) is bool(headers), entry["detail"]+ assert "tok-123" not in json.dumps(entry)+++ "value",+ ["NOT-A-NAME", "DOCS_TOKEN=", "DOCS_TOKEN=bad host", "DOCS_TOKEN=docs.test:443", "DOCS_TOKEN@", "1TOKEN=docs.test"],+)+def test_private_probe_grants_reject_invalid_names_and_scopes(value):+ with pytest.raises(ValueError):+ parse_env_grant(value)diff --git a/tests/tier3/test_plugin_evidence.py b/tests/tier3/test_plugin_evidence.pynew file mode 100644index 00000000..85fef2a4--- /dev/null+++ b/tests/tier3/test_plugin_evidence.py@@ -0,0 +1,549 @@+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+# SPDX-License-Identifier: Apache-2.0+"""Private evidence uses the real preparation and lifecycle with synthetic artifacts."""++from __future__ import annotations++import json+import shutil++import pytest++from skillevaluator.evaluation.options import EvaluationOptions+from skillevaluator.plugin_inventory import plugin_inventory_for_root+from skillevaluator.tier3 import plugin_evidence as evidence+from skillevaluator.tier3 import plugin_execution as execution+from skillevaluator.tier3.harbor.metrics import DEFAULT_METRIC_SET, DEFAULT_METRICS, DEFAULT_SCORE_POLICY+from skillevaluator.tier3.plugin_prepare import prepare_runtime_package+++def prepared(root):+ files = {+ ".claude-plugin/plugin.json": {+ "name": "demo",+ "mcpServers": {"docs": {"url": "https://docs.example.invalid/mcp"}},+ },+ "skills/a/SKILL.md": "---\nname: a\ndescription: A.\n---\nBody.\n",+ "skills/a/evals/evals.json": [+ {"id": "one", "prompt": "do it", "expected_tools": ["MCP:docs/search"]},+ {"id": "two", "prompt": "another", "expected_tools": ["MCP:docs/other"]},+ ],+ }+ for rel, value in files.items():+ path = root / rel+ path.parent.mkdir(parents=True, exist_ok=True)+ path.write_text(value if isinstance(value, str) else json.dumps(value))+ return prepare_runtime_package(plugin_inventory_for_root(root))+++def test_selected_case_mapping_accepts_a_trusted_subset_and_rejects_unknown_or_duplicate_ids(tmp_path):+ from skillevaluator.tier3.plugin_prepare import stage_wrapper_skill++ package = prepared(tmp_path / "plugin")+ wrapper = stage_wrapper_skill(package, tmp_path / "wrapper")+ hooks = execution.NativeRuntimeHooks(package.native_source, {})+ observer = evidence.RuntimeEvidenceObserver(package, wrapper, hooks, evidence.EvidenceOptions())+ observer.configure_run(case_id_by_task_selector={"selected-task": "a-one"})+ assert observer.selector_map == {"selected-task": "a-one"}+ with pytest.raises(ValueError, match="identit"):+ observer.configure_run(case_id_by_task_selector={"unknown-task": "unknown-case"})+ with pytest.raises(ValueError, match="unique"):+ observer.configure_run(case_id_by_task_selector={"first": "a-one", "second": "a-one"})+++def trajectory():+ return {+ "schema_version": "ATIF-v1.7",+ "trajectory_id": "fixture-trial",+ "agent": {"name": "codex", "version": "test"},+ "final_metrics": {"total_prompt_tokens": 23, "total_completion_tokens": 5, "total_cost_usd": None},+ "steps": [+ {+ "step_id": 1,+ "source": "agent",+ "message": "",+ "metrics": {"prompt_tokens": 23, "completion_tokens": 5},+ "tool_calls": [+ {+ "tool_call_id": "call",+ "function_name": "mcp__docs__search",+ "arguments": {"query": "argument-value-should-not-persist"},+ }+ ],+ "observation": {+ "results": [+ {"source_call_id": "call", "content": "found", "extra": {"tool_result_is_error": False}}+ ]+ },+ }+ ],+ }+++def write_trial(jobs, skill, arm="with", *, missing=False, steps=1):+ root = jobs / f"{skill}-codex-{arm}" / "a-one__attempt1"+ for index in range(steps):+ verifier = root / (f"steps/step-{index}/verifier" if steps > 1 else "verifier")+ verifier.mkdir(parents=True)+ reward = {+ **dict.fromkeys(DEFAULT_METRICS, 1.0),+ "entry_id": "a-one",+ "metric_set": DEFAULT_METRIC_SET,+ "score_policy": DEFAULT_SCORE_POLICY,+ }+ (verifier / "reward.json").write_text(json.dumps(reward))+ if not missing:+ for index in range(steps):+ agent = root / (f"steps/step-{index}/agent" if steps > 1 else "agent")+ agent.mkdir()+ (agent / "trajectory.json").write_text(json.dumps(trajectory()))+ if steps > 1:+ (root / "result.json").write_text(+ json.dumps({"step_results": [{"step_name": f"step-{index}"} for index in range(steps)]})+ )+ return root+++def run_fixture(+ tmp_path,+ monkeypatch,+ *,+ missing=False,+ steps=1,+ options=None,+ variants=("with",),+ listed_only=False,+ hosted_search=False,+ forged_case=None,+ empty=False,+):+ package = prepared(tmp_path / "plugin")+ work = tmp_path / "work"+ work.mkdir()+ captured = {}++ def evaluate(settings):+ hooks = execution.active_runtime()+ captured["observer"] = hooks.observer+ captured["wrapper"] = settings.skill_path+ jobs = tmp_path / "results" / "_harbor-jobs"+ jobs.mkdir(parents=True)+ for variant in variants:+ if empty:+ (jobs / f"{settings.skill_path.name}-codex-{variant}").mkdir()+ continue+ trial = write_trial(jobs, settings.skill_path.name, variant, missing=missing, steps=steps)+ if forged_case:+ for path in trial.rglob("reward.json"):+ row = json.loads(path.read_text())+ row["entry_id"] = forged_case+ path.write_text(json.dumps(row))+ if listed_only:+ hooks.trial_censuses[trial] = {+ "loaded": [],+ "listed": [{"type": "mcp", "name": "docs"}],+ "not_loaded": [],+ }+ if hosted_search:+ path = trial / "agent/trajectory.json"+ data = json.loads(path.read_text())+ data["steps"][0]["tool_calls"][0]["function_name"] = "web_search_call"+ path.write_text(json.dumps(data))+ result = {+ "execution_status": "succeeded",+ "agents": {+ "codex": {+ "conditions": {+ "with_skill": {"execution_status": "succeeded", "scored_attempts": 1, "expected_attempts": 1},+ "without_skill": {"execution_status": "skipped", "scored_attempts": 0, "expected_attempts": 0},+ }+ }+ },+ }+ for arm, variant in evidence.ARM_SUFFIXES.items():+ if variant in variants:+ result["agents"]["codex"]["conditions"][arm] = {+ "execution_status": "succeeded",+ "expected_attempts": 1,+ "scored_attempts": 1,+ }+ hooks.collect_evidence(jobs, result)+ hooks.collect_evidence(jobs, result)+ shutil.rmtree(jobs)+ return result++ outcome = execution.evaluate_prepared_plugin(+ package,+ EvaluationOptions(+ skill_path=package.native_source.plugin_root, agents="codex", results_dir=tmp_path / "results"+ ),+ work_dir=work,+ evaluate=evaluate,+ evidence_options=options,+ )+ return outcome, captured+++def test_default_collects_advisory_evidence_without_probing_and_survives_cleanup(tmp_path, monkeypatch):+ monkeypatch.setattr(evidence, "probe_mcp_servers", lambda *_a, **_k: pytest.fail("default host probe"))+ outcome, captured = run_fixture(tmp_path, monkeypatch)+ assert outcome.status == "succeeded" and outcome.attempts == 1+ assert not captured["wrapper"].exists()+ assert outcome.provenance["mcp_proof"]["docs"]["status"] == "used-successfully"+ summary = outcome.result["agents"]["codex"]["plugin_signals_summary"]["with_skill"]+ assert summary["mcp_calls"]["succeeded"] == 1+ record = captured["observer"].logical_trials["codex"]["with_skill"][0]+ assert record.case_id == "a-one" and record.reward["security"] == 1.0+ assert record.usage.get("total_cost_usd") is None+ assert record.first_turn_usage["prompt_tokens"] == 23+ sidecar = tmp_path / "results" / evidence.RUNTIME_EVIDENCE_FILENAME+ persisted = sidecar.read_text()+ assert "argument-value-should-not-persist" not in persisted+ assert json.loads(persisted)["agents"]["codex"]["with_skill"]["trials"][0]["case_id"] == "a-one"+++def test_logical_trial_is_read_and_normalized_once_across_steps_and_repeated_collection(tmp_path, monkeypatch):+ from skillevaluator.tier3.eval_core import plugin_signals+ from skillevaluator.utils.secure_fs import SecureRoot++ reads, normalizations = [], []+ read = SecureRoot.read_bytes+ extract = plugin_signals._extract_calls++ def read_once(self, path, *args, **kwargs):+ if path.name == "trajectory.json":+ reads.append(path)+ return read(self, path, *args, **kwargs)++ def normalized(*args, **kwargs):+ normalizations.append(True)+ return extract(*args, **kwargs)++ monkeypatch.setattr(SecureRoot, "read_bytes", read_once)+ monkeypatch.setattr(plugin_signals, "_extract_calls", normalized)+ outcome, _ = run_fixture(tmp_path, monkeypatch, steps=2)+ assert len(reads) == 2 and len(normalizations) == 1+ assert outcome.result["agents"]["codex"]["plugin_signals_summary"]["with_skill"]["n_trials"] == 1+++def test_missing_trajectory_is_explicit_and_never_proves_a_server(tmp_path, monkeypatch):+ outcome, captured = run_fixture(tmp_path, monkeypatch, missing=True)+ summary = outcome.result["agents"]["codex"]["plugin_signals_summary"]["with_skill"]+ assert summary["n_missing_trajectory"] == 1+ assert outcome.provenance["mcp_proof"]["docs"]["status"] == "declared"+ record = captured["observer"].logical_trials["codex"]["with_skill"][0]+ assert record.signals is None and record.usage is None and record.evidence_status == "missing"+++def test_explicit_probe_forwards_scoped_grants_and_never_changes_execution_status(tmp_path, monkeypatch):+ seen = []++ def probe(targets, **kwargs):+ seen.append((targets, kwargs))+ return {"docs": {"status": "unreachable", "tools": [], "detail": "fake timeout"}}++ monkeypatch.setattr(evidence, "probe_mcp_servers", probe)+ outcome, _ = run_fixture(+ tmp_path,+ monkeypatch,+ missing=True,+ options=evidence.EvidenceOptions(probe_mcp=True, env_grants=("TOKEN@docs",)),+ )+ assert outcome.status == "succeeded" and len(seen) == 1+ assert seen[0][1]["expand_env"] == ("TOKEN@docs",)+ assert outcome.provenance["mcp_proof"]["docs"]["status"] == "unreachable"+++def test_comparison_arm_jobs_use_the_existing_engine_variants(tmp_path, monkeypatch):+ outcome, captured = run_fixture(tmp_path, monkeypatch, variants=("with", "without", "sumofparts", "looseparts"))+ assert set(captured["observer"].logical_trials["codex"]) == {+ "with_skill",+ "without_skill",+ "sum_of_parts",+ "loose_parts",+ }+ assert len(outcome.result["agents"]["codex"]["plugin_signals_summary"]) == 4+++def test_listed_only_census_without_init_keeps_load_unknown(tmp_path, monkeypatch):+ outcome, _ = run_fixture(tmp_path, monkeypatch, missing=True, listed_only=True)+ tally = outcome.result["agents"]["codex"]["plugin_signals_summary"]["with_skill"]["mcp_load"]["docs"]+ assert tally["loaded"] == tally["trials"] == 0+ assert tally["unknown"] == 1+ assert outcome.provenance["mcp_proof"]["docs"]["status"] == "declared"+++def test_first_turn_hosted_search_is_not_a_context_measurement(tmp_path, monkeypatch):+ _, captured = run_fixture(tmp_path, monkeypatch, hosted_search=True)+ record = captured["observer"].logical_trials["codex"]["with_skill"][0]+ assert record.first_turn_usage is None+ assert record.first_turn_excluded_reason == "hosted_search"+++def test_reward_cannot_forge_another_valid_staged_case_for_signals(tmp_path, monkeypatch):+ _, captured = run_fixture(tmp_path, monkeypatch, forged_case="a-two")+ record = captured["observer"].logical_trials["codex"]["with_skill"][0]+ assert record.case_id == record.reward["entry_id"] == "a-one"+ assert record.signals["tool_selection"]["recall"] == 1.0++[email protected]("variants,empty", [((), False), (("with",), True)])+def test_missing_requested_job_or_empty_job_is_explicit_incomplete_evidence(tmp_path, monkeypatch, variants, empty):+ outcome, _ = run_fixture(tmp_path, monkeypatch, variants=variants, empty=empty)+ assert outcome.provenance["runtime_evidence"]["evidence_complete"] is False+ summary = outcome.result["agents"]["codex"]["plugin_signals_summary"]["with_skill"]+ assert summary["n_missing_trials"] == 1 and summary["evidence_complete"] is False+++def test_selected_reward_roots_refuse_escape_and_read_only_the_requested_trial(tmp_path):+ jobs = tmp_path / "jobs"+ first = write_trial(jobs, "demo")+ other = first.with_name("other__attempt1")+ (other / "verifier").mkdir(parents=True)+ (other / "verifier/reward.json").write_text("{}")+ rows = evidence.collector._extract_rewards(first.parent, selected_trial_roots=[first])+ assert {row["_trial_root_name"] for row in rows} == {first.name}+ with pytest.raises(ValueError):+ evidence.collector._extract_rewards(first.parent, selected_trial_roots=[tmp_path])+++def test_setup_retry_replaces_cached_trials_without_reprobing_or_rereading_rewards(tmp_path, monkeypatch):+ package = prepared(tmp_path / "plugin")+ work = tmp_path / "work"+ work.mkdir()+ attempts, probes, observers = [], [], []++ def probe(*_args, **_kwargs):+ probes.append(True)+ return {"docs": {"status": "reachable-host", "tools": [], "detail": "fake host proof"}}++ def evaluate(settings):+ hooks = execution.active_runtime()+ observers.append(hooks.observer)+ index = len(attempts)+ jobs = tmp_path / f"attempt-{index}" / "_harbor-jobs"+ root = write_trial(jobs, settings.skill_path.name, missing=index == 0)+ if index:+ root = root.rename(root.with_name("second-task__attempt1"))+ path = root / "agent/trajectory.json"+ data = json.loads(path.read_text())+ data["steps"][0]["tool_calls"][0]["function_name"] = "mcp__docs__other"+ path.write_text(json.dumps(data))+ selector_map = {"a-one": "a-one", "second-task" if index else "a-two": "a-two"}+ (root / "result.json").write_text(json.dumps({"task_id": {"path": root.name.split("__")[0]}}))+ hooks.configure_run(case_id_by_task_selector=selector_map)+ rows = evidence.collector._extract_rewards(root.parent, selector_map)+ scoreable, _ = evidence.collector._partition_scoreable_rewards(rows)+ hooks.observe_collected_trials("codex", "with_skill", root.parent, rows, scoreable, selector_map)+ condition = {+ "execution_status": "succeeded" if index else "failed",+ "expected_attempts": 1,+ "scored_attempts": 1 if index else 0,+ }+ info = {"conditions": {"with_skill": condition}}+ if not index:+ info["trial_failures"] = {"with_skill": [{"trial": "a-one", "reason": "AgentSetupNetworkError: dns"}]}+ info["failure_detail_metadata"] = {"with_skill_trials": {"details_total": 1, "details_truncated": False}}+ result = {"execution_status": condition["execution_status"], "agents": {"codex": info}}+ with monkeypatch.context() as local:+ local.setattr(evidence.collector, "_extract_rewards", lambda *_a, **_k: pytest.fail("reward reread"))+ hooks.collect_evidence(jobs, result)+ attempts.append(result)+ return result++ monkeypatch.setattr(evidence, "probe_mcp_servers", probe)+ outcome = execution.evaluate_prepared_plugin(+ package,+ EvaluationOptions(skill_path=package.native_source.plugin_root, agents="codex"),+ work_dir=work,+ evaluate=evaluate,+ evidence_options=evidence.EvidenceOptions(probe_mcp=True),+ )+ assert outcome.status == "succeeded" and outcome.attempts == 2+ assert len(probes) == 1 and observers[0] is observers[1]+ observer = observers[-1]+ record = observer.logical_trials["codex"]["with_skill"][0]+ assert record.case_id == "a-two" and record.signals["tool_selection"]["recall"] == 1.0+ assert record.usage["total_prompt_tokens"] == 23+ assert len(observer.artifacts._traces) == len(observer._rewards) == 1+ assert attempts[0]["runtime_evidence"]["mcp_proof"]["docs"]["status"] == "reachable-host"+ assert attempts[0]["agents"]["codex"]["plugin_signals_summary"]["with_skill"]["n_missing_trajectory"] == 1+ assert attempts[1]["runtime_evidence"]["mcp_proof"]["docs"]["status"] == "used-successfully"+++def test_retry_that_fails_before_collection_cannot_publish_previous_runtime_evidence(tmp_path):+ package = prepared(tmp_path / "plugin")+ work = tmp_path / "work"+ work.mkdir()+ attempts, observers = [], []++ def evaluate(settings):+ hooks = execution.active_runtime()+ observers.append(hooks.observer)+ if attempts:+ return {"execution_status": "failed", "error": ["fixture failure before jobs exist"]}+ jobs = tmp_path / "first-attempt" / "_harbor-jobs"+ root = write_trial(jobs, settings.skill_path.name)+ hooks.trial_censuses[root] = {"loaded": [{"type": "mcp", "name": "docs"}]}+ hooks.configure_run(case_id_by_task_selector={"a-one": "a-one"})+ result = {+ "execution_status": "failed",+ "agents": {+ "codex": {+ "conditions": {+ "with_skill": {"execution_status": "failed", "expected_attempts": 1, "scored_attempts": 0}+ },+ "trial_failures": {"with_skill": [{"trial": "a-one", "reason": "AgentSetupNetworkError: dns"}]},+ "failure_detail_metadata": {"with_skill_trials": {"details_total": 1, "details_truncated": False}},+ }+ },+ }+ hooks.collect_evidence(jobs, result)+ hooks.observer.record_failure(ValueError("first attempt only"))+ attempts.append(result)+ return result++ outcome = execution.evaluate_prepared_plugin(+ package,+ EvaluationOptions(skill_path=package.native_source.plugin_root, agents="codex"),+ work_dir=work,+ evaluate=evaluate,+ )+ assert outcome.attempts == 2 and outcome.status == "failed"+ assert attempts[0]["runtime_evidence"]["mcp_proof"]["docs"]["status"] == "used-successfully"+ final = outcome.provenance["runtime_evidence"]+ assert final["mcp_proof"]["docs"]["status"] == "declared"+ assert final["evidence_complete"] is False and final["errors"] == []+ observer = observers[-1]+ assert observer.logical_trials == observer.summaries == observer.arm_jobs == {}+ assert observer.artifacts._traces == observer._rewards == {}+ assert observer.hooks.trial_censuses == observer.hooks.trial_init_events == {}+ assert observer.selector_map is None and observer._sidecar is None+++def test_unscored_trace_keeps_trusted_identity_and_usage(tmp_path, monkeypatch):+ extract = evidence.collector._extract_rewards++ def no_scores(*args, **kwargs):+ rows = extract(*args, **kwargs)+ for row in rows:+ for metric in DEFAULT_METRICS:+ row.pop(metric, None)+ return rows++ monkeypatch.setattr(evidence.collector, "_extract_rewards", no_scores)+ _, captured = run_fixture(tmp_path, monkeypatch)+ record = captured["observer"].logical_trials["codex"]["with_skill"][0]+ assert record.case_id == "a-one" and record.reward is None+ assert record.usage["total_prompt_tokens"] == 23+++def test_private_probe_schema_checks_arguments_without_persisting_values(tmp_path, monkeypatch):+ monkeypatch.setattr(+ evidence,+ "probe_mcp_servers",+ lambda *_a, **_k: {+ "docs": {+ "status": "reachable-host",+ "tools": ["search"],+ "detail": "fixture",+ "input_schemas": {+ "search": {"type": "object", "properties": {"query": {"type": "integer"}}, "required": ["query"]}+ },+ }+ },+ )+ outcome, captured = run_fixture(tmp_path, monkeypatch, options=evidence.EvidenceOptions(probe_mcp=True))+ signals = captured["observer"].logical_trials["codex"]["with_skill"][0].signals+ assert signals["arguments"]["checked"] == 1 and signals["arguments"]["passed"] == 0+ assert outcome.provenance["mcp_proof"]["docs"]["status"] == "used-successfully"+ assert (+ "argument-value-should-not-persist"+ not in (tmp_path / "results" / evidence.RUNTIME_EVIDENCE_FILENAME).read_text()+ )+++def test_trial_cap_bounds_reward_and_trajectory_work_and_marks_truncation(tmp_path, monkeypatch):+ original = write_trial++ def two_trials(*args, **kwargs):+ root = original(*args, **kwargs)+ shutil.copytree(root, root.with_name("a-one__attempt2"))+ return root++ selected = []+ extract = evidence.collector._extract_rewards++ def bounded(*args, **kwargs):+ selected.extend(kwargs["selected_trial_roots"])+ return extract(*args, **kwargs)++ monkeypatch.setattr(evidence, "MAX_TRIALS_PER_ARM", 1)+ monkeypatch.setattr(evidence.collector, "_extract_rewards", bounded)+ monkeypatch.setitem(run_fixture.__globals__, "write_trial", two_trials)+ outcome, captured = run_fixture(tmp_path, monkeypatch)+ assert len(selected) == len(captured["observer"].artifacts._traces) == 1+ summary = outcome.result["agents"]["codex"]["plugin_signals_summary"]["with_skill"]+ assert summary["trials_truncated"] is True and summary["evidence_complete"] is False+++def test_mutation_during_explicit_probe_aborts_before_dispatch_and_cleans_wrapper(tmp_path, monkeypatch):+ package = prepared(tmp_path / "plugin")+ work = tmp_path / "work"+ work.mkdir()++ def probe(*_args, **_kwargs):+ (tmp_path / "plugin/skills/a/SKILL.md").write_text("changed")+ return {}++ monkeypatch.setattr(evidence, "probe_mcp_servers", probe)+ with pytest.raises(ValueError, match="changed"):+ execution.evaluate_prepared_plugin(+ package,+ EvaluationOptions(skill_path=package.native_source.plugin_root, agents="codex"),+ work_dir=work,+ evidence_options=evidence.EvidenceOptions(probe_mcp=True),+ evaluate=lambda _o: pytest.fail("dispatch after source mutation"),+ )+ assert list(work.iterdir()) == [] and execution.active_runtime() is None+++def test_observer_failure_is_advisory_and_does_not_retry(tmp_path, monkeypatch):+ def broken(*_args, **_kwargs):+ raise OSError("fixture observer failure")++ monkeypatch.setattr(evidence, "compute_plugin_signals", broken)+ outcome, _ = run_fixture(tmp_path, monkeypatch)+ assert outcome.status == "succeeded" and outcome.attempts == 1+ assert "fixture observer failure" in outcome.provenance["runtime_evidence"]["errors"][0]+ assert execution.active_runtime() is None+++def test_missing_trial_census_keeps_server_load_unknown():+ loaded = evidence.LogicalTrialEvidence(+ "one", "a-one", None, None, {"loaded": [{"type": "mcp", "name": "docs"}]}, None, None, "missing"+ )+ missing = evidence.LogicalTrialEvidence("two", "a-two", None, None, None, None, None, "missing")+ assert evidence._load_tally((loaded, missing))["docs"] == {"loaded": 1, "trials": 1, "unknown": 1}+ assert evidence._proof_census((loaded, missing))["loaded"] == []+++def test_cancellation_with_observer_restores_context_and_cleans_wrapper(tmp_path):+ package = prepared(tmp_path / "plugin")+ work = tmp_path / "work"+ work.mkdir()+ calls = []++ def cancel(settings):+ assert execution.active_runtime().observer is not None and settings.skill_path.exists()+ calls.append(True)+ raise KeyboardInterrupt++ with pytest.raises(KeyboardInterrupt):+ execution.evaluate_prepared_plugin(+ package,+ EvaluationOptions(skill_path=package.native_source.plugin_root, agents="codex"),+ work_dir=work,+ evaluate=cancel,+ )+ assert calls == [True] and execution.active_runtime() is None and list(work.iterdir()) == []diff --git a/tests/tier3/test_plugin_evidence_review.py b/tests/tier3/test_plugin_evidence_review.pynew file mode 100644index 00000000..f004056a--- /dev/null+++ b/tests/tier3/test_plugin_evidence_review.py@@ -0,0 +1,347 @@+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+# SPDX-License-Identifier: Apache-2.0+"""Evidence attribution remains uncertain when identities or child artifacts are incomplete."""++from __future__ import annotations++import json++import pytest++from skillevaluator.tier3 import plugin_trace_artifacts as artifacts+from skillevaluator.tier3.eval_core import plugin_signals as signals+++def call(call_id, name, args=None, *, refs=None):+ result = {"source_call_id": call_id, "content": "ok", "extra": {"tool_result_is_error": False}}+ if refs is not None:+ result["subagent_trajectory_ref"] = refs+ return {+ "step_id": 1,+ "source": "agent",+ "message": "",+ "tool_calls": [{"tool_call_id": call_id, "function_name": name, "arguments": args or {}}],+ "observation": {"results": [result]},+ }+++def tree(*, external=False):+ child = {+ "schema_version": "ATIF-v1.7",+ "trajectory_id": "child",+ "agent": {"name": "claude-code", "version": "test"},+ "steps": [call("c", "mcp__docs__search")],+ }+ ref = {"trajectory_path": "child.json"} if external else {"trajectory_id": "child"}+ root = {+ "schema_version": "ATIF-v1.7",+ "trajectory_id": "root",+ "agent": {"name": "claude-code", "version": "test"},+ "steps": [call("spawn", "Agent", {"subagent_type": "reviewer"}, refs=[ref])],+ }+ if not external:+ root["subagent_trajectories"] = [child]+ return root, child+++def grade(trajectory):+ return signals.compute_plugin_signals(+ trajectory, {"expected_tools": ["mcp:docs/search"]}, declared={"mcp": ["docs"], "subagent": ["reviewer"]}+ )++[email protected]("server", [None, "docs"])[email protected]("output", ["same answer", "unmatched answer"])+def test_ambiguous_codex_items_do_not_borrow_identity_or_success(server, output):+ items = [+ {"tool": "search", "args": "{}", "server": "docs", "status": "completed", "text": "same answer"},+ {"tool": "search", "args": "{}", "server": server or "kb", "status": "failed", "text": "same answer"},+ ]+ assert artifacts._take_codex_item(items, tool="search", args="{}", server=server, output=output) is None+ assert len(items) == 2+++def test_long_codex_argument_keys_cannot_match_by_a_shared_prefix():+ first, second = {"q": "x" * 5000 + "one"}, {"q": "x" * 5000 + "two"}+ key = artifacts._codex_args_key(first)+ other = artifacts._codex_args_key(second)+ assert key != other or key is None+ assert (+ artifacts._take_codex_item(+ [{"tool": "search", "args": other, "server": "docs", "status": "completed"}],+ tool="search",+ args=key,+ server="docs",+ )+ is None+ )+++def test_embedded_executed_child_contributes_calls_once_without_mutating_source():+ root, _ = tree()+ root["steps"][0]["observation"]["results"][0]["subagent_trajectory_ref"] *= 2+ before = json.dumps(root, sort_keys=True)+ result = grade(root)+ assert result["mcp_calls"]["succeeded"] == result["mcp_calls"]["total"] == 1+ assert result["tool_selection"]["recall"] == 1.0 and result["evidence_complete"] is True+ assert json.dumps(root, sort_keys=True) == before++[email protected]("kind", ["external", "missing", "cycle"])+def test_unresolved_or_cyclic_child_references_cannot_claim_complete_evidence(kind):+ root, child = tree(external=kind == "external")+ if kind == "missing":+ root.pop("subagent_trajectories")+ elif kind == "cycle":+ child["subagent_trajectories"] = [root]+ child["steps"][0]["observation"]["results"][0]["subagent_trajectory_ref"] = [{"trajectory_id": "root"}]+ assert grade(root)["evidence_complete"] is False++[email protected]("external", [False, True])+def test_adapter_materializes_authorized_child_artifacts(tmp_path, external):+ root, child = tree(external=external)+ agent = tmp_path / "trial/agent"+ agent.mkdir(parents=True)+ (agent / "trajectory.json").write_text(json.dumps(root))+ if external:+ (agent / "child.json").write_text(json.dumps(child))+ trace = artifacts.TrialTraceArtifacts().trace(agent.parent)+ assert grade(trace.trajectory)["mcp_calls"]["succeeded"] == 1+++def test_adapter_rejects_root_that_contradicts_authoritative_steps(tmp_path):+ root, _ = tree()+ trial = tmp_path / "trial"+ (trial / "agent").mkdir(parents=True)+ (trial / "agent/trajectory.json").write_text(json.dumps(root))+ (trial / "result.json").write_text(json.dumps({"step_results": [{"step_name": "one"}]}))+ assert artifacts.TrialTraceArtifacts().trace(trial).trajectory is None+++def test_redaction_drops_a_secret_cut_after_a_shrinking_assignment():+ secret, bound = "ghp_" + "B" * 36, 256 * 4 + 256+ prefix = "token=" + "a" * (bound - 39 - 7) + " "+ rendered = signals._safe_text(prefix + secret)+ assert "ghp_" not in rendered++[email protected]("control", ["\x01", "\x9b", "\x1b[31m", "\x1b]0;title\x07"])+def test_diagnostics_redact_credentials_after_removing_controls(control):+ from skillevaluator.tier3 import mcp_proof, plugin_evidence++ value = "ghp_" + "B" * 33 + control + "C" * 3+ for render in (signals._safe_text, plugin_evidence._safe_text, mcp_proof._detail):+ text = render(value)+ assert "ghp_" not in text+ assert "B" * 12 not in text+ assert "redacted" in text+++def test_malformed_codex_types_do_not_discard_valid_log_items(tmp_path):+ root = tmp_path / "trial"+ agent = root / "agent"+ sessions = agent / "sessions"+ sessions.mkdir(parents=True)+ trajectory = {+ "schema_version": "ATIF-v1.7",+ "agent": {"name": "codex", "version": "test"},+ "steps": [call("c", "search")],+ }+ (agent / "trajectory.json").write_text(json.dumps(trajectory))+ (agent / "codex.txt").write_text(+ "\n".join(+ json.dumps(event)+ for event in [+ {"type": [], "mcp": 1},+ {+ "type": "item.completed",+ "item": {+ "type": "mcp_tool_call",+ "id": "i",+ "server": "docs",+ "tool": "search",+ "arguments": {},+ "status": "failed",+ },+ },+ ]+ )+ )+ (sessions / "session.jsonl").write_text(+ "\n".join(+ json.dumps({"payload": payload})+ for payload in [+ {"item": {"type": [], "mcp": 1}},+ {"type": {}, "mcp": 1},+ {+ "type": "function_call",+ "namespace": "mcp__docs",+ "name": "search",+ "call_id": "c",+ "arguments": "{}",+ },+ ]+ )+ )+ trace = artifacts.TrialTraceArtifacts().trace(root)+ result = signals.compute_plugin_signals(+ trace.trajectory, {}, declared={"mcp": ["docs"]}, mcp_call_servers=trace.call_servers+ )+ assert trace.call_servers == {"c": "docs"}+ assert result["mcp_calls"]["failed"] == result["mcp_calls"]["total"] == 1++[email protected]("overlay", ["_codex_mcp_calls", "_opencode_call_errors"])+def test_harness_overlay_failure_keeps_plain_trajectory_and_other_trials(tmp_path, monkeypatch, overlay):+ original = getattr(artifacts, overlay)++ def failing_overlay(root, *args):+ if root.name == "bad":+ raise TypeError("malformed harness event")+ return original(root, *args)++ monkeypatch.setattr(artifacts, overlay, failing_overlay)+ cache = artifacts.TrialTraceArtifacts()+ for name in ("bad", "good"):+ agent = tmp_path / name / "agent"+ agent.mkdir(parents=True)+ (agent / "trajectory.json").write_text(+ json.dumps(+ {+ "schema_version": "ATIF-v1.7",+ "agent": {"name": "codex", "version": "test"},+ "steps": [call("c", "mcp__docs__search")],+ }+ )+ )+ trace = cache.trace(agent.parent)+ assert grade(trace.trajectory)["mcp_calls"]["succeeded"] == 1+ assert cache.trace(agent.parent) is trace+++def test_unstaged_file_server_is_skipped_without_hiding_a_portable_server(tmp_path):+ from dataclasses import replace+ from types import SimpleNamespace++ from tests.tier3.test_plugin_evidence import prepared++ from skillevaluator.plugin_inventory import plugin_inventory_for_root+ from skillevaluator.tier3.plugin_evidence import EvidenceOptions, RuntimeEvidenceObserver+ from skillevaluator.tier3.plugin_native import wrapper_decision+ from skillevaluator.tier3.plugin_prepare import prepare_runtime_package, stage_wrapper_skill++ prepared(tmp_path / "plugin")+ manifest = tmp_path / "plugin/.claude-plugin/plugin.json"+ manifest.write_text(+ json.dumps(+ {+ "name": "demo",+ "mcpServers": {+ "docs": {"url": "https://docs.example.invalid/mcp"},+ "local": {"command": "python", "args": ["./server.py"]},+ },+ }+ )+ )+ (tmp_path / "plugin/server.py").write_text("print('fixture')")+ package = prepare_runtime_package(plugin_inventory_for_root(tmp_path / "plugin"))+ assert {server["name"] for server in package.native_source.plugin_file_mcp_servers} == {"local"}+ wrapper = stage_wrapper_skill(package, tmp_path / "wrapper")+ hooks = SimpleNamespace(decisions={"codex": wrapper_decision("codex", "fixture")}, stagings={})+ observer = RuntimeEvidenceObserver(package, wrapper, hooks, EvidenceOptions())+ observer._context_for_staging()+ declared = observer.context.declared_for("with_skill", "codex")+ assert declared["mcp"] == ["docs"]+ result = signals.compute_plugin_signals(+ {"steps": [], "agent": {"name": "codex"}},+ {"expected_tools": ["mcp:local/tool", "mcp:docs/search"]},+ declared=declared,+ )+ assert result["tool_selection"]["skipped"] == ["mcp:local/tool"]+ assert result["tool_selection"]["recall"] == 0.0+ native = replace(observer.context, agent_mcp_servers={"claude-code": ("local",)})+ assert set(native.declared_for("with_skill", "claude-code")["mcp"]) == {"docs", "local"}++[email protected]("known_owner", [True, False])+def test_child_handoff_uses_its_referenced_parent_not_another_parallel_agent(known_owner):+ root, child = tree()+ parent = root["steps"][0]+ parent["tool_calls"].append(+ {"tool_call_id": "other", "function_name": "Agent", "arguments": {"subagent_type": "distractor"}}+ )+ result = parent["observation"]["results"][0]+ if not known_owner:+ result["source_call_id"] = "missing-parent"+ child["steps"][0]["observation"]["results"][0]["content"] = "carried-value"+ root["steps"].append(call("consume", "mcp__docs__consume", {"value": "carried-value"}))+ result = signals.compute_plugin_signals(+ root,+ {"handoffs": [{"producer": "Agent:reviewer", "consumer": "MCP:docs/consume", "value": "carried-value"}]},+ declared={"mcp": ["docs"], "subagent": ["reviewer", "distractor"]},+ )+ assert result["handoff"]["passed"] == int(known_owner)+ assert result["evidence_complete"] is known_owner+++def test_child_work_shares_the_global_step_budget(monkeypatch):+ root, _ = tree()+ root["steps"].append(call("later", "mcp__docs__search"))+ monkeypatch.setattr(signals, "_MAX_STEPS", 2)+ result = grade(root)+ assert result["mcp_calls"]["total"] == 1+ assert result["evidence_complete"] is False and "steps" in result["limits"]++[email protected]("native", [False, True])+def test_prepared_agent_components_keep_their_declared_names_and_staging_scope(tmp_path, native):+ from types import SimpleNamespace++ from tests.tier3.test_plugin_evidence import prepared++ from skillevaluator.plugin_inventory import plugin_inventory_for_root+ from skillevaluator.tier3.plugin_evidence import EvidenceOptions, RuntimeEvidenceObserver+ from skillevaluator.tier3.plugin_native import wrapper_decision+ from skillevaluator.tier3.plugin_prepare import prepare_runtime_package, stage_wrapper_skill++ prepared(tmp_path / "plugin")+ agents = tmp_path / "plugin/agents"+ agents.mkdir()+ (agents / "reviewer.md").write_text("---\nname: reviewer\ndescription: Review.\n---\nReview the work.\n")+ package = prepare_runtime_package(plugin_inventory_for_root(tmp_path / "plugin"))+ assert any(text.type == "agent" and text.name == "reviewer" for text in package.native_source.texts)+ wrapper = stage_wrapper_skill(package, tmp_path / "wrapper")+ decision = wrapper_decision("codex", "fixture")+ staging = SimpleNamespace(+ bundle=SimpleNamespace(declared=[{"type": "agent", "name": "reviewer"}]), adapter=SimpleNamespace()+ )+ hooks = SimpleNamespace(+ decisions={"codex": SimpleNamespace(native=True) if native else decision},+ stagings={"codex": staging} if native else {},+ )+ observer = RuntimeEvidenceObserver(package, wrapper, hooks, EvidenceOptions())+ observer._context_for_staging()+ assert observer.context.subagents == ("reviewer",)+ declared = observer.context.declared_for("with_skill", "codex")+ result = signals.compute_plugin_signals(+ {"steps": [], "agent": {"name": "claude-code"}}, {"expected_tools": ["Agent:reviewer"]}, declared=declared+ )+ assert result["routing"]["skipped"] == ([] if native else ["Agent:reviewer"])+++def test_ambiguous_same_server_codex_outcomes_remain_unknown_after_overlay(tmp_path):+ from tests.tier3.test_f90_mcp_execution import _codex_items, _codex_step, _collect_with_arm, _traj++ args = {"q": "same"}+ trial = {+ "trajectory": _traj(_codex_step("call", "search", args, "same answer"), agent="codex"),+ "session": None,+ "codex_txt": [+ *_codex_items("one", "docs", "search", args, "same answer", "completed"),+ *_codex_items("two", "docs", "search", args, "same answer", "failed"),+ ],+ }+ summary = _collect_with_arm(tmp_path, {"case-1__attempt1": trial}, servers=["docs"])+ assert summary["mcp_calls"]["succeeded"] == 0+ assert summary["mcp_calls"]["unknown"] == 1diff --git a/tests/tier3/test_plugin_signals.py b/tests/tier3/test_plugin_signals.pynew file mode 100644index 00000000..d6a5e191--- /dev/null+++ b/tests/tier3/test_plugin_signals.py@@ -0,0 +1,1477 @@+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+# SPDX-License-Identifier: Apache-2.0++"""Report-only plugin component signals computed from ATIF trajectories."""++from __future__ import annotations++import json+from typing import Any++import pytest++from skillevaluator.tier3.eval_core import checks, plugin_signals+from skillevaluator.tier3.eval_core.plugin_signals import (+ MAX_TOOL_PATTERNS,+ build_plugin_signals_context,+ compute_plugin_signals,+ declared_mcp_server_matcher,+ match_declared_mcp_server,+ plugin_case_spec,+ summarize_plugin_signals,+ validate_plugin_case_fields,+)++DECLARED = {"skill": ["alpha", "beta"], "mcp": ["github", "jira"]}+++def test_signal_labels_omit_credentials_cut_at_the_redaction_window():+ prefix = ("ghp_" + "a" * 150 + " ") * 8+ prefix += " " * (1274 - len(prefix))+ assert "sk-Ab1" not in plugin_signals._safe_text(prefix + "sk-Ab1Cd2Ef3Gh4Ij5Kl6Mn7Op8")+++def test_signal_scan_reports_a_call_limit_instead_of_a_complete_negative(monkeypatch):+ monkeypatch.setattr(plugin_signals, "_MAX_CALLS", 1)+ signals = _signals(+ _traj(+ _one("Read", {"file_path": "/workspace/a"}), _one("mcp__github__write", {"value": "later"}, call_id="late")+ )+ )+ assert signals["evidence_complete"] is False+ assert "calls" in signals["limits"]+ summary = summarize_plugin_signals([signals])+ assert summary["n_incomplete_trajectory"] == 1+++def _tc(fn: str, args: dict[str, Any] | None = None, call_id: str = "c1") -> dict[str, Any]:+ return {"tool_call_id": call_id, "function_name": fn, "arguments": args or {}}+++def _res(call_id: str, content: Any, **extra: Any) -> dict[str, Any]:+ return {"source_call_id": call_id, "content": content, **extra}+++def _step(tool_calls: list[dict[str, Any]], results: list[dict[str, Any]] | None = None) -> dict[str, Any]:+ return {"source": "agent", "tool_calls": tool_calls, "observation": {"results": results or []}}+++def _traj(*steps: dict[str, Any], prompt: str = "Please help.") -> dict[str, Any]:+ return {"schema_version": "ATIF-v1.2", "steps": [{"source": "user", "message": prompt}, *steps]}+++def _one(fn: str, args: dict[str, Any] | None = None, content: Any = "ok", call_id: str = "c1", **extra: Any):+ return _step([_tc(fn, args, call_id)], [_res(call_id, content, **extra)])+++def _signals(traj: dict[str, Any], case: dict[str, Any] | None = None, **kwargs: Any) -> dict[str, Any]:+ kwargs.setdefault("declared", DECLARED)+ signals = compute_plugin_signals(traj, case or {}, **kwargs)+ assert signals is not None+ return signals+++def _activations(traj: dict[str, Any]) -> list[dict[str, Any]]:+ return _signals(traj)["activations"]+++def _record_regex_timeouts(monkeypatch: pytest.MonkeyPatch) -> list[float]:+ """Record the deadline of every dataset-pattern search (a search without one fails)."""+ timeouts: list[float] = []+ search = plugin_signals.regex.search++ def search_with_deadline(pattern: str, text: str, *, timeout: float) -> Any:+ timeouts.append(timeout)+ return search(pattern, text, timeout=timeout)++ monkeypatch.setattr(plugin_signals.regex, "search", search_with_deadline)+ return timeouts+++def _codex_exec(source: str, call_id: str = "exec-1") -> dict[str, Any]:+ return {"tool_call_id": call_id, "function_name": "exec", "arguments": {"input": source}}+++# ---------------------------------------------------------------------------+# Component activation classifier+# ---------------------------------------------------------------------------+++class TestClassifierClaudeStyle:+ def test_skill_mcp_subagent_and_command_keep_identity_order_and_outcome(self) -> None:+ traj = _traj(+ _one("Skill", {"skill": "alpha"}, "Launching skill: alpha"),+ _one("mcp__github__list_issues", {"repo": "o/r"}, [{"type": "text", "text": "[]"}], call_id="c2"),+ _one("Task", {"subagent_type": "reviewer", "prompt": "review"}, "looks good", call_id="c3"),+ _one("SlashCommand", {"command": "/deploy prod"}, "deployed", call_id="c4"),+ _one("Read", {"file_path": "/workspace/data.csv"}, "a,b", call_id="c5"),+ )++ activations = _activations(traj)++ assert activations == [+ {"type": "skill", "name": "alpha", "tool": "Skill", "server": None, "step_index": 1, "succeeded": True},+ {+ "type": "mcp",+ "name": "github",+ "tool": "mcp__github__list_issues",+ "server": "github",+ "step_index": 2,+ "succeeded": True,+ },+ {+ "type": "subagent",+ "name": "reviewer",+ "tool": "Task",+ "server": None,+ "step_index": 3,+ "succeeded": True,+ },+ {+ "type": "command",+ "name": "deploy",+ "tool": "SlashCommand",+ "server": None,+ "step_index": 4,+ "succeeded": True,+ },+ ]++ def test_read_of_declared_member_manifest_counts_as_skill_activation(self) -> None:+ traj = _traj(+ _one("Read", {"file_path": "/workspace/.claude/skills/beta/SKILL.md"}, "# beta"),+ _one("Read", {"file_path": "/workspace/.claude/skills/other/SKILL.md"}, "# other", call_id="c2"),+ _one("Read", {"file_path": "/workspace/skills/beta/SKILL.md.bak"}, "# old", call_id="c3"),+ )++ activations = _activations(traj)++ assert [(a["type"], a["name"], a["tool"]) for a in activations] == [("skill", "beta", "Read:skill-md-read")]++ def test_ordinary_tools_are_not_activations(self) -> None:+ traj = _traj(_one("Bash", {"command": "ls"}), _one("Write", {"file_path": "x"}, call_id="c2"))+ assert _activations(traj) == []++ def test_mcp_filesystem_read_of_member_manifest_is_both_mcp_and_skill(self) -> None:+ traj = _traj(_one("mcp__filesystem__read_file", {"path": "/workspace/skills/alpha/SKILL.md"}, "# alpha"))++ activations = _activations(traj)++ assert [(a["type"], a["name"]) for a in activations] == [("mcp", "filesystem"), ("skill", "alpha")]+++class TestClassifierCodexStyle:+ def test_shell_read_of_declared_manifest_counts(self) -> None:+ traj = _traj(+ _one("exec_command", {"cmd": ["bash", "-lc", "cd /workspace && sed -n '1,80p' skills/alpha/SKILL.md"]}),+ _one("shell", {"command": "FOO=1 cat skills/beta/SKILL.md | head"}, call_id="c2"),+ )++ activations = _activations(traj)++ assert [(a["name"], a["tool"]) for a in activations] == [+ ("alpha", "exec_command:skill-md-read"),+ ("beta", "shell:skill-md-read"),+ ]++ @pytest.mark.parametrize(+ "command",+ [+ "rm skills/alpha/SKILL.md",+ "ls skills/alpha/SKILL.md",+ "grep name skills/alpha/SKILL.md",+ "printf skills/alpha/SKILL.md",+ "cat skills/alpha/SKILL.md.bak",+ "cat skills/wrapper/SKILL.md",+ "echo hi > skills/alpha/SKILL.md",+ ],+ )+ def test_non_read_or_undeclared_shell_mentions_do_not_count(self, command: str) -> None:+ traj = _traj(_one("exec_command", {"cmd": command}))+ assert _activations(traj) == []++ @pytest.mark.parametrize(+ "command",+ [+ # The apostrophe sits in a here-document body, so the script is valid bash.+ "ls skills\ncat skills/alpha/SKILL.md\ncat > notes.md <<'EOF'\nIt's fine\nEOF",+ # A genuinely unbalanced quote on a later line must not merge the lines.+ "cd /workspace\nsed -n '1,120p' skills/alpha/SKILL.md\necho Don't forget",+ ],+ )+ def test_multiline_script_with_a_stray_quote_still_credits_the_read(self, command: str) -> None:+ (activation,) = _activations(_traj(_one("exec_command", {"cmd": command})))+ assert (activation["type"], activation["name"]) == ("skill", "alpha")++ def test_heredoc_body_is_data_not_a_manifest_read(self) -> None:+ command = "cat > notes.md <<EOF\ncat skills/alpha/SKILL.md\nEOF\nls"+ assert _activations(_traj(_one("exec_command", {"cmd": command}))) == []++ def test_native_exec_wrapper_is_normalized_and_mapped_observation_is_used(self) -> None:+ source = 'const r = await tools.mcp__github__list_issues({repo: "o/r"});\ntext(JSON.stringify(r));'+ traj = _traj(_step([_codex_exec(source)], [_res("exec-1", "[1, 2]")]))++ (activation,) = _activations(traj)++ assert activation["tool"] == "mcp__github__list_issues"+ assert activation["succeeded"] is True++ def test_native_exec_inner_calls_without_proven_observation_are_unknown(self) -> None:+ source = (+ 'const a = await tools.mcp__github__list_issues({repo: "o/r"});\n'+ 'const b = await tools.mcp__jira__create_ticket({title: "x"});\n'+ "text(JSON.stringify(a));\ntext(JSON.stringify(b));"+ )+ traj = _traj(_step([_codex_exec(source)], [_res("exec-1", "403: forbidden")]))++ activations = _activations(traj)++ assert [(a["tool"], a["succeeded"]) for a in activations] == [+ ("mcp__github__list_issues", None),+ ("mcp__jira__create_ticket", None),+ ]++ @pytest.mark.parametrize(+ "fn", ["mcp__fs__get_document", "fs__get_document", "fs.get_document", "mcp_fs_get_document", "fs_get_document"]+ )+ def test_every_spelling_of_a_declared_server_read_credits_the_member(self, fn: str) -> None:+ traj = _traj(_one(fn, {"path": "/workspace/skills/alpha/SKILL.md"}, "# alpha"))++ activations = _signals(traj, declared={**DECLARED, "mcp": ["fs"]})["activations"]++ assert [(a["type"], a["name"], a["tool"]) for a in activations] == [+ ("mcp", "fs", "mcp__fs__get_document"),+ ("skill", "alpha", f"{fn}:skill-md-read"),+ ]++ @pytest.mark.parametrize("fn", ["mcp__fs__bash", "fs__bash"])+ def test_mcp_tool_named_like_a_shell_is_not_parsed_as_one(self, fn: str) -> None:+ traj = _traj(_one(fn, {"command": "cat skills/alpha/SKILL.md"}))++ activations = _signals(traj, declared={**DECLARED, "mcp": ["fs"]})["activations"]++ assert [(a["type"], a["name"]) for a in activations] == [("mcp", "fs")]++ @pytest.mark.parametrize("fn", ["mcp__fs__edit_file", "mcp_fs_edit_file"])+ def test_every_spelling_of_an_mcp_edit_tool_writes_the_artifact(self, fn: str) -> None:+ traj = _traj(+ _one("Skill", {"skill": "alpha"}),+ _one(fn, {"path": "out/report.json", "edits": []}, call_id="c2"),+ _one("Skill", {"skill": "beta"}, call_id="c3"),+ _one("Read", {"file_path": "out/report.json"}, call_id="c4"),+ )+ case = {"handoffs": [{"producer": "Skill:alpha", "consumer": "Skill:beta", "artifact": "out/report.json"}]}++ signals = _signals(traj, case, declared={**DECLARED, "mcp": ["fs"]})++ assert signals["handoff"]["passed"] == 1++ @pytest.mark.parametrize(+ ("agent", "fn", "key"),+ [+ ("claude-code", "mcp__fs__write_file", "destination"),+ ("hermes", "mcp_fs_write_file", "destination"),+ ("hermes", "mcp_fs_save_file", "output"),+ ("opencode", "fs_write_file", "destination"),+ ("opencode", "fs_write_file", "file"),+ ("opencode", "fs_edit_file", "target"),+ ],+ )+ def test_mcp_write_tools_without_a_path_key_still_write_the_artifact(self, agent: str, fn: str, key: str) -> None:+ traj = _traj(+ _one("Skill", {"skill": "alpha"}),+ _one(fn, {key: "out/report.json", "content": "{}"}, call_id="c2"),+ _one("Skill", {"skill": "beta"}, call_id="c3"),+ _one("Read", {"file_path": "out/report.json"}, call_id="c4"),+ )+ case = {"handoffs": [{"producer": "Skill:alpha", "consumer": "Skill:beta", "artifact": "out/report.json"}]}++ signals = _signals({**traj, "agent": {"name": agent}}, case, declared={**DECLARED, "mcp": ["fs"]})++ assert (signals["handoff"]["passed"], signals["handoff"]["failures"]) == (1, [])++ def test_an_mcp_editor_view_does_not_write_the_artifact(self) -> None:+ traj = _traj(+ _one("Skill", {"skill": "alpha"}),+ _one("mcp__fs__str_replace_editor", {"command": "view", "target": "out/report.json"}, call_id="c2"),+ _one("Skill", {"skill": "beta"}, call_id="c3"),+ _one("Read", {"file_path": "out/report.json"}, call_id="c4"),+ )+ case = {"handoffs": [{"producer": "Skill:alpha", "consumer": "Skill:beta", "artifact": "out/report.json"}]}++ signals = _signals(traj, case, declared={**DECLARED, "mcp": ["fs"]})++ assert [failure["detail"] for failure in signals["handoff"]["failures"]] == [+ "artifact was not written by the producer"+ ]++ def test_declared_server_alternate_spellings_are_canonicalized(self) -> None:+ traj = _traj(+ _one("github.search_code", {"q": "x"}),+ _one("jira__create_ticket", {"title": "x"}, call_id="c2"),+ _one("slack.post", {"text": "x"}, call_id="c3"),+ )++ activations = _activations(traj)++ assert [a["tool"] for a in activations] == ["mcp__github__search_code", "mcp__jira__create_ticket"]+++class TestMcpServerNames:+ @pytest.mark.parametrize(+ ("observed", "declared", "expected"),+ [+ ("GitHub", ["github", "jira"], "github"),+ ("my_docs", ["my.docs", "my_docs"], "my_docs"), # an exact name beats another server's spelling+ ("my_docs", ["my.docs"], "my.docs"),+ ("my_docs", ["my.docs", "my-docs"], None), # a spelling two servers share credits neither+ ("plugin_demo-plugin_docs", ["docs"], "docs"),+ ("plugin_demo_my_docs", ["my.docs"], "my.docs"),+ ("plugin_a_b_team_docs", ["team_docs", "docs"], "team_docs"), # the longest declared suffix+ ("plugin_x_y_my_docs", ["my.docs", "my-docs"], None),+ ("plugin_", ["docs"], None),+ ("slack", ["github"], None),+ ("github", [], None),+ ],+ )+ def test_observed_server_names_map_to_one_declared_server(+ self, observed: str, declared: list[str], expected: str | None+ ) -> None:+ assert match_declared_mcp_server(observed, declared) == expected+ assert declared_mcp_server_matcher(declared)(observed) == expected++ def test_declared_servers_are_indexed_once_per_trajectory(self, monkeypatch: pytest.MonkeyPatch) -> None:+ builds: list[object] = []+ build = plugin_signals._McpNames.__init__++ def counting_build(self: Any, declared: Any) -> None:+ builds.append(self)+ build(self, declared)++ monkeypatch.setattr(plugin_signals._McpNames, "__init__", counting_build)+ traj = _traj(_one("mcp__github__get_issue"), _one("jira__create_ticket", call_id="c2"))++ signals = _signals(traj)++ assert len(builds) == 1+ assert signals["activation_coverage"]["exercised"] == ["mcp:github", "mcp:jira"]++ @pytest.mark.parametrize(+ ("agent", "fn", "servers", "tool"),+ [+ ("claude-code", "my.docs__search", ["my.docs", "my"], "mcp__my.docs__search"), # the longest prefix wins+ ("claude-code", "my_docs__search", ["my.docs", "my-docs"], "mcp__my_docs__search"),+ ("hermes", "mcp_my_docs_search", ["my.docs", "my-docs"], "mcp__my_docs__search"),+ ("opencode", "github_team.list", ["github", "github_team"], "mcp__github_team__list"),+ ("opencode", "docs_search", ["docs"], "mcp__docs__search"),+ ("codex", "docs_search", ["docs"], None), # only OpenCode names MCP tools <server>_<tool>+ ("opencode", "web_search", ["web"], None), # a built-in tool, not the web server's+ ("claude-code", "mcp__plugin_demo_my_docs__get", ["my.docs"], "mcp__my.docs__get"),+ ],+ )+ def test_tool_names_map_to_declared_servers(+ self, agent: str, fn: str, servers: list[str], tool: str | None+ ) -> None:+ traj = {**_traj(_one(fn)), "agent": {"name": agent}}++ activations = _signals(traj, declared={"skill": [], "mcp": servers})["activations"]++ assert [a["tool"] for a in activations if a["type"] == "mcp"] == ([tool] if tool else [])+++class TestShellReadsAgainstTheScoredCheck:+ """The report-only SKILL.md read detector is deliberately more lenient than the scored check."""++ @staticmethod+ def _credited(command: str) -> list[str]:+ activations = _signals(_traj(_one("exec_command", {"cmd": command})))["activations"]+ return [a["name"] for a in activations if a["type"] == "skill"]++ @pytest.mark.parametrize("verb", sorted(checks._FILE_READ_VERBS))+ def test_every_scored_reader_verb_credits_the_member(self, verb: str) -> None:+ command = f"{verb} skills/alpha/SKILL.md"+ assert checks._cmd_reads_skill_md(command)+ assert self._credited(command) == ["alpha"]++ @pytest.mark.parametrize(+ "command",+ ["sudo cat skills/alpha/SKILL.md", "bash -lc 'cat skills/alpha/SKILL.md'", "tac skills/alpha/SKILL.md"],+ )+ def test_lenient_reads_credit_the_member_but_not_the_score(self, command: str) -> None:+ assert not checks._cmd_reads_skill_md(command)+ assert self._credited(command) == ["alpha"]++ def test_shell_variables_are_expanded_only_by_the_scored_check(self) -> None:+ command = "D=skills/alpha; cat $D/SKILL.md"+ assert checks._cmd_reads_skill_md(command)+ assert self._credited(command) == []+++class TestOutcomeTriState:+ def test_structured_error_flag_marks_failure(self) -> None:+ traj = _traj(_one("mcp__github__x", content="{}", is_error=True))+ assert _activations(traj)[0]["succeeded"] is False++ @pytest.mark.parametrize(+ "content",+ [+ "403: Failed to decrypt access token",+ "<tool_use_error>No such tool</tool_use_error>",+ "MCP error -32602",+ # A status line needs its reason phrase: "500: boom" is an answer the agent saw (proof M32).+ "500: Internal Server Error",+ "HTTP/1.1 404 - Not Found",+ # An HTTP/x.y status line fails without a reason phrase (#180's 3a46e21 row).+ "HTTP/1.1 404 - x",+ "HTTP/2 500",+ "Done.\nstatus code 502: bad gateway",+ # A status after an error word or a request line, not at the start of the line.+ "Request failed with status 404: Not Found",+ "Error calling tool: 404: not found",+ "GET /repos/x: 404 - Not Found",+ ],+ )+ def test_failure_markers_mark_failure(self, content: str) -> None:+ traj = _traj(_one("mcp__github__x", content=content))+ assert _activations(traj)[0]["succeeded"] is False++ @pytest.mark.parametrize(+ "content",+ [+ "Open issues:\n#451 - Crash",+ "Found in src/app.py:404: def handler()",+ "Total: 500 - all good",+ "The error handler is in src/app.py:404: def handler()",+ # A server's own "Error: ..." text with no flag is an answer; its status still needs a reason phrase.+ "Error: 500 - boom",+ "HTTP/1.1 404 responses are cached by the CDN for an hour.",+ ],+ )+ def test_status_like_numbers_inside_a_successful_answer_are_not_failures(self, content: str) -> None:+ traj = _traj(_one("mcp__github__x", content=content))+ assert _activations(traj)[0]["succeeded"] is True++ def test_failure_marker_mid_way_through_a_long_successful_body_is_ignored(self) -> None:+ # Scanning is bounded: each block's head and the result's tail, not the middle.+ traj = _traj(_one("mcp__github__x", content="x" * 5000 + " permission denied " + "y" * 5000))+ assert _activations(traj)[0]["succeeded"] is True++ def test_failure_marker_at_the_end_of_a_long_body_marks_failure(self) -> None:+ traj = _traj(_one("mcp__github__x", content="x" * 2100 + "\n401 - Unauthorized"))+ assert _activations(traj)[0]["succeeded"] is False++ def test_error_block_after_a_long_first_block_marks_failure(self) -> None:+ content = [+ {"type": "text", "text": "Search results preamble " + "." * 2100},+ {"type": "text", "text": "Error: 403: Failed to decrypt access token"},+ {"type": "text", "text": "z" * 3000},+ ]+ traj = _traj(_one("mcp__github__x", content=content))+ assert _activations(traj)[0]["succeeded"] is False++ def test_missing_sibling_file_does_not_fail_a_shell_manifest_read(self) -> None:+ output = "cat: skills/alpha/REFERENCE.md: No such file or directory\n---\nname: alpha\n---\n"+ traj = _traj(_one("exec_command", {"cmd": "cat skills/alpha/SKILL.md skills/alpha/REFERENCE.md"}, output))+ assert _activations(traj)[0]["succeeded"] is True++ @pytest.mark.parametrize("fn", ["Read", "mcp__github__read_file"])+ def test_missing_file_still_fails_mcp_and_file_read_tools(self, fn: str) -> None:+ traj = _traj(_one(fn, {"file_path": "skills/beta/SKILL.md"}, "Error: file does not exist"))+ assert {a["succeeded"] for a in _activations(traj)} == {False}++ def test_sibling_result_never_proves_success(self) -> None:+ traj = _traj(+ _step(+ [_tc("mcp__github__x", call_id="c1"), _tc("Bash", {"command": "ls"}, call_id="c2")],+ [_res("c2", "files")],+ )+ )+ assert _activations(traj)[0]["succeeded"] is None++ def test_idless_result_is_used_only_for_single_call_steps(self) -> None:+ single = _traj(_step([_tc("mcp__github__x", call_id="")], [{"content": "ok"}]))+ multi = _traj(+ _step([_tc("mcp__github__x", call_id=""), _tc("mcp__jira__y", call_id="")], [{"content": "ok"}]),+ )+ assert _activations(single)[0]["succeeded"] is True+ assert [a["succeeded"] for a in _activations(multi)] == [None, None]++ def test_empty_correlated_body_is_unknown(self) -> None:+ traj = _traj(_one("mcp__github__x", content=" "))+ assert _activations(traj)[0]["succeeded"] is None++ def test_string_arguments_are_decoded(self) -> None:+ traj = _traj(_step([{"tool_call_id": "c1", "function_name": "Skill", "arguments": '{"skill": "beta"}'}]))+ assert _activations(traj)[0]["name"] == "beta"+++def test_unreadable_trajectory_yields_none() -> None:+ assert compute_plugin_signals(None, {}) is None+ assert compute_plugin_signals({"steps": "nope"}, {}) is None+++# ---------------------------------------------------------------------------+# Graders+# ---------------------------------------------------------------------------++EMPTY_CASE_STATUSES = ("routing", "tool_selection", "arguments", "order", "handoff", "conflict")+++def test_case_without_fields_is_not_applicable_everywhere() -> None:+ signals = _signals(_traj(_one("Skill", {"skill": "alpha"})))+ for key in EMPTY_CASE_STATUSES:+ assert signals[key]["status"] == "not_applicable", key+ assert signals["routing"]["precision"] is None+ # Check 15 (routing) sees the skill; check 22 (tool selection) does not.+ assert signals["routing"]["called"] == ["Skill:alpha"]+ assert signals["tool_selection"]["called"] == []+ assert set(signals) == {+ "evidence_complete",+ "limits",+ "activations",+ "routing",+ "tool_selection",+ "arguments",+ "mcp_calls",+ "order",+ "handoff",+ "conflict",+ "activation_coverage",+ }+++class TestToolSelection:+ def test_exact_glob_skill_form_acceptable_and_decoys(self) -> None:+ traj = _traj(+ _one("Skill", {"skill": "alpha"}),+ _one("mcp__github__list_issues", call_id="c2"),+ _one("mcp__github__get_issue", call_id="c3"),+ _one("mcp__slack__post", call_id="c4"),+ _one("mcp__slack__post", call_id="c5"),+ _one("Bash", {"command": "ls"}, call_id="c6"),+ )+ case = {+ "expected_tools": ["Skill:alpha", "mcp__github__list_*", "mcp__jira__create_ticket"],+ "acceptable_tools": ["mcp__github__get_issue"],+ "decoy_tools": ["mcp__slack__*"],+ }++ signals = _signals(traj, case)+ block = signals["tool_selection"]++ # Check 22 scores the tool refs only; the skill ref is check 15's (routing).+ assert block["called"] == [+ "mcp__github__list_issues",+ "mcp__github__get_issue",+ "mcp__slack__post",+ ]+ assert block["precision"] == pytest.approx(0.6667)+ assert block["recall"] == 0.5+ assert block["f1"] == pytest.approx(0.5714, abs=1e-4)+ assert block["decoy_calls"] == 2+ assert block["status"] == "scored"+ routing = signals["routing"]+ assert routing["called"] == ["Skill:alpha"]+ assert (routing["precision"], routing["recall"], routing["decoy_calls"]) == (1.0, 1.0, 0)++ def test_listed_plain_tool_enters_scope_but_unlisted_ones_do_not(self) -> None:+ traj = _traj(_one("Bash", {"command": "ls"}), _one("Read", {"file_path": "x"}, call_id="c2"))+ block = _signals(traj, {"acceptable_tools": ["bash"]})["tool_selection"]+ assert block["called"] == ["Bash"]+ assert block["precision"] == 1.0+ assert block["recall"] is None+ assert block["f1"] is None++ def test_nothing_called_with_expectations_has_undefined_precision(self) -> None:+ block = _signals(_traj(), {"expected_tools": ["Skill:alpha"]})["routing"]+ assert (block["precision"], block["recall"], block["f1"]) == (None, 0.0, 0.0)++ def test_decoy_only_case_with_nothing_in_scope_has_undefined_precision(self) -> None:+ block = _signals(_traj(_one("Bash")), {"decoy_tools": ["mcp__slack__*"]})["tool_selection"]+ assert block["status"] == "scored"+ assert block["precision"] is None+ assert block["decoy_calls"] == 0++ def test_generated_wrapper_skill_is_not_a_selection(self) -> None:+ traj = _traj(_one("Skill", {"skill": "my-plugin"}), _one("Skill", {"skill": "alpha"}, call_id="c2"))+ block = _signals(traj, {"expected_tools": ["Skill:alpha"]}, wrapper_skills=["my-plugin"])["routing"]+ assert block["called"] == ["Skill:alpha"]+ assert block["precision"] == 1.0++ def test_codex_manifest_read_satisfies_skill_ref(self) -> None:+ traj = _traj(_one("exec_command", {"cmd": "cat skills/alpha/SKILL.md"}))+ block = _signals(traj, {"expected_tools": ["skill:ALPHA"]})["routing"]+ assert block["recall"] == 1.0+++class TestArguments:+ def test_required_schema_and_semantic_constraints(self) -> None:+ traj = _traj(+ _one("mcp__github__search", {"query": "bug label:p1", "limit": 50, "opts": {"sort": "new"}}),+ _one("mcp__github__search", {"query": "feature", "limit": 5, "opts": {"sort": "old"}}, call_id="c2"),+ )+ case = {+ "tool_arguments": [+ {+ "tool": "mcp__github__search",+ "required": ["query", "opts.sort"],+ "schema": {+ "type": "object",+ "required": ["limit"],+ "properties": {+ "limit": {"type": "integer", "minimum": 1, "maximum": 20},+ "opts": {"type": "object", "properties": {"sort": {"enum": ["new", "top"]}}},+ },+ },+ "contains": {"query": "label:"},+ "pattern": {"query": "^bug\\b"},+ }+ ]+ }++ block = _signals(traj, case)["arguments"]++ assert block["checked"] == 2+ assert block["passed"] == 0+ rules = {(f["arg"], f["rule"]) for f in block["failures"]}+ assert ("limit", "schema") in rules+ assert ("opts.sort", "schema") in rules+ assert ("query", "contains") in rules+ assert ("query", "pattern") in rules++ def test_passing_call_and_equals_type_strictness(self) -> None:+ traj = _traj(+ _one("mcp__jira__create_ticket", {"project": "OPS", "priority": 2, "dry_run": False}),+ _one("mcp__jira__create_ticket", {"project": "OPS", "priority": 2.0, "dry_run": 0}, call_id="c2"),+ )+ case = {"tool_arguments": [{"tool": "mcp:jira", "equals": {"project": "OPS", "priority": 2, "dry_run": False}}]}++ block = _signals(traj, case)["arguments"]++ assert (block["checked"], block["passed"]) == (2, 1)+ assert block["failures"] == [+ {+ "tool": "mcp__jira__create_ticket",+ "arg": "dry_run",+ "rule": "equals",+ "detail": "expected false, got integer",+ }+ ]++ def test_rule_without_matching_call_is_reported_and_counted(self) -> None:+ block = _signals(_traj(_one("Bash")), {"tool_arguments": [{"tool": "mcp__x__*", "required": ["a"]}]})[+ "arguments"+ ]+ # An uncalled rule counts as a check that did not pass (proof M31), so a rate cannot hide it.+ assert (block["checked"], block["passed"]) == (1, 0)+ assert block["failures"][0]["rule"] == "not_called"+ assert block["status"] == "scored"++ @pytest.mark.parametrize(+ "rule",+ [+ {"equals": {"command": "make test"}},+ {"contains": {"command": "pytest"}},+ {"pattern": {"command": "^pytest"}},+ {"schema": {"properties": {"command": {"enum": ["ls"], "pattern": "^ls"}}}},+ ],+ )+ @pytest.mark.parametrize(+ ("command", "secret"),+ [+ ("git clone https://deploy:[email protected]/x.git", "S3cr3tP4ss"),+ ("psql postgresql://admin:hunter2pw@db:5432/app", "hunter2pw"),+ ("mysql -u root -phunter2pw -h db", "hunter2pw"),+ ("printf '-----BEGIN PRIVATE KEY-----\\nMIIEvQIBADANBgkqhkiG9w0'", "MIIEvQIBADANBgkqhkiG9w0"),+ ("curl -H 'X-Key: sk-" + "Ab1" * 8 + "'", "Ab1Ab1Ab1"),+ ],+ )+ def test_failure_details_never_echo_argument_values(self, rule: dict[str, Any], command: str, secret: str) -> None:+ traj = _traj(_one("Bash", {"command": command}))+ case = {"tool_arguments": [{"tool": "Bash", **rule}]}++ signals = _signals(traj, case)++ assert secret not in json.dumps(signals)+ assert signals["arguments"]["failures"]+ for failure in signals["arguments"]["failures"]:+ assert f"string(len={len(command)})" in failure["detail"]++ def test_failure_detail_is_truncated(self) -> None:+ traj = _traj(_one("mcp__github__call", {"body": "x" * 5000}))+ case = {"tool_arguments": [{"tool": "mcp__github__call", "contains": {"body": "y" * 1000}}]}++ (failure,) = _signals(traj, case)["arguments"]["failures"]++ assert len(failure["detail"]) <= 240++ def test_oversized_value_is_not_regex_scanned(self) -> None:+ traj = _traj(_one("mcp__github__call", {"body": "a" * 10_000}))+ case = {"tool_arguments": [{"tool": "mcp__github__call", "pattern": {"body": "^(a+)+$"}}]}+ (failure,) = _signals(traj, case)["arguments"]["failures"]+ assert failure["detail"] == "value exceeds the pattern-check size limit"++ @pytest.mark.parametrize(+ ("pattern", "subject"),+ [+ (r"^(\w+\s?)+$", "a" * 39 + "!"),+ (r"(a|aa)+$", "a" * 60 + "!"),+ (r"[a-z]*[a-z0-9]*!", "a" * 4096),+ ],+ ids=["nested-quantifier", "overlapping-alternation", "adjacent-quantifiers-at-subject-cap"],+ )+ def test_backtracking_pattern_fails_fast_instead_of_hanging(+ self, monkeypatch: pytest.MonkeyPatch, pattern: str, subject: str+ ) -> None:+ timeouts = _record_regex_timeouts(monkeypatch)+ traj = _traj(_one("Bash", {"command": subject}))+ case = {"tool_arguments": [{"tool": "Bash", "pattern": {"command": pattern}}]}++ block = _signals(traj, case)["arguments"]++ # The search ran under the per-check deadline, so it could not hang.+ assert timeouts+ assert all(0 < timeout <= plugin_signals._PATTERN_TIMEOUT_SECONDS for timeout in timeouts)+ assert (block["checked"], block["passed"]) == (1, 0)+ (failure,) = block["failures"]+ assert failure["detail"] in {+ "pattern check timed out",+ f"value string(len={len(subject)}) does not match {pattern}",+ }++ def test_pattern_checks_share_one_time_budget_per_trial(self, monkeypatch: pytest.MonkeyPatch) -> None:+ monkeypatch.setattr(plugin_signals, "_PATTERN_TIMEOUT_SECONDS", 0.05)+ monkeypatch.setattr(plugin_signals, "_PATTERN_BUDGET_SECONDS", 0.2)+ timeouts = _record_regex_timeouts(monkeypatch)+ steps = [_one("Bash", {"command": "a" * 60 + "!"}, call_id=f"c{index}") for index in range(20)]+ case = {"tool_arguments": [{"tool": "Bash", "pattern": {"command": "(a|aa)+$"}}]}++ block = _signals(_traj(*steps), case)["arguments"]++ # Each search that timed out spent its share of the 0.2 s budget, so the+ # later checks failed without searching at all.+ assert 0 < len(timeouts) < 20+ assert (block["checked"], block["passed"]) == (20, 0)+ assert {failure["detail"] for failure in block["failures"]} == {"pattern check timed out"}++ def test_huge_integers_and_non_ascii_digits_never_crash_grading(self) -> None:+ huge = 10**400+ traj = _traj(_one("Bash", {"n": huge, "items": [1, 2, 3]}))+ case = {+ "tool_arguments": [+ {"tool": "Bash", "equals": {"n": huge}},+ {"tool": "Bash", "equals": {"n": huge + 1}},+ {"tool": "Bash", "required": ["items.²"]},+ {"tool": "Bash", "schema": {"properties": {"n": {"maximum": huge - 1}}}},+ ]+ }+ assert validate_plugin_case_fields(case) == []++ block = _signals(traj, case)["arguments"]++ assert (block["checked"], block["passed"]) == (4, 1)+ assert [(failure["arg"], failure["detail"]) for failure in block["failures"]] == [+ ("n", "expected " + str(huge + 1)[:66] + "...<truncated>, got integer"),+ ("items.²", "argument is missing"),+ ("n", "value is above maximum " + str(huge - 1)[:66] + "...<truncated>"),+ ]++ @pytest.mark.parametrize("bound", ["minimum", "maximum"])+ def test_huge_schema_bounds_are_validated_not_raised(self, bound: str) -> None:+ entry = {"tool_arguments": [{"tool": "Bash", "schema": {"properties": {"n": {bound: 10**400}}}}]}+ assert validate_plugin_case_fields(entry) == []+ inverted = {"tool_arguments": [{"tool": "Bash", "schema": {"minimum": 10**400, "maximum": 10**399}}]}+ assert any("minimum must not exceed" in problem for problem in validate_plugin_case_fields(inverted))+++class TestMcpCalls:+ def test_success_rate_excludes_unknown_and_covers_every_server(self) -> None:+ traj = _traj(+ _one("mcp__github__a", content="ok"),+ _one("mcp__github__b", content="500: Internal Server Error", call_id="c2"),+ _step([_tc("mcp__undeclared__c", call_id="c3")]),+ _one("Bash", call_id="c4"),+ )++ block = _signals(traj)["mcp_calls"]++ assert {k: block[k] for k in ("total", "succeeded", "failed", "unknown", "success_rate")} == {+ "total": 3,+ "succeeded": 1,+ "failed": 1,+ "unknown": 1,+ "success_rate": 0.5,+ }+ assert block["by_server"]["github"]["tools"] == ["mcp__github__a", "mcp__github__b"]+ assert block["by_server"]["undeclared"]["unknown"] == 1++ def test_no_mcp_calls_has_undefined_rate(self) -> None:+ assert _signals(_traj())["mcp_calls"]["success_rate"] is None+++class TestOrder:+ TRAJ = _traj(+ _one("Skill", {"skill": "alpha"}),+ _one("mcp__github__search", call_id="c2"),+ _one("Skill", {"skill": "beta"}, call_id="c3"),+ )++ def test_alternatives_and_violations(self) -> None:+ case = {+ "expected_order": [+ ["Skill:alpha", ["mcp__github__list", "mcp__github__search"]],+ [["Skill:gamma", "Skill:beta"], "Skill:alpha"],+ ["Skill:alpha", "Skill:missing"],+ ]+ }++ block = _signals(self.TRAJ, case)["order"]++ assert (block["edges"], block["satisfied"]) == (3, 1)+ # Each edge that does not hold says why: a real reversal and a missing call differ.+ assert block["violated"] == [+ {"before": "Skill:gamma | Skill:beta", "after": "Skill:alpha", "reason": "reversed"},+ {"before": "Skill:alpha", "after": "Skill:missing", "reason": "after_never_called"},+ ]++ def test_globs_in_refs(self) -> None:+ block = _signals(self.TRAJ, {"expected_order": [["mcp:git*", "skill:b*"]]})["order"]+ assert block["satisfied"] == 1+++class TestHandoff:+ def test_value_flows_from_producer_output_to_later_consumer_input(self) -> None:+ traj = _traj(+ _one("mcp__github__get_issue", {"n": 1}, "Issue ISSUE-4242 is open"),+ _one("mcp__jira__create_ticket", {"title": "Track ISSUE-4242"}, call_id="c2"),+ )+ case = {"handoffs": [{"producer": "mcp:github", "consumer": "mcp:jira", "value": "ISSUE-4242"}]}+ block = _signals(traj, case)["handoff"]+ assert (block["checked"], block["passed"], block["failures"]) == (1, 1, [])++ def test_value_already_in_prompt_is_not_attributable(self) -> None:+ traj = _traj(+ _one("mcp__github__get_issue", {}, "ISSUE-4242"),+ _one("mcp__jira__create_ticket", {"title": "ISSUE-4242"}, call_id="c2"),+ prompt="File ISSUE-4242 in jira",+ )+ case = {"handoffs": [{"producer": "mcp:github", "consumer": "mcp:jira", "value": "ISSUE-4242"}]}+ (failure,) = _signals(traj, case)["handoff"]["failures"]+ assert "task prompt" in failure["detail"]++ def test_consumer_in_the_same_step_cannot_have_seen_the_output(self) -> None:+ traj = _traj(+ _step(+ [+ _tc("mcp__github__get_issue", call_id="c1"),+ _tc("mcp__jira__create_ticket", {"title": "ISSUE-4242"}, call_id="c2"),+ ],+ [_res("c1", "ISSUE-4242"), _res("c2", "ok")],+ )+ )+ case = {"handoffs": [{"producer": "mcp:github", "consumer": "mcp:jira", "value": "ISSUE-4242"}]}+ (failure,) = _signals(traj, case)["handoff"]["failures"]+ assert "did not reach consumer" in failure["detail"]++ def test_artifact_written_in_producer_skill_window_and_read_by_consumer(self) -> None:+ traj = _traj(+ _one("Skill", {"skill": "alpha"}),+ _one("Write", {"file_path": "/workspace/out/report.json", "content": "{}"}, call_id="c2"),+ _one("Skill", {"skill": "beta"}, call_id="c3"),+ _one("Bash", {"command": "jq . out/report.json"}, call_id="c4"),+ )+ case = {"handoffs": [{"producer": "Skill:alpha", "consumer": "Skill:beta", "artifact": "out/report.json"}]}+ assert _signals(traj, case)["handoff"]["passed"] == 1++ def test_artifact_read_outside_the_consumer_window_does_not_count(self) -> None:+ traj = _traj(+ _one("Skill", {"skill": "alpha"}),+ _one("Bash", {"command": "echo '{}' > out/report.json"}, call_id="c2"),+ _one("Read", {"file_path": "out/report.json"}, call_id="c3"),+ _one("Skill", {"skill": "beta"}, call_id="c4"),+ )+ case = {"handoffs": [{"producer": "Skill:alpha", "consumer": "Skill:beta", "artifact": "out/report.json"}]}+ (failure,) = _signals(traj, case)["handoff"]["failures"]+ assert failure == {+ "producer": "Skill:alpha",+ "consumer": "Skill:beta",+ "detail": "artifact was not read by the consumer after the producer wrote it",+ }++ @pytest.mark.parametrize(+ ("fn", "args"),+ [+ ("apply_patch", {"input": "*** Begin Patch\n *** Add File: out/report.json\n+{}\n*** End Patch"}),+ (+ "apply_patch",+ {"input": "*** Begin Patch\n*** Update File: draft.json\n*** Move to: out/report.json\n*** End Patch"},+ ),+ ("apply_patch", {"patchText": "*** Begin Patch\n*** Add File: out/report.json\n+{}\n*** End Patch"}),+ ("apply_patch", {"raw": "*** Begin Patch\n*** Add File: out/report.json\n+{}\n*** End Patch"}),+ # Harbor's converters wrap a tool input that is not a JSON object as {"value": ...}.+ ("apply_patch", {"value": "*** Begin Patch\n*** Add File: out/report.json\n+{}\n*** End Patch"}),+ ("applypatch", {"input": "*** Begin Patch\n*** Add File: out/report.json\n+{}\n*** End Patch"}),+ ("patch", {"mode": "patch", "patch": "*** Begin Patch\n*** Update File: out/report.json\n*** End Patch"}),+ ("patch", {"mode": "replace", "path": "out/report.json", "old_string": "a", "new_string": "b"}),+ ],+ ids=[+ "indented-header",+ "move-to",+ "opencode-patch-text",+ "raw-input",+ "value-input",+ "applypatch",+ "hermes-patch",+ "hermes-replace",+ ],+ )+ def test_patch_tool_writes_count_as_artifact_writes(self, fn: str, args: dict[str, Any]) -> None:+ traj = _traj(+ _one("Skill", {"skill": "alpha"}),+ _one(fn, args, call_id="c2"),+ _one("Skill", {"skill": "beta"}, call_id="c3"),+ _one("Read", {"file_path": "out/report.json"}, call_id="c4"),+ )+ case = {"handoffs": [{"producer": "Skill:alpha", "consumer": "Skill:beta", "artifact": "out/report.json"}]}+ assert _signals(traj, case)["handoff"]["passed"] == 1++ @pytest.mark.parametrize(+ ("headers", "passed"),+ [+ ("*** Add File: out/report.json\n+{}", 1),+ ("*** Update File: out/report.json\n@@\n-a\n+b", 1),+ ("*** Update File: draft.json\n*** Move to: out/report.json", 1),+ ("*** Delete File: out/report.json", 0),+ ("*** Update File: out/other.json\n@@\n-a\n+b\n*** Delete File: out/report.json", 0),+ ("*** Delete File: out/report.json\n*** Add File: out/report.json\n+{}", 1),+ ("*** Add File: out/report.json\n+{}\n*** Delete File: out/report.json", 1),+ ("*** Update File: out/report.json\n*** Move to: archive/report.json\n@@\n-a\n+b", 0),+ (" *** Update File: out/report.json\n *** Move to: archive/report.json", 0),+ ("*** Update File: out/report.json\n*** Move to: archive/report.json\n*** Add File: out/report.json", 1),+ ],+ ids=[+ "add",+ "update",+ "move-to",+ "delete",+ "delete-beside-update",+ "delete-then-add",+ "add-then-delete",+ "moved-away",+ "moved-away-indented",+ "moved-away-then-added",+ ],+ )+ @pytest.mark.parametrize("form", ["tool", "shell"])+ def test_a_patch_that_deletes_or_moves_the_artifact_away_does_not_write_it(+ self, headers: str, passed: int, form: str+ ) -> None:+ patch = f"*** Begin Patch\n{headers}\n*** End Patch"+ call = (+ _one("apply_patch", {"input": patch}, call_id="c2")+ if form == "tool"+ else _one("exec_command", {"cmd": f"apply_patch <<'EOF'\n{patch}\nEOF"}, call_id="c2")+ )+ traj = _traj(+ _one("Skill", {"skill": "alpha"}),+ call,+ _one("Skill", {"skill": "beta"}, call_id="c3"),+ _one("Read", {"file_path": "out/report.json"}, call_id="c4"),+ )+ case = {"handoffs": [{"producer": "Skill:alpha", "consumer": "Skill:beta", "artifact": "out/report.json"}]}+ handoff = _signals(traj, case)["handoff"]+ assert handoff["passed"] == passed+ if not passed:+ assert [failure["detail"] for failure in handoff["failures"]] == [+ "artifact was not written by the producer"+ ]++ @pytest.mark.parametrize(+ ("command", "passed"),+ [+ ("cd /workspace && apply_patch <<'EOF'\n*** Begin Patch\n*** Add File: out/report.json\n+{}\nEOF", 1),+ # A header-like line in a file that is not a patch names no write.+ ("cat > notes.md <<'EOF'\n*** Add File: out/report.json\nEOF", 0),+ ],+ ids=["shell-apply-patch", "plain-heredoc"],+ )+ def test_shell_apply_patch_headers_count_as_artifact_writes(self, command: str, passed: int) -> None:+ traj = _traj(+ _one("Skill", {"skill": "alpha"}),+ _one("exec_command", {"cmd": command}, call_id="c2"),+ _one("Skill", {"skill": "beta"}, call_id="c3"),+ _one("Read", {"file_path": "out/report.json"}, call_id="c4"),+ )+ case = {"handoffs": [{"producer": "Skill:alpha", "consumer": "Skill:beta", "artifact": "out/report.json"}]}+ assert _signals(traj, case)["handoff"]["passed"] == passed++ def test_mcp_consumer_receiving_the_artifact_path_counts_as_a_read(self) -> None:+ traj = _traj(+ _one("Skill", {"skill": "alpha"}),+ _one("Bash", {"command": "python gen.py | tee /workspace/out.csv"}, call_id="c2"),+ _one("mcp__github__upload", {"file": "/workspace/out.csv"}, call_id="c3"),+ )+ case = {"handoffs": [{"producer": "Skill:alpha", "consumer": "mcp:github", "artifact": "out.csv"}]}+ assert _signals(traj, case)["handoff"]["passed"] == 1++ def test_inactive_components_fail_fast(self) -> None:+ case = {"handoffs": [{"producer": "Skill:alpha", "consumer": "Skill:beta", "value": "x"}]}+ (failure,) = _signals(_traj(), case)["handoff"]["failures"]+ assert failure["detail"] == "producer was not activated; consumer was not activated"++ def test_prompts_are_read_only_when_a_handoff_checks_a_value(self, monkeypatch: pytest.MonkeyPatch) -> None:+ reads: list[int] = []+ prompt_texts = plugin_signals._prompt_texts+ monkeypatch.setattr(plugin_signals, "_prompt_texts", lambda traj: reads.append(1) or prompt_texts(traj))+ traj = _traj(_one("Skill", {"skill": "alpha"}))++ _signals(traj, {"handoffs": [{"producer": "Skill:alpha", "consumer": "Skill:beta", "artifact": "a.json"}]})+ assert reads == []+ _signals(traj, {"handoffs": [{"producer": "Skill:alpha", "consumer": "Skill:beta", "value": "v"}]})+ assert reads == [1]+++class TestConflict:+ def test_pass_fail_and_alternatives(self) -> None:+ traj = _traj(_one("Skill", {"skill": "alpha"}), _one("mcp__jira__create", call_id="c2"))+ case = {+ "conflict_probes": [+ {"id": "ok", "must_use": "Skill:alpha", "must_not_use": ["Skill:beta", "mcp:slack"]},+ {"id": "bad", "must_use": "Skill:beta", "must_not_use": "mcp:jira", "description": "d"},+ ]+ }++ block = _signals(traj, case)["conflict"]++ assert (block["checked"], block["passed"]) == (2, 1)+ assert block["failures"] == [+ {"probe": "bad", "detail": "must_use Skill:beta was not activated; must_not_use mcp:jira was activated"}+ ]+++class TestActivationCoverage:+ def test_declared_exercised_unverified_unavailable(self) -> None:+ traj = _traj(+ _one("Skill", {"skill": "alpha"}),+ _one("mcp__jira__create", content="connection refused", call_id="c2"),+ _one("mcp__jira__create", content="failed to connect", call_id="c3"),+ _one("mcp__github__list", content="403: nope", call_id="c4"),+ _step([_tc("mcp__github__list", call_id="c5")]),+ )++ block = _signals(traj)["activation_coverage"]++ # Every jira call failed: it is unavailable, not exercised (the lists do not overlap).+ assert block == {+ "declared": ["skill:alpha", "skill:beta", "mcp:github", "mcp:jira"],+ "exercised": ["skill:alpha", "mcp:github"],+ "unverified": ["skill:beta"],+ "unavailable": ["mcp:jira"],+ }++ def test_every_spelling_and_namespace_credits_the_declared_component(self) -> None:+ traj = _traj(+ _one("Skill", {"skill": "ALPHA"}),+ _one("Skill", {"skill": "demo:deploy"}, call_id="c2"),+ _one("Task", {"subagent_type": "Reviewer"}, call_id="c3"),+ _one("my_docs__search", content="connection refused", call_id="c4"),+ _one("mcp__plugin_demo_my_docs__search", content="403: forbidden", call_id="c5"),+ _one("mcp__tracker__list", content="500: down", call_id="c6"),+ _one("tracker.get", content="ok", call_id="c7"),+ )+ declared = {+ "skill": ["Alpha", "alpha", "beta"],+ "mcp": ["my.docs", "tracker"],+ "subagent": ["reviewer"],+ "command": ["deploy"],+ }++ block = _signals(traj, declared=declared)["activation_coverage"]++ assert block["exercised"] == [+ "skill:Alpha",+ "skill:alpha",+ "mcp:tracker",+ "subagent:reviewer",+ "command:deploy",+ ]+ assert block["unverified"] == ["skill:beta"]+ # Both my.docs spellings name the declared server, but every my.docs call failed, so it is unavailable+ # and not exercised (the lists do not overlap); one tracker call worked.+ assert block["unavailable"] == ["mcp:my.docs"]++ def test_no_declared_components(self) -> None:+ block = _signals(_traj(_one("Skill", {"skill": "alpha"})), declared=None)["activation_coverage"]+ assert block == {"declared": [], "exercised": [], "unverified": [], "unavailable": []}+++# ---------------------------------------------------------------------------+# Per-arm summary+# ---------------------------------------------------------------------------+++def test_summary_aggregates_rates_means_and_coverage_union() -> None:+ case = {+ "expected_tools": ["Skill:alpha"],+ "conflict_probes": [{"id": "p", "must_use": "Skill:alpha", "must_not_use": "Skill:beta"}],+ }+ first = _signals(+ _traj(_one("Skill", {"skill": "alpha"}), _one("mcp__jira__x", content="503 Service Unavailable", call_id="c2")),+ case,+ )+ second = _signals(+ _traj(_one("Skill", {"skill": "beta"}), _one("mcp__jira__x", content="ok", call_id="c2")),+ case,+ )+ third = _signals(_traj(_one("mcp__github__y", content="403: Forbidden")), {})++ summary = summarize_plugin_signals([first, second, third, None])++ assert summary["n_trials"] == 3+ assert summary["n_missing_trajectory"] == 1+ assert summary["activations"] == {+ "total": 5,+ "mean_per_trial": pytest.approx(1.6667),+ "by_type": {"skill": 2, "mcp": 3},+ }+ assert summary["routing"]["n_scored"] == 2+ assert summary["routing"]["recall"] == 0.5+ assert summary["tool_selection"]["n_scored"] == 0+ assert summary["conflict"] == {+ "n_scored": 2,+ "checked": 2,+ "passed": 1,+ "pass_rate": 0.5,+ "status": "scored",+ "failed_probes": [{"probe": "p", "trials": 1}],+ "skipped": 0,+ }+ assert summary["arguments"]["status"] == "not_applicable"+ assert summary["mcp_calls"]["by_server"]["jira"]["success_rate"] == 0.5+ coverage = summary["activation_coverage"]+ assert coverage["exercised"] == ["skill:alpha", "skill:beta", "mcp:jira"]+ assert coverage["unverified"] == []+ # jira worked in one trial, so only github (failed everywhere it ran) is unavailable.+ assert coverage["unavailable"] == ["mcp:github"]+ # The trial where every jira call failed did not exercise it.+ assert coverage["exercise_rate"]["mcp:jira"] == pytest.approx(0.3333)+++def test_summary_of_no_trials_is_empty_but_well_formed() -> None:+ summary = summarize_plugin_signals([])+ assert summary["n_trials"] == 0+ assert summary["tool_selection"]["status"] == "not_applicable"+ assert summary["mcp_calls"]["success_rate"] is None+++# ---------------------------------------------------------------------------+# Dataset case fields+# ---------------------------------------------------------------------------++GOOD_CASE: dict[str, Any] = {+ "id": "case-1",+ "prompt": "p",+ "expected_tools": ["mcp__github__*", "Skill:alpha"],+ "acceptable_tools": ["Read"],+ "decoy_tools": ["mcp__slack__*"],+ "tool_arguments": [+ {+ "tool": "mcp__github__search",+ "required": ["query"],+ "schema": {"type": "object", "properties": {"limit": {"type": ["integer", "null"], "maximum": 10}}},+ "equals": {"sort": "new"},+ "contains": {"query": "bug"},+ "pattern": {"query": "^bug"},+ }+ ],+ "expected_order": [["Skill:alpha", ["mcp:github", "mcp:jira"]]],+ "handoffs": [{"producer": "mcp:github", "consumer": "mcp:jira", "value": "ISSUE-1", "artifact": "out.json"}],+ "conflict_probes": [{"id": "p1", "must_use": "Skill:alpha", "must_not_use": "Skill:beta", "description": "d"}],+}+++def test_valid_case_fields_are_accepted_and_spec_is_idempotent() -> None:+ assert validate_plugin_case_fields(GOOD_CASE) == []+ spec = plugin_case_spec(GOOD_CASE)+ assert set(spec) == {+ "expected_tools",+ "acceptable_tools",+ "decoy_tools",+ "tool_arguments",+ "expected_order",+ "handoffs",+ "conflict_probes",+ }+ assert plugin_case_spec(spec) == spec+ assert validate_plugin_case_fields(spec) == []+ assert json.loads(json.dumps(spec)) == spec+++def test_absent_and_null_fields_are_fine() -> None:+ assert validate_plugin_case_fields({"id": "x", "expected_tools": None}) == []+ assert plugin_case_spec({"id": "x"}) == {}+++ ("field", "value", "fragment"),+ [+ ("expected_tools", "mcp__x__y", "must be a list"),+ ("expected_tools", [""], "non-empty string"),+ ("expected_tools", ["Skill:"], "must be followed by a name"),+ ("acceptable_tools", ["x" * 300], "at most 256"),+ ("decoy_tools", [f"t{i}" for i in range(MAX_TOOL_PATTERNS + 1)], "at most 64"),+ ("tool_arguments", [{"tool": "x"}], "at least one of"),+ ("tool_arguments", [{"tool": "x", "requred": ["a"]}], "unsupported keys"),+ ("tool_arguments", [{"tool": "x", "schema": {"items": {}}}], "unsupported JSON-Schema keyword"),+ ("tool_arguments", [{"tool": "x", "schema": {"type": "float"}}], "must be one of"),+ ("tool_arguments", [{"tool": "x", "schema": {"minimum": 5, "maximum": 1}}], "minimum must not exceed"),+ ("tool_arguments", [{"tool": "x", "pattern": {"a": "("}}], "not a valid regular expression"),+ ("tool_arguments", [{"tool": "x", "contains": {"a": 3}}], "non-empty string"),+ ("tool_arguments", [{"tool": "x", "equals": {}}], "non-empty object"),+ ("expected_order", [["a"]], "two-item"),+ ("expected_order", [["a", []]], "non-empty list"),+ ("handoffs", [{"producer": "a", "consumer": "b"}], "value or an artifact"),+ ("handoffs", [{"producer": "a", "consumer": "b", "value": ""}], "non-empty string"),+ ("conflict_probes", [{"id": "p", "must_use": "a"}], "must_not_use"),+ ("conflict_probes", [{"id": "p", "must_use": "a", "must_not_use": "b", "description": 3}], "description"),+ ("handoffs", [{"producer": "a", "consumer": "b", "value": "v", "description": ["x"]}], "description"),+ ("tool_arguments", [{"tool": "x", "required": ["a"], "description": "d" * 2000}], "description"),+ (+ "conflict_probes",+ [{"id": "p", "must_use": "a", "must_not_use": "b"}, {"id": "p", "must_use": "a", "must_not_use": "b"}],+ "must be unique",+ ),+ ],+)+def test_invalid_case_fields_are_reported_and_dropped(field: str, value: Any, fragment: str) -> None:+ entry = {"id": "case-1", field: value}+ problems = validate_plugin_case_fields(entry)+ assert problems, field+ assert any(fragment in problem for problem in problems), problems+ assert field not in plugin_case_spec(entry)+++ ("pattern", "valid"),+ [+ (r"^\p{Lu}\w+$", True), # a Unicode property: the regex engine runs it, ``re`` cannot compile it+ ("[[:alpha:]", False), # ``re`` reads a set of characters; the regex engine an unclosed POSIX class+ ("(?a)a(?u)", False), # the regex engine raises ValueError, not regex.error+ ("(?V0)(?V1)x", False), # ... KeyError+ ("(?:abc){e<=99999999999999999999}", False), # ... RuntimeError+ ],+)+def test_patterns_are_validated_by_the_engine_that_runs_them(pattern: str, valid: bool) -> None:+ entry = {"tool_arguments": [{"tool": "mcp__jira__create", "pattern": {"title": pattern}}]}+ schema_entry = {"tool_arguments": [{"tool": "t", "schema": {"properties": {"title": {"pattern": pattern}}}}]}++ for problems in (validate_plugin_case_fields(entry), validate_plugin_case_fields(schema_entry)):+ if valid:+ assert problems == []+ else:+ assert any("not a valid regular expression" in problem for problem in problems), problems+ if valid:+ traj = _traj(_one("mcp__jira__create", {"title": "Track"}))+ assert _signals(traj, entry)["arguments"]["passed"] == 1++[email protected]("pattern", ["(?V0)(?V1)x", "(?:abc){e<=99999999999999999999}"])+def test_a_pattern_the_engine_cannot_compile_is_dropped_without_breaking_collection(pattern: str) -> None:+ case = {"id": "c1", "tool_arguments": [{"tool": "mcp:jira/create", "pattern": {"title": pattern}}]}+ traj = _traj(_one("mcp__jira__create", {"title": "abcx"}))++ assert plugin_case_spec(case) == {}+ assert build_plugin_signals_context(entries=[case]).cases == {}+ assert _signals(traj, case)["arguments"]["status"] == "not_applicable"+++ "pattern",+ [+ "(?:a{3000}){3000}",+ "(?:a{10000}){10000}",+ "a{101}",+ "a{1,101}",+ "a{101,}",+ "a{0000101}",+ "a{" + "9" * 200 + "}",+ "(?:a{2}){2}",+ "(?:(?:a{2})+){2}",+ "a{100}{100}",+ "(?:abc){e<=99999999999999999999}",+ "(?x:a{ 10000 })",+ "(?x:a{1 0 0 0 0})",+ "(?x:a{1 # count\n0000})",+ "(?iV1)(?:a{100}){100}",+ "(?V1i)(?:a{100}){100}",+ "(?V1)[[a]{10000}]",+ "a{100}(?i){100}",+ "(?:abc){e<=99999999999999999999:[a]}",+ "(?:abc){1i+1d<99999999999999999999}",+ "(?:abc){0<=e<=99999999999999999999}",+ "a{,10000}",+ "(?:a{,100}){100}",+ "(?x) a{ 10000 }",+ r"(?:(?#abc\))a{100}){100}",+ r"(?:a{100}[[:alpha:] )]){100}",+ ],+)+def test_expensive_counted_repeats_are_rejected_before_compilation(monkeypatch, pattern):+ compiled = []++ def forbidden_compile(value):+ compiled.append(value)+ raise AssertionError("unsafe pattern reached the compiler")++ # Never send these patterns to the real compiler, including during the red run.+ monkeypatch.setattr(plugin_signals.regex, "compile", forbidden_compile)+ case = {"tool_arguments": [{"tool": "t", "pattern": {"title": pattern}}]}+ schema_case = {"tool_arguments": [{"tool": "t", "schema": {"pattern": pattern}}]}+ assert validate_plugin_case_fields(case)+ assert validate_plugin_case_fields(schema_case)+ assert plugin_signals.sanitize_input_schema({"type": "string", "pattern": pattern}) == {"type": "string"}+ assert compiled == []+++ "pattern",+ [+ r"a{100}",+ r"a{1,100}",+ r"a{1,}",+ r"\{3000\}",+ r"[a{3000}]{2}",+ r"a{2}b{2}",+ r"(?i:a){2}",+ r"\p{gc=L}{2}",+ r"a{,100}",+ ],+)+def test_small_counted_repeats_and_literal_braces_remain_supported(pattern):+ case = {"tool_arguments": [{"tool": "t", "pattern": {"title": pattern}}]}+ assert validate_plugin_case_fields(case) == []++[email protected]("error", [KeyError("V0"), RuntimeError("invalid RE code"), IndexError("list index")])+def test_a_pattern_search_that_raises_fails_the_check(monkeypatch: pytest.MonkeyPatch, error: Exception) -> None:+ def failing_search(*_args: Any, **_kwargs: Any) -> Any:+ raise error++ monkeypatch.setattr(plugin_signals.regex, "search", failing_search)+ case = {"tool_arguments": [{"tool": "mcp:jira/create", "pattern": {"title": "^T"}}]}++ block = _signals(_traj(_one("mcp__jira__create", {"title": "Track"})), case)["arguments"]++ assert (block["checked"], block["passed"]) == (1, 0)+ assert [failure["rule"] for failure in block["failures"]] == ["pattern"]+++def test_deeply_nested_schema_is_rejected() -> None:+ schema: dict[str, Any] = {"type": "object"}+ for _ in range(10):+ schema = {"type": "object", "properties": {"x": schema}}+ problems = validate_plugin_case_fields({"tool_arguments": [{"tool": "t", "schema": schema}]})+ assert any("nesting" in problem for problem in problems)+++def test_invalid_fields_are_ignored_by_graders() -> None:+ signals = _signals(_traj(_one("Skill", {"skill": "alpha"})), {"expected_tools": "Skill:alpha"})+ assert signals["tool_selection"]["status"] == "not_applicable"+++def test_context_builder_bounds_and_scopes_declared_components() -> None:+ context = build_plugin_signals_context(+ member_skills=["alpha", "alpha", "", 3, "beta"],+ mcp_servers=["github", None],+ wrapper_skills=["demo-plugin-eval", "demo"],+ entries=[GOOD_CASE, {"id": "plain", "prompt": "p"}, {"id": "bad", "expected_tools": "x"}, "junk"],+ )++ assert context.member_skills == ("alpha", "beta")+ assert context.declared_for("with_skill") == {"skill": ["alpha", "beta"], "mcp": ["github"], "plugin": ["demo"]}+ assert context.declared_for("sum_of_parts") == {+ "skill": ["alpha", "beta"],+ "mcp": [],+ "plugin": ["demo"],+ "unstaged": ["mcp"],+ }+ assert set(context.cases) == {"case-1"}+ assert context.case_spec("missing") == {}+ assert context.arm_enabled("with_skill") and context.arm_enabled("sum_of_parts")+ assert not context.arm_enabled("without_skill")+++class TestUntrustedTrajectoryContent:+ def test_argument_derived_names_persist_only_when_identifier_shaped_or_declared(self) -> None:+ traj = _traj(+ _one("Skill", {"skill": "https://u:SuperSecretPW@host/x"}),+ _one("Task", {"subagent_type": "-----BEGIN RSA PRIVATE KEY-----MIIEowIBAAKCAQEA"}, call_id="c2"),+ _one("SlashCommand", {"command": "postgresql://admin:hunter2pw@db/app --now"}, call_id="c3"),+ _one("Skill", {"skill": "Release Notes"}, call_id="c4"),+ _one("Task", {"subagent_type": "code-reviewer"}, call_id="c5"),+ _one("SlashCommand", {"command": "/plugin:deploy prod"}, call_id="c6"),+ )+ declared = {**DECLARED, "skill": [*DECLARED["skill"], "release notes"]}+ case = {"tool_arguments": [{"tool": "Skill:*", "required": ["missing"]}]}++ signals = _signals(traj, case, declared=declared)++ dumped = json.dumps(signals)+ for secret in ("SuperSecretPW", "MIIEowIBAAKCAQEA", "hunter2pw"):+ assert secret not in dumped+ assert [a["name"] for a in signals["activations"]] == [+ "<non-name>",+ "<non-name>",+ "<non-name>",+ "release notes",+ "code-reviewer",+ "plugin:deploy",+ ]+ assert signals["routing"]["called"] == [+ "Skill:<non-name>",+ "Agent:<non-name>",+ "Command:<non-name>",+ "Skill:release notes",+ "Agent:code-reviewer",+ "Command:plugin:deploy",+ ]+ assert [f["tool"] for f in signals["arguments"]["failures"]] == ["Skill:<non-name>", "Skill:release notes"]++ def test_adversarial_text_for_the_redactor_does_not_stall_collection(self, monkeypatch: pytest.MonkeyPatch) -> None:+ from skillevaluator.utils import redaction++ payload = "a." * 32_768 # quadratic for the credential redactor's assignment patterns+ traj = _traj(+ _one("Skill", {"skill": payload}),+ _one("Task", {"subagent_type": payload}, call_id="c2"),+ _one("Bash", {"command": payload}, call_id="c3"),+ )+ case = {+ "tool_arguments": [+ {"tool": "Bash", "contains": {"command": "pytest"}},+ {"tool": "Bash", "equals": {"command": "make test"}},+ {"tool": "Bash", "pattern": {"command": "^pytest"}},+ {"tool": "Bash", "schema": {"properties": {"command": {"enum": ["ls"]}}}},+ ]+ }++ redacted: list[int] = []+ redact = redaction.redact_sensitive_text++ def recording_redact(text: str, **kwargs: Any) -> str:+ redacted.append(len(text))+ return redact(text, **kwargs)++ monkeypatch.setattr(redaction, "redact_sensitive_text", recording_redact)++ signals = _signals(traj, case)++ # The redactor only ever sees a bounded window (4 x 256 + 256 characters), never the payload.+ assert redacted+ assert max(redacted) <= 4 * plugin_signals._MAX_LABEL_CHARS + 256+ assert len(signals["arguments"]["failures"]) == 4++ def test_unhashable_step_source_is_ignored(self) -> None:+ traj = _traj(_one("Skill", {"skill": "alpha"}))+ traj["steps"].insert(0, {"source": ["user"], "message": "hi"})+ assert _signals(traj)["activations"][0]["name"] == "alpha"diff --git a/tests/tier3/test_private_comparison_artifacts.py b/tests/tier3/test_private_comparison_artifacts.pynew file mode 100644index 00000000..955662ba--- /dev/null+++ b/tests/tier3/test_private_comparison_artifacts.py@@ -0,0 +1,226 @@+# SPDX-License-Identifier: Apache-2.0+"""Durable comparison evidence retains every attempt within the artifact bounds."""++import json+from types import SimpleNamespace++import pytest++from skillevaluator.tier3.harbor import collector, loose_parts_arm+from skillevaluator.tier3.harbor.metrics import DEFAULT_METRIC_SET, DEFAULT_METRICS+from skillevaluator.tier3.plugin_comparisons import ComparisonOptions, ComparisonRuntime+from skillevaluator.tier3.plugin_evidence import LogicalTrialEvidence+++ "cases,note_size,limit", [(100, 80, "node count"), (30, 1024, "encoded JSON"), (1000, 80, "node count")]+)+def test_indexed_evidence_retains_all_attempts_beyond_uncompressed_artifact_limits(tmp_path, cases, note_size, limit):+ evidence = {+ arm: {+ f"case-{case}": [+ {+ f"skill:piece-{piece}": {+ "loaded": (True, False, None)[(case + attempt + piece) % 3],+ "source": "system/init",+ "evidence": f"piece-{piece}: " + "x" * note_size,+ "status": "connected",+ "tools": ["fixture"],+ }+ for piece in range(15)+ }+ for attempt in range(3)+ ]+ for case in range(cases)+ }+ for arm in ("with_skill", "sum_of_parts", "loose_parts")+ }+ with pytest.raises(ValueError, match=limit):+ collector._write_generated_root_json(tmp_path / "raw.json", tmp_path, evidence)++ compact = loose_parts_arm.compact_load_evidence(evidence)+ path = tmp_path / loose_parts_arm.LOAD_EVIDENCE_FILENAME+ collector._write_generated_root_json(path, tmp_path, compact)+ saved = json.loads(path.read_text())+ # Decode using the documented tables, independently of the encoder. Missing+ # attempts, piece names, failures, or unknown load states must all be visible.+ restored = {+ arm: {+ case: [+ {piece: saved["results"][row] for piece, row in saved["attempts"][index].items()} for index in attempts+ ]+ for case, attempts in rows.items()+ }+ for arm, rows in saved["arms"].items()+ }+ assert restored == evidence+ assert saved["format"] == "indexed-v1"+++def _runtime(tmp_path, *, case_count=1, n_attempts=1, n_pieces=0):+ agents = ("claude-code", "codex")+ jobs_dir = tmp_path / "_harbor-jobs"+ jobs_dir.mkdir()+ cases = tuple(f"case-{index}" for index in range(case_count))+ names = [f"piece-{index}" for index in range(n_pieces)]+ records = [+ LogicalTrialEvidence(+ trial_id=f"{case}__attempt-{ordinal}",+ case_id=case,+ reward={+ "entry_id": case,+ "_attempt_ordinal": ordinal,+ "metric_set": DEFAULT_METRIC_SET,+ **dict.fromkeys(DEFAULT_METRICS, 1.0),+ },+ signals=None,+ native_census=None,+ usage=None,+ first_turn_usage=None,+ evidence_status="recorded",+ init_event={"skills": names + [f"fixture:{name}" for name in names], "plugins": [{"name": "fixture"}]},+ )+ for case in cases+ for ordinal in range(1, n_attempts + 1)+ ]+ arms = (+ ("with_skill", "without_skill", "sum_of_parts", "loose_parts") if n_pieces else ("with_skill", "without_skill")+ )+ observer = SimpleNamespace(+ logical_trials={+ agent: dict.fromkeys(arms if agent == "claude-code" else arms[:2], records) for agent in agents+ },+ context=SimpleNamespace(case_spec=lambda _case: {"cross_component": True}),+ )+ native = SimpleNamespace(observer=observer, decisions={}, collect_evidence=lambda *_args: None)+ package = SimpleNamespace(+ native_source=SimpleNamespace(member_skills=(tmp_path / "member",) if n_pieces else ()),+ identity=SimpleNamespace(),+ )+ runtime = ComparisonRuntime(native, package, ComparisonOptions(loose_parts=True), agents, baseline=True)+ runtime.expected_case_ids = cases+ runtime.n_attempts = n_attempts+ for agent in agents:+ runtime.jobs[agent] = {}+ for arm in observer.logical_trials[agent]:+ job = jobs_dir / f"{agent}-{arm}"+ job.mkdir()+ (job / "result.json").write_text(+ json.dumps(+ {+ "n_total_trials": len(records),+ "stats": {+ "n_trials": len(records),+ "n_errors": 0,+ "evals": {+ "fixture": {+ "n_trials": len(records),+ "n_errors": 0,+ "reward_stats": {"score": {"1.0": [record.trial_id for record in records]}},+ }+ },+ },+ }+ )+ )+ runtime.jobs[agent][arm] = job+ # Only the first agent has a loose arm. Its artifact failure must not prevent+ # statistics for the later agent, whose native route cannot stage loose parts.+ placements = [SimpleNamespace(id=f"skill:{name}", kind="skill", name=name) for name in names]+ runtime.loose_plan.arms["claude-code"] = SimpleNamespace(+ parts=SimpleNamespace(_placements=lambda _arm: placements, source=SimpleNamespace(plugin_name="fixture")),+ content={},+ info=lambda evidence: {"measured": True, "load_evidence": loose_parts_arm.load_evidence_summary(evidence)},+ )+ result = {"agents": {agent: {} for agent in agents}}+ return runtime, jobs_dir, result+++def test_large_runtime_collection_saves_complete_evidence_and_statistics(tmp_path):+ runtime, jobs_dir, result = _runtime(tmp_path, case_count=100, n_attempts=3, n_pieces=15)++ runtime.collect_evidence(jobs_dir, result)++ saved = json.loads((tmp_path / "private_comparisons.json").read_text())+ assert saved["status"] == "recorded" and saved["errors"] == []+ agent = saved["agents"]["claude-code"]+ assert agent["lift_ladder"]["n_cases"] == 100+ assert agent["loose_parts"]["load_evidence"]["trials"] == {+ "with_skill": 300,+ "sum_of_parts": 300,+ "loose_parts": 300,+ }+ assert agent["loose_parts"]["load_evidence"]["pieces"]["loose_parts"]["skill:piece-14"]["loaded"] == 300+ evidence = json.loads((tmp_path / "claude-code" / loose_parts_arm.LOAD_EVIDENCE_FILENAME).read_text())+ assert evidence["format"] == "indexed-v1"+ assert {arm: len(cases) for arm, cases in evidence["arms"].items()} == {+ "with_skill": 100,+ "sum_of_parts": 100,+ "loose_parts": 100,+ }+ assert all(len(attempts) == 3 for cases in evidence["arms"].values() for attempts in cases.values())+ assert all(len(attempt) == 15 for attempt in evidence["attempts"])+ assert (tmp_path / "codex/statistics.json").is_file()++[email protected]("artifact", ["statistics.json", loose_parts_arm.LOAD_EVIDENCE_FILENAME])+def test_one_artifact_write_failure_keeps_other_artifacts_and_later_agents(tmp_path, monkeypatch, artifact):+ runtime, jobs_dir, result = _runtime(tmp_path)+ real_write = collector._write_generated_root_json++ def write(path, output, payload):+ if path.parent.name == "claude-code" and path.name == artifact:+ raise ValueError("encoded JSON exceeds limit")+ return real_write(path, output, payload)++ monkeypatch.setattr(collector, "_write_generated_root_json", write)+ runtime.collect_evidence(jobs_dir, result)++ saved = json.loads((tmp_path / "private_comparisons.json").read_text())+ assert saved["status"] == "incomplete"+ assert set(saved["agents"]) == {"claude-code", "codex"}+ assert len(saved["errors"]) == 1 and f"claude-code/{artifact}" in saved["errors"][0]+ assert result["private_comparisons"] == saved+ assert result["agents"]["codex"]["cost"]+ assert json.loads((tmp_path / "codex/statistics.json").read_text())["reliability"]+ other = (+ "statistics.json"+ if artifact == loose_parts_arm.LOAD_EVIDENCE_FILENAME+ else loose_parts_arm.LOAD_EVIDENCE_FILENAME+ )+ assert (tmp_path / "claude-code" / other).is_file()+++def test_standalone_write_failure_is_reported_inline_after_agent_artifacts_are_saved(tmp_path, monkeypatch):+ runtime, jobs_dir, result = _runtime(tmp_path)+ real_write = collector._write_generated_root_json++ def write(path, output, payload):+ if path.name == "private_comparisons.json":+ raise OSError("fixture write failure")+ return real_write(path, output, payload)++ monkeypatch.setattr(collector, "_write_generated_root_json", write)+ runtime.collect_evidence(jobs_dir, result)++ assert result["private_comparisons"]["status"] == "incomplete"+ assert len(result["private_comparisons"]["errors"]) == 1+ assert "private_comparisons.json" in result["private_comparisons"]["errors"][0]+ assert all((tmp_path / agent / "statistics.json").is_file() for agent in ("claude-code", "codex"))+ assert (tmp_path / "claude-code" / loose_parts_arm.LOAD_EVIDENCE_FILENAME).is_file()+++def test_comparison_attempt_reset_preserves_previous_results_and_resets_native_hooks(tmp_path):+ runtime, _jobs_dir, _result = _runtime(tmp_path)+ runtime.native.begin_attempt = lambda: setattr(runtime.native, "evidence", {})+ runtime.native.evidence = {"previous": True}+ runtime.data = {"status": "recorded", "agents": {"previous": {}}, "errors": []}+ previous = runtime.provenance()++ runtime.begin_attempt()++ assert runtime.native.evidence == {}+ assert runtime.data == {"status": "not_collected", "agents": {}, "errors": []}+ assert runtime.jobs == {} and runtime.expected_case_ids == ()+ assert runtime.loose_plan.arms == {}+ assert previous["agents"] == {"previous": {}}diff --git a/tests/tier3/test_private_comparison_contracts.py b/tests/tier3/test_private_comparison_contracts.pynew file mode 100644index 00000000..4b602358--- /dev/null+++ b/tests/tier3/test_private_comparison_contracts.py@@ -0,0 +1,430 @@+# SPDX-License-Identifier: Apache-2.0+"""Private comparison contracts; deterministic rewards and staged files only."""++from __future__ import annotations++import math++import pytest++from skillevaluator.tier3.harbor import stats+from skillevaluator.tier3.harbor.metrics import DEFAULT_METRIC_SET, DEFAULT_METRICS, LEGACY_SCORE_POLICY, overall_score+++def reward(case, value=0.5, *, accuracy=None, judge=None):+ row = {"entry_id": case, "metric_set": DEFAULT_METRIC_SET, **dict.fromkeys(DEFAULT_METRICS, value)}+ if accuracy is not None:+ row["accuracy"] = accuracy+ if judge:+ row["details"] = {"accuracy": {"judge_model_used": judge}}+ return row+++def arms(cases=("a", "b", "c", "d", "e")):+ return {+ arm: stats.ArmObservations(+ "succeeded", tuple(stats.TrialObservation(case, reward(case, value)) for case in cases)+ )+ for arm, value in (("without_skill", 0.2), ("sum_of_parts", 0.4), ("loose_parts", 0.6), ("with_skill", 0.8))+ }+++def test_ladder_uses_one_shared_bootstrap_draw_stream(monkeypatch):+ calls = []+ original = stats.random.Random.choices++ def choices(self, population, *args, **kwargs):+ calls.append(len(population))+ return original(self, population, *args, **kwargs)++ monkeypatch.setattr(stats.random.Random, "choices", choices)+ result = stats.lift_ladder(arms(), None)+ assert len(calls) == result["bootstrap"]["resamples"]+ steps = result["steps"]+ assert math.isclose(+ sum(steps[key]["estimate"] for key in ("skill", "capability", "integration")), steps["plugin"]["estimate"]+ )+++def test_judge_mismatch_does_not_enter_paired_interval_or_ladder():+ observed = {}+ for arm in arms():+ high = arm == "with_skill"+ observed[arm] = stats.ArmObservations(+ "succeeded",+ tuple(+ stats.TrialObservation(+ case, reward(case, accuracy=0.9 if high else 0.1, judge="fallback" if high else "primary")+ )+ for case in ("a", "b", "c", "d", "e")+ ),+ )+ pair = stats.paired_lift(+ observed["with_skill"], observed["without_skill"], treatment_arm="with_skill", control_arm="without_skill"+ )+ assert pair["estimate"] == 0.0+ ladder = stats.lift_ladder(observed, None)+ assert ladder["steps"]["plugin"]["estimate"] == 0.0+ assert ladder["judge_mixed_cases"]["accuracy"] == 5+++def test_within_arm_judge_mixture_is_excluded_when_other_arm_has_no_model():+ treatment = stats.ArmObservations(+ "succeeded",+ tuple(stats.TrialObservation("a", reward("a", accuracy=0.9, judge=model)) for model in ("primary", "fallback")),+ )+ control = stats.ArmObservations("succeeded", (stats.TrialObservation("a", reward("a", accuracy=0.1)),))+ pair = stats.paired_lift(treatment, control, treatment_arm="with_skill", control_arm="without_skill")+ assert pair["estimate"] == 0.0+++def test_unscored_executed_attempt_participates_in_cost():+ usage = {"prompt_tokens": 10, "completion_tokens": 10, "cost_usd": 0.4}+ unscored = {"prompt_tokens": 20, "completion_tokens": 10, "cost_usd": 0.6}+ arm = stats.ArmObservations(+ "succeeded",+ (stats.TrialObservation("a", reward("a"), usage),),+ {"passed_cases": 1},+ unscored=(stats.TrialObservation("b", {}, unscored),),+ )+ result = stats.build_agent_statistics(+ {"with_skill": arm},+ expected_case_ids=["a", "b"],+ n_attempts=1,+ stop_on_pass=False,+ pass_threshold=0.5,+ sum_of_parts_requested=False,+ )+ assert result["cost"]["with_skill"]["total_tokens"] == 50+ assert result["cost"]["with_skill"]["total_usd"] == 1.0+++def test_missing_unscored_counter_cannot_become_a_partial_cost_total():+ arm = stats.ArmObservations(+ "succeeded",+ (stats.TrialObservation("a", reward("a"), {"prompt_tokens": 10, "completion_tokens": 2, "cost_usd": 0.0}),),+ {"passed_cases": 1},+ unscored=(stats.TrialObservation("b", {}, {}),),+ )+ result = stats.build_agent_statistics(+ {"with_skill": arm},+ expected_case_ids=["a", "b"],+ n_attempts=1,+ stop_on_pass=False,+ pass_threshold=0.5,+ sum_of_parts_requested=False,+ )+ assert result["cost"]["with_skill"]["total_tokens"] is None+ assert result["cost"]["with_skill"]["total_usd"] is None+ assert stats.arm_cost(arm.trials, successes=1)["total_usd"] == 0.0+++def test_bounded_or_incomplete_observation_coverage_cannot_claim_total_cost():+ usage = {"prompt_tokens": 10, "completion_tokens": 2, "cost_usd": 0.4, "first_turn_prompt_tokens": 10}+ arm = stats.ArmObservations(+ "succeeded",+ tuple(stats.TrialObservation(case, reward(case), usage) for case in ("a", "b", "c")),+ {"passed_cases": 3},+ observations_complete=False,+ )+ result = stats.build_agent_statistics(+ {"with_skill": arm, "without_skill": arm},+ expected_case_ids=["a", "b", "c"],+ n_attempts=2,+ stop_on_pass=False,+ pass_threshold=0.5,+ sum_of_parts_requested=False,+ )+ assert result["cost"]["with_skill"]["total_tokens"] is None+ assert result["cost"]["with_skill"]["total_usd"] is None+ assert result["token_efficiency"]["with_skill"] is None+ assert result["context_cost_measured"]["partial"] is True+++def test_failed_arm_keeps_cost_when_all_executed_observations_were_retained():+ arm = stats.ArmObservations(+ "failed",+ unscored=(stats.TrialObservation("a", {}, {"prompt_tokens": 3, "completion_tokens": 2, "cost_usd": 0.25}),),+ job_failure="setup failed after a recorded call",+ )+ result = stats.build_agent_statistics(+ {"with_skill": arm},+ expected_case_ids=["a"],+ n_attempts=1,+ stop_on_pass=False,+ pass_threshold=0.5,+ sum_of_parts_requested=False,+ )+ assert result["cost"]["with_skill"]["total_tokens"] == 5+ assert result["cost"]["with_skill"]["total_usd"] == 0.25+ assert result["reliability"] == {}++[email protected]("bad", [float("nan"), float("inf"), True, -1.0, 2.0])+def test_invalid_logical_scores_are_unavailable(bad):+ trial = stats.TrialObservation("a", reward("a"), logical_overall=bad)+ assert stats.case_dimension_means([trial]) == {}+++def test_unidentified_cases_do_not_pair():+ result = stats.common_case_scores(arms(("unknown", "")))+ assert not result.case_ids+++def test_completeness_rejects_missing_attempts():+ observed = arms(("a",))+ result = stats.integration_completeness(+ observed["with_skill"],+ observed["sum_of_parts"],+ expected_case_ids=["a"],+ n_attempts=2,+ stop_on_pass=False,+ pass_threshold=0.5,+ )+ assert result["complete"] is False+ assert len(result["attempt_shortfall"]) == 2+++def test_expected_case_missing_from_both_arms_makes_context_partial():+ usage = {"first_turn_prompt_tokens": 20}+ arm = stats.ArmObservations(+ "succeeded", tuple(stats.TrialObservation(case, reward(case), usage) for case in ("a", "b", "c"))+ )+ result = stats.build_agent_statistics(+ {"with_skill": arm, "without_skill": arm},+ expected_case_ids=["a", "b", "c", "missing"],+ n_attempts=1,+ stop_on_pass=False,+ pass_threshold=0.5,+ sum_of_parts_requested=False,+ )+ assert result["context_cost_measured"]["partial"] is True+ assert result["context_cost_measured"]["missing_cases"] == ["missing"]+++def test_judge_exclusion_cannot_remove_security():+ with pytest.raises(ValueError, match="judge"):+ stats.common_case_scores(arms(), judge_mixed_cases={"security": ["a"]})+++def test_case_index_reads_each_judge_provenance_once(monkeypatch):+ from skillevaluator.tier3.harbor import collector++ calls = []+ original = collector._reward_judge_models++ def read(row, metric):+ calls.append((id(row), metric))+ return original(row, metric)++ monkeypatch.setattr(collector, "_reward_judge_models", read)+ observed = arms(("a", "b"))+ stats.build_agent_statistics(+ observed,+ expected_case_ids=["a", "b"],+ n_attempts=1,+ stop_on_pass=False,+ pass_threshold=0.5,+ sum_of_parts_requested=True,+ cross_component_case_ids=["a"],+ )+ assert len(calls) == 4 * 2 * 3+++def test_pairwise_judges_do_not_inherit_another_arms_fallback():+ observed = {}+ for arm in arms():+ primary = arm in {"with_skill", "without_skill"}+ observed[arm] = stats.ArmObservations(+ "succeeded",+ tuple(+ stats.TrialObservation(+ case,+ reward(+ case, accuracy=0.9 if arm == "with_skill" else 0.1, judge="primary" if primary else "fallback"+ ),+ )+ for case in ("a", "b", "c", "d", "e")+ ),+ )+ result = stats.build_agent_statistics(+ observed,+ expected_case_ids=["a", "b", "c", "d", "e"],+ n_attempts=1,+ stop_on_pass=False,+ pass_threshold=0.5,+ sum_of_parts_requested=True,+ )+ assert result["lift_uncertainty"]["effectiveness"]["estimate"] > 0+ assert result["lift_ladder"]["steps"]["plugin"]["estimate"] == 0+++def test_load_failure_is_excluded_from_each_affected_pair():+ observed = arms(("a", "b"))+ kwargs = {+ "expected_case_ids": ["a", "b"],+ "n_attempts": 1,+ "stop_on_pass": False,+ "pass_threshold": 0.5,+ "sum_of_parts_requested": True,+ }+ result = stats.build_agent_statistics(+ observed, load_failures_by_case={"a": {"with_skill": "MCP did not load"}}, **kwargs+ )+ assert result["lift_uncertainty"]["effectiveness"]["n_cases"] == 1+ assert result["lift_uncertainty"]["integration"]["n_cases"] == 1+ assert result["lift_ladder"]["n_cases"] == 1+ unrelated = stats.build_agent_statistics(+ observed, load_failures_by_case={"a": {"loose_parts": "MCP did not load"}}, **kwargs+ )+ assert unrelated["lift_uncertainty"]["effectiveness"]["n_cases"] == 2+ assert unrelated["lift_ladder"]["n_cases"] == 1+++def test_complete_legacy_policy_keeps_its_metric_mean():+ row = reward("a", 0.0)+ row.update(skill_execution=1.0, score_policy=LEGACY_SCORE_POLICY)+ assert stats.trial_quality_score(row) == overall_score(row)+++def test_custom_only_scores_keep_the_existing_overall_basis():+ treatment = stats.ArmObservations("succeeded", (stats.TrialObservation("a", {"overall": 0.75}),))+ control = stats.ArmObservations("succeeded", (stats.TrialObservation("a", {"overall": 0.25}),))+ pair = stats.paired_lift(treatment, control, treatment_arm="with_skill", control_arm="without_skill")+ assert pair["dimensions"] == ["overall"]+ assert pair["estimate"] == 0.5+++def test_mixed_logical_attempt_retains_the_collectors_overall_score():+ treatment = stats.ArmObservations(+ "succeeded", (stats.TrialObservation("a", reward("a", 0.9), logical_overall=0.7),)+ )+ control = stats.ArmObservations("succeeded", (stats.TrialObservation("a", reward("a", 0.2)),))+ pair = stats.paired_lift(treatment, control, treatment_arm="with_skill", control_arm="without_skill")+ assert pair["treatment_score"] == 0.7+ assert pair["estimate"] == 0.5+++def test_unknown_policy_and_unexplained_missing_dimension_are_unscored():+ row = reward("a")+ row["score_policy"] = "unrecognized-policy"+ assert stats.trial_quality_score(row) is None+ row = reward("a")+ del row["accuracy"]+ assert stats.trial_quality_score(row) is None+ row["accuracy"] = None+ row["details"] = {"accuracy": {"status": "not_applicable"}}+ assert stats.trial_quality_score(row) == pytest.approx(overall_score(row), abs=0.0001)+++def test_different_declared_policies_are_not_called_one_overall_lift():+ normal = reward("a")+ legacy = {**normal, "score_policy": LEGACY_SCORE_POLICY}+ pair = stats.paired_lift(+ stats.ArmObservations("succeeded", (stats.TrialObservation("a", legacy),)),+ stats.ArmObservations("succeeded", (stats.TrialObservation("a", normal),)),+ treatment_arm="with_skill",+ control_arm="without_skill",+ )+ assert pair is None+++def test_common_metric_is_selected_before_dimension_weighting():+ treatment = reward("a", 0.0)+ treatment["goal_accuracy"] = 1.0+ control = reward("a", 0.0)+ control["goal_accuracy"] = None+ control["details"] = {"goal_accuracy": {"status": "not_applicable"}}+ from skillevaluator.tier3.harbor.lift_basis import pair_metric_scores++ expected = pair_metric_scores(+ [("a", treatment)], [("a", control)], arms=("with_skill", "without_skill"), metrics=DEFAULT_METRICS+ )+ assert expected.treatment == expected.control+ paired = stats.paired_lift(+ stats.ArmObservations("succeeded", (stats.TrialObservation("a", treatment),)),+ stats.ArmObservations("succeeded", (stats.TrialObservation("a", control),)),+ treatment_arm="with_skill",+ control_arm="without_skill",+ )+ assert paired["estimate"] == 0.0+ observed = {+ name: stats.ArmObservations(+ "succeeded", (stats.TrialObservation("a", treatment if name == "with_skill" else control),)+ )+ for name in arms()+ }+ ladder = stats.lift_ladder(observed, ["a"])+ assert ladder["steps"]["plugin"]["estimate"] == 0.0+ assert ladder["glue"]["lift"] == 0.0+++def test_per_metric_attempt_means_precede_dimension_weighting():+ scored = reward("a", 0.0)+ scored["goal_accuracy"] = 1.0+ not_applicable = {**scored, "goal_accuracy": None, "details": {"goal_accuracy": {"status": "not_applicable"}}}+ treatment = stats.ArmObservations(+ "succeeded", tuple(stats.TrialObservation("a", row) for row in (scored, not_applicable))+ )+ control = stats.ArmObservations("succeeded", tuple(stats.TrialObservation("a", scored) for _ in range(2)))+ pair = stats.paired_lift(treatment, control, treatment_arm="with_skill", control_arm="without_skill")+ assert pair["estimate"] == 0.0+++ "low,high,verdict",+ [+ (0.05, 0.2, "real_integration"),+ (-0.2, -0.05, "negative_integration"),+ (-0.04, 0.04, "cosmetic_bundling"),+ (-0.05, 0.05, "inconclusive"),+ (0.02, 0.1, "inconclusive"),+ ],+)+def test_integration_claim_uses_the_whole_eligible_interval(low, high, verdict):+ ladder = {+ "partial": False,+ "glue": {+ "n": 5,+ "lift": (low + high) / 2,+ "ci_low": low,+ "ci_high": high,+ "precision": "adequate",+ "partial": False,+ },+ }+ content = {"equal_across_arms": True, "tasks_hashed": 5, "content_basis": "emitted Harbor tasks"}+ complete = {"complete": True, "loose_parts": {"complete": True}, "all_arms": {"complete": True}}+ result = stats.integration_claim(ladder, content, complete, agent="claude-code", full_arm_load="native")+ assert result["verdict"] == verdict+ assert result["eligible"] is (verdict != "inconclusive")+ assert (+ stats.integration_claim(+ ladder, {**content, "equal_across_arms": False}, complete, agent="claude-code", full_arm_load="native"+ )["eligible"]+ is False+ )+ assert (+ stats.integration_claim(+ ladder, content, {**complete, "all_arms": {"complete": False}}, agent="claude-code", full_arm_load="native"+ )["eligible"]+ is False+ )+ wrapper = stats.integration_claim(ladder, content, complete, agent="claude-code", full_arm_load="wrapper")+ assert wrapper["verdict"] is None and "Wrapper effect" in wrapper["label"]+++ "bad_glue",+ [{"n": 4}, {"n": True}, {"ci_low": float("nan")}, {"ci_high": float("inf")}, {"lift": 2.0}, {"partial": True}],+)+def test_integration_claim_rejects_unusable_or_incomplete_glue(bad_glue):+ ladder = {+ "partial": False,+ "glue": {"n": 5, "lift": 0.15, "ci_low": 0.1, "ci_high": 0.2, "precision": "low", "partial": False, **bad_glue},+ }+ content = {"equal_across_arms": True, "tasks_hashed": 5, "content_basis": "emitted Harbor tasks"}+ complete = {"complete": True, "loose_parts": {"complete": True}, "all_arms": {"complete": True}}+ result = stats.integration_claim(ladder, content, complete, agent="claude-code", full_arm_load="native")+ assert result["eligible"] is False+ assert result["verdict"] == "inconclusive"diff --git a/tests/tier3/test_private_comparison_engine.py b/tests/tier3/test_private_comparison_engine.pynew file mode 100644index 00000000..9f685fee--- /dev/null+++ b/tests/tier3/test_private_comparison_engine.py@@ -0,0 +1,232 @@+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+# SPDX-License-Identifier: Apache-2.0++"""Private comparisons share the existing launcher and its trial budget."""++from pathlib import Path+from threading import Barrier, Lock+from types import SimpleNamespace++import pytest++from skillevaluator.tier3.harbor import runner+++def arguments(tmp_path):+ return {+ "skill_name": "bundle",+ "agent": "claude-code",+ "model": "fixture-model",+ "env_mode": "docker",+ "with_skill": tmp_path / "with",+ "baseline": tmp_path / "without",+ "jobs_dir": tmp_path / "jobs",+ "run_env": {},+ "n_attempts": 3,+ "n_concurrent": 2,+ "timeout_multiplier": 1.0,+ "override_cpus": None,+ "override_memory_mb": None,+ "override_storage_mb": None,+ "expected_trials": 6,+ "agent_import_path": "fixture:ordinary",+ "with_agent_import_path": "fixture:native",+ "additional_arms": (+ ("sumofparts", tmp_path / "members", "fixture:ordinary"),+ ("looseparts", tmp_path / "loose", "fixture:loose"),+ ),+ }+++def test_four_arms_share_the_per_agent_trial_budget(tmp_path, monkeypatch):+ calls, active, maximum = [], 0, 0+ lock, barrier = Lock(), Barrier(2)++ def launch(**kwargs):+ nonlocal active, maximum+ with lock:+ calls.append(kwargs)+ active += 1+ maximum = max(maximum, active)+ barrier.wait(timeout=10)+ with lock:+ active -= 1+ return True, ""++ monkeypatch.setattr(runner, "_run_harbor", launch)+ assert runner._run_agent_pair(**arguments(tmp_path)) == []+ assert maximum == 2+ by_variant = {call["job_name"].rsplit("-", 1)[1]: call for call in calls}+ assert set(by_variant) == {"with", "without", "sumofparts", "looseparts"}+ assert {variant: call["agent_import_path"] for variant, call in by_variant.items()} == {+ "with": "fixture:native",+ "without": "fixture:ordinary",+ "sumofparts": "fixture:ordinary",+ "looseparts": "fixture:loose",+ }+ assert all(call["n_concurrent"] == 1 and call["n_attempts"] == 3 for call in calls)+ assert all(call["model"] == "fixture-model" and call["expected_trials"] == 6 for call in calls)+++def test_added_arm_launch_failures_stay_advisory(tmp_path, monkeypatch):+ def launch(**kwargs):+ return (False, "fixture setup failure") if kwargs["job_name"].endswith("looseparts") else (True, "")++ monkeypatch.setattr(runner, "_run_harbor", launch)+ failures = {}+ assert runner._run_agent_pair(**arguments(tmp_path), additional_arm_errors=failures) == []+ assert failures == {"looseparts": "fixture setup failure"}+++def test_stop_on_pass_applies_the_same_policy_to_each_private_arm(tmp_path, monkeypatch):+ calls = []+ monkeypatch.setattr(runner, "_run_stop_on_pass_variant", lambda **kw: calls.append(kw) or [])+ assert runner._run_agent_pair(**arguments(tmp_path), stop_on_pass=True, task_names=["case-a", "case-b"]) == []+ assert [call["variant"] for call in calls] == ["with", "without", "sumofparts", "looseparts"]+ assert all(call["n_attempts"] == 3 and call["task_names"] == ["case-a", "case-b"] for call in calls)+++def test_reserved_or_duplicate_private_variants_are_refused(tmp_path):+ kwargs = arguments(tmp_path)+ kwargs["additional_arms"] = (("with", Path("fixture"), None),)+ with pytest.raises(ValueError, match="variant"):+ runner._run_agent_pair(**kwargs)+++def test_private_comparison_options_refuse_an_incomplete_ladder():+ from skillevaluator.tier3.plugin_comparisons import ComparisonOptions++ with pytest.raises(ValueError, match="member"):+ ComparisonOptions(member_skills=False, loose_parts=True)+ with pytest.raises(ValueError, match="boolean"):+ ComparisonOptions(member_skills="yes")++[email protected]("loose_parts", [False, True])+def test_baseline_only_keeps_one_arm_and_explains_unmeasured_private_controls(tmp_path, monkeypatch, loose_parts):+ from skillevaluator.tier3 import plugin_comparisons as comparisons++ package = SimpleNamespace(native_source=SimpleNamespace(member_skills=(tmp_path / "member",)))+ runtime = comparisons.ComparisonRuntime(+ SimpleNamespace(),+ package,+ comparisons.ComparisonOptions(loose_parts=loose_parts),+ ["claude-code"],+ baseline=True,+ )+ if loose_parts:+ runtime.loose_plan.arms["claude-code"] = object()+ baseline = tmp_path / "baseline" / "case"+ baseline.mkdir(parents=True)+ monkeypatch.setattr(comparisons, "verify_source_identity", lambda _package: None)++ def unexpected_control(*_args):+ pytest.fail("baseline-only execution emitted an additional comparison arm")++ jobs = runtime.stage_additional("claude-code", unexpected_control, {"without_skill": [baseline]}, None)++ assert jobs == ()+ runtime.bind_execution(["case"], n_attempts=1, stop_on_pass=False, pass_threshold=0.5)+ assert runtime.task_sets["claude-code"] == {"without_skill": (baseline,)}+ assert runtime.loose_plan.arm_for("claude-code") is None+ assert "with-skill arm disabled" in runtime.loose_plan.not_measured["claude-code"]+++def test_usage_of_unscored_attempts_is_kept_without_inventing_scores():+ from skillevaluator.tier3.plugin_comparisons import arm_observations++ records = [+ SimpleNamespace(+ case_id="case-a",+ reward=None,+ usage={"total_prompt_tokens": 17, "total_completion_tokens": 3, "total_cost_usd": 0},+ first_turn_usage={"prompt_tokens": 17},+ evidence_status="recorded",+ )+ ]+ observed = arm_observations(+ records,+ expected_case_ids=["case-a"],+ n_attempts=1,+ stop_on_pass=False,+ pass_threshold=0.5,+ job_failure="setup failed",+ )+ assert observed.execution_status == "failed"+ assert not observed.trials+ assert observed.unscored[0].case_id == "case-a"+ assert observed.unscored[0].usage == {+ "prompt_tokens": 17,+ "completion_tokens": 3,+ "cost_usd": 0,+ "first_turn_prompt_tokens": 17,+ }+++def test_first_turn_hosted_search_is_excluded_even_with_a_token_counter():+ from skillevaluator.tier3.plugin_comparisons import observation_usage++ record = SimpleNamespace(+ usage={"prompt_tokens": 10}, first_turn_usage={"prompt_tokens": 10, "first_turn_hosted_search": 1}+ )+ usage = observation_usage(record)+ assert usage["first_turn_hosted_search"] == 1+ assert "first_turn_prompt_tokens" not in usage+++def test_stop_on_pass_extra_exception_preserves_the_completed_primary_arms(tmp_path, monkeypatch):+ calls = []++ def launch(**kwargs):+ calls.append(kwargs["variant"])+ if kwargs["variant"] == "sumofparts":+ raise OSError("fixture control failure")+ return []++ monkeypatch.setattr(runner, "_run_stop_on_pass_variant", launch)+ failures = {}+ assert runner._run_agent_pair(**arguments(tmp_path), stop_on_pass=True, additional_arm_errors=failures) == []+ assert calls == ["with", "without", "sumofparts", "looseparts"]+ assert "fixture control failure" in failures["sumofparts"]++[email protected]("missing_case", [False, True])+def test_missing_recorded_attempts_cannot_become_complete_cost(missing_case):+ from skillevaluator.tier3.plugin_comparisons import arm_observations++ record = SimpleNamespace(+ case_id="case-a", reward=None, usage={"prompt_tokens": 10}, first_turn_usage={}, evidence_status="recorded"+ )+ observed = arm_observations(+ [record],+ expected_case_ids=["case-a", "case-b"] if missing_case else ["case-a"],+ n_attempts=1 if missing_case else 2,+ stop_on_pass=False,+ pass_threshold=0.5,+ )+ assert observed.observations_complete is False+++def test_early_pass_is_complete_only_with_contiguous_attempt_evidence():+ from skillevaluator.tier3.harbor.metrics import DEFAULT_METRIC_SET, DEFAULT_METRICS+ from skillevaluator.tier3.plugin_comparisons import arm_observations++ def observed(ordinal):+ record = SimpleNamespace(+ case_id="case-a",+ usage={},+ first_turn_usage={},+ evidence_status="recorded",+ reward={+ "entry_id": "case-a",+ "_attempt_ordinal": ordinal,+ "metric_set": DEFAULT_METRIC_SET,+ **dict.fromkeys(DEFAULT_METRICS, 1.0),+ },+ )+ return arm_observations(+ [record], expected_case_ids=["case-a"], n_attempts=3, stop_on_pass=True, pass_threshold=0.5+ )++ assert observed(1).observations_complete is True+ assert observed(2).observations_complete is Falsediff --git a/tests/tier3/test_private_comparison_engine_acceptance.py b/tests/tier3/test_private_comparison_engine_acceptance.pynew file mode 100644index 00000000..2d6739ad--- /dev/null+++ b/tests/tier3/test_private_comparison_engine_acceptance.py@@ -0,0 +1,358 @@+# SPDX-License-Identifier: Apache-2.0+"""Private arms use real emission/orchestration with an in-process fake Harbor."""++from __future__ import annotations++import json+import os+import shutil+import subprocess+import threading+import tomllib+from datetime import UTC, datetime+from pathlib import Path+from uuid import UUID++import pytest++from skillevaluator.evaluation.options import EvaluationOptions+from skillevaluator.plugin_inventory import plugin_inventory_for_root+from skillevaluator.tier3 import plugin_execution as px+from skillevaluator.tier3.harbor import adapter, collector, runner+from skillevaluator.tier3.harbor.metrics import DEFAULT_METRIC_SET, DEFAULT_METRICS+from skillevaluator.tier3.plugin_comparisons import ComparisonOptions, ComparisonRuntime+from skillevaluator.tier3.plugin_prepare import prepare_runtime_package, stage_wrapper_skill, wrapper_skill_name+from test_tier3_progress import _stub_runner+++def _package(root: Path):+ files = {+ ".claude-plugin/plugin.json": {"name": "acceptance"},+ "skills/member/SKILL.md": "---\nname: member\ndescription: Fixture member.\n---\nUse the fixture member.\n",+ "skills/member/evals/evals.json": [+ {"id": f"case-{index}", "prompt": f"Perform fixture task {index}.", "cross_component": True}+ for index in range(1, 6)+ ],+ ".mcp.json": {"mcpServers": {"fixture": {"command": "python3", "args": ["-m", "fixture_server"]}}},+ "rules/policy.md": "# Fixture policy\nKeep outputs concise.\n",+ }+ for relative, payload in files.items():+ path = root / relative+ path.parent.mkdir(parents=True, exist_ok=True)+ path.write_text(payload if isinstance(payload, str) else json.dumps(payload), encoding="utf-8")+ inventory = plugin_inventory_for_root(root)+ assert inventory is not None+ return prepare_runtime_package(inventory)+++def _harbor_result(job: Path, tasks: list[Path], attempts: int) -> None:+ """Real Harbor model shapes, deterministic complete rewards, no agent/provider."""+ from harbor.models.job.result import JobResult, JobStats+ from harbor.models.trial.result import TrialResult++ now = datetime(2026, 11, 2, tzinfo=UTC)+ variant = job.name.rsplit("-", 1)[-1]+ score = {"without": 0.2, "sumofparts": 0.4, "looseparts": 0.6, "with": 0.8}[variant]+ init = {+ "type": "system",+ "subtype": "init",+ "skills": ["acceptance:member"] if variant == "with" else [] if variant == "without" else ["member"],+ "plugins": [{"name": "acceptance"}] if variant == "with" else [],+ "mcp_servers": (+ [{"name": "plugin:acceptance:fixture" if variant == "with" else "fixture", "status": "connected"}]+ if variant in {"with", "looseparts"}+ else []+ ),+ }+ trials = []+ for task in tasks:+ entry = json.loads((task / "tests/entry.json").read_text())+ for ordinal in range(1, attempts + 1):+ trial_name = f"{task.name}__attempt-{ordinal}"+ trial = job / trial_name+ (trial / "verifier").mkdir(parents=True)+ (trial / "agent").mkdir()+ setup = task / "environment/skilleval/native/setup.sh"+ if setup.is_file():+ # Fake Harbor skips agent setup. Run only the emitted fixture's+ # copy/listing script, with every writable location in tmp_path,+ # so the collector reads the same census as a real native trial.+ home = trial / "home"+ home.mkdir()+ script = (+ setup.read_text()+ .replace("/skilleval/", str(task / "environment/skilleval") + "/")+ .replace("/logs/agent", str(trial / "agent"))+ )+ completed = subprocess.run(+ ["sh", "-c", script],+ env={"PATH": os.environ["PATH"], "HOME": str(home), "CLAUDE_CONFIG_DIR": str(home / ".claude")},+ capture_output=True,+ text=True,+ timeout=15,+ )+ assert completed.returncode == 0, completed.stderr+ (trial / "agent/claude-code.txt").write_text(json.dumps(init) + "\n")+ reward = {+ "entry_id": entry["id"],+ "_attempt_ordinal": ordinal,+ "metric_set": DEFAULT_METRIC_SET,+ **dict.fromkeys(DEFAULT_METRICS, score),+ }+ (trial / "verifier/reward.json").write_text(json.dumps(reward))+ result = TrialResult.model_validate(+ {+ "id": UUID(int=len(trials) + 2),+ "task_name": f"fixture/{task.name}",+ "trial_name": trial_name,+ "trial_uri": trial.as_uri(),+ "task_id": {"path": str(task)},+ "task_checksum": "acceptance-fixture",+ "config": {"task": {"path": str(task)}, "trial_name": trial_name},+ "agent_info": {+ "name": "claude-code",+ "version": "fixture",+ "model_info": {"name": "fixture-model"},+ },+ "agent_result": {"n_input_tokens": 10, "n_output_tokens": 2},+ "verifier_result": {"rewards": dict.fromkeys(DEFAULT_METRICS, score)},+ "started_at": now,+ "finished_at": now,+ "step_results": None,+ }+ )+ (trial / "result.json").write_text(result.model_dump_json())+ trials.append(result)+ result = JobResult(+ id=UUID(int=1),+ started_at=now,+ updated_at=now,+ finished_at=now,+ n_total_trials=len(trials),+ stats=JobStats.from_trial_results(trials, n_total_trials=len(trials)),+ trial_results=trials,+ )+ (job / "result.json").write_text(result.model_dump_json())+++def test_fake_harbor_native_setup_confirms_the_full_arm_and_caches_its_init_event(tmp_path):+ from skillevaluator.tier3 import plugin_native+ from skillevaluator.tier3.harbor.native_staging import build_native_task_staging, stage_native_bundle++ package = _package(tmp_path / "plugin")+ task = tmp_path / "tasks/fixture"+ (task / "tests").mkdir(parents=True)+ (task / "tests/entry.json").write_text(json.dumps({"id": "case-1"}))+ staging = build_native_task_staging(+ "claude-code", plugin_native.HARNESS_ADAPTERS["claude-code"], package.native_source+ )+ stage_native_bundle(task / "environment", staging)+ job = tmp_path / "jobs/fixture-with"+ _harbor_result(job, [task], 2)++ plan = staging.bundle.census_plan()+ events = {}+ for trial in sorted(job.iterdir()):+ if not trial.is_dir():+ continue+ census = px._trial_census(+ plugin_native, "claude-code", trial / "agent", plan["declared"], plan, init_events=events+ )+ assert {(row["type"], row["name"]) for row in census["loaded"]} == {("skill", "member"), ("mcp", "fixture")}+ assert events[trial]["plugins"] == [{"name": "acceptance"}]+ assert len(events) == 2+++def _engine(monkeypatch, root: Path, *, arm_count: int, route: str | None, keep: bool, ordinary: bool = False):+ package = _package(root / "plugin")+ real_pair = runner._run_agent_pair+ _, skill = _stub_runner(+ monkeypatch,+ root,+ run_agent=real_pair,+ collect=collector.collect_harbor_results,+ task_emitter=adapter.generate_harbor_tasks,+ skill_name=wrapper_skill_name(package.plugin_name),+ )+ shutil.rmtree(skill)+ stage_wrapper_skill(package, skill)+ monkeypatch.setattr(+ runner,+ "load_evals_config",+ lambda _path: (+ {+ "harbor": {+ "task_source": "evals_json",+ "base_image_mode": "disabled",+ "resources": {"cpus": 3, "memory_mb": 5120, "storage_mb": 3072},+ }+ },+ None,+ ),+ )+ monkeypatch.setattr(runner, "_agent_import_path", lambda *_args: route)+ calls, lock, first_wave = [], threading.Lock(), threading.Event()+ active, maximum = 0, 0++ def launch(**kwargs):+ nonlocal active, maximum+ tasks = sorted(path for path in kwargs["dataset"].iterdir() if path.is_dir())+ captured = {+ **kwargs,+ "task_names": [path.name for path in tasks],+ "tasks": tasks,+ "entries": [json.loads((path / "tests/entry.json").read_text()) for path in tasks],+ "tomls": [tomllib.loads((path / "task.toml").read_text()) for path in tasks],+ "files": [+ {path.relative_to(task).as_posix() for path in task.rglob("*") if path.is_file()} for task in tasks+ ],+ }+ with lock:+ calls.append(captured)+ active += kwargs["n_concurrent"]+ maximum = max(maximum, active)+ if len(calls) >= 2:+ first_wave.set()+ assert first_wave.wait(timeout=10), "runner did not share the requested condition budget"+ _harbor_result(kwargs["jobs_dir"] / kwargs["job_name"], tasks, kwargs["n_attempts"])+ with lock:+ active -= kwargs["n_concurrent"]+ return True, ""++ monkeypatch.setattr(runner, "_run_harbor", launch)+ seen = {}++ def evaluate(options):+ seen["runtime"] = px.active_runtime()+ seen["wrapper"] = options.skill_path+ return runner.run_harbor_eval(+ options.skill_path,+ ["claude-code"],+ output_dir=options.results_dir,+ n_attempts=options.n_attempts,+ n_concurrent=options.n_concurrent,+ max_cases=options.max_cases,+ keep_harbor_jobs=options.harbor_keep_jobs,+ override_cpus=options.override_cpus,+ override_memory_mb=options.override_memory_mb,+ override_storage_mb=options.override_storage_mb,+ agent_runtime_preflight=options.agent_runtime_preflight,+ )++ options = EvaluationOptions(+ skill_path=skill,+ agents="claude-code",+ results_dir=root / "results",+ n_attempts=2,+ n_concurrent=2,+ max_cases=2,+ harbor_keep_jobs=keep,+ override_cpus=4,+ override_memory_mb=6144,+ override_storage_mb=4096,+ agent_runtime_preflight=False,+ )+ if ordinary:+ result = evaluate(options)+ assert seen["runtime"] is None+ else:+ work = root / "work"+ work.mkdir()+ kwargs = {"comparison_options": ComparisonOptions(loose_parts=arm_count == 4)} if arm_count > 2 else {}+ outcome = px.evaluate_prepared_plugin(+ package, options, work_dir=work, load="native", evaluate=evaluate, **kwargs+ )+ assert outcome.cleanup_error is None and outcome.publication_error is None+ assert not seen["wrapper"].exists() and list(work.iterdir()) == []+ result = outcome.result+ assert isinstance(seen["runtime"], ComparisonRuntime) is (arm_count > 2)+ assert px.active_runtime() is None+ return result, calls, seen["runtime"], maximum+++ "arm_count,route",+ [+ (2, None),+ (3, "skillevaluator.tier3.harbor.local_agents:SkillEvaluatorClaudeCode"),+ (4, "skillevaluator.tier3.harbor.local_agents:SkillEvaluatorNvidiaBuildClaudeCode"),+ ],+)+def test_real_runner_emits_private_arms_with_shared_cases_resources_and_budget(tmp_path, monkeypatch, arm_count, route):+ result, calls, runtime, maximum = _engine(monkeypatch, tmp_path, arm_count=arm_count, route=route, keep=True)+ assert result["execution_status"] == "succeeded", result+ assert len(calls) == arm_count+ assert maximum == 2+ by_variant = {call["job_name"].rsplit("-", 1)[1]: call for call in calls}+ assert set(by_variant) == {+ "with",+ "without",+ *(["sumofparts"] if arm_count > 2 else []),+ *(["looseparts"] if arm_count > 3 else []),+ }+ selected = by_variant["with"]["task_names"]+ assert len(selected) == 2+ for call in calls:+ assert call["task_names"] == selected+ assert call["n_attempts"] == 2 and call["expected_trials"] == 4+ assert call["n_concurrent"] == 1+ assert (call["override_cpus"], call["override_memory_mb"], call["override_storage_mb"]) == (4, 6144, 4096)+ for task in call["tomls"]:+ assert {key: task["environment"][key] for key in ("cpus", "memory_mb", "storage_mb")} == {+ "cpus": 3,+ "memory_mb": 5120,+ "storage_mb": 3072,+ }+ assert "Native" in by_variant["with"]["agent_import_path"]+ assert "Private" in by_variant["without"]["agent_import_path"]+ if arm_count > 2:+ assert by_variant["sumofparts"]["agent_import_path"] == by_variant["without"]["agent_import_path"]+ for files, task in zip(by_variant["sumofparts"]["files"], by_variant["sumofparts"]["tomls"], strict=True):+ assert "environment/skills/member/SKILL.md" in files+ assert {path for path in files if path.endswith("/SKILL.md")} == {"environment/skills/member/SKILL.md"}+ assert not any(+ "native/claude-code/plugin" in path or path.endswith("mcp.json") or path.endswith("mcp_servers.json")+ for path in files+ )+ assert not task["environment"].get("mcp_servers")+ if arm_count == 4:+ assert by_variant["looseparts"]["agent_import_path"].endswith(":LoosePartsNvidiaBuildClaudeCode")+ assert runtime.loose_plan.arm_for("claude-code").content["equal_across_arms"] is True+++def test_comparison_evidence_is_saved_before_real_runner_cleanup(tmp_path, monkeypatch):+ result, _calls, runtime, _maximum = _engine(monkeypatch, tmp_path, arm_count=4, route=None, keep=False)+ assert result["execution_status"] == "succeeded", result+ run_dir = Path(result["run_dir"])+ assert not (run_dir / "_harbor-tasks").exists()+ assert not (run_dir / "_harbor-jobs").exists()+ saved = json.loads((run_dir / "private_comparisons.json").read_text())+ assert saved["status"] == "recorded" and saved["errors"] == []+ agent = saved["agents"]["claude-code"]+ assert agent["loose_parts"]["equal_across_arms"] is True+ assert agent["integration_claim"]["eligible"] is False+ assert agent["integration_claim"]["verdict"] == "inconclusive"+ ladder = agent["lift_ladder"]+ assert ladder["n_cases"] == 2+ assert ladder["excluded_cases"] == [] and ladder["failed_arms"] == []+ assert {name: step["estimate"] for name, step in ladder["steps"].items()} == pytest.approx(+ {"skill": 0.2, "capability": 0.2, "integration": 0.2, "plugin": 0.6}+ )+ assert {step["n_cases"] for step in ladder["steps"].values()} == {2}+ assert ladder["bundle"]["estimate"] == pytest.approx(0.4)+ assert agent["loose_parts"]["load_evidence"]["dropped_cases"] == []+ assert json.loads((run_dir / "claude-code/statistics.json").read_text())["lift_ladder"] == ladder+ assert runtime.expected_case_ids+ assert json.loads(Path(result["result_path"]).read_text())["private_comparisons"]+++def test_ordinary_runner_has_two_arms_and_no_private_comparison_side_effects(tmp_path, monkeypatch):+ result, calls, runtime, _maximum = _engine(+ monkeypatch, tmp_path, arm_count=2, route=None, keep=False, ordinary=True+ )+ assert result["execution_status"] == "succeeded", result+ assert len(calls) == 2 and runtime is None+ assert all(call["agent_import_path"] is None for call in calls)+ assert "private_comparisons" not in result and "runtime_evidence" not in result+ assert "private_comparisons" not in result["run_config"]+ assert not (Path(result["run_dir"]) / "private_comparisons.json").exists()diff --git a/tests/tier3/test_private_comparison_staging.py b/tests/tier3/test_private_comparison_staging.pynew file mode 100644index 00000000..649e9cf9--- /dev/null+++ b/tests/tier3/test_private_comparison_staging.py@@ -0,0 +1,375 @@+# SPDX-License-Identifier: Apache-2.0+"""Parity reads the emitted private tasks, including file modes."""++import json+from dataclasses import replace++import pytest++from skillevaluator.tier3 import loose_parts+from skillevaluator.tier3.plugin_native import NativePluginSource+++def source(root):+ (root / ".claude-plugin").mkdir(parents=True)+ manifest = {"name": "demo", "description": "A fixture plugin"}+ (root / ".claude-plugin/plugin.json").write_text(json.dumps(manifest))+ member = root / "skills/a"+ member.mkdir(parents=True)+ (member / "SKILL.md").write_text("---\nname: a\ndescription: A skill\n---\nUse a.\n")+ (root / "run.js").write_text("console.log('ok');\n")+ (root / "run.js").chmod(0o755)+ return NativePluginSource(+ "demo",+ "A fixture plugin",+ True,+ root,+ manifest,+ ".claude-plugin/plugin.json",+ member_skills=(member,),+ rules=(("policy", "# Policy\nUse the skill.\n"),),+ plugin_file_mcp_servers=({"name": "tool", "command": "node", "args": ["${CLAUDE_PLUGIN_ROOT}/run.js"]},),+ )+++def test_missing_expected_arm_sets_cannot_claim_equality():+ result = loose_parts.compare_content_hashes({"with_skill": {"skill:a": "same"}})+ assert result["equal_across_arms"] is not True+++def test_mode_only_script_change_is_a_parity_mismatch(tmp_path):+ parts = loose_parts.build_loose_parts(source(tmp_path / "plugin"))+ assert parts.error is None+ tasks = {}+ for arm in ("with_skill", "sum_of_parts", "loose_parts"):+ task = tmp_path / arm+ parts.stage_arm(arm, task / "environment")+ tasks[arm] = [task]+ assert parts.content_report_for_tasks(tasks)["equal_across_arms"] is True+ script = tasks["loose_parts"][0] / "environment/skilleval" / loose_parts.LOOSE_BUNDLE_DIR / "root/run.js"+ script.chmod(0o644)+ result = parts.content_report_for_tasks(tasks)+ assert result["equal_across_arms"] is False+ assert "script:run.js" in result["mismatches"]++[email protected]("kind,mode,suffix", [("mcp", 0o400, ""), ("skill", 0o700, ""), ("skill", 0o700, "nested")])+def test_structured_config_and_directory_permissions_are_part_of_capability_parity(tmp_path, kind, mode, suffix):+ prepared = source(tmp_path / "plugin")+ nested = prepared.plugin_root / "skills/a/nested"+ nested.mkdir()+ (nested / "guide.md").write_text("Nested fixture guidance")+ parts = loose_parts.build_loose_parts(prepared)+ tasks = {}+ for arm in ("with_skill", "sum_of_parts", "loose_parts"):+ task = tmp_path / arm+ parts.stage_arm(arm, task / "environment")+ tasks[arm] = [task]+ assert parts.content_report_for_tasks(tasks)["equal_across_arms"] is True+ placement = next(item for item in parts._loose if item.kind == kind)+ (tasks["loose_parts"][0] / "environment" / placement.path / suffix).chmod(mode)+ result = parts.content_report_for_tasks(tasks)+ assert result["equal_across_arms"] is False+ assert placement.id in result["mismatches"]+++def test_hook_json_permissions_are_part_of_capability_parity(tmp_path, monkeypatch):+ monkeypatch.setenv("SKILLEVALUATOR_ALLOWED_MCP_SECRETS", "RELTOOLS_TOKEN")+ from tests.tier3.private_loose_fixtures import make_plugin, native_source++ parts = loose_parts.build_loose_parts(+ native_source(make_plugin(tmp_path / "plugins"), tmp_path / "stage").native_source+ )+ tasks = {}+ for arm in ("with_skill", "sum_of_parts", "loose_parts"):+ task = tmp_path / arm+ parts.stage_arm(arm, task / "environment")+ tasks[arm] = [task]+ assert parts.content_report_for_tasks(tasks)["equal_across_arms"] is True+ placement = next(item for item in parts._loose if item.kind == "hook")+ (tasks["loose_parts"][0] / "environment" / placement.path).chmod(0o400)+ result = parts.content_report_for_tasks(tasks)+ assert result["equal_across_arms"] is False+ assert placement.id in result["mismatches"]+++def test_empty_emitted_task_set_stays_unknown(tmp_path):+ parts = loose_parts.build_loose_parts(source(tmp_path / "plugin"))+ assert parts.content_report_for_tasks({})["equal_across_arms"] is not True+++def test_census_normalization_cannot_hide_changed_emitted_frontmatter(tmp_path, monkeypatch):+ monkeypatch.setenv("SKILLEVALUATOR_ALLOWED_MCP_SECRETS", "RELTOOLS_TOKEN")+ from tests.tier3.private_loose_fixtures import make_plugin, native_source, write++ plugin = make_plugin(tmp_path / "plugins")+ write(+ plugin / "skills/version-bump/SKILL.md",+ "---\nname: version-bump\ndescription: Bump.\nhooks:\n PreToolUse:\n - matcher: Bash\n hooks:\n - type: command\n command: echo bump\n---\nBump.\n",+ )+ parts = loose_parts.build_loose_parts(native_source(plugin, tmp_path / "stage").native_source)+ tasks = {}+ for arm in ("with_skill", "sum_of_parts", "loose_parts"):+ task = tmp_path / arm+ parts.stage_arm(arm, task / "environment")+ tasks[arm] = [task]+ assert parts.content_report_for_tasks(tasks)["equal_across_arms"] is True+ placement = next(item for item in parts._full if item.id == "skill:version-bump")+ staged = tasks["with_skill"][0] / "environment" / placement.overrides[0][0]+ staged.write_text(staged.read_text() + "\nChanged runtime instructions.\n")+ result = parts.content_report_for_tasks(tasks)+ assert result["equal_across_arms"] is not True+++def test_unexpected_capability_root_is_not_normalized_as_equal(tmp_path):+ parts = loose_parts.build_loose_parts(source(tmp_path / "plugin"))+ tasks = {}+ for arm in ("with_skill", "sum_of_parts", "loose_parts"):+ task = tmp_path / arm+ parts.stage_arm(arm, task / "environment")+ tasks[arm] = [task]+ placement = next(item for item in parts._loose if item.kind == "mcp")+ config = tasks["loose_parts"][0] / "environment" / placement.path+ payload = json.loads(config.read_text())+ payload["mcpServers"][placement.key]["args"] = ["/unexpected/root/run.js"]+ config.write_text(json.dumps(payload))+ result = parts.content_report_for_tasks(tasks)+ assert result["equal_across_arms"] is False+ assert "mcp:tool" in result["mismatches"]+++def test_whole_root_normalization_never_hides_extra_emitted_metadata_content(tmp_path):+ prepared = source(tmp_path / "plugin")+ (prepared.plugin_root / ".claude-plugin/authored.txt").write_text("Not staged by either arm")+ prepared = replace(+ prepared,+ plugin_file_mcp_servers=(+ {"name": "tool", "command": "uv", "args": ["run", "--directory", "${CLAUDE_PLUGIN_ROOT}", "srv"]},+ ),+ )+ parts = loose_parts.build_loose_parts(prepared)+ tasks = {}+ for arm in ("with_skill", "sum_of_parts", "loose_parts"):+ task = tmp_path / arm+ parts.stage_arm(arm, task / "environment")+ tasks[arm] = [task]+ assert parts.content_report_for_tasks(tasks)["equal_across_arms"] is True+ placement = next(item for item in parts._full if item.id == "script:.")+ metadata = tasks["with_skill"][0] / "environment" / placement.path / ".claude-plugin"+ assert {path.name for path in metadata.iterdir()} == {"plugin.json"}+ loose_placement = next(item for item in parts._loose if item.id == "script:.")+ assert not (tasks["loose_parts"][0] / "environment" / loose_placement.path / ".claude-plugin").exists()+ (metadata / "injected.txt").write_text("Unexpected emitted capability")+ result = parts.content_report_for_tasks(tasks)+ assert result["equal_across_arms"] is False+ assert "script:." in result["mismatches"]+++def test_piece_missing_from_every_emitted_arm_cannot_disappear_from_parity(tmp_path):+ parts = loose_parts.build_loose_parts(source(tmp_path / "plugin"))+ tasks = {}+ for arm in ("with_skill", "sum_of_parts", "loose_parts"):+ task = tmp_path / arm+ parts.stage_arm(arm, task / "environment")+ tasks[arm] = [task]+ placement = next(item for item in parts._placements(arm) if item.id == "skill:a")+ (task / "environment" / placement.path / "SKILL.md").unlink()+ result = parts.content_report_for_tasks(tasks)+ assert result["equal_across_arms"] is not True+++def test_parity_rejects_symlink_content_even_when_all_arms_match(tmp_path):+ parts = loose_parts.build_loose_parts(source(tmp_path / "plugin"))+ tasks = {}+ target = tmp_path / "outside.md"+ target.write_text("Unverified external instruction")+ for arm in ("with_skill", "sum_of_parts", "loose_parts"):+ task = tmp_path / arm+ parts.stage_arm(arm, task / "environment")+ tasks[arm] = [task]+ placement = next(item for item in parts._placements(arm) if item.id == "skill:a")+ staged = task / "environment" / placement.path / "SKILL.md"+ staged.unlink()+ staged.symlink_to(target)+ # Direct secure reads refuse redirects; the private plan catches this+ # ValueError and records unavailable parity rather than an equal claim.+ with pytest.raises(ValueError, match="symlink"):+ parts.content_report_for_tasks(tasks)+++def test_same_emitted_task_is_not_read_twice_in_one_parity_check(tmp_path, monkeypatch):+ parts = loose_parts.build_loose_parts(source(tmp_path / "plugin"))+ tasks = {}+ for arm in ("with_skill", "sum_of_parts", "loose_parts"):+ task = tmp_path / arm+ parts.stage_arm(arm, task / "environment")+ tasks[arm] = [task]+ calls = []+ original = loose_parts.SecureRoot.read_bytes++ def read(self, *args, **kwargs):+ calls.append(args)+ return original(self, *args, **kwargs)++ monkeypatch.setattr(loose_parts.SecureRoot, "read_bytes", read)+ assert parts.content_report_for_tasks(tasks)["equal_across_arms"] is True+ first = len(calls)+ calls.clear()+ tasks["with_skill"] *= 2+ assert parts.content_report_for_tasks(tasks)["equal_across_arms"] is True+ assert len(calls) == first++[email protected]("limit,value", [("MAX_PARITY_BYTES", 1), ("MAX_PARITY_FILES", 0)])+def test_parity_read_budget_rejects_before_an_extra_secure_read(tmp_path, monkeypatch, limit, value):+ target = tmp_path / "content.txt"+ target.write_bytes(b"bounded")+ monkeypatch.setattr(loose_parts, limit, value)+ calls = []+ monkeypatch.setattr(loose_parts.SecureRoot, "read_bytes", lambda *_args, **_kwargs: calls.append(True))+ with pytest.raises(ValueError, match="budget"):+ loose_parts._ParityReads().read(target)+ assert calls == []+++ "route,expected",+ [+ ("PrivateClaudeCode", "LoosePartsClaudeCode"),+ ("PrivateContainerClaudeCode", "LoosePartsContainerClaudeCode"),+ ("PrivateNvidiaBuildClaudeCode", "LoosePartsNvidiaBuildClaudeCode"),+ ("NativeNvidiaBuildClaudeCode", "LoosePartsNvidiaBuildClaudeCode"),+ ],+)+def test_private_and_native_route_classes_keep_the_same_loose_backend(route, expected):+ result = loose_parts.loose_parts_agent_import_path(+ "claude-code", "skillevaluator.tier3.harbor.native_agents:" + route+ )+ assert result.endswith(":" + expected)+ from skillevaluator.tier3.harbor.local_agents import SetupNetworkErrorAgent+ from skillevaluator.tier3.harbor.loose_agents import _LoosePartsClaudeCodeMixin++ assert issubclass(_LoosePartsClaudeCodeMixin, SetupNetworkErrorAgent)+++ "root,variable", [(loose_parts.LOOSE_ROOT, "CLAUDE_PLUGIN_ROOT"), (loose_parts.LOOSE_DATA, "CLAUDE_PLUGIN_DATA")]+)+def test_full_mcp_literal_loose_directory_is_not_normalized_as_equal(tmp_path, monkeypatch, root, variable):+ monkeypatch.setenv("SKILLEVALUATOR_ALLOWED_MCP_SECRETS", "CLAUDE_PLUGIN_DATA")+ prepared = source(tmp_path / "plugin")+ prepared = replace(+ prepared,+ plugin_file_mcp_servers=(+ {"name": "tool", "command": "node", "args": ["${CLAUDE_PLUGIN_ROOT}/run.js", "${" + variable + "}/output"]},+ ),+ )+ parts = loose_parts.build_loose_parts(prepared)+ tasks = {}+ for arm in ("with_skill", "sum_of_parts", "loose_parts"):+ task = tmp_path / arm+ parts.stage_arm(arm, task / "environment")+ tasks[arm] = [task]+ assert parts.content_report_for_tasks(tasks)["equal_across_arms"] is True+ placement = next(item for item in parts._full if item.kind == "mcp")+ config = tasks["with_skill"][0] / "environment" / placement.path+ payload = json.loads(config.read_text())+ payload["mcpServers"][placement.key]["args"][1] = root + "/output"+ config.write_text(json.dumps(payload))+ result = parts.content_report_for_tasks(tasks)+ assert result["equal_across_arms"] is False+ assert "mcp:tool" in result["mismatches"]+++def test_mcp_literal_root_text_outside_substituted_field_is_preserved(tmp_path):+ prepared = source(tmp_path / "plugin")+ prepared = replace(+ prepared,+ plugin_file_mcp_servers=(+ {+ "name": "tool",+ "command": "node",+ "args": ["${CLAUDE_PLUGIN_ROOT}/run.js", loose_parts.LOOSE_ROOT + "/label"],+ },+ ),+ )+ parts = loose_parts.build_loose_parts(prepared)+ tasks = {}+ for arm in ("with_skill", "sum_of_parts", "loose_parts"):+ task = tmp_path / arm+ parts.stage_arm(arm, task / "environment")+ tasks[arm] = [task]+ assert parts.content_report_for_tasks(tasks)["equal_across_arms"] is True+ placement = next(item for item in parts._loose if item.kind == "mcp")+ config = tasks["loose_parts"][0] / "environment" / placement.path+ payload = json.loads(config.read_text())+ payload["mcpServers"][placement.key]["args"][1] = "${CLAUDE_PLUGIN_ROOT}/label"+ config.write_text(json.dumps(payload))+ result = parts.content_report_for_tasks(tasks)+ assert result["equal_across_arms"] is False+ assert "mcp:tool" in result["mismatches"]+++def test_full_hook_literal_loose_directory_is_not_normalized_as_equal(tmp_path, monkeypatch):+ monkeypatch.setenv("SKILLEVALUATOR_ALLOWED_MCP_SECRETS", "RELTOOLS_TOKEN")+ from tests.tier3.private_loose_fixtures import make_plugin, native_source++ parts = loose_parts.build_loose_parts(+ native_source(make_plugin(tmp_path / "plugins"), tmp_path / "stage").native_source+ )+ tasks = {}+ for arm in ("with_skill", "sum_of_parts", "loose_parts"):+ task = tmp_path / arm+ parts.stage_arm(arm, task / "environment")+ tasks[arm] = [task]+ assert parts.content_report_for_tasks(tasks)["equal_across_arms"] is True+ placement = next(item for item in parts._full if item.kind == "hook")+ config = tasks["with_skill"][0] / "environment" / placement.path+ config.write_text(config.read_text().replace("${CLAUDE_PLUGIN_ROOT}", loose_parts.LOOSE_ROOT))+ result = parts.content_report_for_tasks(tasks)+ assert result["equal_across_arms"] is False+ assert placement.id in result["mismatches"]++[email protected]("script_name", ["run server.js", "run:server.js", "run,server.js"])+def test_mcp_argument_preserves_exact_referenced_filename_and_detects_missing_copy(tmp_path, script_name):+ prepared = source(tmp_path / "plugin")+ (prepared.plugin_root / "run.js").rename(prepared.plugin_root / script_name)+ prepared = replace(+ prepared,+ plugin_file_mcp_servers=(+ {"name": "tool", "command": "node", "args": ["${CLAUDE_PLUGIN_ROOT}/" + script_name]},+ ),+ )+ parts = loose_parts.build_loose_parts(prepared)+ tasks = {}+ for arm in ("with_skill", "sum_of_parts", "loose_parts"):+ task = tmp_path / arm+ parts.stage_arm(arm, task / "environment")+ tasks[arm] = [task]+ copied = tasks["loose_parts"][0] / "environment/skilleval" / loose_parts.LOOSE_BUNDLE_DIR / "root" / script_name+ assert copied.read_bytes() == b"console.log('ok');\n"+ assert copied.stat().st_mode & 0o777 == 0o755+ assert parts.content_report_for_tasks(tasks)["equal_across_arms"] is True+ copied.unlink()+ result = parts.content_report_for_tasks(tasks)+ assert result["equal_across_arms"] is False+ assert any(piece.startswith("script:") for piece in result["mismatches"])+++def test_loose_mcp_reverted_root_substitution_cannot_match_full_template(tmp_path):+ parts = loose_parts.build_loose_parts(source(tmp_path / "plugin"))+ tasks = {}+ for arm in ("with_skill", "sum_of_parts", "loose_parts"):+ task = tmp_path / arm+ parts.stage_arm(arm, task / "environment")+ tasks[arm] = [task]+ assert parts.content_report_for_tasks(tasks)["equal_across_arms"] is True+ placement = next(item for item in parts._loose if item.kind == "mcp")+ config = tasks["loose_parts"][0] / "environment" / placement.path+ payload = json.loads(config.read_text())+ payload["mcpServers"][placement.key]["args"] = ["${CLAUDE_PLUGIN_ROOT}/run.js"]+ config.write_text(json.dumps(payload))+ result = parts.content_report_for_tasks(tasks)+ assert result["equal_across_arms"] is False+ assert "mcp:tool" in result["mismatches"]diff --git a/tests/tier3/test_private_evidence_review2.py b/tests/tier3/test_private_evidence_review2.pynew file mode 100644index 00000000..db5d97c5--- /dev/null+++ b/tests/tier3/test_private_evidence_review2.py@@ -0,0 +1,237 @@+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+# SPDX-License-Identifier: Apache-2.0+"""Private proof bounds and harness evidence retain their source and uncertainty."""++from __future__ import annotations++import gzip+import json+import threading+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer++import pytest+from tests.tier3.test_f90_mcp_execution import _codex_items, _codex_step, _session_line, _traj++from skillevaluator.tier3.eval_core.plugin_signals import compute_plugin_signals+from skillevaluator.tier3.mcp_proof import MAX_RESPONSE_BYTES, apply_in_agent_mcp_proof, probe_mcp_server+from skillevaluator.tier3.plugin_trace_artifacts import TrialTraceArtifacts+++def test_compressed_reply_cannot_bypass_the_host_probe_body_limit():+ seen_encodings = []++ class Handler(BaseHTTPRequestHandler):+ def log_message(self, *_args):+ pass++ def empty(self, status):+ self.send_response(status)+ self.send_header("Content-Length", "0")+ self.end_headers()++ def do_GET(self):+ self.empty(405)++ def do_DELETE(self):+ self.empty(200)++ def do_POST(self):+ seen_encodings.append(self.headers.get("Accept-Encoding"))+ request = json.loads(self.rfile.read(int(self.headers.get("Content-Length", "0"))))+ if "id" not in request:+ self.empty(202)+ return+ compressed = request["method"] == "tools/list"+ result = (+ {+ "tools": [+ {+ "name": "search",+ "description": "x" * (2 * MAX_RESPONSE_BYTES),+ "inputSchema": {"type": "object"},+ }+ ]+ }+ if compressed+ else {+ "protocolVersion": request["params"]["protocolVersion"],+ "capabilities": {"tools": {}},+ "serverInfo": {"name": "fixture", "version": "1"},+ }+ )+ data = json.dumps({"jsonrpc": "2.0", "id": request["id"], "result": result}).encode()+ if compressed:+ data = gzip.compress(data)+ self.send_response(200)+ self.send_header("Content-Type", "application/json")+ self.send_header("Content-Length", str(len(data)))+ if compressed:+ self.send_header("Content-Encoding", "gzip")+ self.end_headers()+ self.wfile.write(data)++ server = ThreadingHTTPServer(("127.0.0.1", 0), Handler)+ thread = threading.Thread(target=server.serve_forever, daemon=True)+ thread.start()+ try:+ proof = probe_mcp_server(+ {+ "name": "docs",+ "url": f"http://127.0.0.1:{server.server_address[1]}/mcp",+ "headers": {"accept-encoding": "gzip"},+ },+ allowed_private_hosts=("127.0.0.1",),+ total_timeout=10,+ )+ finally:+ server.shutdown()+ server.server_close()+ thread.join(timeout=3)+ assert proof["status"] == "unreachable"+ assert "encoding" in proof["detail"].casefold()+ assert seen_encodings and set(seen_encodings) == {"identity"}++[email protected]("session", [False, True])+def test_multistep_codex_logs_keep_reused_call_ids_in_their_source_step(tmp_path, session):+ for index, (server, status) in enumerate((("docs", "completed"), ("tracker", "failed"))):+ agent = tmp_path / "steps" / f"step-{index}" / "agent"+ agent.mkdir(parents=True)+ args, answer = {"q": "same query"}, "same answer"+ (agent / "trajectory.json").write_text(+ json.dumps(_traj(_codex_step("call", "search", args, answer), agent="codex"))+ )+ (agent / "codex.txt").write_text("\n".join(_codex_items("item_1", server, "search", args, answer, status)))+ if session:+ sessions = agent / "sessions"+ sessions.mkdir()+ (sessions / "rollout.jsonl").write_text(_session_line("call", server, "search", args))+ (tmp_path / "result.json").write_text(+ json.dumps({"step_results": [{"step_name": "step-0"}, {"step_name": "step-1"}]})+ )+ # A log outside the authoritative step set must contribute no evidence.+ unrelated = tmp_path / "agent"+ unrelated.mkdir()+ (unrelated / "codex.txt").write_text(+ "\n".join(_codex_items("item_1", "tracker", "search", args, answer, "completed"))+ )+ trace = TrialTraceArtifacts().trace(tmp_path)+ result = compute_plugin_signals(+ trace.trajectory,+ {"expected_tools": ["MCP:docs/search", "MCP:tracker/search"]},+ declared={"mcp": ["docs", "tracker"]},+ mcp_call_servers=trace.call_servers,+ )+ assert result["mcp_calls"]["total"] == 2+ assert result["mcp_calls"]["by_server"]["docs"]["succeeded"] == 1+ assert result["mcp_calls"]["by_server"]["tracker"]["failed"] == 1+ assert result["tool_selection"]["recall"] == 1.0+ assert result["evidence_complete"] is True+++def test_multistep_opencode_dropped_results_keep_step_local_errors(tmp_path):+ for index in range(2):+ agent = tmp_path / "steps" / f"step-{index}" / "agent"+ agent.mkdir(parents=True)+ step = _codex_step("call", "docs_search", {}, "found")+ if index:+ step.pop("observation")+ (agent / "opencode.txt").write_text(+ json.dumps(+ {+ "type": "tool_use",+ "part": {+ "callID": "call",+ "tool": "docs_search",+ "state": {"status": "error", "error": "MCP error -32000: closed"},+ },+ }+ )+ )+ (agent / "trajectory.json").write_text(json.dumps(_traj(step, agent="opencode")))+ (tmp_path / "result.json").write_text(+ json.dumps({"step_results": [{"step_name": "step-0"}, {"step_name": "step-1"}]})+ )+ trace = TrialTraceArtifacts().trace(tmp_path)+ result = compute_plugin_signals(trace.trajectory, {}, declared={"mcp": ["docs"]})+ assert result["mcp_calls"]["total"] == 2+ assert result["mcp_calls"]["succeeded"] == result["mcp_calls"]["failed"] == 1+ assert result["mcp_calls"]["unknown"] == 0++[email protected]("external", [False, True])+def test_multistep_call_scopes_preserve_child_ownership_and_handoffs(tmp_path, external):+ from tests.tier3.test_plugin_evidence_review import call, tree++ for index in range(2):+ agent = tmp_path / "steps" / f"step-{index}" / "agent"+ agent.mkdir(parents=True)+ root, child = tree(external=external)+ child["steps"][0]["observation"]["results"][0]["content"] = "carried-value"+ root["steps"].append(call("consume", "mcp__docs__consume", {"value": "carried-value"}))+ root["steps"][-1]["step_id"] = 2+ (agent / "trajectory.json").write_text(json.dumps(root))+ if external:+ (agent / "child.json").write_text(json.dumps(child))+ (tmp_path / "result.json").write_text(+ json.dumps({"step_results": [{"step_name": "step-0"}, {"step_name": "step-1"}]})+ )+ trace = TrialTraceArtifacts().trace(tmp_path)+ result = compute_plugin_signals(+ trace.trajectory,+ {"handoffs": [{"producer": "Agent:reviewer", "consumer": "MCP:docs/consume", "value": "carried-value"}]},+ declared={"mcp": ["docs"], "subagent": ["reviewer"]},+ )+ assert result["mcp_calls"]["total"] == 4+ assert result["handoff"]["passed"] == 1+ assert result["evidence_complete"] is True+++def test_single_step_authored_multistep_extras_cannot_select_other_logs(tmp_path):+ trial, foreign = tmp_path / "trial", tmp_path / "foreign"+ for directory, server, status in ((trial, "docs", "failed"), (foreign, "tracker", "completed")):+ agent = directory / "agent"+ agent.mkdir(parents=True)+ args = {"q": "same query"}+ (agent / "codex.txt").write_text(+ "\n".join(_codex_items("item_1", server, "search", args, "same answer", status))+ )+ trajectory = _traj(_codex_step("call", "search", args, "same answer"), agent="codex")+ trajectory["extra"] = {"harbor_multi_step": {"step_names": ["../../foreign"]}}+ (trial / "agent/trajectory.json").write_text(json.dumps(trajectory))+ trace = TrialTraceArtifacts().trace(trial)+ result = compute_plugin_signals(+ trace.trajectory,+ {},+ declared={"mcp": ["docs", "tracker"]},+ mcp_call_servers=trace.call_servers,+ )+ assert result["mcp_calls"]["by_server"]["docs"]["failed"] == 1+ assert "tracker" not in result["mcp_calls"]["by_server"]++[email protected]("unknown", [1, 2])+def test_unknown_trial_load_preserves_host_reachability(unknown):+ final = _load_proof({"loaded": 0, "trials": 1, "unknown": unknown})+ assert final["status"] == "reachable-host"+ assert "unknown" in final["detail"]++[email protected]("tally", [{"loaded": 0, "trials": 2}, {"loaded": 0, "trials": 2, "unknown": 0}])+def test_known_never_loaded_classification_accepts_legacy_tallies(tally):+ assert _load_proof(tally)["status"] == "not-loaded-in-agent"+++def _load_proof(tally):+ engine = {+ "agents": {+ "claude-code": {+ "plugin_load_census": {"not_loaded": [{"type": "mcp", "name": "docs", "reason": "init failed"}]},+ "plugin_signals_summary": {"with_skill": {"mcp_load": {"docs": tally}, "mcp_calls": {"by_server": {}}}},+ }+ }+ }+ return apply_in_agent_mcp_proof(+ {"docs": {"status": "reachable-host", "tools": [], "detail": "initialize and tools/list succeeded"}},+ engine,+ )["docs"]diff --git a/tests/tier3/test_private_ladder_stats.py b/tests/tier3/test_private_ladder_stats.pynew file mode 100644index 00000000..d11905e3--- /dev/null+++ b/tests/tier3/test_private_ladder_stats.py@@ -0,0 +1,614 @@+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+# SPDX-License-Identifier: Apache-2.0++"""The lift ladder: four arms, one common basis, four steps that add up.++no plugin -> skills only -> loose parts -> full plugin. The steps are+``skill`` (skills only minus no plugin), ``capability`` (loose parts minus+skills only), ``integration`` (full plugin minus loose parts, the glue) and+``plugin`` (full plugin minus no plugin). On one shared per-case basis+across all four arms they add up per case, on the means, and in every+bootstrap resample. The glue verdict reads cross-component cases only, on a+second basis over the three skill arms that keeps all five dimensions.+"""++from __future__ import annotations++import math+import random+from fractions import Fraction+from typing import Any++import pytest++from skillevaluator.constants import LIFT_BOOTSTRAP_SEED, LIFT_CI_MIN_PAIRED_CASES+from skillevaluator.tier3.harbor import stats+from skillevaluator.tier3.harbor.metrics import DEFAULT_METRIC_SET, DEFAULT_METRICS+from skillevaluator.tier3.harbor.stats import ArmObservations, TrialObservation++NONE, SKILLS, LOOSE, FULL = "without_skill", "sum_of_parts", "loose_parts", "with_skill"+LADDER_ARMS = (NONE, SKILLS, LOOSE, FULL)+STEP_ARMS = {+ "skill": (SKILLS, NONE),+ "capability": (LOOSE, SKILLS),+ "integration": (FULL, LOOSE),+ "plugin": (FULL, NONE),+}+STEP_FIELDS = {"estimate", "ci_low", "ci_high", "n_cases", "precision"}+++def _reward(case_id: str, value: float | None, **overrides: float | None) -> dict[str, Any]:+ """All six default metrics at *value*; a keyword sets one metric (``None`` = not applicable)."""+ reward: dict[str, Any] = {"entry_id": case_id, "metric_set": DEFAULT_METRIC_SET}+ reward.update(dict.fromkeys(DEFAULT_METRICS, value))+ reward.update(overrides)+ reward["details"] = {+ metric: {"status": "not_applicable"}+ for metric, score in reward.items()+ if metric in ("skill_execution", "skill_efficiency", "accuracy", "goal_accuracy", "behavior_check")+ and score is None+ }+ return reward+++def _no_skill(case_id: str, value: float) -> dict[str, Any]:+ """The no-plugin arm: nothing to discover, so the skill metrics are N/A."""+ return _reward(case_id, value, skill_execution=None, skill_efficiency=None)+++def _arm(trials: list[TrialObservation], status: str = "succeeded", job_failure: str = "") -> ArmObservations:+ return ArmObservations(status, tuple(trials), job_failure=job_failure)+++def _flat_arms(cases: list[str], none: float, skills: float, loose: float, full: float) -> dict[str, ArmObservations]:+ return {+ NONE: _arm([TrialObservation(c, _no_skill(c, none)) for c in cases]),+ SKILLS: _arm([TrialObservation(c, _reward(c, skills)) for c in cases]),+ LOOSE: _arm([TrialObservation(c, _reward(c, loose)) for c in cases]),+ FULL: _arm([TrialObservation(c, _reward(c, full)) for c in cases]),+ }+++CASES = [f"c-{index}" for index in range(1, 10)]+CROSS = CASES[:5]+++# ---------------------------------------------------------------------------+# common_case_scores: one basis for every arm+# ---------------------------------------------------------------------------+++def test_common_basis_keeps_only_the_dimensions_every_arm_scored_for_that_case() -> None:+ arms = _flat_arms(["a", "b"], 0.4, 0.6, 0.7, 0.8)+ # Case b: the loose arm has no accuracy score, so correctness drops out of b for every arm.+ arms[LOOSE] = _arm(+ [TrialObservation("a", _reward("a", 0.7)), TrialObservation("b", _reward("b", 0.7, accuracy=None))]+ )+ arms[FULL] = _arm(+ [TrialObservation("a", _reward("a", 0.8)), TrialObservation("b", _reward("b", 0.8, accuracy=0.2))]+ )++ basis = stats.common_case_scores(arms)++ assert basis.case_ids == ("a", "b")+ assert basis.dimensions == ("security", "correctness", "effectiveness")+ assert basis.excluded_cases == ()+ # a: security, correctness, effectiveness all at 0.8. b: only security and effectiveness, so the 0.2 accuracy+ # never counts.+ assert basis.scores[FULL] == {"a": pytest.approx(0.8), "b": pytest.approx(0.8)}+ assert basis.scores[NONE] == {"a": pytest.approx(0.4), "b": pytest.approx(0.4)}+ # Discoverability and efficiency are not in the common basis, even where every arm has them.+ assert "discoverability" not in basis.dimensions and "efficiency" not in basis.dimensions+++def test_attempts_are_averaged_inside_a_case_before_the_dimensions() -> None:+ arms = _flat_arms(["a"], 0.4, 0.6, 0.7, 0.8)+ arms[FULL] = _arm([TrialObservation("a", _reward("a", 0.6)), TrialObservation("a", _reward("a", 1.0))])++ basis = stats.common_case_scores(arms)++ assert basis.scores[FULL]["a"] == pytest.approx(0.8)+++def test_a_case_one_arm_did_not_score_is_dropped_from_every_arm_and_listed() -> None:+ arms = _flat_arms(CASES, 0.4, 0.6, 0.7, 0.8)+ arms[LOOSE] = _arm([TrialObservation(c, _reward(c, 0.7)) for c in CASES if c != "c-3"])++ basis = stats.common_case_scores(arms)++ assert "c-3" not in basis.case_ids+ assert all("c-3" not in scores for scores in basis.scores.values())+ assert basis.excluded_cases == (+ {"case": "c-3", "reason": "not_scored", "missing_arms": [LOOSE], "load_failures": {}},+ )+++def test_a_case_in_one_arm_only_lists_the_three_arms_it_lacks() -> None:+ arms = _flat_arms(["a", "b"], 0.4, 0.6, 0.7, 0.8)+ arms[FULL] = _arm([*arms[FULL].trials, TrialObservation("only-full", _reward("only-full", 0.9))])++ basis = stats.common_case_scores(arms)++ assert basis.case_ids == ("a", "b")+ assert basis.excluded_cases == (+ {"case": "only-full", "reason": "not_scored", "missing_arms": [NONE, SKILLS, LOOSE], "load_failures": {}},+ )+++def test_unscored_sparse_case_cannot_supply_a_reduced_dimension_basis() -> None:+ arms = _flat_arms(["a", "b"], 0.4, 0.6, 0.7, 0.8)+ # Security is mandatory; an accuracy-only reward has no valid score.+ only_accuracy = _reward("b", None, accuracy=0.5)+ only_security = _reward("b", None, security=0.9, skill_execution=0.9)+ arms[NONE] = _arm([TrialObservation("a", _no_skill("a", 0.4)), TrialObservation("b", only_accuracy)])+ arms[FULL] = _arm([TrialObservation("a", _reward("a", 0.8)), TrialObservation("b", only_security)])++ basis = stats.common_case_scores(arms)++ assert basis.case_ids == ("a",)+ assert basis.excluded_cases == ({"case": "b", "reason": "not_scored", "missing_arms": [NONE], "load_failures": {}},)+++def test_an_expected_case_no_arm_scored_is_listed_with_every_arm() -> None:+ arms = _flat_arms(["a"], 0.4, 0.6, 0.7, 0.8)++ basis = stats.common_case_scores(arms, expected_case_ids=["a", "never-ran"])++ assert basis.excluded_cases == (+ {"case": "never-ran", "reason": "not_scored", "missing_arms": list(LADDER_ARMS), "load_failures": {}},+ )+++def test_a_load_failure_in_any_arm_drops_the_case_from_every_arm() -> None:+ """A part Claude Code did not report loaded (and never called) voids the case in all four arms."""+ arms = _flat_arms(["a", "b", "c"], 0.4, 0.6, 0.7, 0.8)+ arms[SKILLS] = _arm([TrialObservation(c, _reward(c, 0.6)) for c in ("a", "b")])++ basis = stats.common_case_scores(+ arms,+ load_failures_by_case={+ "a": {LOOSE: "rule:release-policy.md not in the startup report and never read"},+ "c": {FULL: "mcp:reltools not loaded"},+ },+ )++ assert basis.case_ids == ("b",)+ assert basis.excluded_cases == (+ {+ "case": "a",+ "reason": "load_failed",+ "missing_arms": [],+ "load_failures": {LOOSE: "rule:release-policy.md not in the startup report and never read"},+ },+ # Not scored by the skills arm AND a load failure: load failure is the reason, both are listed.+ {+ "case": "c",+ "reason": "load_failed",+ "missing_arms": [SKILLS],+ "load_failures": {FULL: "mcp:reltools not loaded"},+ },+ )+++def test_an_empty_load_failure_map_changes_nothing() -> None:+ arms = _flat_arms(CASES, 0.4, 0.6, 0.7, 0.8)+ assert stats.common_case_scores(arms, load_failures_by_case={}) == stats.common_case_scores(arms)+ assert stats.common_case_scores(arms, load_failures_by_case={"c-1": {}}) == stats.common_case_scores(arms)+++def test_a_custom_basis_keeps_discoverability_and_efficiency() -> None:+ arms = {arm: obs for arm, obs in _flat_arms(["a"], 0.4, 0.6, 0.7, 0.8).items() if arm != NONE}+ arms[FULL] = _arm([TrialObservation("a", _reward("a", 0.8, skill_execution=1.0))])++ basis = stats.common_case_scores(arms, dimensions=stats.GLUE_BASIS_DIMENSIONS)++ assert basis.dimensions == ("security", "correctness", "discoverability", "effectiveness", "efficiency")+ assert basis.scores[FULL]["a"] == pytest.approx((0.8 * 4 + 1.0) / 5)+++# ---------------------------------------------------------------------------+# lift_ladder: the block+# ---------------------------------------------------------------------------+++def test_a_clean_four_arm_run_gives_every_step_and_the_bundle() -> None:+ block = stats.lift_ladder(_flat_arms(CASES, 0.4, 0.6, 0.7, 0.8), CROSS, expected_case_ids=CASES)++ assert block["schema_version"] == "1.0"+ assert block["advisory"] is True and block["report_only"] is True+ assert block["arms"] == {"none": NONE, "skills": SKILLS, "loose": LOOSE, "full": FULL}+ assert block["basis"] == "common_dimensions_all_arms"+ assert block["dimensions"] == ["security", "correctness", "effectiveness"]+ assert (block["n_cases"], block["expected_cases"], block["partial"]) == (9, 9, False)+ assert block["excluded_cases"] == [] and block["failed_arms"] == []+ assert set(block["steps"]) == {"skill", "capability", "integration", "plugin"}+ for step in (*block["steps"].values(), block["bundle"]):+ assert set(step) == STEP_FIELDS+ assert step["n_cases"] == 9+ estimates = {name: step["estimate"] for name, step in block["steps"].items()}+ assert estimates == pytest.approx({"skill": 0.2, "capability": 0.1, "integration": 0.1, "plugin": 0.4})+ assert block["bundle"]["estimate"] == pytest.approx(0.2)+ assert block["bootstrap"]["seed"] == LIFT_BOOTSTRAP_SEED+ assert block["bootstrap"]["method"] == "paired_case_bootstrap"+ assert block["bootstrap"]["interval"] == "expanded_percentile"+++def test_case_type_split_has_the_same_steps_on_each_group() -> None:+ block = stats.lift_ladder(_flat_arms(CASES, 0.4, 0.6, 0.7, 0.8), CROSS)++ split = block["by_case_type"]+ assert set(split) == {"cross_component", "other"}+ assert split["cross_component"]["n_cases"] == 5 and split["other"]["n_cases"] == 4+ for group in split.values():+ assert set(group["steps"]) == {"skill", "capability", "integration", "plugin"}+ assert group["bundle"]["estimate"] == pytest.approx(0.2)+++def test_glue_reads_cross_component_cases_on_the_five_dimension_basis() -> None:+ """Glue that only shows in routing (discoverability) is 0 on the common basis but not in the glue verdict."""+ arms = _flat_arms(CASES, 0.4, 0.6, 0.7, 0.7)+ arms[FULL] = _arm([TrialObservation(c, _reward(c, 0.7, skill_execution=1.0)) for c in CASES])++ block = stats.lift_ladder(arms, CROSS)++ assert block["steps"]["integration"]["estimate"] == pytest.approx(0.0)+ glue = block["glue"]+ assert glue["case_type"] == "cross_component"+ assert glue["basis"] == "common_dimensions_skill_arms"+ assert glue["arms"] == [SKILLS, LOOSE, FULL]+ assert glue["basis_dimensions"] == ["security", "correctness", "discoverability", "effectiveness", "efficiency"]+ assert glue["n"] == 5+ assert glue["lift"] == pytest.approx(0.3 / 5)+ assert glue["ci_low"] == pytest.approx(0.06) and glue["ci_high"] == pytest.approx(0.06)+ assert glue["partial"] is False and glue["reason"] is None and glue["excluded_cases"] == []+++def test_fewer_than_five_cross_component_pairs_give_no_usable_glue_interval() -> None:+ block = stats.lift_ladder(_flat_arms(CASES, 0.4, 0.6, 0.7, 0.8), CASES[:4])++ glue = block["glue"]+ assert glue["n"] == 4 < LIFT_CI_MIN_PAIRED_CASES+ assert glue["lift"] == pytest.approx(0.1)+ assert glue["precision"] == "insufficient"+ # No interval below 5 pairs, as for the Bundle lift's case-type split.+ assert (glue["ci_low"], glue["ci_high"]) == (None, None)+ group = block["by_case_type"]["cross_component"]+ assert group["n_cases"] == 4+ assert all((step["ci_low"], step["ci_high"]) == (None, None) for step in group["steps"].values())+ assert group["steps"]["skill"]["estimate"] == pytest.approx(0.2)+ # The whole ladder (9 cases) keeps its intervals.+ assert block["steps"]["skill"]["ci_low"] is not None+ assert glue["reason"] == "Only 4 paired cross-component case(s); at least 5 are needed for an interval."+++def test_no_cross_component_case_leaves_the_glue_unmeasured() -> None:+ block = stats.lift_ladder(_flat_arms(CASES, 0.4, 0.6, 0.7, 0.8), [])++ assert block["by_case_type"]["cross_component"]["n_cases"] == 0+ assert all(step["estimate"] is None for step in block["by_case_type"]["cross_component"]["steps"].values())+ assert block["by_case_type"]["other"]["n_cases"] == 9+ glue = block["glue"]+ assert (glue["n"], glue["lift"], glue["ci_low"], glue["ci_high"]) == (0, None, None, None)+ assert glue["reason"] == "No cross-component case was scored in every arm."+++def test_unrecorded_case_types_record_no_split_and_no_glue() -> None:+ block = stats.lift_ladder(_flat_arms(CASES, 0.4, 0.6, 0.7, 0.8), None)++ assert block["by_case_type"] is None+ assert block["glue"]["lift"] is None+ assert block["glue"]["reason"] == "The run recorded no case types, so no case is known to be cross-component."+ assert block["steps"]["integration"]["estimate"] == pytest.approx(0.1)+++def test_all_cases_cross_component_makes_the_cross_group_the_whole_ladder() -> None:+ block = stats.lift_ladder(_flat_arms(CASES, 0.4, 0.6, 0.7, 0.8), CASES)++ assert block["by_case_type"]["cross_component"]["steps"] == block["steps"]+ assert block["by_case_type"]["other"]["n_cases"] == 0+ assert block["glue"]["n"] == 9+++def test_an_excluded_case_is_out_of_every_step_and_the_glue() -> None:+ arms = _flat_arms(CASES, 0.4, 0.6, 0.7, 0.8)+ # c-2 is cross-component; the no-plugin arm never scored it. The glue does not use that arm, but the case+ # is still dropped everywhere, so the glue and the steps describe one case set.+ arms[NONE] = _arm([TrialObservation(c, _no_skill(c, 0.4)) for c in CASES if c != "c-2"])++ block = stats.lift_ladder(arms, CROSS, expected_case_ids=CASES)++ assert block["n_cases"] == 8 and block["partial"] is True+ assert {step["n_cases"] for step in block["steps"].values()} == {8}+ assert block["excluded_cases"] == [+ {"case": "c-2", "reason": "not_scored", "missing_arms": [NONE], "load_failures": {}}+ ]+ assert block["glue"]["n"] == 4+ assert block["glue"]["partial"] is True+ assert block["glue"]["excluded_cases"] == ["c-2"]+++def test_load_failures_exclude_the_case_from_the_ladder_and_the_glue() -> None:+ block = stats.lift_ladder(+ _flat_arms(CASES, 0.4, 0.6, 0.7, 0.8),+ CROSS,+ load_failures_by_case={"c-1": {LOOSE: "rule not loaded"}},+ )++ assert block["n_cases"] == 8 and block["partial"] is True+ assert block["excluded_cases"][0]["reason"] == "load_failed"+ assert block["glue"]["n"] == 4 and block["glue"]["excluded_cases"] == ["c-1"]+++def test_a_failed_arm_marks_the_ladder_partial_and_names_the_arm() -> None:+ arms = _flat_arms(CASES, 0.4, 0.6, 0.7, 0.8)+ arms[LOOSE] = _arm(list(arms[LOOSE].trials), status="failed", job_failure="looseparts job failed")++ block = stats.lift_ladder(arms, CROSS)++ assert block["partial"] is True+ assert block["failed_arms"] == [LOOSE]+ assert block["glue"]["partial"] is True+ # The scored cases still count: a failed job keeps the cases its trials scored.+ assert block["n_cases"] == 9+++def test_a_missing_arm_leaves_no_case_and_is_named() -> None:+ arms = _flat_arms(CASES, 0.4, 0.6, 0.7, 0.8)+ del arms[LOOSE]++ block = stats.lift_ladder(arms, CROSS, expected_case_ids=CASES)++ assert block["n_cases"] == 0 and block["partial"] is True+ assert block["failed_arms"] == [LOOSE]+ assert all(step["estimate"] is None for step in block["steps"].values())+ assert {entry["reason"] for entry in block["excluded_cases"]} == {"not_scored"}+ assert block["excluded_cases"][0]["missing_arms"] == [LOOSE]+++# ---------------------------------------------------------------------------+# Property: the steps add up, per case, on the means and in every resample+# ---------------------------------------------------------------------------+++def _random_reward(case_id: str, rng: random.Random, *, skill: bool) -> dict[str, Any]:+ reward: dict[str, Any] = {"entry_id": case_id, "metric_set": DEFAULT_METRIC_SET}+ for metric in DEFAULT_METRICS:+ if rng.random() < 0.08 or (not skill and metric in ("skill_execution", "skill_efficiency")):+ reward[metric] = None+ else:+ reward[metric] = round(rng.random(), 3)+ reward["details"] = {+ metric: {"status": "not_applicable"}+ for metric in DEFAULT_METRICS+ if metric != "security" and reward[metric] is None+ }+ return reward+++def _random_ladder_arms(rng: random.Random) -> tuple[dict[str, ArmObservations], set[str], list[str]]:+ """Four random arms: 0-24 cases, 1-3 attempts, some cases missing from an arm, some N/A metrics."""+ cases = [f"case-{index}" for index in range(rng.randint(0, 24))]+ trials: dict[str, list[TrialObservation]] = {arm: [] for arm in LADDER_ARMS}+ for case in cases:+ for arm in LADDER_ARMS:+ if rng.random() < 0.06:+ continue # this arm lost the case+ for _ in range(rng.randint(1, 3)):+ trials[arm].append(TrialObservation(case, _random_reward(case, rng, skill=arm != NONE)))+ share = rng.choice((0.0, 0.3, 0.6, 1.0))+ cross = {case for case in cases if rng.random() < share} | {"not-a-case"}+ return {arm: _arm(observed) for arm, observed in trials.items()}, cross, cases+++def test_steps_add_up_per_case_on_the_means_and_in_every_resample_on_random_data(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """Property, 80 random runs.++ Every step is bootstrapped over the same cases with the same seed, so+ ``rng.choices`` draws the same case indices for each step; the per-case+ deltas telescope, so every resample mean adds up too. The rounded+ estimates add up within 0.0002 (four values rounded to 4 places).+ """+ monkeypatch.setattr(stats, "LIFT_BOOTSTRAP_TAIL_RESAMPLES", 5)+ calls: list[tuple[dict[str, float], dict[str, float], dict[str, Any]]] = []+ real_bootstrap = stats.paired_case_bootstrap++ def spy(treatment: Any, control: Any, **kwargs: Any) -> dict[str, Any]:+ result = real_bootstrap(treatment, control, **kwargs)+ calls.append((dict(treatment), dict(control), result))+ return result++ monkeypatch.setattr(stats, "paired_case_bootstrap", spy)+ rng = random.Random(20261010)+ checked = 0+ for _ in range(80):+ arms, cross, cases = _random_ladder_arms(rng)+ calls.clear()+ block = stats.lift_ladder(arms, cross, expected_case_ids=cases)+ basis = stats.common_case_scores(arms, expected_case_ids=cases)+ # The first four bootstrap calls are the four steps over all common cases, in step order.+ steps = dict(zip(("skill", "capability", "integration", "plugin"), calls[:4], strict=True))+ case_ids = set(basis.case_ids)+ for name, (treatment, control, result) in steps.items():+ treatment_arm, control_arm = STEP_ARMS[name]+ # Identical case sets for every step: the common cases, nothing else.+ assert set(treatment) == set(control) == case_ids+ assert treatment == dict(basis.scores[treatment_arm]) and control == dict(basis.scores[control_arm])+ assert result["n_cases"] == len(case_ids) == block["steps"][name]["n_cases"] == block["n_cases"]+ assert result["seed"] == LIFT_BOOTSTRAP_SEED+ assert len({result["resamples"] for _, _, result in steps.values()}) == 1+ # The steps share their arm scores exactly, so per case they telescope.+ for case in case_ids:+ skill, capability = steps["skill"], steps["capability"]+ integration, plugin = steps["integration"], steps["plugin"]+ assert skill[0][case] == capability[1][case] # skills only+ assert capability[0][case] == integration[1][case] # loose parts+ assert integration[0][case] == plugin[0][case] # full plugin+ assert skill[1][case] == plugin[1][case] # no plugin+ exact = sum(Fraction(t[case]) - Fraction(c[case]) for t, c, _ in (skill, capability, integration))+ assert exact == Fraction(plugin[0][case]) - Fraction(plugin[1][case])+ floats = [t[case] - c[case] for t, c, _ in (skill, capability, integration)]+ assert math.isclose(math.fsum(floats), plugin[0][case] - plugin[1][case], abs_tol=1e-12)+ if not case_ids:+ assert all(step["estimate"] is None for step in block["steps"].values())+ continue+ deltas = {+ name: [treatment[case] - control[case] for case in sorted(case_ids)]+ for name, (treatment, control, _) in steps.items()+ }+ means = {name: math.fsum(values) / len(values) for name, values in deltas.items()}+ assert math.isclose(means["skill"] + means["capability"] + means["integration"], means["plugin"], abs_tol=1e-12)+ estimates = {name: block["steps"][name]["estimate"] for name in steps}+ assert (+ abs(estimates["skill"] + estimates["capability"] + estimates["integration"] - estimates["plugin"]) <= 2e-4+ )+ assert abs(estimates["capability"] + estimates["integration"] - block["bundle"]["estimate"]) <= 2e-4+ # Every resample adds up: the same seed and n draw the same case indices for each step.+ resamples = steps["skill"][2]["resamples"]+ draws = {+ name: stats._resample_means(values, resamples=resamples, seed=LIFT_BOOTSTRAP_SEED)+ for name, values in deltas.items()+ }+ for index in range(resamples):+ total = draws["skill"][index] + draws["capability"][index] + draws["integration"][index]+ assert math.isclose(total, draws["plugin"][index], abs_tol=1e-12)+ # The case-type groups partition the common cases, and add up inside each group too.+ split = block["by_case_type"]+ assert split["cross_component"]["n_cases"] + split["other"]["n_cases"] == len(case_ids)+ for group in split.values():+ group_steps = group["steps"]+ if group["n_cases"]:+ total = sum(group_steps[name]["estimate"] for name in ("skill", "capability", "integration"))+ assert abs(total - group_steps["plugin"]["estimate"]) <= 2e-4+ # The glue reads exactly the cross-component common cases.+ assert block["glue"]["n"] == len(case_ids & cross)+ checked += 1+ assert checked >= 60+++def test_resample_means_draw_the_same_indices_whatever_the_values() -> None:+ """The seam the additivity rests on: ``rng.choices`` picks indices from the seed and n alone."""+ first = stats._resample_means([1.0, 2.0, 3.0, 4.0, 5.0], resamples=50, seed=0)+ second = stats._resample_means([10.0, 20.0, 30.0, 40.0, 50.0], resamples=50, seed=0)+ assert [10 * value for value in first] == pytest.approx(second)+++# ---------------------------------------------------------------------------+# Integration completeness checks loose parts against the full plugin too+# ---------------------------------------------------------------------------+++def _completeness(**kwargs: Any) -> dict[str, Any]:+ arms = _flat_arms(["a", "b"], 0.4, 0.6, 0.7, 0.8)+ params: dict[str, Any] = {+ "expected_case_ids": ["a", "b"],+ "n_attempts": 1,+ "stop_on_pass": False,+ "pass_threshold": 0.5,+ }+ params.update(kwargs)+ return stats.integration_completeness(arms[FULL], arms[SKILLS], **params)+++def test_completeness_without_a_loose_arm_is_unchanged() -> None:+ result = _completeness()+ assert "loose_parts" not in result+ assert result == {"complete": True, "missing_cases": [], "failed_arms": [], "attempt_shortfall": []}+++def test_completeness_checks_loose_parts_against_the_full_plugin() -> None:+ loose = _arm([TrialObservation("a", _reward("a", 0.7))], status="failed", job_failure="boom")++ result = _completeness(loose_parts_arm=loose)++ # The full-versus-skills check is the same as before.+ assert {key: result[key] for key in ("complete", "missing_cases", "failed_arms", "attempt_shortfall")} == {+ "complete": True,+ "missing_cases": [],+ "failed_arms": [],+ "attempt_shortfall": [],+ }+ assert result["loose_parts"] == {+ "complete": False,+ "missing_cases": ["b"],+ "failed_arms": [LOOSE],+ "attempt_shortfall": [{"case": "b", "arm": LOOSE, "expected": 1, "observed": 0}],+ }+++def test_a_complete_loose_arm_is_complete() -> None:+ loose = _arm([TrialObservation(c, _reward(c, 0.7)) for c in ("a", "b")])+ assert _completeness(loose_parts_arm=loose)["loose_parts"]["complete"] is True+++# ---------------------------------------------------------------------------+# build_agent_statistics: the block is optional+# ---------------------------------------------------------------------------+++def _agent_statistics(arms: dict[str, ArmObservations], **kwargs: Any) -> dict[str, Any]:+ return stats.build_agent_statistics(+ arms,+ expected_case_ids=CASES,+ n_attempts=1,+ stop_on_pass=False,+ pass_threshold=0.5,+ sum_of_parts_requested=True,+ cross_component_case_ids=frozenset(CROSS),+ **kwargs,+ )+++def test_statistics_without_loose_observations_carry_no_ladder() -> None:+ arms = _flat_arms(CASES, 0.4, 0.6, 0.7, 0.8)+ del arms[LOOSE]++ result = _agent_statistics(arms)++ assert "lift_ladder" in stats.STATISTICS_BLOCKS+ assert tuple(result) == stats.STATISTICS_BLOCKS+ assert result["lift_ladder"] is None+ assert "loose_parts" not in result["integration_completeness"]+++def test_statistics_with_loose_observations_carry_the_ladder_and_its_completeness() -> None:+ arms = _flat_arms(CASES, 0.4, 0.6, 0.7, 0.8)+ failures = {"c-9": {LOOSE: "rule not loaded"}}++ result = _agent_statistics(arms, load_failures_by_case=failures)++ assert tuple(result) == stats.STATISTICS_BLOCKS+ assert result["lift_ladder"] == stats.lift_ladder(+ arms, frozenset(CROSS), expected_case_ids=CASES, load_failures_by_case=failures+ )+ assert result["lift_ladder"]["n_cases"] == 8+ assert result["integration_completeness"]["loose_parts"]["complete"] is True+ # The loose arm gets its own reliability and cost rows like any arm that ran.+ assert LOOSE in result["reliability"] or LOOSE in result["cost"]+ # The existing Bundle lift is untouched by the extra arm.+ without_loose = {arm: obs for arm, obs in arms.items() if arm != LOOSE}+ assert result["lift_uncertainty"] == _agent_statistics(without_loose)["lift_uncertainty"]+++def test_statistics_respect_an_unrequested_skills_arm() -> None:+ """The skills-only rung uses the arm only when it was requested, like the Bundle lift."""+ result = stats.build_agent_statistics(+ _flat_arms(CASES, 0.4, 0.6, 0.7, 0.8),+ expected_case_ids=CASES,+ n_attempts=1,+ stop_on_pass=False,+ pass_threshold=0.5,+ sum_of_parts_requested=False,+ cross_component_case_ids=frozenset(CROSS),+ )++ ladder = result["lift_ladder"]+ assert ladder["failed_arms"] == [SKILLS]+ assert ladder["n_cases"] == 0 and ladder["partial"] is True+++def test_a_legacy_members_baseline_gets_no_ladder() -> None:+ """Legacy ``--lift-mode integration``: the only baseline holds the member skills, so there is no no-plugin rung."""+ result = _agent_statistics(_flat_arms(CASES, 0.4, 0.6, 0.7, 0.8), baseline_is_sum_of_parts=True)+ assert result["lift_ladder"] is Nonediff --git a/tests/tier3/test_private_lift_small_n.py b/tests/tier3/test_private_lift_small_n.pynew file mode 100644index 00000000..22e3be4c--- /dev/null+++ b/tests/tier3/test_private_lift_small_n.py@@ -0,0 +1,72 @@+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+# SPDX-License-Identifier: Apache-2.0++"""Proof M12: lift intervals were overconfident at small n, and "adequate" said so.++Proof example check-14 edge-05 drew cases from 39 real per-case lift deltas+and asked for the 95% interval: it held the true lift only 82% of the time at+5 cases, and 48% of the intervals labelled "adequate" at 5 cases missed it.+The interval is now an expanded percentile bootstrap (wider at small n), and+"adequate" needs at least 10 paired cases as well as a narrow interval.+"""++from __future__ import annotations++import random+import statistics++import pytest++from skillevaluator.tier3.harbor import stats++# The 39 per-case lift deltas of proof example check-14 edge-05 (population.json).+POPULATION = [+ 0.5067, 0.6742, 0.57, 0.3905, 0.324, 0.3891, 0.3752, 0.3876, 0.0, 0.2945, 0.4719, 0.2325, 0.5522,+ 0.5289, 0.3061, 0.2702, 0.1871, 0.0333, 0.2187, 0.4362, 0.1843, 0.4144, 0.3971, 0.0428, 0.3301,+ 0.338, -0.0173, 0.1923, 0.34, 0.1539, 0.4773, 0.3624, 0.0, 0.1567, 0.0, 0.0333, 0.15, 0.2867, 0.0,+] # fmt: skip+++def _simulate(n_cases: int, sims: int, seed: int) -> tuple[float, list[str]]:+ truth = statistics.fmean(POPULATION)+ rng = random.Random(seed)+ covered = 0+ labels: list[str] = []+ for _ in range(sims):+ deltas = [rng.choice(POPULATION) for _ in range(n_cases)]+ treatment = {f"c{index}": delta for index, delta in enumerate(deltas)}+ interval = stats.paired_case_bootstrap(treatment, dict.fromkeys(treatment, 0.0))+ covered += interval["ci_low"] <= truth <= interval["ci_high"]+ labels.append(interval["precision"])+ return covered / sims, labels++[email protected](("n_cases", "minimum_coverage"), [(5, 0.87), (9, 0.90)])+def test_small_n_intervals_cover_the_true_lift(n_cases: int, minimum_coverage: float) -> None:+ coverage, labels = _simulate(n_cases, sims=400, seed=2026)++ # Before the fix: 81.8% at 5 cases and 89.6% at 9 (proof edge-05, 1,000 runs).+ assert coverage >= minimum_coverage+ # A handful of cases is never "adequate", however narrow the interval looks.+ assert "adequate" not in labels+++def test_adequate_needs_enough_cases_as_well_as_a_narrow_interval() -> None:+ assert stats.lift_precision(9, 0.30, 0.35) == "low"+ assert stats.lift_precision(10, 0.30, 0.35) == "adequate"+ assert stats.lift_precision(30, 0.10, 0.35) == "low"+ assert stats.lift_precision(4, 0.30, 0.31) == "insufficient"+++def test_expanded_interval_is_wider_at_small_n_and_converges() -> None:+ assert stats.expanded_tail(5, 0.95) < stats.expanded_tail(9, 0.95) < stats.expanded_tail(100, 0.95) < 0.025+ assert stats.expanded_tail(100_000, 0.95) == pytest.approx(0.025, abs=1e-4)+ # The t quantiles behind the expansion match the textbook values.+ assert stats._student_t_quantile(0.975, 4) == pytest.approx(2.7764, abs=1e-4)+ assert stats._student_t_quantile(0.975, 8) == pytest.approx(2.3060, abs=1e-4)++ deltas = {f"c{index}": value for index, value in enumerate(POPULATION[:6])}+ interval = stats.paired_case_bootstrap(deltas, dict.fromkeys(deltas, 0.0))+ assert interval["interval"] == "expanded_percentile"+ assert interval["ci_low"] <= interval["estimate"] <= interval["ci_high"]+ assert interval["precision"] == "low"diff --git a/tests/tier3/test_private_loose_dependency_review.py b/tests/tier3/test_private_loose_dependency_review.pynew file mode 100644index 00000000..dcaa35ea--- /dev/null+++ b/tests/tier3/test_private_loose_dependency_review.py@@ -0,0 +1,118 @@+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+# SPDX-License-Identifier: Apache-2.0++"""Local entrypoint dependencies survive staging and participate in parity."""++from __future__ import annotations++import json+import os+import shutil+import subprocess+import sys+from pathlib import Path++import pytest++sys.path.insert(0, str(Path(__file__).resolve().parent))++from private_loose_fixtures import make_skills_only_plugin, native_source, write++from skillevaluator.tier3.loose_parts import LOOSE_BUNDLE_DIR, LOOSE_ROOT, build_loose_parts+from skillevaluator.tier3.plugin_native import CLAUDE_PLUGIN_ROOT_VAR++SUPPORT_FILES = {+ "servers/dependency.js": "module.exports = require('./lib/child.js');\n",+ "servers/lib/child.js": "module.exports = require('../../shared/value.json').value;\n",+ "shared/value.json": '{"value":"relative-dependency-ready"}\n',+}+STAGED_ROOTS = {+ "with_skill": Path("skilleval/native/claude-code/plugin"),+ "loose_parts": Path(f"skilleval/{LOOSE_BUNDLE_DIR}/root"),+}+++def dependency_tasks(tmp_path: Path):+ plugin = make_skills_only_plugin(tmp_path / "plugins")+ write(+ plugin / ".mcp.json",+ json.dumps(+ {"mcpServers": {"reltools": {"command": "node", "args": ["${CLAUDE_PLUGIN_ROOT}/servers/reltools.js"]}}}+ ),+ )+ write(plugin / "servers/reltools.js", "console.log(require('./dependency.js'));\n", mode=0o755)+ for relative, content in SUPPORT_FILES.items():+ write(plugin / relative, content)+ parts = build_loose_parts(native_source(plugin, tmp_path / "stage").native_source)+ assert parts.error is None, parts.error+ tasks = {}+ for arm in ("with_skill", "loose_parts", "sum_of_parts"):+ task = tmp_path / "tasks" / arm / "case-1"+ parts.stage_arm(arm, task / "environment")+ tasks[arm] = [task]+ return parts, tasks+++def test_relative_entrypoint_dependency_closure_is_present_in_both_emitted_arms(dependency_tasks) -> None:+ parts, tasks = dependency_tasks+ for arm, relative_root in STAGED_ROOTS.items():+ root = tasks[arm][0] / "environment" / relative_root+ for relative, content in SUPPORT_FILES.items():+ target = root / relative+ assert target.is_file(), f"{arm} omitted {relative} needed by servers/reltools.js"+ assert target.read_text(encoding="utf-8") == content+ report = parts.content_report_for_tasks(tasks)+ assert report["equal_across_arms"] is True, report["mismatches"]++[email protected]("arm", ["with_skill", "loose_parts"])[email protected]("change", ["modify", "remove", "chmod"])+def test_emitted_dependency_damage_makes_parity_fail(dependency_tasks, arm: str, change: str) -> None:+ parts, tasks = dependency_tasks+ baseline = parts.content_report_for_tasks(tasks)+ assert baseline["equal_across_arms"] is True, baseline+ target = tasks[arm][0] / "environment" / STAGED_ROOTS[arm] / "shared/value.json"+ assert target.is_file(), f"{arm} omitted the dependency before the mutation"+ if change == "modify":+ target.write_text('{"value":"wrong-result"}\n', encoding="utf-8")+ elif change == "remove":+ target.unlink()+ else:+ target.chmod(target.stat().st_mode ^ 0o100)++ report = parts.content_report_for_tasks(tasks)+ assert report["equal_across_arms"] is False, report+ assert report["mismatches"], report+++def test_both_emitted_mcp_entrypoints_execute_with_relative_dependencies(dependency_tasks) -> None:+ node = shutil.which("node")+ if node is None:+ pytest.skip("A local Node executable is needed only for the entrypoint runtime check")+ _, tasks = dependency_tasks+ for arm, relative_root in STAGED_ROOTS.items():+ environment = tasks[arm][0] / "environment"+ root = environment / relative_root+ config = (+ root / ".mcp.json" if arm == "with_skill" else environment / "skilleval" / LOOSE_BUNDLE_DIR / "mcp.json"+ )+ server = json.loads(config.read_text(encoding="utf-8"))["mcpServers"]["reltools"]+ assert server["command"] == "node"+ args = [arg.replace(CLAUDE_PLUGIN_ROOT_VAR, str(root)).replace(LOOSE_ROOT, str(root)) for arg in server["args"]]+ # Translate only the emitted container root to this test's local root.+ assert args == [str(root / "servers/reltools.js")]+ environment_variables = {+ key: value for key, value in os.environ.items() if key not in {"NODE_OPTIONS", "NODE_PATH"}+ }+ completed = subprocess.run(+ [node, *args],+ cwd=environment,+ env=environment_variables,+ capture_output=True,+ text=True,+ timeout=10,+ check=False,+ )+ assert completed.returncode == 0, f"{arm}: {completed.stderr}"+ assert completed.stdout == "relative-dependency-ready\n", armdiff --git a/tests/tier3/test_private_loose_dependency_scaffold_review.py b/tests/tier3/test_private_loose_dependency_scaffold_review.pynew file mode 100644index 00000000..267f5515--- /dev/null+++ b/tests/tier3/test_private_loose_dependency_scaffold_review.py@@ -0,0 +1,84 @@+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+# SPDX-License-Identifier: Apache-2.0++"""Resource parity normalizes generated scaffolding without hiding authored content."""++from __future__ import annotations++import json+from pathlib import Path++from skillevaluator.tier3.loose_parts import LOOSE_BUNDLE_DIR, LooseParts, build_loose_parts+from skillevaluator.tier3.plugin_native import NativePluginSource+++def _stage_relocated_member(tmp_path: Path, *, authored_skills_directory: bool = False):+ plugin = tmp_path / "plugin"+ metadata = plugin / ".claude-plugin"+ metadata.mkdir(parents=True)+ manifest = {"name": "relocated", "description": "A relocated member fixture"}+ (metadata / "plugin.json").write_text(json.dumps(manifest), encoding="utf-8")+ member = plugin / "other" / "a"+ member.mkdir(parents=True)+ (member / "SKILL.md").write_text("---\nname: a\ndescription: A relocated skill\n---\nUse a.\n", encoding="utf-8")+ (plugin / "run.js").write_text("console.log('ready');\n", encoding="utf-8")+ if authored_skills_directory:+ (plugin / "skills").mkdir(mode=0o755)+ (plugin / "skills").chmod(0o755)+ source = NativePluginSource(+ plugin_name="relocated",+ description="A relocated member fixture",+ contained=True,+ plugin_root=plugin,+ manifest=manifest,+ manifest_rel=".claude-plugin/plugin.json",+ member_skills=(member,),+ plugin_file_mcp_servers=({"name": "tool", "command": "node", "args": ["${CLAUDE_PLUGIN_ROOT}/run.js"]},),+ )+ parts = build_loose_parts(source)+ assert parts.error is None, parts.error+ tasks = {}+ for arm in ("with_skill", "loose_parts", "sum_of_parts"):+ task = tmp_path / "tasks" / arm / "case-1"+ parts.stage_arm(arm, task / "environment")+ tasks[arm] = [task]+ return parts, tasks+++def _assert_equal(parts: LooseParts, tasks: dict[str, list[Path]]) -> None:+ report = parts.content_report_for_tasks(tasks)+ assert report["equal_across_arms"] is True, report["mismatches"]+++def test_relocated_member_generated_parent_does_not_create_a_resource_mismatch(tmp_path: Path) -> None:+ parts, tasks = _stage_relocated_member(tmp_path)+ full = tasks["with_skill"][0] / "environment/skilleval/native/claude-code/plugin"+ loose = tasks["loose_parts"][0] / "environment/skilleval" / LOOSE_BUNDLE_DIR / "root"+ assert (full / "skills/a/SKILL.md").is_file()+ assert not (loose / "skills").exists()+ _assert_equal(parts, tasks)+++def test_unexpected_file_in_generated_parent_still_breaks_resource_parity(tmp_path: Path) -> None:+ parts, tasks = _stage_relocated_member(tmp_path)+ _assert_equal(parts, tasks)+ full = tasks["with_skill"][0] / "environment/skilleval/native/claude-code/plugin"+ (full / "skills/unexpected.js").write_text("console.log('unexpected');\n", encoding="utf-8")++ report = parts.content_report_for_tasks(tasks)++ assert report["equal_across_arms"] is False, report+ assert "script:." in report["mismatches"]+++def test_authored_empty_parent_directory_mode_still_breaks_resource_parity(tmp_path: Path) -> None:+ parts, tasks = _stage_relocated_member(tmp_path, authored_skills_directory=True)+ _assert_equal(parts, tasks)+ loose = tasks["loose_parts"][0] / "environment/skilleval" / LOOSE_BUNDLE_DIR / "root"+ assert list((loose / "skills").iterdir()) == []+ (loose / "skills").chmod(0o700)++ report = parts.content_report_for_tasks(tasks)++ assert report["equal_across_arms"] is False, report+ assert "script:." in report["mismatches"]diff --git a/tests/tier3/test_private_loose_source.py b/tests/tier3/test_private_loose_source.pynew file mode 100644index 00000000..ad76ad0f--- /dev/null+++ b/tests/tier3/test_private_loose_source.py@@ -0,0 +1,901 @@+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+# SPDX-License-Identifier: Apache-2.0++"""The loose-parts arm on native Claude Code: staging, content hashes, and load evidence.++The loose arm stages the same skills, rules, MCP servers, subagents, commands+and hooks that Claude Code can load standalone, as plain user-level pieces+under ``$CLAUDE_CONFIG_DIR`` (and ``--mcp-config``), so that full plugin minus+loose parts measures only the plugin's glue. Every test uses fake plugin trees+and fake container layouts. Nothing starts Harbor, Docker, or an agent.+"""++from __future__ import annotations++import dataclasses+import json+import shlex+import shutil+import stat+import subprocess+import sys+from pathlib import Path+from typing import Any++import pytest++sys.path.insert(0, str(Path(__file__).resolve().parent))++from private_loose_fixtures import (+ AGENT_TEXT,+ COMMAND_TEXT,+ GUARD_SCRIPT,+ HOOKS,+ SERVER_SCRIPT,+ make_plugin,+ make_skills_only_plugin,+ native_source,+ write,+)++from skillevaluator.tier3.harbor.native_staging import (+ NativeTaskStaging,+ build_native_task_staging,+ stage_native_bundle,+)+from skillevaluator.tier3.loose_parts import (+ LOOSE_BUNDLE_DIR,+ LOOSE_DATA,+ LOOSE_MCP_CONFIG,+ LOOSE_PARTS_ARM,+ LOOSE_PARTS_LABEL,+ LOOSE_ROOT,+ LoosePartsClaudeCodeAdapter,+ LoosePartsUnsupported,+ build_loose_parts,+ compare_content_hashes,+ load_failures_by_case,+ loose_parts_agent_import_path,+ loose_parts_load_evidence,+ loose_parts_task_staging,+)+from skillevaluator.tier3.plugin_native import (+ CLAUDE_PLUGIN_ROOT_VAR,+ HOOK_CENSUS_SCRIPT,+ ClaudeCodeAdapter,+ PluginLoadError,+ normalize_census,+)++[email protected](autouse=True)+def _approve_the_fixture_plugin_mcp_token(monkeypatch: pytest.MonkeyPatch) -> None:+ """The fixture plugin's MCP server references ``${RELTOOLS_TOKEN}``: approve it, as a plugin user must."""+ monkeypatch.setenv("SKILLEVALUATOR_ALLOWED_MCP_SECRETS", "RELTOOLS_TOKEN")+++ROOT_DIR = f"skilleval/{LOOSE_BUNDLE_DIR}/root"+++def package(tmp_path: Path):+ return native_source(make_plugin(tmp_path / "plugins"), tmp_path / "stage")+++def source(package):+ return package.native_source+++def loose(source):+ parts = build_loose_parts(source)+ assert parts.error is None+ return parts+++def _tree(root: Path) -> dict[str, bytes]:+ return {+ path.relative_to(root).as_posix(): path.read_bytes()+ for path in sorted(root.rglob("*"))+ if path.is_file() and not path.is_symlink()+ }+++def _unwrap(handler: dict[str, Any]) -> str:+ """The original command a census-wrapped shell-form hook handler runs."""+ words = shlex.split(handler["command"])+ assert words[:2] == ["/bin/sh", HOOK_CENSUS_SCRIPT] and words[4] == "--", words+ return words[5]+++def _hook_ids(config: dict[str, Any]) -> list[str]:+ return [+ shlex.split(handler["command"])[2]+ for groups in config["hooks"].values()+ for group in groups+ for handler in group["hooks"]+ ]+++# --------------------------------------------------------------------------- #+# Names #+# --------------------------------------------------------------------------- #+def test_arm_names_are_the_final_names() -> None:+ assert LOOSE_PARTS_ARM == "loose_parts"+ assert LOOSE_PARTS_LABEL == "Loose parts"+ assert LOOSE_ROOT == "/skilleval/native/claude-code/loose/root"+ assert LOOSE_MCP_CONFIG == "/skilleval/native/claude-code/loose/mcp.json"+++# --------------------------------------------------------------------------- #+# What the loose bundle stages #+# --------------------------------------------------------------------------- #+def test_loose_bundle_stages_each_part_where_claude_code_loads_it_standalone(loose) -> None:+ bundle = loose.bundle+ assert bundle is not None+ # Resource files retain their root, but no plugin loader or manifest is installed.+ assert bundle.plugin_tree == f"{LOOSE_BUNDLE_DIR}/root"+ assert not any("plugin.json" in rel or "/plugin/" in rel for rel in bundle.generated)+ assert sorted(bundle.generated) == [+ f"{LOOSE_BUNDLE_DIR}/mcp.json",+ f"{LOOSE_BUNDLE_DIR}/settings.json",+ "native/claude-code/rules/release-files.md",+ "native/claude-code/rules/release-policy.md",+ "native/setup.sh",+ ]+ assert sorted(dest for _source, dest in bundle.files) == [+ f"{LOOSE_BUNDLE_DIR}/agents/release-reviewer.md",+ f"{LOOSE_BUNDLE_DIR}/commands/release-check.md",+ ]+ # Member skills go through the task skills projection, as in the skills-only arm.+ assert bundle.trees == []+ setup = bundle.generated["native/setup.sh"]+ assert '"$CLAUDE_CONFIG_DIR"/agents/release-reviewer.md' in setup+ assert '"$CLAUDE_CONFIG_DIR"/commands/release-check.md' in setup+ assert '"$CLAUDE_CONFIG_DIR"/settings.json' in setup+ assert f"mkdir -p {LOOSE_DATA}" in setup+ assert '"$CLAUDE_CONFIG_DIR/rules/skilleval-shipit/release-policy.md"' in setup+ assert "--plugin-dir" not in setup+ assert bundle.skill_aliases == []+ # The collector must not read this arm with the plugin-arm init check.+ assert bundle.harness["kind"] == "claude-code-loose-init"+++def test_rules_are_byte_identical_to_native_mode_and_on_request_rules_stay_out_of_both(loose, source) -> None:+ full = ClaudeCodeAdapter().build(source)+ for rel in ("native/claude-code/rules/release-policy.md", "native/claude-code/rules/release-files.md"):+ assert loose.bundle.generated[rel] == full.generated[rel]+ # The glob-scoped rule keeps its patterns as Claude Code paths frontmatter.+ assert loose.bundle.generated["native/claude-code/rules/release-files.md"].startswith(+ '---\npaths:\n - "RELEASE*.md"\n - "CHANGELOG.md"\n---\n'+ )+ assert loose.rules == ("release-files.md", "release-policy.md")+ assert not any("semver-help" in rel for rel in (*loose.bundle.generated, *full.generated))+ not_staged = {piece.id: piece.reason for piece in loose.not_staged}+ assert "agent-requested" in not_staged["rule:semver-help.mdc"]+++def test_agent_command_and_script_files_are_byte_and_mode_identical(tmp_path: Path, loose, source) -> None:+ env_dir = tmp_path / "loose" / "environment"+ loose.stage_arm(LOOSE_PARTS_ARM, env_dir)+ staged = env_dir / "skilleval" / Path(LOOSE_BUNDLE_DIR)+ plugin = source.plugin_root+ pairs = {+ "agents/release-reviewer.md": "agents/release-reviewer.md",+ "commands/release-check.md": "commands/release-check.md",+ "root/hooks/guard-push.sh": "hooks/guard-push.sh",+ "root/servers/reltools.js": "servers/reltools.js",+ }+ for staged_rel, source_rel in pairs.items():+ assert (staged / staged_rel).read_bytes() == (plugin / source_rel).read_bytes(), staged_rel+ assert stat.S_IMODE((staged / staged_rel).stat().st_mode) == stat.S_IMODE(+ (plugin / source_rel).stat().st_mode+ ), staged_rel+ assert (staged / "agents/release-reviewer.md").read_text(encoding="utf-8") == AGENT_TEXT+ assert (staged / "commands/release-check.md").read_text(encoding="utf-8") == COMMAND_TEXT+ assert (staged / "root/hooks/guard-push.sh").read_text(encoding="utf-8") == GUARD_SCRIPT+ assert (staged / "root/servers/reltools.js").read_text(encoding="utf-8") == SERVER_SCRIPT+ # Plain skills, as the skills-only arm stages them (the skill's evals/ never reaches the agent).+ skills = env_dir / "skills"+ assert sorted(path.name for path in skills.iterdir()) == ["changelog-collect", "version-bump"]+ assert (skills / "version-bump" / "scripts" / "bump.sh").read_bytes() == (+ plugin / "skills" / "version-bump" / "scripts" / "bump.sh"+ ).read_bytes()+ assert not (skills / "version-bump" / "evals").exists()+++def test_hooks_go_to_user_settings_with_only_the_root_variable_substituted(loose, source) -> None:+ full = json.loads(ClaudeCodeAdapter().build(source).generated["native/claude-code/plugin/hooks/hooks.json"])+ settings = json.loads(loose.bundle.generated[f"{LOOSE_BUNDLE_DIR}/settings.json"])+ # A user settings.json holds the hooks and nothing else (no plugin hooks.json description).+ assert list(settings) == ["hooks"]+ # The same census ids as the plugin arm, so hook runs join the same Tier 1 rows.+ assert _hook_ids(settings) == _hook_ids(full)+ loose_commands = [_unwrap(h) for groups in settings["hooks"].values() for g in groups for h in g["hooks"]]+ source_commands = [h["command"] for groups in HOOKS["hooks"].values() for g in groups for h in g["hooks"]]+ assert loose_commands == [command.replace(CLAUDE_PLUGIN_ROOT_VAR, LOOSE_ROOT) for command in source_commands]+ assert loose_commands[0] == f'sh "{LOOSE_ROOT}/hooks/guard-push.sh"'+ # Matchers and events are unchanged.+ assert {event: [g["matcher"] for g in groups] for event, groups in settings["hooks"].items()} == {+ event: [g["matcher"] for g in groups] for event, groups in full["hooks"].items()+ }+++def test_mcp_config_is_identical_except_the_root_variable(loose, source) -> None:+ full = json.loads(ClaudeCodeAdapter().build(source).generated["native/claude-code/plugin/.mcp.json"])+ staged = json.loads(loose.bundle.generated[f"{LOOSE_BUNDLE_DIR}/mcp.json"])+ assert list(staged) == ["mcpServers"]+ assert staged["mcpServers"]["reltools"] == {+ "type": "stdio",+ "command": "node",+ "args": [f"{LOOSE_ROOT}/servers/reltools.js", "--strict"],+ "env": {"RELTOOLS_MODE": "strict", "RELTOOLS_TOKEN": "${RELTOOLS_TOKEN}"},+ }+ expected = json.loads(json.dumps(full).replace(CLAUDE_PLUGIN_ROOT_VAR, LOOSE_ROOT))+ assert staged == expected+ assert loose.mcp_servers == ("reltools",)+++def test_support_tree_preserves_resources_and_excludes_loader_and_evaluator_data(tmp_path: Path, loose) -> None:+ env_dir = tmp_path / "loose" / "environment"+ loose.stage_arm(LOOSE_PARTS_ARM, env_dir)+ assert sorted(_tree(env_dir / "skilleval")) == [+ "hook_census.sh",+ "native/claude-code/loose/agents/release-reviewer.md",+ "native/claude-code/loose/commands/release-check.md",+ "native/claude-code/loose/mcp.json",+ "native/claude-code/loose/root/agents/release-reviewer.md",+ "native/claude-code/loose/root/bin/shipit-notify",+ "native/claude-code/loose/root/commands/release-check.md",+ "native/claude-code/loose/root/hooks/guard-push.sh",+ "native/claude-code/loose/root/hooks/hooks.json",+ "native/claude-code/loose/root/output-styles/terse.md",+ "native/claude-code/loose/root/rules/release-files.md",+ "native/claude-code/loose/root/rules/release-policy.md",+ "native/claude-code/loose/root/rules/semver-help.mdc",+ "native/claude-code/loose/root/servers/README.md",+ "native/claude-code/loose/root/servers/reltools.js",+ "native/claude-code/loose/root/skills/changelog-collect/SKILL.md",+ "native/claude-code/loose/root/skills/version-bump/SKILL.md",+ "native/claude-code/loose/root/skills/version-bump/scripts/bump.sh",+ "native/claude-code/loose/settings.json",+ "native/claude-code/rules/release-files.md",+ "native/claude-code/rules/release-policy.md",+ "native/setup.sh",+ ]+ assert sorted(_tree(env_dir / "skills")) == [+ "changelog-collect/SKILL.md",+ "version-bump/SKILL.md",+ "version-bump/scripts/bump.sh",+ ]+++def test_dropped_as_glue_lists_exactly_what_cannot_load_standalone(loose) -> None:+ dropped = {piece.id: piece.reason for piece in loose.dropped_as_glue}+ assert sorted(dropped) == [+ "bin:bin/",+ "lsp:go",+ "manifest:.claude-plugin/plugin.json",+ "output_style:terse",+ "settings:agent",+ "settings:subagentStatusLine",+ ]+ assert "PATH" in dropped["bin:bin/"]+ assert ".lsp.json" in dropped["lsp:go"]+ assert "namespac" in dropped["manifest:.claude-plugin/plugin.json"]+ assert all(reason.strip() for reason in dropped.values())+ # Every staged piece has a sha256 and the plugin-relative file it came from.+ pieces = {piece.id: piece for piece in loose.pieces}+ assert sorted(pieces) == [+ "agent:release-reviewer",+ "command:release-check",+ "hook:hooks/hooks.json",+ "mcp:reltools",+ "rule:release-files.md",+ "rule:release-policy.md",+ "script:.",+ "script:hooks/guard-push.sh",+ "script:servers/reltools.js",+ "skill:changelog-collect",+ "skill:version-bump",+ ]+ assert pieces["agent:release-reviewer"].source == "agents/release-reviewer.md"+ assert all(len(piece.sha256) == 64 for piece in pieces.values())+++def test_user_config_is_glue_too(tmp_path: Path) -> None:+ plugin = make_plugin(tmp_path / "plugins")+ manifest = json.loads((plugin / ".claude-plugin" / "plugin.json").read_text(encoding="utf-8"))+ manifest["userConfig"] = {"region": {"type": "string", "title": "Region", "default": "us"}}+ write(plugin / ".claude-plugin" / "plugin.json", json.dumps(manifest))+ parts = build_loose_parts(native_source(plugin, tmp_path / "stage").native_source)+ assert "userConfig" in {piece.id for piece in parts.dropped_as_glue} | {+ piece.id.split(":", 1)[0] for piece in parts.dropped_as_glue+ }+++# --------------------------------------------------------------------------- #+# Content hashes #+# --------------------------------------------------------------------------- #+def test_content_hashes_are_equal_across_arms_for_every_piece(loose) -> None:+ report = loose.content_report()+ hashes = report["content_hashes"]+ assert sorted(hashes) == ["loose_parts", "sum_of_parts", "with_skill"]+ assert report["equal_across_arms"] is True, report["mismatches"]+ assert report["mismatches"] == []+ for piece_id, digest in hashes["loose_parts"].items():+ assert hashes["with_skill"][piece_id] == digest, piece_id+ assert sorted(hashes["sum_of_parts"]) == ["skill:changelog-collect", "skill:version-bump"]+ for piece_id, digest in hashes["sum_of_parts"].items():+ assert hashes["loose_parts"][piece_id] == digest+ # Glue is the only content the full arm has beyond the loose arm.+ assert set(hashes["with_skill"]) == set(hashes["loose_parts"])+ assert loose.content_hashes_for_arm("none") == {}+ assert {piece.id: piece.sha256 for piece in loose.pieces} == hashes["loose_parts"]+++ ("staged_rel", "expected_mismatches"),+ [+ (f"skilleval/{LOOSE_BUNDLE_DIR}/agents/release-reviewer.md", ["agent:release-reviewer"]),+ (f"{ROOT_DIR}/hooks/guard-push.sh", ["script:.", "script:hooks/guard-push.sh"]),+ ("skills/version-bump/scripts/bump.sh", ["skill:version-bump"]),+ (f"skilleval/{LOOSE_BUNDLE_DIR}/mcp.json", ["mcp:reltools"]),+ ],+)+def test_a_changed_byte_makes_that_piece_a_mismatch(+ tmp_path: Path, loose, staged_rel: str, expected_mismatches+) -> None:+ env_dir = tmp_path / "loose" / "environment"+ loose.stage_arm(LOOSE_PARTS_ARM, env_dir)+ target = env_dir / staged_rel+ data = bytearray(target.read_bytes())+ marker = data.index(b"e")+ data[marker] = ord("E")+ target.write_bytes(bytes(data))++ hashes = {+ "with_skill": loose.content_hashes_for_arm("with_skill"),+ "loose_parts": loose.hash_staged_arm(LOOSE_PARTS_ARM, env_dir),+ "sum_of_parts": loose.content_hashes_for_arm("sum_of_parts"),+ }+ report = compare_content_hashes(hashes)++ assert report["equal_across_arms"] is False+ assert report["mismatches"] == expected_mismatches+++def test_a_piece_missing_from_the_loose_arm_is_a_mismatch(loose) -> None:+ hashes = {+ "with_skill": loose.content_hashes_for_arm("with_skill"),+ "loose_parts": {+ k: v for k, v in loose.content_hashes_for_arm("loose_parts").items() if k != "command:release-check"+ },+ "sum_of_parts": loose.content_hashes_for_arm("sum_of_parts"),+ }+ report = compare_content_hashes(hashes)+ assert report["equal_across_arms"] is False+ assert report["mismatches"] == ["command:release-check"]+++# --------------------------------------------------------------------------- #+# Errors and refusals #+# --------------------------------------------------------------------------- #+def test_a_plugin_with_nothing_loose_beyond_skills_gives_the_error(tmp_path: Path) -> None:+ package = native_source(make_skills_only_plugin(tmp_path / "plugins"), tmp_path / "stage")+ parts = build_loose_parts(package.native_source)+ assert parts.error is not None+ assert "loose parts equal the skills alone" in parts.error+ assert parts.bundle is None+ with pytest.raises(PluginLoadError, match="loose parts equal the skills alone"):+ loose_parts_task_staging("claude-code", package.native_source)++[email protected]("agent", ["codex", "opencode", "hermes", "cursor"])+def test_other_harnesses_are_not_supported_with_an_explicit_reason(source, agent: str) -> None:+ parts = build_loose_parts(source, agent=agent)+ assert parts.error is not None and parts.error.startswith("not supported:")+ assert "not measured" in parts.error+ with pytest.raises(LoosePartsUnsupported, match="not supported"):+ loose_parts_task_staging(agent, source)+++def test_wrapper_mode_is_not_supported_use_native(tmp_path: Path, source) -> None:+ parts = build_loose_parts(source, plugin_load="wrapper")+ assert parts.error is not None and parts.error.startswith("not supported: use native")+ with pytest.raises(LoosePartsUnsupported, match="not supported: use native"):+ loose_parts_task_staging("claude-code", source, plugin_load="wrapper")+++def test_staging_is_deterministic(tmp_path: Path, source) -> None:+ first, second = build_loose_parts(source), build_loose_parts(source)+ assert first.bundle.generated == second.bundle.generated+ assert first.bundle.files == second.bundle.files+ assert first.pieces == second.pieces+ assert first.dropped_as_glue == second.dropped_as_glue+ one, two = tmp_path / "one" / "environment", tmp_path / "two" / "environment"+ first.stage_arm(LOOSE_PARTS_ARM, one)+ second.stage_arm(LOOSE_PARTS_ARM, two)+ assert _tree(one) == _tree(two)+ assert first.hash_staged_arm(LOOSE_PARTS_ARM, one) == second.hash_staged_arm(LOOSE_PARTS_ARM, two)+++def test_a_reference_outside_the_plugin_root_is_never_copied(tmp_path: Path) -> None:+ plugin = make_plugin(tmp_path / "plugins")+ write(tmp_path / "plugins" / "outside.sh", "#!/bin/sh\necho secret\n")+ hooks = json.loads(json.dumps(HOOKS))+ hooks["hooks"]["PreToolUse"][0]["hooks"].append(+ {"type": "command", "command": 'sh "${CLAUDE_PLUGIN_ROOT}/../outside.sh"'}+ )+ write(plugin / "hooks" / "hooks.json", json.dumps(hooks))+ parts = build_loose_parts(native_source(plugin, tmp_path / "stage").native_source)+ assert parts.error is None+ env_dir = tmp_path / "loose" / "environment"+ parts.stage_arm(LOOSE_PARTS_ARM, env_dir)+ staged = _tree(env_dir)+ assert not any("outside" in rel for rel in staged)+ assert all(b"echo secret" not in data for data in staged.values())+ unresolved = {piece.id: piece.reason for piece in parts.not_staged}+ assert "script:../outside.sh" in unresolved+ assert "outside the plugin root" in unresolved["script:../outside.sh"]+++def test_a_symlinked_referenced_script_is_refused(tmp_path: Path) -> None:+ plugin = make_plugin(tmp_path / "plugins")+ # The Tier 3 inventory already refuses a linked plugin file; swap one in after the snapshot.+ source = native_source(plugin, tmp_path / "stage").native_source+ target = tmp_path / "plugins" / "elsewhere.sh"+ write(target, "#!/bin/sh\necho elsewhere\n")+ (plugin / "hooks" / "guard-push.sh").unlink()+ (plugin / "hooks" / "guard-push.sh").symlink_to(target)+ with pytest.raises(ValueError, match=r"(?i)refus"):+ build_loose_parts(source)+++def test_a_script_swapped_for_a_link_after_planning_is_never_copied(tmp_path: Path) -> None:+ plugin = make_plugin(tmp_path / "plugins")+ parts = build_loose_parts(native_source(plugin, tmp_path / "stage").native_source)+ target = tmp_path / "plugins" / "elsewhere.sh"+ write(target, "#!/bin/sh\necho elsewhere\n")+ (plugin / "servers" / "reltools.js").unlink()+ (plugin / "servers" / "reltools.js").symlink_to(target)+ env_dir = tmp_path / "loose" / "environment"+ with pytest.raises(ValueError):+ parts.stage_arm(LOOSE_PARTS_ARM, env_dir)+ assert all(b"elsewhere" not in data for data in _tree(env_dir).values())+++def test_a_bypass_hook_is_refused_like_the_plugin_arm(tmp_path: Path, source) -> None:+ # Native preparation already refuses such a plugin; the loose arm refuses the same snapshot on its own too.+ message = "Refusing to stage plugin hooks 'hooks/hooks.json' natively: permission-bypass flag"+ refused = dataclasses.replace(source, refusals=(("hook", "hooks/hooks.json", message),))+ with pytest.raises(PluginLoadError, match=r"(?i)bypass"):+ loose_parts_task_staging("claude-code", refused)+ parts = build_loose_parts(refused)+ assert parts.error == message and parts.bundle is None+ with pytest.raises(PluginLoadError, match=r"(?i)bypass"):+ build_native_task_staging("claude-code", LoosePartsClaudeCodeAdapter(), refused)+++# --------------------------------------------------------------------------- #+# The seam the runner calls #+# --------------------------------------------------------------------------- #+def test_task_staging_has_the_shape_native_staging_consumes(tmp_path: Path, source) -> None:+ staging = loose_parts_task_staging("claude-code", source)+ assert isinstance(staging, NativeTaskStaging)+ assert staging.stage_member_skills is True+ assert staging.stage_wrapper_skill is False+ assert staging.plugin_mcp_via_task is False+ assert staging.workspace_skill_aliases() == []+ env_dir = tmp_path / "task" / "environment"+ env_dir.mkdir(parents=True)+ assert stage_native_bundle(env_dir, staging) == ["COPY skilleval/ /skilleval/"]+ assert (env_dir / "skilleval" / LOOSE_BUNDLE_DIR / "mcp.json").is_file()+ plan = staging.bundle.census_plan()+ declared = {(row["type"], row["name"]) for row in plan["declared"]}+ assert {("agent", "release-reviewer"), ("command", "release-check"), ("mcp", "reltools")} <= declared+ assert {("hook", "hooks/hooks.json"), ("rule", "release-policy.md"), ("skill", "version-bump")} <= declared+++def _run_setup(bundle: Path, root: Path, env: dict[str, str]) -> dict[str, Any]:+ container = root / "skilleval"+ shutil.copytree(bundle, container)+ logs = root / "logs" / "agent"+ logs.mkdir(parents=True)+ script = (container / "native" / "setup.sh").read_text(encoding="utf-8")+ script = script.replace("/skilleval/", f"{container}/").replace("/logs/agent", str(logs))+ (root / "setup.sh").write_text(script, encoding="utf-8")+ completed = subprocess.run(+ ["/bin/sh", str(root / "setup.sh")],+ env={"PATH": "/usr/bin:/bin", **env},+ capture_output=True,+ text=True,+ check=False,+ timeout=30,+ )+ assert completed.returncode == 0, completed.stderr+ census = normalize_census(json.loads((logs / "skilleval-load-census.json").read_text(encoding="utf-8")))+ assert census is not None+ return census+++def test_setup_puts_every_part_under_claude_config_dir_and_lists_it(tmp_path: Path, loose) -> None:+ env_dir = tmp_path / "loose" / "environment"+ loose.stage_arm(LOOSE_PARTS_ARM, env_dir)+ root = tmp_path / "container"+ config_dir = root / "logs-agent-sessions"+ # Harbor copies the task skills into $CLAUDE_CONFIG_DIR/skills before the launch.+ shutil.copytree(env_dir / "skills", config_dir / "skills")+ census = _run_setup(env_dir / "skilleval", root, {"HOME": str(root / "home"), "CLAUDE_CONFIG_DIR": str(config_dir)})+ assert (config_dir / "agents" / "release-reviewer.md").read_text(encoding="utf-8") == AGENT_TEXT+ assert (config_dir / "commands" / "release-check.md").read_text(encoding="utf-8") == COMMAND_TEXT+ assert list(json.loads((config_dir / "settings.json").read_text(encoding="utf-8"))) == ["hooks"]+ assert (config_dir / "rules" / "skilleval-shipit" / "release-policy.md").is_file()+ assert census["loaded"] == []+ listed = {(row["type"], row["name"]) for row in census["listed"]}+ assert listed == {+ ("skill", "changelog-collect"),+ ("skill", "version-bump"),+ ("rule", "release-policy.md"),+ ("rule", "release-files.md"),+ ("agent", "release-reviewer"),+ ("command", "release-check"),+ ("hook", "hooks/hooks.json"),+ ("mcp", "reltools"),+ }+ assert census["not_loaded"] == [] or all(row["type"] == "rule" for row in census["not_loaded"])+++def test_setup_never_overwrites_an_existing_user_settings_file(tmp_path: Path, loose) -> None:+ env_dir = tmp_path / "loose" / "environment"+ loose.stage_arm(LOOSE_PARTS_ARM, env_dir)+ root = tmp_path / "container"+ config_dir = root / "sessions"+ write(config_dir / "settings.json", '{"model": "kept"}')+ census = _run_setup(env_dir / "skilleval", root, {"HOME": str(root / "home"), "CLAUDE_CONFIG_DIR": str(config_dir)})+ assert json.loads((config_dir / "settings.json").read_text(encoding="utf-8")) == {"model": "kept"}+ assert "settings.json" in census.get("setup_errors", "")+ assert ("hook", "hooks/hooks.json") in {(row["type"], row["name"]) for row in census["not_loaded"]}+++def test_loose_agent_import_paths() -> None:+ assert loose_parts_agent_import_path("claude-code", None) == (+ "skillevaluator.tier3.harbor.loose_agents:LoosePartsClaudeCode"+ )+ assert loose_parts_agent_import_path(+ "claude-code", "skillevaluator.tier3.harbor.local_agents:SkillEvaluatorNvidiaBuildClaudeCode"+ ) == ("skillevaluator.tier3.harbor.loose_agents:LoosePartsNvidiaBuildClaudeCode")+ with pytest.raises(LoosePartsUnsupported, match="not supported"):+ loose_parts_agent_import_path("codex", None)+++def test_info_block_has_the_final_json_field_names(loose) -> None:+ info = loose.info()+ assert {"content_hashes", "equal_across_arms", "mismatches", "dropped_as_glue", "load_evidence"} <= set(info)+ assert info["arm"] == "loose_parts"+ assert info["label"] == "Loose parts"+ assert info["dropped_as_glue"][0].keys() == {"id", "reason"}+ assert {"id", "kind", "source", "sha256"} == set(info["pieces"][0])+ assert info["load_evidence"] == {}+++# --------------------------------------------------------------------------- #+# Load evidence (R4) #+# --------------------------------------------------------------------------- #+def _init(**overrides: Any) -> dict[str, Any]:+ event: dict[str, Any] = {+ "type": "system",+ "subtype": "init",+ "skills": ["update-config", "changelog-collect", "version-bump"],+ "slash_commands": ["clear", "changelog-collect", "version-bump", "release-check"],+ "agents": ["Explore", "general-purpose", "release-reviewer"],+ "mcp_servers": [{"name": "reltools", "status": "connected"}],+ "tools": ["Bash", "Read", "mcp__reltools__list_changes", "mcp__reltools__compute_version"],+ "plugins": [],+ }+ event.update(overrides)+ return event+++def _trajectory(*calls: tuple[str, dict[str, Any], Any]) -> dict[str, Any]:+ steps: list[dict[str, Any]] = [{"step_id": 1, "source": "user", "message": "Prepare the release."}]+ for index, (function, arguments, content) in enumerate(calls, start=1):+ result: dict[str, Any] = {"source_call_id": f"c{index}", "content": content}+ steps.append(+ {+ "step_id": index + 1,+ "source": "agent",+ "message": "",+ "tool_calls": [{"tool_call_id": f"c{index}", "function_name": function, "arguments": arguments}],+ "observation": {"results": [result]},+ }+ )+ return {"schema_version": "ATIF-v1.7", "agent": {"name": "claude-code", "version": "2.1"}, "steps": steps}+++def test_init_marks_the_parts_it_reports_loaded(loose) -> None:+ evidence = loose_parts_load_evidence([_init()], None, loose)+ for piece_id in (+ "skill:changelog-collect",+ "skill:version-bump",+ "agent:release-reviewer",+ "command:release-check",+ "mcp:reltools",+ ):+ assert evidence[piece_id]["loaded"] is True, piece_id+ assert evidence[piece_id]["source"] == "system/init", piece_id+ assert evidence["mcp:reltools"]["tools"] == ["compute_version", "list_changes"]+ # Rules, hooks and scripts are not in Claude Code's startup report: never loaded, never a failure.+ for piece_id in ("rule:release-policy.md", "hook:hooks/hooks.json", "script:hooks/guard-push.sh"):+ assert evidence[piece_id]["loaded"] is None, piece_id+ assert evidence[piece_id]["source"] == "unverifiable", piece_id+ assert load_failures_by_case({LOOSE_PARTS_ARM: {"case-1": [evidence]}}) == {}+++def test_a_pending_mcp_server_with_a_successful_call_is_loaded_by_call_evidence(loose) -> None:+ init = _init(mcp_servers=[{"name": "reltools", "status": "pending"}], tools=["Bash", "Read"])+ trajectory = _trajectory(("mcp__reltools__list_changes", {"since": "v1.2.3"}, "3 changes: RK-1, RK-2, RK-3"))++ evidence = loose_parts_load_evidence([init], trajectory, loose)++ assert evidence["mcp:reltools"]["loaded"] is True+ assert evidence["mcp:reltools"]["source"] == "mcp-call"+ assert "pending" in evidence["mcp:reltools"]["evidence"]+ assert "mcp__reltools__list_changes" in evidence["mcp:reltools"]["evidence"]+++def test_a_pending_mcp_server_whose_calls_all_failed_is_not_loaded(loose) -> None:+ init = _init(mcp_servers=[{"name": "reltools", "status": "pending"}])+ trajectory = _trajectory(("mcp__reltools__list_changes", {}, "Error: No such tool available: list_changes"))++ evidence = loose_parts_load_evidence([init], trajectory, loose)++ assert evidence["mcp:reltools"]["loaded"] is False+ assert "pending" in evidence["mcp:reltools"]["evidence"]+++def test_a_part_loaded_nowhere_is_a_load_failure_that_drops_the_case_from_every_arm(loose) -> None:+ init = _init(agents=["Explore"], mcp_servers=[{"name": "reltools", "status": "failed"}])+ evidence = loose_parts_load_evidence([init], _trajectory(), loose)+ assert evidence["agent:release-reviewer"]["loaded"] is False+ assert evidence["mcp:reltools"]["loaded"] is False+ assert "failed" in evidence["mcp:reltools"]["evidence"]+ ok = loose_parts_load_evidence([_init()], None, loose)++ failures = load_failures_by_case(+ {LOOSE_PARTS_ARM: {"case-1": [evidence], "case-2": [ok]}, "with_skill": {"case-1": [], "case-2": []}}+ )++ assert sorted(failures) == ["case-1"]+ rows = failures["case-1"]+ assert {(row["arm"], row["piece"]) for row in rows} == {+ (LOOSE_PARTS_ARM, "agent:release-reviewer"),+ (LOOSE_PARTS_ARM, "mcp:reltools"),+ }+ assert all(row["reason"] and row["attempt"] == 1 for row in rows)+++def test_no_init_event_means_nothing_init_checks_is_loaded(loose) -> None:+ evidence = loose_parts_load_evidence([], None, loose)+ assert evidence["skill:version-bump"]["loaded"] is False+ assert "no system/init event" in evidence["skill:version-bump"]["evidence"]+++def test_the_full_plugin_arm_is_checked_under_its_namespaced_names(loose) -> None:+ init = _init(+ skills=["shipit:changelog-collect", "shipit:version-bump"],+ slash_commands=["shipit:release-check", "shipit:changelog-collect", "shipit:version-bump"],+ agents=["Explore", "shipit:release-reviewer"],+ mcp_servers=[{"name": "plugin:shipit:reltools", "status": "pending"}],+ tools=["Bash"],+ plugins=[{"name": "shipit", "path": "/skilleval/native/claude-code/plugin", "source": "shipit@inline"}],+ )+ trajectory = _trajectory(("mcp__plugin_shipit_reltools__list_changes", {}, "3 changes"))++ evidence = loose_parts_load_evidence([init], trajectory, loose, arm="with_skill")++ assert {piece: row["loaded"] for piece, row in evidence.items() if row["loaded"] is not None} == {+ "skill:changelog-collect": True,+ "skill:version-bump": True,+ "agent:release-reviewer": True,+ "command:release-check": True,+ "mcp:reltools": True,+ }+ assert evidence["mcp:reltools"]["source"] == "mcp-call"+ # The bare loose spelling does not count as the plugin arm's component.+ bare = loose_parts_load_evidence([_init()], None, loose, arm="with_skill")+ assert bare["agent:release-reviewer"]["loaded"] is False+++def test_the_skills_only_arm_checks_only_skills(loose) -> None:+ evidence = loose_parts_load_evidence([_init()], None, loose, arm="sum_of_parts")+ assert sorted(evidence) == ["skill:changelog-collect", "skill:version-bump"]+++# --------------------------------------------------------------------------- #+# Edge layouts #+# --------------------------------------------------------------------------- #+def test_frontmatter_hooks_census_wrapped_in_the_plugin_arm_still_hash_equal(tmp_path: Path) -> None:+ plugin = make_plugin(tmp_path / "plugins")+ write(+ plugin / "skills" / "version-bump" / "SKILL.md",+ "---\nname: version-bump\ndescription: Compute the next version.\nhooks:\n PreToolUse:\n"+ " - matcher: Bash\n hooks:\n - type: command\n command: echo bump\n---\n\nBump.\n",+ )+ parts = build_loose_parts(native_source(plugin, tmp_path / "stage").native_source)+ report = parts.content_report()+ assert report["equal_across_arms"] is True, report["mismatches"]+ assert "skill:version-bump" in report["normalized"]+ # The loose arm stages the skill exactly as the skills-only arm does (its own frontmatter hooks, unwrapped).+ env_dir = tmp_path / "loose" / "environment"+ parts.stage_arm(LOOSE_PARTS_ARM, env_dir)+ assert (env_dir / "skills" / "version-bump" / "SKILL.md").read_bytes() == (+ plugin / "skills" / "version-bump" / "SKILL.md"+ ).read_bytes()+++def test_a_referenced_directory_is_staged_with_the_plugin_copy_rules(tmp_path: Path) -> None:+ plugin = make_plugin(tmp_path / "plugins")+ mcp = {+ "mcpServers": {+ "reltools": {+ "command": "python3",+ "args": ["-m", "reltools"],+ "env": {"PYTHONPATH": "${CLAUDE_PLUGIN_ROOT}/servers"},+ }+ }+ }+ write(plugin / ".mcp.json", json.dumps(mcp))+ write(plugin / "servers" / "lib" / "util.py", "VALUE = 1\n")+ write(plugin / "servers" / ".env", "TOKEN=never-copied\n")+ parts = build_loose_parts(native_source(plugin, tmp_path / "stage").native_source)+ assert "script:servers" in {piece.id for piece in parts.pieces}+ env_dir = tmp_path / "loose" / "environment"+ parts.stage_arm(LOOSE_PARTS_ARM, env_dir)+ staged = sorted(_tree(env_dir / ROOT_DIR / "servers"))+ assert staged == ["README.md", "lib/util.py", "reltools.js"]+ report = parts.content_report()+ assert report["equal_across_arms"] is True, report["mismatches"]+++def test_a_reference_to_the_plugin_root_itself_copies_the_plugin_tree_like_the_plugin_arm(tmp_path: Path) -> None:+ plugin = make_plugin(tmp_path / "plugins")+ mcp = {+ "mcpServers": {"reltools": {"command": "uv", "args": ["run", "--directory", "${CLAUDE_PLUGIN_ROOT}", "srv"]}}+ }+ write(plugin / ".mcp.json", json.dumps(mcp))+ parts = build_loose_parts(native_source(plugin, tmp_path / "stage").native_source)+ assert parts.bundle.plugin_tree == f"{LOOSE_BUNDLE_DIR}/root"+ staged = json.loads(parts.bundle.generated[f"{LOOSE_BUNDLE_DIR}/mcp.json"])+ assert staged["mcpServers"]["reltools"]["args"] == ["run", "--directory", LOOSE_ROOT, "srv"]+ env_dir = tmp_path / "loose" / "environment"+ parts.stage_arm(LOOSE_PARTS_ARM, env_dir)+ root = _tree(env_dir / ROOT_DIR)+ assert "servers/reltools.js" in root and ".claude-plugin/plugin.json" not in root+ report = parts.content_report()+ assert report["equal_across_arms"] is True, report["mismatches"]+++def test_two_hook_sources_are_two_pieces(tmp_path: Path) -> None:+ plugin = make_plugin(tmp_path / "plugins")+ manifest = json.loads((plugin / ".claude-plugin" / "plugin.json").read_text(encoding="utf-8"))+ manifest["hooks"] = {"Stop": [{"hooks": [{"type": "command", "command": "echo stop"}]}]}+ write(plugin / ".claude-plugin" / "plugin.json", json.dumps(manifest))+ parts = build_loose_parts(native_source(plugin, tmp_path / "stage").native_source)+ hooks = sorted(piece.id for piece in parts.pieces if piece.kind == "hook")+ assert len(hooks) == 2 and "hook:hooks/hooks.json" in hooks+ report = parts.content_report()+ assert report["equal_across_arms"] is True, report["mismatches"]+ assert set(report["content_hashes"]["with_skill"]) >= set(hooks)+++def test_plugin_data_variable_points_at_a_fresh_loose_data_dir(tmp_path: Path) -> None:+ plugin = make_plugin(tmp_path / "plugins")+ hooks = json.loads(json.dumps(HOOKS))+ hooks["hooks"]["PostToolUse"][0]["hooks"][0]["command"] = 'echo wrote >> "${CLAUDE_PLUGIN_DATA}/writes.log"'+ write(plugin / "hooks" / "hooks.json", json.dumps(hooks))+ parts = build_loose_parts(native_source(plugin, tmp_path / "stage").native_source)+ settings = parts.bundle.generated[f"{LOOSE_BUNDLE_DIR}/settings.json"]+ assert "${CLAUDE_PLUGIN_DATA}" not in settings and LOOSE_DATA in settings+ assert f"mkdir -p {LOOSE_DATA}" in parts.bundle.generated["native/setup.sh"]+ assert parts.content_report()["equal_across_arms"] is True+++def test_a_file_a_skills_frontmatter_hook_names_is_copied_to_the_loose_root(tmp_path: Path) -> None:+ plugin = make_plugin(tmp_path / "plugins")+ write(+ plugin / "skills" / "version-bump" / "SKILL.md",+ "---\nname: version-bump\ndescription: Compute the next version.\nhooks:\n PostToolUse:\n"+ " - matcher: Write\n hooks:\n - type: command\n"+ ' command: sh "${CLAUDE_PLUGIN_ROOT}/skills/version-bump/scripts/bump.sh"\n---\n\nBump.\n',+ )+ parts = build_loose_parts(native_source(plugin, tmp_path / "stage").native_source)+ assert "script:skills/version-bump/scripts/bump.sh" in {piece.id for piece in parts.pieces}+ env_dir = tmp_path / "loose" / "environment"+ parts.stage_arm(LOOSE_PARTS_ARM, env_dir)+ copied = env_dir / ROOT_DIR / "skills" / "version-bump" / "scripts" / "bump.sh"+ assert copied.read_bytes() == (plugin / "skills" / "version-bump" / "scripts" / "bump.sh").read_bytes()+ assert parts.content_report()["equal_across_arms"] is True+++def test_hook_groups_without_a_command_handler_are_hashed_too(tmp_path: Path) -> None:+ plugin = make_plugin(tmp_path / "plugins")+ hooks = json.loads(json.dumps(HOOKS))+ hooks["hooks"]["Stop"] = [{"hooks": [{"type": "prompt", "prompt": "Check ${CLAUDE_PLUGIN_ROOT}/RELEASE.md"}]}]+ write(plugin / "hooks" / "hooks.json", json.dumps(hooks))+ parts = build_loose_parts(native_source(plugin, tmp_path / "stage").native_source)+ pieces = {piece.id: piece.sha256 for piece in parts.pieces}+ assert pieces["hook:(groups without a command handler)"]+ report = parts.content_report()+ assert report["equal_across_arms"] is True, report["mismatches"]+ assert "hook:(groups without a command handler)" in report["content_hashes"]["with_skill"]+++def test_a_path_list_names_each_referenced_path(tmp_path: Path) -> None:+ plugin = make_plugin(tmp_path / "plugins")+ pythonpath = "${CLAUDE_PLUGIN_ROOT}/lib:${CLAUDE_PLUGIN_ROOT}/servers"+ write(+ plugin / ".mcp.json",+ json.dumps({"mcpServers": {"reltools": {"command": "python3", "env": {"PYTHONPATH": pythonpath}}}}),+ )+ write(plugin / "lib" / "helper.py", "X = 1\n")+ parts = build_loose_parts(native_source(plugin, tmp_path / "stage").native_source)+ assert {"script:lib", "script:servers"} <= {piece.id for piece in parts.pieces}+ staged = json.loads(parts.bundle.generated[f"{LOOSE_BUNDLE_DIR}/mcp.json"])+ assert staged["mcpServers"]["reltools"]["env"]["PYTHONPATH"] == f"{LOOSE_ROOT}/lib:{LOOSE_ROOT}/servers"+++def test_an_unsafe_agent_file_name_cannot_inject_into_setup(tmp_path: Path) -> None:+ plugin = make_plugin(tmp_path / "plugins")+ (plugin / "agents" / "release-reviewer.md").rename(plugin / "agents" / 'rev"$(touch pwned)".md')+ parts = build_loose_parts(native_source(plugin, tmp_path / "stage").native_source)+ setup = parts.bundle.generated["native/setup.sh"]+ assert "$(touch" not in setup+ env_dir = tmp_path / "loose" / "environment"+ parts.stage_arm(LOOSE_PARTS_ARM, env_dir)+ root = tmp_path / "container"+ config_dir = root / "sessions"+ census = _run_setup(env_dir / "skilleval", root, {"HOME": str(root / "home"), "CLAUDE_CONFIG_DIR": str(config_dir)})+ assert not list(tmp_path.rglob("pwned"))+ staged = [path.name for path in (config_dir / "agents").iterdir()]+ assert len(staged) == 1 and "$" not in staged[0] and '"' not in staged[0]+ assert ("agent", "release-reviewer") in {(row["type"], row["name"]) for row in census["listed"]}+++def test_the_bare_root_variable_is_left_for_the_shell_and_its_file_is_still_copied(tmp_path: Path) -> None:+ plugin = make_plugin(tmp_path / "plugins")+ hooks = json.loads(json.dumps(HOOKS))+ hooks["hooks"]["PreToolUse"][0]["hooks"][0]["command"] = "sh $CLAUDE_PLUGIN_ROOT/hooks/guard-push.sh"+ write(plugin / "hooks" / "hooks.json", json.dumps(hooks))+ parts = build_loose_parts(native_source(plugin, tmp_path / "stage").native_source)+ settings = json.loads(parts.bundle.generated[f"{LOOSE_BUNDLE_DIR}/settings.json"])+ assert _unwrap(settings["hooks"]["PreToolUse"][0]["hooks"][0]) == "sh $CLAUDE_PLUGIN_ROOT/hooks/guard-push.sh"+ assert "script:hooks/guard-push.sh" in {piece.id for piece in parts.pieces}+ assert parts.content_report()["equal_across_arms"] is True+++def test_a_file_a_command_body_runs_is_copied(tmp_path: Path) -> None:+ plugin = make_plugin(tmp_path / "plugins")+ write(plugin / "commands" / "release-check.md", COMMAND_TEXT + "\n!`sh ${CLAUDE_PLUGIN_ROOT}/scripts/check.sh`\n")+ write(plugin / "scripts" / "check.sh", "#!/bin/sh\necho ok\n", mode=0o755)+ parts = build_loose_parts(native_source(plugin, tmp_path / "stage").native_source)+ assert "script:scripts/check.sh" in {piece.id for piece in parts.pieces}+ report = parts.content_report()+ assert report["equal_across_arms"] is True, report["mismatches"]+++def test_info_pieces_use_the_hashes_of_the_report_they_sit_next_to(loose) -> None:+ report = loose.content_report()+ report["content_hashes"]["loose_parts"] = {+ **report["content_hashes"]["loose_parts"],+ "skill:version-bump": "f" * 64,+ }+ info = loose.info(content=report)+ pieces = {piece["id"]: piece["sha256"] for piece in info["pieces"]}+ assert pieces["skill:version-bump"] == "f" * 64+++def test_setup_sets_the_config_dir_default_once(loose) -> None:+ setup = loose.bundle.generated["native/setup.sh"]+ assert setup.count(': "${CLAUDE_CONFIG_DIR:=$HOME/.claude}"') == 1diff --git a/tests/tier3/test_private_recovered_timeout_review2.py b/tests/tier3/test_private_recovered_timeout_review2.pynew file mode 100644index 00000000..783a0c71--- /dev/null+++ b/tests/tier3/test_private_recovered_timeout_review2.py@@ -0,0 +1,91 @@+# SPDX-License-Identifier: Apache-2.0+"""Private statistics honor the collector's verified physical timeout recovery."""++from __future__ import annotations++import json+from types import SimpleNamespace++import pytest++from skillevaluator.tier3 import plugin_evidence as evidence+from skillevaluator.tier3.plugin_comparisons import ComparisonOptions, ComparisonRuntime+from skillevaluator.tier3.plugin_trace_artifacts import TrialTraceArtifacts+from test_harbor_timeout_recovery import collect, native_attempt+++def _attempt(root, ordinal, *, timeout):+ job = native_attempt(root, ordinal, timeout=timeout)+ trial = next(path for path in job.iterdir() if path.is_dir())+ payload = json.loads((trial / "result.json").read_text())+ payload["agent_result"] = {"n_input_tokens": 10, "n_cache_tokens": 2, "n_output_tokens": 3, "cost_usd": 0.25}+ (trial / "result.json").write_text(json.dumps(payload))+ aggregate = json.loads((job / "result.json").read_text())+ aggregate["trial_results"] = [payload]+ (job / "result.json").write_text(json.dumps(aggregate))+ return job+++def _runtime(root, result):+ job = root / "jobs" / "demo-opencode-with"+ observer = evidence.RuntimeEvidenceObserver.__new__(evidence.RuntimeEvidenceObserver)+ observer._jobs_dir = job.parent+ observer.logical_trials, observer._rewards, observer.summaries = {}, {}, {}+ observer.selector_map = {"case-a": "case-a"}+ observer.hooks = SimpleNamespace(trial_censuses={}, trial_init_events={})+ observer.artifacts = TrialTraceArtifacts()+ observer.context = SimpleNamespace(case_spec=lambda _: {}, arm_enabled=lambda _: False, mcp_input_schemas={})+ observer.collect_arm("opencode", "with_skill", job)+ runtime = ComparisonRuntime(+ SimpleNamespace(observer=observer, decisions={}),+ SimpleNamespace(native_source=SimpleNamespace(member_skills=())),+ ComparisonOptions(member_skills=False),+ ["opencode"],+ baseline=False,+ )+ runtime.expected_case_ids = ("case-a",)+ runtime.n_attempts, runtime.stop_on_pass = 3, True+ runtime.jobs = {"opencode": {"with_skill": job}}+ runtime._collect_statistics(result, job.parent)+ return runtime.data["agents"]["opencode"], observer.logical_trials["opencode"]["with_skill"]+++def test_verified_recovery_keeps_successful_statistics_and_all_physical_usage(tmp_path):+ failed = _attempt(tmp_path / "attempts", 1, timeout=True)+ passed = _attempt(tmp_path / "attempts", 2, timeout=False)+ result = collect(tmp_path, failed, passed)+ assert result["agents"]["opencode"]["conditions"]["with_skill"]["recovered_attempts_total"] == 1+ stats, records = _runtime(tmp_path, result)+ assert stats["conditions"]["with_skill"]["execution_status"] == "succeeded"+ assert stats["conditions"]["with_skill"]["expected_attempts"] == 1+ assert stats["conditions"]["with_skill"]["physical_expected_attempts"] == 2+ assert stats["reliability"]["with_skill"]["pass_at_k"] == 1+ assert stats["cost"]["with_skill"] == {+ "total_tokens": 22,+ "tokens_per_success": 22,+ "total_usd": 0.5,+ "usd_per_success": 0.5,+ "successes": 1,+ "observations_complete": True,+ }+ assert records[0].reward is None++[email protected]("mutation", ["checksum", "exception"])+def test_unverified_recovery_stays_failed(tmp_path, mutation):+ failed = _attempt(tmp_path / "attempts", 1, timeout=True)+ passed = _attempt(tmp_path / "attempts", 2, timeout=False)+ trial = next(path for path in failed.iterdir() if path.is_dir())+ payload = json.loads((trial / "result.json").read_text())+ if mutation == "checksum":+ payload["task_checksum"] = "different"+ else:+ payload["exception_info"]["exception_type"] = "AgentSetupError"+ (trial / "result.json").write_text(json.dumps(payload))+ aggregate = json.loads((failed / "result.json").read_text())+ aggregate["trial_results"] = [payload]+ (failed / "result.json").write_text(json.dumps(aggregate))+ result = collect(tmp_path, failed, passed)+ stats, _records = _runtime(tmp_path, result)+ assert stats["conditions"]["with_skill"]["execution_status"] == "failed"+ assert "with_skill" not in stats["reliability"]diff --git a/tests/tier3/test_private_usage_recovery_review2.py b/tests/tier3/test_private_usage_recovery_review2.pynew file mode 100644index 00000000..9ba328c2--- /dev/null+++ b/tests/tier3/test_private_usage_recovery_review2.py@@ -0,0 +1,119 @@+# SPDX-License-Identifier: Apache-2.0+"""Recorded Harbor usage survives missing ATIF without inventing observations."""++from __future__ import annotations++import json+from types import SimpleNamespace++import pytest++from skillevaluator.tier3 import plugin_evidence as evidence+from skillevaluator.tier3.plugin_trace_artifacts import TrialTrace, TrialTraceArtifacts+++def _observer(job, *, final=None):+ observer = evidence.RuntimeEvidenceObserver.__new__(evidence.RuntimeEvidenceObserver)+ observer._jobs_dir = job.parent+ observer.logical_trials, observer._rewards, observer.summaries = {}, {job: ([], [])}, {}+ observer.selector_map = {"selected-task": "logical-case"}+ observer.hooks = SimpleNamespace(trial_censuses={}, trial_init_events={})+ observer.artifacts = TrialTraceArtifacts()+ observer.context = SimpleNamespace(case_spec=lambda _: {}, arm_enabled=lambda _: False, mcp_input_schemas={})+ if final is not None:+ observer.artifacts._traces[next(job.iterdir())] = TrialTrace({"final_metrics": final, "steps": []}, {})+ return observer+++def _trial(tmp_path, *, agent_result=None, steps=None, failed=False):+ job = tmp_path / "jobs" / "demo-codex-with"+ root = job / "demo-codex-with-selected-task-attempt001__trial"+ root.mkdir(parents=True)+ payload = {+ "trial_uri": root.as_uri(),+ "task_checksum": "fixture",+ "agent_info": {"name": "codex"},+ "config": {"task": {"path": "/tasks/selected-task"}},+ "agent_result": agent_result,+ "step_results": steps,+ "exception_info": {"exception_type": "AgentTimeoutError"} if failed else None,+ }+ (root / "result.json").write_text(json.dumps(payload))+ return job, root++[email protected]("failed", [False, True])+def test_harbor_usage_and_physical_ordinal_are_kept_without_a_score(tmp_path, failed):+ job, _root = _trial(+ tmp_path,+ agent_result={"n_input_tokens": 10, "n_cache_tokens": 2, "n_output_tokens": 3, "cost_usd": 0.25},+ failed=failed,+ )+ record = _observer(job).collect_arm("codex", "with_skill", job)[0]+ assert record.usage == {"prompt_tokens": 10, "cached_tokens": 2, "completion_tokens": 3, "cost_usd": 0.25}+ assert record.reward is None and record.case_id == "logical-case"+ assert record.attempt_ordinal == 1+ assert record.first_turn_usage is None+++def test_fallback_fills_only_missing_finite_fields_and_preserves_zero(tmp_path):+ job, _root = _trial(+ tmp_path, agent_result={"n_input_tokens": 10, "n_cache_tokens": 2, "n_output_tokens": 3, "cost_usd": 0.25}+ )+ record = _observer(+ job, final={"total_prompt_tokens": 0, "total_completion_tokens": None, "total_cost_usd": 0}+ ).collect_arm("codex", "with_skill", job)[0]+ assert record.usage == {+ "total_prompt_tokens": 0,+ "total_completion_tokens": None,+ "total_cost_usd": 0,+ "completion_tokens": 3,+ "cached_tokens": 2,+ }+++def test_multistep_fallback_uses_each_complete_step_once_and_never_root_mirror(tmp_path):+ job, _root = _trial(+ tmp_path,+ agent_result={"n_input_tokens": 100, "n_output_tokens": 100, "cost_usd": 100},+ steps=[+ {"step_name": "first", "agent_result": {"n_input_tokens": 10, "n_output_tokens": 2, "cost_usd": 0}},+ {"step_name": "second", "agent_result": {"n_input_tokens": 20, "n_output_tokens": 3, "cost_usd": 0.25}},+ ],+ )+ record = _observer(job).collect_arm("codex", "with_skill", job)[0]+ assert record.usage == {"prompt_tokens": 30, "completion_tokens": 5, "cost_usd": 0.25}+++def test_partial_or_invalid_step_counters_do_not_become_totals(tmp_path):+ job, _root = _trial(+ tmp_path,+ steps=[+ {"step_name": "first", "agent_result": {"n_input_tokens": 10, "n_output_tokens": 0, "cost_usd": 0.1}},+ {"step_name": "second", "agent_result": {"n_output_tokens": 0, "cost_usd": True}},+ ],+ )+ assert _observer(job).collect_arm("codex", "with_skill", job)[0].usage == {"completion_tokens": 0}+++def test_result_usage_reuses_the_identity_read_and_arm_cache(tmp_path, monkeypatch):+ job, root = _trial(tmp_path, agent_result={"n_input_tokens": 0, "n_output_tokens": 0, "cost_usd": 0})+ observer = _observer(job, final={})+ read = evidence.collector._read_json+ reads = []++ def counted(path):+ reads.append(path)+ return read(path)++ monkeypatch.setattr(evidence.collector, "_read_json", counted)+ first = observer.collect_arm("codex", "with_skill", job)+ assert first[0].usage == {"prompt_tokens": 0, "completion_tokens": 0, "cost_usd": 0}+ assert observer.collect_arm("codex", "with_skill", job) is first+ assert reads == [root / "result.json"]++[email protected]("value", [None, True, -1, float("inf"), "12"])+def test_invalid_harbor_counters_stay_unavailable(tmp_path, value):+ job, _root = _trial(tmp_path, agent_result={"n_input_tokens": value, "cost_usd": value})+ assert _observer(job).collect_arm("codex", "with_skill", job)[0].usage is Nonediff --git a/uv.lock b/uv.lockindex 39ed4d76..30b9a033 100644--- a/uv.lock+++ b/uv.lock@@ -2566,6 +2566,7 @@ dependencies = [{ name = "packaging" },{ name = "pydantic" },{ name = "pyyaml" },+ { name = "regex" },{ name = "rich" },]@@ -2652,6 +2653,7 @@ requires-dist = [{ name = "pytest-xdist", marker = "extra == 'dev'", specifier = ">=3.6.0" },{ name = "python-dotenv", marker = "extra == 'tier3'", specifier = ">=1.2.0" },{ name = "pyyaml", specifier = ">=6.0" },+ { name = "regex", specifier = ">=2024.11.6" },{ name = "rich", specifier = ">=13.5.0" },{ name = "ruff", marker = "extra == 'dev'", specifier = ">=0.15.4,<0.16" },{ name = "skillevaluator", extras = ["llm"], marker = "extra == 'tier2'" },