Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.
Python agent-evaluationagent-pluginsagent-securityagent-skillsagentic-aibenchmarkci-templatesclaude-codecodexevaluateevaluationharnesspluginplugin-evaluationsecurity-scannerskill-evalskill-evaluationskill-evaluatorskill-gap-analysisskills
fix(tier3): redact persisted reward artifacts
Christopher Kevin3daa3fd2b79e
diff --git a/src/skillevaluator/tier3/harbor/collector.py b/src/skillevaluator/tier3/harbor/collector.pyindex 9a3bc51c..13b8f03d 100644--- a/src/skillevaluator/tier3/harbor/collector.py+++ b/src/skillevaluator/tier3/harbor/collector.py@@ -35,7 +35,7 @@ from skillevaluator.tier3.harbor.metrics import (score_definition,)from skillevaluator.tier3.output_provenance import write_output_file_atomically-from skillevaluator.utils.redaction import redact_sensitive_data, redact_sensitive_text+from skillevaluator.utils.redaction import is_sensitive_key, redact_sensitive_data, redact_sensitive_textfrom skillevaluator.utils.secure_fs import SecurePathError, SecureRoot, stat_is_link_or_reparselogger = logging.getLogger(__name__)@@ -2356,10 +2356,77 @@ def _add_trusted_invocation_evidence(invoked = _trusted_trial_skill_invoked(trial_root, source_reward.get("_step_name"), skill_name)if invoked is None:return- clean_reward["skill_invoked"] = invoked+ if invoked is True:+ clean_reward["skill_invoked"] = True+ elif invoked is False:+ clean_reward["skill_invoked"] = False+ else:+ returnclean_reward["invocation_evidence_source"] = "trajectory"+def _can_restore_custom_metric_name(value: str) -> bool:+ """Allow safe names plus narrow, explicitly metric-shaped secret terms."""+ if not is_sensitive_key(value):+ return True+ camel_split = re.sub(r"([a-z0-9])([A-Z])", r"\1_\2", value)+ normalized = re.sub(r"[^a-zA-Z0-9]+", "_", camel_split).strip("_").lower()+ parts = tuple(part for part in normalized.split("_") if part)+ return (+ len(parts) == 2+ and parts[0] in {"auth", "secret", "token"}+ and parts[1]+ in {+ "accuracy",+ "compliance",+ "count",+ "coverage",+ "efficiency",+ "handling",+ "leakage",+ "precision",+ "quality",+ "rate",+ "ratio",+ "recall",+ "safety",+ "score",+ "usage",+ }+ )+++def _restore_custom_metric_scores(source_reward: dict[str, Any], safe_reward: dict[str, Any]) -> None:+ """Restore only finite numeric values recognized by the custom-metric schema."""+ for field in ("custom_metrics", "metrics"):+ source_metrics = source_reward.get(field)+ safe_metrics = safe_reward.get(field)+ if not isinstance(source_metrics, dict) or not isinstance(safe_metrics, dict):+ continue+ for raw_name, raw_value in source_metrics.items():+ name = str(raw_name)+ if not _can_restore_custom_metric_name(name):+ continue+ score = extract_custom_metrics({field: {name: raw_value}}).get(name)+ if score is None:+ continue+ if isinstance(raw_value, dict):+ safe_value = safe_metrics.get(name)+ if not isinstance(safe_value, dict):+ safe_value = {}+ safe_metrics[name] = safe_value+ safe_value["score"] = score+ else:+ safe_metrics[name] = score++ # Legacy custom-only rewards allowed this report metric at the top level.+ token_efficiency = extract_custom_metrics({"token_efficiency": source_reward.get("token_efficiency")}).get(+ "token_efficiency"+ )+ if token_efficiency is not None:+ safe_reward["token_efficiency"] = token_efficiency++def _save_trials(rewards: list[dict[str, Any]],trials_dir: Path,@@ -2409,11 +2476,11 @@ def _save_trials(str(clean_reward.get("evaluation_status") or "").casefold() in {"error", "failed"}or overall_score(clean_reward) is None)- safe_reward = (- redact_sensitive_data(clean_reward, max_str_len=REWARD_DIAGNOSTIC_STRING_MAX_CHARS)- if diagnostic_reward or "evaluation_errors" in clean_reward- else clean_reward+ safe_reward = redact_sensitive_data(+ clean_reward,+ max_str_len=REWARD_DIAGNOSTIC_STRING_MAX_CHARS if diagnostic_reward else None,)+ _restore_custom_metric_scores(clean_reward, safe_reward)(trial_out / "reward.json").write_text(json.dumps(safe_reward, indent=2), encoding="utf-8")if trial_src:diff --git a/src/skillevaluator/utils/redaction.py b/src/skillevaluator/utils/redaction.pyindex 3f79f78b..33611c11 100644--- a/src/skillevaluator/utils/redaction.py+++ b/src/skillevaluator/utils/redaction.py@@ -90,7 +90,8 @@ def _normalized_key_parts(key: str) -> tuple[str, set[str]]:return normalized, parts-def _is_sensitive_key(key: str) -> bool:+def is_sensitive_key(key: str) -> bool:+ """Return whether a structured-data key conventionally carries a secret."""normalized, parts = _normalized_key_parts(key)if normalized in _TOKEN_COUNT_KEYS:return False@@ -106,7 +107,7 @@ def _is_sensitive_key(key: str) -> bool:def _redact_sensitive_assignment(match: re.Match[str]) -> str:key = match.group("key")- if not _is_sensitive_key(key):+ if not is_sensitive_key(key):return match.group(0)return f"{key}{match.group('sep')}<redacted>"@@ -137,7 +138,7 @@ def redact_sensitive_text(value: str, *, max_len: int | None = None) -> str:def redact_sensitive_data(value: Any, *, parent_key: str = "", max_str_len: int | None = None) -> Any:"""Recursively redact structured data using secret-looking key names."""- if _is_sensitive_key(parent_key):+ if is_sensitive_key(parent_key):return "<redacted>"if isinstance(value, Mapping):return {diff --git a/tests/test_harbor_negative_control_evidence.py b/tests/test_harbor_negative_control_evidence.pyindex 7a381dfc..54977e3e 100644--- a/tests/test_harbor_negative_control_evidence.py+++ b/tests/test_harbor_negative_control_evidence.py@@ -12,7 +12,10 @@ from typing import Anyimport pytest-from skillevaluator.tier3.harbor.collector import collect_harbor_results+from skillevaluator.tier3.harbor.collector import (+ REWARD_DIAGNOSTIC_STRING_MAX_CHARS,+ collect_harbor_results,+)STANDARD_REWARD = {"security": 0.9,@@ -145,6 +148,62 @@ def test_collect_persists_target_invocation_for_both_single_step_variants(tmp_paassert {path.name for path in trial_dir.iterdir()} == {"reward.json", "trajectory.json"}+def test_collect_redacts_successful_standard_reward_without_diagnostic_truncation(tmp_path: Path) -> None:+ jobs_dir = tmp_path / "jobs"+ trial_name = "successful-redaction_attempt001"+ long_note = "x" * (REWARD_DIAGNOSTIC_STRING_MAX_CHARS + 1)+ reward = {+ **STANDARD_REWARD,+ "details": {+ "api_key": "plain-text-test-secret",+ "password": 123456789,+ "message": "Authorization: Bearer secret-token-value",+ "safe_note": long_note,+ },+ "custom_metrics": {+ "secret_safety": 0.91,+ "api_key": 135792468,+ "api_token": 135792468,+ "private_token": 135792468,+ "client_token": 135792468,+ },+ "metrics": {+ "password": 97531,+ "github_token": 97531,+ "gitlab_token": 97531,+ "ssh_key": 97531,+ "signing_key": 97531,+ },+ "evaluation_errors": {},+ "api_key": 246813579,+ }+ _write_trial(+ jobs_dir,+ variant="with",+ trial_name=trial_name,+ reward=reward,+ trajectory=_trajectory(skill="demo"),+ )+ _write_job_result(jobs_dir / "demo-opencode-with", [trial_name])++ _collect(tmp_path)++ persisted = _persisted_reward(tmp_path, "with", trial_name)+ assert persisted["skill_invoked"] is True+ assert persisted["invocation_evidence_source"] == "trajectory"+ assert persisted["details"]["api_key"] == "<redacted>"+ assert "secret-token-value" not in persisted["details"]["message"]+ assert "<redacted>" in persisted["details"]["message"]+ assert persisted["details"]["safe_note"] == long_note+ assert persisted["details"]["password"] == "<redacted>"+ assert persisted["custom_metrics"]["secret_safety"] == 0.91+ for name in ("api_key", "api_token", "private_token", "client_token"):+ assert persisted["custom_metrics"][name] == "<redacted>"+ for name in ("password", "github_token", "gitlab_token", "ssh_key", "signing_key"):+ assert persisted["metrics"][name] == "<redacted>"+ assert persisted["api_key"] == "<redacted>"++def test_collect_persists_native_skill_tool_invocation(tmp_path: Path) -> None:jobs_dir = tmp_path / "jobs"trial_name = "native-skill_attempt001"