Skip to content

Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.

fix(tier3): redact persisted reward artifacts

Christopher Kevin3daa3fd2b79e
diff --git a/src/skillevaluator/tier3/harbor/collector.py b/src/skillevaluator/tier3/harbor/collector.py
index 9a3bc51c..13b8f03d 100644
--- a/src/skillevaluator/tier3/harbor/collector.py
+++ b/src/skillevaluator/tier3/harbor/collector.py
@@ -35,7 +35,7 @@ from skillevaluator.tier3.harbor.metrics import (
score_definition,
)
from skillevaluator.tier3.output_provenance import write_output_file_atomically
-from skillevaluator.utils.redaction import redact_sensitive_data, redact_sensitive_text
+from skillevaluator.utils.redaction import is_sensitive_key, redact_sensitive_data, redact_sensitive_text
from skillevaluator.utils.secure_fs import SecurePathError, SecureRoot, stat_is_link_or_reparse
logger = logging.getLogger(__name__)
@@ -2356,10 +2356,77 @@ def _add_trusted_invocation_evidence(
invoked = _trusted_trial_skill_invoked(trial_root, source_reward.get("_step_name"), skill_name)
if invoked is None:
return
- clean_reward["skill_invoked"] = invoked
+ if invoked is True:
+ clean_reward["skill_invoked"] = True
+ elif invoked is False:
+ clean_reward["skill_invoked"] = False
+ else:
+ return
clean_reward["invocation_evidence_source"] = "trajectory"
+def _can_restore_custom_metric_name(value: str) -> bool:
+ """Allow safe names plus narrow, explicitly metric-shaped secret terms."""
+ if not is_sensitive_key(value):
+ return True
+ camel_split = re.sub(r"([a-z0-9])([A-Z])", r"\1_\2", value)
+ normalized = re.sub(r"[^a-zA-Z0-9]+", "_", camel_split).strip("_").lower()
+ parts = tuple(part for part in normalized.split("_") if part)
+ return (
+ len(parts) == 2
+ and parts[0] in {"auth", "secret", "token"}
+ and parts[1]
+ in {
+ "accuracy",
+ "compliance",
+ "count",
+ "coverage",
+ "efficiency",
+ "handling",
+ "leakage",
+ "precision",
+ "quality",
+ "rate",
+ "ratio",
+ "recall",
+ "safety",
+ "score",
+ "usage",
+ }
+ )
+
+
+def _restore_custom_metric_scores(source_reward: dict[str, Any], safe_reward: dict[str, Any]) -> None:
+ """Restore only finite numeric values recognized by the custom-metric schema."""
+ for field in ("custom_metrics", "metrics"):
+ source_metrics = source_reward.get(field)
+ safe_metrics = safe_reward.get(field)
+ if not isinstance(source_metrics, dict) or not isinstance(safe_metrics, dict):
+ continue
+ for raw_name, raw_value in source_metrics.items():
+ name = str(raw_name)
+ if not _can_restore_custom_metric_name(name):
+ continue
+ score = extract_custom_metrics({field: {name: raw_value}}).get(name)
+ if score is None:
+ continue
+ if isinstance(raw_value, dict):
+ safe_value = safe_metrics.get(name)
+ if not isinstance(safe_value, dict):
+ safe_value = {}
+ safe_metrics[name] = safe_value
+ safe_value["score"] = score
+ else:
+ safe_metrics[name] = score
+
+ # Legacy custom-only rewards allowed this report metric at the top level.
+ token_efficiency = extract_custom_metrics({"token_efficiency": source_reward.get("token_efficiency")}).get(
+ "token_efficiency"
+ )
+ if token_efficiency is not None:
+ safe_reward["token_efficiency"] = token_efficiency
+
+
def _save_trials(
rewards: list[dict[str, Any]],
trials_dir: Path,
@@ -2409,11 +2476,11 @@ def _save_trials(
str(clean_reward.get("evaluation_status") or "").casefold() in {"error", "failed"}
or overall_score(clean_reward) is None
)
- safe_reward = (
- redact_sensitive_data(clean_reward, max_str_len=REWARD_DIAGNOSTIC_STRING_MAX_CHARS)
- if diagnostic_reward or "evaluation_errors" in clean_reward
- else clean_reward
+ safe_reward = redact_sensitive_data(
+ clean_reward,
+ max_str_len=REWARD_DIAGNOSTIC_STRING_MAX_CHARS if diagnostic_reward else None,
)
+ _restore_custom_metric_scores(clean_reward, safe_reward)
(trial_out / "reward.json").write_text(json.dumps(safe_reward, indent=2), encoding="utf-8")
if trial_src:
diff --git a/src/skillevaluator/utils/redaction.py b/src/skillevaluator/utils/redaction.py
index 3f79f78b..33611c11 100644
--- a/src/skillevaluator/utils/redaction.py
+++ b/src/skillevaluator/utils/redaction.py
@@ -90,7 +90,8 @@ def _normalized_key_parts(key: str) -> tuple[str, set[str]]:
return normalized, parts
-def _is_sensitive_key(key: str) -> bool:
+def is_sensitive_key(key: str) -> bool:
+ """Return whether a structured-data key conventionally carries a secret."""
normalized, parts = _normalized_key_parts(key)
if normalized in _TOKEN_COUNT_KEYS:
return False
@@ -106,7 +107,7 @@ def _is_sensitive_key(key: str) -> bool:
def _redact_sensitive_assignment(match: re.Match[str]) -> str:
key = match.group("key")
- if not _is_sensitive_key(key):
+ if not is_sensitive_key(key):
return match.group(0)
return f"{key}{match.group('sep')}<redacted>"
@@ -137,7 +138,7 @@ def redact_sensitive_text(value: str, *, max_len: int | None = None) -> str:
def redact_sensitive_data(value: Any, *, parent_key: str = "", max_str_len: int | None = None) -> Any:
"""Recursively redact structured data using secret-looking key names."""
- if _is_sensitive_key(parent_key):
+ if is_sensitive_key(parent_key):
return "<redacted>"
if isinstance(value, Mapping):
return {
diff --git a/tests/test_harbor_negative_control_evidence.py b/tests/test_harbor_negative_control_evidence.py
index 7a381dfc..54977e3e 100644
--- a/tests/test_harbor_negative_control_evidence.py
+++ b/tests/test_harbor_negative_control_evidence.py
@@ -12,7 +12,10 @@ from typing import Any
import pytest
-from skillevaluator.tier3.harbor.collector import collect_harbor_results
+from skillevaluator.tier3.harbor.collector import (
+ REWARD_DIAGNOSTIC_STRING_MAX_CHARS,
+ collect_harbor_results,
+)
STANDARD_REWARD = {
"security": 0.9,
@@ -145,6 +148,62 @@ def test_collect_persists_target_invocation_for_both_single_step_variants(tmp_pa
assert {path.name for path in trial_dir.iterdir()} == {"reward.json", "trajectory.json"}
+def test_collect_redacts_successful_standard_reward_without_diagnostic_truncation(tmp_path: Path) -> None:
+ jobs_dir = tmp_path / "jobs"
+ trial_name = "successful-redaction_attempt001"
+ long_note = "x" * (REWARD_DIAGNOSTIC_STRING_MAX_CHARS + 1)
+ reward = {
+ **STANDARD_REWARD,
+ "details": {
+ "api_key": "plain-text-test-secret",
+ "password": 123456789,
+ "message": "Authorization: Bearer secret-token-value",
+ "safe_note": long_note,
+ },
+ "custom_metrics": {
+ "secret_safety": 0.91,
+ "api_key": 135792468,
+ "api_token": 135792468,
+ "private_token": 135792468,
+ "client_token": 135792468,
+ },
+ "metrics": {
+ "password": 97531,
+ "github_token": 97531,
+ "gitlab_token": 97531,
+ "ssh_key": 97531,
+ "signing_key": 97531,
+ },
+ "evaluation_errors": {},
+ "api_key": 246813579,
+ }
+ _write_trial(
+ jobs_dir,
+ variant="with",
+ trial_name=trial_name,
+ reward=reward,
+ trajectory=_trajectory(skill="demo"),
+ )
+ _write_job_result(jobs_dir / "demo-opencode-with", [trial_name])
+
+ _collect(tmp_path)
+
+ persisted = _persisted_reward(tmp_path, "with", trial_name)
+ assert persisted["skill_invoked"] is True
+ assert persisted["invocation_evidence_source"] == "trajectory"
+ assert persisted["details"]["api_key"] == "<redacted>"
+ assert "secret-token-value" not in persisted["details"]["message"]
+ assert "<redacted>" in persisted["details"]["message"]
+ assert persisted["details"]["safe_note"] == long_note
+ assert persisted["details"]["password"] == "<redacted>"
+ assert persisted["custom_metrics"]["secret_safety"] == 0.91
+ for name in ("api_key", "api_token", "private_token", "client_token"):
+ assert persisted["custom_metrics"][name] == "<redacted>"
+ for name in ("password", "github_token", "gitlab_token", "ssh_key", "signing_key"):
+ assert persisted["metrics"][name] == "<redacted>"
+ assert persisted["api_key"] == "<redacted>"
+
+
def test_collect_persists_native_skill_tool_invocation(tmp_path: Path) -> None:
jobs_dir = tmp_path / "jobs"
trial_name = "native-skill_attempt001"