Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.
Python agent-evaluationagent-pluginsagent-securityagent-skillsagentic-aibenchmarkci-templatesclaude-codecodexevaluateevaluationharnesspluginplugin-evaluationsecurity-scannerskill-evalskill-evaluationskill-evaluatorskill-gap-analysisskills
fix(rubric): sanitize final rejection diagnostics (#484)
Narendran Raghavan90dfd5a72c51
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 5d6f270f..4bfedbcd 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -584,11 +584,10 @@ All notable changes to SkillEvaluator are documented in this file.asked (1, or 2 after a rejected reply, not counting the client's retries),in the skill's `rubric_eval` success detail or `llm_*` finding and, for asingle skill, also in the `rubric_eval` object. It is left out when the- first request fails, for example with no provider configured.--- Rubric retry diagnostics redact judge-supplied credentials before logging or truncating the rejection reason.--- Rubric retry guidance links to the published environment-variable documentation route.+ first request fails, for example with no provider configured. Rejection+ diagnostics redact judge-supplied credentials before shortening the reason,+ and collapse whitespace and controls to printable single-line text in logs,+ retry notes and final invalid-response findings and summaries.- A default Tier 3 run with a baseline can reach a PASS or NEUTRAL publicationverdict in BENCHMARK.md again. The baseline arm has no skill under test, sodiff --git a/src/skillevaluator/validators/rubric_eval.py b/src/skillevaluator/validators/rubric_eval.pyindex 88e6b619..b85ecb69 100644--- a/src/skillevaluator/validators/rubric_eval.py+++ b/src/skillevaluator/validators/rubric_eval.py@@ -39,6 +39,7 @@ from skillevaluator.inference.diagnostics import llm_failure_diagnostic, safe_llfrom skillevaluator.logging_config import get_loggerfrom skillevaluator.models.result import Finding, Severity, ValidationResultfrom skillevaluator.utils.redaction import redact_sensitive_text+from skillevaluator.utils.rich_markup import strip_terminal_controlsfrom skillevaluator.utils.secure_fs import SecureRootfrom skillevaluator.validators.base import ValidatorBasefrom skillevaluator.validators.markdown import markdown_link_targets, normalized_local_path@@ -212,12 +213,13 @@ def _rubric_report_problem(report: Any, criteria: list[dict]) -> str | None:def _bounded_problem(problem: str) -> str:- """Shorten a rejection reason for the log line and the retry note.+ """Make a rejection reason safe for logs, retry notes and failure reports.The reason quotes the judge's own text, such as an invented criterion id,which has no length limit."""- problem = redact_sensitive_text(problem)+ problem = strip_terminal_controls(redact_sensitive_text(problem))+ problem = " ".join("".join(char if char.isprintable() else " " for char in problem).split())if len(problem) <= _MAX_RETRY_PROBLEM_CHARS:return problemreturn f"{problem[:_MAX_RETRY_PROBLEM_CHARS]} ... [truncated]"@@ -796,7 +798,7 @@ class RubricEvalValidator(ValidatorBase):report=report,skill_path=skill_path,manifest=manifest,- message=validation_error,+ message=_bounded_problem(validation_error),check_name="llm_invalid_response",suggestion="Retry rubric evaluation; the judge response did not contain valid complete criterion evidence",judge_attempts=judge_attempts,diff --git a/tests/validators/test_rubric_eval.py b/tests/validators/test_rubric_eval.pyindex 83565aa4..4fe47d5d 100644--- a/tests/validators/test_rubric_eval.py+++ b/tests/validators/test_rubric_eval.py@@ -1174,6 +1174,43 @@ class TestInvalidReplyRetry:assert "sk-synthetic" not in requests[1]["user"].removeprefix(requests[0]["user"])assert result.passed+ @pytest.mark.parametrize("padding", ["", "x" * 410 + " "])+ def test_final_invalid_reply_diagnostics_redact_before_bounding(self, tmp_path, padding):+ checks = _complete_checks()+ checks.append({**checks[0], "id": padding + "sk-syntheticcredential123456789 " + "z" * 2000})++ result, requests = _validate_with(tmp_path, _reply(checks), _reply(checks))++ assert len(requests) == 2+ (finding,) = _llm_findings(result)+ summary = result.metadata["rubric_eval"]["summary"]+ assert finding.message == summary+ assert "sk-synthetic" not in summary+ assert len(summary) <= 500 + len(" ... [truncated]")+ assert summary.endswith(" ... [truncated]")+ assert not result.passed++ @pytest.mark.parametrize("control", ["\n", "\r", "\t", "\x00", "\x1b[31m", "\u2028", "\u202e"])+ def test_rejection_diagnostics_are_printable_single_lines(self, tmp_path, control, caplog, monkeypatch):+ checks = _complete_checks()+ checks.append({**checks[0], "id": f"invented{control}FORGED"})+ monkeypatch.setattr(rubric_eval.logger, "propagate", True)++ result, requests = _validate_with(tmp_path, _reply(checks), _reply(checks))++ problem = requests[1]["user"].removeprefix(requests[0]["user"]).removeprefix(+ f"\n{_RETRY_MARKER}"+ ).removesuffix(f". {_RETRY_TAIL}")+ assert "invented" in problem and "FORGED" in problem+ assert problem.isprintable()+ assert " " not in problem+ rejected = [record for record in caplog.records if "reply rejected" in record.getMessage()]+ assert rejected+ assert all(record.getMessage().isprintable() for record in rejected)+ (finding,) = _llm_findings(result)+ assert finding.message.isprintable()+ assert result.metadata["rubric_eval"]["summary"] == finding.message+@pytest.mark.parametrize(("first_reply", "problem"),[