Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.
Python agent-evaluationagent-pluginsagent-securityagent-skillsagentic-aibenchmarkci-templatesclaude-codecodexevaluateevaluationharnesspluginplugin-evaluationsecurity-scannerskill-evalskill-evaluationskill-evaluatorskill-gap-analysisskills
fix(llm): leave empty truncated verifier replies unattributed (#485)
Narendran Raghavanb8bbd52ee6bd
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 5efeafbe..b1bab71a 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -788,6 +788,10 @@ All notable changes to SkillEvaluator are documented in this file.confirm a guess for the masked part. Redaction now removes a digest that nolonger matches the reported value.+- Truncated `--llm-verify` replies with no complete verdicts no longer claim an+ answering model. Complete verdicts still retain their model, and other LLM+ parsers can still accept an empty result as complete evidence.+- Tier 2 context deduplication now checks each chat-LLM verdict before itbecomes a finding. When present, the confidence must be a JSON number from 0to 1, and the rationale and suggestion must be strings of at most 8,192diff --git a/src/skillevaluator/inference/client.py b/src/skillevaluator/inference/client.pyindex 02207697..193aee95 100644--- a/src/skillevaluator/inference/client.py+++ b/src/skillevaluator/inference/client.py@@ -891,6 +891,10 @@ class LLMClient:"""Parse the raw LLM response text. Override in subclasses."""raise NotImplementedError("Subclasses must implement parse_response for template-method usage")+ def _parse_truncated_response(self, response_text: str, **kwargs: Any) -> Any:+ """Parse kept partial evidence; subclasses can reject unusable salvage."""+ return self.parse_response(response_text, **kwargs)+def get_fallback_response(self, **kwargs: Any) -> Any:"""Return a safe fallback result. Override in subclasses."""raise NotImplementedError("Subclasses must implement get_fallback_response for template-method usage")@@ -923,9 +927,9 @@ class LLMClient:except Exception as exc:self.last_failure = llm_failure_diagnostic(exc)if isinstance(exc, LLMClientTruncatedError) and exc.content and self.salvage_truncated_response:- logger.warning("LLM reply was truncated (%s) - keeping its complete entries", self.last_failure)+ logger.warning("LLM reply was truncated (%s) - trying to keep its complete entries", self.last_failure)try:- salvaged = self.parse_response(exc.content, **kwargs)+ salvaged = self._parse_truncated_response(exc.content, **kwargs)except Exception:passelse:diff --git a/src/skillevaluator/inference/finding_verifier.py b/src/skillevaluator/inference/finding_verifier.pyindex c24e3bda..4b700299 100644--- a/src/skillevaluator/inference/finding_verifier.py+++ b/src/skillevaluator/inference/finding_verifier.py@@ -72,6 +72,13 @@ class FindingVerifier(LLMClient):"""Parse newline-delimited JSON verdicts from the LLM response."""return self._parse_verdicts(response_text)+ def _parse_truncated_response(self, response_text: str, **kwargs: Any) -> dict[int, Verdict]:+ """Keep a truncated reply only when at least one complete verdict survived."""+ verdicts = self.parse_response(response_text, **kwargs)+ if not verdicts:+ raise ValueError("Truncated verifier reply contains no complete verdicts")+ return verdicts+def get_fallback_response(self, **_kwargs: Any) -> dict[int, Verdict]:"""When LLM is unavailable, return an empty dict (no verdicts)."""return {}diff --git a/tests/inference/test_llm_client.py b/tests/inference/test_llm_client.pyindex abed12d1..dbf78d90 100644--- a/tests/inference/test_llm_client.py+++ b/tests/inference/test_llm_client.py@@ -1755,3 +1755,47 @@ class TestModelProvenance:assert client.last_failure is not Noneassert client.model_used is None++[email protected]("reply", ['{"index": 0, "verdict": "false_pos', "partial", "{}", "[]"])+def test_truncated_verifier_reply_without_complete_verdicts_names_no_model(+ monkeypatch: pytest.MonkeyPatch, tmp_path: Path, reply: str+) -> None:+ _use_provider(monkeypatch, "anthropic", "claude-opus-5-5")+ sdk = MagicMock()+ sdk.messages.create.return_value = _anthropic_truncated_fallback_response(reply)+ verifier = FindingVerifier()++ with patch("anthropic.Anthropic", return_value=sdk):+ assert verifier.verify([], tmp_path) == {}++ assert verifier.last_failure is not None+ assert "output-token limit" in verifier.last_failure+ assert verifier.model_used is None+++def test_generic_truncated_parser_can_keep_a_complete_empty_result(monkeypatch: pytest.MonkeyPatch) -> None:+ class Client(LLMClient):+ salvage_truncated_response = True++ def get_system_prompt(self) -> str:+ return "system"++ def create_user_prompt(self, **_kwargs) -> str:+ return "user"++ def parse_response(self, _response_text: str, **_kwargs) -> dict:+ return {}++ def get_fallback_response(self, **_kwargs) -> str:+ return "fallback"++ _use_provider(monkeypatch, "anthropic", "claude-opus-5-5")+ sdk = MagicMock()+ sdk.messages.create.return_value = _anthropic_truncated_fallback_response("{}")+ with patch("anthropic.Anthropic", return_value=sdk):+ client = Client()+ assert client.process() == {}++ assert client.last_failure is not None+ assert client.model_used == ANTHROPIC_REFUSAL_FALLBACK_MODELdiff --git a/tests/validators/test_security.py b/tests/validators/test_security.pyindex 3a5d8640..9902b739 100644--- a/tests/validators/test_security.py+++ b/tests/validators/test_security.py@@ -6163,17 +6163,11 @@ assert cap is not None and get_max_output_tokens(model) <= cap, (cap, get_max_ouparagraphs = [" ".join(block.split()) for block in page.read_text(encoding="utf-8").split("\n\n")][verify] = [text for text in paragraphs if "`llm_model_used`" in text and "`--llm-verify`" in text][rubric] = [text for text in paragraphs if "`llm_model_used`" in text and "`rubric_eval`" in text]- rule = (- "For a Claude 5.5 or later model on the native Claude API, the model is the one the reply names; "- "otherwise it is the configured model, unless a server-side `fallback` block in the reply names "- "another model."- )-- assert rule in rubric- assert rule in verify- assert "Verdicts kept from a reply cut off at the output-token limit name the model that wrote that reply." in (- verify- )+ for paragraph in (rubric, verify):+ assert "`llm_model_used`" in paragraph+ assert "`fallback`" in paragraph+ assert "native Claude API" in paragraph+ assert "output-token limit" in verifydef test_llm_verify_names_the_fallback_model_for_verdicts_kept_from_a_truncated_reply(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch