Skip to content

Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.

fix(tier3): keep case actions from weakening dataset requirements (#588)

Christopher Kevin53dfb347fa33
diff --git a/CHANGELOG.md b/CHANGELOG.md
index c084e81d..822f2a7c 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -503,6 +503,10 @@ All notable changes to SkillEvaluator are documented in this file.
case names receive the same Markdown escaping, publication-label masking and
terminal secret redaction as visible rows, before their displayed length is
bounded.
+- Tier 3 case actions preserve dataset expectations and assertions, rejecting
+ requirement-weakening model advice in favor of deterministic remedies.
+ Recommendations identify model-written actions and record when generation
+ failed or unsafe advice was replaced by a fallback.
- Tier 3 token usage per case, per agent and over all agents now counts each
attempt in the case it belongs to. An unscored attempt of a native task whose
`[metadata] entry_id` differs from its directory joins that case's row
diff --git a/src/skillevaluator/evaluation/dataset_case_removal.py b/src/skillevaluator/evaluation/dataset_case_removal.py
new file mode 100644
index 00000000..d032b8de
--- /dev/null
+++ b/src/skillevaluator/evaluation/dataset_case_removal.py
@@ -0,0 +1,566 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Spot advice that takes a case out of the evaluation dataset.
+
+Tier 3 advice may only fix the skill, never the test: "delete the case for positive-001" or "replace the case with
+an easier one" would make a failing case pass by removing the requirement. The check is a heuristic over free text,
+so it errs on the side of rejecting: a rejected action falls back to a deterministic remedy.
+
+Detection is linear in the length of the text. Each clause is split into tokens once, and every later check looks at
+a bounded window of tokens or at a table lookup, so no input can make it backtrack. A clause is a removal when
+
+* a removal verb ("delete", "exclude", "get rid of", "replace ... with an easier one", ...) governs a case phrase
+ with at most a few filler words in between, or a case phrase is the subject of a passive removal
+ ("the case should be excluded");
+* the phrase names a dataset case: a test/eval/dataset-qualified noun, a known case id next to "case", an
+ identifier-shaped name next to "case", or a bare "case"/"entry" that the clause ties to the dataset;
+* the phrase is not an implementation branch ("the special case in parser.py"), a failure to repair
+ ("the test case failure"), or negated advice ("do not delete the case").
+"""
+
+from __future__ import annotations
+
+import re
+from collections.abc import Callable, Iterable, Iterator
+
+# A line break inside a sentence ("delete the case\nfor positive-001") does not end the clause.
+_CLAUSE_BREAK = re.compile(r"[.!?;]\s")
+_TOKEN = re.compile(r"\w[\w./'\u2019-]*|[,:;!?()\[\]]")
+_WORD_TOKEN = re.compile(r"\w[\w./'\u2019-]*")
+_PHRASE_END_TOKENS = frozenset({".", ",", ";", ":", "!", "?", ")"})
+
+# Advice that keeps the case: "Do not delete the case", "fix the step instead of deleting the case". A negator only
+# counts when it governs the verb, so the words between the two are limited to modifiers of the verb ("without first
+# deleting", "without ever having to delete") and "run without failures by deleting the case" stays a removal.
+# Only the last few words in front of the verb are read, so the check does not depend on the length of the text.
+_NEGATION_WINDOW = 120
+_NEGATION_WORD = re.compile(r"[\w'\u2019]+|[^\w\s]")
+_PLAIN_WORD = re.compile(r"\w+")
+# "do not"-style negators also cover up to two plain words before the verb: "do not try deleting the case".
+_LOOSE_NEGATORS = (("do", "not"), ("don't",), ("don\u2019t",), ("never",), ("avoid",))
+# "not only deleting the case but also fixing the parser" recommends the deletion. After one of these words a "not"
+# negator no longer forbids the verb, so "don't just delete the case, fix the parser" is rejected as well: it cannot
+# be told apart from the additive form, and a rejection only falls back to the deterministic remedy.
+_ADDITIVE_WORDS = frozenset({"only", "just", "merely", "simply", "solely", "exclusively", "purely"})
+_NOT_NEGATORS = (*_LOOSE_NEGATORS, ("not",))
+_OTHER_NEGATORS = (("without",), ("instead", "of"), ("rather", "than"), ("no", "need", "to"))
+_MAX_MODIFIERS = 5
+_MODIFIER_WORD = re.compile(r"just|also|then|even|ever|only|still|first|outright|again|altogether|always|\w{3,}ly")
+_MODIFIER_PHRASES = frozenset(
+ {("the", "need", "to"), ("any", "need", "to")}
+ | {("being", participle, "to") for participle in ("forced", "asked", "required", "made")}
+ | {
+ (verb, "to")
+ for verb in (
+ "having",
+ "have",
+ "has",
+ "needing",
+ "need",
+ "needs",
+ "trying",
+ "attempting",
+ "resorting",
+ "wanting",
+ "choosing",
+ "bothering",
+ )
+ }
+)
+
+
+def _ends_with(words: list[str], end: int, phrases: tuple[tuple[str, ...], ...]) -> bool:
+ return any(end >= len(phrase) and tuple(words[end - len(phrase) : end]) == phrase for phrase in phrases)
+
+
+def _modifier_start(words: list[str], end: int) -> int | None:
+ """Start of the verb modifier ("first", "having to", "being forced to") that ends at ``end``, if any."""
+ for size in (3, 2):
+ if end >= size and tuple(words[end - size : end]) in _MODIFIER_PHRASES:
+ return end - size
+ if end >= 1 and _MODIFIER_WORD.fullmatch(words[end - 1]):
+ return end - 1
+ return None
+
+
+def negated_before(text: str, start: int) -> bool:
+ """True when advice that begins at ``start`` is negated ("do not ...", "without first ...", "instead of ...")."""
+ words = _NEGATION_WORD.findall(text[max(0, start - _NEGATION_WINDOW) : start].lower())
+ count = len(words)
+ for plain in range(3):
+ end = count - plain
+ if (
+ end >= 1
+ and all(_PLAIN_WORD.fullmatch(word) and word not in _ADDITIVE_WORDS for word in words[end:])
+ and _ends_with(words, end, _LOOSE_NEGATORS)
+ ):
+ return True
+ end = count
+ additive = False
+ for _ in range(_MAX_MODIFIERS + 1):
+ if _ends_with(words, end, _OTHER_NEGATORS):
+ return True
+ if _ends_with(words, end, _NOT_NEGATORS):
+ return not additive
+ modifier = _modifier_start(words, end)
+ if modifier is None:
+ return False
+ additive = additive or any(word in _ADDITIVE_WORDS for word in words[modifier:end])
+ end = modifier
+ return False
+
+
+_STRONG_VERB = re.compile(
+ r"remov(?:e[sd]?|ing)|delet(?:e[sd]?|ing)|drop(?:s|ped|ping)?|skip(?:s|ped|ping)?|disabl(?:e[sd]?|ing)|"
+ r"exclud(?:e[sd]?|ing)|discard(?:s|ed|ing)?|omit(?:s|ted|ting)?|eliminat(?:e[sd]?|ing)|retir(?:e[sd]?|ing)"
+)
+# "ignore case" and "strip case" are also about letter case, so these two verbs only count with a determiner:
+# "ignore the case for positive-001" removes a case, "ignore case when comparing" does not.
+_WEAK_VERB = re.compile(r"ignor(?:e[sd]?|ing)|strip(?:s|ped|ping)?")
+# "take the case out of the dataset", "filter positive-001 out of evals.json": the object sits between verb and "out of".
+_EXTRACTION_VERB = re.compile(
+ r"tak(?:e[sn]?|ing)|took|leav(?:e|es|ing)|left|filter(?:s|ed|ing)?|pull(?:s|ed|ing)?|cut(?:s|ting)?"
+)
+_REPLACEMENT_VERB = re.compile(r"replac(?:e[sd]?|ing)")
+_PHRASAL_VERBS = (
+ (re.compile(r"comment(?:s|ed|ing)?"), ("out",)),
+ (re.compile(r"stop(?:s|ped|ping)?"), ("running",)),
+ (re.compile(r"get(?:s|ting)?|got|gotten"), ("rid", "of")),
+ (re.compile(r"tak(?:e[sn]?|ing)|took"), ("out",)),
+ (re.compile(r"leav(?:e|es|ing)|left"), ("out",)),
+ (re.compile(r"filter(?:s|ed|ing)?"), ("out",)),
+ (re.compile(r"swap(?:s|ped|ping)?"), ("out",)),
+ (re.compile(r"instead"), ("of", "keeping")),
+)
+_REMOVE, _EXTRACT, _REPLACE = "remove", "extract", "replace"
+
+_DETERMINER = re.compile(r"the|a|an|my|your|our|their|its|this|that|these|those|any|each|every|all|both")
+_POSSESSIVE = re.compile(r"[a-z0-9_-]+(?:'s|\u2019s)")
+# Words that end a noun phrase, so "remove temp files per case" is not read as "remove the case".
+_PHRASE_STOP = re.compile(
+ r"(?:in|of|on|at|by|for|to|if|as|and|or|with|without|from|after|before|when|where|that|which|than|then|but|so|"
+ r"while|is|are|was|were|be|per|during|across|within|inside|into|onto|via|using|until|upon|around)\b"
+)
+_QUALIFIER_TOKEN = re.compile(r"[a-z0-9_'-]+")
+
+_CASE_WORDS = frozenset({"case", "cases"})
+_ENTRY_WORDS = frozenset({"entry", "entries", "row", "rows", "example", "examples"})
+_QUALIFIERS = frozenset({"test", "eval", "evaluation", "dataset", "benchmark"})
+_QUALIFIED_NOUN = re.compile(r"(?:test|eval(?:uation)?|dataset|benchmark)[_-]*(?:cases?|entr(?:y|ies)|rows?|examples?)")
+_FAILURE_WORDS = frozenset({"mismatch", "mismatches", "failure", "failures", "violation", "violations"})
+_FAILURE_LEAD = frozenset({"output", "answer", "result"})
+# "case of", "case statement", "case folding", "case-insensitive" ...: letter case or control flow, not a dataset case.
+_CASE_IDIOM = re.compile(r"of|statements?|conversions?|folding|handling|(?:in)?sensitiv.*")
+
+_LINK_WORDS = frozenset({"for", "of", "named", "called", "titled", "labelled", "labeled", "id", "name"})
+# An id that is not a known case id must end its phrase, so "the case for zero-length input" stays implementation prose.
+_NAME_END = re.compile(
+ r"so|to|and|or|but|because|since|as|which|that|if|when|by|then|from|in|within|out|"
+ r"is|are|was|were|be|should|must|can|could|may|might|will|would"
+)
+_NAME_TEXT = re.compile(r"[a-z0-9_-]+")
+_NAME_SEPARATOR = re.compile(r"[_-]")
+
+_DATASET_DETERMINERS = frozenset({"the", "this", "our", "your"})
+_EVAL_COMPONENT = re.compile(r"(?:^|/)evals?(?![a-z])")
+_DATASET_WORD = re.compile(r"datasets?\b")
+_JSON_FILE = re.compile(r"[\w./-]\.jsonl?\b")
+_BENCHMARK = re.compile(r"benchmark\b")
+# A case that sits in a script or source file ("the special case for positive-001 in parser.py") is an
+# implementation branch, not a dataset row.
+_CODE_FILE = re.compile(r"[\w./-]*\.(?:py|js|mjs|cjs|ts|tsx|jsx|sh|bash|rb|go|rs|java|c|cc|cpp|h|hpp|md)\b")
+_CODE_WORD = re.compile(
+ r"(?:script|code|function|parser|module|implementation|handler|logic|class|method|helper|switch|conditional|"
+ r"branch|loop|program|algorithm|routine)\b"
+)
+
+_EASIER_WORD = re.compile(r"(?:easier|simpler|weaker|softer|looser|lighter|trivial)\b")
+_BE_FORMS = frozenset({"is", "are", "was", "were", "be", "been", "being"})
+_MODALS = frozenset({"should", "must", "can", "could", "may", "might", "will", "would"})
+
+# What a case phrase is: a dataset case outright, a bare "case" that needs the clause to mention the dataset, or a
+# known case id on its own (only enough for "replace positive-001 with an easier case").
+_DATASET, _BARE, _ID_ONLY = "dataset", "bare", "id"
+
+
+def _is_dataset_token(token: str) -> bool:
+ return bool(
+ _EVAL_COMPONENT.search(token)
+ or _DATASET_WORD.search(token)
+ or _JSON_FILE.search(token)
+ or _BENCHMARK.match(token)
+ )
+
+
+def _id_shaped(token: str) -> bool:
+ """True for case-id-like tokens: "positive-001", "syllable-count", "case_12", "python3"."""
+ if not _NAME_TEXT.fullmatch(token):
+ return False
+ if any(char.isdigit() for char in token) and any("a" <= char <= "z" for char in token):
+ return True
+ parts = _NAME_SEPARATOR.split(token)
+ if not all(parts):
+ return False
+ return len(parts) >= 2 or parts[0] in {"positive", "negative"}
+
+
+def _clauses(text: str) -> Iterator[str]:
+ start = 0
+ for match in _CLAUSE_BREAK.finditer(text):
+ yield text[start : match.end()]
+ start = match.end()
+ yield text[start:]
+
+
+def _tokenize(clause: str) -> tuple[list[str], list[int], list[int]]:
+ tokens: list[str] = []
+ starts: list[int] = []
+ ends: list[int] = []
+ for match in _TOKEN.finditer(clause):
+ token = match.group()
+ end = match.end()
+ if _WORD_TOKEN.fullmatch(token):
+ stripped = token.rstrip(".'\u2019/")
+ if stripped != token:
+ end -= len(token) - len(stripped)
+ tokens.append(stripped)
+ starts.append(match.start())
+ ends.append(end)
+ if token[len(stripped)] == ".":
+ tokens.append(".")
+ starts.append(end)
+ ends.append(match.end())
+ continue
+ tokens.append(token)
+ starts.append(match.start())
+ ends.append(end)
+ return tokens, starts, ends
+
+
+class _Clause:
+ """One clause as tokens, with the lookups the removal checks share."""
+
+ def __init__(self, text: str, known: frozenset[str], strict: bool) -> None:
+ self.text = text
+ self.known = known
+ self.toks, self.starts, self.ends = _tokenize(text)
+ self.n = len(self.toks)
+ # Where each dataset location ("from evals.json", "out of the dataset") starts and where it ends.
+ self.locations = {p: e for p in range(self.n) if (e := self._location_end(p)) is not None}
+ self.first_location_end = min(self.locations.values(), default=None)
+ # Identifier-shaped names are only evidence of a dataset case when the caller has no case ids to compare
+ # against, or when the clause also points at the dataset: "the special case for UTF-8" is code.
+ self.shape_counts = not strict or bool(self.locations)
+ self.verbs_ending_at: dict[int, list[tuple[int, str, bool]]] = {}
+ self.verbs_starting_at: dict[int, list[tuple[int, str, bool]]] = {}
+ for i in range(self.n):
+ for end, form, weak in self._verbs_at(i):
+ self.verbs_starting_at.setdefault(i, []).append((end, form, weak))
+ self.verbs_ending_at.setdefault(end - 1, []).append((i, form, weak))
+
+ def removes(self) -> bool:
+ return any(self._removal(start, end, kind) for start, end, kind in self._phrases())
+
+ # --- tokens
+
+ def _tok(self, index: int) -> str:
+ return self.toks[index] if 0 <= index < self.n else ""
+
+ def _verbs_at(self, i: int) -> Iterator[tuple[int, str, bool]]:
+ token = self.toks[i]
+ if _STRONG_VERB.fullmatch(token):
+ yield i + 1, _REMOVE, False
+ if _WEAK_VERB.fullmatch(token):
+ yield i + 1, _REMOVE, True
+ if _EXTRACTION_VERB.fullmatch(token):
+ yield i + 1, _EXTRACT, False
+ if _REPLACEMENT_VERB.fullmatch(token):
+ yield i + 1, _REPLACE, False
+ for first, rest in _PHRASAL_VERBS:
+ if first.fullmatch(token) and tuple(self.toks[i + 1 : i + 1 + len(rest)]) == rest:
+ yield i + 1 + len(rest), _REMOVE, False
+
+ def _is_name(self, token: str) -> bool:
+ return token in self.known or (self.shape_counts and _id_shaped(token))
+
+ def _is_known_name(self, token: str) -> bool:
+ return token in self.known
+
+ def _strip_possessive(self, token: str) -> str:
+ return token[:-2] if token.endswith(("'s", "\u2019s")) else token
+
+ # --- dataset locations
+
+ def _target_end(self, i: int) -> int | None:
+ if self._tok(i) in _DATASET_DETERMINERS:
+ i += 1
+ token = self._tok(i)
+ if token and _is_dataset_token(token):
+ return i + 1
+ if token in {"test", "eval", "evaluation"} and self._tok(i + 1) in {"set", "suite", "file"}:
+ return i + 2
+ return None
+
+ def _location_end(self, i: int) -> int | None:
+ token = self.toks[i]
+ if token in {"from", "in", "within", "of"}:
+ return self._target_end(i + 1)
+ if token == "out" and self._tok(i + 1) == "of":
+ return self._target_end(i + 2)
+ return None
+
+ def _location_within(self, noun: int, limit: int) -> bool:
+ """A dataset location starts within ``limit`` characters of the noun, in the same phrase."""
+ end = self.ends[noun]
+ for p in range(noun + 1, self.n):
+ gap = self.text[end : self.starts[p]]
+ if len(gap) > limit or re.search(r"[.;!?]", gap):
+ return False
+ if p in self.locations:
+ return True
+ return False
+
+ def _in_code(self, end: int) -> bool:
+ """The phrase that ends at ``end`` is followed by "in parser.py", "from the script", ..."""
+ token = self._tok(end)
+ if token in {"in", "from", "inside", "within"}:
+ i = end + 1
+ elif token == "out" and self._tok(end + 1) == "of":
+ i = end + 2
+ else:
+ return False
+ if self._tok(i) in _DATASET_DETERMINERS:
+ i += 1
+ token = self._tok(i)
+ return bool(token) and bool(_CODE_FILE.match(token) or _CODE_WORD.match(token))
+
+ # --- case phrases
+
+ def _fixes_failure(self, end: int) -> bool:
+ """Failure phrases ("the test case failure", "the case mismatch") name a failure to repair, not the case."""
+ if self._tok(end) in _FAILURE_LEAD:
+ end += 1
+ return self._tok(end) in _FAILURE_WORDS
+
+ def _is_bare_case(self, index: int) -> bool:
+ following = self._tok(index + 1)
+ return not self._fixes_failure(index + 1) and not _CASE_IDIOM.fullmatch(following)
+
+ def _name_starts(self, link_end: int) -> Iterator[int]:
+ """Where a case id may start after a noun: directly, after ":", or after link words like "for the"."""
+ yield link_end
+ if self._tok(link_end) == ":":
+ yield link_end + 1
+ index = link_end
+ for _ in range(2):
+ token = self._tok(index)
+ if token in _LINK_WORDS:
+ index += 1
+ elif token == "with":
+ index += 1
+ if self._tok(index) == "the":
+ index += 1
+ if self._tok(index) not in {"id", "name"}:
+ return
+ index += 1
+ else:
+ return
+ yield index
+ if self._tok(index) == "the":
+ yield index + 1
+
+ def _chain_start(self, noun: int, is_name: Callable[[str], bool]) -> int | None:
+ """Start of the list of names ("a, b and c's") that ends right before ``noun``, if there is one."""
+ last = noun - 1
+ if last < 0 or not is_name(self._strip_possessive(self.toks[last])):
+ return None
+ start = last
+ while True:
+ index = start - 1
+ if self._tok(index) in {"and", "or"}:
+ index -= 1
+ if self._tok(index) == ",":
+ index -= 1
+ elif self._tok(index) == ",":
+ index -= 1
+ else:
+ return start
+ if index < 0 or not is_name(self.toks[index]):
+ return start
+ start = index
+
+ def _phrases(self) -> Iterator[tuple[int, int, str]]:
+ """Every (start, end, kind) token span that reads as a case in the dataset."""
+ for k, token in enumerate(self.toks):
+ if token in _CASE_WORDS:
+ yield from self._case_phrases(k)
+ elif token in _ENTRY_WORDS:
+ yield from self._entry_phrases(k)
+ elif _QUALIFIED_NOUN.fullmatch(token) and not self._fixes_failure(k + 1):
+ yield k, k + 1, _DATASET
+ if token in self.known:
+ if k + 1 in self.locations:
+ yield k, k + 1, _DATASET
+ yield k, k + 1, _ID_ONLY
+
+ def _qualified(self, noun: int) -> Iterator[tuple[int, int, str]]:
+ if self._tok(noun - 1) in _QUALIFIERS and not self._fixes_failure(noun + 1):
+ yield noun - 1, noun + 1, _DATASET
+
+ def _named_after(self, noun: int, *, any_shape: bool) -> Iterator[tuple[int, int, str]]:
+ """ "the case for positive-001", "case: positive-001", "test case positive-001"."""
+ for name in self._name_starts(noun + 1):
+ token = self._tok(name)
+ if self._strip_possessive(token) in self.known:
+ if not self._in_code(name + 1):
+ yield noun, name + 1, _DATASET
+ elif any_shape and self.shape_counts and token and _id_shaped(token):
+ end = name + 1
+ if (end >= self.n or self.toks[end] in _PHRASE_END_TOKENS or _NAME_END.fullmatch(self.toks[end])) and (
+ not self._in_code(end)
+ ):
+ yield noun, end, _DATASET
+
+ def _case_phrases(self, k: int) -> Iterator[tuple[int, int, str]]:
+ yield from self._qualified(k)
+ yield from self._named_after(k, any_shape=True)
+ if self._is_bare_case(k):
+ chain = self._chain_start(k, self._is_name)
+ if chain is not None and not self._in_code(k + 1):
+ yield chain, k + 1, _DATASET
+ if self._location_within(k, 40):
+ yield k, k + 1, _DATASET
+ yield k, k + 1, _BARE
+
+ def _entry_phrases(self, k: int) -> Iterator[tuple[int, int, str]]:
+ yield from self._qualified(k)
+ yield from self._named_after(k, any_shape=False)
+ if self._location_within(k, 60):
+ yield k, k + 1, _DATASET
+ chain = self._chain_start(k, self._is_known_name)
+ if chain is not None and not self._fixes_failure(k + 1) and not self._in_code(k + 1):
+ yield chain, k + 1, _DATASET
+
+ # --- removal verbs
+
+ def _filler_ok(self, start: int, end: int, weak: bool) -> bool:
+ """Between a verb and its case phrase: up to three determiners, then up to three plain words."""
+ count = end - start
+ if count > 6:
+ return False
+ determiners = 0
+ while determiners < min(count, 3) and self._is_determiner(self.toks[start + determiners]):
+ determiners += 1
+ if weak and determiners == 0:
+ return False
+ rest = self.toks[start + determiners : end]
+ return len(rest) <= 3 and all(
+ _QUALIFIER_TOKEN.fullmatch(token) and not _PHRASE_STOP.match(token) for token in rest
+ )
+
+ def _is_determiner(self, token: str) -> bool:
+ return bool(_DETERMINER.fullmatch(token) or _POSSESSIVE.fullmatch(token))
+
+ def _scope_before(self, index: int) -> bool:
+ return self.first_location_end is not None and self.first_location_end <= index
+
+ def _easier_follows(self, index: int) -> bool:
+ if self._tok(index) in {"a", "an", "another", "some"}:
+ index += 1
+ token = self._tok(index)
+ if _EASIER_WORD.match(token):
+ return True
+ following = self._tok(index + 1)
+ return (token == "more" and following in {"lenient", "forgiving", "permissive"}) or (
+ token == "less" and following in {"strict", "demanding", "difficult", "challenging"}
+ )
+
+ def _replacement_follows(self, end: int) -> bool:
+ for index in (end, self.locations.get(end)):
+ if index is not None and self._tok(index) in {"with", "by"} and self._easier_follows(index + 1):
+ return True
+ return False
+
+ def _suffix_ok(self, form: str, end: int, kind: str) -> bool:
+ if form == _REPLACE:
+ return self._replacement_follows(end)
+ if kind == _ID_ONLY:
+ return False
+ if form == _EXTRACT:
+ return self._tok(end) == "out" and self._tok(end + 1) == "of" and self._target_end(end + 2) is not None
+ return True
+
+ def _removal(self, start: int, end: int, kind: str) -> bool:
+ return self._active(start, end, kind) or self._passive(start, end, kind)
+
+ def _active(self, start: int, end: int, kind: str) -> bool:
+ for gap in range(7):
+ last = start - 1 - gap
+ if last < 0:
+ return False
+ for verb_start, form, weak in self.verbs_ending_at.get(last, ()):
+ if (
+ self._filler_ok(last + 1, start, weak)
+ and self._suffix_ok(form, end, kind)
+ and (kind != _BARE or self._scope_before(verb_start))
+ and not negated_before(self.text, self.starts[verb_start])
+ ):
+ return True
+ return False
+
+ # --- passive removals: "the case should be excluded"
+
+ def _passive_verb_at(self, index: int) -> bool:
+ if any(form == _REMOVE for _, form, _ in self.verbs_starting_at.get(index, ())):
+ return True
+ return (
+ self._tok(index) == "replaced"
+ and self._tok(index + 1) in {"with", "by"}
+ and self._easier_follows(index + 2)
+ )
+
+ def _tail_at(self, index: int) -> bool:
+ if self._tok(index) in {"also", "then"}:
+ index += 1
+ token = self._tok(index)
+ if token in _BE_FORMS:
+ index += 1
+ elif token in _MODALS and self._tok(index + 1) == "be":
+ index += 2
+ elif token in {"ought", "need", "needs"} and self.toks[index + 1 : index + 3] == ["to", "be"]:
+ index += 3
+ else:
+ return False
+ if self._tok(index) in {"just", "simply", "safely"}:
+ index += 1
+ return self._passive_verb_at(index)
+
+ def _passive(self, start: int, end: int, kind: str) -> bool:
+ if kind == _ID_ONLY or (kind == _BARE and not self._scope_before(start)):
+ return False
+ if negated_before(self.text, self.starts[start]):
+ return False
+ return self._tail_at(end) or (end in self.locations and self._tail_at(self.locations[end]))
+
+
+def removes_dataset_case(text: str, case_ids: Iterable[object] = ()) -> bool:
+ """True when ``text`` advises taking a dataset case out of the dataset (deleting, excluding, replacing it).
+
+ ``case_ids`` are the ids of the dataset cases the advice may name. A known id is recognised next to "case" even
+ when it is a plain word ("delete the smoke case"). Without ids, or when the clause points at the dataset
+ ("... from evals.json"), identifier-shaped names ("positive-001", "syllable-count") are recognised too.
+ """
+ lowered = text.lower()
+ supplied = [str(case_id).strip().lower() for case_id in case_ids]
+ present = {name for name in supplied if len(name) >= 2 and name in lowered}
+ # Ids that are not a single token (they contain spaces or other punctuation) are swapped for a placeholder.
+ odd = sorted((name for name in present if not _WORD_TOKEN.fullmatch(name) or name[-1] in ".'\u2019/"), key=len)
+ known = present.difference(odd)
+ if odd:
+ placeholders = {name: f"zzcaseid{index}zz" for index, name in enumerate(odd)}
+ pattern = re.compile(r"(?<![\w-])(?:" + "|".join(re.escape(n) for n in reversed(odd)) + r")(?![\w-])")
+ lowered = pattern.sub(lambda match: placeholders[match.group()], lowered)
+ known |= set(placeholders.values())
+ strict = any(supplied)
+ return any(_Clause(clause, frozenset(known), strict).removes() for clause in _clauses(lowered))
diff --git a/src/skillevaluator/evaluation/insights_judge.py b/src/skillevaluator/evaluation/insights_judge.py
index 5b705559..00dec0ee 100644
--- a/src/skillevaluator/evaluation/insights_judge.py
+++ b/src/skillevaluator/evaluation/insights_judge.py
@@ -19,6 +19,7 @@ from __future__ import annotations
import json
import math
import re
+from collections.abc import Iterable
from typing import Any
from skillevaluator.constants import (
@@ -30,6 +31,7 @@ from skillevaluator.constants import (
INSIGHTS_JUDGE_REASONING_EFFORT,
INSIGHTS_JUDGE_TEMPERATURE,
)
+from skillevaluator.evaluation.dataset_case_removal import negated_before, removes_dataset_case
from skillevaluator.inference.client import LLMClient
from skillevaluator.inference.diagnostics import safe_llm_model
from skillevaluator.logging_config import get_logger
@@ -71,24 +73,31 @@ _REQUIREMENT_NOUN = (
r"(?:expectations?|expected(?:\s+(?:output|answer|result))?|assertions?|ground[\s_-]*truth|"
r"pass[\s_-]*thresholds?|thresholds?|requirements?|criteria)"
)
+# "Remove the expected output mismatch" fixes a failure; it does not remove the requirement itself.
+_NOT_FAILURE_OF_REQUIREMENT = r"(?!(?:\s+(?:output|answer|result))?\s+(?:mismatch(?:es)?|failures?|violations?)\b)"
_WEAKENING_VERB = (
- r"(?:loosen(?:s|ed|ing)?|relax(?:es|ed|ing)?|weaken(?:s|ed|ing)?|lower(?:s|ed|ing)?|soften(?:s|ed|ing)?)"
+ r"(?:loosen(?:s|ed|ing)?|relax(?:es|ed|ing)?|weaken(?:s|ed|ing)?|lower(?:s|ed|ing)?|soften(?:s|ed|ing)?|"
+ r"remov(?:e[sd]?|ing)|drop(?:s|ped|ping)?|delet(?:e[sd]?|ing)|skip(?:s|ped|ping)?|disabl(?:e[sd]?|ing))"
)
_REQUIREMENT_DETERMINER = (
r"(?:the|a|an|my|your|our|their|its|this|that|these|those|any|each|every|all|both|"
r"[A-Za-z0-9_-]+(?:'s|\u2019s))"
)
_WEAKENING_PATTERN = re.compile(
- rf"\b{_WEAKENING_VERB}\b(?:\s+{_REQUIREMENT_DETERMINER}){{0,3}}\s+{_REQUIREMENT_NOUN}\b",
+ rf"\b{_WEAKENING_VERB}\b(?:\s+{_REQUIREMENT_DETERMINER}){{0,3}}\s+"
+ rf"(?:`?[A-Za-z0-9_-]+`?\s+)?{_REQUIREMENT_NOUN}\b{_NOT_FAILURE_OF_REQUIREMENT}",
re.IGNORECASE,
)
_PASSIVE_WEAKENING_PATTERN = re.compile(
- rf"\b{_REQUIREMENT_NOUN}\b[\w\s,'\u2019_-]{{0,48}}\b"
+ rf"\b{_REQUIREMENT_NOUN}\b{_NOT_FAILURE_OF_REQUIREMENT}[\w\s,'\u2019_-]{{0,48}}\b"
rf"(?:is|are|was|were|be|been|being|should\s+be|must\s+be|ought\s+to\s+be|will\s+be)\s+"
rf"{_WEAKENING_VERB}\b",
re.IGNORECASE,
)
+# Inline-code ticks and quotes must not hide a name: "Remove the `expectation`" is still a removal.
+_QUOTE_PATTERN = re.compile(r"[`\"\u201c\u201d]|(?<!\w)['\u2018\u2019]|['\u2018\u2019](?!\w)")
+
_SYSTEM_PROMPT = """\
You are an expert Skill Quality Reviewer summarising a Tier 3 (live agent)
@@ -470,14 +479,16 @@ def _text_case_ids(text: str, known_case_ids: set[str]) -> set[str]:
return references
-def _weakens_requirements(text: str) -> bool:
+def _weakens_requirements(text: str, case_ids: Iterable[object] = ()) -> bool:
+ """True when advice removes or weakens a dataset requirement or takes a dataset case out of the dataset.
+
+ ``case_ids`` are the ids of the dataset cases the advice may name; see :func:`removes_dataset_case`.
+ """
+ text = _QUOTE_PATTERN.sub("", text)
for pattern in (_WEAKENING_PATTERN, _PASSIVE_WEAKENING_PATTERN):
- for match in pattern.finditer(text):
- prefix = text[max(0, match.start() - 48) : match.start()]
- if re.search(r"\b(?:do\s+not|don't|never|avoid)(?:\s+\w+){0,2}\s*$", prefix, re.IGNORECASE):
- continue
+ if any(not negated_before(text, match.start()) for match in pattern.finditer(text)):
return True
- return False
+ return removes_dataset_case(text, case_ids)
def _normalized_text(text: object) -> str:
@@ -581,6 +592,7 @@ class InsightsJudge(LLMClient):
raise ValueError(f"Expected JSON object, got {type(data).__name__}")
canonical = kwargs.get("canonical") if isinstance(kwargs.get("canonical"), dict) else {}
grounding = _negative_control_grounding(canonical)
+ known_case_ids = tuple(grounding[1])
# Each entry names the model that wrote it, which on the native
# Anthropic API can be the server-side fallback.
provenance = {"model": safe_llm_model(self.model_used)} if self.model_used else {}
@@ -619,7 +631,7 @@ class InsightsJudge(LLMClient):
message = (item.get("message") or "").strip()
if not title and not message:
continue
- if _weakens_requirements(title) or _weakens_requirements(message):
+ if _weakens_requirements(title, known_case_ids) or _weakens_requirements(message, known_case_ids):
continue
if (key := _normalized_text(message or title)) in seen_messages:
continue
diff --git a/src/skillevaluator/evaluation/tier3_report.py b/src/skillevaluator/evaluation/tier3_report.py
index c4e49c64..69960749 100644
--- a/src/skillevaluator/evaluation/tier3_report.py
+++ b/src/skillevaluator/evaluation/tier3_report.py
@@ -4396,7 +4396,8 @@ def _apply_case_findings(
if part
),
"severity": "fail" if fix["kind"] == "skill hurts" else "warn",
- "source": "deterministic",
+ "source": fix.get("source") or "deterministic",
+ **({"fallback_reason": fix["fallback_reason"]} if fix.get("fallback_reason") else {}),
}
for fix in fixes
]
diff --git a/src/skillevaluator/tier3/case_summary.py b/src/skillevaluator/tier3/case_summary.py
index 106195da..e711d784 100644
--- a/src/skillevaluator/tier3/case_summary.py
+++ b/src/skillevaluator/tier3/case_summary.py
@@ -21,6 +21,7 @@ from rich.box import SIMPLE
from rich.table import Table
from rich.text import Text
+from skillevaluator.evaluation.insights_judge import _weakens_requirements
from skillevaluator.reporting.summary_copy import pass_at_k_summary
from skillevaluator.reporting.terminal_styles import AMBER
from skillevaluator.tier3.attempt_verdict import attempt_verdict, scored_goal_accuracy
@@ -1202,8 +1203,9 @@ def _llm_actions(
prompt = (
"You help a skill author fix an agent skill after a live evaluation. For each issue give ONE concrete fix, "
"at most 35 words, naming the file and what to change (SKILL.md section, a script, "
- "evals/environment/Dockerfile, or the eval case in evals/evals.json if the case expects something the skill "
- "does not provide). For [skill not used] issues the fix belongs in the SKILL.md frontmatter description, "
+ "or evals/environment/Dockerfile). Preserve the dataset expectations and assertions: do not remove, "
+ "loosen or lower them to make a failing case pass, even if the skill currently lacks the feature. "
+ "For [skill not used] issues the fix belongs in the SKILL.md frontmatter description, "
"which is all the agent sees when deciding whether to load the skill. "
"Write fixes that cover the whole class of requests; do not copy names or details from this one case. "
"Base it only on the evidence; do not invent features.\n\n"
@@ -1334,10 +1336,28 @@ def what_to_fix(
if issue["kind"] not in ("run error", "eval case", "missing input", "skill not used")
and not issue.get("asks_for_input")
]
+ case_ids = {*dataset, *_case_ids(agents)}
actions = _llm_actions(actionable, skill_path, dataset) if use_llm and actionable else None
by_id = dict(zip((id(i) for i in actionable), actions or [], strict=False))
+ actionable_ids = {id(issue) for issue in actionable}
for issue in issues:
- issue["action"] = issue.get("action") or by_id.get(id(issue)) or _fallback_action(issue)
+ existing_action = issue.get("action")
+ action = existing_action or by_id.get(id(issue))
+ if action and not _weakens_requirements(action, case_ids):
+ issue["action"] = action
+ issue["source"] = "deterministic" if existing_action else "llm"
+ else:
+ if action:
+ issue["fallback_reason"] = "requirement_weakening"
+ elif use_llm and id(issue) in actionable_ids:
+ issue["fallback_reason"] = "llm_unavailable_or_invalid"
+ fallback = _fallback_action(issue)
+ issue["action"] = (
+ "Update SKILL.md or the skill's scripts to meet the existing case requirements, then re-run the case."
+ if _weakens_requirements(fallback, case_ids)
+ else fallback
+ )
+ issue["source"] = "deterministic"
secrets = secret_values_from_environment(os.environ)
return [
{
diff --git a/tests/evaluation/test_judges.py b/tests/evaluation/test_judges.py
index 007d05ac..d9dcd32f 100644
--- a/tests/evaluation/test_judges.py
+++ b/tests/evaluation/test_judges.py
@@ -11,6 +11,7 @@ network access.
from __future__ import annotations
import json
+import time
import pytest
@@ -21,6 +22,7 @@ from skillevaluator.evaluation import (
compute_dimensions,
compute_dimensions_deterministic,
)
+from skillevaluator.evaluation.insights_judge import _weakens_requirements
from skillevaluator.evaluation.tier3_report import _normalize_trials
from skillevaluator.tier3.dataset_utils import normalize_dataset_entries
from skillevaluator.tier3.harbor.metrics import DIMENSION_DEFINITIONS
@@ -851,28 +853,120 @@ def test_regressed_dimension_reasoning_names_the_weakest_signal() -> None:
assert improved[-1] == "Strongest signal: behavior_check 0.81."
+# Verb modifiers a negator may govern: "without first deleting", "without ever having to delete the case".
+_NEGATION_CHAINS = (
+ "",
+ "first",
+ "outright",
+ "ever",
+ "even",
+ "actually",
+ "simply",
+ "just",
+ "also",
+ "then",
+ "again",
+ "entirely",
+ "completely",
+ "fully",
+ "altogether",
+ "necessarily",
+ "directly",
+ "having to",
+ "needing to",
+ "need to",
+ "the need to",
+ "having to first",
+ "being forced to",
+ "being asked to",
+ "trying to",
+ "attempting to",
+ "ever having to",
+ "ever even having to first",
+)
+# (negator, whether the verb after it is in its base form): "without first deleting", "do not first delete".
+_STRICT_NEGATORS = (
+ ("without", False),
+ ("instead of", False),
+ ("rather than", False),
+ ("there is no need to", True),
+ ("should not", True),
+)
+_ALL_NEGATORS = (*_STRICT_NEGATORS, ("do not", True), ("never", True), ("don't", True), ("avoid", False))
+
+
+# "not only deleting the case but also fixing the parser" recommends the deletion, so after "not", "do not",
+# "should not", "never", "don't" or "avoid" these words void the negation. "without simply removing the case" and
+# "instead of just dropping it" are different: there the negator governs the verb.
+_ADDITIVE_WORDS = ("only", "just", "merely", "simply", "solely", "exclusively", "purely")
+_ADDITIVE_PREFIXES = (*_ADDITIVE_WORDS, "necessarily just", "necessarily only", "even just")
+_NOT_NEGATORS = (
+ ("not", False),
+ ("do not", True),
+ ("should not", True),
+ ("never", True),
+ ("don't", True),
+ ("avoid", False),
+)
+
+
+def _negated_advice(negators: tuple[tuple[str, bool], ...], case_subject: str) -> list[str]:
+ sentences = []
+ for negator, base_form in negators:
+ for chain in _NEGATION_CHAINS:
+ if negator in dict(_NOT_NEGATORS) and set(chain.split()) & set(_ADDITIVE_WORDS):
+ continue
+ base = base_form or chain.endswith(" to")
+ for verb, subject in (("delete", case_subject), ("lower", "the pass threshold")):
+ form = verb if base else verb.replace("lower", "lowering").replace("delete", "deleting")
+ sentences.append(" ".join(f"Fix the parser {negator} {chain} {form} {subject}.".split()))
+ return sentences
+
+
+def _unrelated_negation_advice(negators: tuple[tuple[str, bool], ...], case_subject: str) -> list[str]:
+ """A noun between the negator (and its modifiers) and the verb: the negator governs the noun, not the removal."""
+ return [
+ " ".join(f"Fix the parser {negator} {chain} {noun} by {verb} {subject}.".split())
+ for negator, _ in negators
+ for chain in _NEGATION_CHAINS
+ for noun in ("failures", "errors")
+ for verb, subject in (("deleting", case_subject), ("lowering", "the pass threshold"))
+ ]
+
+
+def _additive_advice(case_subject: str) -> list[str]:
+ """Additive wording ("not only lowering the threshold but also fixing the parser") still recommends the removal."""
+ sentences = []
+ for negator, base_form in _NOT_NEGATORS:
+ for additive in _ADDITIVE_PREFIXES:
+ for verb, subject in (("delete", case_subject), ("lower", "the pass threshold")):
+ form = verb if base_form else verb.replace("lower", "lowering").replace("delete", "deleting")
+ sentences.append(f"Fix the parser and {negator} {additive} {form} {subject} but also add a test.")
+ return sentences
+
+
class TestInsightsJudgeRecommendationQuality:
@staticmethod
- def _recommendation(message: str, title: str = "Next step") -> dict:
+ def _recommendation(message: str, title: str = "Next step", case_id: str = "positive-001") -> dict:
return {
"claim_type": "general",
"category": "Improve",
"title": title,
"message": message,
"severity": "warn",
- "evidence_case_ids": ["positive-001"],
+ "evidence_case_ids": [case_id],
}
- def _parse(self, *messages: str, deterministic: dict | None = None) -> list[str]:
+ def _parse(self, *messages: str, deterministic: dict | None = None, case_id: str = "positive-001") -> list[str]:
response = json.dumps(
{
"conclusions": [],
- "recommendations": [self._recommendation(message) for message in messages],
+ "recommendations": [self._recommendation(message, case_id=case_id) for message in messages],
}
)
parsed = InsightsJudge().parse_response(
response,
- canonical={"dataset": [{"id": "positive-001", "expected_skill": "calculator"}], "trials": []},
+ canonical={"dataset": [{"id": case_id, "expected_skill": "calculator"}], "trials": []},
deterministic=deterministic or {},
)
return [item["message"] for item in parsed["recommendations"]]
@@ -895,6 +989,442 @@ class TestInsightsJudgeRecommendationQuality:
message = "Make the expected output of positive-001 explicit in evals.json."
assert self._parse(message) == [message]
+ @pytest.mark.parametrize(
+ "message",
+ [
+ "Remove the expected output mismatch by deleting the positive-001 case from evals/evals.json.",
+ "Remove the assertion failure by dropping the `positive-001` case from evals/evals.json.",
+ "Remove the expected output mismatch by implementing the parser, then delete the positive-001 case.",
+ "Fix the mismatch by removing the positive-001 entry from evals/evals.json.",
+ "Remove the expected output mismatch by deleting the positive-001 test case.",
+ "Remove the failure by commenting out the positive-001 case in evals/evals.json.",
+ "Remove the mismatch by disabling the dataset entry for positive-001.",
+ "Delete the positive-001 row from the dataset.",
+ "Skip the positive-001 case in evals/evals.json.",
+ "The positive-001 case should be deleted from evals/evals.json.",
+ "The positive-001 entry in evals/evals.json should be removed.",
+ "The positive-001 test case must be dropped.",
+ "Do not remove the expectation; delete the positive-001 case from the dataset.",
+ "Delete the positive-001 case from evals/evals.json.",
+ "Remove the test case for positive-001 from the dataset.",
+ "Drop the eval case for positive-001.",
+ "The test case should be deleted so positive-001 stops failing.",
+ "Remove the special case for positive-001 from evals/evals.json.",
+ "In evals/evals.json, remove the special case for positive-001.",
+ "In the dataset, delete the edge case for positive-001.",
+ "The special case in evals/evals.json should be removed so positive-001 passes.",
+ ],
+ )
+ def test_drops_advice_that_deletes_a_dataset_case(self, message: str) -> None:
+ assert self._parse(message) == []
+
+ @pytest.mark.parametrize(
+ "message",
+ [
+ "Remove the special case in parser.py for positive-001 and implement the general rounding algorithm.",
+ "Remove the test case failure for positive-001 by implementing the rounding rule in SKILL.md",
+ "Remove the special case in the script for positive-001 and implement the general rounding algorithm.",
+ "Delete the default case branch in parser.py; implement the rounding rule positive-001 needs.",
+ "Remove the base case in the recursive function that positive-001 exercises.",
+ "Remove the corner case for zero in parser.py, which positive-001 hits.",
+ "Remove the error case in parser.py and handle positive-001 generally.",
+ "Remove the case failure in positive-001 by implementing the rounding rule in SKILL.md.",
+ "Remove the case mismatch in positive-001 by implementing the rounding rule in SKILL.md.",
+ "Remove the test case violation in positive-001 by implementing the rounding rule in SKILL.md.",
+ "The positive-001 test case failure should be removed by implementing the rounding rule in SKILL.md.",
+ "The special case in parser.py that positive-001 hits should be removed.",
+ "Never remove the positive-001 test case; fix the rounding step instead.",
+ "Fix the failing positive-001 test case by removing the special case in parser.py.",
+ ],
+ )
+ def test_keeps_advice_that_removes_implementation_cases_and_case_failures(self, message: str) -> None:
+ assert self._parse(message) == [message]
+
+ @pytest.mark.parametrize(
+ "case_id,message",
+ [
+ ("positive-001", "Delete the case for positive-001 so it stops failing."),
+ ("positive-001", "Remove the case named positive-001 so the run passes."),
+ ("positive-001", "Delete case positive-001."),
+ ("positive-001", "Delete test case positive-001."),
+ ("positive-001", "Fix the mismatch by deleting the case for positive-001."),
+ ("positive-001", "Delete the entry for positive-001."),
+ ("positive-001", "Remove positive-001 from evals/evals.json."),
+ ("negative-007", "Remove the case for `negative-007`"),
+ ("negative-007", 'Drop the "negative-007" case.'),
+ ("syllable-count", "Delete the syllable-count case."),
+ ("syllable-count", "Remove the case for syllable-count."),
+ ("smoke", "Delete the smoke case."),
+ ("smoke", "Remove the case for smoke so it stops failing."),
+ ("positive-001", "The case for positive-001 should be deleted."),
+ ("positive-001", "The positive-001 entry should be deleted."),
+ ("positive-001", "The case for positive-001 in evals/evals.json can be removed."),
+ ("syllable-count", "The syllable-count case should be removed."),
+ ("smoke", "The smoke case must be dropped."),
+ ("positive-001", "Exclude the case for positive-001 from the run."),
+ ("positive-001", "Omit the case for positive-001."),
+ ("positive-001", "Get rid of the positive-001 case."),
+ ("positive-001", "Take out the case for positive-001."),
+ ("positive-001", "Take positive-001 out of the dataset."),
+ ("positive-001", "Take the positive-001 case out of evals/evals.json."),
+ ("syllable-count", "Leave out the syllable-count case."),
+ ("smoke", "Leave the smoke case out of the dataset."),
+ ("positive-001", "Filter out the case for positive-001."),
+ ("positive-001", "Filter positive-001 out of evals/evals.json."),
+ ("smoke", "Discard the smoke case."),
+ ("positive-001", "Strip the case for positive-001 from evals/evals.json."),
+ ("positive-001", "Ignore the case for positive-001."),
+ ("smoke", "Ignore the smoke case."),
+ ("positive-001", "Stop running the case for positive-001."),
+ ("syllable-count", "Retire the syllable-count case."),
+ ("positive-001", "Replace the positive-001 case with an easier one."),
+ ("positive-001", "Replace the case for positive-001 in evals/evals.json with a simpler one."),
+ ("positive-001", "Replace positive-001 with a simpler case."),
+ ("smoke", "Swap out the smoke case for an easier one."),
+ ("positive-001", "Instead of keeping the case for positive-001, move on to the next one."),
+ ("positive-001", "The positive-001 case should be excluded."),
+ ("positive-001", "The case for positive-001 should be replaced with an easier one."),
+ ("syllable-count", "The syllable-count case should be discarded."),
+ ("smoke", "The smoke case can be left out."),
+ ("positive-001", "The case for positive-001 should be ignored."),
+ ("positive-001", "The positive-001 case should be taken out of the dataset."),
+ ("syllable-count", "The syllable-count case must be filtered out."),
+ ("positive-001", "The case for positive-001 can be stripped from evals/evals.json."),
+ ("smoke", "The smoke case should be retired."),
+ ],
+ )
+ def test_drops_advice_that_deletes_a_dataset_case_named_by_its_id(self, case_id: str, message: str) -> None:
+ assert self._parse(message, case_id=case_id) == []
+
+ @pytest.mark.parametrize(
+ "case_id,message",
+ [
+ ("positive-001", "Remove the special case in parser.py for positive-001 and implement the general rule."),
+ ("positive-001", "Remove the special case for positive-001 in parser.py and implement the general rule."),
+ ("positive-001", "Remove the case for positive-001 from analyze_text.py and implement the general rule."),
+ ("syllable-count", "Remove the syllable-count case in analyze_text.py and implement it generally."),
+ ("smoke", "Remove the smoke case in analyze_text.py and implement it generally."),
+ ("positive-001", "Remove the test case failure for positive-001 by implementing the rule in SKILL.md"),
+ ("positive-001", "Do not delete the case for positive-001; implement the rounding rule in SKILL.md."),
+ ("syllable-count", "Never remove the syllable-count case; implement the counting rule in SKILL.md."),
+ ("smoke", "Do not drop the smoke case; implement the missing step in SKILL.md."),
+ ("positive-001", "Remove the special case for zero-length input in parser.py, which positive-001 hits."),
+ (
+ "positive-001",
+ "Remove the special case for zero-length input and implement the rule positive-001 needs.",
+ ),
+ ],
+ )
+ def test_keeps_advice_that_names_a_case_id_but_edits_the_implementation(self, case_id: str, message: str) -> None:
+ assert self._parse(message, case_id=case_id) == [message]
+
+ @pytest.mark.parametrize(
+ "case_id,message",
+ [
+ ("positive-001", "Do not exclude the case for positive-001; implement the rule in SKILL.md."),
+ (
+ "positive-001",
+ "Never replace the case for positive-001 with an easier one; implement the rule in SKILL.md.",
+ ),
+ ("smoke", "Never replace the smoke case; implement the missing step in SKILL.md."),
+ ("syllable-count", "Don't ignore the syllable-count case; implement the counting rule in SKILL.md."),
+ ("positive-001", "Do not take positive-001 out of the dataset; implement the rule in SKILL.md."),
+ ("positive-001", "Instead of deleting the case for positive-001, implement the rounding rule in SKILL.md."),
+ ("syllable-count", "Instead of removing the syllable-count case, implement the counting rule in SKILL.md."),
+ ("smoke", "Implement the missing step in SKILL.md instead of dropping the smoke case."),
+ ("positive-001", "Fix the rounding step rather than excluding the case for positive-001."),
+ ("positive-001", "Make positive-001 pass without deleting the case for positive-001."),
+ ("positive-001", "Discard the special case in parser.py for positive-001 and implement the general rule."),
+ ("syllable-count", "Exclude the syllable-count case in analyze_text.py and implement it generally."),
+ ("positive-001", "Take the special case for positive-001 out of parser.py and implement the general rule."),
+ ("smoke", "Retire the smoke case from analyze_text.py and implement it generally."),
+ ("positive-001", "Discard the test case failure for positive-001 by implementing the rule in SKILL.md."),
+ ("positive-001", "Ignore case when comparing units, as positive-001 expects."),
+ ("positive-001", "Strip whitespace from the answer, as positive-001 expects."),
+ ("positive-001", "Take the case for positive-001 into account in SKILL.md."),
+ ("positive-001", "Replace the special case for positive-001 in parser.py with the general rounding rule."),
+ ],
+ )
+ def test_keeps_negated_instead_of_and_implementation_advice_for_the_wider_verbs(
+ self, case_id: str, message: str
+ ) -> None:
+ assert self._parse(message, case_id=case_id) == [message]
+
+ @pytest.mark.parametrize(
+ "case_id,message",
+ [
+ ("positive-001", "Run without failures by deleting the positive-001 case from evals/evals.json."),
+ ("positive-001", "Make the run pass without failures by dropping the case for positive-001."),
+ ("smoke", "Get a green run without extra work by excluding the smoke case."),
+ ("positive-001", "Rather than a fix, delete the case for positive-001."),
+ ("positive-001", "Instead of the old approach, delete the case for positive-001 from the dataset."),
+ ],
+ )
+ def test_drops_a_removal_when_the_negation_word_governs_something_else(self, case_id: str, message: str) -> None:
+ assert self._parse(message, case_id=case_id) == []
+
+ @pytest.mark.parametrize(
+ "case_id,message",
+ [
+ ("positive-001", "Make positive-001 pass without deleting the case for positive-001."),
+ ("positive-001", "Make positive-001 pass without simply removing the case for positive-001."),
+ ("positive-001", "Fix the rounding step for positive-001 without ever dropping its case."),
+ ("syllable-count", "Implement the counting rule instead of just dropping the syllable-count case."),
+ ("smoke", "Fix the missing step in SKILL.md rather than excluding the smoke case."),
+ ],
+ )
+ def test_keeps_advice_whose_negation_word_governs_the_removal_verb(self, case_id: str, message: str) -> None:
+ assert self._parse(message, case_id=case_id) == [message]
+
+ @pytest.mark.parametrize(
+ "case_id,message",
+ [
+ ("smoke", "Implement the missing field without having to delete the smoke case."),
+ ("smoke", "Fix the parser without first deleting the smoke case."),
+ ("smoke", "Fix the parser without outright deleting the smoke case."),
+ ("smoke", "Fix the parser without ever having to delete the smoke case."),
+ ("smoke", "Fix the parser without even needing to drop the smoke case."),
+ ("smoke", "Fix the parser without being forced to exclude the smoke case."),
+ ("smoke", "Fix the parser instead of first deleting the smoke case."),
+ ("smoke", "Fix the parser rather than simply dropping the smoke case."),
+ ("smoke", "Do not first delete the smoke case; fix the parser."),
+ ("smoke", "There is no need to delete the smoke case once the parser is fixed."),
+ ("smoke", "You should not delete the smoke case; fix the parser."),
+ ("positive-001", "Fix the parser without trying to delete the case for positive-001."),
+ ("smoke", "Get smoke to pass without first lowering the pass threshold."),
+ ("smoke", "Fix the parser so smoke passes without having to loosen the expected output."),
+ ("smoke", "Make smoke pass without ever relaxing the assertions."),
+ ("smoke", "Make smoke pass instead of outright softening the ground truth."),
+ ],
+ )
+ def test_keeps_advice_whose_negator_governs_the_verb_through_modifiers(self, case_id: str, message: str) -> None:
+ assert self._parse(message, case_id=case_id) == [message]
+
+ @pytest.mark.parametrize(
+ "case_id,message",
+ [
+ ("smoke", "Run without failures by deleting the smoke case from evals/evals.json."),
+ ("smoke", "Run without errors by removing the smoke case from evals/evals.json."),
+ ("positive-001", "Pass without errors by removing the positive-001 case from evals/evals.json."),
+ ("smoke", "Pass without regressions by dropping the smoke case."),
+ ("smoke", "Fix the parser without delay, deleting the smoke case."),
+ ("smoke", "Fix the parser without having to wait, delete the smoke case."),
+ ("smoke", "Fix the parser without ever fixing the parser by deleting the smoke case."),
+ ("smoke", "Fix the parser without first fixing the parser. Delete the smoke case."),
+ ("smoke", "Get smoke to pass without failures by lowering the pass threshold."),
+ ("smoke", "Get smoke to pass without errors by loosening the expected output."),
+ ("smoke", "Get smoke to pass without first fixing the parser, then relax the assertions."),
+ ],
+ )
+ def test_drops_a_removal_that_a_negator_with_modifiers_does_not_govern(self, case_id: str, message: str) -> None:
+ assert self._parse(message, case_id=case_id) == []
+
+ @pytest.mark.parametrize("known", [("smoke",), ()])
+ def test_a_negator_and_its_verb_modifiers_negate_the_removal_and_the_weakening(self, known: tuple) -> None:
+ subject = "the smoke case" if known else "the case for positive-001"
+ offenders = [s for s in _negated_advice(_ALL_NEGATORS, subject) if _weakens_requirements(s, known)]
+ assert offenders == []
+
+ @pytest.mark.parametrize("known", [("smoke",), ()])
+ def test_a_noun_between_the_negator_and_the_verb_breaks_the_negation(self, known: tuple) -> None:
+ subject = "the smoke case from evals/evals.json" if known else "the case for positive-001 from evals/evals.json"
+ missed = [
+ s for s in _unrelated_negation_advice(_STRICT_NEGATORS, subject) if not _weakens_requirements(s, known)
+ ]
+ assert missed == []
+
+ @pytest.mark.parametrize(
+ "case_id,message",
+ [
+ ("smoke", "Make smoke pass by not only lowering the pass threshold but also fixing the parser."),
+ ("smoke", "Not only delete the smoke case but also fix the parser."),
+ ("smoke", "Make smoke pass by not just lowering the pass threshold but also fixing the parser."),
+ ("smoke", "Make smoke pass by not merely relaxing the assertions but also fixing the parser."),
+ ("smoke", "Make smoke pass by not simply loosening the expected output but also fixing the parser."),
+ ("smoke", "Make smoke pass by not solely lowering the pass threshold but also fixing the parser."),
+ ("smoke", "Make smoke pass by not exclusively softening the ground truth but also fixing the parser."),
+ (
+ "smoke",
+ "Make smoke pass by not necessarily just lowering the pass threshold but also fixing the parser.",
+ ),
+ (
+ "smoke",
+ "Make smoke pass by not necessarily only lowering the pass threshold but also fixing the parser.",
+ ),
+ ("smoke", "Fix the parser by not only dropping the smoke case but also adding a unit test."),
+ ("smoke", "Not just removing the smoke case but also fixing the parser."),
+ ("smoke", "Fix the parser by not merely excluding the smoke case but also adding a unit test."),
+ ("positive-001", "Not simply delete the case for positive-001 but also fix the parser."),
+ ("smoke", "Do not only lower the pass threshold for smoke but also fix the parser."),
+ ("smoke", "Do not just delete the smoke case, fix the parser."),
+ ("smoke", "Don't simply delete the smoke case; fix the parser."),
+ ],
+ )
+ def test_drops_additive_wording_that_still_recommends_the_removal(self, case_id: str, message: str) -> None:
+ assert self._parse(message, case_id=case_id) == []
+
+ @pytest.mark.parametrize(
+ "case_id,message",
+ [
+ ("smoke", "You should not lower the pass threshold for smoke; fix the parser."),
+ ("smoke", "You must not delete the smoke case; fix the parser."),
+ ("smoke", "Do not lower the pass threshold for smoke."),
+ ("smoke", "Do not delete the smoke case but also fix the parser."),
+ ("smoke", "Fix the parser and do not delete the smoke case."),
+ ("smoke", "Fix the parser; it should not drop the smoke case."),
+ ("smoke", "Fix the parser without simply removing the smoke case."),
+ ("smoke", "Fix the parser instead of just dropping the smoke case."),
+ ("smoke", "Fix the parser rather than merely excluding the smoke case."),
+ ],
+ )
+ def test_keeps_genuine_prohibitions_next_to_additive_wording(self, case_id: str, message: str) -> None:
+ assert self._parse(message, case_id=case_id) == [message]
+
+ @pytest.mark.parametrize("known", [("smoke",), ()])
+ def test_additive_not_only_wording_does_not_negate_the_removal_or_the_weakening(self, known: tuple) -> None:
+ subject = "the smoke case" if known else "the case for positive-001"
+ missed = [s for s in _additive_advice(subject) if not _weakens_requirements(s, known)]
+ assert missed == []
+
+ def test_negation_is_read_from_a_bounded_window_before_the_verb(self) -> None:
+ padding = "Fix the parser. " * 2000
+ assert not _weakens_requirements(padding + "Without ever having to first delete the smoke case.", ("smoke",))
+ assert _weakens_requirements(
+ padding + "Without ever having to first fix it, delete the smoke case.", ("smoke",)
+ )
+
+ @pytest.mark.parametrize(
+ "message",
+ [
+ "Remove the special case for UTF-8 and use the standard decoder for positive-001.",
+ "Drop the special case for ISO-8859 input and use the standard decoder for positive-001.",
+ "Delete the utf-16 case and decode with the standard library for positive-001.",
+ "Remove the UNIX case and normalise line endings generally for positive-001.",
+ "Remove the v2-api case and call the current endpoint for positive-001.",
+ "Remove the hard-coded case and compute the answer for positive-001 generally.",
+ "Exclude the base-64 case from the encoder and handle positive-001 with the standard library.",
+ ],
+ )
+ def test_keeps_advice_that_names_an_identifier_that_is_not_a_known_case_id(self, message: str) -> None:
+ assert self._parse(message) == [message]
+
+ @pytest.mark.parametrize(
+ "message",
+ [
+ "Delete the utf-16 case from evals/evals.json, then document what positive-001 expects.",
+ "Remove the special case for UTF-8 from the dataset and document what positive-001 expects.",
+ ],
+ )
+ def test_drops_an_unlisted_identifier_when_the_sentence_points_at_the_dataset(self, message: str) -> None:
+ assert self._parse(message) == []
+
+ def test_identifier_shaped_names_count_only_without_case_ids_or_with_a_dataset_location(self) -> None:
+ assert _weakens_requirements("Delete the syllable-count case.")
+ assert _weakens_requirements("Delete the syllable-count case.", ())
+ assert not _weakens_requirements("Delete the syllable-count case.", ("positive-001",))
+ assert _weakens_requirements("Delete the syllable-count case.", ("syllable-count",))
+ assert _weakens_requirements("Delete the syllable-count case from evals.json.", ("positive-001",))
+
+ def test_case_ids_that_are_not_single_words_are_matched_by_name(self) -> None:
+ ids = ("edge one", "positive-001")
+ assert _weakens_requirements("Delete the edge one case.", ids)
+ assert _weakens_requirements("Remove the case for edge one so it stops failing.", ids)
+ assert not _weakens_requirements("Remove the edge case in parser.py and handle edge one generally.", ids)
+
+ def test_a_list_of_case_ids_without_a_case_noun_is_checked_in_bounded_time(self) -> None:
+ ids = tuple(f"case-{i:02d}" for i in range(20))
+ text = "Remove the " + ", ".join(ids) + " overrides in parser.py and implement the general rule."
+ for known in (ids, ()):
+ started = time.perf_counter()
+ assert _weakens_requirements(text, known) is False
+ assert time.perf_counter() - started < 1.0
+
+ def test_long_adversarial_actions_are_checked_in_bounded_time(self) -> None:
+ ids = tuple(f"case-{i:03d}" for i in range(250))
+ listed = ", ".join(ids[:150])
+ hyphenated = " ".join(["a-b-c-d-e-f-g"] * 700)
+ adversarial = {
+ "list without a noun": (f"Remove the {listed} overrides in parser.py and implement the rule.", False),
+ "list before the noun": (f"Delete the {listed} cases.", True),
+ "repeated hyphen words": (f"Remove {hyphenated} case", False),
+ "repeated lists": (("Remove the foo-bar, " * 300) + "baz-qux case", False),
+ "repeated removals": ("Delete the case for case-001 and exclude the case " * 100, True),
+ "removal verbs only": ("remove delete drop skip exclude discard omit " * 150, False),
+ "nouns only": ("case cases entry row example test case " * 150, False),
+ "locations only": ("in evals.json from the dataset of benchmark out of the test set " * 100, False),
+ "one long identifier": ("remove the " + "positive-" * 600 + "x case", False),
+ "requirement nouns": ("expectations are " * 400, False),
+ "negated removals": ("Do not delete the case for case-001 and " * 300, False),
+ "negators and modifiers without a verb": ("without ever having to first simply " * 300, False),
+ }
+ for label, (text, expected) in adversarial.items():
+ for known in (ids, ()):
+ started = time.perf_counter()
+ result = _weakens_requirements(text, known)
+ elapsed = time.perf_counter() - started
+ assert elapsed < 1.0, f"{label} took {elapsed:.2f}s with {len(known)} case ids"
+ if known:
+ assert result is expected, label
+
+ def test_parse_checks_a_list_of_known_case_ids_in_bounded_time(self) -> None:
+ ids = [f"case-{i:03d}" for i in range(250)]
+ dataset = [{"id": case_id, "expected_skill": "calculator"} for case_id in ids]
+ cited = ids[:10]
+ benign = f"Remove the {', '.join(cited)} overrides in parser.py and implement the general rule."
+ removal = f"Delete the {', '.join(cited)} cases from evals/evals.json."
+
+ def recommendation(message: str) -> dict:
+ return {**self._recommendation(message), "evidence_case_ids": cited}
+
+ response = json.dumps({"conclusions": [], "recommendations": [recommendation(benign), recommendation(removal)]})
+ started = time.perf_counter()
+ parsed = InsightsJudge().parse_response(response, canonical={"dataset": dataset, "trials": []})
+ assert time.perf_counter() - started < 1.0
+ assert [item["message"] for item in parsed["recommendations"]] == [benign]
+
+ @pytest.mark.parametrize("case_id", ["smoke", "syllable-count"])
+ def test_drops_a_named_dataset_case_removal_in_the_title_even_when_the_message_is_benign(
+ self, case_id: str
+ ) -> None:
+ message = f"Implement the missing step that {case_id} exercises in SKILL.md."
+ recommendation = self._recommendation(message, title=f"Delete the {case_id} case", case_id=case_id)
+ response = json.dumps({"conclusions": [], "recommendations": [recommendation]})
+ parsed = InsightsJudge().parse_response(
+ response,
+ canonical={"dataset": [{"id": case_id, "expected_skill": "calculator"}], "trials": []},
+ )
+ assert parsed["recommendations"] == []
+
+ @pytest.mark.parametrize(
+ "message",
+ [
+ "Remove the expected output mismatch in positive-001 by implementing the parser in SKILL.md.",
+ "Remove the assertion failure in positive-001 by implementing the parser in SKILL.md.",
+ "Remove the `expected output` mismatch in positive-001 by implementing the parser in SKILL.md.",
+ "Remove the expected output mismatch by fixing the parser; keep the positive-001 case in evals/evals.json.",
+ "Do not delete the case; document the rounding rule that positive-001 depends on in SKILL.md.",
+ "Never remove the positive-001 entry from evals/evals.json; fix the rounding step instead.",
+ "In this case, document the rounding rule that positive-001 depends on in SKILL.md.",
+ "The edge case in positive-001 should be handled in SKILL.md.",
+ "Delete the cache entry, then document the rounding rule that positive-001 depends on in SKILL.md.",
+ "Remove temp files per case, then document the rounding rule that positive-001 depends on in SKILL.md.",
+ ],
+ )
+ def test_keeps_mismatch_fixes_negated_removals_and_ordinary_case_prose(self, message: str) -> None:
+ assert self._parse(message) == [message]
+
+ def test_drops_a_dataset_case_removal_in_the_title_even_when_the_message_is_benign(self) -> None:
+ recommendation = self._recommendation(
+ "Implement the rounding rule in SKILL.md.",
+ title="Delete the positive-001 case from evals/evals.json",
+ )
+ response = json.dumps({"conclusions": [], "recommendations": [recommendation]})
+ parsed = InsightsJudge().parse_response(
+ response,
+ canonical={"dataset": [{"id": "positive-001", "expected_skill": "calculator"}], "trials": []},
+ )
+ assert parsed["recommendations"] == []
+
def test_keeps_negated_and_non_requirement_uses_of_weakening_verbs(self) -> None:
messages = (
"Do not lower the pass threshold for positive-001; fix the rounding step instead.",
diff --git a/tests/test_tier3_case_summary.py b/tests/test_tier3_case_summary.py
index 751fd692..230b223b 100644
--- a/tests/test_tier3_case_summary.py
+++ b/tests/test_tier3_case_summary.py
@@ -4,12 +4,13 @@
"""Case-first Tier 3 summary and How-to-improve ranking."""
import json
+import time
from pathlib import Path
import pytest
from rich.console import Console
-from skillevaluator.tier3.case_summary import render_case_summary, verdict, what_to_fix
+from skillevaluator.tier3.case_summary import _weakens_requirements, render_case_summary, verdict, what_to_fix
from skillevaluator.tier3.result_display import render_result
@@ -52,6 +53,644 @@ def _reward(activated: bool, reason: str) -> dict:
}
+ "action",
+ [
+ "In evals/evals.json, remove the expectation for syllable_count from the syllable-positive case.",
+ "Loosen the assertion in evals/evals.json.",
+ "Drop the syllable_count expectation from evals/evals.json.",
+ "The assertion should be removed from the dataset.",
+ "Lower the pass threshold to make this case pass.",
+ "Remove the `expectation` for syllable_count from the dataset.",
+ "Drop the `syllable_count` `expectation` from evals/evals.json.",
+ "Loosen the `assertion` in evals/evals.json.",
+ "The `expectation` should be removed from the dataset.",
+ "The expectation for `syllable_count` should be removed from the dataset.",
+ "The `pass threshold` for the syllable-positive case should be lowered.",
+ "Remove the expected output mismatch by dropping the assertion from evals/evals.json.",
+ "Remove the expected output for syllable_count from evals/evals.json.",
+ "Remove the expected output mismatch by deleting the syllable-positive case from evals/evals.json.",
+ "Remove the assertion failure by dropping the `syllable-positive` case from evals/evals.json.",
+ "Remove the expected output mismatch by implementing syllable_count, then delete the syllable-positive case.",
+ "Fix the mismatch by removing the syllable-positive entry from evals/evals.json.",
+ "Remove the expected output mismatch by deleting the syllable-positive test case.",
+ "Remove the failure by commenting out the syllable-positive case in evals/evals.json.",
+ "Remove the mismatch by disabling the dataset entry for syllable-positive.",
+ "Delete the syllable-positive row from the dataset.",
+ "Skip the syllable-positive case in evals/evals.json.",
+ "The syllable-positive case should be deleted from evals/evals.json.",
+ "The syllable-positive entry in evals/evals.json should be removed.",
+ "The syllable-positive test case must be dropped.",
+ "Do not remove the expectation; delete the syllable-positive case from the dataset.",
+ "Delete the syllable-positive case from evals/evals.json.",
+ "Remove the test case from the dataset.",
+ "Drop the eval case.",
+ "The test case should be deleted.",
+ "Remove the special case from evals/evals.json.",
+ "In evals/evals.json, remove the special case for syllable-positive.",
+ "In the dataset, delete the edge case.",
+ "The special case in evals/evals.json should be removed.",
+ ],
+)
+def test_case_action_rejects_requirement_weakening(tmp_path, monkeypatch, action):
+ from skillevaluator.tier3.eval_core import llm_judge
+
+ _trial(tmp_path, "a", "t0", _reward(True, "Missing syllable_count"))
+ monkeypatch.setattr(llm_judge, "call_public_llm", lambda *_a, **_k: (json.dumps([action]), None))
+ source = {"agents": {"a": _agent({"syllable-positive": _case(False)}, {})}}
+ [fix] = what_to_fix(source, run_dir=tmp_path)
+
+ assert fix["action"] == "Address in SKILL.md: Missing syllable_count"
+ assert fix["source"] == "deterministic"
+ assert fix["fallback_reason"] == "requirement_weakening"
+
+
+ "case_id,action",
+ [
+ ("positive-001", "Delete the case for positive-001 so it stops failing."),
+ ("positive-001", "Remove the case named positive-001 so the run passes."),
+ ("positive-001", "Delete case positive-001."),
+ ("positive-001", "Delete test case positive-001."),
+ ("positive-001", "Fix the mismatch by deleting the case for positive-001."),
+ ("positive-001", "Delete the entry for positive-001."),
+ ("positive-001", "Remove positive-001 from evals/evals.json."),
+ ("negative-007", "Remove the case for `negative-007`"),
+ ("negative-007", 'Drop the "negative-007" case.'),
+ ("syllable-count", "Delete the syllable-count case."),
+ ("syllable-count", "Remove the case for syllable-count."),
+ ("smoke", "Delete the smoke case."),
+ ("smoke", "Remove the case for smoke so it stops failing."),
+ ("positive-001", "The case for positive-001 should be deleted."),
+ ("positive-001", "The positive-001 entry should be deleted."),
+ ("positive-001", "The case for positive-001 in evals/evals.json can be removed."),
+ ("syllable-count", "The syllable-count case should be removed."),
+ ("smoke", "The smoke case must be dropped."),
+ ("positive-001", "Exclude the case for positive-001 from the run."),
+ ("positive-001", "Omit the case for positive-001."),
+ ("positive-001", "Get rid of the positive-001 case."),
+ ("positive-001", "Take out the case for positive-001."),
+ ("positive-001", "Take positive-001 out of the dataset."),
+ ("positive-001", "Take the positive-001 case out of evals/evals.json."),
+ ("syllable-count", "Leave out the syllable-count case."),
+ ("smoke", "Leave the smoke case out of the dataset."),
+ ("positive-001", "Filter out the case for positive-001."),
+ ("positive-001", "Filter positive-001 out of evals/evals.json."),
+ ("smoke", "Discard the smoke case."),
+ ("positive-001", "Strip the case for positive-001 from evals/evals.json."),
+ ("positive-001", "Ignore the case for positive-001."),
+ ("smoke", "Ignore the smoke case."),
+ ("positive-001", "Stop running the case for positive-001."),
+ ("syllable-count", "Retire the syllable-count case."),
+ ("positive-001", "Replace the positive-001 case with an easier one."),
+ ("positive-001", "Replace the case for positive-001 in evals/evals.json with a simpler one."),
+ ("positive-001", "Replace positive-001 with a simpler case."),
+ ("smoke", "Swap out the smoke case for an easier one."),
+ ("positive-001", "Instead of keeping the case for positive-001, move on to the next one."),
+ ("positive-001", "The positive-001 case should be excluded."),
+ ("positive-001", "The case for positive-001 should be replaced with an easier one."),
+ ("syllable-count", "The syllable-count case should be discarded."),
+ ("smoke", "The smoke case can be left out."),
+ ("positive-001", "The case for positive-001 should be ignored."),
+ ("positive-001", "The positive-001 case should be taken out of the dataset."),
+ ("syllable-count", "The syllable-count case must be filtered out."),
+ ("positive-001", "The case for positive-001 can be stripped from evals/evals.json."),
+ ("smoke", "The smoke case should be retired."),
+ ],
+)
+def test_case_action_rejects_deleting_a_dataset_case_named_by_its_id(tmp_path, monkeypatch, case_id, action):
+ from skillevaluator.tier3.eval_core import llm_judge
+
+ _trial(tmp_path, "a", "t0", _reward(True, "Missing syllable_count"))
+ monkeypatch.setattr(llm_judge, "call_public_llm", lambda *_a, **_k: (json.dumps([action]), None))
+ source = {"agents": {"a": _agent({case_id: _case(False)}, {})}}
+ [fix] = what_to_fix(source, run_dir=tmp_path)
+
+ assert fix["action"] == "Address in SKILL.md: Missing syllable_count"
+ assert fix["source"] == "deterministic"
+ assert fix["fallback_reason"] == "requirement_weakening"
+
+
+def test_case_action_rejects_deleting_another_case_named_in_the_dataset_snapshot(tmp_path, monkeypatch):
+ from skillevaluator.tier3.eval_core import llm_judge
+
+ _trial(tmp_path, "a", "t0", _reward(True, "Missing syllable_count"))
+ (tmp_path / "dataset_snapshot.json").write_text(
+ json.dumps({"dataset": [{"id": "syllable-positive", "question": "q"}, {"id": "smoke", "question": "q"}]})
+ )
+ action = "Delete the smoke case so syllable-positive stops failing."
+ monkeypatch.setattr(llm_judge, "call_public_llm", lambda *_a, **_k: (json.dumps([action]), None))
+ [fix] = what_to_fix({"agents": {"a": _agent({"syllable-positive": _case(False)}, {})}}, run_dir=tmp_path)
+
+ assert fix["action"] == "Address in SKILL.md: Missing syllable_count"
+ assert fix["fallback_reason"] == "requirement_weakening"
+
+
+ "case_id,action",
+ [
+ ("positive-001", "Remove the special case in parser.py for positive-001 and implement the general rule."),
+ ("positive-001", "Remove the special case for positive-001 in parser.py and implement the general rule."),
+ ("positive-001", "Remove the case for positive-001 from analyze_text.py and implement the general rule."),
+ ("syllable-count", "Remove the syllable-count case in analyze_text.py and implement it generally."),
+ ("smoke", "Remove the smoke case in analyze_text.py and implement it generally."),
+ ("positive-001", "Remove the test case failure for positive-001 by implementing the rule in SKILL.md"),
+ ("positive-001", "Do not delete the case for positive-001; implement syllable_count in analyze_text.py."),
+ ("syllable-count", "Never remove the syllable-count case; implement syllable_count in analyze_text.py."),
+ ("smoke", "Do not drop the smoke case; implement syllable_count in analyze_text.py."),
+ ("positive-001", "Remove the special case for zero-length input in analyze_text.py."),
+ ("positive-001", "Remove the special case for zero-length input and implement syllable_count generally."),
+ ],
+)
+def test_case_action_keeps_advice_that_names_a_case_id_but_edits_the_implementation(
+ tmp_path, monkeypatch, case_id, action
+):
+ from skillevaluator.tier3.eval_core import llm_judge
+
+ _trial(tmp_path, "a", "t0", _reward(True, "Missing syllable_count"))
+ monkeypatch.setattr(llm_judge, "call_public_llm", lambda *_a, **_k: (json.dumps([action]), None))
+ source = {"agents": {"a": _agent({case_id: _case(False)}, {})}}
+ [fix] = what_to_fix(source, run_dir=tmp_path)
+
+ assert fix["action"] == action
+ assert fix["source"] == "llm"
+ assert "fallback_reason" not in fix
+
+
+ "case_id,action",
+ [
+ ("positive-001", "Do not exclude the case for positive-001; implement the rule in SKILL.md."),
+ ("positive-001", "Never replace the case for positive-001 with an easier one; implement the rule in SKILL.md."),
+ ("smoke", "Never replace the smoke case; implement the missing step in SKILL.md."),
+ ("syllable-count", "Don't ignore the syllable-count case; implement the counting rule in SKILL.md."),
+ ("positive-001", "Do not take positive-001 out of the dataset; implement the rule in SKILL.md."),
+ ("positive-001", "Instead of deleting the case for positive-001, implement the rounding rule in SKILL.md."),
+ ("syllable-count", "Instead of removing the syllable-count case, implement the counting rule in SKILL.md."),
+ ("smoke", "Implement the missing step in SKILL.md instead of dropping the smoke case."),
+ ("positive-001", "Fix the rounding step rather than excluding the case for positive-001."),
+ ("positive-001", "Make positive-001 pass without deleting the case for positive-001."),
+ ("positive-001", "Discard the special case in parser.py for positive-001 and implement the general rule."),
+ ("syllable-count", "Exclude the syllable-count case in analyze_text.py and implement it generally."),
+ ("positive-001", "Take the special case for positive-001 out of parser.py and implement the general rule."),
+ ("smoke", "Retire the smoke case from analyze_text.py and implement it generally."),
+ ("positive-001", "Discard the test case failure for positive-001 by implementing the rule in SKILL.md."),
+ ("positive-001", "Ignore case when comparing units, as positive-001 expects."),
+ ("positive-001", "Strip whitespace from the answer, as positive-001 expects."),
+ ("positive-001", "Take the case for positive-001 into account in SKILL.md."),
+ ("positive-001", "Replace the special case for positive-001 in parser.py with the general rounding rule."),
+ ],
+)
+def test_case_action_keeps_negated_instead_of_and_implementation_advice_for_the_wider_verbs(
+ tmp_path, monkeypatch, case_id, action
+):
+ from skillevaluator.tier3.eval_core import llm_judge
+
+ _trial(tmp_path, "a", "t0", _reward(True, "Missing syllable_count"))
+ monkeypatch.setattr(llm_judge, "call_public_llm", lambda *_a, **_k: (json.dumps([action]), None))
+ source = {"agents": {"a": _agent({case_id: _case(False)}, {})}}
+ [fix] = what_to_fix(source, run_dir=tmp_path)
+
+ assert fix["action"] == action
+ assert fix["source"] == "llm"
+ assert "fallback_reason" not in fix
+
+
+ "case_id,action",
+ [
+ ("positive-001", "Run without failures by deleting the positive-001 case from evals/evals.json."),
+ ("positive-001", "Make the run pass without failures by dropping the case for positive-001."),
+ ("smoke", "Get a green run without extra work by excluding the smoke case."),
+ ("positive-001", "Rather than a fix, delete the case for positive-001."),
+ ("positive-001", "Instead of the old approach, delete the case for positive-001 from the dataset."),
+ ],
+)
+def test_case_action_rejects_a_removal_when_the_negation_word_governs_something_else(
+ tmp_path, monkeypatch, case_id, action
+):
+ from skillevaluator.tier3.eval_core import llm_judge
+
+ _trial(tmp_path, "a", "t0", _reward(True, "Missing syllable_count"))
+ monkeypatch.setattr(llm_judge, "call_public_llm", lambda *_a, **_k: (json.dumps([action]), None))
+ [fix] = what_to_fix({"agents": {"a": _agent({case_id: _case(False)}, {})}}, run_dir=tmp_path)
+
+ assert fix["action"] == "Address in SKILL.md: Missing syllable_count"
+ assert fix["fallback_reason"] == "requirement_weakening"
+
+
+ "case_id,action",
+ [
+ ("positive-001", "Make positive-001 pass without deleting the case for positive-001."),
+ ("positive-001", "Make positive-001 pass without simply removing the case for positive-001."),
+ ("positive-001", "Fix the rounding step for positive-001 without ever dropping its case."),
+ ("syllable-count", "Implement the counting rule instead of just dropping the syllable-count case."),
+ ("smoke", "Fix the missing step in SKILL.md rather than excluding the smoke case."),
+ ("positive-001", "Remove the special case for UTF-8 and use the standard decoder for positive-001."),
+ ("positive-001", "Drop the special case for ISO-8859 input and use the standard decoder for positive-001."),
+ ("positive-001", "Delete the utf-16 case and decode with the standard library for positive-001."),
+ ("positive-001", "Remove the UNIX case and normalise line endings generally for positive-001."),
+ ("positive-001", "Remove the v2-api case and call the current endpoint for positive-001."),
+ (
+ "positive-001",
+ "Exclude the base-64 case from the encoder and handle positive-001 with the standard library.",
+ ),
+ ],
+)
+def test_case_action_keeps_advice_whose_negation_or_identifier_is_not_about_a_dataset_case(
+ tmp_path, monkeypatch, case_id, action
+):
+ from skillevaluator.tier3.eval_core import llm_judge
+
+ _trial(tmp_path, "a", "t0", _reward(True, "Missing syllable_count"))
+ monkeypatch.setattr(llm_judge, "call_public_llm", lambda *_a, **_k: (json.dumps([action]), None))
+ [fix] = what_to_fix({"agents": {"a": _agent({case_id: _case(False)}, {})}}, run_dir=tmp_path)
+
+ assert fix["action"] == action
+ assert fix["source"] == "llm"
+ assert "fallback_reason" not in fix
+
+
+def test_case_action_rejects_an_unlisted_identifier_when_the_action_points_at_the_dataset(tmp_path, monkeypatch):
+ from skillevaluator.tier3.eval_core import llm_judge
+
+ _trial(tmp_path, "a", "t0", _reward(True, "Missing syllable_count"))
+ action = "Delete the utf-16 case from evals/evals.json, then implement syllable_count."
+ monkeypatch.setattr(llm_judge, "call_public_llm", lambda *_a, **_k: (json.dumps([action]), None))
+ [fix] = what_to_fix({"agents": {"a": _agent({"syllable-positive": _case(False)}, {})}}, run_dir=tmp_path)
+
+ assert fix["action"] == "Address in SKILL.md: Missing syllable_count"
+ assert fix["fallback_reason"] == "requirement_weakening"
+
+
+def test_case_action_with_a_long_list_of_known_case_ids_is_checked_in_bounded_time(tmp_path, monkeypatch):
+ from skillevaluator.tier3.eval_core import llm_judge
+
+ ids = [f"case-{i:02d}" for i in range(20)]
+ dataset = [{"id": case_id, "question": "q"} for case_id in ids + [f"case-{i:03d}" for i in range(230)]]
+ (tmp_path / "dataset_snapshot.json").write_text(json.dumps({"dataset": dataset}))
+ _trial(tmp_path, "a", "t0", _reward(True, "Missing syllable_count"))
+ actions = {
+ "Remove the " + ", ".join(ids) + " overrides in parser.py and implement the general rule.": "llm",
+ "Remove " + " ".join(["a-b-c-d-e-f-g"] * 50) + " case": "llm",
+ "Delete the " + ", ".join(ids[:5]) + " cases.": "deterministic",
+ }
+ for action, expected_source in actions.items():
+ monkeypatch.setattr(llm_judge, "call_public_llm", lambda *_a, action=action, **_k: (json.dumps([action]), None))
+ started = time.perf_counter()
+ [fix] = what_to_fix({"agents": {"a": _agent({"case-00": _case(False)}, {})}}, run_dir=tmp_path)
+ assert time.perf_counter() - started < 1.0
+ assert fix["source"] == expected_source
+
+
+def test_case_action_fallback_text_with_a_long_judge_reason_is_checked_in_bounded_time(tmp_path):
+ ids = [f"case-{i:03d}" for i in range(250)]
+ (tmp_path / "dataset_snapshot.json").write_text(json.dumps({"dataset": [{"id": i, "question": "q"} for i in ids]}))
+ reason = "Missing " + ", ".join(ids[:100]) + " and " + " ".join(["a-b-c-d-e-f-g"] * 200)
+ _trial(tmp_path, "a", "t0", _reward(True, reason))
+ started = time.perf_counter()
+ [fix] = what_to_fix({"agents": {"a": _agent({"case-000": _case(False)}, {})}}, run_dir=tmp_path, use_llm=False)
+ assert time.perf_counter() - started < 1.0
+ assert fix["source"] == "deterministic"
+
+
+# Verb modifiers a negator may govern: "without first deleting", "without ever having to delete the case".
+_NEGATION_CHAINS = (
+ "",
+ "first",
+ "outright",
+ "ever",
+ "even",
+ "actually",
+ "simply",
+ "just",
+ "also",
+ "then",
+ "again",
+ "entirely",
+ "completely",
+ "fully",
+ "altogether",
+ "necessarily",
+ "directly",
+ "having to",
+ "needing to",
+ "need to",
+ "the need to",
+ "having to first",
+ "being forced to",
+ "being asked to",
+ "trying to",
+ "attempting to",
+ "ever having to",
+ "ever even having to first",
+)
+# (negator, whether the verb after it is in its base form): "without first deleting", "do not first delete".
+_STRICT_NEGATORS = (
+ ("without", False),
+ ("instead of", False),
+ ("rather than", False),
+ ("there is no need to", True),
+ ("should not", True),
+)
+_ALL_NEGATORS = (*_STRICT_NEGATORS, ("do not", True), ("never", True), ("don't", True), ("avoid", False))
+
+
+# "not only deleting the case but also fixing the parser" recommends the deletion, so after "not", "do not",
+# "should not", "never", "don't" or "avoid" these words void the negation. "without simply removing the case" and
+# "instead of just dropping it" are different: there the negator governs the verb.
+_ADDITIVE_WORDS = ("only", "just", "merely", "simply", "solely", "exclusively", "purely")
+_ADDITIVE_PREFIXES = (*_ADDITIVE_WORDS, "necessarily just", "necessarily only", "even just")
+_NOT_NEGATORS = (
+ ("not", False),
+ ("do not", True),
+ ("should not", True),
+ ("never", True),
+ ("don't", True),
+ ("avoid", False),
+)
+
+
+def _negated_advice(negators: tuple[tuple[str, bool], ...], case_subject: str) -> list[str]:
+ sentences = []
+ for negator, base_form in negators:
+ for chain in _NEGATION_CHAINS:
+ if negator in dict(_NOT_NEGATORS) and set(chain.split()) & set(_ADDITIVE_WORDS):
+ continue
+ base = base_form or chain.endswith(" to")
+ for verb, subject in (("delete", case_subject), ("lower", "the pass threshold")):
+ form = verb if base else verb.replace("lower", "lowering").replace("delete", "deleting")
+ sentences.append(" ".join(f"Fix the parser {negator} {chain} {form} {subject}.".split()))
+ return sentences
+
+
+def _unrelated_negation_advice(negators: tuple[tuple[str, bool], ...], case_subject: str) -> list[str]:
+ """A noun between the negator (and its modifiers) and the verb: the negator governs the noun, not the removal."""
+ return [
+ " ".join(f"Fix the parser {negator} {chain} {noun} by {verb} {subject}.".split())
+ for negator, _ in negators
+ for chain in _NEGATION_CHAINS
+ for noun in ("failures", "errors")
+ for verb, subject in (("deleting", case_subject), ("lowering", "the pass threshold"))
+ ]
+
+
+def _additive_advice(case_subject: str) -> list[str]:
+ """Additive wording ("not only lowering the threshold but also fixing the parser") still recommends the removal."""
+ sentences = []
+ for negator, base_form in _NOT_NEGATORS:
+ for additive in _ADDITIVE_PREFIXES:
+ for verb, subject in (("delete", case_subject), ("lower", "the pass threshold")):
+ form = verb if base_form else verb.replace("lower", "lowering").replace("delete", "deleting")
+ sentences.append(f"Fix the parser and {negator} {additive} {form} {subject} but also add a test.")
+ return sentences
+
+
+ "case_id,action",
+ [
+ ("smoke", "Implement the missing field without having to delete the smoke case."),
+ ("smoke", "Fix the parser without first deleting the smoke case."),
+ ("smoke", "Fix the parser without outright deleting the smoke case."),
+ ("smoke", "Fix the parser without ever having to delete the smoke case."),
+ ("smoke", "Fix the parser without even needing to drop the smoke case."),
+ ("smoke", "Fix the parser without being forced to exclude the smoke case."),
+ ("smoke", "Fix the parser instead of first deleting the smoke case."),
+ ("smoke", "Fix the parser rather than simply dropping the smoke case."),
+ ("smoke", "Do not first delete the smoke case; fix the parser."),
+ ("smoke", "There is no need to delete the smoke case once the parser is fixed."),
+ ("smoke", "You should not delete the smoke case; fix the parser."),
+ ("positive-001", "Fix the parser without trying to delete the case for positive-001."),
+ ("smoke", "Get smoke to pass without first lowering the pass threshold."),
+ ("smoke", "Fix the parser so smoke passes without having to loosen the expected output."),
+ ("smoke", "Make smoke pass without ever relaxing the assertions."),
+ ("smoke", "Make smoke pass instead of outright softening the ground truth."),
+ ],
+)
+def test_case_action_keeps_advice_whose_negator_governs_the_verb_through_modifiers(
+ tmp_path, monkeypatch, case_id, action
+):
+ from skillevaluator.tier3.eval_core import llm_judge
+
+ _trial(tmp_path, "a", "t0", _reward(True, "Missing syllable_count"))
+ monkeypatch.setattr(llm_judge, "call_public_llm", lambda *_a, **_k: (json.dumps([action]), None))
+ [fix] = what_to_fix({"agents": {"a": _agent({case_id: _case(False)}, {})}}, run_dir=tmp_path)
+
+ assert fix["action"] == action
+ assert fix["source"] == "llm"
+ assert "fallback_reason" not in fix
+
+
+ "case_id,action",
+ [
+ ("smoke", "Run without failures by deleting the smoke case from evals/evals.json."),
+ ("smoke", "Run without errors by removing the smoke case from evals/evals.json."),
+ ("positive-001", "Pass without errors by removing the positive-001 case from evals/evals.json."),
+ ("smoke", "Pass without regressions by dropping the smoke case."),
+ ("smoke", "Fix the parser without delay, deleting the smoke case."),
+ ("smoke", "Fix the parser without having to wait, delete the smoke case."),
+ ("smoke", "Fix the parser without ever fixing the parser by deleting the smoke case."),
+ ("smoke", "Fix the parser without first fixing the parser. Delete the smoke case."),
+ ("smoke", "Get smoke to pass without failures by lowering the pass threshold."),
+ ("smoke", "Get smoke to pass without errors by loosening the expected output."),
+ ("smoke", "Get smoke to pass without first fixing the parser, then relax the assertions."),
+ ],
+)
+def test_case_action_rejects_a_removal_that_a_negator_with_modifiers_does_not_govern(
+ tmp_path, monkeypatch, case_id, action
+):
+ from skillevaluator.tier3.eval_core import llm_judge
+
+ _trial(tmp_path, "a", "t0", _reward(True, "Missing syllable_count"))
+ monkeypatch.setattr(llm_judge, "call_public_llm", lambda *_a, **_k: (json.dumps([action]), None))
+ [fix] = what_to_fix({"agents": {"a": _agent({case_id: _case(False)}, {})}}, run_dir=tmp_path)
+
+ assert fix["action"] == "Address in SKILL.md: Missing syllable_count"
+ assert fix["fallback_reason"] == "requirement_weakening"
+
+
[email protected]("known", [("smoke",), ()])
+def test_a_negator_and_its_verb_modifiers_negate_the_removal_and_the_weakening(known):
+ subject = "the smoke case" if known else "the case for positive-001"
+ offenders = [s for s in _negated_advice(_ALL_NEGATORS, subject) if _weakens_requirements(s, known)]
+ assert offenders == []
+
+
[email protected]("known", [("smoke",), ()])
+def test_a_noun_between_the_negator_and_the_verb_breaks_the_negation(known):
+ subject = "the smoke case from evals/evals.json" if known else "the case for positive-001 from evals/evals.json"
+ missed = [s for s in _unrelated_negation_advice(_STRICT_NEGATORS, subject) if not _weakens_requirements(s, known)]
+ assert missed == []
+
+
+ "case_id,action",
+ [
+ ("smoke", "Make smoke pass by not only lowering the pass threshold but also fixing the parser."),
+ ("smoke", "Not only delete the smoke case but also fix the parser."),
+ ("smoke", "Make smoke pass by not just lowering the pass threshold but also fixing the parser."),
+ ("smoke", "Make smoke pass by not merely relaxing the assertions but also fixing the parser."),
+ ("smoke", "Make smoke pass by not simply loosening the expected output but also fixing the parser."),
+ ("smoke", "Make smoke pass by not solely lowering the pass threshold but also fixing the parser."),
+ ("smoke", "Make smoke pass by not exclusively softening the ground truth but also fixing the parser."),
+ ("smoke", "Make smoke pass by not necessarily just lowering the pass threshold but also fixing the parser."),
+ ("smoke", "Make smoke pass by not necessarily only lowering the pass threshold but also fixing the parser."),
+ ("smoke", "Fix the parser by not only dropping the smoke case but also adding a unit test."),
+ ("smoke", "Not just removing the smoke case but also fixing the parser."),
+ ("smoke", "Fix the parser by not merely excluding the smoke case but also adding a unit test."),
+ ("positive-001", "Not simply delete the case for positive-001 but also fix the parser."),
+ ("smoke", "Do not only lower the pass threshold for smoke but also fix the parser."),
+ ("smoke", "Do not just delete the smoke case, fix the parser."),
+ ("smoke", "Don't simply delete the smoke case; fix the parser."),
+ ],
+)
+def test_case_action_rejects_additive_wording_that_still_recommends_the_removal(tmp_path, monkeypatch, case_id, action):
+ from skillevaluator.tier3.eval_core import llm_judge
+
+ _trial(tmp_path, "a", "t0", _reward(True, "Missing syllable_count"))
+ monkeypatch.setattr(llm_judge, "call_public_llm", lambda *_a, **_k: (json.dumps([action]), None))
+ [fix] = what_to_fix({"agents": {"a": _agent({case_id: _case(False)}, {})}}, run_dir=tmp_path)
+
+ assert fix["action"] == "Address in SKILL.md: Missing syllable_count"
+ assert fix["fallback_reason"] == "requirement_weakening"
+
+
+ "case_id,action",
+ [
+ ("smoke", "You should not lower the pass threshold for smoke; fix the parser."),
+ ("smoke", "You must not delete the smoke case; fix the parser."),
+ ("smoke", "Do not lower the pass threshold for smoke."),
+ ("smoke", "Do not delete the smoke case but also fix the parser."),
+ ("smoke", "Fix the parser and do not delete the smoke case."),
+ ("smoke", "Fix the parser; it should not drop the smoke case."),
+ ("smoke", "Fix the parser without simply removing the smoke case."),
+ ("smoke", "Fix the parser instead of just dropping the smoke case."),
+ ("smoke", "Fix the parser rather than merely excluding the smoke case."),
+ ],
+)
+def test_case_action_keeps_genuine_prohibitions_next_to_additive_wording(tmp_path, monkeypatch, case_id, action):
+ from skillevaluator.tier3.eval_core import llm_judge
+
+ _trial(tmp_path, "a", "t0", _reward(True, "Missing syllable_count"))
+ monkeypatch.setattr(llm_judge, "call_public_llm", lambda *_a, **_k: (json.dumps([action]), None))
+ [fix] = what_to_fix({"agents": {"a": _agent({case_id: _case(False)}, {})}}, run_dir=tmp_path)
+
+ assert fix["action"] == action
+ assert fix["source"] == "llm"
+ assert "fallback_reason" not in fix
+
+
[email protected]("known", [("smoke",), ()])
+def test_additive_not_only_wording_does_not_negate_the_removal_or_the_weakening(known):
+ subject = "the smoke case" if known else "the case for positive-001"
+ missed = [s for s in _additive_advice(subject) if not _weakens_requirements(s, known)]
+ assert missed == []
+
+
+ "action,error,expected_source,expected_fallback",
+ [
+ ("Implement syllable_count in analyze_text.py.", None, "llm", None),
+ ("Do not remove the expectation; implement it in analyze_text.py.", None, "llm", None),
+ ("Do not remove the `expectation`; implement it in analyze_text.py.", None, "llm", None),
+ ("Never drop the `expected output` for syllable_count; implement it in analyze_text.py.", None, "llm", None),
+ ("Remove the expected output mismatch by implementing syllable_count in analyze_text.py.", None, "llm", None),
+ ("Remove the assertion failure by implementing syllable_count in analyze_text.py.", None, "llm", None),
+ ("Remove the `expected output` mismatch by implementing syllable_count in analyze_text.py.", None, "llm", None),
+ (
+ "The requirement violation should be removed by implementing syllable_count in analyze_text.py.",
+ None,
+ "llm",
+ None,
+ ),
+ ("Remove temporary files after analysis.", None, "llm", None),
+ ("Do not delete the case; implement syllable_count in analyze_text.py.", None, "llm", None),
+ (
+ "Never remove the syllable-positive entry from evals/evals.json; fix analyze_text.py instead.",
+ None,
+ "llm",
+ None,
+ ),
+ (
+ "Remove the expected output mismatch by implementing syllable_count in analyze_text.py; "
+ "keep the syllable-positive case in evals/evals.json.",
+ None,
+ "llm",
+ None,
+ ),
+ ("In this case, implement syllable_count in analyze_text.py.", None, "llm", None),
+ ("The edge case should be handled in analyze_text.py.", None, "llm", None),
+ ("Delete the cache entry, then implement syllable_count in analyze_text.py.", None, "llm", None),
+ ("Remove temp files per case, then implement syllable_count in analyze_text.py.", None, "llm", None),
+ (
+ "Remove the special case in parser.py for positive-001 and implement the general rounding algorithm.",
+ None,
+ "llm",
+ None,
+ ),
+ (
+ "Remove the test case failure for positive-001 by implementing the rounding rule in SKILL.md",
+ None,
+ "llm",
+ None,
+ ),
+ ("Remove the special case in the script and implement syllable_count generally.", None, "llm", None),
+ ("Delete the default case branch in analyze_text.py; implement syllable_count.", None, "llm", None),
+ ("Remove the case mismatch by implementing syllable_count in analyze_text.py.", None, "llm", None),
+ ("Remove the test case violation by implementing syllable_count in analyze_text.py.", None, "llm", None),
+ (
+ "The test case failure should be removed by implementing syllable_count in analyze_text.py.",
+ None,
+ "llm",
+ None,
+ ),
+ ("Never remove the test case; implement syllable_count in analyze_text.py.", None, "llm", None),
+ (None, "Connection error", "deterministic", "llm_unavailable_or_invalid"),
+ ("", None, "deterministic", "llm_unavailable_or_invalid"),
+ ],
+)
+def test_case_action_provenance_reaches_canonical_recommendation(
+ tmp_path, monkeypatch, action, error, expected_source, expected_fallback
+):
+ from skillevaluator.evaluation.tier3_report import _apply_case_findings
+ from skillevaluator.tier3.eval_core import llm_judge
+
+ _trial(tmp_path, "a", "t0", _reward(True, "Missing syllable_count"))
+ monkeypatch.setattr(llm_judge, "call_public_llm", lambda *_a, **_k: (json.dumps([action]), error))
+ source = {"agents": {"a": _agent({"syllable-positive": _case(False)}, {"syllable-positive": _case(False)})}}
+ payload = {}
+ _apply_case_findings(payload, source, tmp_path, tmp_path, use_llm=True)
+ [fix] = json.loads((tmp_path / "what_to_fix.json").read_text())
+ [recommendation] = payload["recommendations"]
+
+ for record in (fix, recommendation):
+ assert record["source"] == expected_source
+ assert record.get("fallback_reason") == expected_fallback
+ assert fix["action"] in recommendation["message"]
+
+
+def test_case_action_disabled_is_not_a_failed_llm_fallback(tmp_path, monkeypatch):
+ from skillevaluator.tier3.eval_core import llm_judge
+
+ def unexpected(*args, **kwargs):
+ pytest.fail("LLM should not be called")
+
+ monkeypatch.setattr(llm_judge, "call_public_llm", unexpected)
+ _trial(tmp_path, "a", "t0", _reward(True, "Missing syllable_count"))
+ [fix] = what_to_fix({"agents": {"a": _agent({"c": _case(False)}, {})}}, run_dir=tmp_path, use_llm=False)
+ assert fix["source"] == "deterministic"
+ assert "fallback_reason" not in fix
+
+
+def test_case_action_deterministic_remedy_does_not_repeat_weakening_reason(tmp_path):
+ _trial(tmp_path, "a", "t0", _reward(True, "Remove the expectation for syllable_count"))
+ [fix] = what_to_fix({"agents": {"a": _agent({"c": _case(False)}, {})}}, run_dir=tmp_path, use_llm=False)
+ assert "Remove the expectation" not in fix["action"]
+
+
@pytest.mark.parametrize("source", ["result", "saved", "provenance", "variant", "other_variant", "default"])
def test_incomplete_reconstruction_preserves_configured_threshold(tmp_path: Path, source: str) -> None:
from skillevaluator.tier3.case_summary import _cases