The agent harness performance optimization system. Skills, instincts, memory, security, and research-first development for Claude Code, Codex, Opencode, Cursor and beyond.
fix(skill-comply): stop a failed step supplying evidence downstream (#3109)
Lukas8b951d3bf63b
diff --git a/skills/skill-comply/scripts/grader.py b/skills/skill-comply/scripts/grader.pyindex 516beb18b..1209d042a 100644--- a/skills/skill-comply/scripts/grader.py+++ b/skills/skill-comply/scripts/grader.py@@ -2,7 +2,7 @@from __future__ import annotations-from dataclasses import dataclass+from dataclasses import dataclass, replacefrom scripts.classifier import classify_eventsfrom scripts.parser import ComplianceSpec, ObservationEvent, Step@@ -30,23 +30,38 @@ def _check_temporal_order(event: ObservationEvent,resolved: dict[str, list[ObservationEvent]],classified: dict[str, list[ObservationEvent]],+ graded: set[str],) -> str | None:"""Check before_step/after_step constraints. Returns failure reason or None."""if step.detector.after_step is not None:- after_events = resolved.get(step.detector.after_step)+ after_step = step.detector.after_step+ after_events = resolved.get(after_step)if after_events is None:- after_events = classified.get(step.detector.after_step, [])+ if after_step in graded:+ # Graded and missing from `resolved` means it failed its own checks.+ # Its classified events are not evidence for anything: reusing them+ # here let a dependant pass on the strength of a failed prerequisite,+ # and the overstatement carried down the whole chain.+ return f"after_step '{after_step}' did not pass its own checks"+ # Not graded yet, so this is a forward reference to a step declared later.+ # The classifier's own output is the only thing available, and using it is+ # what makes an out-of-order declaration work.+ after_events = classified.get(after_step, [])if not after_events:- return f"after_step '{step.detector.after_step}' not yet detected"+ return f"after_step '{after_step}' not yet detected"latest_after = max(e.timestamp for e in after_events)if event.timestamp <= latest_after:return (- f"must occur after '{step.detector.after_step}' "+ f"must occur after '{after_step}' "f"(last at {latest_after}), but found at {event.timestamp}")if step.detector.before_step is not None:- # Look ahead using LLM classification results+ # Look ahead using LLM classification results. A failed reference step is NOT+ # excluded here the way it is above: the two fall in opposite directions. An+ # `after_step` fallback can only turn a failure into a pass, while a+ # `before_step` one can only turn a pass into a failure, so dropping it would+ # relax a constraint because some other step failed.before_events = resolved.get(step.detector.before_step)if before_events is None:before_events = classified.get(step.detector.before_step, [])@@ -61,6 +76,46 @@ def _check_temporal_order(return None+def _demote_steps_resting_on_failures(+ step_results: tuple[StepResult, ...],+ after_steps: dict[str, str],+) -> tuple[StepResult, ...]:+ """Undo passes that rest on an `after_step` which ended up failing.++ A step declared before its prerequisite is graded against the classifier's raw+ events for that step, because `resolved` has nothing for it yet. That fallback is+ what makes an out-of-order declaration work, but the prerequisite can go on to+ fail its own checks, and then the dependant is left passing on evidence that+ never held. Repeat until nothing changes: demoting one step can invalidate+ whatever depended on it, whichever order the two were declared in. Demotion only+ ever removes passes, so the loop terminates.+ """+ results = step_results+ while True:+ failed = {result.step_id for result in results if not result.detected}+ demote = {+ result.step_id+ for result in results+ if result.detected and after_steps.get(result.step_id) in failed+ }+ if not demote:+ return results+ results = tuple(+ replace(+ result,+ detected=False,+ evidence=(),+ failure_reason=(+ f"after_step '{after_steps[result.step_id]}' "+ "did not pass its own checks"+ ),+ )+ if result.step_id in demote+ else result+ for result in results+ )++def grade(spec: ComplianceSpec,trace: list[ObservationEvent],@@ -80,6 +135,9 @@ def grade(# Step 2: Check temporal ordering (deterministic)resolved: dict[str, list[ObservationEvent]] = {}+ # Steps already graded, pass or fail. `resolved` alone cannot tell "failed" from+ # "not reached yet", and those need opposite answers in `_check_temporal_order`.+ graded: set[str] = set()step_results: list[StepResult] = []for step in spec.steps:@@ -88,7 +146,7 @@ def grade(failure_reason: str | None = Nonefor event in candidates:- temporal_fail = _check_temporal_order(step, event, resolved, classified)+ temporal_fail = _check_temporal_order(step, event, resolved, classified, graded)if temporal_fail is None:matched.append(event)break@@ -101,6 +159,7 @@ def grade(elif failure_reason is None:failure_reason = f"no matching event classified for step '{step.id}'"+ graded = graded | {step.id}step_results.append(StepResult(step_id=step.id,detected=detected,@@ -108,8 +167,19 @@ def grade(failure_reason=failure_reason if not detected else None,))+ # `graded` catches a prerequisite that had already failed when its dependant was+ # graded. The other direction needs a second pass: a dependant declared first is+ # graded against the classifier's raw events for a prerequisite that has not run+ # yet, and only later does that prerequisite fail.+ after_steps = {+ step.id: step.detector.after_step+ for step in spec.steps+ if step.detector.after_step is not None+ }+ resolved_results = _demote_steps_resting_on_failures(tuple(step_results), after_steps)+required_ids = {s.id for s in spec.steps if s.required}- required_steps = [s for s in step_results if s.step_id in required_ids]+ required_steps = [s for s in resolved_results if s.step_id in required_ids]detected_required = sum(1 for s in required_steps if s.detected)total_required = len(required_steps)@@ -117,7 +187,7 @@ def grade(return ComplianceResult(spec_id=spec.id,- steps=tuple(step_results),+ steps=resolved_results,compliance_rate=compliance_rate,recommend_hook_promotion=compliance_rate < spec.threshold_promote_to_hook,classification=classification,diff --git a/skills/skill-comply/tests/test_grader.py b/skills/skill-comply/tests/test_grader.pyindex a95825266..8ca056452 100644--- a/skills/skill-comply/tests/test_grader.py+++ b/skills/skill-comply/tests/test_grader.py@@ -1,7 +1,7 @@"""Tests for grader module — compliance scoring with LLM classification."""from pathlib import Path-from unittest.mock import patch+from unittest.mock import MagicMock, patchimport pytest@@ -137,7 +137,7 @@ class TestGradeEdgeCases:assert result.spec_id == "tdd-workflow"@patch("scripts.grader.classify_events")- def test_after_step_can_reference_later_declared_spec_step(self, mock_cls) -> None:+ def test_after_step_can_reference_later_declared_spec_step(self, mock_cls: MagicMock) -> None:spec = ComplianceSpec(id="out-of-order-after-step",name="Out of order after_step",@@ -195,3 +195,248 @@ class TestGradeEdgeCases:assert step_a.failure_reason is Noneassert step_b.detected is Trueassert result.compliance_rate == 1.0++ @patch("scripts.grader.classify_events")+ def test_after_step_does_not_reuse_a_step_that_failed_its_own_constraint(+ self, mock_cls: MagicMock+ ) -> None:+ """A step that failed may not supply evidence to a step that depends on it (#3108).++ `resolved` only receives a step once it passes, so a dependant fell back to the+ raw classifier output and could pass on an event belonging to a failed parent.+ The fallback exists for forward references, which the case above covers; a+ parent that has already been graded and failed is a different thing.+ """+ spec = ComplianceSpec(+ id="failed-parent-evidence",+ name="Failed parent evidence",+ source_rule="rules/common/testing.md",+ version="1.0",+ steps=(+ Step(+ id="C",+ description="Reference step with no constraint of its own",+ required=True,+ detector=Detector(description="Event C"),+ ),+ Step(+ id="A",+ description="Must occur before C, and does not",+ required=True,+ detector=Detector(description="Event A", before_step="C"),+ ),+ Step(+ id="B",+ description="Depends on A",+ required=True,+ detector=Detector(description="Event B", after_step="A"),+ ),+ ),+ threshold_promote_to_hook=0.5,+ )+ trace = [+ ObservationEvent(+ timestamp=f"2026-03-20T10:00:0{index}Z",+ event="tool_complete",+ tool="Write",+ session="sess-evidence",+ input=f'{{"file_path":"src/{name}.py"}}',+ output=f"step {name}",+ )+ for index, name in enumerate(("c", "a", "b"))+ ]+ mock_cls.return_value = {"C": [0], "A": [1], "B": [2]}++ result = grade(spec, trace)++ detected = {step.step_id: step.detected for step in result.steps}+ assert detected["C"] is True+ assert detected["A"] is False+ assert detected["B"] is False+ assert result.compliance_rate == pytest.approx(1 / 3)+ step_b = next(step for step in result.steps if step.step_id == "B")+ assert "A" in (step_b.failure_reason or "")++ @patch("scripts.grader.classify_events")+ def test_a_failed_prerequisite_does_not_carry_a_chain_of_passes(+ self, mock_cls: MagicMock+ ) -> None:+ """The overstatement compounds: B on A, C on B, D on C (#3108).++ Each link used to pass on the classifier's raw output for the link above it, so+ one failed prerequisite could leave a four-step workflow reading 3/4 compliant.+ """+ steps = (+ Step(+ id="Z",+ description="Reference step with no constraint of its own",+ required=True,+ detector=Detector(description="Event Z"),+ ),+ Step(+ id="A",+ description="Must occur before Z, and does not",+ required=True,+ detector=Detector(description="Event A", before_step="Z"),+ ),+ *(+ Step(+ id=later,+ description=f"Depends on {earlier}",+ required=True,+ detector=Detector(description=f"Event {later}", after_step=earlier),+ )+ for earlier, later in (("A", "B"), ("B", "C"), ("C", "D"))+ ),+ )+ spec = ComplianceSpec(+ id="failed-prerequisite-chain",+ name="Failed prerequisite chain",+ source_rule="rules/common/testing.md",+ version="1.0",+ steps=steps,+ threshold_promote_to_hook=0.5,+ )+ names = ("z", "a", "b", "c", "d")+ trace = [+ ObservationEvent(+ timestamp=f"2026-03-20T10:00:0{index}Z",+ event="tool_complete",+ tool="Write",+ session="sess-chain",+ input=f'{{"file_path":"src/{name}.py"}}',+ output=f"step {name}",+ )+ for index, name in enumerate(names)+ ]+ mock_cls.return_value = {"Z": [0], "A": [1], "B": [2], "C": [3], "D": [4]}++ result = grade(spec, trace)++ detected = {step.step_id: step.detected for step in result.steps}+ assert detected == {"Z": True, "A": False, "B": False, "C": False, "D": False}+ assert result.compliance_rate == pytest.approx(1 / 5)++ @patch("scripts.grader.classify_events")+ def test_forward_reference_is_revoked_when_the_prerequisite_later_fails(+ self, mock_cls: MagicMock+ ) -> None:+ """The mirror image of the case above, raised in review of #3109.++ `graded` only catches a prerequisite that had already failed. A step declared+ *before* its prerequisite is graded against the classifier's raw events for a+ step that has not run yet — the fallback that makes an out-of-order+ declaration work — and nothing revisited it once that step failed.+ """+ spec = ComplianceSpec(+ id="forward-reference-revoked",+ name="Forward reference revoked",+ source_rule="rules/common/testing.md",+ version="1.0",+ steps=(+ Step(+ id="Z",+ description="Reference step with no constraint of its own",+ required=True,+ detector=Detector(description="Event Z"),+ ),+ Step(+ id="B",+ description="Depends on A, which is declared after it",+ required=True,+ detector=Detector(description="Event B", after_step="A"),+ ),+ Step(+ id="A",+ description="Must occur before Z, and does not",+ required=True,+ detector=Detector(description="Event A", before_step="Z"),+ ),+ ),+ threshold_promote_to_hook=0.5,+ )+ trace = [+ ObservationEvent(+ timestamp=f"2026-03-20T10:00:0{index}Z",+ event="tool_complete",+ tool="Write",+ session="sess-forward",+ input=f'{{"file_path":"src/{name}.py"}}',+ output=f"step {name}",+ )+ for index, name in enumerate(("z", "a", "b"))+ ]+ mock_cls.return_value = {"Z": [0], "A": [1], "B": [2]}++ result = grade(spec, trace)++ detected = {step.step_id: step.detected for step in result.steps}+ assert detected == {"Z": True, "A": False, "B": False}+ assert result.compliance_rate == pytest.approx(1 / 3)+ step_b = next(step for step in result.steps if step.step_id == "B")+ assert step_b.evidence == ()+ assert "A" in (step_b.failure_reason or "")++ @patch("scripts.grader.classify_events")+ def test_revocation_reaches_a_step_that_referenced_the_dependant_backwards(+ self, mock_cls: MagicMock+ ) -> None:+ """One demotion invalidates the next, in either declaration order.++ C is declared after B and passes on `resolved`, the ordinary backward+ reference — B was still detected at the time. B is the forward-reference case+ above and comes down with A, so C has to follow, which takes a second pass.+ """+ spec = ComplianceSpec(+ id="revocation-propagates",+ name="Revocation propagates",+ source_rule="rules/common/testing.md",+ version="1.0",+ steps=(+ Step(+ id="Z",+ description="Reference step with no constraint of its own",+ required=True,+ detector=Detector(description="Event Z"),+ ),+ Step(+ id="B",+ description="Depends on A, which is declared after it",+ required=True,+ detector=Detector(description="Event B", after_step="A"),+ ),+ Step(+ id="A",+ description="Must occur before Z, and does not",+ required=True,+ detector=Detector(description="Event A", before_step="Z"),+ ),+ Step(+ id="C",+ description="Depends on B, which is declared before it",+ required=True,+ detector=Detector(description="Event C", after_step="B"),+ ),+ ),+ threshold_promote_to_hook=0.5,+ )+ trace = [+ ObservationEvent(+ timestamp=f"2026-03-20T10:00:0{index}Z",+ event="tool_complete",+ tool="Write",+ session="sess-propagate",+ input=f'{{"file_path":"src/{name}.py"}}',+ output=f"step {name}",+ )+ for index, name in enumerate(("z", "a", "b", "c"))+ ]+ mock_cls.return_value = {"Z": [0], "A": [1], "B": [2], "C": [3]}++ result = grade(spec, trace)++ detected = {step.step_id: step.detected for step in result.steps}+ assert detected == {"Z": True, "A": False, "B": False, "C": False}+ assert result.compliance_rate == pytest.approx(1 / 4)+ step_c = next(step for step in result.steps if step.step_id == "C")+ assert "B" in (step_c.failure_reason or "")