Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.
Python agent-evaluationagent-pluginsagent-securityagent-skillsagentic-aibenchmarkci-templatesclaude-codecodexevaluateevaluationharnesspluginplugin-evaluationsecurity-scannerskill-evalskill-evaluationskill-evaluatorskill-gap-analysisskills
fix(task-list): preserve status words inside task descriptions (#680)
PMee5cc602132d
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 48e53735..ec50e61a 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -492,6 +492,7 @@ All notable changes to SkillEvaluator are documented in this file.- Partially executed evaluation arms retain scored attempts and per-arm scores without treating missing attempts as failures or enabling aggregate comparisons.- Script lint now runs advisory AST checks on Python files with one leading UTF-8 BOM.+- Task-list descriptions containing status words retain their text and pending status after reload.- Failed Tier 3 arms retain scored and errored trial activation diagnostics without restoring aggregate quality or lift, or double-counting token usage.- Budget-limited Tier 3 coverage checks retain admitted attempt ordinals, avoiding false missing/excess errors when a denied slot leaves a gap.- HTML reports retain budget context for fully budget-denied runs withoutdiff --git a/src/skillevaluator/tier3/reference_skills/task-list/scripts/tasks.py b/src/skillevaluator/tier3/reference_skills/task-list/scripts/tasks.pyindex 5989faba..94db9b09 100644--- a/src/skillevaluator/tier3/reference_skills/task-list/scripts/tasks.py+++ b/src/skillevaluator/tier3/reference_skills/task-list/scripts/tasks.py@@ -62,7 +62,7 @@ def _load_state(state_path: Path) -> tuple[dict, str | None]:counter = 0# Pattern: - [X] task_ID: content (status)- pattern = r"^- \[(.)\] (task_\d+): (.+?) \((pending|in_progress|completed|cancelled)\)"+ pattern = r"^- \[(.)\] (task_\d+): (.+?) \((pending|in_progress|completed|cancelled)\)$"for line in content.split("\n"):match = re.match(pattern, line.strip())diff --git a/tests/test_reference_skill_scripts.py b/tests/test_reference_skill_scripts.pyindex a7d8a17e..1634adb9 100644--- a/tests/test_reference_skill_scripts.py+++ b/tests/test_reference_skill_scripts.py@@ -75,6 +75,44 @@ def test_task_content_containing_error_label_remains_successful(tmp_path: Path)assert result.stderr == ""+ ("content", "operation", "status", "checkbox"),+ [+ ("Investigate (pending) labels after merge", None, "pending", " "),+ ("Investigate (in_progress) labels after merge", None, "pending", " "),+ ("Investigate (completed) labels after merge", None, "pending", " "),+ ("Investigate (cancelled) labels after merge", None, "pending", " "),+ ("Check (completed)", "complete", "completed", "x"),+ ("Check (completed)", "cancel", "cancelled", "~"),+ ],+)+def test_task_status_words_survive_persistence(+ content: str, operation: str | None, status: str, checkbox: str, tmp_path: Path+) -> None:+ state_path = tmp_path / "todo.md"+ env = {"TASK_LIST_STATE_PATH": str(state_path)}++ added = _run_script("tasks", "add", "--content", content, env=env)+ if operation:+ updated = _run_script("tasks", operation, "--task_id", "task_1", env=env)+ assert updated.returncode == 0+ assert updated.stderr == ""+ listed = _run_script("tasks", "list", env=env)+ state, error = _load_script("tasks")._load_state(state_path)++ assert added.returncode == listed.returncode == 0+ assert added.stderr == listed.stderr == ""+ assert error is None+ assert state["tasks"]["task_1"] == {+ "id": "task_1",+ "content": content,+ "status": status,+ }+ if status != "cancelled":+ assert content in listed.stdout+ assert f"- [{checkbox}] task_1: {content} ({status})" in state_path.read_text(encoding="utf-8")++@pytest.mark.parametrize(("name", "args", "expected_error"),[