Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.
Python agent-evaluationagent-pluginsagent-securityagent-skillsagentic-aibenchmarkci-templatesclaude-codecodexevaluateevaluationharnesspluginplugin-evaluationsecurity-scannerskill-evalskill-evaluationskill-evaluatorskill-gap-analysisskills
fix(ci): avoid benchmark scanner regex backtracking
Narendran Raghavan29cbc336091c
diff --git a/scripts/ci/check_public_benchmarks.py b/scripts/ci/check_public_benchmarks.pyindex 80f6d5d6..20f8ff24 100644--- a/scripts/ci/check_public_benchmarks.py+++ b/scripts/ci/check_public_benchmarks.py@@ -70,9 +70,8 @@ LINE_RULES = (),)-_AGENT_MODEL_STATES = re.compile(- r"^[^,]+ \((?:`[^`]+`|model not recorded)\)"- r"(?:,\s*[^,]+ \((?:`[^`]+`|model not recorded)\))*$",+_AGENT_MODEL_STATE = re.compile(+ r"^[^,]+ \((?:`[^`,]+`|model not recorded)\)$",flags=re.IGNORECASE,)_OVERALL_PASS = re.compile(@@ -185,10 +184,16 @@ def _check_metadata_semantics(path: Path, text: str, offenders: list[Offender])elif lowered.startswith("requested but not run"):if "model not recorded" not in lowered:offenders.append(Offender(path, line_number, "agent model identity not recorded"))- elif not _AGENT_MODEL_STATES.fullmatch(value):+ elif not _valid_agent_model_states(value):offenders.append(Offender(path, line_number, "agent model identity not recorded"))+def _valid_agent_model_states(value: str) -> bool:+ """Validate a comma-delimited agent list without a backtracking list regex."""+ agents = [agent.strip() for agent in value.split(",")]+ return bool(agents) and all(_AGENT_MODEL_STATE.fullmatch(agent) for agent in agents)++def _metadata_section_lines(text: str) -> list[tuple[int, str]]:"""Return line-numbered content from the first Evaluation Metadata section."""lines = text.splitlines()diff --git a/tests/test_public_benchmark_gate.py b/tests/test_public_benchmark_gate.pyindex 4e56e30f..f80a1018 100644--- a/tests/test_public_benchmark_gate.py+++ b/tests/test_public_benchmark_gate.py@@ -122,6 +122,21 @@ def test_gate_requires_explicit_agent_model_state(tmp_path: Path) -> None:assert {offender.reason for offender in offenders} == {"agent model identity not recorded"}+def test_gate_accepts_multiple_explicit_agent_model_states(tmp_path: Path) -> None:+ benchmark = tmp_path / "BENCHMARK.md"+ benchmark.write_text(+ _valid_benchmark().replace(+ "Agents: Codex (`gpt-codex`)",+ "Agents: Codex (`gpt-codex`), Claude Code (model not recorded)",+ ),+ encoding="utf-8",+ )++ _files, offenders = benchmark_gate.find_offenders([benchmark])++ assert offenders == []++@pytest.mark.parametrize("field",[