Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.
Python agent-evaluationagent-pluginsagent-securityagent-skillsagentic-aibenchmarkci-templatesclaude-codecodexevaluateevaluationharnesspluginplugin-evaluationsecurity-scannerskill-evalskill-evaluationskill-evaluatorskill-gap-analysisskills
ci: run main test jobs in parallel (-n auto) (#261)
Narendran Raghavand5818845bc6d
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.ymlindex a3e78e01..48c9ea89 100644--- a/.github/workflows/ci.yml+++ b/.github/workflows/ci.yml@@ -115,7 +115,7 @@ jobs:version: "0.11.27"- run: uv sync --all-extras --locked --python 3.13- name: Run tests- run: uv run pytest -q+ run: uv run pytest -q -n autohtml-parser-compatibility:name: HTML parser compatibility (Python ${{ matrix.python-version }})diff --git a/tests/ci/test_release_image.py b/tests/ci/test_release_image.pyindex 2664d988..2e15ed78 100644--- a/tests/ci/test_release_image.py+++ b/tests/ci/test_release_image.py@@ -119,7 +119,7 @@ def test_release_shell_never_pushes_an_existing_or_unverified_tag(tmp_path, scen["bash", "-e", "-o", "pipefail", "-c", script], env=environment, capture_output=True, text=True)commands = (tmp_path / "commands").read_text()- assert ("push " in commands) is (scenario == "new")+ assert ("push " in commands) is (scenario == "new"), (commands, result.returncode, result.stderr)assert (result.returncode == 0) is (scenario in {"existing", "new"})if scenario == "existing":assert "pull " + environment["EXISTING_IMAGE"] in commandsdiff --git a/tests/validators/test_plugin_hook_risk_bounds.py b/tests/validators/test_plugin_hook_risk_bounds.pyindex 872cae97..32105d4e 100644--- a/tests/validators/test_plugin_hook_risk_bounds.py+++ b/tests/validators/test_plugin_hook_risk_bounds.py@@ -76,25 +76,25 @@ def test_redos_matchers_are_classified_quickly() -> None:"(" * 17 + ".*" + ")*" * 17 + "!","(?:.?){64}" * 25,]- start = time.perf_counter()+ start = time.process_time()scopes = [matcher_scope(pattern) for pattern in patterns]- assert time.perf_counter() - start < 2.0+ assert time.process_time() - start < 2.0assert scopes[:3] == ["bash", "bash", "narrow"]assert scopes[3:] == ["all", "all"] # fail closed: nested too deep, over the work budgetdef test_remote_code_scan_is_linear_on_padded_scripts() -> None:- start = time.perf_counter()+ start = time.process_time()for payload in ("curl ", "iex ", "bash $(", "python3.", "curl -o a | ", "'"):_shell_facts((payload * (MAX_SCRIPT_BYTES // len(payload)))[: MAX_SCRIPT_BYTES - 10])- assert time.perf_counter() - start < 3.0+ assert time.process_time() - start < 10.0def test_escape_decoding_is_linear_on_a_long_backslash_run() -> None:# A run of backslashes with no \u or \x after it used to be rescanned from every backslash (quadratic).- start = time.perf_counter()+ start = time.process_time()assert _allow_shapes("# " + "\\" * 65536) == frozenset()- assert time.perf_counter() - start < 1.0+ assert time.process_time() - start < 1.0# Escapes are still decoded, also behind several backslashes, so an escaped 'allow' still counts.assert _allow_shapes('echo {"permissionDecision": "\\u0061llow"}') == {"claude"}assert _allow_shapes('echo {\\"permissionDecision\\": \\"\\\\u0061llow\\"}') == {"claude"}@@ -106,10 +106,10 @@ def test_a_hook_script_of_backslashes_at_the_scan_limit_is_scanned_quickly(tmp_pcommand = {"type": "command", "command": "sh ${CLAUDE_PLUGIN_ROOT}/scripts/check.sh"}root = _plugin(tmp_path, {"hooks/hooks.json": _pre_tool_use(command), "scripts/check.sh": script})- start = time.perf_counter()+ start = time.process_time()checks = _checks(_validate(root))- assert time.perf_counter() - start < 10.0+ assert time.process_time() - start < 10.0assert "plugin_hook_script_unanalyzed" not in checks