Skip to content

Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.

fix(ci): require Semgrep in scanner subprocesses (#324)

Christopher Kevine4cda13a85f3
diff --git a/CHANGELOG.md b/CHANGELOG.md
index a0b29f9b..a2b1cb20 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -243,6 +243,9 @@ All notable changes to SkillEvaluator are documented in this file.
### Fixed
+- Reusable Tier 1 CI always requires Semgrep in scanner subprocesses, so local
+ optional-scanner defaults cannot hide missing CI code-scanning coverage.
+
- Use public, commit-pinned README image URLs so the wordmark and three-tier
diagram can render in PyPI package descriptions as well as on GitHub.
- On Windows, repository root and source-link detection no longer break for
diff --git a/ci/run_tier1.py b/ci/run_tier1.py
index 4408ebfb..9d2a8319 100644
--- a/ci/run_tier1.py
+++ b/ci/run_tier1.py
@@ -147,6 +147,8 @@ def child_environment(environ: dict[str, str]) -> tuple[dict[str, str], bool]:
env["PYTHON_DOTENV_DISABLED"] = "1"
env["SEMGREP_SEND_METRICS"] = "off"
env["SEMGREP_ENABLE_VERSION_CHECK"] = "0"
+ # The complete CI runtime requires Semgrep even when local installs make it optional.
+ env["SKILLEVALUATOR_REQUIRE_SEMGREP"] = "1"
env["NO_COLOR"] = "1"
Path(env["HOME"]).mkdir(parents=True, exist_ok=True)
if enabled == "true":
diff --git a/tests/ci/test_run_tier1.py b/tests/ci/test_run_tier1.py
index 432afdb3..072c29f5 100644
--- a/tests/ci/test_run_tier1.py
+++ b/tests/ci/test_run_tier1.py
@@ -481,6 +481,16 @@ def test_static_run_does_not_receive_ambient_model_credentials():
assert env["SEMGREP_ENABLE_VERSION_CHECK"] == "0"
[email protected]("llm", ["false", "true"])
[email protected]("ambient_requirement", [None, "0", "1"])
+def test_ci_requires_semgrep_even_when_ambient_configuration_disables_it(llm, ambient_requirement):
+ configuration = {"LLM_ENABLED": llm, "LLM_PROVIDER": "openai", "LLM_API_KEY": "test-key"}
+ if ambient_requirement is not None:
+ configuration["SKILLEVALUATOR_REQUIRE_SEMGREP"] = ambient_requirement
+ env, _ = runner.child_environment(configuration)
+ assert env["SKILLEVALUATOR_REQUIRE_SEMGREP"] == "1"
+
+
def test_incomplete_runtime_rollup_is_not_mislabeled_as_inconsistent_evidence(tmp_path):
workspace = tmp_path / "workspace"
skill(workspace, "skills/a")