Skip to content

Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.

fix(ci): align setup selection and paid-run guidance (#481)

Narendran Raghavan63eaccd8423b
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 4bfedbcd..d8ef26de 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -56,8 +56,10 @@ All notable changes to SkillEvaluator are documented in this file.
root skips the paid jobs (LLM-enriched Tier 1, Tier 2 and Tier 3): the keyless
scan remains the Tier 1 gate and reports list Tier 2 and Tier 3 as skipped.
GitHub fork PRs never run the paid tiers and keep their existing Tier 2/3
- reason. The enriched scan applies reviewed security baselines like the keyless
- scan.
+ reason. Setup guidance explains when provider configuration is exercised and
+ how to request an eligible all-skill check before merging.
+ When tiers run, GitLab keyless and native selectors both include changed
+ setup gitlinks despite committed submodule ignore settings.
- HTML, Markdown, and terminal findings show recorded LLM verification verdicts,
confidence, and reasoning, with an explanation of retained or downgraded severity.
diff --git a/ci/gitlab/all-tiers.yml b/ci/gitlab/all-tiers.yml
index dbdab87b..7912856a 100644
--- a/ci/gitlab/all-tiers.yml
+++ b/ci/gitlab/all-tiers.yml
@@ -100,7 +100,7 @@ skill-evaluation:
base, head = os.environ["DIFF_BASE_SHA"], os.environ["DIFF_HEAD_SHA"]
if any(not re.fullmatch(r"[0-9a-f]{40}", value) for value in (base, head)):
raise SystemExit("Changed mode requires exact base and head commits")
- changed = _git(Path(os.environ["CI_PROJECT_DIR"]).resolve(), "diff", "--name-only", "-z", "--no-renames", base, head, "--").split(b"\0")
+ changed = _git(Path(os.environ["CI_PROJECT_DIR"]).resolve(), "diff", "--name-only", "-z", "--no-renames", "--ignore-submodules=none", base, head, "--").split(b"\0")
setup = any(path.startswith((b".github/", b".gitlab/", b".skillevaluator/ci")) or path == b".gitlab-ci.yml" for path in changed)
print("all" if setup else "changed")
PYTHON
diff --git a/ci/run_tiers.py b/ci/run_tiers.py
index b99b543e..cfb32d42 100644
--- a/ci/run_tiers.py
+++ b/ci/run_tiers.py
@@ -41,7 +41,7 @@ try:
from select_changed_skills import (
SelectionError,
_git,
- _paths,
+ changed_paths,
select_changed_skills,
setup_changed,
skill_roots,
@@ -55,7 +55,7 @@ except ModuleNotFoundError:
from ci.select_changed_skills import (
SelectionError,
_git,
- _paths,
+ changed_paths,
select_changed_skills,
setup_changed,
skill_roots,
@@ -108,7 +108,8 @@ def select_skills(workspace: Path, env: dict) -> tuple[list[Path], dict]:
if base and mode == "auto":
selected, selection = select_changed_skills(workspace, roots, base, head)
# Tier 1 applies the same setup predicate, so every tier that runs selects the same skills.
- changed = _paths(_git(workspace, "diff", "--name-only", "-z", "--no-renames", f"{base}...{head}", "--"))
+ # The shared list is the one the paid-job skip reads (gitlinks included).
+ changed = changed_paths(workspace, env, head)
if not setup_changed(changed):
return selected, selection
reason = "ci_configuration_changed"
diff --git a/ci/templates/all-tiers/README.md b/ci/templates/all-tiers/README.md
index 981b16b9..bacd5c79 100644
--- a/ci/templates/all-tiers/README.md
+++ b/ci/templates/all-tiers/README.md
@@ -281,7 +281,8 @@ contains only a secret name, never the key itself. The caller resolves
`secrets[vars.SKILL_EVAL_GATEWAY_SECRET]` into the reusable workflow's generic
`LLM_API_KEY`; other repositories can select their own gateway and secret names
during local onboarding. Missing
-prerequisites remain visible in the reports. A passing Tier 1 gate is not an
+prerequisites remain visible in the reports of runs that start the provider-backed
+jobs (see below for which PRs skip them). A passing Tier 1 gate is not an
all-tier evaluation pass. Forks retain keyless Tier 1 checks. On PRs, ordinary software changes
select no skills, skill changes select the affected skills, and CI configuration
changes select the collection. Pushes to `main` and manual runs scan the collection.
@@ -292,13 +293,26 @@ skill, and its report lists Tier 2 and Tier 3 as skipped because no skill
changed. Fork PRs never run the paid tiers, so their reports keep the existing
Tier 2 and Tier 3 reason.
+Because the generated caller only starts on skill or caller changes, the
+same-repository PRs that now skip the paid jobs are the setup PRs: onboarding,
+pin bumps, and model, secret or gateway changes to the caller. Such a PR no
+longer shows a wrong model or a missing key. The first provider-backed run is the push to the
+default branch after it merges, a manual run there, or the next PR that changes
+a skill. To exercise a caller change with providers before merging, set
+`scan-mode: all` in the caller on that PR and read the results in the run's
+artifact; that PR gets no automatic comment, because the listener approves only
+the default branch's caller. Before merging, restore `auto` and then refresh
+`trusted-caller-blobs` for the final caller.
+
If a provider returns HTTP 404, verify that the selected model accepts chat
requests with your account. Presence in the public model catalog alone does not
prove serving availability. Change the caller's `model` input to an accessible
model and refresh `trusted-caller-blobs` before retrying; do not change global
-provider defaults to troubleshoot one repository.
+provider defaults to troubleshoot one repository. A PR that only changes the
+caller skips the provider-backed jobs, so the new model is first exercised by the
+push after it merges, unless that PR temporarily uses `scan-mode: all` as described above.
-The setup PR can exercise checks, but its first automatic comment requires
+The setup PR runs the keyless Tier 1 scan of every skill, but its first automatic comment requires
the consumer feedback listener on `main` and a subsequent PR run. Repository
maintainers configure branch protection separately if they want the Tier 1
aggregate check to block merges.
diff --git a/docs/advanced-ci.mdx b/docs/advanced-ci.mdx
index 344622ca..ada63548 100644
--- a/docs/advanced-ci.mdx
+++ b/docs/advanced-ci.mdx
@@ -35,8 +35,15 @@ It also documents SkillEvaluator's own callers and alternative feedback filename
Only quality and security checks gate this CI integration. Tier 2 and Tier 3
remain advisory even when failed or incomplete; deprecated Tier 3 required settings are ignored.
-Missing credentials, runtime or datasets must remain visible as incomplete or
-needs-action evidence. A passing scan does not mean all evaluation stages passed.
+When a tier is selected, missing credentials, runtime or datasets remain visible
+as incomplete or needs-action evidence. A passing scan does not mean all
+evaluation stages passed.
+With `scan-mode: auto`, a pull request that changes no skill, such as a setup,
+pin, model or secret change, skips the provider-backed jobs. Their prerequisites
+are first checked by the push to the default branch after it merges, a manual
+run there, or the next skill change. To check them before merging, set
+`scan-mode: all` (GitLab: `SKILL_EVAL_SCAN_MODE: all`) on that pull request and
+restore `auto` before merging.
Reports retain native evidence and an aggregate artifact for 14 days. Feedback
links to the authenticated artifact for that run; download and extract it,
diff --git a/docs/tier1-validation.mdx b/docs/tier1-validation.mdx
index 68d5b8a2..f8ee09b3 100644
--- a/docs/tier1-validation.mdx
+++ b/docs/tier1-validation.mdx
@@ -29,8 +29,11 @@ as a required gate. GitHub calls the hardened legacy scanner workflow at the
same evaluator commit; GitLab runs the same digest-pinned scanner image.
Static CI checks receive no model key. Tier 2 and Tier 3 execute independently
and retain their own native evidence, so one tier's failure does not erase the
-other reports. Changes to onboarding configuration trigger a full Tier 1 scan
-so the setup review exercises actual skills.
+other reports. Changes to onboarding configuration trigger a full keyless
+Tier 1 scan so the setup review exercises actual skills. With `scan-mode: auto`,
+the provider-backed LLM Tier 1 pass, Tier 2 and Tier 3 run on a pull request only
+when it changes a skill. Eligible pushes and manual runs on the default branch
+run them for the whole collection.
Existing released Tier 1-only consumers keep their current workflow and
bundled image pins. See [GitHub Actions](github-actions.mdx) or
diff --git a/tests/ci/test_all_tier_workflows.py b/tests/ci/test_all_tier_workflows.py
index 855a479f..2b373a67 100644
--- a/tests/ci/test_all_tier_workflows.py
+++ b/tests/ci/test_all_tier_workflows.py
@@ -577,6 +577,8 @@ def test_gitlab_tier1_and_native_tiers_select_the_same_skills(tmp_path):
"workflow": ([(".github/workflows/skills.yml", "name: skills\n")], []),
"root pipeline": ([(".gitlab-ci.yml", "include: []\n")], []),
"renamed feedback include": ([], [(".gitlab/skill-feedback.yml", "docs/skill-feedback.yml")]),
+ "ignored setup gitlink": ([("skills/alpha/SKILL.md", "# alpha with setup gitlink\n")], []),
+ "visible setup gitlink": ([("skills/alpha/SKILL.md", "# alpha with setup gitlink\n")], []),
}
for name, (writes, renames) in cases.items():
repo = tmp_path / name.replace(" ", "-")
@@ -588,7 +590,15 @@ def test_gitlab_tier1_and_native_tiers_select_the_same_skills(tmp_path):
(repo / "skills" / skill / "SKILL.md").write_text(f"# {skill}\n")
(repo / ".gitlab").mkdir()
(repo / ".gitlab/skill-feedback.yml").write_text("include: []\n" * 20)
+ gitlink = name in {"ignored setup gitlink", "visible setup gitlink"}
+ if gitlink:
+ ignore = "all" if name == "ignored setup gitlink" else "none"
+ (repo / ".gitmodules").write_text(
+ f'[submodule "actions"]\n path = .github/actions\n url = ./actions\n ignore = {ignore}\n'
+ )
subprocess.run([*git, "add", "-A"], check=True)
+ if gitlink:
+ subprocess.run([*git, "update-index", "--add", "--cacheinfo", f"160000,{'a' * 40},.github/actions"], check=True)
subprocess.run([*git, "commit", "-qm", "base"], check=True)
base = subprocess.check_output([*git, "rev-parse", "HEAD"], text=True).strip()
for relative, text in writes:
@@ -598,6 +608,8 @@ def test_gitlab_tier1_and_native_tiers_select_the_same_skills(tmp_path):
(repo / target).parent.mkdir(parents=True, exist_ok=True)
subprocess.run([*git, "mv", source, target], check=True)
subprocess.run([*git, "add", "-A"], check=True)
+ if gitlink:
+ subprocess.run([*git, "update-index", "--add", "--cacheinfo", f"160000,{'b' * 40},.github/actions"], check=True)
subprocess.run([*git, "commit", "-qm", "head"], check=True)
head = subprocess.check_output([*git, "rev-parse", "HEAD"], text=True).strip()
tier1 = subprocess.run(
@@ -616,3 +628,5 @@ def test_gitlab_tier1_and_native_tiers_select_the_same_skills(tmp_path):
assert (native == "all") == (name not in {"skill only", "consumer ci script"}), name
if native == "changed":
assert [p.name for p in selected] == ["alpha"], name
+ else:
+ assert [p.name for p in selected] == ["alpha", "beta"], name
diff --git a/tests/ci/test_run_tiers.py b/tests/ci/test_run_tiers.py
index d61523cb..ba504554 100644
--- a/tests/ci/test_run_tiers.py
+++ b/tests/ci/test_run_tiers.py
@@ -186,6 +186,23 @@ def test_setup_workflow_only_change_selects_full_collection(consumer):
assert selection["reason"] == "ci_configuration_changed"
+def test_hidden_setup_gitlink_still_selects_full_collection(consumer):
+ """Setup detection reads the same list as the paid-job skip, so .gitmodules cannot hide it."""
+ base = subprocess.check_output(["git", "-C", str(consumer), "rev-parse", "HEAD"], text=True).strip()
+ (consumer / ".gitmodules").write_text(
+ '[submodule "actions"]\n\tpath = .github/actions\n\turl = ./actions\n\tignore = all\n'
+ )
+ git = ["git", "-C", str(consumer), "-c", "user.name=CI tests", "-c", "[email protected]"]
+ subprocess.run([*git, "add", ".gitmodules"], check=True)
+ subprocess.run([*git, "update-index", "--add", "--cacheinfo", f"160000,{'b' * 40},.github/actions"], check=True)
+ subprocess.run([*git, "commit", "-m", "setup gitlink"], check=True, capture_output=True)
+ selected, selection = run_tiers.select_skills(
+ consumer, env_for(consumer, DIFF_BASE_SHA=base, SKILL_EVAL_SCAN_MODE="auto")
+ )
+ assert [p.name for p in selected] == ["alpha", "beta"]
+ assert selection["reason"] == "ci_configuration_changed"
+
+
def test_changed_selection_without_changes_is_explicitly_skipped(consumer, tmp_path):
env = env_for(consumer)
env["DIFF_BASE_SHA"] = env["REPORT_COMMIT"]