Skip to content

Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.

Add reusable GitLab skill evaluation and repository onboarding (#248)

Christopher Kevin34d84cd5f350
diff --git a/.github/workflows/evaluate-skills.yml b/.github/workflows/evaluate-skills.yml
index 728d47f2..e874b2b0 100644
--- a/.github/workflows/evaluate-skills.yml
+++ b/.github/workflows/evaluate-skills.yml
@@ -14,6 +14,10 @@ on:
description: Newline-separated skill directories or roots containing skills
type: string
default: skills
+ min-score:
+ description: Minimum Tier 1 quality score from 0 to 100
+ type: number
+ default: 70
llm:
description: Enable additional LLM security, verification, and rubric checks
type: boolean
@@ -41,8 +45,8 @@ on:
secrets:
LLM_API_KEY:
required: false
-permissions:
- contents: read
+# Inherit the caller's permissions: basic scans need only contents: read.
+# Hosted reports additionally opt in to id-token: write in the caller.
jobs:
tier1:
name: Tier 1 scan
@@ -51,6 +55,51 @@ jobs:
env:
SCAN_MODE: ${{ inputs.scan-mode == 'auto' && (github.event_name == 'pull_request' && 'changed' || 'all') || inputs.scan-mode }}
steps:
+ - name: Capture the optional hosted-report execution receipt
+ id: provenance
+ continue-on-error: true
+ run: |
+ python3 -I - <<'PY'
+ import json, os, re, tempfile, urllib.parse, urllib.request
+
+ url = os.environ.get("ACTIONS_ID_TOKEN_REQUEST_URL")
+ credential = os.environ.get("ACTIONS_ID_TOKEN_REQUEST_TOKEN")
+ if not url or not credential:
+ print("Hosted-report receipt is disabled; scan artifacts remain available.")
+ raise SystemExit(0)
+
+ class NoRedirect(urllib.request.HTTPRedirectHandler):
+ def redirect_request(self, *args, **kwargs):
+ return None
+
+ try:
+ parsed = urllib.parse.urlsplit(url)
+ if (parsed.scheme != "https" or not parsed.hostname
+ or not parsed.hostname.endswith(".actions.githubusercontent.com")
+ or parsed.port not in (None, 443) or parsed.username or parsed.password
+ or parsed.fragment):
+ raise ValueError("Unexpected GitHub identity endpoint")
+ audience = "urn:skillevaluator:report-provenance:v1"
+ url += ("&" if parsed.query else "?") + "audience=" + urllib.parse.quote(audience, safe="")
+ request = urllib.request.Request(url, headers={"Authorization": "Bearer " + credential})
+ with urllib.request.build_opener(NoRedirect()).open(request, timeout=30) as response:
+ payload = response.read(65537)
+ if len(payload) > 65536:
+ raise ValueError("Oversized identity response")
+ token = json.loads(payload)["value"]
+ if (not isinstance(token, str) or len(token) > 32768
+ or not re.fullmatch(r"[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+", token)):
+ raise ValueError("Invalid identity response")
+ directory = tempfile.mkdtemp(prefix="skill-provenance-", dir=os.environ["RUNNER_TEMP"])
+ descriptor = os.open(os.path.join(directory, "execution.jwt"), os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
+ with os.fdopen(descriptor, "w") as handle:
+ handle.write(token)
+ with open(os.environ["GITHUB_OUTPUT"], "a") as handle:
+ handle.write("directory=" + directory + "\n")
+ print("Captured the signed execution receipt for hosted reports.")
+ except (OSError, ValueError, KeyError, TypeError):
+ raise SystemExit("Could not capture the hosted-report receipt. Scan artifacts remain available; hosting will be skipped.") from None
+ PY
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0
with:
persist-credentials: false
@@ -63,7 +112,7 @@ jobs:
env:
IMAGE: ${{ inputs.image }}
run: |
- python3 - <<'PY'
+ python3 -I - <<'PY'
import os, re
if not re.fullmatch(r"ghcr\.io/[a-z0-9][a-z0-9._/-]*@sha256:[0-9a-f]{64}", os.environ["IMAGE"]):
raise SystemExit("image must be a published GHCR image pinned to its sha256 digest")
@@ -80,6 +129,7 @@ jobs:
DIFF_BASE_SHA: ${{ github.event.pull_request.base.sha }}
DIFF_HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }}
SKILL_PATHS: ${{ inputs.paths }}
+ MIN_SCORE: ${{ inputs.min-score }}
LLM_ENABLED: ${{ inputs.llm }}
LLM_PROVIDER: ${{ inputs.llm-provider }}
LLM_MODEL: ${{ inputs.llm-model }}
@@ -107,8 +157,8 @@ jobs:
--cap-drop ALL --security-opt no-new-privileges \
--mount "type=bind,src=$GITHUB_WORKSPACE,dst=/workspace,readonly" \
--mount "type=bind,src=$RUNNER_TEMP/skill-reports,dst=/reports" \
- --env SKILL_PATHS --env LLM_ENABLED --env LLM_PROVIDER --env LLM_MODEL "${llm_secret_args[@]}" \
- --env SCAN_MODE --env DIFF_BASE_SHA --env DIFF_HEAD_SHA --env SCAN_WORKERS \
+ --env SKILL_PATHS --env MIN_SCORE --env LLM_ENABLED --env LLM_PROVIDER --env LLM_MODEL "${llm_secret_args[@]}" \
+ --env SCAN_MODE --env DIFF_BASE_SHA --env DIFF_HEAD_SHA \
--env CALLER_WORKFLOW_SHA \
--env GITHUB_ACTIONS --env REPORT_IMAGE --env REPORT_URL --env REPORT_RUN_URL --env REPORT_COMMIT \
"$IMAGE"
@@ -128,14 +178,11 @@ jobs:
path: ${{ runner.temp }}/skill-reports/
if-no-files-found: warn
retention-days: 14
- # The optional report publisher downloads only this JSON evidence, not the full reports.
- - name: Upload publisher evidence even when validation fails
- if: always()
+ - name: Retain the separate hosted-report execution receipt
+ if: ${{ always() && steps.provenance.outputs.directory != '' }}
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
with:
- name: ${{ inputs.artifact-name }}-evidence-attempt-${{ github.run_attempt }}
- path: |
- ${{ runner.temp }}/skill-reports/summary.json
- ${{ runner.temp }}/skill-reports/skill-*/skillevaluator-tier1.json
- if-no-files-found: warn
+ name: ${{ inputs.artifact-name }}-provenance-attempt-${{ github.run_attempt }}
+ path: ${{ steps.provenance.outputs.directory }}/execution.jwt
+ if-no-files-found: error
retention-days: 14
diff --git a/.github/workflows/publish-skill-reports.yml b/.github/workflows/publish-skill-reports.yml
index 05d9f26e..d50c5c13 100644
--- a/.github/workflows/publish-skill-reports.yml
+++ b/.github/workflows/publish-skill-reports.yml
@@ -80,10 +80,8 @@ jobs:
# No dependency cache is restored: other default-branch workflows can write
# this job's cache scope, and uv does not recheck files from its cache.
run: |
- python3 -m venv "$RUNNER_TEMP/uv"
- "$RUNNER_TEMP/uv/bin/python" -m pip install --disable-pip-version-check \
- --require-hashes --no-deps --only-binary :all: -r ci/requirements/uv.txt
- "$RUNNER_TEMP/uv/bin/uv" sync --frozen --no-dev --no-editable
+ python3 -m pip install --require-hashes -r ci/requirements/uv.txt
+ uv sync --frozen --no-dev --no-editable --extra publisher
- name: Checkout retained report history
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0
with:
diff --git a/.github/workflows/publish-tier1-image.yml b/.github/workflows/publish-tier1-image.yml
index 72b110db..f6d7a517 100644
--- a/.github/workflows/publish-tier1-image.yml
+++ b/.github/workflows/publish-tier1-image.yml
@@ -167,7 +167,7 @@ jobs:
docker --config "$RUNNER_TEMP/anonymous-docker-config" pull "$published_image"
python3 ci/render_templates.py --workflow-ref "$GITHUB_SHA" \
--image "$published_image" --output "$RUNNER_TEMP/onboarding"
- printf 'Published image: %s. Download **github-actions-onboarding** for pinned consumer workflows.\n' \
+ printf 'Published image: %s. Download **github-actions-onboarding** for pinned GitHub and GitLab configurations.\n' \
"$published_image" >> "$GITHUB_STEP_SUMMARY"
- name: Upload pinned onboarding templates
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
diff --git a/CHANGELOG.md b/CHANGELOG.md
index e798397e..0558a287 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -6,42 +6,8 @@ All notable changes to SkillEvaluator are documented in this file.
### Added
-- Opt-in `SKILL_EVAL_JUDGE_CHARACTER_FACTS=1` adds bounded, escaped Unicode
- character facts to Tier 3 accuracy, goal-accuracy, and behavior-check prompts
- without normalizing the evidence. The operator setting is verifier-only;
- enabling it selects the prompt-based goal judge instead of the optional RAGAS
- route. Metadata uses the same text rendering as the judge prompts, including
- numeric and nullable fields. Existing prompts remain unchanged when the
- setting is unset.
-
-- `tier3 evaluate` now leads with a case-first summary: a verdict line and one
- row per `evals.json` case. With one agent, each row shows, for with and
- without the skill, the pass count, per-attempt marks (`✓` pass, `✗` fail,
- `⊘` did not run, `·` skipped after a pass) and mean score, plus an effect
- column (`helps`, `hurts`, `both fail`, `better score`, `worse score`,
- `no change`, `inconclusive`). With several agents, each agent gets a column
- showing without-skill → with-skill marks and the pass-rate change. Success
- rate, quality score, and dimensions follow. A "How to improve" list follows,
- ranked by attempts lost and capped at three; each item names the case,
- agent, issue type (`skill`, `skill not used`, `skill hurts`, `environment`,
- `run error`, `eval case`, `missing input`, `goal not met`), one action, and an evidence
- path, and is also written to `what_to_fix.json` in the run directory. When
- nothing needs fixing it prints a one-line "nothing to fix" instead. The
- verdict is `inconclusive` when attempts did not run. The per-evaluator and
- per-dimension tables and metric findings moved behind the new `--verbose`
- flag and remain in `report.html`.
-- `goal not met` flags with-skill attempts that pass on the aggregate score
- while the judge scored goal accuracy 0.00 (for example, the agent only asked
- for details).
-- `report.html` leads with the same case-based findings: "Verdict at a glance"
- adds the per-case result, the highest-scoring agent versus the biggest skill
- lift, and the goal-not-met count; "Conclusion & Next Steps" and the Insights
- recommendations list the `what_to_fix.json` fixes (or "Nothing to fix").
- Data fixes elsewhere: Pass@K shows attempts passed, a regressed passing
- dimension reads "PASS · regressed" with its weakest signal, resource totals
- are formatted with units and percent change, Attempt Details groups each
- case's with/without rows and marks passes that missed the goal, and Error
- Recovery lists only trials with faults, retries or warnings.
+- Add reusable GitLab CI onboarding and configurable Tier 1 quality thresholds,
+ alongside a shared repository integration guide.
- Reusable GitHub Actions onboarding for complete Tier 1 checks, with
pinned runtime/workflow consumer templates, affected-skill PR selection,
optional hosted reports and updated PR feedback. The new guide separates
@@ -79,6 +45,14 @@ All notable changes to SkillEvaluator are documented in this file.
### Changed
+- Hosted GitHub reports verify a separate GitHub-signed execution receipt.
+ Enable `id-token: write` in the scan workflow and refresh its approved blob
+ when adding hosting; ordinary scans keep their existing read-only permissions.
+- Link GitHub and GitLab source templates directly from the README and onboarding
+ guides. Keep report, log and PR comment examples in the public repository,
+ removing onboarding dependencies on personal demo links.
+- GitHub onboarding shows the PR comment example directly on the page and
+ explains the two templates and required one-time publisher setup.
- HTML reports use a verdict-led layout with a sidebar and expandable check
details. The verdict replaces the pass percentage; checks marked Not
applicable are excluded from passed and executed counts. Publication status
diff --git a/README.md b/README.md
index 9843fc7f..116a2adb 100644
--- a/README.md
+++ b/README.md
@@ -10,17 +10,33 @@
> 📺 **Livestream:** [Ask the Experts: Evaluating Agent Skills](https://www.youtube.com/watch?v=4rI0zATFxYI)
-SkillEvaluator is an open-source tool that checks AI agent skills and measures their effect on an agent's
-task results. Start with a quality check or model-backed feedback, then compare
-live runs with and without a skill.
+SkillEvaluator is an open-source tool that checks AI agent skills and measures their effect on task results:
+start with quality checks, then compare live runs with and without a skill.
Agent skills are folders of instructions and supporting files defined by the
[Agent Skills specification](https://agentskills.io/). SkillEvaluator is part of
the [NVIDIA Verified Skills pipeline](https://github.com/NVIDIA/skills).
+## Bring skill evaluation to your repository
+
+Automatically check changed skills in GitHub pull requests or GitLab merge
+requests, with Tier 1 quality checks, SkillSpector security scanning and retained
+reports. Choose your skill folders and minimum score. No evaluation code, local
+installation or onboarding CLI is required: [copy a template](ci/templates/README.md).
+
+| Your repository | Source template | Copy destination |
+| --- | --- | --- |
+| GitHub | [Skill checks](ci/templates/skill-checks.yml) | `.github/workflows/skill-checks.yml` |
+| GitLab | [Skill checks](ci/templates/gitlab-skill-checks.yml) | `.gitlab-ci.yml`, or merge into your existing pipeline |
+| GitHub reports and PR comments | [Optional publisher](ci/templates/publish-skill-reports.yml) | Add a second workflow; complete publisher setup |
+
+Source templates contain release placeholders; use the filled-in copies from an
+official CI image release. See [setup and availability](docs/repository-integration.mdx)
+and [public report, log and comment examples](ci/examples/README.md).
+
## Quickstart
-Choose a starting path. Both use the bundled `text-analyzer` skill; try your own afterward.
+Both paths use the bundled `text-analyzer` skill.
| Start here | Requires | First result |
| --- | --- | --- |
@@ -35,8 +51,7 @@ git clone --depth 1 https://github.com/NVIDIA/SkillEvaluator.git
cd SkillEvaluator
```
-uv provisions Python 3.13 and installs each path's dependencies. Run from this
-checkout. For a standalone CLI, see
+uv provisions Python 3.13 and dependencies. Run from this checkout; for a standalone CLI, see
[Installation](https://docs.nvidia.com/skills/skillevaluator/installation).
### Run without an LLM
@@ -47,25 +62,21 @@ uv run --python 3.13 skillevaluator quality-check \
-r cli -r json -o reports/first-quality
```
-No API key, external scanner, GPU, or Docker is needed. The example scores
-**84.5/100 (B)** and exits `0`, with seven non-blocking findings. The default
-passing threshold is 70. Read the terminal or saved
-`reports/first-quality/skillevaluator-quality.json` report.
+No API key, scanner, GPU or Docker is needed. The example scores **84.5/100 (B)**,
+exits `0` and saves `reports/first-quality/skillevaluator-quality.json`. The
+passing threshold defaults to 70.
This checks static quality; publication evidence stays `INCOMPLETE`. Security
-coverage needs scanners, and agent impact needs live evaluation. To check your
-own skill, replace the example path with your `SKILL.md` directory. See the
+needs scanners; agent impact needs live evaluation. Replace the example path
+with your skill directory. See the
[keyless walkthrough](docs/quickstart.mdx#run-without-an-llm).
### Use NVIDIA Build
-NVIDIA Developer Program members can use eligible NVIDIA-hosted endpoints for
-**free prototyping**, subject to current access terms and rate limits. See the
-[official access FAQ](https://docs.api.nvidia.com/nim/docs/product) and your
-NVIDIA Build account for details.
+Eligible NVIDIA Developer Program members can use NVIDIA-hosted endpoints for
+**free prototyping**, subject to [access terms and rate limits](https://docs.api.nvidia.com/nim/docs/product).
-1. Sign in or create an account at [build.nvidia.com](https://build.nvidia.com/)
- and complete any enrollment prompts.
+1. Sign in at [build.nvidia.com](https://build.nvidia.com/) and complete enrollment.
2. Open a model page, select **Get API Key**, and generate your key.
3. Set the provider and key in the same terminal, then run the rubric check:
@@ -77,14 +88,12 @@ uv run --python 3.13 --extra llm skillevaluator rubric-eval \
-r cli -r json -o reports/first-build
```
-The `llm` extra adds model clients. The command sends skill documentation to the
-model and writes feedback to the terminal and
+The command sends skill documentation to the model and saves feedback in
`reports/first-build/skillevaluator-rubric.json`. No agent runtime or Docker is
-needed. Completed evaluations can still fail rubric thresholds; inspect the findings.
+needed. Completed evaluations can still fail rubric thresholds.
-Build supplies model defaults; the same key supports Tier 2 embeddings and
-supported Tier 3 agents. See the [Build walkthrough](docs/quickstart.mdx#use-nvidia-build)
-or [Providers & Credentials](docs/configuration.mdx) for overrides and other providers.
+The same key supports Tier 2 embeddings and supported Tier 3 agents. See the
+[Build walkthrough](docs/quickstart.mdx#use-nvidia-build) or [provider configuration](docs/configuration.mdx).
## Three-tier overview
@@ -106,9 +115,8 @@ Missing required scanner evidence is reported as `INCOMPLETE` and exits nonzero.
security scanning; [Harbor](https://github.com/harbor-framework/harbor), the
open-source agent evaluation framework, powers sandboxed agent runs.
-Tier 2 reports when inter-skill comparison did not run. Tier 3 reuses valid
-evaluation sources or creates one starter case, then runs with and without the
-skill. Starter cases provide limited coverage. See the
+Tier 2 reports when inter-skill comparison did not run. Tier 3 reuses evaluation
+sources or creates one starter case with limited coverage. See the
[tier guides](https://docs.nvidia.com/skills/skillevaluator/) for details.
## Run the combined pipeline
@@ -120,17 +128,14 @@ uv run --python 3.13 --extra all skillevaluator doctor --env-mode docker
uv run --python 3.13 --extra all skillevaluator validate ./my-skill
```
-`doctor` checks readiness, not inference success. `validate` runs all tiers
-with autopilot by default for skills; `--full` remains supported but is unnecessary.
-Use `--tiers 1,2` or `--no-tier3` to skip live evaluation, or `--tiers 1` for static
-checks. `--no-autopilot` requires an existing source. Other artifact types retain
-their applicable stages without automatically enabling skill evaluation.
+`doctor` checks readiness, not inference success. `validate` runs all tiers with
+autopilot for skills. Use `--tiers 1,2` to skip live evaluation or `--tiers 1` for
+static checks. `--no-autopilot` requires an existing source. Other artifact types
+retain their applicable stages.
-Tier 1 always gates `validate`. Tier 2 gates by default;
-`--no-block-on-dedup` makes its findings advisory. Tier 3 is advisory by default;
-`--block-on-agent-eval` makes its findings gate too. Model and managed-sandbox
-usage depends on the selected provider's terms and can incur charges. Start with
-one agent and a small dataset; see [runtime and cost](docs/tier3-live-evaluation.mdx#plan-for-cost).
+Tier 1 always gates `validate`. Tier 2 gates unless `--no-block-on-dedup` is set;
+Tier 3 gates only with `--block-on-agent-eval`. Model and managed-sandbox usage
+can incur charges; see [runtime and cost](docs/tier3-live-evaluation.mdx#plan-for-cost).
For broader coverage, generate and review additional cases. Templates omit
negatives unless authored in `EVAL.md`:
@@ -151,15 +156,13 @@ Resources: [paper](https://arxiv.org/abs/2608.20614),
[DOI](https://doi.org/10.48550/arXiv.2608.20614),
[BibTeX](https://arxiv.org/bibtex/2608.20614), and the
[NVIDIA Developer blog](https://developer.nvidia.com/blog/evaluating-ai-agent-skill-performance-with-nvidia-skillevaluator/).
-Cite the paper for methodology and results. For reproducibility, record the
-SkillEvaluator release or commit. See [CITATION.cff](CITATION.cff).
+Cite the paper and record the SkillEvaluator release or commit; see [CITATION.cff](CITATION.cff).
## Documentation
See [the documentation](https://docs.nvidia.com/skills/skillevaluator/) for
setup, tier guides, reports, CI integration, CLI reference, and
[performance dashboard](docs/reports.mdx#performance-dashboard).
-For reusable CI, see [GitHub Actions](docs/github-actions.mdx).
## Installation and third-party software
@@ -178,16 +181,14 @@ Read [CONTRIBUTING.md](CONTRIBUTING.md) and run:
make lint && make test && make build
```
-Project governance is described in [GOVERNANCE.md](GOVERNANCE.md). Participation
-is governed by the [Code of Conduct](CODE_OF_CONDUCT.md).
+See [project governance](GOVERNANCE.md) and the [Code of Conduct](CODE_OF_CONDUCT.md).
## Support
-Support level: **Experimental**. SkillEvaluator is community-supported on a
-best-effort basis with no SLA or NVIDIA enterprise support entitlement. Report
-reproducible bugs and feature requests through
+Support level: **Experimental**. Community-supported, best-effort, with no SLA
+or NVIDIA enterprise support entitlement. Report bugs and feature requests through
[GitHub Issues](https://github.com/NVIDIA/SkillEvaluator/issues); see
-[SUPPORT.md](SUPPORT.md) for details.
+[SUPPORT.md](SUPPORT.md).
## Security
diff --git a/ci/README.md b/ci/README.md
index 40a5f659..9d187b8c 100644
--- a/ci/README.md
+++ b/ci/README.md
@@ -1,11 +1,17 @@
-# Reusable GitHub Actions CI
+# Reusable repository CI
-This directory supplies the public GitHub CI runtime and consumer
-onboarding templates. Start with the [short GitHub Actions guide](../docs/github-actions.mdx).
+This directory supplies the shared Tier 1 CI runtime and consumer onboarding
+templates. Start with [repository integration](../docs/repository-integration.mdx)
+for GitHub Actions or GitLab CI/CD. The GitLab adapter is in `ci/gitlab/`;
+the image and report generation are shared by both platforms.
The upstream workflow release and public package must be reviewed and published
before consumers can use this path. Source templates are not runnable while
`RELEASE_WORKFLOW_SHA` or `RELEASE_IMAGE_DIGEST` remains unresolved.
+For manual setup, start with the [copyable template catalog](templates/README.md).
+The [report and feedback examples](examples/README.md) are kept here in the public
+repository. Neither requires a local onboarding CLI or personal-demo access.
+
## Consumer contract
Copy the release-generated `skill-checks.yml` to
@@ -115,19 +121,23 @@ Pages is not an inherent requirement for GitHub PR comments.
`NVIDIA/SkillEvaluator`; an explicitly reviewed fork override is available
for integration testing, not needed for standard onboarding.
6. Generated release files include the caller blob for their unedited scan
- template. After **any edits** to that consumer file, compute its exact Git blob:
+ template. For hosting, first uncomment `id-token: write` under `permissions`
+ in `.github/workflows/skill-checks.yml`. This lets GitHub sign the workflow's
+ execution identity; no personal token or cloud account is needed.
+ After **any edits** to that consumer file, compute its exact Git blob:
`git hash-object .github/workflows/skill-checks.yml`. Put that value in
`trusted-caller-blobs`. Every later scan-file edit needs review and a new
approved blob. Preserve older reviewed SHA/blob entries only when needed for
historical backfill. Both allowlists accept comma- or newline-separated
- entries; do not copy the personal demo's trust hashes. The publisher verifies
- the caller from recorded PR base/head revisions. Conflicting workflow edits
- or unavailable historical metadata are rejected: update/rebase the consumer
- branch and rerun its scan.
+ entries; compute trust hashes from your own reviewed configuration. The
+ publisher verifies a separate GitHub-signed execution receipt and checks the
+ caller at that receipt's immutable workflow revision. It never reconstructs
+ the executed workflow from a pull request's current base or head.
7. The publisher caller must grant `contents: write`, `actions: read`,
`pages: write`, `id-token: write` and `pull-requests: write`. Reusable jobs
narrow these to history preparation, Pages deployment and PR feedback. The
- scan remains `contents: read`. The automatic GitHub token is sufficient;
+ scan uses `contents: read`, plus `id-token: write` only when hosting is enabled.
+ The automatic GitHub token is sufficient;
no personal access token or model credential is required for publication.
The publisher rebuilds native HTML from bounded, validated JSON using pinned
@@ -155,6 +165,19 @@ Fork PR scans can run without provider secrets; fork PR bot comments and current
PR aliases are not enabled. Comment-only operation and hosting alongside an
arbitrary existing Pages deployment need further integration work.
+Basic scans do not request identity credentials. If GitHub withholds the signing
+permission (including under fork policies), the scan and downloadable reports
+remain available, but hosting is skipped with a setup message. Historical runs
+without a signed receipt cannot be backfilled. Commit the updated configuration
+and trigger a new scan from that revision; rerunning an old job keeps its old
+workflow. Receipts are separate,
+attempt-specific artifacts and are never copied to the report site. The publisher
+checks their signature, repository, attempt, workflow revisions and issuance
+time; token expiration does not invalidate historical evidence. If GitHub has
+retired the signing key, that evidence can no longer be verified and publication
+stops. The receipt establishes execution identity; the approved complete caller
+workflow and GitHub artifact metadata also remain required.
+
## Optional LLM checks
The generated `skill-checks-llm.yml` is a separate trusted manual run. Set
@@ -172,10 +195,9 @@ then.
1. Review and merge source, wrapper, scanner pins, renderer and reusable
workflows. Run focused selection/report/trust tests plus practical E2E
- checks. A personal-fork demo is prior evidence, not proof of this new upstream
- revision. Its [consumer](https://github.com/chrisknvidia/skill-evaluator-consumer-demo)
- illustrates the earlier full-path scan and hosted-feedback flow; it does not
- establish changed-skill selection for a newer release.
+ checks using the candidate revision. The [public examples](examples/README.md)
+ illustrate the report and feedback surfaces; they are not proof that a new
+ release has passed end-to-end verification.
2. From a reviewed commit on upstream `main`, manually run **Build and publish
Tier 1 image** with `publish: true`. Publishing is allowed only from `main`
in `NVIDIA/SkillEvaluator`; PR runs build and smoke-test without uploading.
@@ -205,7 +227,7 @@ then.
being treated as an absent image.
4. Select the reviewed immutable reusable-workflow/tools revision and the
verified image digest. Use `ci/render_templates.py` to generate release
- copies of the three files in `ci/templates/`, replacing
+ copies of the files in `ci/templates/`, replacing
`RELEASE_WORKFLOW_SHA` and `RELEASE_IMAGE_DIGEST` with those identities.
Follow the helper's current `--help` for its CLI. Reject unresolved release
placeholders or mutable references in generated files. The consumer's Pages
diff --git a/ci/examples/README.md b/ci/examples/README.md
new file mode 100644
index 00000000..f31b9d7c
--- /dev/null
+++ b/ci/examples/README.md
@@ -0,0 +1,70 @@
+# SkillEvaluator CI examples
+
+These examples live in the public NVIDIA/SkillEvaluator repository. You do not
+need access to a personal demo repository, private pipeline or hosted report to
+view them. For setup, [copy a template](../templates/README.md).
+
+## Report preview
+
+![SkillEvaluator report showing a failed skill with a quality score of 98.2](../../docs/assets/repository-ci-results-example.png)
+
+This captured example shows a skill scoring **98.2/100** against a **100-point**
+threshold. The scan fails because it is below the configured threshold. Quality
+score and scan verdict are separate; security and other required checks also
+affect the verdict. This preview is an example, not a result for your repository.
+
+GitHub and GitLab retain the HTML report as a job artifact. Open `index.html` in
+the downloaded artifact, then select a skill for its detailed findings.
+
+## GitHub Actions summary
+
+A shortened illustrative example:
+
+```text
+Tier 1 scan result: FAILED
+1 skill · 0 passed · 1 failed
+Problem: Quality score 98.2/100 is below the required 100/100
+```
+
+The full report identifies the quality checks that need attention. Improve the
+skill to meet the repository's chosen threshold, then submit the change again.
+
+## Optional GitHub PR comment
+
+The following is a shortened illustrative comment, not a live pull request:
+
+> **github-actions[bot]**
+>
+> **Tier 1 scan result: FAILED**
+>
+> 1 skill · 0 passed · 1 failed · 0 incomplete
+>
+> | Skill | Status | Quality / 100 |
+> | --- | --- | --- |
+> | `skills/glossary-builder` | FAIL | 98.2 / A |
+>
+> **Problem:** Quality score 98.2/100 is below the required 100/100.
+>
+> **Suggested fix:** Open the skill report and address the quality findings.
+>
+> **Open full report** · **View workflow run**
+
+The last line shows link labels only. In an actual comment, they link to your
+repository's hosted report and workflow run. The publisher updates one comment
+on an eligible current same-repository PR after successful report publication.
+Follow the [optional publisher setup](../../docs/github-actions.mdx#enable-pr-comments).
+
+## GitLab job log
+
+A shortened excerpt corresponding to the report preview:
+
+```text
+Skill result: "skills/glossary-builder" — FAILED
+ Quality: 98.2/100, grade "A"
+ Check: "QUALITY" — "failed"
+ Scanner error: "Quality score 98.2/100 is below the required 100/100 ..."
+```
+
+Download `skill-evaluator-report` from the job for the complete findings. The
+GitLab integration provides job artifacts; automatic MR comments and hosted
+reports are not included.
diff --git a/ci/gitlab/skill-evaluation.yml b/ci/gitlab/skill-evaluation.yml
new file mode 100644
index 00000000..a4b02869
--- /dev/null
+++ b/ci/gitlab/skill-evaluation.yml
@@ -0,0 +1,66 @@
+# SPDX-License-Identifier: Apache-2.0
+# Reusable GitLab job. Include a reviewed commit and provide SKILL_EVAL_IMAGE.
+# Configure merge-request rules in the root .gitlab-ci.yml; GitLab requires it.
+variables:
+ SKILL_EVAL_PATHS: "skills"
+ SKILL_EVAL_SCAN_MODE: "auto"
+ SKILL_EVAL_MIN_SCORE: "70"
+
+skill-evaluation:
+ stage: test
+ image:
+ name: "$SKILL_EVAL_IMAGE"
+ entrypoint: [""]
+ variables:
+ GIT_DEPTH: "0"
+ LLM_ENABLED: "false"
+ script:
+ - |
+ set -eu
+ artifact_dir="$CI_PROJECT_DIR/skill-evaluator-report"
+ if [ -e "$artifact_dir" ] || [ -L "$artifact_dir" ]; then
+ echo "skill-evaluator-report already exists; choose a clean checkout for the report." >&2
+ exit 1
+ fi
+ reports_dir="$(mktemp -d /tmp/skillevaluator-reports.XXXXXX)"
+ trap 'rm -rf "$reports_dir"' EXIT
+ export SKILL_PATHS="$SKILL_EVAL_PATHS"
+ export MIN_SCORE="$SKILL_EVAL_MIN_SCORE"
+ export REPORT_RUN_URL="${CI_JOB_URL:-}"
+ export REPORT_COMMIT="${CI_COMMIT_SHA:-}"
+ export REPORT_IMAGE="$SKILL_EVAL_IMAGE"
+ case "$SKILL_EVAL_SCAN_MODE" in
+ auto)
+ if [ "${CI_PIPELINE_SOURCE:-}" = "merge_request_event" ]; then
+ export SCAN_MODE=changed
+ else
+ export SCAN_MODE=all
+ fi
+ ;;
+ all|changed) export SCAN_MODE="$SKILL_EVAL_SCAN_MODE" ;;
+ *) echo "SKILL_EVAL_SCAN_MODE must be auto, all, or changed." >&2; exit 1 ;;
+ esac
+ if [ "$SCAN_MODE" = changed ]; then
+ if [ "${CI_PIPELINE_SOURCE:-}" != "merge_request_event" ] || [ "${CI_MERGE_REQUEST_EVENT_TYPE:-detached}" != detached ]; then
+ echo "Changed-skill selection requires a detached merge-request pipeline." >&2
+ exit 1
+ fi
+ export DIFF_BASE_SHA="${CI_MERGE_REQUEST_DIFF_BASE_SHA:-}"
+ export DIFF_HEAD_SHA="${CI_COMMIT_SHA:-}"
+ fi
+ set +e
+ python /opt/skillevaluator/ci/run_tier1.py --workspace "$CI_PROJECT_DIR" --reports "$reports_dir"
+ result=$?
+ set -e
+ mkdir "$artifact_dir"
+ cp -R "$reports_dir/." "$artifact_dir/"
+ exit "$result"
+ artifacts:
+ when: always
+ paths:
+ - skill-evaluator-report/
+ expire_in: 14 days
+ rules:
+ - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
+ - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
+ - if: '$CI_PIPELINE_SOURCE == "web"'
diff --git a/ci/native_reports.py b/ci/native_reports.py
index 31f1b83b..4a8383b2 100644
--- a/ci/native_reports.py
+++ b/ci/native_reports.py
@@ -246,12 +246,8 @@ def render_native_reports(reports: Path, summary: dict, context: dict) -> dict[s
f"<title>Skills · SkillEvaluator</title><style>{_STYLE}</style></head><body>",
f'<aside class="rail">{brand_html("se-logo-index")}<p>Evaluation reports</p><a href="#skills" aria-current="page">All skills</a></aside><main>',
'<header class="hero"><div><div class="eyebrow">Evaluation report · Tier 1</div><h1>Skill scan results</h1>',
- f'<span class="status {verdict_class}"><span class="sr-only">Verdict: </span>{verdict}</span>',
- # A no-op run explains itself once, in the skills section below.
- "</div>"
- if no_changes
- else '<p class="muted">Open a skill to see its findings and the evidence for each check.</p></div>',
- f'<a class="button" href="{h(run_url)}">View workflow run ↗</a>' if run_url else "",
+ f'<span class="status {verdict_class}">{verdict}</span><p class="muted">Review findings, quality dimensions, and scanner evidence for each skill.</p></div>',
+ f'<a class="button" href="{h(run_url)}">View CI run ↗</a>' if run_url else "",
'</header><div class="metrics">',
]
for count, label in (
@@ -279,9 +275,10 @@ def render_native_reports(reports: Path, summary: dict, context: dict) -> dict[s
page.append('<section class="attention"><h2>Needs attention</h2>')
if summary.get("error"):
page.append(f"<p>{h(summary['error'])}</p>")
- for row, _data, issue, _status, url, _quality in attention:
- fix = first_fixes.get(row["report_dir"]) if not issue else None
- reason = issue or row.get("error") or (fix or {}).get("message") or "Open the report to inspect the checks."
+ for row, data, issue, _status, url, _quality in attention:
+ reason, _location, _fix = views._first_action(
+ {"row": row, "results": data.get("results", []), "issue": issue}
+ )
label = f'<a href="{h(url)}">{h(row["path"])}</a>' if url else h(row["path"])
line = h(str(reason)[:2000])
if fix and url:
diff --git a/ci/publish_reports.py b/ci/publish_reports.py
index 3c1962f4..f0f7b8b4 100644
--- a/ci/publish_reports.py
+++ b/ci/publish_reports.py
@@ -55,7 +55,7 @@ class PublicationError(ValueError):
class StalePublication(PublicationError):
- """A PR has closed or moved since the selected run completed."""
+ """Required historical evidence is unavailable, so hosting is skipped."""
def allowed_download_url(url: str) -> bool:
@@ -76,10 +76,41 @@ def allowed_download_url(url: str) -> bool:
DOWNLOAD_TOO_LARGE = "The downloaded artifact exceeds its size limit."
-class GitHubAPI(github_api.GitHubAPI):
- error = PublicationError
- json_error = "A required JSON document is invalid."
- user_agent = "SkillEvaluator-report-publisher"
+def _bounded_read(response, limit: int) -> bytes:
+ declared = response.headers.get("Content-Length")
+ if declared and (not declared.isdigit() or int(declared) > limit):
+ raise PublicationError("The downloaded artifact or API response exceeds its size limit.")
+ payload = response.read(limit + 1)
+ if len(payload) > limit:
+ raise PublicationError("The downloaded artifact or API response exceeds its size limit.")
+ return payload
+
+
+class GitHubAPI:
+ def __init__(self, repository: str, token: str):
+ self.repository = repository
+ self.token = token
+ self.opener = urllib.request.build_opener(_NoRedirect())
+
+ def _request(self, path: str):
+ root = f"/repos/{self.repository}"
+ prefix = root + "/"
+ if (path != root and not path.startswith(prefix)) or ".." in path or "\\" in path:
+ raise PublicationError("Only this repository's GitHub API endpoints are supported.")
+ request = urllib.request.Request(
+ f"https://api.github.com{path}",
+ headers={
+ "Authorization": f"Bearer {self.token}",
+ "Accept": "application/vnd.github+json",
+ "X-GitHub-Api-Version": "2022-11-28",
+ "User-Agent": "SkillEvaluator-report-publisher",
+ },
+ )
+ return self.opener.open(request, timeout=60)
+
+ def get(self, path: str):
+ with self._request(path) as response:
+ return _json(_bounded_read(response, MAX_FILE_BYTES))
def download_artifact(self, artifact_id: int) -> bytes:
try:
@@ -492,76 +523,6 @@ def _caller_blob(api, repository: str, sha: str) -> str:
return caller["sha"]
-def _fork_associations(api, repository: str, run: dict) -> list:
- """Find an open fork PR from API run metadata; GitHub never records it on the run."""
- head_repository = run.get("head_repository")
- if head_repository is None:
- raise StalePublication("The fork for this PR run was deleted, so the run cannot be verified.")
- name = head_repository.get("full_name") if isinstance(head_repository, dict) else None
- if name == repository:
- raise StalePublication("The PR is closed or no longer associated with this run.")
- branch = run.get("head_branch")
- if not isinstance(name, str) or REPO_PATTERN.fullmatch(name) is None or not isinstance(branch, str) or not branch:
- raise PublicationError("The PR run lacks a valid head repository and branch.")
- head = urllib.parse.quote(name.split("/", 1)[0], safe="") + ":" + urllib.parse.quote(branch, safe="")
- pulls = api.get(f"/repos/{repository}/pulls?state=open&head={head}&per_page=100")
- if not isinstance(pulls, list) or any(not isinstance(pr, dict) for pr in pulls):
- raise PublicationError("The fork PR run has an unsupported pull request listing.")
- matches = [
- pr
- for pr in pulls
- if ((pr.get("head") or {}).get("repo") or {}).get("full_name") == name
- and ((pr.get("base") or {}).get("repo") or {}).get("full_name") == repository
- ]
- if not matches:
- raise StalePublication("The fork PR is closed or merged; no open PR uses this run's fork branch.")
- return matches
-
-
-def _executed_caller_blob(api, repository: str, run: dict) -> str:
- """Reconstruct unambiguous PR merge content from GitHub-recorded revisions.
-
- The run API exposes the PR head, not GITHUB_WORKFLOW_SHA. Never accept a
- merge revision asserted by an artifact or the PR's current mutable ref.
- Fork runs have no recorded PR, so the PR API supplies the same revisions.
- """
- if run["event"] != "pull_request":
- return _caller_blob(api, repository, run["head_sha"])
- associations = run.get("pull_requests")
- if associations == []:
- associations = _fork_associations(api, repository, run)
- if not isinstance(associations, list) or len(associations) != 1:
- raise PublicationError("The PR run lacks an unambiguous recorded base revision; update the branch and rerun.")
- association = associations[0]
- base = (association.get("base") or {}).get("sha") if isinstance(association, dict) else None
- head = (association.get("head") or {}).get("sha") if isinstance(association, dict) else None
- if (
- not isinstance(base, str)
- or SHA_PATTERN.fullmatch(base) is None
- or not isinstance(head, str)
- or SHA_PATTERN.fullmatch(head) is None
- ):
- raise PublicationError("The PR run lacks trustworthy recorded merge revisions; update the branch and rerun.")
- if head != run["head_sha"]:
- raise StalePublication("The PR head moved after this scan.")
- head_blob = _caller_blob(api, repository, run["head_sha"])
- base_blob = _caller_blob(api, repository, base)
- if head_blob == base_blob:
- return head_blob
- comparison = api.get(f"/repos/{repository}/compare/{base}...{head}")
- ancestor = (comparison.get("merge_base_commit") or {}).get("sha") if isinstance(comparison, dict) else None
- if not isinstance(ancestor, str) or SHA_PATTERN.fullmatch(ancestor) is None:
- raise PublicationError("GitHub did not return the caller workflow merge base.")
- ancestor_blob = _caller_blob(api, repository, ancestor)
- if head_blob == ancestor_blob:
- return base_blob
- if base_blob == ancestor_blob:
- return head_blob
- raise PublicationError(
- "Both branches changed the caller workflow; update the PR branch and rerun before publishing."
- )
-
-
def _validate_workflow_provenance(api, repository: str, run: dict) -> None:
evaluator_pins = _trusted_pins("TRUSTED_EVALUATOR_SHA")
caller_blobs = _trusted_pins("TRUSTED_CALLER_BLOBS")
@@ -576,7 +537,23 @@ def _validate_workflow_provenance(api, repository: str, run: dict) -> None:
workflow = f"{evaluator_repository}/.github/workflows/evaluate-skills.yml"
if sha not in evaluator_pins or reference.get("path") != f"{workflow}@{sha}":
raise PublicationError("The selected run did not use an approved immutable evaluator workflow.")
- if _executed_caller_blob(api, repository, run) not in caller_blobs:
+ # The receipt is a separate artifact made by the trusted reusable workflow.
+ # PR associations and current merge refs are mutable, so they never identify
+ # which caller configuration produced a historical report.
+ from verify_provenance import ProvenanceError, verify_receipt
+
+ artifact = _select_artifact(api, repository, run, provenance=True)
+ try:
+ identity = verify_receipt(
+ api.download_artifact(artifact["id"]),
+ repository=repository,
+ run=run,
+ evaluator_workflow=workflow,
+ evaluator_sha=sha,
+ )
+ except ProvenanceError as exc:
+ raise PublicationError(str(exc)) from exc
+ if _caller_blob(api, repository, identity["workflow_sha"]) not in caller_blobs:
raise PublicationError("The caller workflow differs from the approved workflow blobs.")
@@ -590,7 +567,7 @@ def _timestamp(value: str) -> datetime:
raise PublicationError("The run or artifact lacks a valid timestamp.") from exc
-def _select_artifact(api, repository: str, run: dict) -> dict:
+def _select_artifact(api, repository: str, run: dict, *, provenance=False) -> dict:
listing = api.get(f"/repos/{repository}/actions/runs/{run['id']}/artifacts?per_page=100")
if (
not isinstance(listing, dict)
@@ -602,10 +579,17 @@ def _select_artifact(api, repository: str, run: dict) -> dict:
finished = _timestamp(run.get("updated_at"))
if finished < started:
raise PublicationError("The selected run attempt has an invalid time interval.")
- name = f"{EVIDENCE_ARTIFACT}-attempt-{run['run_attempt']}"
+ names = (
+ {f"{ARTIFACT_NAME}-provenance-attempt-{run['run_attempt']}"}
+ if provenance
+ else {ARTIFACT_NAME, f"{ARTIFACT_NAME}-attempt-{run['run_attempt']}"}
+ )
+ size_limit = 64 * 1024 if provenance else MAX_TOTAL_BYTES
candidates = []
for artifact in listing["artifacts"]:
- if artifact.get("name") != name or artifact.get("expired") is not False:
+ if not isinstance(artifact, dict):
+ raise PublicationError("The selected run has malformed artifact metadata.")
+ if artifact.get("name") not in names or artifact.get("expired") is not False:
continue
identity = artifact.get("workflow_run", {})
if identity.get("id") != run["id"] or identity.get("head_sha") != run["head_sha"]:
@@ -615,16 +599,19 @@ def _select_artifact(api, repository: str, run: dict) -> dict:
if (
not _integer(artifact.get("id"), 1)
or not _integer(artifact.get("size_in_bytes"))
- or artifact["size_in_bytes"] > MAX_TOTAL_BYTES
+ or artifact["size_in_bytes"] > size_limit
):
raise PublicationError("The selected report artifact exceeds its size limit.")
candidates.append(artifact)
if not candidates:
- raise PublicationError(
- f"The selected run attempt has no current report evidence artifact named {name}. "
- "The scan may have stopped before writing reports, the artifact may have expired, "
- "or the scan used an evaluator revision that does not upload it."
- )
+ if provenance:
+ raise StalePublication(
+ "No signed execution receipt is available for this scan attempt. "
+ "For hosted reports, commit a receipt-capable evaluator and id-token: write in the scan caller, "
+ "approve its updated caller blob, then trigger a new scan. "
+ "Downloadable scan reports remain in Actions."
+ )
+ raise PublicationError("The selected run attempt has no current report artifact.")
if len(candidates) != 1:
raise PublicationError("The selected run attempt has ambiguous report artifacts.")
return candidates[0]
diff --git a/ci/render_templates.py b/ci/render_templates.py
index 553581bb..e0aab3c2 100644
--- a/ci/render_templates.py
+++ b/ci/render_templates.py
@@ -49,12 +49,37 @@ def render_templates(output: Path, workflow_ref: str, image: str, repository: st
receipt = {"repository": repository, "workflow_ref": workflow_ref, "image": image, "caller_blob": blob}
(output / "release.json").write_text(json.dumps(receipt, indent=2) + "\n", encoding="utf-8")
(output / "README.md").write_text(
- "# GitHub Actions onboarding\n\n"
- "Copy skill-checks.yml to .github/workflows/skill-checks.yml and set paths to your skill roots. "
- "The workflow and image pins are already filled in. No API key is needed.\n\n"
- "For optional hosted reports and PR comments, follow ci/README.md in the evaluator repository. "
- "Set YOUR_REPORT_BASE_URL in the publisher template. If you edit skill-checks.yml, update "
- "trusted-caller-blobs using git hash-object .github/workflows/skill-checks.yml.\n",
+ "# Repository CI onboarding\n\n"
+ "## GitHub: choose your templates\n\n"
+ "- Checks and downloadable reports: copy `skill-checks.yml` to `.github/workflows/skill-checks.yml`. "
+ "Set `paths` to your skill roots and choose `min-score` (default 70).\n"
+ "- Hosted reports and PR comments: also copy `publish-skill-reports.yml` to `.github/workflows/` "
+ "and complete the one-time setup below.\n\n"
+ "The workflow and image pins are already filled in. Both templates use GitHub's automatic token; "
+ "no personal token or model key is needed for the default checks. "
+ "Your organization must allow the referenced actions and the provided permissions.\n\n"
+ "### One-time publisher setup\n\n"
+ "1. Enable Settings → Pages → Source: GitHub Actions. Reports can include source excerpts; "
+ "a private repository does not automatically make its Pages site private. "
+ "Use scan artifacts if hosting is unsuitable. Coordinate any existing Pages site first.\n"
+ "2. Initialize a new `gh-pages` history branch with `.nojekyll` and `history.json` containing `[]`. "
+ "Preserve existing site content and history.\n"
+ "3. Set `YOUR_REPORT_BASE_URL` in the publisher template to your Pages URL.\n"
+ "4. Uncomment `id-token: write` under `permissions` in `skill-checks.yml` for signed execution receipts. "
+ "After customizing the scan file, run `git hash-object .github/workflows/skill-checks.yml` "
+ "and put the result in `trusted-caller-blobs`. Refresh this value after every scan-file edit.\n"
+ "5. Merge both files into your default branch, keeping the supplied scan filename, "
+ "workflow name and artifact name. The publisher already declares its required permissions.\n\n"
+ "New eligible PR scans trigger publication automatically. After successful deployment, the bot "
+ "creates or updates one comment on the current, open, same-repository PR. Fork PR comments "
+ "and comment-only operation are not supported by this publisher.\n\n"
+ "Without the signing permission, ordinary scans and downloadable reports still run; hosting is skipped. "
+ "Commit the setup and trigger a new scan; rerunning an old job keeps its original workflow.\n\n"
+ f"[Full publisher setup](https://github.com/{repository}/blob/{workflow_ref}/ci/README.md"
+ "#optional-hosted-reports-and-pr-feedback).\n\n"
+ "## GitLab\n\n"
+ "For GitLab, merge gitlab-skill-checks.yml into your root .gitlab-ci.yml and set "
+ "SKILL_EVAL_PATHS. Its rules must remain in the root file to enable merge-request pipelines.\n",
encoding="utf-8",
)
return receipt
diff --git a/ci/report_views.py b/ci/report_views.py
index 5ac9fc2d..08f50f44 100644
--- a/ci/report_views.py
+++ b/ci/report_views.py
@@ -424,6 +424,23 @@ def finding_repository_path(skill_path: object, file_path: object) -> str | None
return str(path)
+def _issue_for_result(result: dict) -> tuple[object, dict]:
+ """Use an explicit failed gate before unrelated advisory findings."""
+ gates = _items(result.get("gate_failures"))
+ findings = [_dict(item) for item in _items(result.get("findings"))]
+ if gates:
+ for finding in findings:
+ message = finding.get("message")
+ if isinstance(message, str) and message and message in _text(gates[0]):
+ return gates[0], finding
+ return gates[0], {}
+ severity = {"critical": 0, "high": 1, "medium": 2, "low": 3, "info": 4}
+ findings.sort(key=lambda item: severity.get(_text(item.get("severity")).lower(), 5))
+ finding = findings[0] if findings else {}
+ errors = _items(_dict(result.get("legacy")).get("errors"))
+ return finding.get("message") or (errors[0] if errors else result.get("skip_reason")), finding
+
+
def _first_action(model: dict) -> tuple[str, str, str]:
"""Select one recorded problem, preferring unavailable scans over findings."""
row = model["row"]
@@ -456,12 +473,8 @@ def _first_action(model: dict) -> tuple[str, str, str]:
"",
"Check this validator's diagnostics and rerun after restoring the missing scan.",
)
- findings = [_dict(f) for f in _items(result.get("findings"))]
- severity = {"critical": 0, "high": 1, "medium": 2, "low": 3, "info": 4}
- findings.sort(key=lambda f: severity.get(_text(f.get("severity")).lower(), 5))
- finding = findings[0] if findings else {}
- errors = _items(legacy.get("errors"))
- reason = finding.get("message") or (errors[0] if errors else "This validator failed without a recorded reason.")
+ reason, finding = _issue_for_result(result)
+ reason = reason or "This validator failed without a recorded reason."
fix = (
finding.get("suggestion")
or "Open the full report and inspect this validator's diagnostics before rerunning."
@@ -668,10 +681,7 @@ def render_reports(
for result, outcome in zip(model["results"], model["statuses"], strict=True):
if outcome not in {"FAIL", "INCOMPLETE", "UNKNOWN"}:
continue
- findings = _items(result.get("findings"))
- finding = _dict(findings[0]) if findings else {}
- errors = _items(_dict(result.get("legacy")).get("errors"))
- reason = finding.get("message") or (errors[0] if errors else result.get("skip_reason"))
+ reason, finding = _issue_for_result(result)
reason = reason or "Execution evidence is incomplete; inspect the validator details."
location = _text(finding.get("file_path"))
if finding.get("line_number") is not None:
@@ -760,14 +770,10 @@ def render_reports(
if outcome in {"FAIL", "INCOMPLETE", "UNKNOWN"}
]
for result in failures:
- findings = _items(result.get("findings"))
- errors = _items(_dict(result.get("legacy")).get("errors"))
- message = (
- _dict(findings[0]).get("message") if findings else (errors[0] if errors else "See retained evidence.")
- )
+ message, finding = _issue_for_result(result)
+ message = message or "See retained evidence."
comments.append(f"**{_md(skill)} / {_md(result.get('validator'))}:** {_md(_text(message)[:2000])}")
- if findings:
- finding = _dict(findings[0])
+ if finding:
location = _text(finding.get("file_path"))
if finding.get("line_number") is not None:
location += ":" + _text(finding["line_number"])
diff --git a/ci/run_tier1.py b/ci/run_tier1.py
index 4706f6be..1562e129 100644
--- a/ci/run_tier1.py
+++ b/ci/run_tier1.py
@@ -514,7 +514,10 @@ def run(workspace: Path, reports: Path, environ: dict[str, str], *, execute=None
if reports.is_relative_to(workspace):
raise ConfigurationError("The report directory must be outside the read-only workspace.")
env, llm = child_environment(environ)
- workers = scan_workers(environ)
+ score = environ.get("MIN_SCORE", "70")
+ if not score.isascii() or not score.isdigit() or len(score) > 3 or int(score) > 100:
+ raise ConfigurationError("MIN_SCORE must be an integer between 0 and 100.")
+ min_score = str(int(score))
mode = environ.get("SCAN_MODE", "all")
if mode not in {"all", "changed"}:
raise ConfigurationError("SCAN_MODE must be all or changed.")
@@ -758,6 +761,79 @@ def run(workspace: Path, reports: Path, environ: dict[str, str], *, execute=None
f"SkillEvaluator: scan time limit reached; {len(rows) - len(started)} skills were not started.",
flush=True,
)
+ output = reports / f"skill-{index:03d}"
+ output.mkdir()
+ row["error"] = "Evaluation interrupted before complete evidence was retained."
+ data = None
+ try:
+ remaining = deadline - clock()
+ if remaining <= 0:
+ raise subprocess.TimeoutExpired("skillevaluator", SCAN_TIMEOUT_SECONDS)
+ # Raw scanner files and logs live only in a private temporary
+ # directory, never in the upload directory. Docker mounts /audit-tmp
+ # as tmpfs, so cancellation cannot upload half-written secrets.
+ with tempfile.TemporaryDirectory(prefix="skillevaluator-private-", dir=env.get("TMPDIR")) as temporary:
+ raw = Path(temporary).resolve(strict=True)
+ if raw.is_relative_to(workspace) or raw.is_relative_to(reports):
+ raise ValueError("A private scanner directory must be outside the workspace and reports.")
+ argv = [
+ "skillevaluator",
+ "tier1",
+ str(target),
+ "--llm" if llm else "--no-llm",
+ "--profile",
+ "external",
+ "--min-score",
+ min_score,
+ "-r",
+ "json,html,markdown,sarif",
+ "-o",
+ str(raw),
+ ]
+ with (raw / "stdout.log").open("xb") as stdout, (raw / "stderr.log").open("xb") as stderr:
+ process = execute(
+ argv, cwd=workspace, env=env, stdout=stdout, stderr=stderr, check=False, timeout=remaining
+ )
+ safe = sanitized_outputs(raw, tokens)
+ for name, content in safe.items():
+ (output / name).write_bytes(content)
+ row["exit_code"] = process.returncode
+ data = _load_report(output / "skillevaluator-tier1.json")
+ for key in (
+ "overall_status",
+ "incomplete_scans",
+ "total_validators",
+ "total_errors",
+ "total_warnings",
+ "severity_counts",
+ ):
+ row[key] = data[key]
+ row["passed"] = (
+ process.returncode == 0
+ and data["overall_passed"] is True
+ and data["overall_status"] == "passed"
+ and not data["incomplete_scans"]
+ and all(
+ result.get("passed") is True
+ and result.get("status") not in {"incomplete", "failed"}
+ and not result.get("incomplete_scans")
+ for result in data["results"]
+ )
+ )
+ if not row["passed"] and row["overall_status"] == "passed":
+ row["overall_status"] = "error"
+ row["incomplete_scans"] = []
+ row["error"] = "Process exit or validator results contradict the report verdict."
+ else:
+ row.pop("error", None)
+ except subprocess.TimeoutExpired:
+ row["error"] = "Scan time limit reached; this skill did not finish."
+ checkpoint()
+ break
+ except (OSError, ValueError, UnicodeError, RecursionError):
+ row["error"] = "Evaluator execution failed or its JSON report was missing, unreadable, or invalid."
+ log_result(row, data, annotations=environ.get("GITHUB_ACTIONS") == "true", budget=annotation_budget)
+ checkpoint()
summary = _summary(rows)
summary["selection"] = selection
summary["execution"] = context
diff --git a/ci/templates/README.md b/ci/templates/README.md
new file mode 100644
index 00000000..6bc7742e
--- /dev/null
+++ b/ci/templates/README.md
@@ -0,0 +1,55 @@
+# Copy a CI template
+
+Bring SkillEvaluator to your repository by copying a configuration file. You do
+not need to install SkillEvaluator locally, use an onboarding CLI, or copy
+evaluation code into your project. GitHub and GitLab templates are maintained
+together in this public NVIDIA/SkillEvaluator repository.
+
+| Platform and result | Source template | Where to put it |
+| --- | --- | --- |
+| GitHub: checks and downloadable reports | [skill-checks.yml](skill-checks.yml) | `.github/workflows/skill-checks.yml` |
+| GitHub: also host reports and update a PR comment | [publish-skill-reports.yml](publish-skill-reports.yml) | `.github/workflows/publish-skill-reports.yml`, alongside the checks file |
+| GitHub: optional trusted manual LLM checks | [skill-checks-llm.yml](skill-checks-llm.yml) | `.github/workflows/skill-checks-llm.yml` |
+| GitLab: checks and downloadable reports | [gitlab-skill-checks.yml](gitlab-skill-checks.yml) | `.gitlab-ci.yml`, or merge into your existing root configuration |
+
+Open a source file to inspect it; GitHub's **Raw** view lets you copy its full
+contents. See [example results](../examples/README.md) without needing access to
+a separate demo repository.
+
+## Get a version you can run
+
+**These source templates contain release placeholders.** Before running them,
+use the generated copies attached to an official CI image release on
+[NVIDIA/SkillEvaluator Releases](https://github.com/NVIDIA/SkillEvaluator/releases).
+Choose a release that explicitly includes the CI onboarding bundle; a regular
+Python package release may not include it. If no such release is available, the
+official CI integration is not yet ready to install from these templates.
+
+The generated files already contain the reviewed workflow commit and container
+image digest. If you copy a source template instead, replace
+`RELEASE_WORKFLOW_SHA` and `RELEASE_IMAGE_DIGEST` with the values recorded in that
+CI release. Do not leave placeholders or substitute `main` or `latest`.
+
+## Customize and commit
+
+- **GitHub:** set `paths` and optionally `min-score` (default `70`). Change
+ `branches: [main]` if needed, then commit the workflow to your default branch.
+ Existing workflows remain separate. Follow the
+ [GitHub guide](../../docs/github-actions.mdx).
+- **GitLab:** set `SKILL_EVAL_PATHS` and optionally `SKILL_EVAL_MIN_SCORE`
+ (default `70`). For an existing pipeline, merge the include, variables and job
+ rules; preserve your other jobs and `workflow: rules`. The job uses `test` by
+ default, so override its stage if your pipeline does not define it. Follow the
+ [GitLab guide](../../docs/gitlab-ci.mdx).
+- **Optional GitHub comments:** the second file needs the
+ [one-time hosting and trust setup](../../docs/github-actions.mdx#enable-pr-comments).
+ Uncomment `id-token: write` in the scan file for signed execution receipts,
+ then approve that file's new blob. Copying the second file alone does not
+ enable comments.
+- **Optional LLM checks:** configure a provider key in the separate manual
+ workflow. The default checks need no model key.
+
+Open a pull request or merge request that changes a skill to see the checks.
+Enable the platform's CI service, allow the referenced workflows or includes,
+and provide a compatible runner. The [platform guides](../../docs/repository-integration.mdx)
+cover results and merge-gating settings.
diff --git a/ci/templates/gitlab-skill-checks.yml b/ci/templates/gitlab-skill-checks.yml
new file mode 100644
index 00000000..1e162092
--- /dev/null
+++ b/ci/templates/gitlab-skill-checks.yml
@@ -0,0 +1,18 @@
+# SPDX-License-Identifier: Apache-2.0
+# Merge this into an existing .gitlab-ci.yml or use it as the complete file.
+# The release-generated copy pins both the reusable job and its container image.
+include:
+ - remote: https://raw.githubusercontent.com/NVIDIA/SkillEvaluator/RELEASE_WORKFLOW_SHA/ci/gitlab/skill-evaluation.yml
+
+variables:
+ SKILL_EVAL_IMAGE: ghcr.io/nvidia/skillevaluator-tier1@sha256:RELEASE_IMAGE_DIGEST
+ SKILL_EVAL_PATHS: "skills"
+ SKILL_EVAL_MIN_SCORE: "70"
+
+# GitLab requires merge-request rules directly in the root file, even when a
+# job is included. Preserve your existing workflow rules if you have them.
+skill-evaluation:
+ rules:
+ - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
+ - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
+ - if: '$CI_PIPELINE_SOURCE == "web"'
diff --git a/ci/templates/publish-skill-reports.yml b/ci/templates/publish-skill-reports.yml
index 8401b0e2..8a89c4e3 100644
--- a/ci/templates/publish-skill-reports.yml
+++ b/ci/templates/publish-skill-reports.yml
@@ -3,6 +3,7 @@
# Pages reports are public unless the site is explicitly private (GitHub Enterprise Cloud).
# Reports include source evidence; private repositories should otherwise use scan artifacts.
# Use the release-generated file; set your own URL and refresh the caller blob after scan-file edits.
+# Also grant id-token: write in skill-checks.yml so scans retain signed execution receipts.
name: Publish skill reports
on:
workflow_run:
diff --git a/ci/templates/skill-checks-llm.yml b/ci/templates/skill-checks-llm.yml
index de872b3b..5d4967e2 100644
--- a/ci/templates/skill-checks-llm.yml
+++ b/ci/templates/skill-checks-llm.yml
@@ -12,6 +12,7 @@ jobs:
with:
image: ghcr.io/nvidia/skillevaluator-tier1@sha256:RELEASE_IMAGE_DIGEST
paths: skills
+ min-score: 70
scan-mode: all
llm: true
llm-provider: openai
diff --git a/ci/templates/skill-checks.yml b/ci/templates/skill-checks.yml
index 33ce5d3f..4e5c6478 100644
--- a/ci/templates/skill-checks.yml
+++ b/ci/templates/skill-checks.yml
@@ -8,11 +8,14 @@ on:
workflow_dispatch:
permissions:
contents: read
+ # Hosted reports/PR comments: uncomment this, then refresh trusted-caller-blobs.
+ # id-token: write
jobs:
skills:
uses: NVIDIA/SkillEvaluator/.github/workflows/evaluate-skills.yml@RELEASE_WORKFLOW_SHA
with:
image: ghcr.io/nvidia/skillevaluator-tier1@sha256:RELEASE_IMAGE_DIGEST
paths: skills
+ min-score: 70
scan-mode: auto
llm: false
diff --git a/ci/verify_provenance.py b/ci/verify_provenance.py
new file mode 100644
index 00000000..fea90f45
--- /dev/null
+++ b/ci/verify_provenance.py
@@ -0,0 +1,204 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+# ruff: noqa: INP001 -- standalone trusted publisher helper.
+"""Verify GitHub-signed execution identity without trusting report content.
+
+The receipt is historical evidence, not a credential used for authentication.
+Its signature and issuance during the selected run attempt remain mandatory
+after expiration. A removed signing key makes old receipts unverifiable.
+"""
+
+from __future__ import annotations
+
+import io
+import json
+import re
+import stat
+import time
+import urllib.error
+import urllib.request
+import zipfile
+from datetime import datetime
+
+ISSUER = "https://token.actions.githubusercontent.com"
+JWKS_URL = ISSUER + "/.well-known/jwks"
+AUDIENCE = "urn:skillevaluator:report-provenance:v1"
+WORKFLOW_PATH = ".github/workflows/skill-checks.yml"
+MAX_TOKEN_BYTES = 32 * 1024
+MAX_RECEIPT_BYTES = 64 * 1024
+MAX_JWKS_BYTES = 256 * 1024
+CLOCK_SKEW_SECONDS = 30
+MAX_TOKEN_LIFETIME_SECONDS = 900
+SHA_PATTERN = re.compile(r"[0-9a-f]{40}")
+
+
+class ProvenanceError(ValueError):
+ """A fixed diagnostic that never embeds receipt or provider response data."""
+
+
+class _NoRedirect(urllib.request.HTTPRedirectHandler):
+ def redirect_request(self, _request, _file, _code, _message, _headers, _url):
+ return None
+
+
+def fetch_signing_keys() -> dict:
+ """Fetch only GitHub's fixed public key endpoint, without credentials."""
+ request = urllib.request.Request(
+ JWKS_URL,
+ headers={"Accept": "application/json", "User-Agent": "SkillEvaluator-report-publisher"},
+ )
+ try:
+ with urllib.request.build_opener(_NoRedirect()).open(request, timeout=30) as response:
+ declared = response.headers.get("Content-Length")
+ if declared and (not declared.isdecimal() or int(declared) > MAX_JWKS_BYTES):
+ raise ProvenanceError("GitHub signing keys exceed the response limit.")
+ payload = response.read(MAX_JWKS_BYTES + 1)
+ if len(payload) > MAX_JWKS_BYTES:
+ raise ProvenanceError("GitHub signing keys exceed the response limit.")
+ result = json.loads(payload)
+ except ProvenanceError:
+ raise
+ except (OSError, ValueError, urllib.error.URLError) as exc:
+ raise ProvenanceError("GitHub signing keys are unavailable; retry report publication.") from exc
+ if not isinstance(result, dict) or not isinstance(result.get("keys"), list) or not 1 <= len(result["keys"]) <= 32:
+ raise ProvenanceError("GitHub returned an unsupported signing key document.")
+ return result
+
+
+def read_receipt(payload: bytes) -> str:
+ """Read exactly one bounded, regular execution.jwt member without extraction."""
+ if len(payload) > MAX_RECEIPT_BYTES:
+ raise ProvenanceError("The execution receipt archive exceeds its size limit.")
+ try:
+ with zipfile.ZipFile(io.BytesIO(payload)) as archive:
+ entries = archive.infolist()
+ if len(entries) != 1:
+ raise ProvenanceError("The execution receipt must contain exactly one JWT file.")
+ entry = entries[0]
+ kind = stat.S_IFMT(entry.external_attr >> 16)
+ if (
+ entry.filename != "execution.jwt"
+ or entry.is_dir()
+ or entry.flag_bits & 1
+ or kind not in {0, stat.S_IFREG}
+ or not 1 <= entry.file_size <= MAX_TOKEN_BYTES
+ ):
+ raise ProvenanceError("The execution receipt contains an unsupported member.")
+ with archive.open(entry) as handle:
+ raw = handle.read(MAX_TOKEN_BYTES + 1)
+ if len(raw) > MAX_TOKEN_BYTES:
+ raise ProvenanceError("The execution receipt exceeds its size limit.")
+ token = raw.decode("ascii").strip()
+ except (OSError, ValueError, UnicodeError, RuntimeError, zipfile.BadZipFile) as exc:
+ raise ProvenanceError("The execution receipt archive is invalid.") from exc
+ if re.fullmatch(r"[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+", token) is None:
+ raise ProvenanceError("The execution receipt is not a signed JWT.")
+ return token
+
+
+def _timestamp(value) -> float:
+ try:
+ result = datetime.fromisoformat(value)
+ if result.tzinfo is None:
+ raise ValueError
+ return result.timestamp()
+ except (ValueError, TypeError, AttributeError) as exc:
+ raise ProvenanceError("The run attempt lacks a valid time interval.") from exc
+
+
+def verify_receipt(
+ payload: bytes,
+ *,
+ repository: str,
+ run: dict,
+ evaluator_workflow: str,
+ evaluator_sha: str,
+) -> dict:
+ """Return signed identity claims only after signature and context validation."""
+ # This optional publisher dependency is never imported by static scans.
+ import jwt
+
+ token = read_receipt(payload)
+ try:
+ header = jwt.get_unverified_header(token)
+ kid = header.get("kid")
+ if (
+ header.get("alg") != "RS256"
+ or header.get("typ") != "JWT"
+ or not isinstance(kid, str)
+ or not 1 <= len(kid) <= 256
+ ):
+ raise ProvenanceError("The execution receipt uses an unsupported signing header.")
+ keys = fetch_signing_keys()["keys"]
+ matches = [key for key in keys if isinstance(key, dict) and key.get("kid") == kid]
+ if len(matches) != 1 or matches[0].get("kty") != "RSA" or matches[0].get("alg") != "RS256":
+ raise ProvenanceError("The execution receipt signing key is unavailable; rerun the scan.")
+ if matches[0].get("use") != "sig":
+ raise ProvenanceError("The execution receipt signing key is not a signature key.")
+ key = jwt.PyJWK.from_dict(matches[0], algorithm="RS256").key
+ if not 2048 <= key.key_size <= 8192:
+ raise ProvenanceError("The execution receipt signing key has an unsupported size.")
+ claims = jwt.decode(
+ token,
+ key,
+ algorithms=["RS256"],
+ audience=AUDIENCE,
+ issuer=ISSUER,
+ options={
+ "strict_aud": True,
+ "verify_exp": False,
+ "verify_nbf": False,
+ "verify_iat": False,
+ "require": ["iss", "aud", "iat", "nbf", "exp"],
+ },
+ )
+ except ProvenanceError:
+ raise
+ except (jwt.PyJWTError, TypeError, ValueError, KeyError) as exc:
+ raise ProvenanceError("The execution receipt signature or audience is invalid.") from exc
+
+ repository_id = run.get("repository", {}).get("id")
+ if type(repository_id) is not int or repository_id <= 0:
+ raise ProvenanceError("The workflow run lacks an immutable repository identity.")
+ expected = {
+ "repository": repository,
+ "repository_id": str(repository_id),
+ "run_id": str(run["id"]),
+ "run_attempt": str(run["run_attempt"]),
+ "event_name": run["event"],
+ "job_workflow_ref": f"{evaluator_workflow}@{evaluator_sha}",
+ "job_workflow_sha": evaluator_sha,
+ }
+ if any(type(claims.get(name)) is not str or claims[name] != value for name, value in expected.items()):
+ raise ProvenanceError("The execution receipt does not match the selected run and approved evaluator.")
+ workflow_sha = claims.get("workflow_sha")
+ ref = claims.get("ref")
+ if (
+ not isinstance(workflow_sha, str)
+ or SHA_PATTERN.fullmatch(workflow_sha) is None
+ or claims.get("sha") != workflow_sha
+ or not isinstance(ref, str)
+ or len(ref) > 1024
+ or claims.get("workflow_ref") != f"{repository}/{WORKFLOW_PATH}@{ref}"
+ ):
+ raise ProvenanceError("The execution receipt lacks a valid caller workflow identity.")
+ if run["event"] == "pull_request":
+ if re.fullmatch(r"refs/pull/[1-9][0-9]*/merge", ref) is None:
+ raise ProvenanceError("The execution receipt lacks a pull-request merge identity.")
+ elif re.fullmatch(r"refs/(?:heads|tags)/[^\x00-\x20\x7f]+", ref) is None or workflow_sha != run["head_sha"]:
+ raise ProvenanceError("The execution receipt does not match the scanned revision.")
+
+ iat, nbf, exp = (claims.get(name) for name in ("iat", "nbf", "exp"))
+ started, finished = (_timestamp(run.get(name)) for name in ("run_started_at", "updated_at"))
+ if (
+ any(type(value) is not int for value in (iat, nbf, exp))
+ or finished < started
+ or not started - CLOCK_SKEW_SECONDS <= iat <= finished + CLOCK_SKEW_SECONDS
+ or iat > time.time() + CLOCK_SKEW_SECONDS
+ or not 0 < exp - iat <= MAX_TOKEN_LIFETIME_SECONDS
+ or not 0 <= iat - nbf <= MAX_TOKEN_LIFETIME_SECONDS
+ ):
+ raise ProvenanceError("The execution receipt was not issued during the selected run attempt.")
+ return {
+ name: claims[name] for name in ("workflow_sha", "job_workflow_sha", "repository_id", "run_id", "run_attempt")
+ }
diff --git a/docs/assets/repository-ci-results-example.png b/docs/assets/repository-ci-results-example.png
new file mode 100644
index 00000000..83352733
Binary files /dev/null and b/docs/assets/repository-ci-results-example.png differ
diff --git a/docs/github-actions.mdx b/docs/github-actions.mdx
index d562ed79..a03a1997 100644
--- a/docs/github-actions.mdx
+++ b/docs/github-actions.mdx
@@ -1,118 +1,238 @@
---
title: "GitHub Actions"
-description: "Add reusable Tier 1 skill checks to your repository, then optionally enable hosted reports and PR feedback."
+description: "Bring SkillEvaluator checks and reports to your GitHub repository with a reusable workflow."
---
-Add one workflow and choose your skill directories. SkillEvaluator supplies the
-scanner runtime; your repository gets a check result, a short Actions summary
-and downloadable reports.
+Bring skill evaluation to your GitHub repository with one workflow file. Point
+it at your skill directories; pull requests check changed skills automatically.
+The shared container includes Tier 1 and SkillSpector security scanning.
-**Release availability:** use the consumer files attached to an official CI
-image release. Until that release is available, source templates contain
-`RELEASE_WORKFLOW_SHA` and `RELEASE_IMAGE_DIGEST` placeholders and are not runnable.
+<Note>
+ Use the generated workflow from an official CI image release. Until that
+ release is available, the source template contains placeholders and is not
+ runnable.
+</Note>
-## Basic setup: one workflow
+## Choose what to add
-1. When a CI release is available, download its generated `skill-checks.yml` and
- copy it to `.github/workflows/skill-checks.yml` in your repository. The release
- file pins the shared workflow to a reviewed commit and the public container
- to an immutable digest.
-2. Set `paths` to your skill directory. For multiple roots, use a YAML multiline
- string. Adapt `branches: [main]` if your default branch has another name.
-3. Commit the file and open a pull request. GitHub Actions must be enabled, and
- your organization must allow the external reusable workflow and its actions.
+The release bundle provides these two GitHub templates. Copy them into
+`.github/workflows/`; you do not need to copy evaluator code or build an image.
-The job configuration in the [source template](../ci/templates/skill-checks.yml)
-uses GitHub's [reusable workflow](https://docs.github.com/en/actions/how-tos/reuse-automations/reuse-workflows)
-syntax:
+| You want | Add this file | Setup |
+| --- | --- | --- |
+| Automatic checks and downloadable reports | [skill-checks.yml](https://github.com/NVIDIA/SkillEvaluator/blob/main/ci/templates/skill-checks.yml) | Choose your skill folders and score threshold. |
+| Hosted reports and an updated PR comment | Also add [publish-skill-reports.yml](https://github.com/NVIDIA/SkillEvaluator/blob/main/ci/templates/publish-skill-reports.yml) | Complete the [one-time publisher setup](#enable-pr-comments). |
-```yaml
+These links open the source templates in NVIDIA/SkillEvaluator. The
+[template catalog](https://github.com/NVIDIA/SkillEvaluator/blob/main/ci/templates/README.md)
+explains how to get release-generated copies from an official CI image release.
+You can copy the files directly; no local installation or onboarding CLI is
+required.
+
+The release fills in the workflow and image versions. Both templates use
+GitHub's automatic token; the default checks need no model key or paid LLM.
+
+## Get started
+
+<Steps>
+
+### Add the workflow
+
+Copy the release-generated `skill-checks.yml` to
+`.github/workflows/skill-checks.yml`. It pins the shared workflow and image.
+
+### Choose your skills and score
+
+Set `paths` to your skill directory and `min-score` to the minimum quality score
+from 0 to 100 (default 70). For multiple roots, use a YAML multiline string.
+Change `branches: [main]` if your default branch has a different name.
+
+### Open a pull request
+
+GitHub Actions evaluates changed skills and provides a concise result. Open the
+run and download its report artifact for details. Enable GitHub Actions and
+allow the external reusable workflow and its actions in your organization.
+
+</Steps>
+
+The generated job has this shape; the release fills in the immutable pins:
+
+```yaml title=".github/workflows/skill-checks.yml"
jobs:
skills:
uses: NVIDIA/SkillEvaluator/.github/workflows/evaluate-skills.yml@RELEASE_WORKFLOW_SHA
with:
image: ghcr.io/nvidia/skillevaluator-tier1@sha256:RELEASE_IMAGE_DIGEST
paths: skills
+ min-score: 70
scan-mode: auto
llm: false
```
-The workflow downloads the public image automatically. You do not build an image,
-install scanners or supply a model API key for static Tier 1. No registry login
-is needed for a public package. Keyless does not mean offline: dependency checks
-can contact vulnerability services. Dependency files are audited without
-installing packages or running their build scripts. Exact package/version pins
-are required; unresolved requirements are reported as incomplete.
-
-`scan-mode: auto` selects skills affected by pull-request changes and evaluates
-all configured skills on pushes and manual runs. Set `scan-mode: all` to check
-the whole configured collection on every run. A partial selection establishes
-results only for the evaluated skills. A PR with no affected skills succeeds
-with **No skills changed**; it does not claim a scanner pass. Fully deleted skills
-are listed as deleted/skipped. Removing `SKILL.md` while leaving tracked files
-behind is an error. Selection follows changed files inside skill directories;
-shared files outside the configured roots are not inferred dependencies. Use
-`scan-mode: all` when those changes should trigger a full evaluation.
-Tier 2 and Tier 3 are not run. The 100-skill limit is a selection limit, not a
-runtime guarantee. Scans have a time budget and preserve completed results plus
-explicit incomplete entries if it is exhausted. The workflow scans up to four
-skills at once. Set `scan-workers` (1 to 16) to change this; `1` scans one skill
-at a time. Results and logs keep the same skill order either way.
-
-## Read the result
-
-Open **Actions → Skill checks**. The summary shows the outcome, affected skills
-and a bounded set of actionable reasons and fixes. Download the
-`skill-evaluator-tier1-attempt-N` artifact for that run attempt (`N`) to get
-`index.html`, per-skill HTML and JSON,
-detailed Markdown, SARIF and scanner logs. The smaller
-`skill-evaluator-tier1-evidence-attempt-N` artifact holds only the JSON that
-the optional report publisher reads. Failed and incomplete scans retain
-their available reports; missing evidence is not a pass. A quality score below
-the minimum fails the Quality score check, so it blocks the verdict. The HTML
-index and reports use the verdicts Passed, Changes required, Incomplete and
-Skipped.
-
-Known credential values detected by the PII patterns, NVIDIA API keys, and
-unquoted `KEY=value` and `Bearer` credentials are redacted from CI reports,
-including parts of them that a scanner cuts or escapes. When a credential or an
-LLM key is present, raw scanner logs are withheld. This is not a guarantee that
-every possible credential or transformed value will be detected. Remove and
-rotate exposed credentials; previously published reports and Git history are not
-retroactively redacted.
-
-The dedicated workflow runs independently alongside existing build and test
-workflows. It does not automatically wait for them. GitHub's `needs` connects
-jobs in the same workflow; adding the scan to a different pipeline requires
-adapting that workflow and the optional publisher's trust configuration. Make
-the scan check required in branch rules separately if it should block merging.
-
-## Optional: hosted reports and one updated PR comment
-
-The basic file does **not** publish a site or post comments. That experience
-requires a second workflow, Pages configured for GitHub Actions, a retained
-history branch, publisher permissions, your own report URL and approved
-workflow revisions. Follow the [publisher setup](../ci/README.md#optional-hosted-reports-and-pr-feedback)
-before adding the generated `publish-skill-reports.yml` to your default branch.
-
-**Check visibility first:** reports can contain skill content, source excerpts
-and unmerged PR findings. A private repository does not make its Pages site
-private. Use an explicitly private GitHub Enterprise Cloud Pages site, or keep
-reports in the scan artifact if public hosting is inappropriate. See
-[Pages access control](https://docs.github.com/en/pages/getting-started-with-github-pages/changing-the-visibility-of-your-github-pages-site).
-
-The publisher follows completed **Skill checks** runs, including failures. It
-validates retained JSON, generates HTML and updates its own bot comment after
-successful deployment for an eligible current same-repository PR. It does not
-execute PR code with a publisher write token. Existing Pages sites need an
-integration plan: this publisher deploys the whole report site. Fork PR bot
-comments are not enabled.
-
-For optional model-backed Tier 1 checks, use the separate trusted manual
-[LLM workflow template](../ci/templates/skill-checks-llm.yml) and explicitly pass
-a repository `LLM_API_KEY` secret. This adds provider cost and sends the selected
-content to that provider. It is not needed for basic scanning and is not
-included in the dedicated publisher's default workflow allowlist.
-
-See [Gate Your CI](ci-integration.mdx) for CLI integration in other CI systems,
-and the [maintainer guide](../ci/README.md) for release and runtime details.
+## See the results
+
+Open **Actions → Skill checks** for the result and suggested next step. Download
+`skill-evaluator-tier1-attempt-N` for `index.html`, per-skill HTML and JSON,
+Markdown, SARIF and available scanner logs.
+
+<Frame caption="Example: a Tier 1 report shows the verdict and quality score separately. This example was captured from a GitLab demo using the same report renderer.">
+ <img src="./assets/repository-ci-results-example.png" alt="Example results table showing a failed skill, its quality score, and a link to its report" />
+</Frame>
+
+### In the Actions summary
+
+A failed skill is identified with a reason and suggested fix. Open its report
+for all findings. The quality score is separate from the scan verdict.
+
+```text
+Tier 1 scan result: FAILED
+1 skill · 0 passed · 1 failed
+Problem: Quality score 98.2/100 is below the required 100/100
+```
+
+This is an illustrative summary; the actual findings depend on the run.
+
+### On the pull request
+
+The optional publisher posts one bot comment and updates it for new results.
+Reviewers can see the failed skill, its score, and what to fix without opening
+the Actions logs.
+
+<Frame caption="Shortened illustrative PR comment. This example is included in the public SkillEvaluator documentation; no demo-repository access is needed.">
+<div style={{ width: "100%", minWidth: 0, padding: "16px" }}>
+
+**github-actions[bot]**
+
+**Tier 1 scan result: FAILED**
+
+1 skill · 0 passed · 1 failed · 0 incomplete
+
+| Skill | Status | Quality / 100 |
+| --- | --- | --- |
+| `skills/glossary-builder` | FAIL | 98.2 / A |
+
+**Problem:** Quality score 98.2/100 is below the required 100/100.
+
+**Suggested fix:** Open the skill report and address the quality findings.
+
+**Open full report** · **View workflow run**
+
+</div>
+</Frame>
+
+The last line shows link labels. In a real comment, they open your repository's
+hosted report and workflow run. Browse the
+[public examples](https://github.com/NVIDIA/SkillEvaluator/blob/main/ci/examples/README.md)
+for the report preview and log excerpts. With `scan-mode: auto`, a pull request
+evaluates only affected skills in the configured folders.
+
+## Enable PR comments
+
+Add the second template for **hosted reports and PR comments together**.
+The current publisher posts the comment after the report site deploys
+successfully, so copying the second file alone is not enough.
+
+<Steps>
+
+### Enable report hosting once
+
+Select **Settings → Pages → Source: GitHub Actions**. Create a new `gh-pages`
+history branch containing an empty `.nojekyll` file and `history.json` with
+`[]`. Keep any existing site or report history intact.
+
+### Configure the publisher template
+
+Copy the release-generated `publish-skill-reports.yml` into
+`.github/workflows/`. Replace `YOUR_REPORT_BASE_URL` with your repository's
+Pages URL, for example `https://your-org.github.io/your-repo`.
+
+In `skill-checks.yml`, uncomment the permission for signed report receipts:
+
+```yaml
+permissions:
+ contents: read
+ id-token: write
+```
+
+GitHub uses this to prove which workflow ran. No personal token or cloud account
+is needed. After choosing your skill folders and other scan options, run:
+
+```bash
+git hash-object .github/workflows/skill-checks.yml
+```
+
+Paste the output into `trusted-caller-blobs` in the publisher file. This
+approves that exact scan configuration; refresh it whenever you edit the scan
+file. The release supplies the evaluator and publisher version pins.
+
+### Merge the setup and open a skill PR
+
+Put both workflow files on your default branch. Keep the **Skill checks**
+workflow name, `skill-checks.yml` filename, and default artifact name.
+The publisher template already includes the needed permissions; your
+organization must allow them and the referenced actions. No extra token is
+required.
+
+The next eligible PR scan triggers publication automatically. After deployment,
+the bot posts or updates its comment. Only open, current-head PRs from branches
+in the same repository receive comments; fork PRs retain their scan results in
+Actions.
+
+</Steps>
+
+<Note>
+ Reports can include source excerpts and unmerged findings. Pages is public
+ unless explicitly configured as private; a private repository alone does not
+ make its Pages site private. This publisher owns the report site's root, so
+ coordinate an existing Pages site before enabling it. Use downloadable
+ artifacts if hosting is unsuitable.
+</Note>
+
+For history initialization, permission details and troubleshooting, follow the
+[publisher setup guide](https://github.com/NVIDIA/SkillEvaluator/blob/main/ci/README.md#optional-hosted-reports-and-pr-feedback).
+
+<AccordionGroup>
+ <Accordion title="When does it run?">
+ `scan-mode: auto` evaluates skills affected by a pull request and the full
+ configured collection on pushes and manual runs. `all` evaluates the whole
+ collection each time. No changed skills is a successful skip, not a scanner
+ pass. Completely deleted skills are listed as skipped; removing `SKILL.md`
+ while leaving tracked files is an error. Shared files outside skill roots are
+ not inferred dependencies; use `all` when they should trigger a full scan.
+ The 100-skill cap is a selection limit, not a runtime guarantee. Timed-out
+ scans retain completed results and mark the remainder incomplete.
+ </Accordion>
+ <Accordion title="Control whether it blocks merging">
+ The workflow runs independently of existing workflows; it does not wait for
+ them automatically. Configure a required check in GitHub branch rules if a
+ failed or incomplete result should block merging. `min-score` controls the
+ quality threshold and also applies to the rubric when LLM checks are enabled;
+ required security, schema and other checks remain independent. Tier 2 and
+ Tier 3 are not run.
+ </Accordion>
+ <Accordion title="A scan finished, but hosting was skipped">
+ Check that the scan workflow grants `id-token: write` and refresh
+ `trusted-caller-blobs` after editing that file. Commit the setup and start
+ a new scan; rerunning an old job keeps its original workflow and cannot add
+ a missing receipt. GitHub can withhold this permission under fork
+ policies; the downloadable report remains available. Signing receipts are
+ separate artifacts and are never published on the report site.
+ </Accordion>
+ <Accordion title="Privacy, dependencies, and optional LLM checks">
+ Static Tier 1 needs no model key, but dependency checks can contact
+ vulnerability services and submit package metadata. Dependencies are audited
+ without installing packages or running build scripts; unpinned or unresolved
+ requirements are incomplete. Known credential values detected by the PII
+ patterns are redacted, but not every secret or transformation is detectable.
+ Raw scanner logs are withheld when a credential or LLM key is present.
+ Remove and rotate exposed credentials; old reports and Git history are not
+ rewritten. Optional model-backed checks use the separate trusted manual
+ [LLM template](https://github.com/NVIDIA/SkillEvaluator/blob/main/ci/templates/skill-checks-llm.yml),
+ a provider key and provider charges; they send selected content to that
+ provider and are not included in the default publisher allowlist.
+ </Accordion>
+</AccordionGroup>
+
+See [Repository Integration](repository-integration.mdx) for GitLab and other
+CI options, or the [maintainer guide](https://github.com/NVIDIA/SkillEvaluator/blob/main/ci/README.md)
+for the release and runtime details.
diff --git a/docs/gitlab-ci.mdx b/docs/gitlab-ci.mdx
new file mode 100644
index 00000000..d0cdf9d7
--- /dev/null
+++ b/docs/gitlab-ci.mdx
@@ -0,0 +1,134 @@
+---
+title: "GitLab CI/CD"
+description: "Add reusable SkillEvaluator Tier 1 checks to a GitLab repository."
+---
+
+Bring skill evaluation to GitLab with a reusable CI job. Point it at your skills
+and choose a quality threshold; merge requests evaluate changed skills
+without installing scanners or adding evaluation code to your repository.
+The shared container includes SkillSpector security scanning.
+
+<Note>
+ Use the generated configuration from an official CI image release. Until
+ that release is available, the source template contains placeholders and is
+ not runnable.
+</Note>
+
+## Get started
+
+Open the [GitLab source template](https://github.com/NVIDIA/SkillEvaluator/blob/main/ci/templates/gitlab-skill-checks.yml)
+in the public NVIDIA/SkillEvaluator repository. It is maintained alongside the
+GitHub templates; you do not need a GitLab mirror or a local onboarding CLI.
+The [template catalog](https://github.com/NVIDIA/SkillEvaluator/blob/main/ci/templates/README.md)
+explains how to get the release-generated version with the workflow and image
+versions filled in.
+
+<Steps>
+
+### Add the reusable job
+
+Download `gitlab-skill-checks.yml` from the release bundle. For a new pipeline,
+use its contents as `.gitlab-ci.yml`. For an existing pipeline, merge its
+`include`, variables and `skill-evaluation` rules into your root file.
+
+### Choose your skills and score
+
+Set `SKILL_EVAL_PATHS` to your skill directory and `SKILL_EVAL_MIN_SCORE` to the
+required quality score from 0 to 100 (default 70). For multiple roots, use a
+YAML multiline string.
+
+### Open a merge request
+
+The job evaluates changed skills, shows the result in the pipeline, and retains
+reports as job artifacts, including when a check fails.
+
+</Steps>
+
+The generated file has this shape; the release fills in the immutable pins:
+
+```yaml title=".gitlab-ci.yml"
+include:
+ - remote: https://raw.githubusercontent.com/NVIDIA/SkillEvaluator/RELEASE_WORKFLOW_SHA/ci/gitlab/skill-evaluation.yml
+variables:
+ SKILL_EVAL_IMAGE: ghcr.io/nvidia/skillevaluator-tier1@sha256:RELEASE_IMAGE_DIGEST
+ SKILL_EVAL_PATHS: "skills"
+ SKILL_EVAL_MIN_SCORE: "70"
+skill-evaluation:
+ rules:
+ - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
+ - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
+ - if: '$CI_PIPELINE_SOURCE == "web"'
+```
+
+A compatible runner must be able to pull the image and run Linux containers.
+If it requires tags, add them to `skill-evaluation`. Static scanning needs no
+model key, but dependency vulnerability checks may contact external services.
+
+## See the results
+
+Open the `skill-evaluation` job and download its `skill-evaluator-report`
+artifact. Open `index.html` for the consolidated view, then select a skill for
+its detailed HTML report. JSON, Markdown and SARIF are included.
+
+<Frame caption="Real demo result: the skill scored 98.2 and failed a 100-point threshold. The score and the scan verdict are separate.">
+ <img src="./assets/repository-ci-results-example.png" alt="Example report table showing a failed skill with a quality score of 98.2 and a link to its report" />
+</Frame>
+
+The [public examples](https://github.com/NVIDIA/SkillEvaluator/blob/main/ci/examples/README.md)
+include this report preview and a log excerpt without requiring access to a
+separate demo project.
+
+<Tabs>
+ <Tab title="Job log">
+
+ The log identifies the skill, result and quality score. This shortened
+ excerpt is from the real demo run shown above:
+
+```text
+Skill result: "skills/glossary-builder" — FAILED
+ Quality: 98.2/100, grade "A"
+ Check: "QUALITY" — "failed"
+ Scanner error: "Quality score 98.2/100 is below the required 100/100 ..."
+```
+
+ </Tab>
+ <Tab title="Report artifact">
+
+ Download the artifact from the job to inspect the overview and per-skill
+ findings. Artifacts expire according to the job's retention settings.
+
+ </Tab>
+</Tabs>
+
+<AccordionGroup>
+ <Accordion title="When does it run?">
+ `SKILL_EVAL_SCAN_MODE: auto` evaluates changed skills in detached
+ merge-request pipelines and all configured skills on the default branch or
+ a manual pipeline. Set it to `all` for a full scan each time. Changed-only
+ selection needs the merge-request diff base and full Git history; missing
+ history is an error, not a pass. Merged-results and merge-train pipelines
+ must use `all` until changed-only selection is supported for them. Shared
+ files outside skill directories do not select dependent skills; use `all`
+ when those changes require a complete scan.
+ </Accordion>
+ <Accordion title="Fit the job into an existing pipeline">
+ GitLab requires merge-request `rules` in the root `.gitlab-ci.yml`, even
+ when a job is included. The generated file provides them. If your project
+ has `workflow: rules`, ensure they allow merge-request pipelines while
+ preserving your existing policy. The job uses the `test` stage by default;
+ override its stage if your pipeline does not define it. A failed or
+ incomplete job fails the pipeline; configure your GitLab merge policy if it
+ should also block merging.
+ </Accordion>
+ <Accordion title="Scores, reports and privacy">
+ The threshold applies to the quality score; security, schema and other
+ required checks remain independent gates. The first version runs Tier 1
+ only. This integration does not host reports or post merge-request comments
+ automatically. Known values detected by credential patterns are redacted,
+ but not every secret or transformation is detectable. Raw scanner logs are
+ withheld when a credential is present. Remove and rotate exposed
+ credentials; previous artifacts and Git history are not rewritten.
+ </Accordion>
+</AccordionGroup>
+
+For GitHub and other CI systems, see [Repository Integration](repository-integration.mdx).
diff --git a/docs/repository-integration.mdx b/docs/repository-integration.mdx
new file mode 100644
index 00000000..f0584541
--- /dev/null
+++ b/docs/repository-integration.mdx
@@ -0,0 +1,78 @@
+---
+title: "Bring Skill Evaluation to Your Repository"
+description: "Evaluate changed skills automatically with reusable GitHub and GitLab CI configurations."
+---
+
+Bring SkillEvaluator to the repository where your team builds skills. Add a
+reusable CI configuration, point it at your skill directories, and get
+results automatically when skills change.
+
+No evaluation code to write. No scanner installations to manage. The shared
+container brings the Tier 1 runtime to your CI runner, including SkillSpector
+for specialized skill security scanning.
+
+## How it works
+
+```mermaid
+flowchart TD
+ repo["Skill changes"] --> runtime["CI runs SkillEvaluator<br/>with SkillSpector"]
+ runtime --> report["Results and reports"]
+ classDef step fill:#f3f9e8,stroke:#76b900,color:#1f2b16,stroke-width:2px
+ class repo,runtime,report step
+```
+
+The job reads your skills, checks the changes, and saves a report. It does not
+require copying SkillEvaluator or SkillSpector source code into your repository.
+
+<CardGroup cols={2}>
+ <Card title="GitHub Actions" href="github-actions.mdx">
+ Add one workflow file to evaluate changed skills in pull requests and open the results in GitHub Actions.
+ </Card>
+ <Card title="GitLab CI/CD" href="gitlab-ci.mdx">
+ Include the reusable job in your pipeline to evaluate changed skills in merge requests and download the report.
+ </Card>
+</CardGroup>
+
+## Copy a template
+
+No local installation or onboarding CLI is required. Both platforms' templates
+live in the public NVIDIA/SkillEvaluator repository. Open the file, copy the
+release-generated version, and set your skill folders and score threshold.
+
+| Your platform | Source template | Put it in your repository |
+| --- | --- | --- |
+| GitHub | [skill-checks.yml](https://github.com/NVIDIA/SkillEvaluator/blob/main/ci/templates/skill-checks.yml) | `.github/workflows/skill-checks.yml` |
+| GitLab | [gitlab-skill-checks.yml](https://github.com/NVIDIA/SkillEvaluator/blob/main/ci/templates/gitlab-skill-checks.yml) | `.gitlab-ci.yml`, or merge into your existing root configuration |
+
+The [template catalog](https://github.com/NVIDIA/SkillEvaluator/blob/main/ci/templates/README.md)
+explains release availability and includes the optional GitHub publisher and
+manual LLM workflow. Source files contain placeholders; official CI releases
+provide copies with the workflow and image versions filled in.
+
+## What you get
+
+- **Automatic checks:** evaluate changed skills in pull requests or merge requests; run the configured collection on the default branch.
+- **Quality and security:** run Tier 1 checks, including SkillSpector security scanning, without adding evaluator code to the repository.
+- **Control:** choose skill directories and a minimum quality score; the job fails when required checks fail or cannot complete.
+- **Results you can inspect:** see a concise outcome in CI and download the detailed HTML, JSON, Markdown and SARIF reports.
+
+<Frame caption="A real demo result: one changed skill scored 98.2 but failed a 100-point quality threshold.">
+ <img src="./assets/repository-ci-results-example.png" alt="Example report table showing a failed skill with a quality score of 98.2" />
+</Frame>
+
+Browse [report, log and PR comment examples](https://github.com/NVIDIA/SkillEvaluator/blob/main/ci/examples/README.md)
+in the public repository. No demo-repository access is needed.
+
+<Note>
+ The first release covers Tier 1. It does not run Tier 2 or Tier 3. An official
+ CI image release is required before the release-generated configurations can
+ be used. The source templates contain placeholders until then.
+</Note>
+
+## Other CI systems
+
+The container contains the same evaluator runtime, and the CLI can also run in
+other CI systems. The [CI integration guide](ci-integration.mdx) covers the CLI,
+exit codes and report formats. The GitHub and GitLab integrations above have
+separate platform-specific configurations; another platform needs its own
+integration and validation.
diff --git a/fern/docs.yml b/fern/docs.yml
index 3930360d..13fe8779 100644
--- a/fern/docs.yml
+++ b/fern/docs.yml
@@ -71,9 +71,15 @@ navigation:
- page: "BENCHMARK.md Rollout and Backfill"
path: ../docs/benchmark-rollout.mdx
slug: benchmark-rollout
+ - page: Repository Integration
+ path: ../docs/repository-integration.mdx
+ slug: repository-integration
- page: GitHub Actions
path: ../docs/github-actions.mdx
slug: github-actions
+ - page: GitLab CI/CD
+ path: ../docs/gitlab-ci.mdx
+ slug: gitlab-ci
- page: Gate Your CI
path: ../docs/ci-integration.mdx
slug: ci-integration
diff --git a/pyproject.toml b/pyproject.toml
index 7941ae27..d5215851 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -77,6 +77,10 @@ security = [
"bandit>=1.9.4",
"pip-audit>=2.10.0",
]
+# Verify GitHub-signed execution receipts for the optional CI report publisher.
+publisher = [
+ "pyjwt[crypto]>=2.13.0",
+]
# Optional local UI for comparing saved reports; never imported by base commands.
dashboard = [
# server.allowedHosts, which limits the dashboard to loopback host names, needs 1.61.
@@ -87,6 +91,7 @@ all = [
"skillevaluator[tier2,tier3,security]",
]
dev = [
+ "skillevaluator[publisher]",
# PyPA build backs the `make build` target.
"build>=1.2",
"httpx>=0.27",
diff --git a/tests/ci/test_evaluate_workflow.py b/tests/ci/test_evaluate_workflow.py
index f8af3c55..f01e8f3e 100644
--- a/tests/ci/test_evaluate_workflow.py
+++ b/tests/ci/test_evaluate_workflow.py
@@ -107,6 +107,56 @@ def test_smoke_container_executes_with_the_same_hardening(tmp_path, monkeypatch)
assert {entry.split("=", 1)[0] for entry in environment}.issuperset({"SCAN_MODE", "DIFF_BASE_SHA", "DIFF_HEAD_SHA"})
-def test_scan_worker_input_defaults_to_four_parallel_scans():
- workers = _workflow()["on"]["workflow_call"]["inputs"]["scan-workers"]
- assert (workers["type"], workers["default"]) == ("number", "4")
+def test_quality_threshold_is_wired_into_the_container():
+ workflow = yaml.load((ROOT / ".github/workflows/evaluate-skills.yml").read_text(), Loader=yaml.BaseLoader)
+ assert workflow["on"]["workflow_call"]["inputs"]["min-score"]["default"] == "70"
+ scan = next(step for step in workflow["jobs"]["tier1"]["steps"] if step.get("name") == "Evaluate selected skills")
+ assert scan["env"]["MIN_SCORE"] == "${{ inputs.min-score }}"
+ assert "--env MIN_SCORE" in scan["run"]
+
+
+def test_image_validation_does_not_import_consumer_python_modules(tmp_path):
+ step = next(step for step in _job()["steps"] if step.get("name") == "Validate the immutable image reference")
+ (tmp_path / "re.py").write_text("raise RuntimeError('consumer module executed on the runner')\n")
+ result = subprocess.run(
+ ["bash", "-e", "-o", "pipefail", "-c", step["run"]],
+ cwd=tmp_path,
+ env={**os.environ, "IMAGE": "ghcr.io/example/scanner@sha256:" + "a" * 64},
+ capture_output=True,
+ text=True,
+ )
+ assert result.returncode == 0, result.stderr
+
+
+def test_basic_scan_does_not_require_identity_credentials(tmp_path):
+ workflow = yaml.load((ROOT / ".github/workflows/evaluate-skills.yml").read_text(), Loader=yaml.BaseLoader)
+ job = workflow["jobs"]["tier1"]
+ assert "permissions" not in workflow and "permissions" not in job
+ step = job["steps"][0]
+ assert step["id"] == "provenance"
+ # Run the actual trusted step from a consumer checkout containing shadow modules.
+ for module in ("re", "json", "tempfile"):
+ (tmp_path / f"{module}.py").write_text("raise RuntimeError('consumer module executed')\n")
+ env = {key: value for key, value in os.environ.items() if not key.startswith("ACTIONS_ID_TOKEN_REQUEST_")}
+ result = subprocess.run(
+ ["bash", "-e", "-o", "pipefail", "-c", step["run"]],
+ cwd=tmp_path,
+ env=env,
+ capture_output=True,
+ text=True,
+ )
+ assert result.returncode == 0, result.stderr
+ assert "scan artifacts remain available" in result.stdout
+
+
+def test_signed_receipt_is_separate_and_retained_after_a_failed_scan():
+ steps = _job()["steps"]
+ capture = next(step for step in steps if step.get("id") == "provenance")
+ assert capture["continue-on-error"] == "true"
+ upload = next(step for step in steps if step.get("name") == "Retain the separate hosted-report execution receipt")
+ assert "always()" in upload["if"]
+ assert upload["with"]["name"] == "${{ inputs.artifact-name }}-provenance-attempt-${{ github.run_attempt }}"
+ assert upload["with"]["path"] == "${{ steps.provenance.outputs.directory }}/execution.jwt"
+ scan = next(step for step in steps if step.get("name") == "Evaluate selected skills")
+ assert "provenance" not in scan["run"]
+ assert "ID_TOKEN" not in scan["run"]
diff --git a/tests/ci/test_gitlab_template.py b/tests/ci/test_gitlab_template.py
new file mode 100644
index 00000000..6dd95381
--- /dev/null
+++ b/tests/ci/test_gitlab_template.py
@@ -0,0 +1,104 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+"""Contracts for the reusable GitLab job and consumer overrides."""
+from pathlib import Path
+
+import yaml
+
+ROOT = Path(__file__).parents[2]
+
+
+def test_user_can_override_included_global_defaults():
+ config = yaml.safe_load((ROOT / "ci/gitlab/skill-evaluation.yml").read_text())
+ job = config["skill-evaluation"]
+ for name in ("SKILL_EVAL_PATHS", "SKILL_EVAL_MIN_SCORE", "SKILL_EVAL_SCAN_MODE"):
+ assert name in config["variables"]
+ assert name not in job.get("variables", {})
+ assert config["variables"]["SKILL_EVAL_MIN_SCORE"] == "70"
+ assert "LLM_ENABLED" not in config["variables"]
+
+
+def _run_template(tmp_path, extra_env, *, scanner_exit=0, existing_report=False):
+ import os
+ import subprocess
+
+ config = yaml.safe_load((ROOT / "ci/gitlab/skill-evaluation.yml").read_text())
+ script = config["skill-evaluation"]["script"][0]
+ workspace = tmp_path / "workspace"
+ workspace.mkdir()
+ if existing_report:
+ (workspace / "skill-evaluator-report").mkdir()
+ bin_dir = tmp_path / "bin"
+ bin_dir.mkdir()
+ stub = bin_dir / "python"
+ stub.write_text(
+ "#!/bin/sh\n"
+ 'while [ "$#" -gt 0 ]; do\n'
+ ' if [ "$1" = --reports ]; then shift; report_dir=$1; fi\n'
+ " shift\n"
+ "done\n"
+ "printf '%s\\n%s\\n%s\\n%s\\n' \"$SCAN_MODE\" \"${DIFF_BASE_SHA:-}\" \"${DIFF_HEAD_SHA:-}\" \"$MIN_SCORE\" > \"$report_dir/evidence.txt\"\n"
+ f"exit {scanner_exit}\n"
+ )
+ stub.chmod(0o755)
+ env = {
+ **os.environ,
+ "PATH": str(bin_dir) + os.pathsep + os.environ["PATH"],
+ "CI_PROJECT_DIR": str(workspace),
+ "CI_PIPELINE_SOURCE": "push",
+ "CI_COMMIT_SHA": "b" * 40,
+ "SKILL_EVAL_IMAGE": "registry.example/test@sha256:" + "a" * 64,
+ "SKILL_EVAL_PATHS": "skills",
+ "SKILL_EVAL_MIN_SCORE": "80",
+ "SKILL_EVAL_SCAN_MODE": "auto",
+ **extra_env,
+ }
+ result = subprocess.run(["bash", "-c", script], env=env, capture_output=True, text=True, check=False)
+ report = workspace / "skill-evaluator-report/evidence.txt"
+ return result, report.read_text().splitlines() if report.exists() else None
+
+
+def test_gitlab_job_runs_all_skills_on_push_and_keeps_output(tmp_path):
+ result, evidence = _run_template(tmp_path, {})
+ assert result.returncode == 0
+ assert evidence == ["all", "", "", "80"]
+
+
+def test_gitlab_job_uses_mr_diff_and_retains_failed_results(tmp_path):
+ result, evidence = _run_template(
+ tmp_path,
+ {"CI_PIPELINE_SOURCE": "merge_request_event", "CI_MERGE_REQUEST_DIFF_BASE_SHA": "a" * 40},
+ scanner_exit=1,
+ )
+ assert result.returncode == 1
+ assert evidence == ["changed", "a" * 40, "b" * 40, "80"]
+
+
+def test_gitlab_job_rejects_merged_results_for_changed_selection(tmp_path):
+ result, evidence = _run_template(
+ tmp_path,
+ {"CI_PIPELINE_SOURCE": "merge_request_event", "CI_MERGE_REQUEST_EVENT_TYPE": "merged_result"},
+ )
+ assert result.returncode != 0
+ assert evidence is None
+ assert "detached merge-request" in result.stderr
+
+
+def test_gitlab_job_all_mode_supports_merged_result_pipelines(tmp_path):
+ result, evidence = _run_template(
+ tmp_path,
+ {
+ "CI_PIPELINE_SOURCE": "merge_request_event",
+ "CI_MERGE_REQUEST_EVENT_TYPE": "merged_result",
+ "SKILL_EVAL_SCAN_MODE": "all",
+ },
+ )
+ assert result.returncode == 0
+ assert evidence == ["all", "", "", "80"]
+
+
+def test_gitlab_job_does_not_overwrite_existing_report_directory(tmp_path):
+ result, evidence = _run_template(tmp_path, {}, existing_report=True)
+ assert result.returncode != 0
+ assert evidence is None
+ assert "already exists" in result.stderr
diff --git a/tests/ci/test_native_reports.py b/tests/ci/test_native_reports.py
index aa3ff7b9..314a7361 100644
--- a/tests/ci/test_native_reports.py
+++ b/tests/ci/test_native_reports.py
@@ -275,63 +275,18 @@ def test_failed_and_incomplete_counts_agree_across_all_surfaces(tmp_path):
assert "<strong>1</strong><span>Incomplete</span>" in page
-def test_index_uses_the_report_vocabulary_logo_and_first_fix(tmp_path):
- summary, _ = _write_fixture(tmp_path, failed=True)
- render_native_reports(tmp_path, summary, {})
- page = (tmp_path / "index.html").read_text()
- report = (tmp_path / "skill-001/skillevaluator-tier1.html").read_text()
-
- assert '<span class="status fail"><span class="sr-only">Verdict: </span>Changes required</span>' in page
- assert '<span class="status fail">Changes required</span>' in page
- assert 'class="mark"' not in page
- assert re.search(r'<aside class="rail"><div class="brand"><svg class="se-logo"[^>]*aria-hidden="true"', page)
- assert "most severe first" not in page
- # "Needs attention" names the skill report's first fix and links to its card.
- anchor = re.search(r'href="#(f-[^"]+)" class="fix-link"', report).group(1)
- assert f'href="skill-001/skillevaluator-tier1.html#{anchor}"' in page
- assert f'id="{anchor}"' in report
-
-
-def test_repository_root_skill_lists_each_fix_first_location_once(tmp_path):
- """Code Risk Analysis records absolute scan paths; other scanners record skill-relative ones."""
- risk = ValidationResult(validator_name="Code Risk Analysis", validator_description="Static analysis")
- risk.add_finding(
- Finding(
- category="BANDIT",
- severity="high",
- check_name="B602",
- message="subprocess call with shell=True",
- file_path="/workspace/scripts/fetch.py",
- line_number=21,
- )
- )
- scan = ValidationResult(validator_name="Security Scan", validator_description="Detect vulnerabilities")
- scan.add_finding(
- Finding(
- category="SECURITY",
- severity="high",
- check_name="tool_misuse",
- message="Tool Misuse: subprocess.run(command, shell=True",
- file_path="scripts/fetch.py",
- line_number=21,
- )
- )
- summary, _ = _write_fixture(tmp_path)
- payload = json.loads(JSONReporter(include_timestamp=False).render_all([risk, scan]))
- (tmp_path / "skill-001/skillevaluator-tier1.json").write_text(json.dumps(payload))
- summary["skills"][0].update(path=".", passed=False, overall_status="failed")
-
- render_native_reports(tmp_path, summary, {})
- report = (tmp_path / "skill-001/skillevaluator-tier1.html").read_text()
- fix_first = report.split('class="fix-first"', 1)[1].split("</section>", 1)[0]
-
- assert re.findall(r"<code>([^<]+)</code>", fix_first) == ["scripts/fetch.py:21"]
- assert "/workspace/" not in fix_first
-
-
-def test_index_dark_theme_never_applies_to_print():
- from ci.native_reports import _STYLE
-
- assert "@media screen and (prefers-color-scheme:dark)" in _STYLE
- assert "@media(prefers-color-scheme:dark)" not in _STYLE
- assert all(int(size) >= 11 for size in re.findall(r"font-size:(\d+)px", _STYLE))
+def test_native_overview_displays_quality_threshold_failure_not_an_advisory(tmp_path):
+ result = ValidationResult(validator_name="QUALITY", validator_description="Quality score")
+ result.add_finding(Finding(category="QUALITY", severity="medium", check_name="version", message="Optional version missing", file_path="skills/example/SKILL.md"))
+ result.fail_gate("Quality score 98.2/100 is below the required 100/100")
+ payload = json.loads(JSONReporter(include_timestamp=False).render_all([result]))
+ directory = tmp_path / "skill-001"
+ directory.mkdir()
+ (directory / "skillevaluator-tier1.json").write_text(json.dumps(payload))
+ row = {"path": "skills/example", "report_dir": "skill-001", "exit_code": 1, "passed": False}
+ row.update({key: payload[key] for key in ("overall_status", "incomplete_scans", "total_validators", "total_errors", "total_warnings", "severity_counts")})
+ render_native_reports(tmp_path, _summary([row]), {})
+ index = (tmp_path / "index.html").read_text()
+ attention = index.split('<section class="attention">', 1)[1].split("</section>", 1)[0]
+ assert "below the required 100/100" in attention
+ assert "Optional version missing" not in attention
diff --git a/tests/ci/test_publish_reports.py b/tests/ci/test_publish_reports.py
index 06bc7d5f..2f77ec7c 100644
--- a/tests/ci/test_publish_reports.py
+++ b/tests/ci/test_publish_reports.py
@@ -13,7 +13,7 @@ import zipfile
from pathlib import Path
import pytest
-import yaml
+from test_verify_provenance import JWKS, signed_receipt
ROOT = Path(__file__).parents[2]
SPEC = importlib.util.spec_from_file_location("publish_reports", ROOT / "ci/publish_reports.py")
@@ -23,6 +23,10 @@ SPEC.loader.exec_module(publisher)
@pytest.fixture(autouse=True)
def trusted_workflow_pins(monkeypatch):
+ monkeypatch.syspath_prepend(str(Path(__file__).parents[2] / "ci"))
+ import verify_provenance
+
+ monkeypatch.setattr(verify_provenance, "fetch_signing_keys", lambda: JWKS)
monkeypatch.setenv("TRUSTED_EVALUATOR_SHA", "c" * 40)
monkeypatch.setenv("TRUSTED_CALLER_BLOBS", "b" * 40)
@@ -268,32 +272,12 @@ def test_redirect_does_not_forward_api_authorization():
assert requests[1].get_header("Authorization") is None
-def test_pr_provenance_compares_recorded_revisions_through_real_github_client():
- api = publisher.GitHubAPI("owner/consumer", "private-token")
- record = run_record()
- record["event"] = "pull_request"
- record["pull_requests"] = [{"base": {"sha": "d" * 40}, "head": {"sha": "a" * 40}}]
- paths = []
-
- class Response(io.BytesIO):
- @property
- def headers(self):
- return {}
-
- class Opener:
- def open(self, request, timeout):
- assert timeout == 60
- paths.append(request.full_url)
- if "/compare/" in request.full_url:
- data = {"merge_base_commit": {"sha": "e" * 40}}
- else:
- sha = request.full_url.rsplit("?ref=", 1)[1]
- data = {"type": "file", "path": publisher.WORKFLOW_PATH, "sha": ("b" if sha == "a" * 40 else "f") * 40}
- return Response(json.dumps(data).encode())
-
- api.opener = Opener()
+def test_pr_provenance_uses_the_signed_merge_without_mutable_pr_metadata():
+ record = {**run_record(), "event": "pull_request", "pull_requests": []}
+ api = FakeAPI(record=record)
publisher._validate_workflow_provenance(api, "owner/consumer", record)
- assert any("/compare/" in path for path in paths)
+ assert f"/repos/owner/consumer/contents/{publisher.WORKFLOW_PATH}?ref=" + "d" * 40 in api.paths
+ assert not any("/compare/" in path or "/pulls" in path for path in api.paths)
@pytest.mark.parametrize(
@@ -347,7 +331,7 @@ def run_record():
"event": "push",
"head_branch": "main",
"head_sha": "a" * 40,
- "repository": {"full_name": "owner/consumer"},
+ "repository": {"full_name": "owner/consumer", "id": 1234},
"head_repository": {"full_name": "owner/consumer"},
"pull_requests": [],
"run_started_at": "2026-10-21T12:00:00Z",
@@ -392,7 +376,7 @@ class FakeAPI:
return {"type": "file", "path": publisher.WORKFLOW_PATH, "sha": "b" * 40}
if path.endswith("/artifacts?per_page=100"):
return {
- "total_count": 1,
+ "total_count": 2,
"artifacts": [
{
"id": 456,
@@ -401,7 +385,15 @@ class FakeAPI:
"size_in_bytes": len(valid_archive()),
"created_at": "2026-10-21T12:01:00Z",
"workflow_run": {"id": 123, "head_sha": "a" * 40},
- }
+ },
+ {
+ "id": 457,
+ "name": f"skill-evaluator-tier1-provenance-attempt-{self.record['run_attempt']}",
+ "expired": False,
+ "size_in_bytes": 4096,
+ "created_at": self.record["run_started_at"],
+ "workflow_run": {"id": 123, "head_sha": "a" * 40},
+ },
],
}
if path == "/repos/owner/consumer":
@@ -422,6 +414,8 @@ class FakeAPI:
raise AssertionError(path)
def download_artifact(self, artifact_id):
+ if artifact_id == 457:
+ return signed_receipt(self.record)
assert artifact_id == 456
return valid_archive()
@@ -460,7 +454,9 @@ def test_error_row_with_rejected_json_gets_a_native_unavailable_report(tmp_path,
)
class ErrorAPI(FakeAPI):
- def download_artifact(self, _artifact_id):
+ def download_artifact(self, artifact_id):
+ if artifact_id == 457:
+ return super().download_artifact(artifact_id)
return archive([("summary.json", payload), ("skill-001/skillevaluator-tier1.json", b"not JSON")])
publisher.publish(
@@ -479,7 +475,9 @@ def test_error_row_with_rejected_json_gets_a_native_unavailable_report(tmp_path,
def test_bad_typed_child_evidence_is_rejected_before_writing_a_receipt(tmp_path):
class BadTypedAPI(FakeAPI):
- def download_artifact(self, _artifact_id):
+ def download_artifact(self, artifact_id):
+ if artifact_id == 457:
+ return super().download_artifact(artifact_id)
child = report()
child["results"][0]["findings"] = "invalid findings"
return archive([("summary.json", summary()), ("skill-001/skillevaluator-tier1.json", child)])
@@ -501,7 +499,9 @@ def test_unexpected_render_failure_cannot_publish_a_passing_receipt(tmp_path, mo
monkeypatch.syspath_prepend(str(Path(__file__).parents[2] / "ci"))
class BadDisplayAPI(FakeAPI):
- def download_artifact(self, _artifact_id):
+ def download_artifact(self, artifact_id):
+ if artifact_id == 457:
+ return super().download_artifact(artifact_id)
payload = summary()
payload["skills"][0]["path"] = "skills/bad\tpath"
return archive([("summary.json", payload), ("skill-001/skillevaluator-tier1.json", report())])
@@ -556,12 +556,13 @@ def test_requested_second_attempt_gets_only_its_own_artifact(tmp_path):
class SecondAttemptAPI(FakeAPI):
def get(self, path):
if path.endswith("/actions/runs/123/attempts/2"):
- return {
+ self.record = {
**run_record(),
"run_attempt": 2,
"run_started_at": "2026-10-21T13:00:00Z",
"updated_at": "2026-10-21T13:02:00Z",
}
+ return self.record
result = super().get(path)
if path.endswith("/artifacts?per_page=100"):
result["artifacts"].append(
@@ -576,6 +577,8 @@ def test_requested_second_attempt_gets_only_its_own_artifact(tmp_path):
return result
def download_artifact(self, artifact_id):
+ if artifact_id == 457:
+ return super().download_artifact(artifact_id)
assert artifact_id == 789
return valid_archive()
@@ -726,41 +729,12 @@ def test_fork_and_deleted_fork_from_commit_fallback_get_no_alias_or_comment(repo
assert publisher._current_routes(ForkAPI(), "owner/consumer", {**run_record(), "event": "pull_request"}) == ([], "")
- "blobs,expected",
- [
- ({"a": "1", "d": "2", "e": "1"}, "2"),
- ({"a": "2", "d": "1", "e": "1"}, "2"),
- ({"a": "2", "d": "2", "e": "1"}, "2"),
- ({"a": "2", "d": "3", "e": "1"}, None),
- ],
-)
-def test_pr_caller_uses_unambiguous_merged_blob_not_pr_head(blobs, expected):
- class MergeAPI(FakeAPI):
- def get(self, path):
- if "/compare/" in path:
- assert path.endswith("d" * 40 + "..." + "a" * 40)
- return {"merge_base_commit": {"sha": "e" * 40}}
- result = super().get(path)
- if "/contents/" in path:
- result["sha"] = blobs[path.rsplit("=", 1)[1][0]] * 40
- return result
-
- run = {
- **run_record(),
- "event": "pull_request",
- "pull_requests": [{"number": 1, "head": {"sha": "a" * 40}, "base": {"sha": "d" * 40}}],
- }
- if expected is None:
- with pytest.raises(publisher.PublicationError, match="Both branches changed"):
- publisher._executed_caller_blob(MergeAPI(), "owner/consumer", run)
- else:
- assert publisher._executed_caller_blob(MergeAPI(), "owner/consumer", run) == expected * 40
-
-
-def test_pr_caller_does_not_trust_missing_or_current_mutable_base():
- with pytest.raises(publisher.StalePublication, match="no longer associated"):
- publisher._executed_caller_blob(FakeAPI(), "owner/consumer", {**run_record(), "event": "pull_request"})
[email protected]("associations", [None, [], [{"base": {"sha": "e" * 40}, "head": {"sha": "f" * 40}}]])
+def test_pr_caller_identity_ignores_current_mutable_associations(associations):
+ record = {**run_record(), "event": "pull_request", "pull_requests": associations}
+ api = FakeAPI(record=record)
+ publisher._validate_workflow_provenance(api, "owner/consumer", record)
+ assert not any("/pulls" in path or "/compare/" in path for path in api.paths)
def test_open_fork_pr_run_is_published_without_alias_or_comment(tmp_path):
@@ -779,8 +753,8 @@ def test_open_fork_pr_run_is_published_without_alias_or_comment(tmp_path):
assert (tmp_path / "site/runs/123/1/index.html").is_file()
assert not (tmp_path / "site/pr").exists()
assert [item["run_id"] for item in json.loads((tmp_path / "site/history.json").read_text())] == [123]
- # The fork PR is found from API run metadata, never from artifact data.
- assert "/repos/owner/consumer/pulls?state=open&head=contributor:feature%2Ffork&per_page=100" in api.paths
+ # Signed execution identity remains verifiable without mutable PR associations.
+ assert not any("/pulls?" in path for path in api.paths)
def test_history_retention_preserves_current_main_and_open_pr_and_prunes_closed(tmp_path):
@@ -927,7 +901,9 @@ def test_recorded_image_provenance_wins_and_legacy_stays_unknown(tmp_path, recor
document["execution"] = {"image": recorded}
class ImageAPI(FakeAPI):
- def download_artifact(self, _artifact_id):
+ def download_artifact(self, artifact_id):
+ if artifact_id == 457:
+ return super().download_artifact(artifact_id)
return archive([("summary.json", document), ("skill-001/skillevaluator-tier1.json", report())])
contexts = []
@@ -961,7 +937,9 @@ def test_recorded_image_provenance_wins_and_legacy_stays_unknown(tmp_path, recor
)
def test_invalid_recorded_image_provenance_is_rejected(tmp_path, execution):
class BadImageAPI(FakeAPI):
- def download_artifact(self, _artifact_id):
+ def download_artifact(self, artifact_id):
+ if artifact_id == 457:
+ return super().download_artifact(artifact_id)
return archive(
[
("summary.json", {**summary(), "execution": execution}),
@@ -1163,6 +1141,8 @@ def test_no_change_requires_consistent_selection_evidence(change):
def test_no_change_publication_matches_run_head_and_success(tmp_path, head, conclusion, accepted):
class NoChangeAPI(FakeAPI):
def download_artifact(self, artifact_id):
+ if artifact_id == 457:
+ return super().download_artifact(artifact_id)
assert artifact_id == 456
payload = skipped_summary()
payload["selection"]["head"] = head
@@ -1183,49 +1163,21 @@ def test_explicit_fork_evaluator_identity_is_required(monkeypatch):
record = run_record()
record["referenced_workflows"][0]["path"] = "example/evaluator/.github/workflows/evaluate-skills.yml@" + "c" * 40
with pytest.raises(publisher.PublicationError):
- publisher._validate_workflow_provenance(FakeAPI(), "owner/consumer", record)
+ publisher._validate_workflow_provenance(FakeAPI(record=record), "owner/consumer", record)
monkeypatch.setenv("EVALUATOR_REPOSITORY", "example/evaluator")
- publisher._validate_workflow_provenance(FakeAPI(), "owner/consumer", record)
-
-
-class ForkLookupAPI(FakeAPI):
- def __init__(self, pulls, **kwargs):
- super().__init__(**kwargs)
- self.pulls = pulls
+ publisher._validate_workflow_provenance(FakeAPI(record=record), "owner/consumer", record)
- def get(self, path):
- if "/pulls?state=open&head=" in path:
- self.paths.append(path)
- return self.pulls
- return super().get(path)
+def test_missing_receipt_is_a_skipped_publisher_run(tmp_path, monkeypatch):
+ class LegacyAPI(FakeAPI):
+ def get(self, path):
+ result = super().get(path)
+ if path.endswith("/artifacts?per_page=100"):
+ result["artifacts"] = result["artifacts"][:1]
+ result["total_count"] = 1
+ return result
- "record,pulls,reason",
- [
- pytest.param(
- {**run_record(), "event": "pull_request"},
- None,
- "The PR is closed or no longer associated",
- id="same-repo-pr-merged-after-scan",
- ),
- pytest.param(
- {
- **run_record(),
- "event": "pull_request",
- "pull_requests": [{"base": {"sha": "d" * 40}, "head": {"sha": "e" * 40}}],
- },
- None,
- "The PR head moved",
- id="same-repo-push-during-scan",
- ),
- pytest.param(fork_record(), [], "The fork PR is closed or merged", id="fork-pr-closed-after-scan"),
- pytest.param(fork_record(), [fork_pull("e" * 40)], "The PR head moved", id="fork-push-during-scan"),
- pytest.param(fork_record(head_repository=None), None, "fork for this PR run was deleted", id="deleted-fork"),
- ],
-)
-def test_stale_or_closed_pr_is_a_skipped_publisher_run(tmp_path, monkeypatch, record, pulls, reason):
- api = ForkLookupAPI(pulls, record=record)
+ api = LegacyAPI()
monkeypatch.setattr(publisher, "GitHubAPI", lambda *_args: api)
for key, value in {
"GH_TOKEN": "dummy-token",
@@ -1240,49 +1192,58 @@ def test_stale_or_closed_pr_is_a_skipped_publisher_run(tmp_path, monkeypatch, re
monkeypatch.setenv(key, value)
assert publisher.main() == 0
assert "skipped=true" in (tmp_path / "github-output").read_text()
- assert "skipped" in (tmp_path / "output/summary.md").read_text().lower()
- assert reason in (tmp_path / "output/summary.md").read_text()
+ assert "id-token: write" in (tmp_path / "output/summary.md").read_text()
+ assert "trigger a new scan" in (tmp_path / "output/summary.md").read_text()
assert not (tmp_path / "site/runs").exists()
- # Only a fork run with a known head repository looks up its PR.
- assert any("/pulls?" in path for path in api.paths) == (pulls is not None)
@pytest.mark.parametrize(
- "record,pulls",
+ "change",
[
- pytest.param({**run_record(), "event": "pull_request", "pull_requests": None}, None, id="same-repo-missing"),
- pytest.param(
- {
- **run_record(),
- "event": "pull_request",
- "pull_requests": [{"base": {"sha": "d" * 40}, "head": {"sha": "a" * 40}}] * 2,
- },
- None,
- id="same-repo-two-prs",
- ),
- pytest.param(
- {**run_record(), "event": "pull_request", "pull_requests": [{"base": {"sha": "d" * 40}, "head": {}}]},
- None,
- id="same-repo-malformed-head",
- ),
- pytest.param(fork_record(pull_requests=None), None, id="fork-missing"),
- pytest.param(fork_record(head_repository="contributor/consumer"), None, id="fork-malformed-head-repository"),
- pytest.param(fork_record(head_branch=None), None, id="fork-missing-branch"),
- pytest.param(fork_record(), {"items": []}, id="fork-malformed-listing"),
- pytest.param(fork_record(), [fork_pull(), fork_pull()], id="fork-two-prs"),
- pytest.param(fork_record(), [fork_pull(base_sha="main")], id="fork-malformed-base"),
+ {"workflow_run": {"id": 124, "head_sha": "a" * 40}},
+ {"workflow_run": {"id": 123, "head_sha": "b" * 40}},
+ {"size_in_bytes": 65537},
],
)
-def test_malformed_pr_association_still_fails(record, pulls):
+def test_receipt_artifact_identity_and_size_are_enforced(change):
+ class BadReceiptAPI(FakeAPI):
+ def get(self, path):
+ result = super().get(path)
+ if path.endswith("/artifacts?per_page=100"):
+ result["artifacts"][1].update(change)
+ return result
+
with pytest.raises(publisher.PublicationError) as raised:
- publisher._validate_workflow_provenance(ForkLookupAPI(pulls, record=record), "owner/consumer", record)
+ publisher._validate_workflow_provenance(BadReceiptAPI(), "owner/consumer", run_record())
assert not isinstance(raised.value, publisher.StalePublication)
+ "change",
+ [
+ {"name": "skill-evaluator-tier1-provenance-attempt-2"},
+ {"expired": True},
+ {"created_at": "2026-10-20T12:00:00Z"},
+ ],
+)
+def test_other_attempt_or_unavailable_receipt_cannot_publish(change):
+ class OtherReceiptAPI(FakeAPI):
+ def get(self, path):
+ result = super().get(path)
+ if path.endswith("/artifacts?per_page=100"):
+ result["artifacts"][1].update(change)
+ return result
+
+ with pytest.raises(publisher.StalePublication):
+ publisher._validate_workflow_provenance(OtherReceiptAPI(), "owner/consumer", run_record())
+
+
@pytest.mark.parametrize("enabled", [True, False, None])
def test_published_report_preserves_recorded_llm_request(tmp_path, enabled):
class RecordedAPI(FakeAPI):
def download_artifact(self, artifact_id):
+ if artifact_id == 457:
+ return super().download_artifact(artifact_id)
assert artifact_id == 456
payload = summary()
if enabled is not None:
diff --git a/tests/ci/test_publisher_workflow.py b/tests/ci/test_publisher_workflow.py
index 59cbb208..dc23fb4b 100644
--- a/tests/ci/test_publisher_workflow.py
+++ b/tests/ci/test_publisher_workflow.py
@@ -72,245 +72,5 @@ def test_privileged_publisher_installs_the_reviewed_hash_locked_bootstrap():
steps = load(".github/workflows/publish-skill-reports.yml")["jobs"]["prepare"]["steps"]
install = next(step for step in steps if step.get("name") == "Install the trusted native report renderer")
assert install["working-directory"] == "publisher"
- lines = [line.strip() for line in install["run"].replace("\\\n", " ").splitlines()]
- # uv comes from its hash-locked requirement, installed into a private environment.
- assert lines[0] == 'python3 -m venv "$RUNNER_TEMP/uv"'
- assert lines[1].startswith('"$RUNNER_TEMP/uv/bin/python" -m pip install ')
- for flag in ("--require-hashes", "--no-deps", "--only-binary :all:", "-r ci/requirements/uv.txt"):
- assert flag in lines[1]
- assert lines[2] == '"$RUNNER_TEMP/uv/bin/uv" sync --frozen --no-dev --no-editable'
- assert len(lines) == 3
- # The runner's system Python and pip are never modified.
- assert "python3 -m pip" not in install["run"]
- # This job can push to gh-pages, so it never restores a dependency cache that
- # other default-branch workflows could have written.
- for step in steps:
- assert "actions/cache" not in step.get("uses", "")
- assert "setup-uv" not in step.get("uses", "")
-
-
-def expression(text: str, context: dict) -> bool:
- """Evaluate the small GitHub expression subset these job conditions use."""
- tokens = re.findall(r"'[^']*'|==|!=|&&|\|\||!|\(|\)|always\(\)|[A-Za-z_][\w.-]*", text)
- assert "".join(tokens) == re.sub(r"\s+", "", text)
- translated = {"&&": " and ", "||": " or ", "!": " not ", "always()": "True"}
- python = []
- for token in tokens:
- if token.startswith("'"):
- python.append(repr(token[1:-1]))
- elif token in translated:
- python.append(translated[token])
- elif token in {"==", "!=", "(", ")"}:
- python.append(token)
- else:
- python.append(repr(context[token]))
- # The input is a workflow file from this repository, translated token by token.
- return eval("".join(python), {"__builtins__": {}})
-
-
-def job_context(committed, event="workflow_run", attempt="1", deploy="success", skipped=""):
- return {
- "needs.prepare.result": "success",
- "needs.prepare.outputs.skipped": skipped,
- "needs.prepare.outputs.committed": committed,
- "needs.prepare.outputs.pr_number": "1",
- "needs.prepare.outputs.is_current": "true",
- "needs.deploy.result": deploy,
- "github.event_name": event,
- "github.run_attempt": attempt,
- }
-
-
- ("committed", "event", "attempt", "deploys"),
- [
- ("true", "workflow_run", "1", True),
- ("false", "workflow_run", "1", False),
- ("false", "workflow_dispatch", "1", True),
- ("false", "workflow_run", "2", True),
- ("true", "workflow_run", "3", True),
- ],
-)
-def test_deploy_is_skipped_only_when_an_automatic_first_attempt_committed_nothing(committed, event, attempt, deploys):
- deploy = load(".github/workflows/publish-skill-reports.yml")["jobs"]["deploy"]
- assert deploy["needs"] == "prepare"
- # No status function, so GitHub also requires prepare to succeed.
- assert "always()" not in deploy["if"]
- assert expression(deploy["if"], job_context(committed, event, attempt)) is deploys
-
-
- ("committed", "deploy", "comments"),
- [
- ("true", "success", True),
- ("false", "success", True),
- # The run that pushed this content owns its deploy and its comment.
- ("false", "skipped", False),
- ("true", "skipped", False),
- ("true", "failure", False),
- ("false", "failure", False),
- ("true", "cancelled", False),
- ],
-)
-def test_comment_waits_for_this_runs_own_successful_deploy(committed, deploy, comments):
- job = load(".github/workflows/publish-skill-reports.yml")["jobs"]["comment"]
- assert expression(job["if"], job_context(committed, deploy=deploy)) is comments
- assert not expression(job["if"], {**job_context(committed, deploy=deploy), "needs.prepare.result": "failure"})
-
-
[email protected]("committed", ["true", "false"])
- ("event", "attempt"), [("workflow_run", "1"), ("workflow_dispatch", "1"), ("workflow_run", "2")]
-)
-def test_skipped_stale_publication_never_deploys_or_comments(committed, event, attempt):
- # A stale run can follow an earlier attempt that already wrote pr_number and is_current.
- jobs = load(".github/workflows/publish-skill-reports.yml")["jobs"]
- for deploy in ("success", "skipped"):
- context = job_context(committed, event, attempt, deploy, skipped="true")
- assert not expression(jobs["deploy"]["if"], context)
- assert not expression(jobs["comment"]["if"], context)
-
-
-def run_publish_loop(tmp_path, *, diffs, pushes):
- """Run the real publish step shell with stand-ins for git and the publisher."""
- step = next(
- step
- for step in load(".github/workflows/publish-skill-reports.yml")["jobs"]["prepare"]["steps"]
- if step.get("id") == "publish"
- )
- workspace, temp, bin_dir = tmp_path / "workspace", tmp_path / "temp", tmp_path / "bin"
- python = workspace / "publisher/.venv/bin/python"
- for directory in (workspace / "site", python.parent, temp, bin_dir):
- directory.mkdir(parents=True, exist_ok=True)
- python.write_text('#!/bin/sh\nmkdir -p "$OUTPUT_DIR"\necho summary > "$OUTPUT_DIR/summary.md"\n')
- git = bin_dir / "git"
- # Each `diff --cached --quiet` and `push` pops its exit code from a queue file.
- git.write_text(
- "#!/bin/sh\n"
- 'echo "$*" >> "$CALLS"\n'
- 'pop() { code=$(head -n 1 "$1"); tail -n +2 "$1" > "$1.next"; mv "$1.next" "$1"; exit "$code"; }\n'
- 'case "$3" in diff) pop "$DIFFS";; push) pop "$PUSHES";; esac\n'
- )
- for script in (python, git):
- script.chmod(0o755)
- (tmp_path / "diffs").write_text("".join(f"{code}\n" for code in diffs))
- (tmp_path / "pushes").write_text("".join(f"{code}\n" for code in pushes))
- output = tmp_path / "github-output"
- env = {
- "PATH": f"{bin_dir}{os.pathsep}{os.environ['PATH']}",
- "GITHUB_WORKSPACE": str(workspace),
- "RUNNER_TEMP": str(temp),
- "GITHUB_OUTPUT": str(output),
- "GITHUB_STEP_SUMMARY": str(tmp_path / "step-summary"),
- "REPORT_RUN_ID": "123",
- "CALLS": str(tmp_path / "calls"),
- "DIFFS": str(tmp_path / "diffs"),
- "PUSHES": str(tmp_path / "pushes"),
- }
- subprocess.run(
- ["bash", "--noprofile", "--norc", "-eo", "pipefail", "-c", step["run"]],
- cwd=workspace,
- env=env,
- check=True,
- capture_output=True,
- )
- calls = (tmp_path / "calls").read_text().splitlines()
- return output.read_text().splitlines(), calls
-
-
- ("diffs", "pushes", "committed", "commits"),
- [
- ([0], [], "false", 0),
- ([1], [0], "true", 1),
- ([1, 1], [1, 0], "true", 2),
- # A concurrent writer already published this exact run attempt.
- ([1, 0], [1], "false", 1),
- ],
-)
-def test_publish_step_reports_whether_it_pushed_new_content(tmp_path, diffs, pushes, committed, commits):
- output, calls = run_publish_loop(tmp_path, diffs=diffs, pushes=pushes)
- assert output == [f"committed={committed}"]
- assert sum(call.startswith("-C site commit") for call in calls) == commits
- assert load(".github/workflows/publish-skill-reports.yml")["jobs"]["prepare"]["outputs"]["committed"] == (
- "${{ steps.publish.outputs.committed }}"
- )
-
-
- ("event", "attempt", "deploys"),
- [("workflow_run", "1", False), ("workflow_dispatch", "1", True), ("workflow_run", "2", True)],
-)
-def test_losing_a_push_race_to_the_same_attempt_never_comments_before_a_deploy(tmp_path, event, attempt, deploys):
- # Attempt 1's push is rejected because another publisher run pushed this scan
- # attempt first. After the reset, attempt 2 has nothing left to commit.
- output, _calls = run_publish_loop(tmp_path, diffs=[1, 0], pushes=[1])
- assert output == ["committed=false"]
- jobs = load(".github/workflows/publish-skill-reports.yml")["jobs"]
- context = job_context("false", event, attempt)
- assert expression(jobs["deploy"]["if"], context) is deploys
- deploy = "success" if deploys else "skipped"
- assert expression(jobs["comment"]["if"], {**context, "needs.deploy.result": deploy}) is deploys
-
-
-def checkout(job: str) -> dict:
- steps = load(".github/workflows/publish-skill-reports.yml")["jobs"][job]["steps"]
- return next(
- step for step in steps if step.get("uses", "").startswith("actions/checkout@") and "repository" in step["with"]
- )
-
-
-def tracked_files() -> list[str]:
- listing = subprocess.run(["git", "ls-files", "-z"], cwd=ROOT, capture_output=True, check=True).stdout
- return [name for name in listing.decode().split("\0") if name and (ROOT / name).is_file()]
-
-
-def test_prepare_sparse_checkout_has_the_build_inputs_and_tools(tmp_path):
- options = checkout("prepare")["with"]
- assert options.get("sparse-checkout-cone-mode", "true") == "true"
- directories = options["sparse-checkout"].split()
- assert directories == ["ci", "src"]
- # Cone mode checks out every root file plus the listed directories.
- selected = [name for name in tracked_files() if "/" not in name or name.split("/", 1)[0] in directories]
- for name in selected:
- (tmp_path / name).parent.mkdir(parents=True, exist_ok=True)
- shutil.copy2(ROOT / name, tmp_path / name)
- project = tomllib.loads((tmp_path / "pyproject.toml").read_text())
- build_inputs = [project["project"]["readme"], *project["project"]["license-files"], "uv.lock"]
- build_inputs.append("ci/requirements/uv.txt")
- for name in build_inputs:
- assert (tmp_path / name).is_file(), name
- for package, data in project["tool"]["setuptools"]["package-data"].items():
- assert (tmp_path / "src" / package.replace(".", "/")).is_dir(), package
- assert data
- code = (
- "import sys; sys.path[:0] = [sys.argv[1] + '/src', sys.argv[1] + '/ci'];"
- "import publish_reports, native_reports, report_views, github_api, skillevaluator.reporting.retained as r;"
- "mods = (publish_reports, native_reports, report_views, github_api, r);"
- "assert all(m.__file__.startswith(sys.argv[1]) for m in mods), [m.__file__ for m in mods]"
- )
- subprocess.run([sys.executable, "-c", code, str(tmp_path)], check=True, capture_output=True, cwd=tmp_path)
-
-
-def test_comment_sparse_checkout_runs_with_only_the_standard_library(tmp_path):
- options = checkout("comment")["with"]
- assert options["sparse-checkout-cone-mode"] == "false"
- names = options["sparse-checkout"].split()
- assert names == ["ci/comment_report.py", "ci/github_api.py"]
- for name in names:
- (tmp_path / name).parent.mkdir(parents=True, exist_ok=True)
- shutil.copy2(ROOT / name, tmp_path / name)
- run = load(".github/workflows/publish-skill-reports.yml")["jobs"]["comment"]["steps"][-1]["run"]
- assert run == "python3 publisher/ci/comment_report.py"
- # No site-packages, user site or PYTHON* variables, like the runner's bare system Python.
- result = subprocess.run(
- [sys.executable, "-S", "-s", "-E", str(tmp_path / "ci/comment_report.py")],
- cwd=tmp_path,
- env={"REPORT_RUN_ID": "invalid"},
- capture_output=True,
- text=True,
- check=False,
- )
- assert result.returncode == 1
- assert "PR feedback rejected" in result.stderr
- assert "Traceback" not in result.stderr
+ assert "python3 -m pip install --require-hashes -r ci/requirements/uv.txt" in install["run"]
+ assert "uv sync --frozen --no-dev --no-editable --extra publisher" in install["run"]
diff --git a/tests/ci/test_render_templates.py b/tests/ci/test_render_templates.py
index 20bad56f..58f3ffd6 100644
--- a/tests/ci/test_render_templates.py
+++ b/tests/ci/test_render_templates.py
@@ -56,3 +56,15 @@ def test_fork_verification_templates_point_to_the_fork(tmp_path):
def test_existing_release_output_is_preserved(tmp_path):
with pytest.raises(ValueError, match="new output directory"):
release.render_templates(tmp_path, "b" * 40, "ghcr.io/nvidia/scan@sha256:" + "a" * 64)
+
+
+def test_gitlab_consumer_pins_image_and_reviewed_job_and_enables_mr_rules(tmp_path):
+ output = tmp_path / "out"
+ image = "ghcr.io/nvidia/skillevaluator-tier1@sha256:" + "a" * 64
+ release.render_templates(output, "b" * 40, image)
+ config = yaml.safe_load((output / "gitlab-skill-checks.yml").read_text())
+ assert config["variables"]["SKILL_EVAL_IMAGE"] == image
+ assert config["include"][0]["remote"] == (
+ "https://raw.githubusercontent.com/NVIDIA/SkillEvaluator/" + "b" * 40 + "/ci/gitlab/skill-evaluation.yml"
+ )
+ assert any("merge_request_event" in rule["if"] for rule in config["skill-evaluation"]["rules"])
diff --git a/tests/ci/test_report_views.py b/tests/ci/test_report_views.py
index ba9ca54e..75ddd5c8 100644
--- a/tests/ci/test_report_views.py
+++ b/tests/ci/test_report_views.py
@@ -878,3 +878,20 @@ def test_noop_requires_complete_consistent_selection_evidence(tmp_path, mutate):
views.render_reports(tmp_path, summary, {})
assert "NO SKILLS CHANGED" not in (tmp_path / "summary.md").read_text()
assert "PASSED" not in compact_header((tmp_path / "summary.md").read_text())
+
+
+def test_quality_threshold_failure_is_not_replaced_by_advisory_finding(tmp_path):
+ result = {
+ "validator": "QUALITY",
+ "status": "failed",
+ "passed": False,
+ "summary": {"errors": 1, "warnings": 1, "checks_performed": 1},
+ "findings": [{"severity": "medium", "message": "Optional version missing", "suggestion": "Add a version"}],
+ "legacy": {"errors": ["Quality score 98.2/100 is below the required 100/100"], "warnings": []},
+ "gate_failures": ["Quality score 98.2/100 is below the required 100/100"],
+ }
+ summary, _ = example(tmp_path, result=result)
+ views.render_reports(tmp_path, summary, {})
+ for filename in ("summary.md", "details.md", "pr-comment.md"):
+ assert "below the required 100/100" in visible_markdown((tmp_path / filename).read_text())
+ assert "Optional version missing" not in visible_markdown((tmp_path / "summary.md").read_text())
diff --git a/tests/ci/test_run_tier1.py b/tests/ci/test_run_tier1.py
index f4d30ba6..d586cd7a 100644
--- a/tests/ci/test_run_tier1.py
+++ b/tests/ci/test_run_tier1.py
@@ -342,6 +342,7 @@ def test_llm_maps_only_explicit_provider_and_credentials(tmp_path):
"LLM_PROVIDER": "nv_build",
"LLM_API_KEY": "test-key",
"LLM_MODEL": "test-model",
+ "MIN_SCORE": "85",
"OPENAI_API_KEY": "ambient-key",
"SKILLSPECTOR_PROVIDER": "untrusted-ambient-provider",
"SKILL_EVAL_LLM_BASE_URL": "https://unexpected.invalid",
@@ -349,6 +350,7 @@ def test_llm_maps_only_explicit_provider_and_credentials(tmp_path):
assert runner.run(workspace, tmp_path / "reports", environment, execute=fake_cli([(0, payload())], calls)) == 0
argv, kwargs = calls[0]
assert "--llm" in argv
+ assert argv[argv.index("--min-score") + 1] == "85"
assert kwargs["env"]["NVIDIA_API_KEY"] == "test-key"
assert kwargs["env"]["SKILL_EVAL_LLM_PROVIDER"] == "nv_build"
assert kwargs["env"]["SKILL_EVAL_LLM_MODEL"] == "test-model"
@@ -730,588 +732,32 @@ def test_relative_annotation_location_is_confined_to_skill(capsys, path, expecte
assert "file=" not in command and "line=" not in command
-def parallel_cli(responses, calls, *, before=None):
- """Fake CLI for concurrent runs; ``before(name)`` runs inside the call."""
- lock = threading.Lock()
-
- def invoke(argv, **kwargs):
- output = Path(argv[argv.index("-o") + 1])
- name = report_dir(output)
- with lock:
- calls.append((name, dict(kwargs["env"])))
- if before:
- before(name)
- exit_code, report = responses[name]
- kwargs["stdout"].write(b"::error::untrusted scanner text\n")
- (output / "skillevaluator-tier1.json").write_text(json.dumps(report), encoding="utf-8")
- return CompletedProcess(argv, exit_code)
-
- return invoke
-
-
-def scan_outputs(reports):
- names = ["summary.json", "summary.md", "details.md", "pr-comment.md", "index.html"]
- names += sorted(path.relative_to(reports).as_posix() for path in reports.glob("skill-*/skillevaluator-tier1.html"))
- return {name: (reports / name).read_text() for name in names}
-
-
-def test_parallel_scan_matches_sequential_rows_logs_annotations_and_reports(tmp_path, capsys):
- workspace = tmp_path / "workspace"
- names = [f"skills/s{index}" for index in range(1, 7)]
- for name in names:
- skill(workspace, name)
- failing = {"skill-002", "skill-003", "skill-005"}
- # Three failing skills with four findings each exceed the 10-annotation budget.
- responses = {
- f"skill-{index:03d}": (1, payload(False, "failed", findings=4))
- if f"skill-{index:03d}" in failing
- else (0, payload())
- for index in range(1, 7)
- }
- environment = {"SKILL_PATHS": "skills", "GITHUB_ACTIONS": "true"}
- outputs, logs, finished = {}, {}, {}
- for workers in ("1", "4"):
- order = finished[workers] = []
- done = {name: threading.Event() for name in responses}
-
- def before(name, workers=workers, order=order, done=done):
- if workers != "1" and name == "skill-001":
- # Finish skill 1 last so completion order differs from skill order.
- assert done["skill-002"].wait(WAIT)
- order.append(name)
- done[name].set()
-
- reports = tmp_path / f"reports-{workers}"
- result = runner.run(
- workspace,
- reports,
- {**environment, "SCAN_WORKERS": workers},
- execute=parallel_cli(responses, [], before=before),
- )
- assert result == 1
- outputs[workers] = scan_outputs(reports)
- logs[workers] = [line for line in capsys.readouterr().out.splitlines() if "at a time" not in line]
- assert finished["1"] == sorted(responses)
- assert finished["4"].index("skill-002") < finished["4"].index("skill-001")
- assert logs["1"] == logs["4"]
- assert outputs["1"] == outputs["4"]
- annotations = [line for line in logs["4"] if line.startswith("::error")]
- assert len(annotations) == 10
- assert all("file=skills/s2/SKILL.md" in line or "file=skills/s3/SKILL.md" in line for line in annotations[:8])
- headers = [line for line in logs["4"] if line.startswith("SkillEvaluator: evaluating skill")]
- assert headers == [
- f"SkillEvaluator: evaluating skill {index} of 6: {json.dumps(name)}" for index, name in enumerate(names, 1)
- ]
-
-
-def test_default_scan_runs_four_skills_at_once(tmp_path, capsys):
- workspace = tmp_path / "workspace"
- for index in range(5):
- skill(workspace, f"skills/s{index}")
- responses = {f"skill-{index:03d}": (0, payload()) for index in range(1, 6)}
- barrier = threading.Barrier(4, timeout=WAIT)
- lock = threading.Lock()
- state = {"active": 0, "peak": 0, "calls": 0}
-
- def before(name):
- with lock:
- state["calls"] += 1
- state["active"] += 1
- state["peak"] = max(state["peak"], state["active"])
- first_wave = state["calls"] <= 4
- if first_wave:
- barrier.wait()
- with lock:
- state["active"] -= 1
-
- calls = []
- assert runner.run(workspace, tmp_path / "reports", {}, execute=parallel_cli(responses, calls, before=before)) == 0
- assert state["peak"] == 4 and len(calls) == 5
- assert "SkillEvaluator: scanning 5 skills, up to 4 at a time." in capsys.readouterr().out
-
-
-def test_one_worker_runs_skills_strictly_one_at_a_time(tmp_path):
- workspace = tmp_path / "workspace"
- for index in range(3):
- skill(workspace, f"skills/s{index}")
- reports = tmp_path / "reports"
- responses = {f"skill-{index:03d}": (0, payload()) for index in range(1, 4)}
- seen = []
-
- def before(name):
- # The previous skill's result is saved before this one starts.
- seen.append(json.loads((reports / "summary.json").read_text())["passed"])
-
- assert (
- runner.run(workspace, reports, {"SCAN_WORKERS": "1"}, execute=parallel_cli(responses, [], before=before)) == 0
- )
- assert seen == [0, 1, 2]
-
-
-def test_each_worker_gets_its_own_home_and_temporary_directory(tmp_path):
- workspace = tmp_path / "workspace"
- for index in range(4):
- skill(workspace, f"skills/s{index}")
- audit = tmp_path / "audit-tmp"
- audit.mkdir()
- responses = {f"skill-{index:03d}": (0, payload()) for index in range(1, 5)}
- barrier = threading.Barrier(2, timeout=WAIT)
- pairs = []
-
- def before(name):
- barrier.wait()
- pairs.append(name)
-
- calls = []
- environment = {"SCAN_WORKERS": "2", "TMPDIR": str(audit)}
- assert (
- runner.run(workspace, tmp_path / "reports", environment, execute=parallel_cli(responses, calls, before=before))
- == 0
- )
- assert len(calls) == 4
- homes = {env["HOME"] for _, env in calls}
- assert homes == {"/tmp/evaluator-home/worker-1", "/tmp/evaluator-home/worker-2"}
- for _, env in calls:
- worker = env["HOME"].rsplit("-", 1)[1]
- assert env["TMPDIR"] == str(audit / f"skillevaluator-worker-{worker}")
- assert Path(env["HOME"]).is_dir() and Path(env["TMPDIR"]).is_dir()
- # Calls 1-2 and 3-4 ran together, each pair on two different workers.
- assert calls[0][1]["HOME"] != calls[1][1]["HOME"] and calls[2][1]["HOME"] != calls[3][1]["HOME"]
-
-
-def test_concurrent_timeout_keeps_finished_results_and_starts_no_more_skills(tmp_path, capsys):
- workspace = tmp_path / "workspace"
- for name in "abcde":
- skill(workspace, f"skills/{name}")
- reports = tmp_path / "reports"
- now = [0.0]
- second_started, first_timed_out = threading.Event(), threading.Event()
- calls = []
-
- def execute(argv, **kwargs):
- output = Path(argv[argv.index("-o") + 1])
- calls.append(report_dir(output))
- if report_dir(output) == "skill-001":
- assert second_started.wait(WAIT)
- now[0] = runner.SCAN_TIMEOUT_SECONDS + 1
- first_timed_out.set()
- raise subprocess.TimeoutExpired(argv, kwargs["timeout"])
- second_started.set()
- # Skill 2 was already running when the budget ran out; its result counts.
- assert first_timed_out.wait(WAIT)
- (output / "skillevaluator-tier1.json").write_text(json.dumps(payload()), encoding="utf-8")
- return CompletedProcess(argv, 0)
-
- environment = {"SCAN_WORKERS": "2"}
- assert runner.run(workspace, reports, environment, execute=execute, clock=lambda: now[0]) == 1
- assert sorted(calls) == ["skill-001", "skill-002"]
- summary = json.loads((reports / "summary.json").read_text())
- assert (summary["passed"], summary["incomplete"]) == (1, 4)
- rows = summary["skills"]
- assert "time limit" in rows[0]["error"] and rows[1]["passed"] is True
- assert all("Not evaluated" in row["error"] for row in rows[2:])
- output = capsys.readouterr().out
- assert 'evaluating skill 2 of 5: "skills/b"' in output and "evaluating skill 3" not in output
- assert "Scan time limit reached; this skill did not finish." in output
- assert "3 skills were not started" in output
-
-
-def test_checkpoint_saves_a_later_skill_before_an_earlier_one_finishes(tmp_path):
- workspace = tmp_path / "workspace"
- skill(workspace, "skills/a")
- skill(workspace, "skills/b")
- reports = tmp_path / "reports"
- responses = {"skill-001": (0, payload()), "skill-002": (0, payload())}
- snapshots = []
-
- def before(name):
- if name != "skill-001":
- return
- deadline = time.monotonic() + WAIT
- while time.monotonic() < deadline:
- rows = json.loads((reports / "summary.json").read_text())["skills"]
- if rows[1]["passed"]:
- snapshots.append(rows)
- return
- time.sleep(0.01)
-
- assert (
- runner.run(workspace, reports, {"SCAN_WORKERS": "2"}, execute=parallel_cli(responses, [], before=before)) == 0
- )
- assert len(snapshots) == 1 and snapshots[0][0]["passed"] is False
-
-
[email protected]("value", ["0", "17", "four", "-1", "2.5", "٣"])
-def test_invalid_scan_worker_count_fails_before_scanning(tmp_path, value):
[email protected]("score", ["0", "75", "100"])
+def test_quality_threshold_is_forwarded_to_evaluator(tmp_path, score):
workspace = tmp_path / "workspace"
skill(workspace, "skills/a")
- reports, calls = tmp_path / "reports", []
- assert runner.run(workspace, reports, {"SCAN_WORKERS": value}, execute=fake_cli([], calls)) == 1
- assert not calls
- assert "SCAN_WORKERS" in json.loads((reports / "summary.json").read_text())["error"]
-
-
-posix_only = pytest.mark.skipif(os.name != "posix", reason="uses a shell script as the evaluator")
-
-
-def scan_shim(tmp_path, *, fast=(), ignore_term=False):
- """Put a fake ``skillevaluator`` on a PATH and return that PATH.
-
- A scan records its PID and its background tool's PID in ``scan.pids``,
- then runs for five minutes. Skills named in ``fast`` pass at once instead.
- """
- tool = tmp_path / "bin"
- tool.mkdir()
- report = tmp_path / "passing-report.json"
- report.write_text(json.dumps(payload()), encoding="utf-8")
- cases = "".join(f'*/{name}) cp "{report}" "$out/skillevaluator-tier1.json"; exit 0 ;;\n' for name in fast)
- script = tool / "skillevaluator"
- script.write_text(
- "#!/bin/sh\n"
- + ("trap '' TERM\n" if ignore_term else "")
- + "for out; do :; done\n"
- + f'case "$2" in\n{cases}*) ;;\nesac\n'
- + "sleep 300 &\n"
- + 'echo "$$ $!" > "$out/scan.tmp" && mv "$out/scan.tmp" "$out/scan.pids"\n'
- + "exec sleep 300\n",
- encoding="utf-8",
- )
- script.chmod(0o755)
- return f"{tool}{os.pathsep}/usr/bin{os.pathsep}/bin"
-
-
-def scan_pids(output, deadline):
- pids = output / "scan.pids"
- while not pids.exists():
- assert time.monotonic() < deadline, "the scan did not start"
- time.sleep(0.02)
- return [int(pid) for pid in pids.read_text().split()]
-
-
-def private_scan_pids(temporary, name, deadline):
- """Read the PIDs of the scan for report directory ``name`` from its private directory under TMPDIR."""
- while not (found := sorted(temporary.rglob(f"skillevaluator-private-{name}-*/scan.pids"))):
- assert time.monotonic() < deadline, "the scan did not start"
- time.sleep(0.02)
- return scan_pids(found[0].parent, deadline)
-
-
-def still_running(pids):
- """Wait up to WAIT for the processes to end; return those still running."""
- deadline = time.monotonic() + WAIT
- running = [pid for pid in pids if not ended(pid)]
- while running and time.monotonic() < deadline:
- time.sleep(0.05)
- running = [pid for pid in running if not ended(pid)]
- return running
-
-
-def ended(pid):
- try:
- os.kill(pid, 0)
- except ProcessLookupError:
- return True
- return False
-
-
-def kill_quietly(pids):
- # Only for processes not seen to end: a PID that ended may be reused.
- for pid in pids:
- with contextlib.suppress(OSError):
- os.kill(pid, signal.SIGKILL)
-
-
-@posix_only
-def test_scan_timeout_also_ends_the_scanner_tools(tmp_path, monkeypatch):
- path = scan_shim(tmp_path)
- output = tmp_path / "out"
- output.mkdir()
- clock = types.SimpleNamespace(offset=0.0)
- clock.monotonic = lambda: time.monotonic() + clock.offset
- monkeypatch.setattr(runner, "time", clock)
- outcome = {}
-
- def scan():
- try:
- runner.run_scan(
- ["skillevaluator", "tier1", "skills/a", "-o", str(output)],
- abort=threading.Event(),
- timeout=WAIT,
- env={"PATH": path},
- stdout=subprocess.DEVNULL,
- stderr=subprocess.DEVNULL,
- )
- except subprocess.TimeoutExpired as exc:
- outcome["error"] = exc
-
- thread = threading.Thread(target=scan, daemon=True)
- thread.start()
- pids = survivors = scan_pids(output, time.monotonic() + WAIT)
- try:
- clock.offset = WAIT
- thread.join(WAIT)
- assert not thread.is_alive() and "error" in outcome
- survivors = still_running(pids)
- assert not survivors
- finally:
- kill_quietly(survivors)
-
-
-@posix_only
-def test_aborted_scan_is_killed_after_the_grace_period(tmp_path, monkeypatch):
- monkeypatch.setattr(runner, "STOP_GRACE_SECONDS", 0.5)
- path = scan_shim(tmp_path, ignore_term=True)
- output = tmp_path / "out"
- output.mkdir()
- abort, outcome = threading.Event(), {}
-
- def scan():
- outcome["process"] = runner.run_scan(
- ["skillevaluator", "tier1", "skills/a", "-o", str(output)],
- abort=abort,
- timeout=WAIT,
- env={"PATH": path},
- stdout=subprocess.DEVNULL,
- stderr=subprocess.DEVNULL,
- )
-
- thread = threading.Thread(target=scan, daemon=True)
- thread.start()
- pids = survivors = scan_pids(output, time.monotonic() + WAIT)
- try:
- abort.set()
- thread.join(WAIT)
- assert not thread.is_alive()
- # The scan and its tool ignore SIGTERM, so only SIGKILL ends them.
- assert outcome["process"].returncode == -signal.SIGKILL
- survivors = still_running(pids)
- assert not survivors
- finally:
- kill_quietly(survivors)
-
-
-@posix_only
-def test_checkpoint_error_ends_running_scans_at_once(tmp_path, monkeypatch):
- workspace = tmp_path / "workspace"
- skill(workspace, "skills/fast")
- skill(workspace, "skills/slow")
- audit = tmp_path / "audit-tmp"
- audit.mkdir()
- reports = tmp_path / "reports"
- environment = {"PATH": scan_shim(tmp_path, fast=["fast"]), "TMPDIR": str(audit)}
- render, pids, outcome = runner.render_reports, [], {}
- survivors = pids
-
- def render_reports(*args):
- if (reports / "skill-001" / "skillevaluator-tier1.json").exists():
- # The slow scan is running when saving the fast result fails.
- pids.extend(private_scan_pids(audit, "skill-002", time.monotonic() + WAIT))
- raise OSError(errno.ENOSPC, "No space left on device")
- render(*args)
-
- monkeypatch.setattr(runner, "render_reports", render_reports)
-
- def scan():
- try:
- runner.run(workspace, reports, environment)
- except BaseException as exc:
- outcome["error"] = exc
-
- thread = threading.Thread(target=scan, daemon=True)
- thread.start()
- try:
- thread.join(WAIT)
- assert not thread.is_alive(), "the run waited for a running scan"
- assert isinstance(outcome["error"], OSError)
- survivors = still_running(pids)
- assert pids and not survivors
- finally:
- kill_quietly(survivors)
-
-
-def start_runner_with_endless_scans(tmp_path, *, ignore_term=False):
- """Start the runner alone in its session, as Docker runs it, and wait for two endless scans."""
- workspace = tmp_path / "workspace"
- for name in ("a", "b"):
- skill(workspace, f"skills/{name}")
- audit = tmp_path / "audit-tmp"
- audit.mkdir()
- reports = tmp_path / "reports"
- environment = {"PATH": scan_shim(tmp_path, ignore_term=ignore_term), "TMPDIR": str(audit)}
- environment.update(
- (name, os.environ[name]) for name in ("PYTHONPATH", "PYTHONDONTWRITEBYTECODE") if name in os.environ
- )
- with (tmp_path / "runner.log").open("wb") as log:
- process = subprocess.Popen(
- [sys.executable, str(MODULE_PATH), "--workspace", str(workspace), "--reports", str(reports)],
- env=environment,
- stdout=log,
- stderr=subprocess.STDOUT,
- start_new_session=True,
- )
- pids = []
- try:
- deadline = time.monotonic() + WAIT
- for name in ("skill-001", "skill-002"):
- pids += private_scan_pids(audit, name, deadline)
- except BaseException:
- process.kill()
- process.wait()
- kill_quietly(pids)
- raise
- return process, pids
-
-
-@posix_only
-def test_ctrl_c_ends_running_scans_and_their_tools(tmp_path):
- process, pids = start_runner_with_endless_scans(tmp_path)
- survivors = pids
- try:
- # Docker sends Ctrl-C to the runner (PID 1) only, not to its scans.
- os.kill(process.pid, signal.SIGINT)
- assert process.wait(WAIT) == -signal.SIGINT
- survivors = still_running(pids)
- assert not survivors
- finally:
- if process.poll() is None:
- process.kill()
- process.wait()
- kill_quietly(survivors)
-
-
-@posix_only
-def test_second_ctrl_c_does_not_leave_the_runner_waiting(tmp_path):
- # The scans ignore SIGTERM, so the second Ctrl-C arrives while the
- # runner is still waiting for them to end.
- process, pids = start_runner_with_endless_scans(tmp_path, ignore_term=True)
- survivors = pids
- try:
- os.kill(process.pid, signal.SIGINT)
- time.sleep(1)
- os.kill(process.pid, signal.SIGINT)
- assert process.wait(WAIT) == -signal.SIGINT
- survivors = still_running(pids)
- assert not survivors
- finally:
- if process.poll() is None:
- process.kill()
- process.wait()
- kill_quietly(survivors)
-
-
-def test_each_report_is_released_once_its_result_is_logged(tmp_path, monkeypatch):
- # Reports can be 32 MiB each; 100 of them must not stay in memory.
- workspace = tmp_path / "workspace"
- for index in range(5):
- skill(workspace, f"skills/s{index}")
- responses = {f"skill-{index:03d}": (0, payload()) for index in range(1, 6)}
- load, reports, alive = runner._load_report, [], []
-
- class Report(dict):
- pass
-
- def tracked(path):
- report = Report(load(path))
- reports.append(weakref.ref(report))
- return report
-
- def before(name):
- gc.collect()
- alive.append(sum(report() is not None for report in reports))
-
- monkeypatch.setattr(runner, "_load_report", tracked)
- environment = {"SCAN_WORKERS": "1"}
- assert (
- runner.run(workspace, tmp_path / "reports", environment, execute=parallel_cli(responses, [], before=before))
- == 0
- )
- # Only the latest result may still be referenced when the next scan starts.
- assert alive == [0, 1, 1, 1, 1]
-
-
-def worker_directory(env, name):
- return Path(env[name]) / ("worker-1" if name == "HOME" else "skillevaluator-worker-1")
-
-
[email protected]("name", ["HOME", "TMPDIR"])
-def test_symlinked_worker_directory_is_rejected(tmp_path, name):
- env = {"HOME": str(tmp_path / "home"), "TMPDIR": str(tmp_path / "tmp")}
- directory = worker_directory(env, name)
- directory.parent.mkdir()
- (tmp_path / "elsewhere").mkdir(mode=0o700)
- directory.symlink_to(tmp_path / "elsewhere", target_is_directory=True)
- with pytest.raises(runner.ConfigurationError):
- runner.worker_environment(env, 1)
-
-
-@posix_only
[email protected]("name", ["HOME", "TMPDIR"])
-def test_existing_worker_directory_open_to_others_is_rejected(tmp_path, name):
- env = {"HOME": str(tmp_path / "home"), "TMPDIR": str(tmp_path / "tmp")}
- directory = worker_directory(env, name)
- directory.mkdir(parents=True)
- directory.chmod(0o777)
- with pytest.raises(runner.ConfigurationError):
- runner.worker_environment(env, 1)
- assert stat.S_IMODE(directory.stat().st_mode) == 0o777
-
-
-@posix_only
-def test_worker_directory_owned_by_another_user_is_rejected(tmp_path, monkeypatch):
- env = {"HOME": str(tmp_path / "home"), "TMPDIR": str(tmp_path / "tmp")}
- uid = os.getuid()
- monkeypatch.setattr(runner.os, "getuid", lambda: uid + 1)
- with pytest.raises(runner.ConfigurationError):
- runner.worker_environment(env, 1)
-
-
-@posix_only
-def test_existing_private_worker_directories_are_reused(tmp_path):
- env = {"HOME": str(tmp_path / "home"), "TMPDIR": str(tmp_path / "tmp")}
- for name in ("HOME", "TMPDIR"):
- worker_directory(env, name).mkdir(mode=0o700, parents=True)
- (worker_directory(env, name) / "kept").write_text("", encoding="utf-8")
- isolated = runner.worker_environment(env, 1)
- for name in ("HOME", "TMPDIR"):
- assert isolated[name] == str(worker_directory(env, name))
- assert (worker_directory(env, name) / "kept").is_file()
+ calls = []
+ assert runner.run(
+ workspace,
+ tmp_path / "reports",
+ {"SKILL_PATHS": "skills", "MIN_SCORE": score},
+ execute=fake_cli([(0, payload())], calls),
+ ) == 0
+ argv = calls[0][0]
+ assert argv[argv.index("--min-score") + 1] == score
-def test_symlinked_worker_directory_fails_before_scanning(tmp_path):
[email protected]("score", ["-1", "101", "12.5", "abc", "\uff17\uff10", ""])
+def test_invalid_quality_threshold_fails_without_running_scanner(tmp_path, score):
workspace = tmp_path / "workspace"
skill(workspace, "skills/a")
- audit = tmp_path / "audit-tmp"
- audit.mkdir()
- (tmp_path / "elsewhere").mkdir(mode=0o700)
- (audit / "skillevaluator-worker-1").symlink_to(tmp_path / "elsewhere", target_is_directory=True)
- reports, calls = tmp_path / "reports", []
- assert runner.run(workspace, reports, {"TMPDIR": str(audit)}, execute=fake_cli([], calls)) == 1
+ calls = []
+ assert runner.run(
+ workspace,
+ tmp_path / "reports",
+ {"SKILL_PATHS": "skills", "MIN_SCORE": score},
+ execute=fake_cli([(0, payload())], calls),
+ ) == 1
assert not calls
- summary = json.loads((reports / "summary.json").read_text())
+ summary = json.loads((tmp_path / "reports/summary.json").read_text())
assert summary["overall_status"] == "error"
- assert summary["error"] == "Scan worker directories must be private directories owned by this user."
-
-
[email protected]("workers", ["1", "4"])
-def test_unexpected_worker_error_is_a_row_error_and_later_skills_still_run(tmp_path, capsys, workers):
- workspace = tmp_path / "workspace"
- for name in ("a", "b", "c"):
- skill(workspace, f"skills/{name}")
- reports, calls = tmp_path / "reports", []
- passing = parallel_cli({"skill-002": (0, payload()), "skill-003": (0, payload())}, calls)
-
- def execute(argv, **kwargs):
- if report_dir(Path(argv[argv.index("-o") + 1])) == "skill-001":
- raise RuntimeError("untrusted worker text")
- return passing(argv, **kwargs)
-
- assert runner.run(workspace, reports, {"SCAN_WORKERS": workers}, execute=execute) == 1
- assert sorted(name for name, _ in calls) == ["skill-002", "skill-003"]
- rows = json.loads((reports / "summary.json").read_text())["skills"]
- assert rows[0]["error"] == "Evaluator execution failed unexpectedly; no result is claimed."
- assert [row["passed"] for row in rows] == [False, True, True]
- captured = capsys.readouterr()
- for name in ("a", "b", "c"):
- assert f'Skill result: "skills/{name}"' in captured.out
- assert "untrusted worker text" not in captured.out + captured.err
- assert all("untrusted worker text" not in text for text in scan_outputs(reports).values())
diff --git a/tests/ci/test_secret_redaction.py b/tests/ci/test_secret_redaction.py
index 4453f20f..30dd7adb 100644
--- a/tests/ci/test_secret_redaction.py
+++ b/tests/ci/test_secret_redaction.py
@@ -130,12 +130,17 @@ def test_runner_and_retained_publisher_never_export_detected_value(tmp_path, cap
spec.loader.exec_module(fixtures)
class ArtifactAPI(fixtures.FakeAPI):
- def download_artifact(self, _artifact_id):
+ def download_artifact(self, artifact_id):
+ if artifact_id == 457:
+ return super().download_artifact(artifact_id)
return old_archive.getvalue()
monkeypatch.setenv("TRUSTED_EVALUATOR_SHA", "c" * 40)
monkeypatch.setenv("TRUSTED_CALLER_BLOBS", "b" * 40)
monkeypatch.syspath_prepend(str(ROOT / "ci"))
+ import verify_provenance
+
+ monkeypatch.setattr(verify_provenance, "fetch_signing_keys", lambda: fixtures.JWKS)
site, feedback = tmp_path / "site", tmp_path / "feedback"
fixtures.publisher.publish(
ArtifactAPI(record={**fixtures.run_record(), "conclusion": "failure"}),
@@ -385,12 +390,17 @@ def _publish(tmp_path: Path, monkeypatch, reports: Path) -> tuple[Path, Path]:
zipped.write(path, path.relative_to(reports).as_posix())
class ArtifactAPI(fixtures.FakeAPI):
- def download_artifact(self, _artifact_id):
+ def download_artifact(self, artifact_id):
+ if artifact_id == 457:
+ return super().download_artifact(artifact_id)
return archive.getvalue()
monkeypatch.setenv("TRUSTED_EVALUATOR_SHA", "c" * 40)
monkeypatch.setenv("TRUSTED_CALLER_BLOBS", "b" * 40)
monkeypatch.syspath_prepend(str(ROOT / "ci"))
+ import verify_provenance
+
+ monkeypatch.setattr(verify_provenance, "fetch_signing_keys", lambda: fixtures.JWKS)
site, feedback = tmp_path / "site", tmp_path / "feedback"
fixtures.publisher.publish(
ArtifactAPI(record={**fixtures.run_record(), "conclusion": "failure"}),
diff --git a/tests/ci/test_verify_provenance.py b/tests/ci/test_verify_provenance.py
new file mode 100644
index 00000000..3c6d6e6c
--- /dev/null
+++ b/tests/ci/test_verify_provenance.py
@@ -0,0 +1,194 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+"""Signed execution receipts reject tampering and cross-run replay."""
+
+import io
+import json
+import stat
+import zipfile
+from datetime import datetime
+from pathlib import Path
+
+import jwt
+import pytest
+from cryptography.hazmat.primitives.asymmetric import rsa
+
+KEY = rsa.generate_private_key(public_exponent=65537, key_size=2048)
+JWK = {
+ **json.loads(jwt.algorithms.RSAAlgorithm.to_jwk(KEY.public_key())),
+ "kid": "test-key",
+ "alg": "RS256",
+ "use": "sig",
+}
+JWKS = {"keys": [JWK]}
+
+
+def zip_receipt(token, name="execution.jwt"):
+ output = io.BytesIO()
+ with zipfile.ZipFile(output, "w") as archive:
+ archive.writestr(name, token)
+ return output.getvalue()
+
+
+def signed_receipt(run, **changes):
+ started = int(datetime.fromisoformat(run["run_started_at"]).timestamp())
+ sha = "d" * 40 if run["event"] == "pull_request" else run["head_sha"]
+ ref = "refs/pull/7/merge" if run["event"] == "pull_request" else "refs/heads/main"
+ claims = {
+ "iss": "https://token.actions.githubusercontent.com",
+ "aud": "urn:skillevaluator:report-provenance:v1",
+ "iat": started + 30,
+ "nbf": started - 570,
+ "exp": started + 330,
+ "repository": "owner/consumer",
+ "repository_id": "1234",
+ "run_id": str(run["id"]),
+ "run_attempt": str(run["run_attempt"]),
+ "event_name": run["event"],
+ "workflow_ref": f"owner/consumer/.github/workflows/skill-checks.yml@{ref}",
+ "workflow_sha": sha,
+ "sha": sha,
+ "ref": ref,
+ "job_workflow_ref": run["referenced_workflows"][0]["path"],
+ "job_workflow_sha": run["referenced_workflows"][0]["sha"],
+ **changes,
+ }
+ return zip_receipt(jwt.encode(claims, KEY, algorithm="RS256", headers={"kid": "test-key"}))
+
+
+def verifier(monkeypatch):
+ monkeypatch.syspath_prepend(str(Path(__file__).parents[2] / "ci"))
+ import verify_provenance
+
+ monkeypatch.setattr(verify_provenance, "fetch_signing_keys", lambda: JWKS)
+ return verify_provenance
+
+
+def record():
+ return {
+ "id": 123,
+ "repository": {"full_name": "owner/consumer", "id": 1234},
+ "run_attempt": 1,
+ "event": "pull_request",
+ "head_sha": "a" * 40,
+ "run_started_at": "2026-10-21T12:00:00Z",
+ "updated_at": "2026-10-21T12:02:00Z",
+ "referenced_workflows": [
+ {"path": "NVIDIA/SkillEvaluator/.github/workflows/evaluate-skills.yml@" + "c" * 40, "sha": "c" * 40}
+ ],
+ }
+
+
+def verify(verifier, record, payload):
+ return verifier.verify_receipt(
+ payload,
+ repository="owner/consumer",
+ run=record,
+ evaluator_workflow="NVIDIA/SkillEvaluator/.github/workflows/evaluate-skills.yml",
+ evaluator_sha="c" * 40,
+ )
+
+
+def test_historical_expired_receipt_attests_the_executed_merge(verifier, record):
+ identity = verify(verifier, record, signed_receipt(record))
+ assert identity["workflow_sha"] == "d" * 40
+ assert identity["workflow_sha"] != record["head_sha"]
+ assert "token" not in identity
+
+
+ "changes",
+ [
+ {"aud": "other-purpose"},
+ {"aud": ["urn:skillevaluator:report-provenance:v1"]},
+ {"iss": "https://attacker.invalid"},
+ {"repository": "attacker/consumer"},
+ {"repository_id": "9999"},
+ {"repository_id": 1234},
+ {"run_id": "124"},
+ {"run_attempt": "2"},
+ {"event_name": "push"},
+ {"job_workflow_sha": "e" * 40},
+ {"job_workflow_ref": "NVIDIA/SkillEvaluator/.github/workflows/evaluate-skills.yml@main"},
+ {"workflow_ref": "owner/consumer/.github/workflows/other.yml@refs/pull/7/merge"},
+ {"workflow_sha": "main"},
+ {"sha": "e" * 40},
+ {"ref": "refs/heads/main", "workflow_ref": "owner/consumer/.github/workflows/skill-checks.yml@refs/heads/main"},
+ {"iat": True},
+ {"iat": 1},
+ {"iat": 9999999999},
+ {"exp": 9999999999},
+ {"nbf": 1},
+ ],
+)
+def test_signed_wrong_identity_or_time_is_rejected(verifier, record, changes):
+ with pytest.raises(verifier.ProvenanceError):
+ verify(verifier, record, signed_receipt(record, **changes))
+
+
+def test_tampered_signature_and_unsigned_token_are_rejected(verifier, record):
+ token = verifier.read_receipt(signed_receipt(record))
+ parts = token.split(".")
+ signature = parts[2]
+ parts[2] = ("A" if signature[0] != "A" else "B") + signature[1:]
+ with pytest.raises(verifier.ProvenanceError):
+ verify(verifier, record, zip_receipt(".".join(parts)))
+ unsigned = jwt.encode({"aud": verifier.AUDIENCE}, "", algorithm="none")
+ with pytest.raises(verifier.ProvenanceError):
+ verify(verifier, record, zip_receipt(unsigned))
+
+
+def test_unknown_key_and_wrong_key_type_are_rejected(verifier, record, monkeypatch):
+ for keys in ({"keys": []}, {"keys": [{**JWK, "kty": "oct"}]}, {"keys": [JWK, JWK]}):
+ monkeypatch.setattr(verifier, "fetch_signing_keys", lambda keys=keys: keys)
+ with pytest.raises(verifier.ProvenanceError):
+ verify(verifier, record, signed_receipt(record))
+
+
[email protected]("name", ["../execution.jwt", "/execution.jwt", "dir/execution.jwt", "execution.jwt/"])
+def test_receipt_member_names_are_bounded(verifier, name):
+ with pytest.raises(verifier.ProvenanceError):
+ verifier.read_receipt(zip_receipt("a.b.c", name))
+
+
+def test_receipt_rejects_extra_members_symlinks_and_oversize(verifier):
+ for kind in ("extra", "symlink", "large"):
+ output = io.BytesIO()
+ with zipfile.ZipFile(output, "w", zipfile.ZIP_DEFLATED) as archive:
+ entry = zipfile.ZipInfo("execution.jwt")
+ if kind == "symlink":
+ entry.create_system = 3
+ entry.external_attr = (stat.S_IFLNK | 0o777) << 16
+ archive.writestr(entry, b"a" * (verifier.MAX_TOKEN_BYTES + 1) if kind == "large" else b"a.b.c")
+ if kind == "extra":
+ archive.writestr("another", b"a.b.c")
+ with pytest.raises(verifier.ProvenanceError):
+ verifier.read_receipt(output.getvalue())
+ with pytest.raises(verifier.ProvenanceError):
+ verifier.read_receipt(b"a" * (verifier.MAX_RECEIPT_BYTES + 1))
+
+
+def test_jwks_uses_only_fixed_host_without_credentials(monkeypatch):
+ monkeypatch.syspath_prepend(str(Path(__file__).parents[2] / "ci"))
+ import verify_provenance
+
+ class Response(io.BytesIO):
+ @property
+ def headers(self):
+ return {}
+
+ class Opener:
+ def open(self, request, timeout):
+ assert request.full_url == "https://token.actions.githubusercontent.com/.well-known/jwks"
+ assert request.get_header("Authorization") is None
+ assert timeout == 30
+ return Response(json.dumps(JWKS).encode())
+
+ def opener(handler):
+ assert handler.redirect_request(None, None, None, None, None, "https://elsewhere.invalid") is None
+ return Opener()
+
+ monkeypatch.setattr(verify_provenance.urllib.request, "build_opener", opener)
+ assert verify_provenance.fetch_signing_keys() == JWKS
diff --git a/tests/test_ci_workflows.py b/tests/test_ci_workflows.py
index 4d71307f..e29c7858 100644
--- a/tests/test_ci_workflows.py
+++ b/tests/test_ci_workflows.py
@@ -370,19 +370,28 @@ def test_publish_docs_installs_the_fern_cli_from_the_committed_lockfile() -> Non
assert "fern/node_modules/.bin/fern" in publish_step["run"], "run the installed CLI, not one from PATH"
-def test_every_workflow_declares_explicit_permissions() -> None:
+def test_workflows_declare_permissions_or_inherit_the_explicit_scan_caller() -> None:
"""A job-level permissions: block overrides the workflow-level one, so check both.
- This rejects the write-all shorthand and the inherited-default token scope, not
- every broad grant -- a granular write like `contents: write` is a legitimate
- choice for a release workflow and is not this test's concern.
+ The callable-only scanner inherits its explicit consumer permissions so
+ hosting can opt into OIDC without requiring that permission for basic scans.
+ All event-triggered workflows still declare their scope; no job uses write-all.
"""
for workflow_name in _workflow_names():
workflow = _load(workflow_name)
- assert workflow.get("permissions") is not None, (
- f"{workflow_name} inherits the repository default token scope"
- )
- for scope, permissions in [("workflow", workflow["permissions"])] + [
+ if workflow_name == "evaluate-skills.yml":
+ assert set(workflow["on"]) == {"workflow_call"}
+ assert "permissions" not in workflow
+ assert all("permissions" not in job for job in workflow["jobs"].values())
+ for template in ("skill-checks.yml", "skill-checks-llm.yml"):
+ caller = yaml.load((ROOT / "ci/templates" / template).read_text(), Loader=yaml.BaseLoader)
+ assert caller["permissions"] == {"contents": "read"}
+ assert all("permissions" not in job for job in caller["jobs"].values())
+ else:
+ assert workflow.get("permissions") is not None, (
+ f"{workflow_name} inherits the repository default token scope"
+ )
+ for scope, permissions in [("workflow", workflow.get("permissions"))] + [
(f"job {job_id}", job["permissions"]) for job_id, job in workflow["jobs"].items() if "permissions" in job
]:
assert permissions != "write-all", f"{workflow_name}: {scope}"
diff --git a/uv.lock b/uv.lock
index 932c9890..1303225b 100644
--- a/uv.lock
+++ b/uv.lock
@@ -438,10 +438,10 @@ resolution-markers = [
"sys_platform == 'linux'",
]
dependencies = [
- { name = "license-expression", marker = "sys_platform == 'linux'" },
- { name = "packageurl-python", marker = "sys_platform == 'linux'" },
- { name = "py-serializable", marker = "sys_platform == 'linux'" },
- { name = "sortedcontainers", marker = "sys_platform == 'linux'" },
+ { name = "license-expression" },
+ { name = "packageurl-python" },
+ { name = "py-serializable" },
+ { name = "sortedcontainers" },
]
sdist = { url = "https://files.pythonhosted.org/packages/66/fc/abaad5482f7b59c9a0a9d8f354ce4ce23346d582a0d85730b559562bbeb4/cyclonedx_python_lib-9.1.0.tar.gz", hash = "sha256:86935f2c88a7b47a529b93c724dbd3e903bc573f6f8bd977628a7ca1b5dadea1", size = 1048735, upload-time = "2025-02-27T17:23:40.367Z" }
wheels = [
@@ -458,11 +458,11 @@ resolution-markers = [
"sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32'",
]
dependencies = [
- { name = "license-expression", marker = "sys_platform != 'linux'" },
- { name = "packageurl-python", marker = "sys_platform != 'linux'" },
- { name = "py-serializable", marker = "sys_platform != 'linux'" },
- { name = "sortedcontainers", marker = "sys_platform != 'linux'" },
- { name = "typing-extensions", marker = "python_full_version < '3.13' and sys_platform != 'linux'" },
+ { name = "license-expression" },
+ { name = "packageurl-python" },
+ { name = "py-serializable" },
+ { name = "sortedcontainers" },
+ { name = "typing-extensions", marker = "python_full_version < '3.13'" },
]
sdist = { url = "https://files.pythonhosted.org/packages/75/c9/5d0ccdd19bc7d8ab803b90695c1706aa2ea8529685d18e682dc2524d2630/cyclonedx_python_lib-11.11.0.tar.gz", hash = "sha256:4b3194db72b613717f2912447e67ab618c75ff7dcac6c4af3c0e9e1ac617c102", size = 1442983, upload-time = "2026-06-17T11:57:49.055Z" }
wheels = [
@@ -2120,6 +2120,7 @@ dashboard = [
dev = [
{ name = "build" },
{ name = "httpx" },
+ { name = "pyjwt", extra = ["crypto"] },
{ name = "pytest" },
{ name = "pytest-cov" },
{ name = "pytest-xdist" },
@@ -2131,6 +2132,9 @@ llm = [
{ name = "litellm" },
{ name = "openai" },
]
+publisher = [
+ { name = "pyjwt", extra = ["crypto"] },
+]
security = [
{ name = "bandit" },
{ name = "pip-audit" },
@@ -2172,6 +2176,7 @@ requires-dist = [
{ name = "packaging", specifier = ">=24" },
{ name = "pip-audit", marker = "extra == 'security'", specifier = ">=2.10.0" },
{ name = "pydantic", specifier = ">=2.11.0" },
+ { name = "pyjwt", extras = ["crypto"], marker = "extra == 'publisher'", specifier = ">=2.13.0" },
{ name = "pyjwt", extras = ["crypto"], marker = "extra == 'tier3'", specifier = ">=2.13.0" },
{ name = "pytest", marker = "extra == 'dev'", specifier = ">=8.4.0" },
{ name = "pytest-cov", marker = "extra == 'dev'", specifier = ">=6.2.0" },
@@ -2182,10 +2187,11 @@ requires-dist = [
{ name = "ruff", marker = "extra == 'dev'", specifier = ">=0.15.4,<0.16" },
{ name = "skillevaluator", extras = ["llm"], marker = "extra == 'tier2'" },
{ name = "skillevaluator", extras = ["llm"], marker = "extra == 'tier3'" },
+ { name = "skillevaluator", extras = ["publisher"], marker = "extra == 'dev'" },
{ name = "skillevaluator", extras = ["tier2", "tier3", "security"], marker = "extra == 'all'" },
{ name = "streamlit", marker = "extra == 'dashboard'", specifier = ">=1.61,<2" },
]
-provides-extras = ["llm", "tier2", "tier3", "security", "dashboard", "all", "dev"]
+provides-extras = ["llm", "tier2", "tier3", "security", "publisher", "dashboard", "all", "dev"]
[[package]]
name = "sniffio"