Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.
Python agent-evaluationagent-pluginsagent-securityagent-skillsagentic-aibenchmarkci-templatesclaude-codecodexevaluateevaluationharnesspluginplugin-evaluationsecurity-scannerskill-evalskill-evaluationskill-evaluatorskill-gap-analysisskills
perf(ci): faster Tier 1 scans and a clearer HTML report (#256)
Narendran Raghavan1aee929eb31a
diff --git a/.dockerignore b/.dockerignoreindex 86411014..1ffb9c95 100644--- a/.dockerignore+++ b/.dockerignore@@ -14,3 +14,22 @@ log/*.egg-info/.pytest_cache/.ruff_cache/+# Neither image reads these. Leaving them out of the build context also stops+# test, documentation and tooling edits from invalidating cached image layers.+.claude/+.github/+.gitleaks.toml+config/+docs/+fern/+scripts/+tests/+CHANGELOG.md+CITATION.cff+CODE_OF_CONDUCT.md+CONTRIBUTING.md+GOVERNANCE.md+Makefile+SECURITY.md+SUPPORT.md+sonar-project.propertiesdiff --git a/.github/workflows/evaluate-skills.yml b/.github/workflows/evaluate-skills.ymlindex 5f9bf4c3..728d47f2 100644--- a/.github/workflows/evaluate-skills.yml+++ b/.github/workflows/evaluate-skills.yml@@ -34,6 +34,10 @@ on:description: Optional GitHub Pages report site for stable linkstype: stringdefault: ''+ scan-workers:+ description: Skills scanned at once, 1 to 16; 1 scans one skill at a time+ type: number+ default: 4secrets:LLM_API_KEY:required: false@@ -44,11 +48,16 @@ jobs:name: Tier 1 scanruns-on: ubuntu-24.04timeout-minutes: 60+ env:+ SCAN_MODE: ${{ inputs.scan-mode == 'auto' && (github.event_name == 'pull_request' && 'changed' || 'all') || inputs.scan-mode }}steps:- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0with:persist-credentials: false- fetch-depth: 0+ # Changed mode needs commits and trees for the merge base, not old+ # file contents. Full scans need only the checked-out commit.+ fetch-depth: ${{ env.SCAN_MODE != 'changed' && 1 || 0 }}+ filter: ${{ env.SCAN_MODE == 'changed' && 'blob:none' || '' }}ref: ${{ github.event.pull_request.head.sha || github.sha }}- name: Validate the immutable image referenceenv:@@ -68,7 +77,6 @@ jobs:- name: Evaluate selected skillsenv:IMAGE: ${{ inputs.image }}- SCAN_MODE: ${{ inputs.scan-mode == 'auto' && (github.event_name == 'pull_request' && 'changed' || 'all') || inputs.scan-mode }}DIFF_BASE_SHA: ${{ github.event.pull_request.base.sha }}DIFF_HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }}SKILL_PATHS: ${{ inputs.paths }}@@ -77,6 +85,7 @@ jobs:LLM_MODEL: ${{ inputs.llm-model }}LLM_API_KEY: ${{ inputs.llm && secrets.LLM_API_KEY || '' }}REPORT_BASE_URL: ${{ inputs.report-base-url }}+ SCAN_WORKERS: ${{ inputs.scan-workers }}REPORT_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}REPORT_COMMIT: ${{ github.event.pull_request.head.sha || github.sha }}CALLER_WORKFLOW_SHA: ${{ github.workflow_sha }}@@ -99,7 +108,7 @@ jobs:--mount "type=bind,src=$GITHUB_WORKSPACE,dst=/workspace,readonly" \--mount "type=bind,src=$RUNNER_TEMP/skill-reports,dst=/reports" \--env SKILL_PATHS --env LLM_ENABLED --env LLM_PROVIDER --env LLM_MODEL "${llm_secret_args[@]}" \- --env SCAN_MODE --env DIFF_BASE_SHA --env DIFF_HEAD_SHA \+ --env SCAN_MODE --env DIFF_BASE_SHA --env DIFF_HEAD_SHA --env SCAN_WORKERS \--env CALLER_WORKFLOW_SHA \--env GITHUB_ACTIONS --env REPORT_IMAGE --env REPORT_URL --env REPORT_RUN_URL --env REPORT_COMMIT \"$IMAGE"@@ -119,3 +128,14 @@ jobs:path: ${{ runner.temp }}/skill-reports/if-no-files-found: warnretention-days: 14+ # The optional report publisher downloads only this JSON evidence, not the full reports.+ - name: Upload publisher evidence even when validation fails+ if: always()+ uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2+ with:+ name: ${{ inputs.artifact-name }}-evidence-attempt-${{ github.run_attempt }}+ path: |+ ${{ runner.temp }}/skill-reports/summary.json+ ${{ runner.temp }}/skill-reports/skill-*/skillevaluator-tier1.json+ if-no-files-found: warn+ retention-days: 14diff --git a/.github/workflows/publish-skill-reports.yml b/.github/workflows/publish-skill-reports.ymlindex 1b937d58..05d9f26e 100644--- a/.github/workflows/publish-skill-reports.yml+++ b/.github/workflows/publish-skill-reports.yml@@ -50,6 +50,7 @@ jobs:pr_number: ${{ steps.publish.outputs.pr_number }}is_current: ${{ steps.publish.outputs.is_current }}skipped: ${{ steps.publish.outputs.skipped }}+ committed: ${{ steps.publish.outputs.committed }}steps:- name: Validate trusted tool identityenv:@@ -70,11 +71,19 @@ jobs:ref: ${{ inputs.tools-ref }}path: publisherpersist-credentials: false+ # Cone mode also keeps every root file, including the package build inputs.+ sparse-checkout: |+ ci+ src- name: Install the trusted native report rendererworking-directory: publisher+ # No dependency cache is restored: other default-branch workflows can write+ # this job's cache scope, and uv does not recheck files from its cache.run: |- python3 -m pip install --require-hashes -r ci/requirements/uv.txt- uv sync --frozen --no-dev --no-editable+ python3 -m venv "$RUNNER_TEMP/uv"+ "$RUNNER_TEMP/uv/bin/python" -m pip install --disable-pip-version-check \+ --require-hashes --no-deps --only-binary :all: -r ci/requirements/uv.txt+ "$RUNNER_TEMP/uv/bin/uv" sync --frozen --no-dev --no-editable- name: Checkout retained report historyuses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0with:@@ -100,6 +109,7 @@ jobs:git -C site config user.email '41898282+github-actions[bot]@users.noreply.github.com'# Rebuild against the newest history after a concurrent writer wins.# Every run is retained; a busy deployment cannot discard its report.+ committed=falsefor attempt in 1 2 3 4 5; dopublisher/.venv/bin/python publisher/ci/publish_reports.pygit -C site add .@@ -108,6 +118,7 @@ jobs:figit -C site commit -m "Record skill scan $REPORT_RUN_ID"if git -C site push origin HEAD:gh-pages; then+ committed=truebreakfiif [ "$attempt" -eq 5 ]; then@@ -117,6 +128,7 @@ jobs:git -C site reset --hard origin/gh-pagesgit -C site clean -fddone+ echo "committed=$committed" >> "$GITHUB_OUTPUT"cat "$OUTPUT_DIR/summary.md" >> "$GITHUB_STEP_SUMMARY"- name: Retain feedback for the comment jobif: steps.publish.outputs.skipped != 'true'@@ -128,7 +140,13 @@ jobs:retention-days: 14deploy:needs: prepare- if: needs.prepare.outputs.skipped != 'true'+ # A skipped (stale) publication never deploys. Otherwise skip only an automatic first+ # attempt that committed nothing new. Manual runs and re-runs always deploy, so a re-run+ # after a failed deploy still publishes.+ if: >-+ needs.prepare.outputs.skipped != 'true' &&+ (needs.prepare.outputs.committed == 'true' || github.event_name == 'workflow_dispatch' ||+ github.run_attempt != '1')runs-on: ubuntu-24.04timeout-minutes: 10concurrency:@@ -158,8 +176,10 @@ jobs:uses: actions/deploy-pages@368f82528645a54fb793d4d04e342629a3f51346 # v5.0.1comment:needs: [prepare, deploy]+ # Comment only after this run's own deploy. A skipped deploy means another+ # publisher run pushed this content; that run deploys it and then comments.if: >-- always() && needs.prepare.result == 'success' &&+ always() && needs.prepare.result == 'success' && needs.prepare.outputs.skipped != 'true' &&needs.prepare.outputs.pr_number != '' && needs.prepare.outputs.is_current == 'true' &&needs.deploy.result == 'success'runs-on: ubuntu-24.04@@ -180,6 +200,10 @@ jobs:ref: ${{ inputs.tools-ref }}path: publisherpersist-credentials: false+ sparse-checkout: |+ ci/comment_report.py+ ci/github_api.py+ sparse-checkout-cone-mode: false- name: Download this publication's feedbackenv:GH_TOKEN: ${{ github.token }}diff --git a/.github/workflows/publish-tier1-image.yml b/.github/workflows/publish-tier1-image.ymlindex 54ad889e..72b110db 100644--- a/.github/workflows/publish-tier1-image.yml+++ b/.github/workflows/publish-tier1-image.yml@@ -9,6 +9,17 @@ on:- uv.lock- .dockerignore- .github/workflows/publish-tier1-image.yml+ # Builds on main seed the layer cache that pull requests read.+ push:+ branches:+ - main+ paths:+ - ci/**+ - src/skillevaluator/**+ - pyproject.toml+ - uv.lock+ - .dockerignore+ - .github/workflows/publish-tier1-image.ymlworkflow_dispatch:inputs:publish:@@ -21,17 +32,32 @@ permissions:jobs:verify:+ # A release dispatch builds, verifies and pushes one image in the publish job instead.+ if: >-+ ${{ !(github.event_name == 'workflow_dispatch' && inputs.publish &&+ github.repository == 'NVIDIA/SkillEvaluator' && github.ref == 'refs/heads/main') }}runs-on: ubuntu-24.04timeout-minutes: 40steps:- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0with:persist-credentials: false+ - uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1- name: Build the complete Tier 1 runtime- run: |- docker build -f ci/Dockerfile.tier1 \- --label org.opencontainers.image.revision="$GITHUB_SHA" \- -t skillevaluator-tier1:verify .+ uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0+ with:+ context: .+ file: ci/Dockerfile.tier1+ labels: org.opencontainers.image.revision=${{ github.sha }}+ tags: skillevaluator-tier1:verify+ load: true+ provenance: false+ # Every verify run reads the layer cache that main writes. Only runs on main write it, so a pull+ # request cannot replace the layers that later main builds reuse.+ cache-from: type=gha,scope=tier1-image+ cache-to: >-+ ${{ github.event_name != 'pull_request' && github.ref == 'refs/heads/main' &&+ 'type=gha,scope=tier1-image,mode=max,ignore-error=true' || '' }}- name: Exercise real scans and changed-skill selectionrun: python3 ci/smoke_image.py --image skillevaluator-tier1:verify --output "$RUNNER_TEMP/image-verification"- name: Retain integration evidence@@ -44,7 +70,6 @@ jobs:retention-days: 14publish:- needs: verifyoutputs:image_digest: ${{ steps.release.outputs.image_digest }}concurrency:@@ -69,14 +94,26 @@ jobs:env:GHCR_TOKEN: ${{ github.token }}run: python3 ci/release_image.py --image "$IMAGE" --revision "$GITHUB_SHA" --output "$GITHUB_OUTPUT"+ - uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1+ if: steps.existing.outputs.exists != 'true'+ # Built once: the smoke test below runs this local image and the release step pushes it.+ # It reuses no cached layers, so apt installs the packages current at release time and no+ # earlier job's layers reach the release. It still writes the cache that later builds read.- name: Build the reviewed release revisionif: steps.existing.outputs.exists != 'true'- run: |- docker build -f ci/Dockerfile.tier1 \- --label org.opencontainers.image.source=https://github.com/NVIDIA/SkillEvaluator \- --label org.opencontainers.image.revision="$GITHUB_SHA" \- --label org.opencontainers.image.description='SkillEvaluator complete Tier 1 CI runtime' \- -t "$IMAGE:sha-$GITHUB_SHA" .+ uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0+ with:+ context: .+ file: ci/Dockerfile.tier1+ labels: |+ org.opencontainers.image.source=https://github.com/NVIDIA/SkillEvaluator+ org.opencontainers.image.revision=${{ github.sha }}+ org.opencontainers.image.description=SkillEvaluator complete Tier 1 CI runtime+ tags: ${{ env.IMAGE }}:sha-${{ github.sha }}+ load: true+ provenance: false+ no-cache: true+ cache-to: type=gha,scope=tier1-image,mode=max,ignore-error=true- name: Verify the exact image being publishedif: steps.existing.outputs.exists != 'true'run: python3 ci/smoke_image.py --image "$IMAGE:sha-$GITHUB_SHA" --output "$RUNNER_TEMP/release-verification"diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 580b1522..a095a2c7 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -38,7 +38,18 @@ All notable changes to SkillEvaluator are documented in this file.pinned runtime/workflow consumer templates, affected-skill PR selection,optional hosted reports and updated PR feedback. The new guide separatesbasic scanning from publisher setup; public image publication and generated- release files require a reviewed manual release before consumer use.+ release files require a reviewed manual release before consumer use. The+ Tier 1 image is a multi-stage build with precompiled bytecode that ships only+ the locked Semgrep, SkillSpector and SkillEvaluator environments, Gitleaks,+ `ci/` and the license files. A release builds it without cached layers and+ pushes the image it smoke-tested.+- The reusable Tier 1 workflow scans up to four skills at once. The+ `scan-workers` input (1 to 16) changes this, and `1` scans one skill at a+ time. Each worker gets its own private home and temporary directory. Logs,+ annotations and report rows keep skill order, and skills not started when the+ scan time budget runs out are still reported as not evaluated. A timeout,+ Ctrl-C or an error while saving results ends running scans together with+ the scanner tools they started.- `--quality-config PATH` on `validate`, `quality-check`, and `tier1` loads thedeterministic quality check's expected Markdown headings from a YAML fileinstead of the built-in list. Each `expected_headers` entry sets a `heading`@@ -54,6 +65,9 @@ All notable changes to SkillEvaluator are documented in this file.`quality_config` (`packaged` or `custom`, the custom file's name, a`sha256:` digest of the normalized rules, and the `--min-score` passthreshold, which is kept out of the digest).+- CI-native per-skill HTML pages link Download JSON to the retained+ `skillevaluator-tier1.json` and do not embed a second copy of the report+ JSON. Standalone CLI HTML reports embed it for Export JSON.### Changed@@ -62,6 +76,26 @@ All notable changes to SkillEvaluator are documented in this file.applicable are excluded from passed and executed counts. Publication statusremains visible on mobile and in print, and blocking/advisory findings retaintheir separate counts.+- The HTML report opens on the answer. The skill name is the page title, and+ the header shows one verdict, the date and version, and Download JSON. A+ summary gives the verdict with a one-line reason, four numbers with one+ definition of a finding, and a Fix first list of up to five places, most+ severe first and linked to their evidence. A scanner that did not finish is+ listed first with its own reason, and two scanners reporting one location+ take one entry. Scanner diagnostics always show and print.+ Failed checks are listed first, raw check IDs get plain names, and+ publication eligibility sits next to the verdict.+- The HTML report, the CI index and the Pages history use one verdict+ vocabulary: Passed, Changes required, Incomplete, Skipped and Neutral. When+ no check ran, the header and the summary both say Skipped. A failing tier+ whose gate does not block shows Failed (advisory). The CI index names the+ same first fix as each skill's report and links to it.+- The SkillEvaluator logo replaces the letter tile and the green square on+ every report page, the CI index and the Pages history. Colors use one+ palette built on the logo greens, with WCAG AA text contrast in light and+ dark themes. The report follows the system theme until the reader picks+ one, and prints dark text on white paper, Tier 3 charts included. No text is+ smaller than 11 px, and short table cells no longer break mid-word.- Dependency audits inspect pinned package versions without installing packagesor executing build scripts. Unpinned, URL and other unresolved requirementsreport incomplete coverage instead of executing package resolution.@@ -74,6 +108,22 @@ All notable changes to SkillEvaluator are documented in this file.with eligible free prototyping access. Installation adds extras as needed,and the guides separate quality results from full scanner and live-agentevidence.+- The optional GitHub report publisher downloads only a small+ `skill-evaluator-tier1-evidence-attempt-N` artifact holding `summary.json`+ and each skill's JSON. At 99 skills the download drops from 4.2 MB to+ 0.46 MB, and HTML, SARIF and logs no longer count toward its 64 MiB limit.+ A scan from an evaluator revision that does not upload this artifact cannot+ be published. Re-running it reuses that evaluator, so update the scan caller+ and the publisher allowlists, then start a new scan. When an automatic run+ commits nothing new, the publisher skips its Pages deploy and PR comment and+ leaves both to the run that pushed that content. The publisher also installs+ uv into a private environment instead of the runner's system Python, checks+ out only the tool files each job needs and replaces site files atomically.+ It and the PR comment helper share one GitHub client, which rejects sibling+ repository endpoints and NaN or Infinity in JSON for both.+- HTML reports rendered in one process share one compiled template+ environment, so each template compiles once per process instead of once per+ report.### Documentationdiff --git a/ci/Dockerfile.tier1 b/ci/Dockerfile.tier1index 56621178..84399087 100644--- a/ci/Dockerfile.tier1+++ b/ci/Dockerfile.tier1@@ -1,27 +1,31 @@# Complete Tier 1 runtime for reusable GitHub Actions scans.-FROM python:3.12-slim-bookworm@sha256:62ed8e07e544e8555c5d93748d5275015136b7f5c52b5e8679b4de339c3779d3+# The build stages install the locked environments. The runtime stage copies only what scans use:+# the three virtual environments, gitleaks, ci/ and the license files. It has no uv, uv cache or source tree.+FROM python:3.12-slim-bookworm@sha256:62ed8e07e544e8555c5d93748d5275015136b7f5c52b5e8679b4de339c3779d3 AS build+# The runtime image is read-only, so Python can never cache bytecode there: compile it at install time.+# Hash-based .pyc files do not embed source mtimes, so unchanged inputs give identical files. Installed+# sources never change inside the image, so the hashes do not need to be checked on import.ENV PIP_NO_INPUT=1 PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1 \- PYTHON_DOTENV_DISABLED=1 SEMGREP_SEND_METRICS=off SEMGREP_ENABLE_VERSION_CHECK=0 \- HOME=/tmp/evaluator-home+ UV_COMPILE_BYTECODE=1 PYC_INVALIDATION_MODE=UNCHECKED_HASH \+ UV_NO_CACHE=1 UV_PYTHON_DOWNLOADS=never-RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates curl git \+RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates curl \&& rm -rf /var/lib/apt/lists/*COPY ci/requirements/uv.txt /opt/scanner-requirements/uv.txtRUN python -m pip install --no-cache-dir --require-hashes -r /opt/scanner-requirements/uv.txt-WORKDIR /opt/skillevaluator-COPY . .-RUN uv sync --frozen --no-dev --no-editable --extra security --extra llm-+# Scanner layers come before any SkillEvaluator source, so a source edit does not rebuild them.# Independent environments avoid conflicting scanner dependency requirements.+COPY ci/requirements/semgrep.txt /opt/scanner-requirements/semgrep.txtRUN uv venv /opt/semgrep && uv pip install --python /opt/semgrep/bin/python \- --require-hashes -r ci/requirements/semgrep.txt+ --require-hashes -r /opt/scanner-requirements/semgrep.txt# Build tooling is locked too: do not resolve an isolated build environment.+COPY ci/requirements/skillspector-build.txt ci/requirements/skillspector.txt /opt/scanner-requirements/RUN uv venv /opt/skillspector && uv pip install --python /opt/skillspector/bin/python \- --require-hashes -r ci/requirements/skillspector-build.txt \+ --require-hashes -r /opt/scanner-requirements/skillspector-build.txt \&& uv pip install --python /opt/skillspector/bin/python --no-build-isolation \- --require-hashes -r ci/requirements/skillspector.txt+ --require-hashes -r /opt/scanner-requirements/skillspector.txtRUN set -eu; arch="$(dpkg --print-architecture)"; \case "$arch" in \amd64) asset=x64; checksum=79a3ab579b53f71efd634f3aaf7e04a0fa0cf206b7ed434638d1547a2470a66e ;; \@@ -35,14 +39,50 @@ RUN set -eu; arch="$(dpkg --print-architecture)"; \tar -xzf /tmp/gitleaks.tar.gz -C /usr/local/share/licenses/gitleaks LICENSE; \rm /tmp/gitleaks.tar.gz+# Locked SkillEvaluator dependencies. They need only the project metadata and the lock.+WORKDIR /opt/skillevaluator+COPY pyproject.toml uv.lock ./+RUN uv sync --frozen --no-dev --no-editable --no-install-project --extra security --extra llm++# The project itself goes into /overlay, laid out at its final paths, so a source edit changes one small+# layer. uv compiles only the files under --prefix, and the console script runs the venv's Python.+FROM build AS project+COPY README.md LICENSE NOTICE THIRD_PARTY_NOTICES.md ./+COPY src ./src+RUN uv pip install --python /opt/skillevaluator/.venv/bin/python --no-deps \+ --prefix /overlay/opt/skillevaluator/.venv .++FROM python:3.12-slim-bookworm@sha256:62ed8e07e544e8555c5d93748d5275015136b7f5c52b5e8679b4de339c3779d3++ENV PIP_NO_INPUT=1 PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1 \+ PYTHON_DOTENV_DISABLED=1 SEMGREP_SEND_METRICS=off SEMGREP_ENABLE_VERSION_CHECK=0 \+ HOME=/tmp/evaluator-home++# git is needed for changed-skill selection. curl and uv stay in the build stage.+RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates git \+ && rm -rf /var/lib/apt/lists/*+# The base image ships its standard library without .pyc files. Its own site-packages are unused.+RUN python -m compileall -q -j 0 --invalidation-mode unchecked-hash -x /site-packages/ /usr/local/lib/python3.12++COPY --from=build /opt/semgrep /opt/semgrep+COPY --from=build /opt/skillspector /opt/skillspector+COPY --from=build /usr/local/bin/gitleaks /usr/local/bin/gitleaks+COPY --from=build /usr/local/share/licenses/gitleaks /usr/local/share/licenses/gitleaks+COPY --from=build /opt/skillevaluator/.venv /opt/skillevaluator/.venvENV PATH="/opt/skillevaluator/.venv/bin:/opt/semgrep/bin:/opt/skillspector/bin:${PATH}"-RUN skillevaluator --version && bandit --version && pip-audit --version \- && semgrep --version && skillspector --version && gitleaks version \- && /opt/skillspector/bin/python -c 'import yara'+RUN bandit --version && pip-audit --version && semgrep --version && skillspector --version \+ && gitleaks version && /opt/skillspector/bin/python -c 'import yara'RUN useradd --uid 10001 --create-home --shell /usr/sbin/nologin evaluator \&& rm -rf /tmp/evaluator-home \&& mkdir -p /workspace /reports /tmp/evaluator-home \&& chown evaluator:evaluator /reports /tmp/evaluator-home++# Source-dependent layers come last.+COPY --from=project /overlay/ /+COPY LICENSE NOTICE THIRD_PARTY_NOTICES.md /opt/skillevaluator/+COPY ci /opt/skillevaluator/ci+RUN python -m compileall -q --invalidation-mode unchecked-hash /opt/skillevaluator/ci \+ && skillevaluator --versionUSER evaluatorWORKDIR /workspaceENTRYPOINT ["python", "/opt/skillevaluator/ci/run_tier1.py"]diff --git a/ci/README.md b/ci/README.mdindex a18924b8..40a5f659 100644--- a/ci/README.md+++ b/ci/README.md@@ -18,11 +18,12 @@ mutable image tag or an unreviewed branch reference.| Input | Basic configuration || --- | --- |-| `paths` | `skills`; newline-separated repository-relative skill directories or roots |+| `paths` | `skills`; up to 100 newline-separated repository-relative skill directories or roots || `scan-mode` | `auto`: affected skills on PRs, all configured skills on pushes/manual runs; `all` overrides selection || `llm` | `false`; static Tier 1 requires no model key || `artifact-name` | Keep `skill-evaluator-tier1` for the optional publisher || `report-base-url` | Optional own Pages URL, only after hosted publication is configured |+| `scan-workers` | `4`; skills scanned at once, from 1 to 16. `1` scans one skill at a time |The runtime discovers exact-case `SKILL.md` files, rejects unsafe paths andsymlinks, deduplicates overlapping roots and bounds each invocation to 100@@ -30,12 +31,19 @@ skills. This is a selection cap, not a runtime guarantee: scans have a boundedtime budget and retain completed results with explicit incomplete entries forunprocessed skills. Do not interpret a changed-skill selection as a newfull-catalog pass.+Up to `scan-workers` skills are scanned at once, each with its own private home+and temporary directory. Logs, annotations and report rows keep skill order. When+the time budget runs out, running scans stop and skills not yet started are+reported as not evaluated. Ctrl-C also stops running scans at once, including+the scanner tools they started.With no affected skills, the check succeeds with **No skills changed** andrecords that validation did not run. Fully removed skills are listed asdeleted/skipped. A removed manifest with tracked files left in its skillfolder is an error. Selection uses changed files inside discovered skills;shared files outside the configured roots are not inferred as dependencies.Use `scan-mode: all` when changes to those shared resources require evaluation.+Changed-skill selection checks out full history without old file contents (a+blobless clone); full scans fetch only the scanned commit.Static scanning is network connected: vulnerability databases may receivepackage metadata. Dependency auditing never installs packages or runs buildscripts: requirements must provide exact package/version pins. Unpinned, URL,@@ -49,7 +57,9 @@ incomplete scans and missing or contradictory report evidence fail the job.The short `summary.md` is the Actions landing view. `details.md`, originalper-skill JSON, native HTML, SARIF and scanner logs retain detailed evidence inthe scan artifact, named `skill-evaluator-tier1-attempt-N` for attempt `N`.-Artifacts expire after the configured retention period;+A second, small artifact, `skill-evaluator-tier1-evidence-attempt-N`, holds+only `summary.json` and each skill's `skillevaluator-tier1.json`. The optional+publisher downloads only that one. Artifacts expire after the configured retention period;hosted history is an optional separate surface. Quality scores are not scannerpass percentages or proof of useful live-agent behavior.@@ -93,7 +103,12 @@ Pages is not an inherent requirement for GitHub PR comments.**Skill checks**, and artifact name **skill-evaluator-tier1**. The dedicatedpublisher validates this caller and accepts exactly one approved referencedevaluator workflow. A differently named manual LLM workflow is not accepted- automatically.+ automatically. The publisher reads only the attempt's+ `skill-evaluator-tier1-evidence-attempt-N` artifact. A scan from an+ evaluator revision that does not upload it cannot be published. Re-running+ that scan reuses its commit and pinned evaluator. Instead, update the scan+ caller to a release that uploads the artifact, approve that evaluator SHA+ and the new caller blob (steps 5 and 6), and start a new scan.5. Set `trusted-evaluator-shas` to the reviewed evaluator workflow commit. Set`tools-ref` to the reviewed immutable commit containing the publisher helpers;generated release files supply both. `tools-repository` defaults to@@ -117,7 +132,9 @@ Pages is not an inherent requirement for GitHub PR comments.The publisher rebuilds native HTML from bounded, validated JSON using pinnedtrusted tools. It does not execute consumer PR code or trust uploaded HTML as-site code. Each completion event binds publication to its recorded run attempt;+site code. Its jobs use sparse checkouts of the tools repository: preparation+gets the root files, `ci/` and `src/`; the comment job gets only+`ci/comment_report.py` and `ci/github_api.py`, which use only the standard library. Each completion event binds publication to its recorded run attempt;manual backfills must specify that attempt. It retains the newest 100 runs bydefault (`retention-runs`), within a 512 MiB site budget, plus current`main/latest/` and open-PR reports. Older unprotected runs are pruned first;@@ -126,7 +143,14 @@ aliases are retired. Retained `runs/<run-id>/<attempt>/` paths are immutable; deruns cannot replace newer aliases. Current default-branch reports use the`main/latest/` route even if the actual default branch has a different name.A bot comment is created once and updated in place only for an open,-current-head, same-repository PR after deployment. Human comments are untouched.+current-head, same-repository PR after that publication's own deployment. Human+comments are untouched. When an automatic publication commits nothing new to+`gh-pages` because another publisher run already pushed the same run attempt,+its Pages deploy and PR comment are skipped. The run that pushed it deploys and+then comments. Manual runs and re-runs always deploy, so re-running after a+failed deploy still publishes and comments.+A publication skipped because its PR closed or moved after the scan neither+deploys nor comments.Fork PR scans can run without provider secrets; fork PR bot comments and currentPR aliases are not enabled. Comment-only operation and hosting alongside anarbitrary existing Pages deployment need further integration work.@@ -156,6 +180,12 @@ then.Tier 1 image** with `publish: true`. Publishing is allowed only from `main`in `NVIDIA/SkillEvaluator`; PR runs build and smoke-test without uploading.The default `publish: false` builds and verifies without publication.+ A release run builds the image once, smoke-tests that local image and pushes+ the same image. Pushes to `main` also build and smoke-test the image; they+ write the GitHub Actions layer cache that later builds read. Pull request+ runs read that cache but never write it. Release builds do not reuse cached+ layers, so `git` and `ca-certificates` come from the Debian packages current+ at release time.Build the complete Tier 1 runtime with required external scanners. The baseimage digest, uv bootstrap, scanner runtime dependencies and SkillSpectorbuild dependencies are pinned; scanner installs require package hashes.@@ -234,6 +264,19 @@ are unit tested separately; only a run against a real image establishes runtimeproof. The script does not publish an image, modify a remote repository ortrigger a GitHub workflow.+## Image layout++`ci/Dockerfile.tier1` is a multi-stage build. The build stages install the+locked environments with uv. The runtime stage copies only the three virtual+environments, Gitleaks, `ci/` and the license files; uv, its cache and the+source tree are not in the published image. Scanner layers come before any+SkillEvaluator source, so a source edit rebuilds only the small project layer.+The runtime filesystem is read-only, so Python bytecode is compiled at build+time, including the standard library. The `.pyc` files are hash-based and do+not embed source timestamps. Installed sources never change inside the image,+so Python does not re-check them on import. Do not bind-mount replacement+Python files over single installed modules; rebuild the image instead.+## Updating scanner locks`ci/requirements/*.in` records direct scanner inputs; the corresponding `.txt`diff --git a/ci/comment_report.py b/ci/comment_report.pyindex 41d47e41..891ad900 100644--- a/ci/comment_report.py+++ b/ci/comment_report.py@@ -14,17 +14,20 @@ import jsonimport osimport reimport sys-import urllib.requestfrom pathlib import Path+try:+ import github_api+except ModuleNotFoundError:+ from ci import github_api+MARKER = "<!-- skillevaluator-tier1-report -->"RUN_MARKER = re.compile(r"<!-- skillevaluator-tier1-run:([1-9][0-9]*):([1-9][0-9]*) -->")-REPOSITORY_PATTERN = re.compile(r"[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+")-SHA_PATTERN = re.compile(r"[0-9a-f]{40}")+REPOSITORY_PATTERN = github_api.REPOSITORY+SHA_PATTERN = github_api.COMMIT_SHABOT_PATTERN = re.compile(r"[A-Za-z0-9_-]+\[bot\]")MAX_BODY_BYTES = 60000MAX_METADATA_BYTES = 64 * 1024-MAX_API_BYTES = 16 * 1024 * 1024MAX_PAGES = 10@@ -32,60 +35,14 @@ class FeedbackError(ValueError):"""A fixed diagnostic safe to print without feedback content or credentials."""-class _NoRedirect(urllib.request.HTTPRedirectHandler):- def redirect_request(self, _request, _file, _code, _message, _headers, _url):- return None---class GitHubAPI:- def __init__(self, repository: str, token: str):- self.repository, self.token = repository, token- self.opener = urllib.request.build_opener(_NoRedirect())-- def _request(self, method: str, path: str, payload=None):- prefix = f"/repos/{self.repository}/"- if not path.startswith(prefix) or ".." in path or "\\" in path:- raise FeedbackError("Only this repository's GitHub API endpoints are supported.")- headers = {- "Authorization": f"Bearer {self.token}",- "Accept": "application/vnd.github+json",- "X-GitHub-Api-Version": "2022-11-28",- "User-Agent": "SkillEvaluator-PR-feedback",- }- data = None- if payload is not None:- data = json.dumps(payload, ensure_ascii=True, allow_nan=False).encode("utf-8")- headers["Content-Type"] = "application/json"- request = urllib.request.Request(f"https://api.github.com{path}", data=data, headers=headers, method=method)- with self.opener.open(request, timeout=60) as response:- raw = response.read(MAX_API_BYTES + 1)- if len(raw) > MAX_API_BYTES:- raise FeedbackError("The GitHub API response exceeded its size limit.")- return _json(raw)-- def get(self, path: str):- return self._request("GET", path)-- def post(self, path: str, payload: dict):- return self._request("POST", path, payload)-- def patch(self, path: str, payload: dict):- return self._request("PATCH", path, payload)+class GitHubAPI(github_api.GitHubAPI):+ error = FeedbackError+ json_error = "Feedback metadata or the GitHub API response is invalid JSON."+ user_agent = "SkillEvaluator-PR-feedback"def _json(raw: bytes):- def pairs(items):- value = {}- for key, item in items:- if key in value:- raise FeedbackError("Feedback metadata contains duplicate JSON keys.")- value[key] = item- return value-- try:- return json.loads(raw.decode("utf-8"), object_pairs_hook=pairs)- except (UnicodeError, ValueError, RecursionError) as exc:- raise FeedbackError("Feedback metadata or the GitHub API response is invalid JSON.") from exc+ return github_api.strict_json(raw, FeedbackError, GitHubAPI.json_error)def _read(path: Path, limit: int) -> bytes:diff --git a/ci/github_api.py b/ci/github_api.pynew file mode 100644index 00000000..505b929a--- /dev/null+++ b/ci/github_api.py@@ -0,0 +1,110 @@+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+# SPDX-License-Identifier: Apache-2.0+# ruff: noqa: INP001 -- standalone helper shared by the trusted CI entry points.++"""GitHub REST client and identity patterns shared by the trusted CI helpers.++Standard library only. The PR comment job runs this module with the runner's+system Python from a sparse checkout that holds just it and comment_report.py.+"""++from __future__ import annotations++import json+import re+import urllib.request++# GitHub commit, tree and blob identities are SHA-1. SHA-256 repositories are not supported.+COMMIT_SHA = re.compile(r"[0-9a-f]{40}")+REPOSITORY = re.compile(r"[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+")+GHCR_DIGEST = re.compile(r"ghcr\.io/[a-z0-9][a-z0-9._/-]*@sha256:[0-9a-f]{64}")+MAX_API_BYTES = 16 * 1024 * 1024+TIMEOUT_SECONDS = 60+++class _NoRedirect(urllib.request.HTTPRedirectHandler):+ def redirect_request(self, _request, _file, _code, _message, _headers, _url):+ return None+++def bounded_read(+ response,+ limit: int,+ error: type[Exception] = ValueError,+ message: str = "The GitHub API response exceeds its size limit.",+) -> bytes:+ """Read at most ``limit`` bytes, rejecting a larger declared or actual body."""+ declared = response.headers.get("Content-Length")+ if declared and (not declared.isdigit() or int(declared) > limit):+ raise error(message)+ payload = response.read(limit + 1)+ if len(payload) > limit:+ raise error(message)+ return payload+++def strict_json(raw: bytes, error: type[Exception] = ValueError, message: str = "A JSON document is invalid."):+ """Parse UTF-8 JSON and reject duplicate keys and NaN or Infinity."""++ def pairs(items):+ value = {}+ for key, item in items:+ if key in value:+ raise ValueError("duplicate key")+ value[key] = item+ return value++ def invalid_constant(_value):+ raise ValueError("non-finite number")++ try:+ return json.loads(raw.decode("utf-8"), object_pairs_hook=pairs, parse_constant=invalid_constant)+ except (ValueError, UnicodeError, RecursionError) as exc:+ raise error(message) from exc+++class GitHubAPI:+ """Token-authenticated REST client limited to one repository's endpoints.++ Subclasses set ``error`` to their own fixed-diagnostic exception type.+ """++ error: type[Exception] = ValueError+ json_error = "The GitHub API response is invalid JSON."+ user_agent = "SkillEvaluator-CI"++ def __init__(self, repository: str, token: str):+ self.repository, self.token = repository, token+ self.opener = urllib.request.build_opener(_NoRedirect())++ def _request(self, method: str, path: str, payload=None):+ base = f"/repos/{self.repository}"+ # A commit comparison is the one endpoint whose path needs "..": allow exactly two full SHAs.+ comparison = re.fullmatch(re.escape(base) + r"/compare/[0-9a-f]{40}\.\.\.[0-9a-f]{40}", path)+ if not (path == base or path.startswith(base + "/")) or (".." in path and not comparison) or "\\" in path:+ raise self.error("Only this repository's GitHub API endpoints are supported.")+ headers = {+ "Authorization": f"Bearer {self.token}",+ "Accept": "application/vnd.github+json",+ "X-GitHub-Api-Version": "2022-11-28",+ "User-Agent": self.user_agent,+ }+ data = None+ if payload is not None:+ data = json.dumps(payload, ensure_ascii=True, allow_nan=False).encode("utf-8")+ headers["Content-Type"] = "application/json"+ request = urllib.request.Request(f"https://api.github.com{path}", data=data, headers=headers, method=method)+ return self.opener.open(request, timeout=TIMEOUT_SECONDS)++ def _call(self, method: str, path: str, payload=None):+ with self._request(method, path, payload) as response:+ return strict_json(bounded_read(response, MAX_API_BYTES, self.error), self.error, self.json_error)++ def get(self, path: str):+ return self._call("GET", path)++ def post(self, path: str, payload: dict):+ return self._call("POST", path, payload)++ def patch(self, path: str, payload: dict):+ return self._call("PATCH", path, payload)diff --git a/ci/native_reports.py b/ci/native_reports.pyindex ed724f9d..31f1b83b 100644--- a/ci/native_reports.py+++ b/ci/native_reports.py@@ -15,15 +15,19 @@ from functools import lru_cachefrom html import unescapefrom html.parser import HTMLParserfrom importlib import resources-from pathlib import Path+from pathlib import Path, PurePosixPath-from skillevaluator.reporting.retained import render_retained_report+from skillevaluator.reporting.html import VERDICT_WORDS, fix_first_for_results, render_logo+from skillevaluator.reporting.retained import render_retained_report, retained_resultstry:import report_views as viewsexcept ModuleNotFoundError:from ci import report_views as views+# The scan job mounts the repository here, so scanners record finding paths below it.+WORKSPACE = PurePosixPath("/workspace")+class _InlineCode(HTMLParser):def __init__(self):@@ -106,17 +110,48 @@ def constrain_native_html(document: str) -> str:_STYLE = """-:root{font-family:ui-sans-serif,system-ui,-apple-system,BlinkMacSystemFont,'Segoe UI',sans-serif;color:#24292f;background:#f5f6f7;font-size:14px}-*{box-sizing:border-box}body{margin:0}a{color:#487600;text-underline-offset:3px}a:focus-visible,summary:focus-visible{outline:3px solid #76b900;outline-offset:4px}-.rail{position:fixed;inset:0 auto 0 0;width:210px;background:#1e2328;color:white;padding:32px 24px}.brand{font-size:17px;font-weight:700;letter-spacing:-.5px}.mark{display:inline-block;background:#76b900;width:11px;height:11px;border-radius:3px;margin-right:8px}.rail p{color:#bfc6ce;font-size:12px;line-height:1.6}.rail a{display:block;color:#e6f2d4;margin-top:30px;padding:12px;background:#34402d;border-radius:6px;text-decoration:none}-main{margin-left:210px;max-width:1400px;padding:36px 44px 60px}.eyebrow{text-transform:uppercase;letter-spacing:.1em;font-size:10px;font-weight:700;color:#59636e}h1{font-size:28px;letter-spacing:-.7px;margin:8px 0 12px}h2{font-size:19px;letter-spacing:-.3px;margin:0 0 16px}p{line-height:1.6}.muted{color:#59636e;font-size:12px}.status{display:inline-block;border-radius:5px;padding:5px 9px;font-size:11px;font-weight:700;background:#fff5df;color:#8a5800}.pass{background:#edf5e4;color:#357000}.fail{background:#fff0ef;color:#b4232d}.hero{display:flex;justify-content:space-between;align-items:start;gap:20px}.button{display:inline-block;padding:10px 14px;border:1px solid #dfe3e6;border-radius:6px;background:#fff;font-size:12px;font-weight:600;text-decoration:none;white-space:nowrap}-.metrics{display:grid;grid-template-columns:repeat(4,1fr);gap:12px;margin:24px 0}.metric,section{background:#fff;border:1px solid #dfe3e6;border-radius:8px}.metric{padding:17px 20px}.metric strong{display:block;font-size:26px;letter-spacing:-.5px;margin-bottom:5px}.metric span{font-size:12px;color:#59636e}section{padding:24px;margin-bottom:20px}.scope{font-size:12px;color:#59636e;line-height:1.7;padding:0 0 20px}.scope strong{color:#24292f}.attention{border-left:3px solid #b4232d}.attention p:last-child{margin-bottom:0}.table-wrap{overflow:auto}table{width:100%;border-collapse:collapse;text-align:left}th{font-size:10px;text-transform:uppercase;letter-spacing:.07em;color:#59636e;background:#fafbfc}th,td{padding:14px 12px;border-bottom:1px solid #eaecef}td{font-size:12px}td:first-child{font-weight:600;min-width:180px;max-width:320px;overflow-wrap:anywhere}td .muted{font-weight:400}tr:last-child td{border-bottom:0}.score{font-variant-numeric:tabular-nums;font-weight:600}.evidence{white-space:nowrap}summary{cursor:pointer;font-size:12px;font-weight:600}details p{font-size:12px;overflow-wrap:anywhere}code{font-family:ui-monospace,monospace;font-size:11px;overflow-wrap:anywhere}.empty{padding:20px;background:#fff5df;border-radius:6px}footer{font-size:11px;color:#59636e;margin-top:24px}+:root{--brand-core:#76b900;--brand-green:#3e6500;--bg:#fff;--page:#f5f6f7;--tx:#24292f;--muted:#59636e;--line:#dfe3e6;--line-soft:#eaecef;--th:#fafbfc;--link:#3e6500;--focus:#3d6b00;+--pass-bg:#edf5e4;--pass-tx:#315b25;--fail-bg:#fff0ef;--fail-tx:#a3232b;--warn-bg:#fff5df;--warn-tx:#7a4e00;--rail:#1e2328;--rail-tx:#bfc6ce;--rail-link:#34402d;--rail-link-tx:#e6f2d4;+--se-logo-panel:#152117;--se-logo-panel-edge:#304631;--se-logo-ring-outer:#39523a;--se-logo-ring-middle:#6f9264;--se-logo-ring-inner:#dbe8d6;--se-logo-core:#76b900;--se-logo-core-edge:#9bdb34;--se-logo-check:#fff;+font-family:ui-sans-serif,system-ui,-apple-system,BlinkMacSystemFont,'Segoe UI',sans-serif;color:var(--tx);background:var(--page);font-size:14px}+@media screen and (prefers-color-scheme:dark){:root{--bg:#1e252c;--page:#151a1f;--tx:#e1e6eb;--muted:#aab5bf;--line:#3b454f;--line-soft:#303b46;--th:#252d35;--link:#b0d684;--focus:#b0d684;+--pass-bg:#293e27;--pass-tx:#c4e7a0;--fail-bg:#412a2d;--fail-tx:#ffc1c4;--warn-bg:#403826;--warn-tx:#f4d792;--rail:#0f1317}}+*{box-sizing:border-box}body{margin:0;background:var(--page);color:var(--tx)}a{color:var(--link);text-underline-offset:3px}a:focus-visible,summary:focus-visible{outline:3px solid var(--focus);outline-offset:4px}+.rail{position:fixed;inset:0 auto 0 0;width:210px;background:var(--rail);color:white;padding:32px 24px}.brand{display:flex;align-items:center;gap:10px;font-size:17px;font-weight:700;letter-spacing:-.5px;color:#fff;white-space:nowrap}.se-logo{flex:none;display:block}.rail p{color:var(--rail-tx);font-size:12px;line-height:1.6}.rail a{display:block;color:var(--rail-link-tx);margin-top:30px;padding:12px;background:var(--rail-link);border-radius:6px;text-decoration:none}+main{margin-left:210px;max-width:1400px;padding:36px 44px 60px}.eyebrow{text-transform:uppercase;letter-spacing:.1em;font-size:11px;font-weight:700;color:var(--muted)}h1{font-size:28px;letter-spacing:-.7px;margin:8px 0 12px}h2{font-size:19px;letter-spacing:-.3px;margin:0 0 16px}p{line-height:1.6}.muted{color:var(--muted);font-size:12px}.status{display:inline-block;border-radius:999px;border:1px solid currentColor;padding:4px 10px;font-size:12px;font-weight:700;white-space:nowrap;background:var(--warn-bg);color:var(--warn-tx);print-color-adjust:exact;-webkit-print-color-adjust:exact}.pass{background:var(--pass-bg);color:var(--pass-tx)}.fail{background:var(--fail-bg);color:var(--fail-tx)}.neutral{background:var(--th);color:var(--muted)}.hero{display:flex;justify-content:space-between;align-items:start;gap:20px}.button{display:inline-block;padding:10px 14px;border:1px solid var(--line);border-radius:6px;background:var(--bg);color:var(--tx);font-size:12px;font-weight:600;text-decoration:none;white-space:nowrap}+.metrics{display:grid;grid-template-columns:repeat(4,1fr);gap:12px;margin:24px 0}.metric,section{background:var(--bg);border:1px solid var(--line);border-radius:8px}.metric{padding:17px 20px}.metric strong{display:block;font-size:26px;letter-spacing:-.5px;margin-bottom:5px}.metric span{font-size:12px;color:var(--muted)}section{padding:24px;margin-bottom:20px}.scope{font-size:12px;color:var(--muted);line-height:1.7;padding:0 0 20px}.scope strong{color:var(--tx)}.attention{border-left:3px solid var(--fail-tx)}.attention p:last-child{margin-bottom:0}.table-wrap{overflow:auto}table{width:100%;border-collapse:collapse;text-align:left}th{font-size:11px;text-transform:uppercase;letter-spacing:.07em;color:var(--muted);background:var(--th);white-space:nowrap}th,td{padding:14px 12px;border-bottom:1px solid var(--line-soft)}td{font-size:13px}td:first-child{font-weight:600;min-width:180px;max-width:320px;overflow-wrap:anywhere}td .muted{font-weight:400}tr:last-child td{border-bottom:0}.score{font-variant-numeric:tabular-nums;font-weight:600;white-space:nowrap}.evidence{white-space:nowrap}summary{cursor:pointer;font-size:12px;font-weight:600}details p{font-size:12px;overflow-wrap:anywhere}code{font-family:ui-monospace,monospace;font-size:12px;overflow-wrap:anywhere}.empty{padding:20px;background:var(--warn-bg);color:var(--warn-tx);border-radius:6px}footer{font-size:12px;color:var(--muted);margin-top:24px}+.sr-only{position:absolute;width:1px;height:1px;overflow:hidden;clip:rect(0,0,0,0);white-space:nowrap}@media(max-width:850px){.rail{position:static;width:auto;padding:16px 20px}.rail p,.rail a{display:none}main{margin:0;padding:24px 20px}.hero{flex-wrap:wrap}.metrics{grid-template-columns:repeat(2,1fr)}section{padding:18px}.metric{padding:16px}h1{font-size:25px}}-@media(max-width:600px){table,tbody{display:block}thead{display:none}tr{display:grid;grid-template-columns:1fr 1fr;border-bottom:1px solid #dfe3e6;padding:12px 0;gap:10px}td{display:block;padding:0;border:0}td:first-child{grid-column:1/-1;min-width:0;max-width:none}td:nth-child(3)::before{content:'Quality: ';font-weight:400;color:#59636e}td:nth-child(4)::before{content:'Errors: ';color:#59636e}td:nth-child(5)::before{content:'Warnings: ';color:#59636e}.evidence{grid-column:1/-1}tr:last-child{border-bottom:0}}-@media print{.rail,.button{display:none}main{margin:0;padding:0;max-width:none}.table-wrap{overflow:visible}section,.metric{break-inside:avoid}body{background:white}a{color:inherit}}+@media(max-width:600px){table,tbody{display:block}thead{display:none}tr{display:grid;grid-template-columns:1fr 1fr;border-bottom:1px solid var(--line);padding:12px 0;gap:10px}td{display:block;padding:0;border:0}td:first-child{grid-column:1/-1;min-width:0;max-width:none}td:nth-child(3)::before{content:'Quality: ';font-weight:400;color:var(--muted)}td:nth-child(4)::before{content:'Errors: ';color:var(--muted)}td:nth-child(5)::before{content:'Warnings: ';color:var(--muted)}.evidence{grid-column:1/-1}tr:last-child{border-bottom:0}}+@media print{.rail{position:static;width:auto;padding:0 0 8px;background:none;color:#111}.rail p,.rail a,.button{display:none}.brand{color:#111}.rail .se-logo{--se-logo-panel:#f2f7ec;--se-logo-panel-edge:#d7e5cc;--se-logo-ring-outer:#b8d3a5;--se-logo-ring-middle:#83aa66;--se-logo-ring-inner:#315b25;--se-logo-core-edge:#3e6500}main{margin:0;padding:0;max-width:none}.table-wrap{overflow:visible}section,.metric{break-inside:avoid}body{background:white}a{color:inherit}}"""+def brand_html(uid: str) -> str:+ """Logo and product name for the rail of the CI index and the Pages history.++ The logo is decorative next to the visible name; ``uid`` is kept so a+ labelled copy on the same page would still get unique ids.+ """+ return f'<div class="brand">{render_logo(uid, 28, "")}<span>SkillEvaluator</span></div>'+++# Run and skill statuses in the report vocabulary. Anything unrecognised is+# shown as Incomplete, never as a pass.+_STATUS_VERDICTS = {+ "PASS": "pass",+ "FAIL": "fail",+ "passed": "pass",+ "failed": "fail",+ "skipped": "skipped",+}+++def verdict_word(status: object) -> str:+ """Return the report's verdict word for a run or skill status."""+ return VERDICT_WORDS[_STATUS_VERDICTS.get(str(status), "incomplete")]++def render_native_reports(reports: Path, summary: dict, context: dict) -> dict[str, str]:"""Write all available views and return per-skill rendering diagnostics."""reports = Path(reports)@@ -124,19 +159,24 @@ def render_native_reports(reports: Path, summary: dict, context: dict) -> dict[sraise ValueError("A regular report directory is required.")models = []render_errors = {}+ first_fixes = {}for row in summary.get("skills", []):if row.get("overall_status") == "error" and re.fullmatch(r"skill-[0-9]{3}", row.get("report_dir", "")):# The publisher intentionally discards rejected child JSON, so its# directory may not exist yet. Create only this validated leaf.(reports / row["report_dir"]).mkdir(exist_ok=True)- directory = views._directory(reports, row.get("report_dir"))- data, issue = views._read(directory)+ # Read each skill's JSON once; the Markdown views reuse these parsed reports.+ skills = views.read_skill_models(reports, summary)+ for skill in skills:+ row, directory, data, issue = skill["row"], skill["directory"], skill["data"], skill["issue"]status = views._skill_status(row, data, issue)native_url = Noneif row.get("overall_status") == "error":issue = row.get("error") or "The evaluator did not produce consistent report evidence."elif data and directory is not None:try:+ # Finding paths are shown relative to the skill, also for a skill at the repository root.+ scan_root = (WORKSPACE / row["path"]).as_posix()document = render_retained_report(data,skill_name=row["path"],@@ -147,9 +187,19 @@ def render_native_reports(reports: Path, summary: dict, context: dict) -> dict[s"report_index_url": "../index.html","canonical_json_url": "skillevaluator-tier1.json",},+ scan_root=scan_root,)views._write(directory / "skillevaluator-tier1.html", constrain_native_html(document))native_url = f"{directory.name}/skillevaluator-tier1.html"+ # Name the same first fix as the skill's own report, with its anchor.+ fixes = fix_first_for_results(+ retained_results(data),+ skill_name=row["path"],+ expected_skill_name=Path(row["path"]).name or "workspace",+ scan_root=scan_root,+ )+ if fixes:+ first_fixes[row["report_dir"]] = fixes[0]except (ValueError, TypeError, KeyError) as exc:issue = f"Native report could not be rendered: {exc}"if issue:@@ -172,37 +222,42 @@ def render_native_reports(reports: Path, summary: dict, context: dict) -> dict[squality = views._items(data.get("quality_summary"))models.append((row, data, issue, status, native_url, views._dict(quality[0]) if len(quality) == 1 else {}))# Retain GitHub's concise native surface and detailed text evidence.- views.render_reports(reports, summary, context, render_errors=render_errors)+ views.render_reports(reports, summary, context, render_errors=render_errors, models=skills)h = views._hpassed = sum(item[3] == "PASS" for item in models)incomplete = sum(item[3] in {"INCOMPLETE", "ERROR", "UNKNOWN"} for item in models)failed = len(models) - passed - incompleteno_changes = views.is_noop_summary(summary)- verdict = (- "No skills changed"+ verdict_key = (+ "skipped"if no_changes- else "Passed"+ else "pass"if models and passed == len(models)- else "Incomplete"+ else "incomplete"if incomplete or not models- else "Failed"+ else "fail")- verdict_class = "pass" if verdict == "Passed" else "fail" if verdict == "Failed" else ""+ verdict = VERDICT_WORDS[verdict_key]+ verdict_class = {"pass": "pass", "fail": "fail", "skipped": "neutral"}.get(verdict_key, "")run_url = views._url(context.get("run_url"))page = ['<!doctype html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">',"<meta http-equiv=\"Content-Security-Policy\" content=\"default-src 'none'; style-src 'unsafe-inline'; base-uri 'none'; form-action 'none'\">",f"<title>Skills · SkillEvaluator</title><style>{_STYLE}</style></head><body>",- '<aside class="rail"><div class="brand"><span class="mark"></span>SkillEvaluator</div><p>Evaluation reports</p><a href="#skills" aria-current="page">All skills</a></aside><main>',+ f'<aside class="rail">{brand_html("se-logo-index")}<p>Evaluation reports</p><a href="#skills" aria-current="page">All skills</a></aside><main>','<header class="hero"><div><div class="eyebrow">Evaluation report · Tier 1</div><h1>Skill scan results</h1>',- f'<span class="status {verdict_class}">{verdict}</span><p class="muted">Review findings, quality dimensions, and scanner evidence for each skill.</p></div>',+ f'<span class="status {verdict_class}"><span class="sr-only">Verdict: </span>{verdict}</span>',+ # A no-op run explains itself once, in the skills section below.+ "</div>"+ if no_changes+ else '<p class="muted">Open a skill to see its findings and the evidence for each check.</p></div>',f'<a class="button" href="{h(run_url)}">View workflow run ↗</a>' if run_url else "",'</header><div class="metrics">',]for count, label in ((len(models), "Skills scanned"),(passed, "Passed"),- (failed, "Failed"),+ (failed, VERDICT_WORDS["fail"]),(incomplete, "Incomplete"),):page.append(f'<div class="metric"><strong>{count}</strong><span>{label}</span></div>')@@ -224,32 +279,37 @@ def render_native_reports(reports: Path, summary: dict, context: dict) -> dict[spage.append('<section class="attention"><h2>Needs attention</h2>')if summary.get("error"):page.append(f"<p>{h(summary['error'])}</p>")- for row, data, issue, _status, url, _quality in attention:- reasons = [- finding.get("message")- for result in data.get("results", [])- if not result.get("passed")- for finding in result.get("findings", [])- if finding.get("message")- ]- reason = issue or row.get("error") or next(iter(reasons), "Open the report to inspect the required checks.")+ for row, _data, issue, _status, url, _quality in attention:+ fix = first_fixes.get(row["report_dir"]) if not issue else None+ reason = issue or row.get("error") or (fix or {}).get("message") or "Open the report to inspect the checks."label = f'<a href="{h(url)}">{h(row["path"])}</a>' if url else h(row["path"])- page.append(f"<p><strong>{label}</strong><br>{h(str(reason)[:2000])}</p>")+ line = h(str(reason)[:2000])+ if fix and url:+ line = f'<a href="{h(url)}#{h(fix["anchor"])}">{line}</a>'+ if fix:+ line = f'{line} <span class="muted">{h(fix["check"])}</span>'+ page.append(f"<p><strong>{label}</strong><br>{line}</p>")page.append("</section>")- page.append(- '<section id="skills"><h2>All skills</h2><p class="muted">Quality measures content quality out of 100. The scan verdict is a separate result.</p>'- )+ page.append('<section id="skills"><h2>All skills</h2>')+ if models:+ page.append(+ '<p class="muted">Quality is a content score out of 100. A score below the minimum fails the'+ " Quality score check, which blocks the verdict.</p>"+ )if models:page.append(- '<div class="table-wrap"><table><thead><tr><th scope="col">Skill</th><th scope="col">Scan result</th><th scope="col">Quality</th><th scope="col">Errors</th><th scope="col">Warnings</th><th scope="col">Report</th></tr></thead><tbody>'+ '<div class="table-wrap"><table><thead><tr><th scope="col">Skill</th><th scope="col">Verdict</th><th scope="col">Quality</th><th scope="col">Errors</th><th scope="col">Warnings</th><th scope="col">Report</th></tr></thead><tbody>')for row, data, _issue, status, url, quality in models:css = "pass" if status == "PASS" else "fail" if status == "FAIL" else ""score = views._score(quality.get("overall_score"))grade = quality.get("grade", "")+ minimum = views._score(views._dict(data.get("quality_config")).get("min_score"))+ if minimum != "Not recorded" and score != "Not recorded":+ grade = " · ".join(part for part in (str(grade), f"min {minimum}") if part)link = f'<a href="{h(url)}">View report →</a>' if url else "Unavailable"page.append(- f'<tr id="{h(row["report_dir"])}"><td>{h(row["path"])}</td><td><span class="status {css}">{h(status)}</span></td><td class="score">{h(score)} <span class="muted">{h(grade)}</span></td><td>{h(data.get("total_errors", "—"))}</td><td>{h(data.get("total_warnings", "—"))}</td><td class="evidence">{link}</td></tr>'+ f'<tr id="{h(row["report_dir"])}"><td>{h(row["path"])}</td><td><span class="status {css}">{h(verdict_word(status))}</span></td><td class="score">{h(score)} <span class="muted">{h(grade)}</span></td><td>{h(data.get("total_errors", "—"))}</td><td>{h(data.get("total_warnings", "—"))}</td><td class="evidence">{link}</td></tr>')page.append("</tbody></table></div>")elif no_changes:diff --git a/ci/publish_reports.py b/ci/publish_reports.pyindex 5bdfd403..3c1962f4 100644--- a/ci/publish_reports.py+++ b/ci/publish_reports.py@@ -17,6 +17,7 @@ import ioimport jsonimport osimport re+import secretsimport shutilimport statimport sys@@ -27,6 +28,10 @@ import zipfilefrom datetime import datetimefrom pathlib import Path, PurePosixPath+try:+ import github_api+except ModuleNotFoundError:+ from ci import github_apifrom skillevaluator.utils.secret_redaction import REDACTED, SENSITIVE_PII_CATEGORIES, Redactor, sensitive_valuesMAX_FILES = 1000@@ -35,10 +40,14 @@ MAX_TOTAL_BYTES = 64 * 1024 * 1024MAX_FILE_BYTES = 32 * 1024 * 1024MAX_HISTORY_BYTES = 512 * 1024 * 1024ARTIFACT_NAME = "skill-evaluator-tier1"+# The scan uploads this small artifact next to the full one. It holds only the+# JSON read here, so HTML and logs never count against the download limits.+EVIDENCE_ARTIFACT = f"{ARTIFACT_NAME}-evidence"WORKFLOW_PATH = ".github/workflows/skill-checks.yml"-SHA_PATTERN = re.compile(r"[0-9a-f]{40}")-REPO_PATTERN = re.compile(r"[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+")+SHA_PATTERN = github_api.COMMIT_SHA+REPO_PATTERN = github_api.REPOSITORYREPORT_DIRECTORY = re.compile(r"skill-(?:00[1-9]|0[1-9][0-9]|100)")+REPORT_MEMBER = re.compile(rf"(?:{REPORT_DIRECTORY.pattern})/skillevaluator-tier1\.json")class PublicationError(ValueError):@@ -64,52 +73,18 @@ def allowed_download_url(url: str) -> bool:)-class _NoRedirect(urllib.request.HTTPRedirectHandler):- def redirect_request(self, _request, _file, _code, _message, _headers, _url):- return None---def _bounded_read(response, limit: int) -> bytes:- declared = response.headers.get("Content-Length")- if declared and (not declared.isdigit() or int(declared) > limit):- raise PublicationError("The downloaded artifact or API response exceeds its size limit.")- payload = response.read(limit + 1)- if len(payload) > limit:- raise PublicationError("The downloaded artifact or API response exceeds its size limit.")- return payload---class GitHubAPI:- def __init__(self, repository: str, token: str):- self.repository = repository- self.token = token- self.opener = urllib.request.build_opener(_NoRedirect())-- def _request(self, path: str):- root = f"/repos/{self.repository}"- prefix = root + "/"- comparison = re.fullmatch(re.escape(prefix) + r"compare/[0-9a-f]{40}\.\.\.[0-9a-f]{40}", path)- if (path != root and not path.startswith(prefix)) or (".." in path and not comparison) or "\\" in path:- raise PublicationError("Only this repository's GitHub API endpoints are supported.")- request = urllib.request.Request(- f"https://api.github.com{path}",- headers={- "Authorization": f"Bearer {self.token}",- "Accept": "application/vnd.github+json",- "X-GitHub-Api-Version": "2022-11-28",- "User-Agent": "SkillEvaluator-report-publisher",- },- )- return self.opener.open(request, timeout=60)+DOWNLOAD_TOO_LARGE = "The downloaded artifact exceeds its size limit."- def get(self, path: str):- with self._request(path) as response:- return _json(_bounded_read(response, MAX_FILE_BYTES))++class GitHubAPI(github_api.GitHubAPI):+ error = PublicationError+ json_error = "A required JSON document is invalid."+ user_agent = "SkillEvaluator-report-publisher"def download_artifact(self, artifact_id: int) -> bytes:try:- with self._request(f"/repos/{self.repository}/actions/artifacts/{artifact_id}/zip") as response:- return _bounded_read(response, MAX_TOTAL_BYTES)+ with self._request("GET", f"/repos/{self.repository}/actions/artifacts/{artifact_id}/zip") as response:+ return github_api.bounded_read(response, MAX_TOTAL_BYTES, PublicationError, DOWNLOAD_TOO_LARGE)except urllib.error.HTTPError as exc:if exc.code not in {301, 302, 303, 307, 308}:raise@@ -118,10 +93,10 @@ class GitHubAPI:for _ in range(5):if not allowed_download_url(url):raise PublicationError("The artifact download redirected to an unsupported host.")- request = urllib.request.Request(url, headers={"User-Agent": "SkillEvaluator-report-publisher"})+ request = urllib.request.Request(url, headers={"User-Agent": self.user_agent})try:- with self.opener.open(request, timeout=60) as response:- return _bounded_read(response, MAX_TOTAL_BYTES)+ with self.opener.open(request, timeout=github_api.TIMEOUT_SECONDS) as response:+ return github_api.bounded_read(response, MAX_TOTAL_BYTES, PublicationError, DOWNLOAD_TOO_LARGE)except urllib.error.HTTPError as exc:if exc.code not in {301, 302, 303, 307, 308}:raise@@ -130,21 +105,7 @@ class GitHubAPI:def _json(payload: bytes):- def object_pairs(pairs):- result = {}- for key, value in pairs:- if key in result:- raise PublicationError("Duplicate JSON keys are not supported.")- result[key] = value- return result-- def invalid_constant(_value):- raise PublicationError("Non-finite JSON numbers are not supported.")-- try:- return json.loads(payload.decode("utf-8"), object_pairs_hook=object_pairs, parse_constant=invalid_constant)- except (ValueError, UnicodeError, RecursionError) as exc:- raise PublicationError("A required JSON document is invalid.") from exc+ return github_api.strict_json(payload, PublicationError, GitHubAPI.json_error)def _integer(value, minimum=0):@@ -357,7 +318,7 @@ def read_artifact(payload: bytes) -> tuple[dict, dict[str, dict]]:total += entry.file_sizeif entry.file_size > MAX_FILE_BYTES or total > MAX_TOTAL_BYTES:raise PublicationError("The artifact exceeds its expanded size limit.")- if name == "summary.json" or re.fullmatch(r"skill-[0-9]{3}/skillevaluator-tier1\.json", name):+ if name == "summary.json" or REPORT_MEMBER.fullmatch(name):with archive.open(entry) as handle:content = handle.read(MAX_FILE_BYTES + 1)if len(content) > MAX_FILE_BYTES:@@ -431,17 +392,20 @@ def is_newer(candidate: dict, existing: dict | None) -> bool:def _write(path: Path, text: str) -> None:- current = path.parent- while True:- if current.is_symlink():+ """Replace a site file atomically; a failed write leaves the old file intact."""+ for ancestor in (path, *path.parents):+ if ancestor.is_symlink():raise PublicationError("The site contains a symlink in an output path.")- if current == current.parent:- break- current = current.parent- if path.is_symlink():- raise PublicationError("The site contains a symlink in an output path.")path.parent.mkdir(parents=True, exist_ok=True)- path.write_text(text, encoding="utf-8")+ temporary = path.with_name(f".{path.name}.{secrets.token_hex(8)}.tmp")+ descriptor = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0), 0o666)+ try:+ with os.fdopen(descriptor, "w", encoding="utf-8") as handle:+ handle.write(text)+ temporary.replace(path)+ except BaseException:+ temporary.unlink(missing_ok=True)+ raisedef _dump(path: Path, data) -> None:@@ -638,12 +602,10 @@ def _select_artifact(api, repository: str, run: dict) -> dict:finished = _timestamp(run.get("updated_at"))if finished < started:raise PublicationError("The selected run attempt has an invalid time interval.")+ name = f"{EVIDENCE_ARTIFACT}-attempt-{run['run_attempt']}"candidates = []for artifact in listing["artifacts"]:- if (- artifact.get("name") not in {ARTIFACT_NAME, f"{ARTIFACT_NAME}-attempt-{run['run_attempt']}"}- or artifact.get("expired") is not False- ):+ if artifact.get("name") != name or artifact.get("expired") is not False:continueidentity = artifact.get("workflow_run", {})if identity.get("id") != run["id"] or identity.get("head_sha") != run["head_sha"]:@@ -658,7 +620,11 @@ def _select_artifact(api, repository: str, run: dict) -> dict:raise PublicationError("The selected report artifact exceeds its size limit.")candidates.append(artifact)if not candidates:- raise PublicationError("The selected run attempt has no current report artifact.")+ raise PublicationError(+ f"The selected run attempt has no current report evidence artifact named {name}. "+ "The scan may have stopped before writing reports, the artifact may have expired, "+ "or the scan used an evaluator revision that does not upload it."+ )if len(candidates) != 1:raise PublicationError("The selected run attempt has ambiguous report artifacts.")return candidates[0]@@ -693,15 +659,15 @@ def _current_routes(api, repository: str, run: dict) -> tuple[list[str], str]:def _history_index(site: Path, history: list[dict]) -> None:try:- from native_reports import _STYLE+ from native_reports import _STYLE, brand_html, verdict_wordexcept ModuleNotFoundError:- from ci.native_reports import _STYLE+ from ci.native_reports import _STYLE, brand_html, verdict_wordlinks = []for record in sorted(history, key=lambda item: (item["run_id"], item["run_attempt"]), reverse=True):url = html.escape(record["report_url"], quote=True)label = html.escape(- f"Run {record['run_id']} / attempt {record['run_attempt']} — {record['overall_status']} — {record['head_sha'][:12]}"+ f"Run {record['run_id']} / attempt {record['run_attempt']} · {verdict_word(record['overall_status'])} · {record['head_sha'][:12]}")links.append(f'<li><a href="{url}">{label}</a></li>')latest = []@@ -717,7 +683,7 @@ def _history_index(site: Path, history: list[dict]) -> None:'<!doctype html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">'"<meta http-equiv=\"Content-Security-Policy\" content=\"default-src 'none'; style-src 'unsafe-inline'; base-uri 'none'; form-action 'none'\">"f"<title>Report history · SkillEvaluator</title><style>{_STYLE}li{{padding:14px 0;line-height:1.6;overflow-wrap:anywhere}}ul{{padding-left:18px}}</style></head><body>"- '<aside class="rail"><div class="brand"><span class="mark"></span>SkillEvaluator</div><p>Evaluation reports</p><a href="#history" aria-current="page">Report history</a></aside><main>'+ f'<aside class="rail">{brand_html("se-logo-history")}<p>Evaluation reports</p><a href="#history" aria-current="page">Report history</a></aside><main>''<header><p class="eyebrow">Evaluation reports</p><h1>Report history</h1><p class="muted">Open the latest scan or inspect the evidence recorded for a specific commit.</p></header><section><h2>Latest reports</h2><p>'+ " · ".join(latest)+ '</p></section><section id="history"><h2>Retained runs</h2><ul>'@@ -817,9 +783,7 @@ def _recorded_image(summary: dict, fallback: str) -> str:if not isinstance(recorded, str) or not isinstance(fallback, str):raise PublicationError("The report's runtime image identity must be a string.")selected = recorded or fallback- if selected and (- len(selected) > 1024 or re.fullmatch(r"ghcr\.io/[a-z0-9][a-z0-9._/-]*@sha256:[0-9a-f]{64}", selected) is None- ):+ if selected and (len(selected) > 1024 or github_api.GHCR_DIGEST.fullmatch(selected) is None):raise PublicationError("The report's runtime image identity must be an immutable GHCR digest.")return selecteddiff --git a/ci/render_templates.py b/ci/render_templates.pyindex 4a9589df..553581bb 100644--- a/ci/render_templates.py+++ b/ci/render_templates.py@@ -8,19 +8,23 @@ from __future__ import annotationsimport argparseimport hashlibimport json-import refrom pathlib import Path+try:+ import github_api+except ModuleNotFoundError:+ from ci import github_api+TEMPLATES = Path(__file__).parent / "templates"IMAGE_PLACEHOLDER = "ghcr.io/nvidia/skillevaluator-tier1@sha256:RELEASE_IMAGE_DIGEST"def render_templates(output: Path, workflow_ref: str, image: str, repository: str = "NVIDIA/SkillEvaluator") -> dict:- if not re.fullmatch(r"[0-9a-f]{40}", workflow_ref):+ if not github_api.COMMIT_SHA.fullmatch(workflow_ref):raise ValueError("A full reviewed workflow commit SHA is required.")- if not re.fullmatch(r"ghcr\.io/[a-z0-9][a-z0-9._/-]*@sha256:[0-9a-f]{64}", image):+ if not github_api.GHCR_DIGEST.fullmatch(image):raise ValueError("A published GHCR image digest is required.")- if not re.fullmatch(r"[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+", repository):+ if not github_api.REPOSITORY.fullmatch(repository):raise ValueError("A GitHub owner/repository is required.")if output.exists():raise ValueError("Use a new output directory to preserve previous release files.")diff --git a/ci/report_views.py b/ci/report_views.pyindex 4beefc5c..5ac9fc2d 100644--- a/ci/report_views.py+++ b/ci/report_views.py@@ -19,6 +19,7 @@ DETAIL_LIMIT = 500 * 1024SUMMARY_SKILL_LIMIT = 5COMMENT_LIMIT = 50 * 1024REPORT_LIMIT = 32 * 1024 * 1024+_CONTROL_CHARACTERS = dict.fromkeys(range(32), " ")def _dict(value: object) -> dict:@@ -42,7 +43,7 @@ def _h(value: object) -> str:def _md(value: object) -> str:- text = "".join(c if ord(c) >= 32 else " " for c in _text(value))+ text = _text(value).translate(_CONTROL_CHARACTERS)# Break GitHub mention tokens while retaining readable author and finding text.text = text.replace("@", "@\u200b")escaped = re.sub(r"([\\`*_{}\[\]()#+.!|~-])", r"\\\1", text)@@ -139,6 +140,19 @@ def _read(directory: Path | None) -> tuple[dict, str | None]:return {}, "JSON report could not be read; evidence is incomplete."+def read_skill_models(reports: Path, summary: dict) -> list[dict]:+ """Read each skill's retained JSON once, so every view of a run shares one parse."""+ models = []+ for index, row in enumerate(_items(summary.get("skills")), 1):+ row = _dict(row)+ directory = _directory(reports, row.get("report_dir"))+ data, issue = _read(directory)+ models.append(+ {"row": row, "directory": directory, "data": data, "issue": issue, "anchor": f"skill-{index:03d}"}+ )+ return models++def _links(directory: Path | None, report_url: str | None = None, *, anchor: str | None = None) -> list[tuple[str, str]]:@@ -234,8 +248,12 @@ def _markdown_blocks(lines: list[str]) -> list[str]:return blocks+def _reserve(footer: str) -> int:+ return len(footer.encode("utf-8")) + 160++def _bounded(blocks: list[str], limit: int, footer: str) -> str:- reserve = len(footer.encode("utf-8")) + 160+ reserve = _reserve(footer)parts = []size = 0truncated = False@@ -254,6 +272,29 @@ def _bounded(blocks: list[str], limit: int, footer: str) -> str:return "\n\n".join(parts) + "\n"+class _Budget:+ """Tell when _bounded() will drop every further line, so callers can stop building them.++ Each block costs at least one byte more than its lines, so the running sum+ of line sizes plus one is a lower bound. Once that bound passes the limit,+ _bounded() truncates within the lines already collected, whatever follows.+ """++ def __init__(self, lines: list[str], limit: int, footer: str) -> None:+ self._lines = lines+ self._counted = 0+ self._left = limit - _reserve(footer)++ def spent(self) -> bool:+ new = self._lines[self._counted :]+ self._counted += len(new)+ # JSON can carry lone surrogates. _bounded() never encodes a line past+ # its cut, so counting one here must not raise. surrogatepass gives+ # the same size for every line _bounded() can encode.+ self._left -= sum(len(line.encode("utf-8", "surrogatepass")) + 1 for line in new)+ return self._left < 0++def _write(path: Path, content: str) -> None:if path.is_symlink():raise ValueError("Refusing to replace a symlink report view.")@@ -392,11 +433,15 @@ def _first_action(model: dict) -> tuple[str, str, str]:"","Inspect the failed workflow step and report artifacts, then rerun the scan.",)- failed = [r for r in model["results"] if _status(r) in {"FAIL", "INCOMPLETE", "UNKNOWN"}]- failed.sort(key=lambda r: _status(r) == "FAIL")- for result in failed:+ failed = [+ (result, outcome)+ for result, outcome in zip(model["results"], model["statuses"], strict=True)+ if outcome in {"FAIL", "INCOMPLETE", "UNKNOWN"}+ ]+ failed.sort(key=lambda item: item[1] == "FAIL")+ for result, outcome in failed:legacy = _dict(result.get("legacy"))- if _status(result) in {"INCOMPLETE", "UNKNOWN"}:+ if outcome in {"INCOMPLETE", "UNKNOWN"}:errors = _items(legacy.get("errors"))warnings = _items(legacy.get("warnings"))reason = errors[0] if errors else result.get("skip_reason")@@ -455,7 +500,7 @@ def _compact_summary(models: list[dict], summary: dict, context: dict, *, statusoutcomes = ["INCOMPLETE"if model["skill_status"] == "PASS"- and any(_status(r) in {"FAIL", "INCOMPLETE", "UNKNOWN"} for r in model["results"])+ and any(outcome in {"FAIL", "INCOMPLETE", "UNKNOWN"} for outcome in model["statuses"])else model["skill_status"]for model in models]@@ -528,12 +573,21 @@ def _compact_summary(models: list[dict], summary: dict, context: dict, *, statusreturn _bounded(lines, SUMMARY_LIMIT, footer)-def render_reports(reports: Path, summary: dict, context: dict, *, render_errors: dict[str, str] | None = None) -> None:+def render_reports(+ reports: Path,+ summary: dict,+ context: dict,+ *,+ render_errors: dict[str, str] | None = None,+ models: list[dict] | None = None,+) -> None:"""Write a short Actions summary, detailed Markdown and a PR comment.Optional context keys: run_url, commit, image, report_url, llm_enabled, ci_conclusion.report_url is the hosted directory containing index.html and the artifacts.All report payloads are displayed as text, never executable markup.+ models is the read_skill_models() result when the caller has already read+ the reports; it is not modified."""reports = Path(reports)if reports.is_symlink() or not reports.is_dir():@@ -543,27 +597,18 @@ def render_reports(reports: Path, summary: dict, context: dict, *, render_errorsreturnreport_url = _url(context.get("report_url"))run_url = _url(context.get("run_url"))- models = []- for index, row in enumerate(_items(summary.get("skills")), 1):- row = _dict(row)- directory = _directory(reports, row.get("report_dir"))- data, issue = _read(directory)+ models = [dict(model) for model in (read_skill_models(reports, summary) if models is None else models)]+ for model in models:+ row = model["row"]if row.get("overall_status") == "error" or (render_errors or {}).get(row.get("report_dir")):- issue = (render_errors or {}).get(row.get("report_dir")) or row.get("error") or "Report unavailable"- data = {}- results = [_dict(item) for item in _items(data.get("results"))]- quality = [_dict(item) for item in _items(data.get("quality_summary"))]- models.append(- {- "row": row,- "directory": directory,- "data": data,- "issue": issue,- "results": results,- "quality": quality,- "anchor": f"skill-{index:03d}",- }- )+ model["issue"] = (+ (render_errors or {}).get(row.get("report_dir")) or row.get("error") or "Report unavailable"+ )+ model["data"] = {}+ model["results"] = [_dict(item) for item in _items(model["data"].get("results"))]+ # Each result's display status is computed once and reused by every view below.+ model["statuses"] = [_status(result) for result in model["results"]]+ model["quality"] = [_dict(item) for item in _items(model["data"].get("quality_summary"))]for model in models:model["skill_status"] = _skill_status(model["row"], model["data"], model["issue"])if (@@ -620,8 +665,7 @@ def render_reports(reports: Path, summary: dict, context: dict, *, render_errorscomment_header = list(markdown)attention = []for model in models:- for result in model["results"]:- outcome = _status(result)+ for result, outcome in zip(model["results"], model["statuses"], strict=True):if outcome not in {"FAIL", "INCOMPLETE", "UNKNOWN"}:continuefindings = _items(result.get("findings"))@@ -665,6 +709,7 @@ def render_reports(reports: Path, summary: dict, context: dict, *, render_errorserrors, warnings = data.get("total_errors"), data.get("total_warnings")markdown.append(f"| {_md(skill)} | {skill_status} | {score} | {_md(grade)} | {_md(errors)} | {_md(warnings)} |")comments.append(f"| {_md(skill)} | {skill_status} | {score} / {_md(grade)} | {_md(errors)} | {_md(warnings)} |")+ details = _Budget(markdown, DETAIL_LIMIT, footer)for model in models:row = model["row"]skill = _text(row.get("path"))@@ -693,18 +738,27 @@ def render_reports(reports: Path, summary: dict, context: dict, *, render_errors"| --- | --- | --- | --- | --- |",])- for result in model["results"]:+ for result, outcome in zip(model["results"], model["statuses"], strict=True):numbers = _dict(result.get("summary"))- name, outcome = result.get("validator"), _status(result)+ name = result.get("validator")block.append(f"| {_md(name)} | {outcome} | {_md(numbers.get('checks_performed'))} | {_md(numbers.get('errors'))} | {_md(numbers.get('warnings'))} |")+ markdown.extend(block)+ # details.md keeps only a prefix that fits. Skip escaping evidence it would drop.for result in model["results"]:- block.append(f"### {_md(result.get('validator'))} — evidence")+ if details.spent():+ break+ markdown.append(f"### {_md(result.get('validator'))} — evidence")for label, value in _evidence(result):- block.append(f"**{_md(label)}:** {_md(value)}")- markdown.extend(block)- failures = [result for result in model["results"] if _status(result) in {"FAIL", "INCOMPLETE", "UNKNOWN"}]+ if details.spent():+ break+ markdown.append(f"**{_md(label)}:** {_md(value)}")+ failures = [+ result+ for result, outcome in zip(model["results"], model["statuses"], strict=True)+ if outcome in {"FAIL", "INCOMPLETE", "UNKNOWN"}+ ]for result in failures:findings = _items(result.get("findings"))errors = _items(_dict(result.get("legacy")).get("errors"))diff --git a/ci/run_tier1.py b/ci/run_tier1.pyindex cf2297b2..4706f6be 100644--- a/ci/run_tier1.py+++ b/ci/run_tier1.py@@ -5,16 +5,23 @@"""Run complete Tier 1 checks on configured repository skills and retain evidence.Raw scanner output belongs in artifacts. Structured findings are escaped before-being shown in logs or GitHub annotations.+being shown in logs or GitHub annotations. SCAN_WORKERS (default 4) bounds how+many skills are scanned at once; rows, logs and annotations keep skill order."""from __future__ import annotationsimport argparse+import functoolsimport jsonimport os+import queue+import re+import signal+import statimport subprocessimport tempfile+import threadingimport timefrom pathlib import Path@@ -42,13 +49,31 @@ except ModuleNotFoundError:from ci.report_views import finding_repository_path, render_reportstry:- from select_changed_skills import SelectionError, select_changed_skills+ from select_changed_skills import (+ SelectionError,+ has_control_character,+ select_changed_skills,+ skill_root,+ skill_roots,+ )except ModuleNotFoundError:- from ci.select_changed_skills import SelectionError, select_changed_skills+ from ci.select_changed_skills import (+ SelectionError,+ has_control_character,+ select_changed_skills,+ skill_root,+ skill_roots,+ )MAX_SKILLS = 100MAX_REPORT_BYTES = 32 * 1024 * 1024SCAN_TIMEOUT_SECONDS = 3000+# An interrupted run asks running scans to stop, then kills them after this.+STOP_GRACE_SECONDS = 5+# How often a running scan checks whether the run is ending.+POLL_SECONDS = 0.1+DEFAULT_SCAN_WORKERS = 4+MAX_SCAN_WORKERS = 16PROVIDER_KEYS = {"openai": "OPENAI_API_KEY", "anthropic": "ANTHROPIC_API_KEY", "nv_build": "NVIDIA_API_KEY"}PROCESS_ENV = {"PATH",@@ -74,28 +99,16 @@ class ConfigurationError(ValueError):def discover_skills(workspace: Path, roots: str) -> list[Path]:- """Discover a bounded unique set; any linked scan content is rejected."""+ """Discover a bounded unique set from SKILL_PATHS; linked scan content is rejected."""workspace = workspace.resolve(strict=True)- configured = [line.strip() for line in roots.splitlines() if line.strip()]- if not configured:- raise ConfigurationError("SKILL_PATHS must contain at least one relative directory.")+ return skill_trees(workspace, skill_roots(workspace, roots, ConfigurationError))+++def skill_trees(workspace: Path, roots: list[str]) -> list[Path]:+ """Walk validated relative roots and return the skills found under them."""discovered: set[Path] = set()- for raw in configured:- relative = Path(raw)- if (- relative.is_absolute()- or ".." in relative.parts- or "\\" in raw- or ":" in raw- or any(ord(character) < 32 or ord(character) == 127 for character in raw)- ):- raise ConfigurationError("Skill roots must be relative directories without parent traversal.")- current = workspace- for part in relative.parts:- current /= part- if current.is_symlink():- raise ConfigurationError("Symlinks are not supported in skill roots or scan content.")- root = workspace / relative+ for relative in roots:+ root = workspace / skill_root(workspace, relative, ConfigurationError)if not root.is_dir() or not root.resolve(strict=True).is_relative_to(workspace):raise ConfigurationError("A configured skill root is missing or outside the workspace.")@@ -107,7 +120,7 @@ def discover_skills(workspace: Path, roots: str) -> list[Path]:inside_skill = any(parent.is_relative_to(skill) for skill in discovered) or "SKILL.md" in filesfor name in [*directories, *files]:child = parent / name- if any(ord(character) < 32 or ord(character) == 127 for character in name):+ if has_control_character(name):raise ConfigurationError("Skill paths must not contain control characters.")if child.is_symlink() and (inside_skill or name == "SKILL.md"):raise ConfigurationError("Symlinks are not supported in skill roots or scan content.")@@ -153,6 +166,34 @@ def child_environment(environ: dict[str, str]) -> tuple[dict[str, str], bool]:return env, enabled == "true"+def scan_workers(environ: dict[str, str]) -> int:+ """Read the bound on concurrent skill scans; 1 scans one skill at a time."""+ value = environ.get("SCAN_WORKERS", "").strip() or str(DEFAULT_SCAN_WORKERS)+ if not re.fullmatch(r"[0-9]{1,2}", value) or not 1 <= int(value) <= MAX_SCAN_WORKERS:+ raise ConfigurationError(f"SCAN_WORKERS must be a whole number from 1 to {MAX_SCAN_WORKERS}.")+ return int(value)+++def worker_environment(env: dict[str, str], worker: int) -> dict[str, str]:+ """Give each scan worker its own private home and temporary directory.++ The paths are fixed, so outside Docker another local user could create+ them first in a shared /tmp. An existing directory is used only if this+ user owns it and no one else has access to it.+ """+ isolated = dict(env)+ isolated["HOME"] = str(Path(env["HOME"]) / f"worker-{worker}")+ isolated["TMPDIR"] = str(Path(env.get("TMPDIR") or tempfile.gettempdir()) / f"skillevaluator-worker-{worker}")+ for name in ("HOME", "TMPDIR"):+ directory = Path(isolated[name])+ directory.mkdir(mode=0o700, parents=True, exist_ok=True)+ info = directory.lstat()+ private = os.name != "posix" or (info.st_uid == os.getuid() and not info.st_mode & 0o077)+ if not stat.S_ISDIR(info.st_mode) or not private:+ raise ConfigurationError("Scan worker directories must be private directories owned by this user.")+ return isolated++def _write_json(path: Path, payload: dict) -> None:temporary = path.with_suffix(path.suffix + ".tmp")with temporary.open("x", encoding="utf-8") as handle:@@ -189,7 +230,8 @@ def _load_report(path: Path) -> dict:raise ValueError("The evaluator JSON report has invalid severity counts.")from skillevaluator.reporting.retained import retained_results- retained_results(data)+ # Validation only: the results are discarded, so skip copying their metadata.+ retained_results(data, copy_metadata=False)return data@@ -286,10 +328,166 @@ def execution_context(environ: dict[str, str]) -> dict:}-def run(- workspace: Path, reports: Path, environ: dict[str, str], *, execute=subprocess.run, clock=time.monotonic-) -> int:- """Run all selected skills; return nonzero on every incomplete or failed run."""+def _signal_scan(process: subprocess.Popen, *, kill: bool) -> None:+ """Signal an evaluator's process group, which also holds the scanner tools it started."""+ if os.name != "posix":+ if process.poll() is None:+ if kill:+ process.kill()+ else:+ process.terminate()+ return+ try:+ os.killpg(process.pid, signal.SIGKILL if kill else signal.SIGTERM)+ except ProcessLookupError:+ pass+ except PermissionError:+ # macOS reports EPERM instead of ESRCH once the group leader has exited.+ if process.poll() is None:+ raise+++def run_scan(+ argv: list[str], *, abort: threading.Event, timeout: float, check: bool = False, **options+) -> subprocess.CompletedProcess:+ """Run one evaluator process like subprocess.run, and end it early when the run aborts.++ Scan worker threads never see Ctrl-C, and Docker sends it only to this+ runner (PID 1), so the runner ends running scans itself. The evaluator+ leads its own process group, so a timeout or abort also ends its tools.+ """+ process = subprocess.Popen(argv, start_new_session=os.name == "posix", **options)+ deadline = time.monotonic() + timeout+ grace_ends = None+ try:+ while True:+ now = time.monotonic()+ if now >= deadline:+ raise subprocess.TimeoutExpired(argv, timeout)+ if abort.is_set() and grace_ends is None:+ grace_ends = now + STOP_GRACE_SECONDS+ _signal_scan(process, kill=False)+ if grace_ends is not None and now >= grace_ends:+ break+ try:+ process.wait(timeout=min(POLL_SECONDS, deadline - now))+ break+ except subprocess.TimeoutExpired:+ pass+ finally:+ if process.returncode is None or grace_ends is not None:+ # Also end scanner tools that outlived an aborted evaluator.+ _signal_scan(process, kill=True)+ if process.returncode is None:+ process.wait()+ if check and process.returncode:+ raise subprocess.CalledProcessError(process.returncode, argv)+ return subprocess.CompletedProcess(argv, process.returncode)+++def evaluate_skill(+ target: Path,+ output: Path,+ *,+ workspace: Path,+ env: dict[str, str],+ llm: bool,+ timeout: float,+ execute,+ tokens: set[str],+) -> tuple[dict, dict | None, bool]:+ """Run one evaluator process; return row fields, its report and whether time ran out.++ Scan workers call this concurrently. Raw scanner output stays in a private+ directory under the worker's TMPDIR; only redacted files reach the skill's+ own report directory. An ``error`` field of None means the row has no error.+ """+ fields: dict = {"error": "Evaluation interrupted before complete evidence was retained."}+ data = None+ try:+ output.mkdir()+ # Raw scanner files and logs live only in a private temporary+ # directory, never in the upload directory. Docker mounts /audit-tmp+ # as tmpfs, so cancellation cannot upload half-written secrets. The+ # name carries the report directory so concurrent scans are traceable.+ with tempfile.TemporaryDirectory(+ prefix=f"skillevaluator-private-{output.name}-", dir=env.get("TMPDIR")+ ) as temporary:+ raw = Path(temporary).resolve(strict=True)+ if raw.is_relative_to(workspace) or raw.is_relative_to(output.parent):+ raise ValueError("A private scanner directory must be outside the workspace and reports.")+ argv = [+ "skillevaluator",+ "tier1",+ str(target),+ "--llm" if llm else "--no-llm",+ "--profile",+ "external",+ # native_reports renders each skill's HTML from the retained JSON+ # and replaces any CLI copy, so the CLI does not render it.+ "-r",+ "json,markdown,sarif",+ "-o",+ str(raw),+ ]+ with (raw / "stdout.log").open("xb") as stdout, (raw / "stderr.log").open("xb") as stderr:+ process = execute(+ argv, cwd=workspace, env=env, stdout=stdout, stderr=stderr, check=False, timeout=timeout+ )+ safe = sanitized_outputs(raw, tokens)+ for name, content in safe.items():+ (output / name).write_bytes(content)+ fields["exit_code"] = process.returncode+ data = _load_report(output / "skillevaluator-tier1.json")+ for key in (+ "overall_status",+ "incomplete_scans",+ "total_validators",+ "total_errors",+ "total_warnings",+ "severity_counts",+ ):+ fields[key] = data[key]+ fields["passed"] = (+ process.returncode == 0+ and data["overall_passed"] is True+ and data["overall_status"] == "passed"+ and not data["incomplete_scans"]+ and all(+ result.get("passed") is True+ and result.get("status") not in {"incomplete", "failed"}+ and not result.get("incomplete_scans")+ for result in data["results"]+ )+ )+ try:+ # The publisher rejects the whole artifact for one contradictory+ # report. Check it the same way here so only this skill is lost.+ validate_report(data, fields)+ consistent = True+ except PublicationError:+ consistent = False+ if not consistent or (not fields["passed"] and fields["overall_status"] == "passed"):+ fields["overall_status"] = "error"+ fields["incomplete_scans"] = []+ fields["passed"] = False+ fields["error"] = "Process exit or validator results contradict the report verdict."+ else:+ fields["error"] = None+ except subprocess.TimeoutExpired:+ fields["error"] = "Scan time limit reached; this skill did not finish."+ return fields, None, True+ except (OSError, ValueError, UnicodeError, RecursionError):+ fields["error"] = "Evaluator execution failed or its JSON report was missing, unreadable, or invalid."+ return fields, data, False+++def run(workspace: Path, reports: Path, environ: dict[str, str], *, execute=None, clock=time.monotonic) -> int:+ """Run all selected skills; return nonzero on every incomplete or failed run.++ ``execute`` is called like subprocess.run. The default, run_scan, also+ ends running scans when the run stops early on an interrupt or error.+ """if reports.is_symlink():raise ConfigurationError("The report directory must not be a symlink.")reports.mkdir(parents=True, exist_ok=True)@@ -316,6 +514,7 @@ def run(if reports.is_relative_to(workspace):raise ConfigurationError("The report directory must be outside the read-only workspace.")env, llm = child_environment(environ)+ workers = scan_workers(environ)mode = environ.get("SCAN_MODE", "all")if mode not in {"all", "changed"}:raise ConfigurationError("SCAN_MODE must be all or changed.")@@ -326,7 +525,7 @@ def run()if selected:# Validate scan trees without adding nested unchanged targets.- discover_skills(workspace, "\n".join(path.relative_to(workspace).as_posix() for path in selected))+ skill_trees(workspace, [path.relative_to(workspace).as_posix() for path in selected])else:selected = discover_skills(workspace, roots)selection = {@@ -361,6 +560,7 @@ def run(raise ConfigurationError("Credential preflight could not safely prepare the scan; no results were published.") from exc+ environments = [worker_environment(env, slot) for slot in range(1, min(workers, len(selected)) + 1)]except (ConfigurationError, SelectionError, OSError) as exc:message = (str(exc)@@ -404,93 +604,160 @@ def run(render_reports(reports, summary, context)checkpoint()- for index, (target, row) in enumerate(zip(selected, rows, strict=True), 1):- remaining = deadline - clock()- if remaining <= 0:- break- print(- f"SkillEvaluator: evaluating skill {index} of {len(selected)}: {json.dumps(target.relative_to(workspace).as_posix())}",- flush=True,- )- output = reports / f"skill-{index:03d}"- output.mkdir()- row["error"] = "Evaluation interrupted before complete evidence was retained."- data = None+ pending: queue.SimpleQueue[int] = queue.SimpleQueue()+ for index in range(len(rows)):+ pending.put(index)+ events: queue.SimpleQueue[tuple[str, int, tuple | None]] = queue.SimpleQueue()+ stop = threading.Event()+ # Set when the run stops early on an interrupt or error: running scans end.+ abort = threading.Event()+ if execute is None:+ execute = functools.partial(run_scan, abort=abort)++ def ending() -> bool:+ return stop.is_set() or abort.is_set()++ def work(worker_env: dict[str, str]) -> None:+ # Workers only run processes. The main thread owns rows, logs and+ # report files, so nothing races and logs keep skill order.try:- remaining = deadline - clock()- if remaining <= 0:- raise subprocess.TimeoutExpired("skillevaluator", SCAN_TIMEOUT_SECONDS)- # Raw scanner files and logs live only in a private temporary- # directory, never in the upload directory. Docker mounts /audit-tmp- # as tmpfs, so cancellation cannot upload half-written secrets.- with tempfile.TemporaryDirectory(prefix="skillevaluator-private-", dir=env.get("TMPDIR")) as temporary:- raw = Path(temporary).resolve(strict=True)- if raw.is_relative_to(workspace) or raw.is_relative_to(reports):- raise ValueError("A private scanner directory must be outside the workspace and reports.")- argv = [- "skillevaluator",- "tier1",- str(target),- "--llm" if llm else "--no-llm",- "--profile",- "external",- "-r",- "json,html,markdown,sarif",- "-o",- str(raw),- ]- with (raw / "stdout.log").open("xb") as stdout, (raw / "stderr.log").open("xb") as stderr:- process = execute(- argv, cwd=workspace, env=env, stdout=stdout, stderr=stderr, check=False, timeout=remaining+ while not ending():+ try:+ index = pending.get_nowait()+ except queue.Empty:+ return+ remaining = deadline - clock()+ if remaining <= 0:+ stop.set()+ return+ events.put(("started", index, None))+ try:+ outcome = evaluate_skill(+ selected[index],+ reports / rows[index]["report_dir"],+ workspace=workspace,+ env=worker_env,+ llm=llm,+ timeout=remaining,+ execute=execute,+ tokens=tokens,)- safe = sanitized_outputs(raw, tokens)- for name, content in safe.items():- (output / name).write_bytes(content)- row["exit_code"] = process.returncode- data = _load_report(output / "skillevaluator-tier1.json")- for key in (- "overall_status",- "incomplete_scans",- "total_validators",- "total_errors",- "total_warnings",- "severity_counts",- ):- row[key] = data[key]- row["passed"] = (- process.returncode == 0- and data["overall_passed"] is True- and data["overall_status"] == "passed"- and not data["incomplete_scans"]- and all(- result.get("passed") is True- and result.get("status") not in {"incomplete", "failed"}- and not result.get("incomplete_scans")- for result in data["results"]+ except Exception:+ outcome = ({"error": "Evaluator execution failed unexpectedly; no result is claimed."}, None, False)+ if outcome[2]:+ # The budget is spent: running scans end at the deadline.+ stop.set()+ saved = threading.Event()+ events.put(("finished", index, (outcome, saved)))+ # Start another skill only after this result is checkpointed.+ # Once the run is ending no skill starts, and an interrupted+ # run never checkpoints again, so stop waiting.+ while not saved.wait(POLL_SECONDS):+ if ending():+ return+ finally:+ events.put(("exited", -1, None))++ threads = [+ threading.Thread(target=work, args=(worker_env,), name=f"skill-scan-{slot}")+ for slot, worker_env in enumerate(environments, 1)+ ]+ if len(threads) > 1:+ print(f"SkillEvaluator: scanning {len(rows)} skills, up to {len(threads)} at a time.", flush=True)+ started: set[int] = set()+ announced: set[int] = set()+ results: dict[int, dict | None] = {}+ logged = 0++ def flush_logs(*, final: bool = False) -> None:+ # Print each skill's lines in skill order, whatever order scans end in,+ # so logs and the annotation budget match a sequential run.+ nonlocal logged+ while logged < len(rows):+ if logged in started:+ if logged not in announced:+ announced.add(logged)+ print(+ f"SkillEvaluator: evaluating skill {logged + 1} of {len(rows)}: {json.dumps(rows[logged]['path'])}",+ flush=True,+ )+ if logged not in results:+ return+ # Drop each report once it is logged; only unlogged ones stay in memory.+ log_result(+ rows[logged],+ results.pop(logged),+ annotations=environ.get("GITHUB_ACTIONS") == "true",+ budget=annotation_budget,)- )- try:- # The publisher rejects the whole artifact for one contradictory- # report. Check it the same way here so only this skill is lost.- validate_report(data, row)- consistent = True- except PublicationError:- consistent = False- if not consistent or (not row["passed"] and row["overall_status"] == "passed"):- row["overall_status"] = "error"- row["incomplete_scans"] = []- row["passed"] = False- row["error"] = "Process exit or validator results contradict the report verdict."- else:- row.pop("error", None)- except subprocess.TimeoutExpired:- row["error"] = "Scan time limit reached; this skill did not finish."- checkpoint()- break- except (OSError, ValueError, UnicodeError, RecursionError):- row["error"] = "Evaluator execution failed or its JSON report was missing, unreadable, or invalid."- log_result(row, data, annotations=environ.get("GITHUB_ACTIONS") == "true", budget=annotation_budget)- checkpoint()+ elif not final:+ return+ logged += 1++ running = 0+ live: list[threading.Thread] = []+ unsaved: list[threading.Event] = []+ try:+ for thread in threads:+ thread.start()+ live.append(thread)+ running += 1+ while running:+ batch = [events.get()]+ while True:+ try:+ batch.append(events.get_nowait())+ except queue.Empty:+ break+ for kind, index, payload in batch:+ if kind == "exited":+ running -= 1+ elif kind == "started":+ started.add(index)+ rows[index]["error"] = "Evaluation interrupted before complete evidence was retained."+ else:+ (fields, data, _), saved = payload+ for key, value in fields.items():+ if key == "error" and value is None:+ rows[index].pop("error", None)+ else:+ rows[index][key] = value+ results[index] = data+ unsaved.append(saved)+ flush_logs()+ if unsaved:+ # One checkpoint covers every result in the batch.+ checkpoint()+ for saved in unsaved:+ saved.set()+ unsaved.clear()+ except BaseException:+ # Ctrl-C or an error such as a full report disk: end running scans+ # now instead of waiting up to the whole scan budget for them.+ abort.set()+ raise+ finally:+ stop.set()+ for saved in unsaved:+ saved.set()+ # Workers exit within the stop grace once the run ends. A repeated+ # Ctrl-C must not leave them behind, so it is raised after they exit.+ interrupt = None+ for thread in live:+ while thread.is_alive():+ try:+ thread.join()+ except KeyboardInterrupt as exc:+ abort.set()+ interrupt = exc+ if interrupt is not None:+ raise interrupt+ flush_logs(final=True)+ if len(started) < len(rows):+ print(+ f"SkillEvaluator: scan time limit reached; {len(rows) - len(started)} skills were not started.",+ flush=True,+ )summary = _summary(rows)summary["selection"] = selectionsummary["execution"] = contextdiff --git a/ci/select_changed_skills.py b/ci/select_changed_skills.pyindex 2cab75c1..17148865 100644--- a/ci/select_changed_skills.py+++ b/ci/select_changed_skills.py@@ -21,6 +21,8 @@ MAX_GIT_BYTES = 16 * 1024 * 1024MAX_FILES = 100_000MAX_DISCOVERED_SKILLS = 10_000MAX_SELECTED_SKILLS = 100+MAX_ROOTS = 100+MAX_ROOTS_TEXT = 16384GIT_TIMEOUT = 30_SHA = re.compile(r"(?:[0-9a-fA-F]{40}|[0-9a-fA-F]{64})")@@ -33,6 +35,44 @@ class SelectionError(ValueError):self.selection = selection+def has_control_character(text: str) -> bool:+ """The one control-character rule for configured roots and Git paths, DEL included."""+ return any(ord(character) < 32 or ord(character) == 127 for character in text)+++def skill_root(workspace: Path, value: str, error: type[ValueError] = SelectionError) -> str:+ """Validate one workspace-relative skill root and return its normalized form.++ The Tier 1 runner and changed-skill selection share these rules. A root may+ be missing, since a pull request can delete it, but never a non-directory.+ """+ relative = PurePosixPath(value)+ if (+ relative.is_absolute()+ or ".." in relative.parts+ or "\\" in value+ or ":" in value+ or has_control_character(value)+ ):+ raise error("Skill roots must be relative directories without parent traversal.")+ current = workspace+ for part in relative.parts:+ current /= part+ if current.is_symlink():+ raise error("Symlinks are not supported in skill roots.")+ if current.exists() and not current.is_dir():+ raise error("A configured skill root is not a directory.")+ return relative.as_posix()+++def skill_roots(workspace: Path, configured: str, error: type[ValueError] = SelectionError) -> list[str]:+ """Parse SKILL_PATHS into 1 to 100 unique, sorted and validated roots."""+ values = [line.strip() for line in configured.splitlines() if line.strip()]+ if not values or len(values) > MAX_ROOTS or len(configured) > MAX_ROOTS_TEXT:+ raise error("SKILL_PATHS must specify between 1 and 100 bounded relative directories.")+ return sorted({skill_root(workspace, value, error) for value in values})++def _git(workspace: Path, *arguments: str) -> bytes:# Do not inherit GIT_DIR, GIT_CONFIG_COUNT, external diff commands or user# config. The checkout is read-only and optional index updates are disabled.@@ -102,40 +142,27 @@ def _paths(output: bytes) -> list[str]:or len(path.encode()) > 4096or PurePosixPath(path).is_absolute()or ".." in PurePosixPath(path).parts- or any(ord(character) < 32 or ord(character) == 127 for character in path)+ or has_control_character(path)):raise SelectionError("Git returned an unsupported or unsafe path.")return paths-def _roots(workspace: Path, configured: str) -> list[str]:- values = [line.strip() for line in configured.splitlines() if line.strip()]- if not values or len(values) > 100 or len(configured) > 16384:- raise SelectionError("SKILL_PATHS must specify between 1 and 100 bounded relative directories.")- roots = set()- for value in values:- relative = PurePosixPath(value)- if (- relative.is_absolute()- or ".." in relative.parts- or "\\" in value- or ":" in value- or any(ord(character) < 32 or ord(character) == 127 for character in value)- ):- raise SelectionError("Skill roots must be relative directories without parent traversal.")- current = workspace- for part in relative.parts:- current /= part- if current.is_symlink():- raise SelectionError("Symlinks are not supported in skill roots.")- if current.exists() and not current.is_dir():- raise SelectionError("A configured skill root is not a directory.")- roots.add(relative.as_posix())- return sorted(roots)+def _containing(paths: list[str]) -> set[str]:+ """Return every path plus each directory that contains one, including ".".--def _within(path: str, directory: str) -> bool:- return directory in {".", path} or path.startswith(directory + "/")+ ``directory in _containing(paths)`` answers "is any path inside directory"+ with one lookup instead of comparing every directory with every path.+ """+ found: set[str] = set()+ for path in paths:+ current = path+ # A known entry already has all of its ancestors in the set.+ while current not in found:+ found.add(current)+ slash = current.rfind("/")+ current = current[:slash] if slash > 0 else "."+ return founddef _manifests(paths: list[str]) -> set[str]:@@ -152,18 +179,24 @@ def select_changed_skills(workspace: Path, roots: str, base: str, head: str) ->The unique merge base defines the PR delta. No network fetch or full-scan fallback is performed."""workspace = workspace.resolve(strict=True)- configured = _roots(workspace, roots)+ configured = skill_roots(workspace, roots)if not isinstance(base, str) or not isinstance(head, str) or not _SHA.fullmatch(base) or not _SHA.fullmatch(head):raise SelectionError("Changed selection requires full DIFF_BASE_SHA and DIFF_HEAD_SHA commit IDs.")- top = _git(workspace, "rev-parse", "--show-toplevel").decode().strip()+ requested_base, requested_head = base.lower(), head.lower()+ # One process resolves everything. The trailing "--" makes Git reject any+ # argument that is not a revision instead of reading it as a path.+ resolved = _git(+ workspace, "rev-parse", "--show-toplevel", base + "^{commit}", head + "^{commit}", "HEAD", "--"+ ).decode()+ lines = resolved.split("\n")+ if len(lines) != 6 or lines[4:] != ["--", ""]:+ raise SelectionError("Git could not resolve the requested selection; verify checkout history and commits.")+ top, base, head, checked_out = lines[:4]if Path(top).resolve(strict=True) != workspace:raise SelectionError("The workspace must be the root of its Git checkout.")- requested_base, requested_head = base.lower(), head.lower()- base = _git(workspace, "rev-parse", "--verify", base + "^{commit}").decode().strip()- head = _git(workspace, "rev-parse", "--verify", head + "^{commit}").decode().strip()if base != requested_base or head != requested_head:raise SelectionError("DIFF_BASE_SHA and DIFF_HEAD_SHA must identify commits, not tag objects.")- if _git(workspace, "rev-parse", "HEAD").decode().strip() != head:+ if checked_out != head:raise SelectionError("The checked-out HEAD does not match DIFF_HEAD_SHA; no scan was substituted.")base_ref = basetry:@@ -185,8 +218,9 @@ def select_changed_skills(workspace: Path, roots: str, base: str, head: str) ->)before_files = _paths(_git(workspace, "ls-tree", "-r", "-z", "--name-only", base, "--", *configured))after_files = _paths(_git(workspace, "ls-tree", "-r", "-z", "--name-only", head, "--", *configured))+ before_tree = _containing(before_files)for root in configured:- if not (workspace / root).is_dir() and not any(_within(path, root) for path in before_files):+ if not (workspace / root).is_dir() and root not in before_tree:raise SelectionError("A configured skill root is missing from both the checkout and base revision.")before = _manifests(before_files)after = _manifests(after_files)@@ -209,8 +243,9 @@ def select_changed_skills(workspace: Path, roots: str, base: str, head: str) ->))removed = sorted(before - after)- deleted = [directory for directory in removed if not any(_within(path, directory) for path in after_files)]- selected = sorted(directory for directory in after if any(_within(path, directory) for path in changed))+ after_tree, changed_tree = _containing(after_files), _containing(changed)+ deleted = [directory for directory in removed if directory not in after_tree]+ selected = sorted(directory for directory in after if directory in changed_tree)selection = {"mode": "changed","base": base,diff --git a/docs/github-actions.mdx b/docs/github-actions.mdxindex 4f222d17..d562ed79 100644--- a/docs/github-actions.mdx+++ b/docs/github-actions.mdx@@ -55,7 +55,9 @@ shared files outside the configured roots are not inferred dependencies. Use`scan-mode: all` when those changes should trigger a full evaluation.Tier 2 and Tier 3 are not run. The 100-skill limit is a selection limit, not aruntime guarantee. Scans have a time budget and preserve completed results plus-explicit incomplete entries if it is exhausted.+explicit incomplete entries if it is exhausted. The workflow scans up to four+skills at once. Set `scan-workers` (1 to 16) to change this; `1` scans one skill+at a time. Results and logs keep the same skill order either way.## Read the result@@ -63,9 +65,13 @@ Open **Actions → Skill checks**. The summary shows the outcome, affected skilland a bounded set of actionable reasons and fixes. Download the`skill-evaluator-tier1-attempt-N` artifact for that run attempt (`N`) to get`index.html`, per-skill HTML and JSON,-detailed Markdown, SARIF and scanner logs. Failed and incomplete scans retain-their available reports; missing evidence is not a pass. Quality scores and the-pass/fail decision are separate.+detailed Markdown, SARIF and scanner logs. The smaller+`skill-evaluator-tier1-evidence-attempt-N` artifact holds only the JSON that+the optional report publisher reads. Failed and incomplete scans retain+their available reports; missing evidence is not a pass. A quality score below+the minimum fails the Quality score check, so it blocks the verdict. The HTML+index and reports use the verdicts Passed, Changes required, Incomplete and+Skipped.Known credential values detected by the PII patterns, NVIDIA API keys, andunquoted `KEY=value` and `Bearer` credentials are redacted from CI reports,diff --git a/docs/reports.mdx b/docs/reports.mdxindex b415794c..e138a0b2 100644--- a/docs/reports.mdx+++ b/docs/reports.mdx@@ -85,6 +85,41 @@ timestamp is `YYYYMMDDHHMMSS`, so`-r json` produces something like `skillevaluator-output-20260709141530.json`and never clobbers the previous run.+## HTML report++The HTML report is one self-contained file. It opens on the answer and puts+the evidence below it:++- **Header.** The skill name is the page title. The header shows one verdict,+ the date and version, and **Download JSON**.+- **Summary.** The verdict with a one-line reason, four numbers (checks passed,+ findings, blocking critical and high findings, quality score with its+ minimum), and **Fix first**: up to five places to fix, most severe first,+ each linking to its evidence. When a required scanner did not finish, it is+ listed first with the scanner's own reason, such as a timeout. Two scanners+ reporting the same file and line take one entry.+- **Findings and evidence.** Every check with its findings and suggested fixes.+ Failed checks come first. Checks that report a raw ID, such as+ `SCRIPT_LINT`, show a plain name with the ID beside it. Scanner diagnostics+ and gate failures are always shown and printed; raw scanner lines that only+ repeat a finding are folded away.+- **Run details.** Scope, the target path, provenance and the gate policy.++A finding is one problem a check reports at one location, with a severity. A+finding in a failed check blocks the verdict; a finding in a passed check is+advisory. A quality score below the configured minimum fails the Quality score+check, so it also blocks the verdict.++The HTML report, the CI index and the Pages history use the same verdict+words: **Passed**, **Changes required**, **Incomplete**, **Skipped** and+**Neutral**. When no check ran, the verdict is Skipped, not Passed. A tier+whose gate does not block, such as deduplication with `--no-block-on-dedup` or+live evaluation without `--block-on-agent-eval`, shows **Failed (advisory)**+when it fails, because it does not require changes. Publication eligibility is+shown next to the verdict, because a report can pass its checks and still not+be eligible for publication. The report follows the system light or dark theme+until you pick one, and always prints dark text on white paper.+## BENCHMARK.mdWhen the target is a skill, `validate` always writes `BENCHMARK.md` to thediff --git a/src/skillevaluator/reporting/base.py b/src/skillevaluator/reporting/base.pyindex 42322d72..a22017ef 100644--- a/src/skillevaluator/reporting/base.py+++ b/src/skillevaluator/reporting/base.py@@ -1330,15 +1330,13 @@ def _agent_eval_publication_aggregates_consistent(raw_agents_run: list[list[str]] = []for container in (payload, summary):agents_run = container.get("agents_run")- if (- not isinstance(agents_run, list)- or any(not isinstance(name, str) for name in agents_run)- ):+ if not isinstance(agents_run, list) or any(not isinstance(name, str) for name in agents_run):return Falseraw_agents_run.append(agents_run)- if raw_agents_run[0] != raw_agents_run[1] or [- publication_semantic_text(name).strip() for name in raw_agents_run[0]- ] != expected_agents:+ if (+ raw_agents_run[0] != raw_agents_run[1]+ or [publication_semantic_text(name).strip() for name in raw_agents_run[0]] != expected_agents+ ):return Falsebest_agent = agents[best_name]@@ -2437,12 +2435,8 @@ def passes_required_gate(result: ValidationResult) -> bool:return bool(result.passed or is_advisory_agent_eval_skip(result))-def additional_errors(result: ValidationResult) -> list[str]:- """Return errors not already displayed through a structured finding.-- Findings also populate the legacy error list. Keep independent errors,- including execution diagnostics, without repeating those mirrored entries.- """+def _finding_legacy_lines(result: ValidationResult) -> set[str]:+ """Return the legacy error and warning lines that only repeat a structured finding."""represented = {finding.to_legacy_string() for finding in result.findings}for finding in result.findings:# merge_with_prefix puts skill labels before legacy messages but inside@@ -2453,9 +2447,29 @@ def additional_errors(result: ValidationResult) -> list[str]:label, _, location = location.partition("] ")prefix += label + "] "represented.add(f"{prefix}{finding.tag} {finding.message} in {location}")+ return represented+++def additional_errors(result: ValidationResult) -> list[str]:+ """Return errors not already displayed through a structured finding.++ Findings also populate the legacy error list. Keep independent errors,+ including execution diagnostics, without repeating those mirrored entries.+ """+ represented = _finding_legacy_lines(result)return [error for error in result.errors if error not in represented]+def additional_warnings(result: ValidationResult) -> list[str]:+ """Return warnings not already displayed through a structured finding.++ Medium and low findings also populate the legacy warning list. Some+ scanners report why they did not run as a warning, so keep those lines.+ """+ represented = _finding_legacy_lines(result)+ return [warning for warning in result.warnings if warning not in represented]++class ReporterBase(ABC):"""Abstract base class for validation result reporters.diff --git a/src/skillevaluator/reporting/html.py b/src/skillevaluator/reporting/html.pyindex a96da6b3..7c7fd018 100644--- a/src/skillevaluator/reporting/html.py+++ b/src/skillevaluator/reporting/html.py@@ -31,6 +31,7 @@ from collections.abc import Iteratorfrom dataclasses import dataclassfrom datetime import UTC, datetimefrom decimal import Decimal, InvalidOperation+from functools import cachefrom importlib import resourcesfrom pathlib import Pathfrom typing import TYPE_CHECKING, Any, ClassVar@@ -39,12 +40,13 @@ from urllib.parse import urlparsefrom jinja2 import BaseLoader, Environmentfrom skillevaluator import __version__-from skillevaluator.constants import TIER3_LIFT_FAIL_THRESHOLD, TIER3_LIFT_PASS_THRESHOLD+from skillevaluator.constants import QUALITY_GRADE_THRESHOLDS, TIER3_LIFT_FAIL_THRESHOLD, TIER3_LIFT_PASS_THRESHOLDfrom skillevaluator.publication_evidence import result_publication_evidence_dictfrom skillevaluator.reporting.base import (AGENT_EVAL_EXACT_TEXT_KEYS,ReporterBase,additional_errors,+ additional_warnings,agent_eval_publication_evidence_complete,agent_eval_report_serialization_issue,agent_eval_report_serialization_limits,@@ -1261,6 +1263,194 @@ def _adaptive_interval_percent(interval: object) -> str:return f"{_bound(lower)}\N{EN DASH}{_bound(upper)}"+# Plain-language names for checks that report a raw upper-case ID. The report+# shows the plain name with the raw ID beside it.+_PLAIN_CHECK_NAMES = {+ "QUALITY": "Quality score",+ "SCRIPT_LINT": "Script lint",+ "SCHEMA": "Schema",+ "RUBRIC_EVAL": "Rubric evaluation",+ "AGENT_EVAL": "Live agent evaluation",+ "DEDUPLICATION": "Duplicate detection",+}+_RAW_CHECK_ID = re.compile(r"[A-Z][A-Z0-9_]*")+# One verdict vocabulary for every surface: the HTML report, the CI index and+# the Pages history. Keys are the report's verdict keys.+VERDICT_WORDS = {+ "pass": "Passed",+ "fail": "Changes required",+ "incomplete": "Incomplete",+ "skipped": "Skipped",+ "neutral": "Neutral",+}+PUBLICATION_WORDS = {"pass": "Eligible", "fail": "Not eligible", "incomplete": "Incomplete", "neutral": "Neutral"}+# A tier whose gate does not block can fail without requiring changes.+ADVISORY_FAIL_WORD = "Failed (advisory)"+# "A 90+, B 80+, C 70+, D 60+, F below 60", from the thresholds the grades use.+QUALITY_GRADE_LEGEND = ", ".join(+ f"{grade} {threshold}+" if threshold else f"{grade} below {QUALITY_GRADE_THRESHOLDS[index - 1][0]}"+ for index, (threshold, grade) in enumerate(QUALITY_GRADE_THRESHOLDS)+)+_SEVERITY_RANK = {"critical": 4, "high": 3, "medium": 2, "low": 1, "info": 0}+FIX_FIRST_LIMIT = 5+# How the tool runner and the scanners say that a scanner itself is missing.+_MISSING_SCANNER = re.compile(r"\bnot installed\b|\bnot found in PATH\b", re.IGNORECASE)+++def _scan_stop_reason(errors: list[str], warnings: list[str]) -> str:+ """Return the diagnostic that says why a scan did not finish, or "".++ *errors* and *warnings* are the lines that do not repeat a finding.+ Errors come first. A line saying a scanner is missing is used only when+ nothing else explains the gap, because a scanner that ran and failed+ needs a different fix.+ """+ lines = [" ".join(str(line).split()) for line in [*errors, *warnings]]+ lines = [line for line in lines if line]+ return next((line for line in lines if not _MISSING_SCANNER.search(line)), lines[0] if lines else "")+++def _sentence(text: str, limit: int = 200) -> str:+ """Bound *text* to *limit* characters and end it as a sentence."""+ if len(text) > limit:+ text = text[: limit - 1].rstrip() + "\N{HORIZONTAL ELLIPSIS}"+ return text if text.endswith((".", "!", "?", "\N{HORIZONTAL ELLIPSIS}")) else f"{text}."+++def plain_check_name(name: object) -> str:+ """Return a readable name for a check that reports a raw ID such as ``SCRIPT_LINT``."""+ text = str(name or "")+ if text in _PLAIN_CHECK_NAMES:+ return _PLAIN_CHECK_NAMES[text]+ if _RAW_CHECK_ID.fullmatch(text):+ return text.replace("_", " ").capitalize()+ return text+++def _skill_relative_path(path: object, skill_name: object, scan_root: str | None = None) -> str:+ """Show a location relative to the skill folder instead of the machine path.++ *scan_root* is the folder the scanners ran on, when it is known. Without+ it, the last part of the skill name marks the skill folder, which cannot+ work for a skill at the repository root.+ """+ text = str(path or "")+ root = (scan_root or "").rstrip("/")+ if root and text.startswith(f"{root}/"):+ return text[len(root) + 1 :]+ base = str(skill_name or "").rstrip("/").rsplit("/", 1)[-1]+ marker = f"/{base}/" if base else ""+ if marker and marker in text:+ return text.split(marker, 1)[1]+ return text+++def _anchor_slug(*parts: object) -> str:+ slug = re.sub(r"[^a-z0-9]+", "-", " ".join(str(part) for part in parts).lower()).strip("-")+ return slug or "item"+++def _evidence_anchors(+ skills: dict[str, dict[str, Any]],+) -> tuple[dict[str, dict[str, list[str]]], dict[str, dict[str, str]]]:+ """Give every check and finding card in the skill view a stable, unique id.++ Returns ``(findings, checks)``: ``findings[skill][check]`` lists one id per+ finding card in template order, and ``checks[skill][check]`` is the id of+ the check itself. Ids are readable slugs, so a link to a finding survives+ a re-render of the same evidence.+ """+ used: set[str] = set()++ def unique(base: str) -> str:+ candidate, suffix = base, 2+ while candidate in used:+ candidate, suffix = f"{base}-{suffix}", suffix + 1+ used.add(candidate)+ return candidate++ findings: dict[str, dict[str, list[str]]] = {}+ checks: dict[str, dict[str, str]] = {}+ for skill_name in sorted(skills):+ for check_name in sorted(skills[skill_name].get("validators", {})):+ check = skills[skill_name]["validators"][check_name]+ base = _anchor_slug(skill_name, check_name)+ checks.setdefault(skill_name, {})[check_name] = unique(f"c-{base}")+ findings.setdefault(skill_name, {})[check_name] = [+ unique(f"f-{base}-{index}") for index in range(1, len(check.get("findings") or []) + 1)+ ]+ return findings, checks+++def _fix_first(+ skills: dict[str, dict[str, Any]],+ finding_anchors: dict[str, dict[str, list[str]]],+ check_anchors: dict[str, dict[str, str]],+ *,+ limit: int = FIX_FIRST_LIMIT,+ scan_root: str | None = None,+) -> list[dict[str, Any]]:+ """Pick the few places to fix first, most severe first, one entry per location.++ A check that could not finish comes first, with the scanner's own reason,+ because no pass can be claimed until it runs. Findings then rank by severity, then blocking before+ advisory, then occurrences. A failed check with no located finding is+ listed as a high, blocking item so its reason is not lost. Two findings at+ the same file and line (for example one secret reported by two scanners)+ take one slot.+ """+ candidates: list[tuple[tuple[int, ...], tuple, dict[str, Any]]] = []+ for skill_name in sorted(skills):+ for check_name in sorted(skills[skill_name].get("validators", {})):+ check = skills[skill_name]["validators"][check_name]+ label = plain_check_name(check_name)+ status = check.get("status") or ("pass" if check.get("passed") else "fail")+ findings = check.get("findings") or []+ blocking = not check.get("passed", True)+ item = {"check": label, "skill": skill_name, "location": "", "blocking": True}+ check_key = (skill_name, "check", check_name)+ if status == "incomplete":+ scans = ", ".join(str(scan) for scan in check.get("incomplete_scans") or [])+ reason = _scan_stop_reason(check.get("additional_errors") or [], check.get("additional_warnings") or [])+ message = f"{label} did not finish" + (f": {reason}" if reason else f" ({scans})" if scans else "")+ item |= {"kind": "incomplete", "severity": "incomplete", "message": message}+ candidates.append(((2, 0, 1, 0), check_key, item | {"anchor": check_anchors[skill_name][check_name]}))+ elif status == "fail" and not findings:+ errors = [str(error) for error in check.get("errors") or [] if str(error).strip()]+ message = errors[0] if errors else f"{label} failed without a located finding"+ item |= {"kind": "check", "severity": "high", "message": message}+ rank = (1, _SEVERITY_RANK["high"], 1, 1)+ candidates.append((rank, check_key, item | {"anchor": check_anchors[skill_name][check_name]}))+ for index, finding in enumerate(findings):+ severity = str(finding.get("severity") or "medium").lower()+ location = _skill_relative_path(finding.get("file_path"), skill_name, scan_root)+ if location and finding.get("line_number"):+ location = f"{location}:{finding['line_number']}"+ occurrences = finding.get("occurrences", 1)+ occurrences = min(occurrences, 9999) if isinstance(occurrences, int) else 1+ rank = (1, _SEVERITY_RANK.get(severity, _SEVERITY_RANK["medium"]), int(blocking), occurrences)+ key = (skill_name, "file", location) if location else (skill_name, check_name, finding.get("message"))+ entry = {+ "kind": "finding",+ "severity": severity,+ "message": str(finding.get("message") or ""),+ "check": label,+ "skill": skill_name,+ "location": location,+ "blocking": blocking,+ "anchor": finding_anchors[skill_name][check_name][index],+ }+ candidates.append((rank, key, entry))+ items: list[dict[str, Any]] = []+ seen: set[tuple] = set()+ # sorted() is stable, so ties keep skill, check and evidence order.+ for _rank, key, entry in sorted(candidates, key=lambda candidate: candidate[0], reverse=True):+ if key not in seen:+ seen.add(key)+ items.append(entry)+ return items[:limit]++class PackageLoader(BaseLoader):"""Custom Jinja2 loader that loads templates from package resources."""@@ -1355,6 +1545,7 @@ class HTMLReporter(ReporterBase):expected_skill_name: str | None = None,explicit_skill_name: str | None = None,ci_context: dict[str, str] | None = None,+ scan_root: str | None = None,) -> None:self.include_timestamp = include_timestampself.title = title or "SkillEvaluator Validation Report"@@ -1369,9 +1560,21 @@ class HTMLReporter(ReporterBase):self.expected_skill_name = expected_skill_nameself.explicit_skill_name = explicit_skill_nameself.ci_context = _validated_ci_context(ci_context)+ # The folder the scanners ran on, used to show finding locations+ # relative to the skill in "Fix first".+ self.scan_root = scan_rootself._env = self._create_environment()- def _create_environment(self) -> Environment:+ @staticmethod+ @cache+ def _create_environment() -> Environment:+ """Return one shared environment, so each template compiles once per process.++ Compiling report.html.j2 costs far more than rendering it. The loader+ serves fixed package templates and the filters are pure functions, so+ reporters with different options can share it. Jinja environments are+ safe to share between threads once they are configured.+ """loader = PackageLoader("skillevaluator.reporting", "templates")environment = Environment(loader=loader, autoescape=True)environment.filters["cleanly_skipped"] = is_cleanly_skipped@@ -1467,6 +1670,9 @@ class HTMLReporter(ReporterBase):"status": display_status,"errors": list(result.errors),"warnings": list(result.warnings),+ # The lines that do not repeat a finding: scanner diagnostics and gate failures.+ "additional_errors": additional_errors(result),+ "additional_warnings": additional_warnings(result),"incomplete_scans": list(result.incomplete_scans),"description": result.validator_description,"details": [],@@ -1782,7 +1988,8 @@ class HTMLReporter(ReporterBase):``issue_key`` (matching the row it rolls up to) so the template candeep-link from the executive summary directly to the finding card.- Returns a list of dicts sorted by total_count descending.+ Returns a list of dicts, most severe first; the occurrence count only+ breaks ties within a severity."""sev_order = {"critical": 4, "high": 3, "medium": 2, "low": 1, "info": 0}issue_map: dict[tuple, dict[str, Any]] = {}@@ -1821,7 +2028,11 @@ class HTMLReporter(ReporterBase):"issue_key": issue_key,}- return sorted(issue_map.values(), key=lambda x: x["total_count"], reverse=True)+ return sorted(+ issue_map.values(),+ key=lambda issue: (sev_order.get(issue["severity"], 0), issue["total_count"]),+ reverse=True,+ )@staticmethoddef _extract_contributors(@@ -2663,6 +2874,17 @@ class HTMLReporter(ReporterBase):advisory_high = advisory_totals["high"]advisory_medium = advisory_totals["medium"]advisory_low = advisory_totals["low"]+ # A tier whose every result is explicitly non-blocking is advisory: its+ # failure does not require changes, so the report labels it "Failed (advisory)".+ nonblocking_tiers = [+ tier_name+ for tier_name, tier_results in (("tier2", tier2_results), ("tier3", tier3_results))+ if tier_results+ and all(+ isinstance(result.metadata.get("gating"), dict) and result.metadata["gating"].get("blocking") is False+ for result in tier_results+ )+ ]gating = {"blocking_tiers": blocking_tiers,"advisory_tiers": advisory_tiers,@@ -2710,6 +2932,53 @@ class HTMLReporter(ReporterBase):_attach_quality_scores(skills_by_name)_attach_quality_scores(tier1_skills)+ # The quality gate's minimum, shown next to every score it applies to.+ quality_min_score = next(+ (+ r.metadata["quality_config"]["min_score"]+ for r in results+ if isinstance(r.metadata, dict)+ and isinstance(r.metadata.get("quality_config"), dict)+ and isinstance(r.metadata["quality_config"].get("min_score"), int | float)+ ),+ None,+ )+ check_names = {+ r.validator_name: plain_check_name(r.validator_name)+ for r in results+ if plain_check_name(r.validator_name) != r.validator_name+ }+ tier1_finding_anchors, tier1_check_anchors = _evidence_anchors(tier1_skills)+ tier1_fix_first = _fix_first(tier1_skills, tier1_finding_anchors, tier1_check_anchors, scan_root=self.scan_root)+ tier1_failed_checks = [+ plain_check_name(r.validator_name) for r in tier1_results if _result_display_status(r) == "fail"+ ]+ tier1_blocking_findings = sum(+ 1+ for r in tier1_results+ if not passes_required_gate(r)+ for f in r.findings+ if (f.severity.value if hasattr(f.severity, "value") else str(f.severity).lower()) in {"critical", "high"}+ )+ # Why each incomplete Tier 1 check stopped, in the scanner's words. The+ # summary asks for an install only when every scanner is just missing.+ tier1_stop_reasons = [+ (plain_check_name(r.validator_name), _scan_stop_reason(additional_errors(r), additional_warnings(r)))+ for r in tier1_results+ if r.is_incomplete+ ]+ tier1_scanners_missing = bool(tier1_stop_reasons) and all(+ _MISSING_SCANNER.search(reason) for _check, reason in tier1_stop_reasons+ )+ tier1_scan_stop = next(+ (+ f"{check}: {_sentence(reason)}"+ for check, reason in tier1_stop_reasons+ if reason and not _MISSING_SCANNER.search(reason)+ ),+ "",+ )+report_data = {"title": self.title,"timestamp": timestamp,@@ -2755,12 +3024,16 @@ class HTMLReporter(ReporterBase):},"skillspector_versions": tier1_skillspector_versions,}- report_json = (- json.dumps(report_data, indent=2, allow_nan=False)- .replace("<", "\\u003c")- .replace(">", "\\u003e")- .replace("&", "\\u0026")- )+ # CI-native pages link Download JSON to the retained canonical file, so+ # nothing on them reads an embedded copy. Only standalone reports embed it.+ report_json = None+ if not self.ci_context.get("canonical_json_url"):+ report_json = (+ json.dumps(report_data, indent=2, allow_nan=False)+ .replace("<", "\\u003c")+ .replace(">", "\\u003e")+ .replace("&", "\\u0026")+ )# Dynamically add Tier 3 tab when agent eval data is presentactive_tabs = list(self.tabs)@@ -2853,7 +3126,55 @@ class HTMLReporter(ReporterBase):tier2_results=tier2_results,tier3_lift_pass_threshold=TIER3_LIFT_PASS_THRESHOLD,tier3_lift_fail_threshold=TIER3_LIFT_FAIL_THRESHOLD,+ quality_min_score=quality_min_score,+ quality_grade_legend=QUALITY_GRADE_LEGEND,+ verdict_words=VERDICT_WORDS,+ advisory_fail_word=ADVISORY_FAIL_WORD,+ nonblocking_tiers=nonblocking_tiers,+ publication_words=PUBLICATION_WORDS,+ tier3_status_reason=publication.tier3.reason,+ tier1_failed_checks=tier1_failed_checks,+ tier1_incomplete_scans=list(dict.fromkeys(tool for r in tier1_results for tool in r.incomplete_scans)),+ tier1_scanners_missing=tier1_scanners_missing,+ tier1_scan_stop=tier1_scan_stop,+ tier1_blocking_findings=tier1_blocking_findings,+ check_names=check_names,+ finding_anchors=tier1_finding_anchors,+ check_anchors=tier1_check_anchors,+ fix_first=tier1_fix_first,)def get_file_extension(self) -> str:return ".html"+++def fix_first_for_results(+ results: list[ValidationResult],+ *,+ skill_name: str | None = None,+ expected_skill_name: str | None = None,+ scan_root: str | None = None,+) -> list[dict[str, Any]]:+ """Return the Tier 1 "Fix first" entries exactly as the report for *results* lists them.++ The CI index uses this so its "Needs attention" line names the same first+ fix as the skill's own report, with the same anchor. Pass the same+ *scan_root* as the report.+ """+ reporter = HTMLReporter(+ include_timestamp=False, explicit_skill_name=skill_name, expected_skill_name=expected_skill_name+ )+ tier1_results, _tier2, _tier3 = reporter._split_results_by_tier(results)+ skills = reporter._reorganize_by_skill(tier1_results)+ return _fix_first(skills, *_evidence_anchors(skills), scan_root=scan_root)+++def render_logo(uid: str, size: int = 28, label: str = "SkillEvaluator logo") -> str:+ """Return the inline SkillEvaluator logo for pages built outside report.html.j2.++ ``uid`` keeps the logo's ids unique on the page. An empty ``label`` marks+ the logo as decorative, for use next to visible "SkillEvaluator" text.+ """+ # The shared reporter environment compiles the partial once per process.+ environment = HTMLReporter._create_environment()+ return str(environment.get_template("logo.svg.j2").module.se_logo(uid, size, label))diff --git a/src/skillevaluator/reporting/retained.py b/src/skillevaluator/reporting/retained.pyindex 28a57072..898a8a62 100644--- a/src/skillevaluator/reporting/retained.py+++ b/src/skillevaluator/reporting/retained.py@@ -10,6 +10,7 @@ from __future__ import annotationsimport copyimport mathfrom dataclasses import fields+from functools import partialfrom typing import Anyfrom skillevaluator.models import Finding, SuccessDetail, ValidationResult, ValidationSummary@@ -52,9 +53,9 @@ def _text(value: Any, field: str, *, optional: bool = False) -> Any:return value-def _mapping(value: Any, field: str) -> dict:+def _mapping(value: Any, field: str, *, copy_value: bool = True) -> dict:_require(isinstance(value, dict), f"Invalid {field}: expected object")- return copy.deepcopy(value)+ return copy.deepcopy(value) if copy_value else valuedef _strings(value: Any, field: str) -> list[str]:@@ -112,10 +113,16 @@ def _typed_metadata(metadata: dict) -> None:_require(all(type(value) is bool for value in benchmark.values()), "Invalid benchmark policy")-def retained_results(payload: dict) -> list[ValidationResult]:- """Rehydrate the JSONReporter's Tier 1 contract using explicit field maps."""+def retained_results(payload: dict, *, copy_metadata: bool = True) -> list[ValidationResult]:+ """Rehydrate the JSONReporter's Tier 1 contract using explicit field maps.++ Results own deep copies of the payload's metadata by default. A caller that+ only validates and discards the results can pass copy_metadata=False; the+ results then share the payload's mappings, which neither side modifies.+ """_require(isinstance(payload, dict), "Report must be an object")_json(payload)+ mapping = partial(_mapping, copy_value=copy_metadata)rows = payload.get("results")_require(isinstance(rows, list) and 0 < len(rows) <= 1000, "Report requires nonempty results")results = []@@ -137,7 +144,7 @@ def retained_results(payload: dict) -> list[ValidationResult]:_require(bool(name) and name not in seen, "Validator names must be nonempty and unique")seen.add(name)_require(type(row.get("passed")) is bool, "Result passed must be boolean")- summary = _mapping(row.get("summary"), "summary")+ summary = mapping(row.get("summary"), "summary")for field in fields(ValidationSummary):_require(type(summary.get(field.name)) is int and summary[field.name] >= 0, f"Invalid summary.{field.name}")_require(set(summary) == {f.name for f in fields(ValidationSummary)}, "Unexpected summary fields")@@ -159,7 +166,7 @@ def retained_results(payload: dict) -> list[ValidationResult]:line_number=line,line_content=_text(item.get("line_content"), "line_content", optional=True),suggestion=_text(item.get("suggestion"), "suggestion", optional=True),- metadata=_mapping(+ metadata=mapping(item.get("metadata") if item.get("metadata") is not None else {}, "finding.metadata"),)@@ -172,16 +179,16 @@ def retained_results(payload: dict) -> list[ValidationResult]:SuccessDetail(check_name=_text(item.get("check"), "success.check"),message=_text(item.get("message"), "success.message"),- metadata=_mapping(+ metadata=mapping(item.get("metadata") if item.get("metadata") is not None else {}, "success.metadata"),))- legacy = _mapping(row.get("legacy"), "legacy")+ legacy = mapping(row.get("legacy"), "legacy")metadata = {}for source, dest in mappings.items():if source in row:- metadata[dest] = _mapping(row[source], source)+ metadata[dest] = mapping(row[source], source)if "skillspector_version" in row:metadata["skillspector_version"] = _text(row["skillspector_version"], "skillspector_version")if "publication_target_conflict" in row:@@ -190,10 +197,10 @@ def retained_results(payload: dict) -> list[ValidationResult]:)for key in ("policy", "quality_config"):if key not in metadata and payload.get(key) is not None:- metadata[key] = _mapping(payload[key], key)+ metadata[key] = mapping(payload[key], key)# Retain source identity under the exact carrier consumed by reporters.if payload.get("evaluated_source") is not None:- metadata["evaluated_source"] = _mapping(payload["evaluated_source"], "evaluated_source")+ metadata["evaluated_source"] = mapping(payload["evaluated_source"], "evaluated_source")for key in ("incomplete_scans", "gate_failures"):if key in row:metadata[key] = _strings(row[key], key)@@ -203,7 +210,8 @@ def retained_results(payload: dict) -> list[ValidationResult]:and all(isinstance(x, dict) for x in row["dependency_coverage"]),"Invalid dependency_coverage",)- metadata["dependency_coverage"] = copy.deepcopy(row["dependency_coverage"])+ coverage = row["dependency_coverage"]+ metadata["dependency_coverage"] = copy.deepcopy(coverage) if copy_metadata else coverageif "skipped" in row:_require(type(row["skipped"]) is bool, "skipped must be boolean")metadata["skipped"] = row["skipped"]@@ -262,9 +270,18 @@ def retained_results(payload: dict) -> list[ValidationResult]:def render_retained_report(- payload: dict, *, skill_name: str, expected_skill_name: str | None = None, context: dict | None = None+ payload: dict,+ *,+ skill_name: str,+ expected_skill_name: str | None = None,+ context: dict | None = None,+ scan_root: str | None = None,) -> str:- """Render a single skill from retained evidence without inspecting its paths."""+ """Render a single skill from retained evidence without inspecting its paths.++ *scan_root* is the folder the scanners ran on. It is only compared with+ recorded finding paths, never opened.+ """_require(isinstance(skill_name, str)and 0 < len(skill_name.encode("utf-8")) <= 4096@@ -284,4 +301,5 @@ def render_retained_report(expected_skill_name=canonical_name,explicit_skill_name=skill_name,ci_context=context,+ scan_root=scan_root,).render_all(results)diff --git a/src/skillevaluator/reporting/templates/logo.svg.j2 b/src/skillevaluator/reporting/templates/logo.svg.j2new file mode 100644index 00000000..10d45e6e--- /dev/null+++ b/src/skillevaluator/reporting/templates/logo.svg.j2@@ -0,0 +1,30 @@+{#- SPDX-License-Identifier: Apache-2.0 -#}+{#-+ SkillEvaluator logo, from docs/assets/skillevaluator-logo.svg.++ The logo is inline SVG so it ships in the wheel and needs no image source:+ the CI pages' Content-Security-Policy does not allow image files.++ Colors come from the page's --se-logo-* theme variables instead of a+ prefers-color-scheme query, so the logo follows the report's theme toggle.+ The fallbacks are the light colors, for pages that define no variables.++ ``uid`` prefixes the <title> and <desc> ids, so a page can show the logo+ more than once and keep its ids unique. Pass ``label=''`` when the logo+ sits next to visible "SkillEvaluator" text: it then has no ids and is+ hidden from assistive technology, so the name is not read twice.+-#}+{%- macro se_logo(uid, size=28, label='SkillEvaluator logo') -%}+<svg class="se-logo" xmlns="http://www.w3.org/2000/svg" width="{{ size }}" height="{{ size }}" viewBox="10 10 108 108" focusable="false"+{%- if label %} role="img" aria-labelledby="{{ uid }}-title {{ uid }}-desc"{% else %} aria-hidden="true"{% endif %}>+{%- if label %}<title id="{{ uid }}-title">{{ label }}</title><desc id="{{ uid }}-desc">Three concentric evaluation levels surrounding a verified core.</desc>{% endif -%}+<rect x="10.5" y="10.5" width="107" height="107" rx="53.5" stroke-width="1" style="fill:var(--se-logo-panel,#f2f7ec);stroke:var(--se-logo-panel-edge,#d7e5cc)"/>+<circle cx="64" cy="64" r="42" fill="none" stroke-width="5" style="stroke:var(--se-logo-ring-outer,#b8d3a5)"/>+<circle cx="64" cy="64" r="30" fill="none" stroke-width="5" style="stroke:var(--se-logo-ring-middle,#83aa66)"/>+<circle cx="64" cy="64" r="18" fill="none" stroke-width="5" style="stroke:var(--se-logo-ring-inner,#315b25)"/>+<circle cx="64" cy="64" r="12" stroke-width="1.8" style="fill:var(--se-logo-core,#76b900);stroke:var(--se-logo-core-edge,#3e6500)"/>+<path d="m57.5 64 4.4 4.6 8.8-9.3" fill="none" stroke-width="2.8" stroke-linecap="round" stroke-linejoin="round" style="stroke:var(--se-logo-check,#fff)"/>+<circle cx="97" cy="38" r="5" stroke-width="1.4" style="fill:var(--se-logo-core,#76b900);stroke:var(--se-logo-core-edge,#3e6500)"/>+<circle cx="34" cy="82" r="5" stroke-width="1.4" style="fill:var(--se-logo-core,#76b900);stroke:var(--se-logo-core-edge,#3e6500)"/>+</svg>+{%- endmacro -%}diff --git a/src/skillevaluator/reporting/templates/report.html.j2 b/src/skillevaluator/reporting/templates/report.html.j2index 0c0800b1..3b0fa1e2 100644--- a/src/skillevaluator/reporting/templates/report.html.j2+++ b/src/skillevaluator/reporting/templates/report.html.j2@@ -1,5 +1,5 @@<!DOCTYPE html>-<html lang="en">+{% from "logo.svg.j2" import se_logo %}<html lang="en"><head><meta charset="UTF-8"><meta name="viewport" content="width=device-width, initial-scale=1.0">@@ -41,6 +41,7 @@--badge-fail-bg: #fee2e2;--badge-fail-tx: #991b1b;}+ @media screen {[data-theme="dark"] {--primary-color: #0c111b;--secondary-color: #76b900;@@ -75,6 +76,7 @@--badge-fail-bg: #7f1d1d;--badge-fail-tx: #fecaca;}+ }* { box-sizing: border-box; margin: 0; padding: 0; }html { scroll-behavior: smooth; }body {@@ -188,8 +190,8 @@.header-status-badge.pass { background: rgba(255,255,255,0.95); color: var(--success-color); }.header-status-badge.fail { background: #fecaca; color: #991b1b; border: 1px solid #fca5a5; }.header-status-badge.warn { background: #fef3c7; color: #92400e; border: 1px solid #f59e0b; }- [data-theme="dark"] .header-status-badge.fail { background: #450a0a; color: #fca5a5; border-color: #991b1b; }- [data-theme="dark"] .header-status-badge.warn { background: #451a03; color: #fcd34d; border-color: #92400e; }+ @media screen { [data-theme="dark"] .header-status-badge.fail { background: #450a0a; color: #fca5a5; border-color: #991b1b; } }+ @media screen { [data-theme="dark"] .header-status-badge.warn { background: #451a03; color: #fcd34d; border-color: #92400e; } }.container { max-width: 1200px; margin: 0 auto; padding: 2rem; }.nav-tabs { display: flex; gap: 0.5rem; margin: 1.5rem 0; flex-wrap: wrap; }.nav-tab {@@ -204,7 +206,7 @@transition: all 0.2s ease;}.nav-tab:hover { border-color: var(--secondary-color); color: var(--secondary-color); }- .nav-tab.active { background: var(--secondary-color); color: white; border-color: var(--secondary-color); }+ .nav-tab.active { background: var(--secondary-color); color: var(--on-accent); border-color: var(--secondary-color); }.tab-content { display: none; }.tab-content.active { display: block; }.dashboard {@@ -259,9 +261,9 @@.status-badge.pass { background: #dcfce7; color: #166534; }.status-badge.fail { background: #fee2e2; color: #991b1b; }.status-badge.warn { background: #fef3c7; color: #92400e; }- [data-theme="dark"] .status-badge.pass { background: #14532d; color: #86efac; }- [data-theme="dark"] .status-badge.fail { background: #450a0a; color: #fca5a5; }- [data-theme="dark"] .status-badge.warn { background: #451a03; color: #fcd34d; }+ @media screen { [data-theme="dark"] .status-badge.pass { background: #14532d; color: #86efac; } }+ @media screen { [data-theme="dark"] .status-badge.fail { background: #450a0a; color: #fca5a5; } }+ @media screen { [data-theme="dark"] .status-badge.warn { background: #451a03; color: #fcd34d; } }.section-header {color: var(--dark-text);font-weight: 600;@@ -287,7 +289,7 @@transition: all 0.2s ease;}.filter-btn:hover { border-color: var(--secondary-color); }- .filter-btn.active { background: var(--secondary-color); color: white; border-color: var(--secondary-color); }+ .filter-btn.active { background: var(--secondary-color); color: var(--on-accent); border-color: var(--secondary-color); }.filter-btn.success.active { background: var(--success-color); border-color: var(--success-color); }.filter-btn.danger.active { background: var(--danger-color); border-color: var(--danger-color); }.table-section { margin: 1.25rem 0; }@@ -360,7 +362,7 @@.validator-collapsible:hover { background: var(--bg-secondary); }.validator-collapsible.active {background: var(--secondary-color);- color: white;+ color: var(--on-accent);border-color: var(--secondary-color);border-radius: 8px 8px 0 0;}@@ -411,10 +413,10 @@.severity-medium { background: #fef3c7; color: #92400e; }.severity-low { background: #f3f4f6; color: #4b5563; }.severity-info { background: #dbeafe; color: #1e40af; }- [data-theme="dark"] .severity-critical, [data-theme="dark"] .severity-high { background: #450a0a; color: #fca5a5; }- [data-theme="dark"] .severity-medium { background: #451a03; color: #fcd34d; }- [data-theme="dark"] .severity-low { background: #374151; color: #9ca3af; }- [data-theme="dark"] .severity-info { background: #1e3a8a; color: #93c5fd; }+ @media screen { [data-theme="dark"] .severity-critical, [data-theme="dark"] .severity-high { background: #450a0a; color: #fca5a5; } }+ @media screen { [data-theme="dark"] .severity-medium { background: #451a03; color: #fcd34d; } }+ @media screen { [data-theme="dark"] .severity-low { background: #374151; color: #9ca3af; } }+ @media screen { [data-theme="dark"] .severity-info { background: #1e3a8a; color: #93c5fd; } }.finding-meta { display: flex; flex-wrap: wrap; gap: 0.75rem; color: var(--text-secondary); font-size: 0.8rem; margin-top: 0.375rem; }.finding-meta span { display: flex; align-items: center; gap: 0.25rem; }.meta-icon { width: 0.8rem; height: 0.8rem; flex-shrink: 0; }@@ -438,7 +440,7 @@border-left: 3px solid var(--success-color);color: #166534;}- [data-theme="dark"] .suggestion { background: #14532d; color: #bbf7d0; }+ @media screen { [data-theme="dark"] .suggestion { background: #14532d; color: #bbf7d0; } }footer {margin-top: 2rem;padding: 1.25rem 2rem;@@ -455,7 +457,7 @@padding: 0.5rem 0.75rem;border-radius: 6px;background: var(--secondary-color);- color: white;+ color: var(--on-accent);border: none;cursor: pointer;font-size: 0.75rem;@@ -581,7 +583,7 @@font-size: 0.75rem;color: #166534;}- [data-theme="dark"] .check-finding .finding-suggestion { background: #14532d; color: #bbf7d0; }+ @media screen { [data-theme="dark"] .check-finding .finding-suggestion { background: #14532d; color: #bbf7d0; } }.metadata-tag {display: inline-block;background: var(--bg-secondary);@@ -610,7 +612,7 @@transition: all 0.2s ease;}.view-toggle-btn:hover { border-color: var(--secondary-color); }- .view-toggle-btn.active { background: var(--secondary-color); color: white; border-color: var(--secondary-color); }+ .view-toggle-btn.active { background: var(--secondary-color); color: var(--on-accent); border-color: var(--secondary-color); }.results-view { display: none; }.results-view.active { display: block; }.skills-summary-bar {@@ -700,7 +702,7 @@cursor: pointer;transition: all 0.15s ease;}- .skill-pill:hover { background: var(--secondary-color); color: white; }+ .skill-pill:hover { background: var(--secondary-color); color: var(--on-accent); }/* Pair the skill pill (jumps to finding) with a small quality-scorelink so the two destinations stay distinct: the issue lives in themain pill, the quality score lives in the trailing ``Q`` chip. */@@ -730,7 +732,7 @@transition: all 0.15s ease;border-left: 1px solid var(--border-color);}- .quality-pill:hover { background: var(--warning-color); color: white; }+ .quality-pill:hover { background: var(--warning-color); color: var(--on-warning); }.top-issues-active-row { background: var(--bg-secondary) !important; outline: 2px solid var(--secondary-color); outline-offset: -2px; }/* Contributor Summary */@@ -799,8 +801,8 @@}.contributor-item-pill.pass { background: #dcfce7; color: #166534; }.contributor-item-pill.fail { background: #fee2e2; color: #991b1b; }- [data-theme="dark"] .contributor-item-pill.pass { background: #14532d; color: #86efac; }- [data-theme="dark"] .contributor-item-pill.fail { background: #450a0a; color: #fca5a5; }+ @media screen { [data-theme="dark"] .contributor-item-pill.pass { background: #14532d; color: #86efac; } }+ @media screen { [data-theme="dark"] .contributor-item-pill.fail { background: #450a0a; color: #fca5a5; } }.contributor-item-pill:hover { opacity: 0.8; }.skill-filter-banner {@@ -816,7 +818,7 @@color: #1e40af;position: sticky; top: 8rem; z-index: 49;}- [data-theme="dark"] .skill-filter-banner { background: #1e3a5f; border-color: #2563eb; color: #93c5fd; }+ @media screen { [data-theme="dark"] .skill-filter-banner { background: #1e3a5f; border-color: #2563eb; color: #93c5fd; } }.skill-filter-banner.visible { display: flex; }.skill-filter-banner .clear-filter {margin-left: auto;@@ -830,11 +832,11 @@font-size: 0.8rem;}.skill-filter-banner .clear-filter:hover { background: rgba(0,0,0,0.08); }- [data-theme="dark"] .skill-filter-banner .clear-filter:hover { background: rgba(255,255,255,0.1); }+ @media screen { [data-theme="dark"] .skill-filter-banner .clear-filter:hover { background: rgba(255,255,255,0.1); } }.count-badge {display: inline-block;background: var(--danger-color);- color: white;+ color: var(--on-danger);padding: 0.125rem 0.5rem;border-radius: 10px;font-size: 0.7rem;@@ -864,7 +866,7 @@transition: all 0.2s ease;}.severity-filter-btn:hover { border-color: var(--secondary-color); }- .severity-filter-btn.active { background: var(--secondary-color); color: white; border-color: var(--secondary-color); }+ .severity-filter-btn.active { background: var(--secondary-color); color: var(--on-accent); border-color: var(--secondary-color); }.severity-filter-btn.active.critical-btn { background: var(--critical-color); border-color: var(--critical-color); }.severity-filter-btn.active.high-btn { background: var(--high-color); border-color: var(--high-color); }@@ -882,9 +884,9 @@.confidence-badge.high-conf { background: #fee2e2; color: #991b1b; }.confidence-badge.medium-conf { background: #fef3c7; color: #92400e; }.confidence-badge.low-conf { background: #f3f4f6; color: #6b7280; border: 1px dashed #d1d5db; }- [data-theme="dark"] .confidence-badge.high-conf { background: #450a0a; color: #fca5a5; }- [data-theme="dark"] .confidence-badge.medium-conf { background: #451a03; color: #fcd34d; }- [data-theme="dark"] .confidence-badge.low-conf { background: #374151; color: #9ca3af; border-color: #4b5563; }+ @media screen { [data-theme="dark"] .confidence-badge.high-conf { background: #450a0a; color: #fca5a5; } }+ @media screen { [data-theme="dark"] .confidence-badge.medium-conf { background: #451a03; color: #fcd34d; } }+ @media screen { [data-theme="dark"] .confidence-badge.low-conf { background: #374151; color: #9ca3af; border-color: #4b5563; } }/* Low-confidence finding styling */.check-finding.low-confidence {@@ -929,7 +931,7 @@font-weight: 600;color: var(--secondary-color);}- [data-theme="dark"] .finding-suggestion.truncatable::after { background: #14532d; }+ @media screen { [data-theme="dark"] .finding-suggestion.truncatable::after { background: #14532d; } }.finding-suggestion.truncatable.expanded {max-height: none;}@@ -972,10 +974,9 @@.severity-breakdown-item.sev-high { background: #fee2e2; color: #991b1b; }.severity-breakdown-item.sev-medium { background: #fef3c7; color: #92400e; }.severity-breakdown-item.sev-low { background: #f3f4f6; color: #4b5563; }- [data-theme="dark"] .severity-breakdown-item.sev-critical,- [data-theme="dark"] .severity-breakdown-item.sev-high { background: #450a0a; color: #fca5a5; }- [data-theme="dark"] .severity-breakdown-item.sev-medium { background: #451a03; color: #fcd34d; }- [data-theme="dark"] .severity-breakdown-item.sev-low { background: #374151; color: #9ca3af; }+ @media screen { [data-theme="dark"] .severity-breakdown-item.sev-critical, [data-theme="dark"] .severity-breakdown-item.sev-high { background: #450a0a; color: #fca5a5; } }+ @media screen { [data-theme="dark"] .severity-breakdown-item.sev-medium { background: #451a03; color: #fcd34d; } }+ @media screen { [data-theme="dark"] .severity-breakdown-item.sev-low { background: #374151; color: #9ca3af; } }.severity-breakdown-label {font-size: 0.65rem;font-weight: 700;@@ -985,7 +986,7 @@margin-top: 0.5rem;}.severity-breakdown-label.advisory { color: #92400e; }- [data-theme="dark"] .severity-breakdown-label.advisory { color: #fcd34d; }+ @media screen { [data-theme="dark"] .severity-breakdown-label.advisory { color: #fcd34d; } }.advisory-note {font-size: 0.65rem;font-style: italic;@@ -1002,8 +1003,8 @@/* Dedup finding collapsible cards */.dedup-finding-high { border: 1px solid #fca5a5; background: #fef2f2; }.dedup-finding-medium { border: 1px solid #fde68a; background: #fffbeb; }- [data-theme="dark"] .dedup-finding-high { border-color: #991b1b; background: #450a0a; }- [data-theme="dark"] .dedup-finding-medium { border-color: #92400e; background: #451a03; }+ @media screen { [data-theme="dark"] .dedup-finding-high { border-color: #991b1b; background: #450a0a; } }+ @media screen { [data-theme="dark"] .dedup-finding-medium { border-color: #92400e; background: #451a03; } }.dedup-recommendation {margin-top: 0.75rem; padding: 0.75rem;background: var(--bg-secondary); border-radius: 6px;@@ -1140,9 +1141,9 @@border-radius: 999px;letter-spacing: 0.05em;}- .tier-card.pass .tier-card-verdict { background: var(--success-color); color: white; }- .tier-card.fail .tier-card-verdict { background: var(--danger-color); color: white; }- .tier-card.warn .tier-card-verdict { background: var(--warning-color); color: white; }+ .tier-card.pass .tier-card-verdict { background: var(--badge-pass-bg); color: var(--badge-pass-tx); border: 1px solid currentColor; }+ .tier-card.fail .tier-card-verdict { background: var(--badge-fail-bg); color: var(--badge-fail-tx); border: 1px solid currentColor; }+ .tier-card.warn .tier-card-verdict { background: var(--badge-warn-bg); color: var(--badge-warn-tx); border: 1px solid currentColor; }.tier-card.skipped .tier-card-verdict { background: var(--bg-tertiary); color: var(--text-secondary); }.tier-card-title {font-size: 1.05rem;@@ -1195,7 +1196,7 @@background: var(--bg);}.tier-card-sev-pill.crit { color: var(--danger-color); border-color: var(--danger-color); }- .tier-card-sev-pill.high { color: #d29922; border-color: #d29922; }+ .tier-card-sev-pill.high { color: var(--high-color); border-color: var(--high-color); }.tier-card-sev-pill.med { color: var(--warning-color); border-color: var(--warning-color); }.tier-card-sev-pill.low { color: var(--text-secondary); }.chart-grid {@@ -1231,15 +1232,6 @@<body>{% set ci = ci_context | default({}, true) %}<a class="skip-link" href="#report-workspace">Skip to report</a>- <aside class="report-sidebar" aria-label="Report navigation">- <a class="sidebar-brand" href="#report-workspace"><span class="brand-mark" aria-hidden="true">S</span><span>SkillEvaluator<small>EVALUATION REPORT</small></span></a>- <span class="sidebar-label">THIS REPORT</span>- <a href="#report-workspace" class="sidebar-current">Overview</a>- <a href="#report-evidence">Findings & evidence</a>- <a href="#run-metadata">Run details</a>- {% if ci.get('report_index_url') %}<a href="{{ ci.report_index_url }}">All skills ↗</a>{% endif %}- <div class="sidebar-footer">Evidence you can inspect.<br>Self-contained report.</div>- </aside>{% set has_tier3 = tier3 is defined and tier3 %}{% set tier3_summary_data = (tier3.get('summary') or {}) if has_tier3 else {} %}{% set tier3_provenance_data = (tier3.get('provenance') or {}) if has_tier3 else {} %}@@ -1247,35 +1239,55 @@{% set tier3_is_skipped = tier3_execution_status == 'skipped' or (tier3_provenance_data.get('advisory') and tier3_provenance_data.get('reason') == 'skipped') %}{% set tier3_skip_message = tier3_provenance_data.get('message') or 'Live evaluation did not run.' %}{% set tier3_verdict = (tier3_effective_status or tier3.get('verdict') or 'unknown') | lower if has_tier3 else 'unknown' %}- <div class="main-header">+ {#- A failing tier whose gate does not block is "Failed (advisory)", never "Changes required". -#}+ {% set tier3_advisory_fail = tier3_verdict == 'fail' and 'tier3' in nonblocking_tiers %}+ {% set has_t12_tab = tabs | selectattr('id', 'in', ['tier1', 'tier2']) | list | length > 0 %}+ {% if has_t12_tab %}+ {#- The summary's rule: when no Tier 1 or Tier 2 check ran, the verdict is Skipped, not Passed. -#}+ {% set t12_executed = tier1_summary.executed_count + tier2_summary.executed_count %}+ {% set verdict_key = 'incomplete' if has_incomplete else ('fail' if not all_passed else ('pass' if t12_executed else 'skipped')) %}+ {% elif has_tier3 %}+ {% set verdict_key = 'skipped' if tier3_is_skipped else (tier3_verdict if tier3_verdict in ['pass', 'fail', 'neutral'] else 'incomplete') %}+ {% else %}+ {% set verdict_key = 'incomplete' %}+ {% endif %}+ {% set verdict_class = {'pass': 'pass', 'fail': 'fail'}.get(verdict_key, 'warn') %}+ {% set verdict_label = advisory_fail_word if not has_t12_tab and verdict_key == 'fail' and tier3_advisory_fail else verdict_words[verdict_key] %}+ {% set check_words = {'pass': 'Passed', 'fail': 'Failed', 'incomplete': 'Incomplete', 'skipped': 'Skipped', 'not_applicable': 'N/A'} %}+ <header class="main-header"><div class="header-content">- <div class="logo-section">- <h1 class="header-title">SkillEvaluator</h1>- <div class="header-subtitle-row">- <p class="header-subtitle">Evaluation report</p>+ <div class="header-brand">+ {{ se_logo("se-logo-header", 28, "") }}+ <div>+ <p class="header-title">SkillEvaluator</p>+ <p class="header-subtitle">{% if timestamp %}Generated <time>{{ timestamp }}</time> · {% endif %}Version {{ version }}</p></div></div><div class="header-meta">+ <span class="header-status-badge verdict-badge {{ verdict_class }}" data-verdict="{{ verdict_key }}"><span class="verdict-dot" aria-hidden="true"></span><span class="sr-only">Verdict: </span>{{ verdict_label }}</span><div class="header-controls">- <button class="theme-toggle" onclick="toggleTheme()" title="Switch color theme" aria-label="Switch color theme">- <span id="theme-icon">☀</span><span id="theme-text">Light</span>+ <button type="button" class="theme-toggle" onclick="toggleTheme()" aria-pressed="false" title="Switch between the light and dark themes">+ <span id="theme-icon" aria-hidden="true">☾</span><span id="theme-text">Dark theme</span></button>{% if ci.get('canonical_json_url') %}<a class="export-btn" href="{{ ci.canonical_json_url }}" download="skillevaluator-report.json">↓ Download JSON</a>{% else %}- <button class="export-btn" onclick="exportJSON()" title="Export report as JSON">- <span>↓</span><span>Download JSON</span>+ <button type="button" class="export-btn" onclick="exportJSON()" title="Export report as JSON">+ <span aria-hidden="true">↓</span><span>Download JSON</span></button>{% endif %}</div>- <span class="header-status-badge {{ 'warn' if has_incomplete else ('pass' if all_passed else 'fail') }}">Process: {{ 'Incomplete' if has_incomplete else ('All Passed' if all_passed else 'Issues Found') }}</span>- <span class="header-status-badge {{ 'pass' if publication_status == 'pass' else ('fail' if publication_status == 'fail' else 'warn') }}"- data-publication-status="{{ publication_status }}"- {% if publication_reasons %}title="{{ publication_reasons | join(' ') }}"{% endif %}>- Publication: {{ publication_status | title }}- </span></div></div>- </div>+ </header>+ <aside class="report-sidebar" aria-label="Report navigation">+ <a class="sidebar-brand" href="#report-workspace">{{ se_logo("se-logo-sidebar", 28, "") }}<span class="sidebar-brand-text">SkillEvaluator<small>Evaluation report</small></span></a>+ <span class="sidebar-label">THIS REPORT</span>+ <a href="#report-workspace" class="sidebar-current">Summary</a>+ <a href="#report-evidence">Findings & evidence</a>+ <a href="#run-metadata">Run details</a>+ {% if ci.get('report_index_url') %}<a class="sidebar-index-link" href="{{ ci.report_index_url }}">← All skills</a>{% endif %}+ <div class="sidebar-footer">Evidence you can inspect.<br>Self-contained report.</div>+ </aside><main class="container" id="report-workspace">- <div class="report-breadcrumb">Evaluation / {{ content_label | default('Skill') }} report</div>+ {% if ci.get('report_index_url') %}<nav class="report-breadcrumb" aria-label="Breadcrumb"><a href="{{ ci.report_index_url }}">All skills</a> / <span aria-current="page">{{ content_label | default('Skill') }} report</span></nav>{% endif %}{% set has_tier1 = tabs | selectattr('id', 'equalto', 'tier1') | list | length > 0 %}{% set has_tier2 = tabs | selectattr('id', 'equalto', 'tier2') | list | length > 0 %}{% set ui_t1 = tier1_execution_summary | default({}) %}@@ -1297,11 +1309,11 @@{% set hero_harbor = (tier3.get('harbor_viewer') or tier3.get('summary', {}).get('harbor_viewer') or {}) if has_tier3 else {} %}{% set hero_harbor_job_url = hero_harbor.get('job_url') %}{% set hero_harbor_analysis_url = hero_harbor.get('analysis_url') %}- <section class="agent-hero">+ <section class="agent-hero" aria-labelledby="report-title"><div class="agent-hero-top"><div class="agent-hero-title-block"><span class="agent-hero-eyebrow">{{ content_label }} evaluation</span>- <h2 class="agent-hero-title">{{ hero_title }}</h2>+ <h1 class="agent-hero-title" id="report-title">{{ hero_title }}</h1>{#-Show the *full* path/URL only when it actually addsinformation (i.e. it differs from the friendly label).@@ -1309,32 +1321,34 @@the repo root the two strings are identical and thesubtitle becomes noise.-#}- {% if target_path and target_path != hero_title and target_display != hero_title %}+ {% if target_path and target_path.startswith('https://') and target_path != hero_title and target_display != hero_title %}<div class="agent-hero-subtitle">- {% if target_path.startswith('https://') %}<a href="{{ target_path }}" target="_blank" title="{{ target_path }}">{{ target_display or target_path }}</a>{% else %}<span title="{{ target_path }}">{{ target_path }}</span>{% endif %}+ <a href="{{ target_path }}" target="_blank" rel="noopener noreferrer" title="{{ target_path }}">{{ target_display or target_path }}</a></div>{% endif %}+ {#- In a folder scan this is the only sign that some skills were not reported. -#}+ {% if has_tier1 or has_tier2 %}<p class="hero-skills">{{ content_label_plural }}: {{ passed_skills }} of {{ total_skills }} passed</p>{% endif %}</div>{#-- Right-side score / verdict block. Tier 3 takes precedence- when present (it is the most expressive single metric);- otherwise we surface the cross-tier pass percentage so- Tier-1-only and Tier-1+Tier-2 runs still get a verdict- badge.+ Right-side block: the Tier 3 verdict and score when Tier 3+ ran, and publication eligibility for every report, kept+ next to the verdict.-#}+ {% set publication_class = {'pass': 'pass', 'fail': 'fail'}.get(publication_status, 'warn') %}+ <div class="hero-side">{% if has_tier3 %}{% set tier3_environment = (tier3.summary.environment if tier3.summary is defined and tier3.summary else None) or tier3.environment %}{% if tier3_is_skipped %}- <div>- <div class="agent-hero-score score-warn">SKIPPED</div>+ <span class="verdict-chip warn"><span class="sr-only">Tier 3 verdict: </span>{{ verdict_words['skipped'] }}</span><div class="agent-hero-detail"><div><strong>Reason:</strong> {{ tier3_skip_message }}</div><div>Tier 3 is advisory and does not block required validation.</div></div>- </div>{% else %}- <div>- <div class="agent-hero-score {{ 'score-pass' if hero_score is not none and hero_score >= 0.8 else ('score-warn' if hero_score is not none and hero_score >= 0.6 else 'score-fail') }}">{{ "%.2f" | format(hero_score) if hero_score is not none else "N/A" }}</div>+ {% set t3_key = tier3_verdict if tier3_verdict in ['pass', 'fail', 'neutral'] else 'incomplete' %}+ <span class="verdict-chip {{ {'pass': 'pass', 'fail': 'fail'}.get(t3_key, 'warn') }}"><span class="sr-only">Tier 3 verdict: </span>{{ advisory_fail_word if tier3_advisory_fail else verdict_words[t3_key] }}</span>+ {% if t3_key == 'incomplete' and tier3_status_reason %}<p class="hero-reason">{{ tier3_status_reason }}</p>{% endif %}+ <div class="agent-hero-score {{ 'score-pass' if hero_score is not none and hero_score >= 0.8 else ('score-warn' if hero_score is not none and hero_score >= 0.6 else 'score-fail') }}">{{ "%.2f" | format(hero_score) if hero_score is not none else "N/A" }}<span class="hero-score-label">overall score</span></div><div class="agent-hero-detail"><div><strong>Best performing agent:</strong> {{ tier3.best_agent or 'N/A' }}</div><div><strong>Trials:</strong> {{ hero_trials }} across {{ (tier3.agents_run or []) | length }} agent(s)</div>@@ -1342,20 +1356,10 @@{% if tier3_environment %}<div><strong>Environment:</strong> <code>{{ tier3_environment }}</code></div>{% endif %}{% if tier3.get('overall_lift') is not none %}<div><strong>Overall lift:</strong> {{ "%+.2f" | format(tier3.get('overall_lift')) }}</div>{% endif %}</div>- </div>{% endif %}- {% else %}- <div>- <div class="agent-hero-score verdict-text {{ 'score-warn' if has_incomplete else ('score-pass' if all_passed else 'score-fail') }}">{{ 'Incomplete' if has_incomplete else ('Passed' if all_passed else 'Changes required') }}</div>- <p class="verdict-caption">Evaluation outcome</p>- <div class="agent-hero-detail">- {% if has_tier1 and not has_tier2 %}<div><strong>Checks passed:</strong> {{ ui_t1_pass }}/{{ ui_t1_executed }}</div>{% if ui_t1_na %}<div><strong>Not applicable:</strong> {{ ui_t1_na }}</div>{% endif %}{% else %}<div><strong>Checks passed:</strong> {{ passed_count }}/{{ executed_count }}</div>{% endif %}- {% if skipped_count %}<div><strong>Skipped:</strong> {{ skipped_count }}</div>{% endif %}- <div><strong>{{ content_label_plural }}:</strong> {{ passed_skills }}/{{ total_skills }} passed</div>- {% if total_issues %}<div><strong>Issues:</strong> {{ total_issues }}</div>{% endif %}- </div>- </div>{% endif %}+ <p class="publication-line" data-publication-status="{{ publication_status }}"><span class="publication-label">Publication eligibility</span> <span class="status-badge {{ publication_class }}">{{ publication_words.get(publication_status, publication_status | title) }}</span>{% if publication_reasons %} <span class="publication-reasons">{{ publication_reasons | join(' ') }}</span>{% endif %}</p>+ </div></div>{#-Per-tier summary tiles. Each tier renders its own card iff its@@ -1373,7 +1377,9 @@with whichever tab content is rendered active by default.-#}{% set initial_tab = tabs[0].id if tabs else 'tier1' %}- <div class="tier-cards">+ {#- With one tier, the executive summary below carries the numbers. -#}+ {% set single_tier = tabs | length < 2 %}+ <div class="tier-cards{{ ' single-tier' if single_tier else '' }}{{ ' tier-line' if single_tier and has_tier1 else '' }}">{% if has_tier1 %}{% set t1_class = 'warn' if tier1_summary.incomplete_count else ('skipped' if tier1_summary.executed_count == 0 else ('pass' if tier1_summary.all_passed else 'fail')) %}<div class="tier-card {{ t1_class }}{% if initial_tab == 'tier1' %} active{% endif %}"@@ -1386,16 +1392,18 @@title="Open Tier 1: Security and Static Validation"><div class="tier-card-head"><span class="tier-card-label">Tier 1</span>- <span class="tier-card-verdict">{{ 'PASS' if t1_class == 'pass' else ('FAIL' if t1_class == 'fail' else ('INCOMPLETE' if t1_class == 'warn' else 'SKIPPED')) }}</span>+ {% if not single_tier %}<span class="tier-card-verdict">{{ verdict_words[{'pass': 'pass', 'fail': 'fail', 'warn': 'incomplete'}.get(t1_class, 'skipped')] }}</span>{% endif %}</div>- <h3 class="tier-card-title">Security & Static Validation</h3>+ <p class="tier-card-title">Security & Static Validation</p><div class="tier-card-stats">- <span><strong>{{ ui_t1_pass }}</strong>/{{ ui_t1_executed }} validators passed</span>{% if ui_t1_na %}<span><strong>{{ ui_t1_na }}</strong> not applicable</span>{% endif %}+ {% if not single_tier %}+ <span><strong>{{ ui_t1_pass }}</strong> of {{ ui_t1_executed }} checks passed</span>{% if ui_t1_na %}<span><strong>{{ ui_t1_na }}</strong> not applicable</span>{% endif %}{% if tier1_summary.skipped_count %}<span><strong>{{ tier1_summary.skipped_count }}</strong> skipped</span>{% endif %}- {% if tier1_summary.issue_count %}<span><strong>{{ tier1_summary.issue_count }}</strong> issue(s)</span>{% endif %}+ {% if tier1_summary.issue_count %}<span><strong>{{ tier1_summary.issue_count }}</strong> finding{{ 's' if tier1_summary.issue_count != 1 else '' }}</span>{% endif %}+ {% endif %}{% if skillspector_versions %}<span title="SkillSpector version that ran the Tier 1 security scan">SkillSpector <strong>v{{ skillspector_versions | join(', v') }}</strong></span>{% endif %}</div>- {% if tier1_summary.critical or tier1_summary.high or tier1_summary.medium or tier1_summary.low %}+ {% if not single_tier and (tier1_summary.critical or tier1_summary.high or tier1_summary.medium or tier1_summary.low) %}<div class="tier-card-sev">{% if tier1_summary.critical %}<span class="tier-card-sev-pill crit">{{ tier1_summary.critical }} critical</span>{% endif %}{% if tier1_summary.high %}<span class="tier-card-sev-pill high">{{ tier1_summary.high }} high</span>{% endif %}@@ -1417,18 +1425,18 @@title="Open Tier 2: Deduplication"><div class="tier-card-head"><span class="tier-card-label">Tier 2</span>- <span class="tier-card-verdict">{{ 'PASS' if t2_class == 'pass' else ('FAIL' if t2_class == 'fail' else ('INCOMPLETE' if t2_class == 'warn' else 'SKIPPED')) }}</span>+ <span class="tier-card-verdict">{{ advisory_fail_word if t2_class == 'fail' and 'tier2' in nonblocking_tiers else verdict_words[{'pass': 'pass', 'fail': 'fail', 'warn': 'incomplete'}.get(t2_class, 'skipped')] }}</span></div>- <h3 class="tier-card-title">Deduplication</h3>+ <p class="tier-card-title">Deduplication</p><div class="tier-card-stats">- <span><strong>{{ tier2_summary.passed_count }}</strong>/{{ tier2_summary.executed_count }} checks passed</span>+ <span><strong>{{ tier2_summary.passed_count }}</strong> of {{ tier2_summary.executed_count }} checks passed</span>{% if tier2_summary.skipped_count %}<span><strong>{{ tier2_summary.skipped_count }}</strong> skipped</span>{% endif %}{% if tier2_summary.issue_count %}<span><strong>{{ tier2_summary.issue_count }}</strong> duplicate(s)</span>{% endif %}</div></div>{% endif %}{% if has_tier3 %}- {% set tier3_display_verdict = 'SKIPPED' if tier3_is_skipped else tier3_verdict | upper %}+ {% set tier3_display_verdict = verdict_words['skipped'] if tier3_is_skipped else (advisory_fail_word if tier3_advisory_fail else verdict_words[tier3_verdict if tier3_verdict in ['pass', 'fail', 'neutral'] else 'incomplete']) %}{% set t3_class = 'skipped' if tier3_is_skipped else ('pass' if tier3_verdict == 'pass' else ('fail' if tier3_verdict == 'fail' else 'warn')) %}<div class="tier-card {{ t3_class }}{% if initial_tab == 'tier3' %} active{% endif %}"role="button"@@ -1442,7 +1450,7 @@<span class="tier-card-label">Tier 3</span><span class="tier-card-verdict">{{ tier3_display_verdict }}</span></div>- <h3 class="tier-card-title">Live Agent Evaluation</h3>+ <p class="tier-card-title">Live Agent Evaluation</p><div class="tier-card-stats">{% if tier3_is_skipped %}<span>{{ tier3_skip_message }}</span>@@ -1462,41 +1470,7 @@{% endif %}</div></section>- <div class="scope-strip" aria-label="Evaluation scope">- <span><strong>Scope</strong></span>- <span>Tier 1 · {{ 'Static validation' if has_tier1 else 'Not run' }}</span>- <span>Tier 2 · {{ 'Deduplication' if has_tier2 else 'Not run' }}</span>- <span>Tier 3 · {{ 'Live evaluation' if has_tier3 and not tier3_is_skipped else 'Not run' }}</span>- </div>- <details class="run-metadata" id="run-metadata">- <summary>Run details & provenance</summary>- <div class="metadata-grid">- {% if target_path %}<p class="header-meta-item"><strong>Target:</strong> {% if target_path.startswith('https://') %}<a href="{{ target_path }}" target="_blank" class="header-target-link" title="{{ target_path }}">{{ target_display or target_path }}</a>{% else %}{{ target_path }}{% endif %}</p>{% endif %}- {% if profile %}<p class="header-meta-item"><strong>Profile:</strong> <span class="header-profile-badge profile-{{ profile }}">{{ profile }}</span></p>{% endif %}- {% if timestamp %}<p class="header-meta-item">Generated: {{ timestamp }}{% if version %} · SkillEvaluator v{{ version }}{% endif %}</p>{% endif %}-- {% if ci.get('run_url') %}<p><strong>CI run</strong><a href="{{ ci.run_url }}" target="_blank" rel="noopener noreferrer">Open run ↗</a></p>{% endif %}- {% if ci.get('commit') %}<p><strong>Commit</strong><code>{{ ci.commit }}</code></p>{% endif %}- {% if ci.get('image') %}<p><strong>Evaluator image</strong><code>{{ ci.image }}</code></p>{% endif %}- {% if ci.get('ci_conclusion') %}<p><strong>CI conclusion</strong>{{ ci.ci_conclusion }}</p>{% endif %}- </div>- </details>- {% if has_tier2 or has_tier3 %}- {% set included_tiers = [] %}- {% if has_tier1 %}{% set _ = included_tiers.append('tier1') %}{% endif %}- {% if has_tier2 %}{% set _ = included_tiers.append('tier2') %}{% endif %}- {% if has_tier3 %}{% set _ = included_tiers.append('tier3') %}{% endif %}- {% set tier_labels = {'tier1': 'Tier 1', 'tier2': 'Tier 2', 'tier3': 'Tier 3 agent-eval'} %}- {% set blocking_labels = [] %}{% set advisory_labels = [] %}- {% for tier in gating.blocking_tiers if tier in included_tiers %}{% set _ = blocking_labels.append(tier_labels[tier]) %}{% endfor %}- {% for tier in gating.advisory_tiers if tier in included_tiers %}{% set _ = advisory_labels.append(tier_labels[tier]) %}{% endfor %}- <div class="gate-policy" aria-label="Gate policy for included tiers">- <strong>Gate policy</strong>- {% if blocking_labels %}<span>Blocking ({{ blocking_labels | join(' + ') }}): <strong>{{ blocking_critical }} critical · {{ blocking_high }} high · {{ blocking_medium }} medium · {{ blocking_low }} low</strong></span>{% endif %}- {% if advisory_labels %}<span>Advisory ({{ advisory_labels | join(' + ') }}): <strong>{{ advisory_critical }} critical · {{ advisory_high }} high · {{ advisory_medium }} medium · {{ advisory_low }} low</strong>.{% if has_tier3 and 'tier3' in gating.advisory_tiers %} Tier 3 agent-eval findings are advisory and don't block merge.{% endif %}</span>{% endif %}- </div>- {% endif %}- <div id="report-evidence" class="workspace-heading"><h2>Findings & evidence</h2><p>Inspect the outcome and supporting checks. Quality scores are separate from the pass / fail decision.</p></div>+ {% if not has_tier1 %}<div id="report-evidence" class="workspace-heading"><h2>Findings & evidence</h2><p>Each check with its results and the evidence behind them.</p></div>{% endif %}{% for tab in tabs %}<div id="tab-{{ tab.id }}" class="tab-content {{ 'active' if loop.first else '' }}">{% if tab.id == 'tier1' %}@@ -1515,11 +1489,82 @@{% set passed_skills = tier1_display_summary.passed_skills %}{% set incomplete_skills = skills.values() | selectattr('status', 'defined') | selectattr('status', 'equalto', 'incomplete') | list | length %}{% set failed_skills = [tier1_display_summary.failed_skills - incomplete_skills, 0] | max %}- {# ==================== TOP ISSUES EXECUTIVE SUMMARY ==================== #}+ {# ==================== EXECUTIVE SUMMARY ==================== #}+ {% set t1_key = 'incomplete' if tier1_summary.incomplete_count else ('skipped' if tier1_summary.executed_count == 0 else ('pass' if tier1_summary.all_passed else 'fail')) %}+ {% set t1_class = {'pass': 'pass', 'fail': 'fail'}.get(t1_key, 'warn') %}+ {% set t1_all_checks = total_validators + ui_t1_na + (skipped_count or 0) %}+ {% set qualities = skills.values() | selectattr('quality', 'defined') | map(attribute='quality') | selectattr('overall_score', 'defined') | list %}+ <section class="exec-summary {{ t1_class }}" aria-labelledby="exec-summary-title">+ <div class="exec-verdict">+ <h2 id="exec-summary-title" class="exec-title"><span class="sr-only">Tier 1 result: </span><span class="verdict-chip {{ t1_class }}">{{ verdict_words[t1_key] }}</span></h2>+ <p class="exec-reason">+ {%- if t1_key == 'incomplete' -%}+ A required scan did not finish{% if tier1_incomplete_scans %} ({{ tier1_incomplete_scans | join(', ') }}){% endif %}, so no pass is claimed.+ {%- if tier1_scanners_missing %} Install or enable the scanner and run the evaluation again.+ {%- elif tier1_scan_stop %} {{ tier1_scan_stop }} Fix what stopped the scan and run the evaluation again.+ {%- else %} Find out why the scan stopped and run the evaluation again.{% endif %}+ {%- elif t1_key == 'fail' -%}+ {{ failed_count }} of {{ total_validators }} checks failed: {{ tier1_failed_checks[:3] | join(', ') }}{% if tier1_failed_checks | length > 3 %} and {{ tier1_failed_checks | length - 3 }} more{% endif %}.{% if tier1_blocking_findings %} {{ tier1_blocking_findings }} critical or high finding{{ 's' if tier1_blocking_findings != 1 else '' }} in those checks block the verdict.{% endif %}+ {%- elif t1_key == 'skipped' -%}+ No Tier 1 check ran, so there is no result to report.+ {%- else -%}+ All {{ total_validators }} checks that apply passed.{% if tier1_summary.issue_count %} {{ tier1_summary.issue_count }} advisory finding{{ 's do' if tier1_summary.issue_count != 1 else ' does' }} not block.{% endif %}+ {%- endif -%}+ </p>+ </div>+ <dl class="kpis">+ <div class="kpi">+ <dt>Checks passed</dt>+ <dd class="kpi-value"><strong>{{ passed_count }}</strong> of {{ total_validators }}</dd>+ <dd class="kpi-sub">{{ total_validators }} of {{ t1_all_checks }} checks ran{% if ui_t1_na %}, {{ ui_t1_na }} not applicable{% endif %}{% if skipped_count %}, {{ skipped_count }} skipped{% endif %}{% if failed_count %}; {{ failed_count }} failed{% endif %}{% if incomplete_count %}; {{ incomplete_count }} incomplete{% endif %}</dd>+ </div>+ <div class="kpi">+ <dt>Findings</dt>+ <dd class="kpi-value"><strong>{{ tier1_summary.issue_count or 0 }}</strong></dd>+ <dd class="kpi-sub sev-line">{% for level in ['critical', 'high', 'medium', 'low'] %}{% if tier1_summary[level] %}<span class="sev-item"><span class="sev-dot {{ level }}" aria-hidden="true"></span>{{ tier1_summary[level] }} {{ level }}</span> {% endif %}{% endfor %}{% if not tier1_summary.issue_count %}None reported{% endif %}</dd>+ </div>+ <div class="kpi">+ <dt>Blocking critical + high</dt>+ <dd class="kpi-value"><strong class="{{ 'score-fail' if tier1_blocking_findings else '' }}">{{ tier1_blocking_findings }}</strong></dd>+ <dd class="kpi-sub">in checks that failed</dd>+ </div>+ <div class="kpi">+ <dt>Quality</dt>+ {% if qualities | length == 1 %}{% set q = qualities[0] %}+ <dd class="kpi-value"><strong class="{{ 'score-fail' if quality_min_score is not none and q.overall_score < quality_min_score else '' }}">{{ q.overall_score | round(1) }}</strong> / 100{% if q.grade %} · grade {{ q.grade }}{% endif %}</dd>+ <dd class="kpi-sub">{% if quality_min_score is not none %}Minimum {{ quality_min_score }} to pass{% else %}Content quality score{% endif %}</dd>+ {% elif qualities %}+ <dd class="kpi-value"><strong>{{ qualities | selectattr('overall_score', 'ge', quality_min_score if quality_min_score is not none else 0) | list | length }}</strong> of {{ qualities | length }}</dd>+ <dd class="kpi-sub">{{ content_label_plural | lower }} meet {% if quality_min_score is not none %}the minimum of {{ quality_min_score }}{% else %}the quality bar{% endif %}</dd>+ {% else %}+ <dd class="kpi-value"><strong>—</strong></dd>+ <dd class="kpi-sub">Not scored in this run</dd>+ {% endif %}+ </div>+ </dl>+ <p class="kpi-legend">A finding is one problem a check reports at one location, with a severity from critical to low. Findings in a failed check block the verdict; findings in a passed check are advisory.</p>+ {% if fix_first %}+ <div class="fix-first">+ <h3>{{ 'Worth a look' if t1_key == 'pass' else 'Fix first' }}</h3>+ <ol>+ {% for item in fix_first %}+ <li>+ <span class="severity-tag severity-{{ item.severity }}">{{ 'Incomplete' if item.kind == 'incomplete' else ('Failed' if item.kind == 'check' else item.severity | title) }}</span>+ <a href="#{{ item.anchor }}" class="fix-link" onclick="return openFinding(this.getAttribute('href').slice(1))">{{ item.message | truncate(160) }}</a>+ <span class="fix-meta">{{ item.check }}{% if skills | length > 1 %} · {{ item.skill }}{% endif %}{% if item.location %} · <code>{{ item.location }}</code>{% endif %}{% if not item.blocking %} · advisory{% endif %}</span>+ </li>+ {% endfor %}+ </ol>+ {% if tier1_summary.issue_count > fix_first | selectattr('kind', 'equalto', 'finding') | list | length %}<p class="fix-more">The other findings are listed under their checks below.</p>{% endif %}+ </div>+ {% endif %}+ </section>+ <div id="report-evidence" class="workspace-heading"><h2>Findings & evidence</h2><p>Each check with its findings, evidence and suggested fixes.</p></div>+ {# ==================== FINDING TYPES ==================== #}{% if top_issues %}<div class="top-issues-section {{ 'advisory-findings' if all_passed and not has_incomplete else '' }}"><div class="top-issues-header" onclick="toggleTopIssues(this)">- <h3>{{ 'Advisory findings' if all_passed and not has_incomplete else 'Needs attention' }} · {{ top_issues | length }} finding types</h3>+ <h3>{{ 'Advisory findings' if all_passed and not has_incomplete else 'Finding types' }} · {{ top_issues | length }}, most severe first</h3><span class="expand-text">▲</span></div><div class="top-issues-content active">@@ -1534,7 +1579,7 @@</tr></thead><tbody>- {% for issue in top_issues[:5] %}+ {% for issue in top_issues[:10] %}<tr data-severity="{{ issue.severity }}" data-affected-skills="{{ issue.skills_affected | join(',') }}" data-issue-key="{{ issue.issue_key }}" onclick="filterToAffectedSkills(this)" title="Click to jump to the first occurrence of this issue"><td style="font-weight: 500;">{{ issue.message }}</td><td><span class="metadata-tag">{{ issue.category }}</span></td>@@ -1545,7 +1590,7 @@{% for skill in issue.skills_affected[:5] %}<span class="skill-pill-group"><span class="skill-pill" data-skill-name="{{ skill }}" data-issue-key="{{ issue.issue_key }}" onclick="event.stopPropagation(); scrollToFinding(this.getAttribute('data-skill-name'), this.getAttribute('data-issue-key'))" title="Jump to this issue in {{ skill }}">{{ skill }}</span>- <span class="quality-pill" data-skill-name="{{ skill }}" onclick="event.stopPropagation(); scrollToQualityScore(this.getAttribute('data-skill-name'))" title="View quality score for {{ skill }}">Q</span>+ <span class="quality-pill" data-skill-name="{{ skill }}" onclick="event.stopPropagation(); scrollToQualityScore(this.getAttribute('data-skill-name'))" title="View quality score for {{ skill }}" aria-label="Quality score for {{ skill }}">Quality</span></span>{% endfor %}{% if issue.skills_affected | length > 5 %}@@ -1557,43 +1602,11 @@{% endfor %}</tbody></table>- {% if top_issues | length > 5 %}<p class="findings-overflow-note">Showing 5 of {{ top_issues | length }} finding types. Full evidence and suggested fixes are included below.</p>{% endif %}+ {% if top_issues | length > 10 %}<p class="findings-overflow-note">Showing 10 of {{ top_issues | length }} finding types. Full evidence and suggested fixes are included below.</p>{% endif %}</div></div>{% endif %}- <div class="dashboard">- <div class="dashboard-card">- <h3 class="dashboard-card-title">Validators Run</h3>- <p class="dashboard-card-value">{{ total_validators }}</p>- <p class="dashboard-card-subtitle">Included in this report</p>- </div>- <div class="dashboard-card success">- <h3 class="dashboard-card-title">Passed</h3>- <p class="dashboard-card-value">{{ passed_count }}</p>- <p class="dashboard-card-subtitle">{{ pass_percentage }}% of executed validators</p>- </div>- {% if ui_t1_na %}<div class="dashboard-card"><h3 class="dashboard-card-title">Not applicable</h3><p class="dashboard-card-value">{{ ui_t1_na }}</p><p class="dashboard-card-subtitle">No matching input to scan</p></div>{% endif %}- {% if skipped_count %}- <div class="dashboard-card">- <h3 class="dashboard-card-title">Skipped</h3>- <p class="dashboard-card-value">{{ skipped_count }}</p>- <p class="dashboard-card-subtitle">Not executed</p>- </div>- {% endif %}- {% if incomplete_count %}<div class="dashboard-card"><h3 class="dashboard-card-title">Incomplete</h3><p class="dashboard-card-value score-warn">{{ incomplete_count }}</p><p class="dashboard-card-subtitle">Required evidence unavailable</p></div>{% endif %}- <div class="dashboard-card danger">- <h3 class="dashboard-card-title">Failed</h3>- <p class="dashboard-card-value">{{ failed_count }}</p>- <p class="dashboard-card-subtitle">Requires attention</p>- </div>- <div class="dashboard-card">- <h3 class="dashboard-card-title">Overall Status</h3>- <p class="dashboard-card-value"><span class="status-badge {{ 'warn' if has_incomplete else ('pass' if all_passed else 'fail') }}">{{ 'Incomplete' if has_incomplete else ('Passed' if all_passed else 'Failed') }}</span></p>- <p class="dashboard-card-subtitle">{{ total_issues }} total issues</p>- <div class="progress-bar"><div class="progress-bar-fill" style="width: {{ pass_percentage }}%"></div></div>- </div>- </div><div class="report-searchbar"><label for="report-search">Search evidence</label><input id="report-search" type="search" placeholder="Find a skill, check, or finding…" oninput="applyReportSearch()" autocomplete="off">@@ -1603,8 +1616,8 @@<p id="report-no-matches" class="report-no-matches" hidden>No matching evidence. Try a different search or reset filters.</p><div class="view-toggle-section"><span style="font-size: 0.875rem; color: var(--text-secondary); margin-right: 0.5rem;">View by:</span>- <button class="view-toggle-btn active" id="view-skill" onclick="switchView('skill')">By {{ content_label }}</button>- <button class="view-toggle-btn" id="view-validator" onclick="switchView('validator')">By Validator</button>+ <button class="view-toggle-btn active" id="view-skill" onclick="switchView('skill')">By {{ content_label | lower }}</button>+ <button class="view-toggle-btn" id="view-validator" onclick="switchView('validator')">By check</button></div>{# ==================== SKILL-CENTRIC VIEW ==================== #}@@ -1613,7 +1626,7 @@<div class="filter-group"><button class="filter-btn active" id="skill-filter-all" onclick="filterSkills('all')">All {{ content_label_plural }}</button><button class="filter-btn success" id="skill-filter-pass" onclick="filterSkills('pass')">Passed</button>- <button class="filter-btn danger" id="skill-filter-fail" onclick="filterSkills('fail')">Failed</button>+ <button class="filter-btn danger" id="skill-filter-fail" onclick="filterSkills('fail')">Changes required</button><button class="filter-btn" id="skill-filter-incomplete" onclick="filterSkills('incomplete')">Incomplete</button><button class="bulk-action-btn" onclick="toggleAllSkills(true)">Expand All</button><button class="bulk-action-btn" onclick="toggleAllSkills(false)">Collapse All</button>@@ -1625,15 +1638,15 @@<button class="severity-filter-btn" id="sev-filter-low" onclick="filterSeverity('low')">Low+Info</button></div></div>- <div class="skills-summary-bar">+ <div class="skills-summary-bar"{% if total_skills < 2 %} hidden{% endif %}><div class="summary-item"><span>Total {{ content_label_plural }}:</span> <strong>{{ total_skills }}</strong></div><div class="summary-item pass"><span>Passed:</span> <strong>{{ passed_skills }}</strong></div>- <div class="summary-item fail"><span>Failed:</span> <strong>{{ failed_skills }}</strong></div>+ <div class="summary-item fail"><span>Changes required:</span> <strong>{{ failed_skills }}</strong></div>{% if incomplete_skills %}<div class="summary-item"><span>Incomplete:</span> <strong>{{ incomplete_skills }}</strong></div>{% endif %}</div>{# ==================== CONTRIBUTOR SUMMARY ==================== #}- {% if contributors %}+ {% if contributors and not (contributors | length == 1 and (contributors[0].author | lower) == 'unknown') %}<div class="contributor-section"><div class="contributor-header" onclick="toggleContributors(this)"><h3>Contributor Summary ({{ contributors | length }} contributor{{ 's' if contributors | length != 1 else '' }})</h3>@@ -1706,13 +1719,13 @@{% set qg = skill_data.quality.grade %}{% set qs_val = skill_data.quality.overall_score %}{% set gc = 'pass' if qg in ['A', 'B'] else ('warn' if qg == 'C' else 'fail') %}- <span class="status-badge {{ gc }}" title="Quality score, separate from evaluation outcome">Quality {{ qs_val | round(1) }} / {{ qg }}</span>+ <span class="status-badge {{ gc }}" title="Quality score out of 100{% if quality_min_score is not none %}, minimum {{ quality_min_score }} to pass{% endif %}">Quality {{ qs_val | round(1) }} / {{ qg }}</span>{% endif %}{% set skill_status = skill_data.status | default('pass' if skill_data.passed else 'fail') %}- <span class="status-badge {{ 'warn' if skill_status == 'incomplete' else skill_status }}">{{ skill_status | title }}</span>+ <span class="status-badge {{ 'warn' if skill_status == 'incomplete' else skill_status }}">{{ verdict_words.get(skill_status, skill_status | title) }}</span><span style="color: var(--text-secondary);">{{ skill_data.validators | length }} checks</span>{% if not skill_data.passed and skill_data.issue_count is defined and skill_data.issue_count > 0 %}- <span class="issue-count-badge">{{ skill_data.issue_count }} issue{{ 's' if skill_data.issue_count != 1 else '' }}</span>+ <span class="issue-count-badge">{{ skill_data.issue_count }} finding{{ 's' if skill_data.issue_count != 1 else '' }}</span>{% endif %}<span class="expand-text">{{ '▲' if not skill_data.passed else '▼' }}</span></div>@@ -1722,12 +1735,12 @@<div class="quality-score-block" data-quality-skill-name="{{ skill_name }}" style="background: var(--bg-secondary); border-radius: 8px; padding: 1rem; margin-bottom: 1rem; border-left: 4px solid {% if skill_data.quality.grade in ['A','B'] %}var(--success-color){% elif skill_data.quality.grade == 'C' %}var(--warning-color){% else %}var(--danger-color){% endif %};"><div style="display: flex; justify-content: space-between; align-items: center; margin-bottom: 0.75rem;"><div>- <strong style="font-size: 1.1rem;">Quality Score: {{ skill_data.quality.overall_score | round(1) }}/100</strong>- <span style="margin-left: 0.5rem; padding: 2px 8px; border-radius: 4px; font-weight: bold; color: white; background: {% if skill_data.quality.grade in ['A','B'] %}var(--success-color){% elif skill_data.quality.grade == 'C' %}var(--warning-color){% else %}var(--danger-color){% endif %};">{{ skill_data.quality.grade }}</span>+ <strong style="font-size: 1.1rem;">Quality score: {{ skill_data.quality.overall_score | round(1) }} / 100{% if quality_min_score is not none %}, minimum {{ quality_min_score }} to pass{% endif %}</strong>+ <span style="margin-left: 0.5rem; padding: 2px 8px; border-radius: 4px; font-weight: bold; border: 1px solid currentColor; {% if skill_data.quality.grade in ['A','B'] %}color: var(--badge-pass-tx); background: var(--badge-pass-bg){% elif skill_data.quality.grade == 'C' %}color: var(--badge-warn-tx); background: var(--badge-warn-bg){% else %}color: var(--badge-fail-tx); background: var(--badge-fail-bg){% endif %};">{{ skill_data.quality.grade }}</span></div><span style="color: var(--text-secondary); font-size: 0.85rem;">Type: {{ skill_data.quality.skill_type }}</span></div>- <p class="quality-outcome-note">Quality is assessed separately from the evaluation outcome. Required checks must also complete and pass.</p>+ <p class="quality-outcome-note">{% if quality_min_score is not none %}A score below {{ quality_min_score }} fails the Quality score check, which blocks the verdict. {% endif %}Grades: {{ quality_grade_legend }}.</p><table style="width: 100%; font-size: 0.85rem; border-collapse: collapse;"><tr style="border-bottom: 1px solid var(--border-color);"><th style="text-align: left; padding: 4px 8px;">Dimension</th>@@ -1762,20 +1775,25 @@</div>{% endif %}<div class="skill-checks-grid">- {% for validator_name, validator_data in skill_data.validators.items() | sort %}- <details class="check-result"{% if not validator_data.passed %} open{% endif %}>+ {% for validator_name, validator_data in skill_data.validators.items() | sort | sort(attribute='1.passed') %}+ <details class="check-result" id="{{ check_anchors[skill_name][validator_name] }}"{% if not validator_data.passed %} open{% endif %}><summary class="check-header"><span class="check-label">- <span class="check-name">{{ validator_name }}</span>+ <span class="check-name">{{ check_names.get(validator_name, validator_name) }}{% if validator_name in check_names %} <code class="check-id">{{ validator_name }}</code>{% endif %}</span>{% if validator_data.description %}<span class="check-description">{{ validator_data.description }}</span>{% endif %}</span>{% set check_status = validator_data.status | default('pass' if validator_data.passed else 'fail') %}- <span class="status-badge {{ 'pass' if check_status == 'pass' else ('fail' if check_status == 'fail' else 'warn') }}">{{ 'N/A' if check_status == 'not_applicable' else check_status | title }}</span>+ <span class="status-badge {{ 'pass' if check_status == 'pass' else ('fail' if check_status == 'fail' else 'warn') }}">{{ check_words.get(check_status, check_status | title) }}</span></summary><div class="check-details">{% if check_status == 'incomplete' %}<p class="incomplete-notice"><strong>Scan incomplete.</strong> {% if validator_data.incomplete_scans | default([]) %}Required scanner(s): {{ validator_data.incomplete_scans | join(', ') }}.{% endif %} The required evidence is unavailable.</p>{% endif %}- {% for error in validator_data.errors | default([]) %}<p class="scan-diagnostic">{{ error }}</p>{% endfor %}- {% if check_status == 'incomplete' %}{% for warning in validator_data.warnings | default([]) %}<p class="scan-diagnostic">{{ warning }}</p>{% endfor %}{% endif %}+ {#- Scanner diagnostics and gate failures always show and print. Lines that only+ repeat a finding card below fold into the raw scanner output. -#}+ {% set diagnostics = validator_data.additional_errors | default(validator_data.errors | default([])) %}+ {% set diagnostic_warnings = validator_data.additional_warnings | default(validator_data.warnings | default([])) if check_status == 'incomplete' else [] %}+ {% set finding_lines = (validator_data.errors | default([]) | reject('in', diagnostics) | list) + ((validator_data.warnings | default([]) | reject('in', diagnostic_warnings) | list) if check_status == 'incomplete' else []) %}+ {% for line in diagnostics + diagnostic_warnings %}<p class="scan-diagnostic">{{ line }}</p>{% endfor %}+ {% if finding_lines %}<details class="raw-output"><summary>Scanner output for the findings below ({{ finding_lines | length }} line{{ 's' if finding_lines | length != 1 else '' }})</summary>{% for line in finding_lines %}<p class="scan-diagnostic">{{ line }}</p>{% endfor %}</details>{% endif %}{% if validator_data.passed %}{% if validator_data.details %}{% set detail = validator_data.details[0] %}@@ -1810,7 +1828,7 @@<div style="font-size: 0.8rem; color: var(--text-secondary); margin-bottom: 0.5rem;">{{ validator_data.findings | length }} advisory finding(s) — not blocking:</div>{% for finding in validator_data.findings %}{% set confidence = finding.confidence if finding.confidence is defined else 'high' %}- <div class="check-finding warning {{ 'low-confidence' if confidence == 'low' else '' }}" data-finding-severity="{{ finding.severity }}" data-finding-confidence="{{ confidence }}" data-issue-key="{{ finding.issue_key | default('') }}">+ <div class="check-finding warning {{ 'low-confidence' if confidence == 'low' else '' }}" id="{{ finding_anchors[skill_name][validator_name][loop.index0] }}" data-finding-severity="{{ finding.severity }}" data-finding-confidence="{{ confidence }}" data-issue-key="{{ finding.issue_key | default('') }}"><div class="finding-msg">{{ finding.message }}{% if finding.occurrences is defined and finding.occurrences > 1 %}@@ -1830,7 +1848,7 @@{% if validator_data.findings %}{% for finding in validator_data.findings %}{% set confidence = finding.confidence if finding.confidence is defined else 'high' %}- <div class="check-finding {{ finding.severity }} {{ 'low-confidence' if confidence == 'low' else '' }}" data-finding-severity="{{ finding.severity }}" data-finding-confidence="{{ confidence }}" data-issue-key="{{ finding.issue_key | default('') }}">+ <div class="check-finding {{ finding.severity }} {{ 'low-confidence' if confidence == 'low' else '' }}" id="{{ finding_anchors[skill_name][validator_name][loop.index0] }}" data-finding-severity="{{ finding.severity }}" data-finding-confidence="{{ confidence }}" data-issue-key="{{ finding.issue_key | default('') }}"><div class="finding-msg">{{ finding.message }}{% if finding.occurrences is defined and finding.occurrences > 1 %}@@ -1871,7 +1889,7 @@{# ==================== VALIDATOR-CENTRIC VIEW (ORIGINAL) ==================== #}<div id="results-by-validator" class="results-view"><div class="filter-group">- <button class="filter-btn active" id="filter-all" onclick="filterValidators('all')">All Validators</button>+ <button class="filter-btn active" id="filter-all" onclick="filterValidators('all')">All checks</button><button class="filter-btn success" id="filter-pass" onclick="filterValidators('pass')">Passed</button><button class="filter-btn danger" id="filter-fail" onclick="filterValidators('fail')">Failed</button><button class="filter-btn" id="filter-incomplete" onclick="filterValidators('incomplete')">Incomplete</button>@@ -1889,8 +1907,8 @@{% set result_is_skipped = result | cleanly_skipped %}{% set result_execution_status = result | execution_status %}<tr data-validator-index="{{ loop.index }}" onclick="scrollToValidator(Number(this.dataset.validatorIndex))" data-status="{{ result_execution_status }}">- <td class="name-col">{{ result.validator_name }}</td>- <td><span class="status-badge {{ 'warn' if (result_execution_status in ['skipped', 'incomplete', 'not_applicable']) else ('pass' if result.passed else 'fail') }}">{{ 'N/A' if result_execution_status == 'not_applicable' else result_execution_status | title }}</span></td>+ <td class="name-col"><a class="validator-link" href="#validator-{{ loop.index }}">{{ check_names.get(result.validator_name, result.validator_name) }}</a>{% if result.validator_name in check_names %} <code class="check-id">{{ result.validator_name }}</code>{% endif %}</td>+ <td><span class="status-badge {{ 'warn' if (result_execution_status in ['skipped', 'incomplete', 'not_applicable']) else ('pass' if result.passed else 'fail') }}">{{ check_words.get(result_execution_status, result_execution_status | title) }}</span></td><td>{{ result.summary.files_scanned }}</td><td>{{ result.summary.checks_performed }}</td><td class="{{ 'error-text' if result.summary.errors > 0 else '' }}">{{ result.summary.errors }}</td>@@ -1909,11 +1927,11 @@<button class="validator-collapsible {{ 'active' if (result_is_skipped or not result.passed) else '' }}" onclick="toggleValidator(this)"><div class="collapsible-header"><span class="status-indicator {{ 'warn' if (result_execution_status in ['skipped', 'incomplete', 'not_applicable']) else ('pass' if result.passed else 'fail') }}"></span>- <span>{{ result.validator_name }}</span>+ <span>{{ check_names.get(result.validator_name, result.validator_name) }}{% if result.validator_name in check_names %} <code class="check-id">{{ result.validator_name }}</code>{% endif %}</span>{% if result.validator_description %}<span style="opacity:0.6;font-weight:400;font-size:0.8rem;margin-left:0.25rem">— {{ result.validator_description }}</span>{% endif %}</div><div style="display:flex;align-items:center;gap:0.75rem">- <span class="status-badge {{ 'warn' if (result_execution_status in ['skipped', 'incomplete', 'not_applicable']) else ('pass' if result.passed else 'fail') }}">{{ 'N/A' if result_execution_status == 'not_applicable' else result_execution_status | title }}</span>+ <span class="status-badge {{ 'warn' if (result_execution_status in ['skipped', 'incomplete', 'not_applicable']) else ('pass' if result.passed else 'fail') }}">{{ check_words.get(result_execution_status, result_execution_status | title) }}</span><span class="expand-text">{{ '▲ Collapse' if (result_is_skipped or not result.passed) else '▼ Expand' }}</span></div></button>@@ -2171,7 +2189,7 @@.t3-next-steps-head h3 { margin:0; color:var(--dark-text); font-size:1rem; }.t3-next-steps-conclusion { margin:0 0 0.65rem; color:var(--dark-text); line-height:1.5; }.t3-next-steps-list { margin:0; padding-left:1.15rem; color:var(--dark-text); display:grid; gap:0.35rem; }- .t3-link-btn { background:var(--secondary-color); border:1px solid var(--secondary-color); color:white; border-radius:7px; padding:0.38rem 0.75rem; font-size:0.78rem; font-weight:800; cursor:pointer; }+ .t3-link-btn { background:var(--secondary-color); border:1px solid var(--secondary-color); color:var(--on-accent); border-radius:7px; padding:0.38rem 0.75rem; font-size:0.78rem; font-weight:800; cursor:pointer; }.t3-link-btn:hover { filter:brightness(0.95); }.t3-agent-link { background:transparent; border:0; color:var(--secondary-color); font:inherit; font-weight:800; padding:0; cursor:pointer; text-decoration:underline; text-decoration-thickness:1px; text-underline-offset:2px; }.t3-agent-link:hover { color:var(--accent-color); }@@ -2394,7 +2412,7 @@Scores are diagnostic only; no lift verdict was computed. Check skill loading and agent support, then rerun the evaluation.{% else %}<strong>Verdict at a glance:</strong>- {{ tier3_verdict | upper }}{% if t3_effect %} · {{ t3_effect.message }}{% else %} with an overall score of+ {{ advisory_fail_word if tier3_advisory_fail else (verdict_words.get(tier3_verdict) or tier3_verdict | title) }}{% if t3_effect %} · {{ t3_effect.message }}{% else %} with an overall score of{{ "%.2f" | format(tier3.overall_score) if tier3.overall_score is not none else "N/A" }}.{% if t3_best_name %}Best performing agent is <strong>{{ t3_best_name }}</strong>{% else %}No successfully scored agent is available{% endif %}{% if tier3.get('overall_lift') is not none %} with {{ "%+.2f" | format(tier3.get('overall_lift')) }} skill lift{% endif %}.@@ -2416,7 +2434,7 @@<div class="dashboard"><div class="dashboard-card {{ 'success' if tier3_verdict == 'pass' else ('danger' if tier3_verdict == 'fail' else '') }}"><h3 class="dashboard-card-title">Verdict</h3>- <p class="dashboard-card-value">{{ 'SKIPPED' if tier3_is_skipped else tier3_verdict | upper }}</p>+ <p class="dashboard-card-value">{{ verdict_words['skipped'] if tier3_is_skipped else (advisory_fail_word if tier3_advisory_fail else (verdict_words.get(tier3_verdict) or tier3_verdict | title)) }}</p><p class="dashboard-card-subtitle">Evaluation outcome</p></div><div class="dashboard-card">@@ -3847,13 +3865,27 @@function labelForDimension(dim) {return dim.replace(/_/g, " ").replace(/\b\w/g, function(c){ return c.toUpperCase(); });}+ // While printing, charts use the light theme's colors: paper is white+ // whatever theme is on screen, and a canvas does not restyle itself.+ var printing = false;function cssColors() {- var s = getComputedStyle(document.documentElement);- return {+ var root = document.documentElement;+ var theme = root.getAttribute("data-theme");+ // beforeprint fires while screen styles still apply, so read the light values.+ if (printing && theme === "dark") root.setAttribute("data-theme", "light");+ var s = getComputedStyle(root);+ var colors = {grid: s.getPropertyValue("--chart-grid").trim() || "rgba(0,0,0,0.1)",tick: s.getPropertyValue("--chart-tick").trim() || "#64748b",label: s.getPropertyValue("--chart-label").trim() || "#0f172a"};+ if (printing && theme === "dark") root.setAttribute("data-theme", theme);+ return colors;+ }+ // A chart drawn for print must be complete when the page is captured.+ function chartOptions(options) {+ if (printing) options.animation = false;+ return options;}function agentEntries() {return Object.entries(tier3Data.agents || {});@@ -3950,11 +3982,11 @@};})},- options: {+ options: chartOptions({responsive: true,scales: { r: { min: 0, max: 1, ticks: { stepSize: 0.2, color: colors.tick, backdropColor: "transparent" }, grid: { color: colors.grid }, angleLines: { color: colors.grid }, pointLabels: { color: colors.label, font: { size: 11 } } } },plugins: { legend: { labels: { color: colors.label } } }- }+ })}));}var metricBar = document.getElementById("tier3-metric-bar");@@ -3967,11 +3999,11 @@return { label: item[0], data: agentDimensionData(item[1]), backgroundColor: palette[idx % palette.length] + "cc", borderRadius: 4 };})},- options: {+ options: chartOptions({responsive: true,scales: { y: { min: 0, max: 1, ticks: { stepSize: 0.2, color: colors.tick }, grid: { color: colors.grid } }, x: { ticks: { color: colors.label, maxRotation: 45 }, grid: { display: false } } },plugins: { legend: { labels: { color: colors.label } } }- }+ })}));}var tokenChart = document.getElementById("tier3-token-chart");@@ -3988,11 +4020,11 @@{ label: "Cached (included in prompt)", data: tokenAgents.map(function(a){ return tokenTotals[a].cachedComplete ? tokenTotals[a].cached : null; }), backgroundColor: "#8b5cf6cc", borderRadius: 3 }]},- options: {+ options: chartOptions({responsive: true,scales: { x: { ticks: { color: colors.tick }, grid: { color: colors.grid } }, y: { beginAtZero: true, ticks: { color: colors.label }, grid: { display: false } } },plugins: { legend: { labels: { color: colors.label } } }- }+ })}));}var stepsChart = document.getElementById("tier3-steps-chart");@@ -4002,16 +4034,29 @@charts.push(new Chart(stepsChart, {type: "bar",data: stepsData,- options: {+ options: chartOptions({responsive: true,scales: { x: { ticks: { color: colors.tick, maxRotation: 45 }, grid: { color: colors.grid } }, y: { ticks: { color: colors.label }, grid: { display: false } } },plugins: { legend: { labels: { color: colors.label } } }- }+ })}));}}};document.addEventListener("DOMContentLoaded", function(){ window.rebuildTier3Charts(); });+ function redrawForPrint(isPrinting) {+ if (printing === isPrinting) return;+ printing = isPrinting;+ window.rebuildTier3Charts();+ }+ window.addEventListener("beforeprint", function(){ redrawForPrint(true); });+ window.addEventListener("afterprint", function(){ redrawForPrint(false); });+ if (window.matchMedia) {+ var printQuery = window.matchMedia("print");+ var onPrintChange = function(event){ redrawForPrint(event.matches); };+ if (printQuery.addEventListener) printQuery.addEventListener("change", onPrintChange);+ else if (printQuery.addListener) printQuery.addListener(onPrintChange);+ }})();</script>@@ -4022,10 +4067,44 @@{% endif %}</div>{% endfor %}+ <div class="scope-strip" aria-label="Evaluation scope">+ <span><strong>Scope</strong></span>+ <span>Tier 1 · {{ 'Static validation' if has_tier1 else 'Not run' }}</span>+ <span>Tier 2 · {{ 'Deduplication' if has_tier2 else 'Not run' }}</span>+ <span>Tier 3 · {{ 'Live evaluation' if has_tier3 and not tier3_is_skipped else 'Not run' }}</span>+ </div>+ <details class="run-metadata" id="run-metadata">+ <summary>Run details & provenance</summary>+ <div class="metadata-grid">+ {% if target_path %}<p class="header-meta-item"><strong>Target:</strong> {% if target_path.startswith('https://') %}<a href="{{ target_path }}" target="_blank" class="header-target-link" title="{{ target_path }}">{{ target_display or target_path }}</a>{% else %}{{ target_path }}{% endif %}</p>{% endif %}+ {% if profile %}<p class="header-meta-item"><strong>Profile:</strong> <span class="header-profile-badge profile-{{ profile }}">{{ profile }}</span></p>{% endif %}+ {% if timestamp %}<p class="header-meta-item">Generated: {{ timestamp }}{% if version %} · SkillEvaluator v{{ version }}{% endif %}</p>{% endif %}++ {% if ci.get('run_url') %}<p><strong>CI run</strong><a href="{{ ci.run_url }}" target="_blank" rel="noopener noreferrer">Open run ↗</a></p>{% endif %}+ {% if ci.get('commit') %}<p><strong>Commit</strong><code>{{ ci.commit }}</code></p>{% endif %}+ {% if ci.get('image') %}<p><strong>Evaluator image</strong><code>{{ ci.image }}</code></p>{% endif %}+ {% if ci.get('ci_conclusion') %}<p><strong>CI conclusion</strong>{{ ci.ci_conclusion }}</p>{% endif %}+ </div>+ </details>+ {% if has_tier2 or has_tier3 %}+ {% set included_tiers = [] %}+ {% if has_tier1 %}{% set _ = included_tiers.append('tier1') %}{% endif %}+ {% if has_tier2 %}{% set _ = included_tiers.append('tier2') %}{% endif %}+ {% if has_tier3 %}{% set _ = included_tiers.append('tier3') %}{% endif %}+ {% set tier_labels = {'tier1': 'Tier 1', 'tier2': 'Tier 2', 'tier3': 'Tier 3 agent-eval'} %}+ {% set blocking_labels = [] %}{% set advisory_labels = [] %}+ {% for tier in gating.blocking_tiers if tier in included_tiers %}{% set _ = blocking_labels.append(tier_labels[tier]) %}{% endfor %}+ {% for tier in gating.advisory_tiers if tier in included_tiers %}{% set _ = advisory_labels.append(tier_labels[tier]) %}{% endfor %}+ <div class="gate-policy" aria-label="Gate policy for included tiers">+ <strong>Gate policy</strong>+ {% if blocking_labels %}<span>Blocking ({{ blocking_labels | join(' + ') }}): <strong>{{ blocking_critical }} critical · {{ blocking_high }} high · {{ blocking_medium }} medium · {{ blocking_low }} low</strong></span>{% endif %}+ {% if advisory_labels %}<span>Advisory ({{ advisory_labels | join(' + ') }}): <strong>{{ advisory_critical }} critical · {{ advisory_high }} high · {{ advisory_medium }} medium · {{ advisory_low }} low</strong>.{% if has_tier3 and 'tier3' in gating.advisory_tiers %} Tier 3 agent-eval findings are advisory and don't block merge.{% endif %}</span>{% endif %}+ </div>+ {% endif %}</main><footer>Generated by <strong>SkillEvaluator</strong> — NVIDIA Benchmark for Agent Skills Evaluation</footer><button id="backToTop" onclick="scrollToTop()" title="Back to Top">↑ Top</button>- <script id="report-data" type="application/json">{{ report_json | safe }}</script>+ {% if report_json is not none %}<script id="report-data" type="application/json">{{ report_json | safe }}</script>{% endif %}<script>function toggleTheme() {const html = document.documentElement;@@ -4035,13 +4114,18 @@updateThemeButton(newTheme);if (window.rebuildTier3Charts) window.rebuildTier3Charts();}+ // The button is a "Dark theme" toggle: its name stays fixed and+ // aria-pressed says whether the dark theme is on.function updateThemeButton(theme) {- document.getElementById('theme-icon').textContent = theme === 'dark' ? '☀' : '☾';- document.getElementById('theme-text').textContent = theme === 'dark' ? 'Light' : 'Dark';+ const toggle = document.querySelector('.theme-toggle');+ if (toggle) toggle.setAttribute('aria-pressed', theme === 'dark' ? 'true' : 'false');}function initTheme() {- let savedTheme = 'light';- try { savedTheme = localStorage.getItem('skillevaluator-theme') === 'dark' ? 'dark' : 'light'; } catch (err) { /* Standalone reports work without storage. */ }+ let savedTheme = null;+ try { savedTheme = localStorage.getItem('skillevaluator-theme'); } catch (err) { /* Standalone reports work without storage. */ }+ if (savedTheme !== 'dark' && savedTheme !== 'light') {+ savedTheme = window.matchMedia && window.matchMedia('(prefers-color-scheme: dark)').matches ? 'dark' : 'light';+ }document.documentElement.setAttribute('data-theme', savedTheme);updateThemeButton(savedTheme);if (window.rebuildTier3Charts) window.rebuildTier3Charts();diff --git a/src/skillevaluator/reporting/templates/report_ui.css.j2 b/src/skillevaluator/reporting/templates/report_ui.css.j2index 56b60287..7b36c088 100644--- a/src/skillevaluator/reporting/templates/report_ui.css.j2+++ b/src/skillevaluator/reporting/templates/report_ui.css.j2@@ -1,22 +1,23 @@/* SPDX-License-Identifier: Apache-2.0 *//* Report workspace: static evidence, readable without a network connection. */-:root{--light-bg:#f5f6f7;--bg:#fff;--bg-secondary:#f5f6f7;--bg-tertiary:#e8ebed;--dark-text:#24292f;--text-secondary:#59636e;--border-color:#dfe3e6;--primary-color:#26302b;--secondary-color:#487600;--success-color:#357000;--danger-color:#b4232d;--warning-color:#8a5800;--shadow:none;--shadow-lg:0 4px 16px #1f293708;--badge-pass-bg:#edf5e4;--badge-pass-tx:#356300;--badge-fail-bg:#fff0ef;--badge-fail-tx:#a3232b;--badge-warn-bg:#fff5df;--badge-warn-tx:#805300}+/* One palette built on the logo greens. #76b900 is decoration only; text colors meet WCAG AA (4.5:1) on --bg and --bg-secondary in both themes. */+:root{--brand-core:#76b900;--brand-green:#3e6500;--brand-deep:#315b25;--light-bg:#f5f6f7;--bg:#fff;--bg-secondary:#f5f6f7;--bg-tertiary:#e8ebed;--dark-text:#24292f;--text-secondary:#59636e;--border-color:#dfe3e6;--primary-color:#26302b;--secondary-color:#3e6500;--accent-color:#3e6500;--nv:#76b900;--success-color:#3e6500;--danger-color:#b4232d;--warning-color:#8a5800;--critical-color:#a3232b;--high-color:#b54708;--medium-color:#8a5800;--low-color:#59636e;--info-color:#0b6b9a;--on-accent:#fff;--on-success:#fff;--on-danger:#fff;--on-warning:#fff;--focus-ring:#3d6b00;--chart-grid:#e8ebed;--chart-tick:#59636e;--chart-label:#24292f;--table-header-bg:#f5f6f7;--table-row-hover:#f5f6f7;--shadow:none;--shadow-lg:0 4px 16px #1f293708;--badge-pass-bg:#edf5e4;--badge-pass-tx:#315b25;--badge-fail-bg:#fff0ef;--badge-fail-tx:#a3232b;--badge-warn-bg:#fff5df;--badge-warn-tx:#7a4e00;--sev-critical-bg:#fdecec;--sev-critical-tx:#a3232b;--sev-high-bg:#fff4ec;--sev-high-tx:#b54708;--sev-medium-bg:#fff5df;--sev-medium-tx:#7a4e00;--sev-low-bg:#eef0f2;--sev-low-tx:#4b5563;--sev-info-bg:#e8f2f8;--sev-info-tx:#0b5a82;--llm-tx:#6d28d9;--sidebar-bg:#1e2328;--sidebar-tx:#c7ced5;--se-logo-panel:#f2f7ec;--se-logo-panel-edge:#d7e5cc;--se-logo-ring-outer:#b8d3a5;--se-logo-ring-middle:#83aa66;--se-logo-ring-inner:#315b25;--se-logo-core:#76b900;--se-logo-core-edge:#3e6500;--se-logo-check:#fff}body{font-family:ui-sans-serif,-apple-system,BlinkMacSystemFont,"Segoe UI",sans-serif;font-size:14px;line-height:1.55;padding-left:216px;transition:none}-button,input,summary{font:inherit}button{cursor:pointer}a{color:var(--secondary-color)}button:focus-visible,a:focus-visible,input:focus-visible,summary:focus-visible,[tabindex]:focus-visible{outline:3px solid #668fbb;outline-offset:3px}code,pre{font-family:ui-monospace,SFMono-Regular,Consolas,monospace}code{overflow-wrap:anywhere}h1,h2,h3{letter-spacing:-.025em}table{width:100%}td,th{overflow-wrap:anywhere}pre{max-width:100%;overflow:auto}p{overflow-wrap:anywhere}[hidden]{display:none!important}+button,input,summary{font:inherit}button{cursor:pointer}a{color:var(--secondary-color)}button:focus-visible,a:focus-visible,input:focus-visible,summary:focus-visible,[tabindex]:focus-visible{outline:3px solid var(--focus-ring);outline-offset:3px}code,pre{font-family:ui-monospace,SFMono-Regular,Consolas,monospace}code{overflow-wrap:anywhere}h1,h2,h3{letter-spacing:-.025em}table{width:100%}td,th{overflow-wrap:break-word}th{white-space:nowrap}td code,td .code-snippet{overflow-wrap:anywhere}pre{max-width:100%;overflow:auto}p{overflow-wrap:anywhere}[hidden]{display:none!important}.skip-link{position:fixed;left:228px;top:-80px;background:var(--bg);z-index:1000;padding:12px 20px}.skip-link:focus{top:8px}-.report-sidebar{position:fixed;inset:0 auto 0 0;width:216px;background:#1e2328;color:#c7ced5;display:flex;flex-direction:column;padding:29px 16px;z-index:110}.report-sidebar a{color:#c7ced5;text-decoration:none;border-radius:6px;padding:10px 12px;margin-bottom:4px;font-size:13px}.report-sidebar a:hover{background:#2a3036;color:#fff}.report-sidebar .sidebar-brand{display:flex;align-items:center;gap:10px;color:white;padding:0 6px 37px;font-weight:650;font-size:16px;letter-spacing:-.5px}.sidebar-brand small{display:block;font-size:8px;letter-spacing:1.2px;color:#9da7af;margin-top:6px}.brand-mark{border:1px solid #76b900;border-radius:8px;color:#98cd45;width:30px;height:33px;display:grid;place-items:center;font-family:ui-monospace,monospace;font-size:19px;font-weight:500}.sidebar-label{color:#89949e;font-size:9px;letter-spacing:1.5px;margin:0 12px 12px}.report-sidebar .sidebar-current{background:#30392b;color:#c3e498;border-left:2px solid #76b900}.sidebar-footer{margin-top:auto;padding:14px 12px 0;color:#939da6;font-size:11px;line-height:1.8}-.main-header{position:relative;top:auto;background:var(--bg);color:var(--dark-text);padding:15px 0;border-bottom:1px solid var(--border-color);box-shadow:none;z-index:10}.header-content{max-width:1320px;padding:0 40px;flex-wrap:nowrap}.header-title{font-size:15px;font-weight:650;letter-spacing:-.4px}.header-subtitle{font-size:11px;color:var(--text-secondary)}.header-meta{display:flex;flex-direction:row;flex-wrap:wrap;gap:8px;justify-content:flex-end}.header-controls{display:flex;gap:8px}.theme-toggle,.export-btn{color:var(--dark-text);background:var(--bg);border:1px solid var(--border-color);font-size:12px;padding:7px 11px;box-shadow:none}.theme-toggle:hover,.export-btn:hover{background:var(--bg-secondary)}.header-status-badge{font-size:10px;padding:5px 8px;letter-spacing:0;text-transform:none;border:1px solid var(--border-color)}.header-status-badge.pass{color:var(--badge-pass-tx);background:var(--badge-pass-bg)}.header-status-badge.fail{background:var(--badge-fail-bg);color:var(--badge-fail-tx);border-color:#f2d6d6}.header-status-badge.warn{background:var(--badge-warn-bg);color:var(--badge-warn-tx);border-color:#ead9ae}-.container{max-width:1320px;margin:0 auto;padding:30px 40px 64px;min-width:0}.report-breadcrumb{font-size:11px;color:var(--text-secondary);margin-bottom:20px}.agent-hero{background:var(--bg);border:1px solid var(--border-color);border-radius:10px;padding:26px 28px;margin-bottom:16px;box-shadow:none}.agent-hero-eyebrow{font-size:10px;letter-spacing:1.4px;color:var(--text-secondary);text-transform:uppercase}.agent-hero-title{font-size:28px;line-height:1.22;font-weight:650;margin-top:8px;overflow-wrap:anywhere}.agent-hero-top{gap:24px}.agent-hero-subtitle{font-size:12px;overflow-wrap:anywhere;max-width:650px;margin-top:12px}.agent-hero-score{font-size:35px}.agent-hero-score.verdict-text{font-size:23px;font-weight:650;max-width:250px;line-height:1.15}.verdict-caption{font-size:10px;color:var(--text-secondary);text-align:right;margin:7px 0 12px;text-transform:uppercase;letter-spacing:1px}.agent-hero-detail{font-size:11px;line-height:1.8}.score-pass{color:var(--success-color)}.score-fail{color:var(--danger-color)}.score-warn{color:var(--warning-color)}-.tier-cards{display:flex;gap:8px;flex-wrap:wrap;margin-top:24px;padding-top:20px;border-top:1px solid var(--border-color)}.tier-card{flex:1;min-width:180px;border:1px solid var(--border-color);border-radius:6px;padding:12px 14px;background:var(--bg);box-shadow:none;border-left:1px solid var(--border-color);cursor:pointer}.tier-card.active{border-color:#7c9c52;box-shadow:inset 0 0 0 1px #7c9c52;background:var(--bg-secondary)}.tier-card:hover{transform:none;box-shadow:none}.tier-card-label{font-size:10px;color:var(--text-secondary)}.tier-card-title{font-size:12px;font-weight:600;color:var(--dark-text);margin:6px 0}.tier-card-stats{font-size:10px;line-height:1.6}.tier-card-verdict{font-size:9px}.tier-card-sev-pill{font-size:9px}.scope-strip{display:flex;flex-wrap:wrap;gap:8px 20px;font-size:11px;color:var(--text-secondary);padding:0 2px 16px}.scope-strip span+span{border-left:1px solid var(--border-color);padding-left:18px}-.run-metadata{border-top:1px solid var(--border-color);border-bottom:1px solid var(--border-color);padding:12px 2px;margin-bottom:28px;font-size:12px;scroll-margin-top:20px}.run-metadata summary{cursor:pointer;font-weight:600}.metadata-grid{display:grid;grid-template-columns:repeat(2,minmax(0,1fr));gap:15px 28px;margin-top:20px;padding-bottom:8px}.metadata-grid p,.metadata-grid .header-meta-item{font-size:11px;opacity:1;color:var(--text-secondary);text-align:left}.metadata-grid strong{display:block;color:var(--dark-text);font-size:10px;margin-bottom:5px}.metadata-grid code{font-size:10px;word-break:break-all}.header-target-link{color:var(--secondary-color)}.workspace-heading{margin:0 0 20px;scroll-margin-top:20px}.workspace-heading h2{font-size:19px;font-weight:650;margin-bottom:5px}.workspace-heading p{color:var(--text-secondary);font-size:12px}-.dashboard{display:flex;flex-wrap:wrap;gap:0;margin:18px 0 24px;border:1px solid var(--border-color);background:var(--bg);border-radius:8px;padding:14px 0}.dashboard-card{flex:1;min-width:130px;border:0;border-right:1px solid var(--border-color);border-radius:0;padding:0 18px;box-shadow:none;background:transparent}.dashboard-card:last-child{border-right:0}.dashboard-card:hover{box-shadow:none}.dashboard-card-title{font-size:9px;letter-spacing:.8px}.dashboard-card-value{font-size:21px}.dashboard-card-subtitle{font-size:10px}.dashboard .progress-bar{display:none}.severity-breakdown-label{font-size:9px}-.top-issues-section{border:1px solid #ead6d5;border-radius:8px;box-shadow:none;margin:0 0 18px;background:var(--bg);overflow:hidden}.top-issues-header{background:var(--badge-fail-bg);padding:13px 18px;color:var(--badge-fail-tx);border:0}.top-issues-header h3{font-size:13px;color:inherit;font-weight:650}.top-issues-content.active{max-height:none}.top-issues-table{font-size:11px}.top-issues-table th{background:var(--bg);font-size:9px;letter-spacing:.4px;text-transform:uppercase;color:var(--text-secondary);padding:12px 15px}.top-issues-table td{padding:12px 15px;vertical-align:top}.top-issues-table tbody tr{cursor:pointer}.skills-pills{gap:5px}.skill-pill,.quality-pill{font-size:9px}.count-badge,.metadata-tag,.severity-tag{font-size:9px;box-shadow:none}.contributor-section{margin-top:16px;border:1px solid var(--border-color);box-shadow:none}.contributor-header{background:var(--bg-secondary);padding:12px 16px}.contributor-header h3{font-size:12px}.contributor-content.active{max-height:none}-.report-searchbar{display:flex;align-items:center;gap:10px;flex-wrap:wrap;margin-bottom:15px}.report-searchbar label{font-size:11px;font-weight:600}.report-searchbar input{min-width:190px;flex:1;padding:10px 12px;border:1px solid var(--border-color);border-radius:6px;background:var(--bg);color:var(--dark-text);font-size:12px}.report-searchbar #report-search-count{font-size:10px;color:var(--text-secondary)}.report-no-matches{padding:24px;border:1px dashed var(--border-color);border-radius:8px;margin-bottom:18px;background:var(--bg);color:var(--text-secondary)}.view-toggle-section{background:none;border:0;padding:0;gap:6px}.view-toggle-btn,.filter-btn,.bulk-action-btn,.severity-filter-btn{font-size:11px;padding:7px 11px;min-height:32px;border-radius:5px;background:var(--bg);color:var(--dark-text);border:1px solid var(--border-color);box-shadow:none}.view-toggle-btn.active,.filter-btn.active,.filter-btn.success.active,.filter-btn.danger.active,.severity-filter-btn.active{background:#eaf1df;border-color:#9db67a;color:#385800}.severity-filter-group{margin-left:auto}.skills-summary-bar{background:transparent;border:0;padding:10px 0;margin-bottom:12px;font-size:11px}.section-header{font-size:14px;letter-spacing:-.1px}.skill-card,.validator-card{border:1px solid var(--border-color);box-shadow:none;margin-bottom:10px;border-radius:8px;background:var(--bg)}.skill-header{padding:15px 18px;background:var(--bg);gap:14px}.skill-name{font-size:13px;overflow-wrap:anywhere;min-width:0;font-weight:600}.skill-checks-summary{font-size:10px;flex-wrap:wrap;gap:7px}.skill-checks-summary .status-badge{font-size:10px}.status-badge{font-size:10px;border-radius:4px;padding:3px 7px;text-transform:none}.skill-content.active{max-height:none;padding:18px;border-top:1px solid var(--border-color)}.validator-collapsible{padding:14px 18px;background:var(--bg);font-size:12px;gap:12px;text-align:left}.validator-content.active{max-height:none;padding:18px}.validator-name{font-size:13px}.check-finding{box-shadow:none;border-radius:5px}.quality-score-block{background:var(--bg-secondary)!important}.quality-score-block strong{font-size:14px!important}.finding-message{font-size:12px}.finding-suggestion{font-size:11px}.finding-suggestion.truncatable{max-height:none;overflow:visible;cursor:auto}.summary-table-wrapper,.top-issues-content,.contributor-content{overflow-x:auto}.skill-card,.validator-card{scroll-margin-top:20px}footer{margin-left:0;padding:20px 40px;border-top:1px solid var(--border-color);font-size:10px;background:var(--bg);text-align:center}#backToTop{background:#303b28;color:white;border-radius:6px;box-shadow:none}-[data-theme="dark"]{--light-bg:#151a1f;--bg:#1e252c;--bg-secondary:#252d35;--bg-tertiary:#303b46;--dark-text:#e1e6eb;--text-secondary:#aab5bf;--border-color:#3b454f;--success-color:#a2cf72;--danger-color:#ffadb1;--warning-color:#e7c778;--secondary-color:#b0d684;--badge-pass-bg:#293e27;--badge-pass-tx:#c4e7a0;--badge-fail-bg:#412a2d;--badge-fail-tx:#ffc1c4;--badge-warn-bg:#403826;--badge-warn-tx:#f4d792}[data-theme="dark"] .filter-btn.active,[data-theme="dark"] .severity-filter-btn.active,[data-theme="dark"] .view-toggle-btn.active{background:#344a2e;color:#d4e9c2}-@media(min-width:1600px){.container{padding-top:40px}}@media(max-width:1150px){body{padding-left:180px}.report-sidebar{width:180px}.report-sidebar .sidebar-brand{font-size:13px;gap:7px}.brand-mark{width:25px}.container{padding:25px}.header-content{padding:0 25px}.header-content{flex-wrap:wrap}.header-status-badge{display:inline-flex;white-space:normal}.dashboard-card{min-width:110px;padding:0 12px}.agent-hero-title{font-size:25px}.top-issues-table th,.top-issues-table td{padding:10px}.severity-filter-group{margin-left:0}}-@media(max-width:760px){body{padding-left:0}.report-sidebar{position:relative;width:100%;padding:13px 18px;flex-direction:row;align-items:center;gap:6px;flex-wrap:wrap}.report-sidebar .sidebar-brand{padding:0;margin-right:auto;font-size:14px}.sidebar-brand small,.sidebar-label,.sidebar-footer,.report-sidebar>a:not(.sidebar-brand){display:none}.main-header{padding:10px 0}.header-content{padding:0 18px;gap:8px}.logo-section{display:none}.header-meta{width:100%;justify-content:space-between}.header-controls{width:100%;justify-content:flex-end}.header-controls button{min-height:36px}.container{padding:20px 16px 40px}.agent-hero{padding:20px}.agent-hero-top{display:block}.agent-hero-title{font-size:25px}.agent-hero-score.verdict-text,.agent-hero-score,.agent-hero-detail,.verdict-caption{text-align:left}.agent-hero-score.verdict-text{margin-top:22px;font-size:22px}.agent-hero-detail{margin-top:8px}.verdict-caption{margin-bottom:5px}.tier-cards{display:block}.tier-card{margin-top:8px;min-width:0}.tier-card-stats{display:flex;flex-wrap:wrap;gap:8px}.scope-strip{gap:6px 12px;font-size:10px}.scope-strip span+span{padding-left:0;border:0}.scope-strip span:first-child{width:100%}.metadata-grid{grid-template-columns:1fr}.dashboard{gap:16px 0}.dashboard-card{min-width:45%;padding-left:14px}.dashboard-card:nth-child(2n){border-right:0}.dashboard-card-value{font-size:20px}.report-searchbar input{width:100%;flex-basis:100%;order:2}.report-searchbar label{flex:1}.report-searchbar #report-search-count{order:3}.filter-group{gap:6px}.filter-group .severity-filter-group{width:100%;padding-top:8px}.skill-header{display:block;padding:14px}.skill-name{font-size:12px;word-break:break-word}.skill-checks-summary{margin-top:12px}.skill-content.active{padding:12px}.top-issues-table{min-width:620px}.quality-score-block>div:first-child{flex-wrap:wrap;gap:8px}.validator-collapsible{flex-wrap:wrap}.workspace-heading h2{font-size:18px}.workspace-heading p{font-size:11px}.skip-link{left:10px}footer{padding:18px;font-size:9px}}+.report-sidebar{position:fixed;inset:0 auto 0 0;width:216px;background:var(--sidebar-bg);color:var(--sidebar-tx);display:flex;flex-direction:column;padding:29px 16px;z-index:110}.report-sidebar a{color:#c7ced5;text-decoration:none;border-radius:6px;padding:10px 12px;margin-bottom:4px;font-size:13px}.report-sidebar a:hover{background:#2a3036;color:#fff}.report-sidebar .sidebar-brand{display:flex;align-items:center;gap:10px;color:white;padding:0 6px 37px;font-weight:650;font-size:16px;letter-spacing:-.5px;white-space:nowrap}.sidebar-brand-text{min-width:0;line-height:1.25}.sidebar-brand small{display:block;font-size:11px;font-weight:500;letter-spacing:0;color:#9da7af;margin-top:3px;overflow:hidden;text-overflow:ellipsis}.sidebar-label{color:#89949e;font-size:11px;letter-spacing:1.5px;margin:0 12px 12px}.report-sidebar .sidebar-current{background:#30392b;color:#c3e498;border-left:2px solid #76b900}.sidebar-footer{margin-top:auto;padding:14px 12px 0;color:#939da6;font-size:11px;line-height:1.8}+.main-header{position:relative;top:auto;background:var(--bg);color:var(--dark-text);padding:15px 0;border-bottom:1px solid var(--border-color);box-shadow:none;z-index:10}.header-content{max-width:1320px;padding:0 40px;flex-wrap:nowrap}.header-title{font-size:15px;font-weight:650;letter-spacing:-.4px}.header-subtitle{font-size:11px;color:var(--text-secondary)}.header-meta{display:flex;flex-direction:row;flex-wrap:wrap;gap:8px;justify-content:flex-end}.header-controls{display:flex;gap:8px}.theme-toggle,.export-btn{color:var(--dark-text);background:var(--bg);border:1px solid var(--border-color);font-size:12px;padding:7px 11px;box-shadow:none}.theme-toggle:hover,.export-btn:hover{background:var(--bg-secondary)}.header-status-badge{font-size:12px;padding:5px 8px;letter-spacing:0;text-transform:none;border:1px solid var(--border-color)}.header-status-badge.pass{color:var(--badge-pass-tx);background:var(--badge-pass-bg)}.header-status-badge.fail{background:var(--badge-fail-bg);color:var(--badge-fail-tx);border-color:#f2d6d6}.header-status-badge.warn{background:var(--badge-warn-bg);color:var(--badge-warn-tx);border-color:#ead9ae}+.container{max-width:1320px;margin:0 auto;padding:30px 40px 64px;min-width:0}.report-breadcrumb{font-size:11px;color:var(--text-secondary);margin-bottom:20px}.agent-hero{background:var(--bg);border:1px solid var(--border-color);border-radius:10px;padding:26px 28px;margin-bottom:16px;box-shadow:none}.agent-hero-eyebrow{font-size:12px;letter-spacing:1.4px;color:var(--text-secondary);text-transform:uppercase}.agent-hero-title{font-size:28px;line-height:1.22;font-weight:650;margin-top:8px;overflow-wrap:anywhere}.agent-hero-top{gap:24px}.agent-hero-subtitle{font-size:12px;overflow-wrap:anywhere;max-width:650px;margin-top:12px}.agent-hero-score{font-size:35px}.agent-hero-score.verdict-text{font-size:23px;font-weight:650;max-width:250px;line-height:1.15}.verdict-caption{font-size:12px;color:var(--text-secondary);text-align:right;margin:7px 0 12px;text-transform:uppercase;letter-spacing:1px}.agent-hero-detail{font-size:11px;line-height:1.8}.score-pass{color:var(--success-color)}.score-fail{color:var(--danger-color)}.score-warn{color:var(--warning-color)}+.tier-cards{display:flex;gap:8px;flex-wrap:wrap;margin-top:24px;padding-top:20px;border-top:1px solid var(--border-color)}.tier-card{flex:1;min-width:180px;border:1px solid var(--border-color);border-radius:6px;padding:12px 14px;background:var(--bg);box-shadow:none;border-left:1px solid var(--border-color);cursor:pointer}.tier-card.active{box-shadow:inset 0 0 0 1px currentColor;background:var(--bg-secondary)}.tier-card.pass{border-color:var(--success-color);color:var(--success-color)}.tier-card.fail{border-color:var(--danger-color);color:var(--danger-color)}.tier-card.warn{border-color:var(--warning-color);color:var(--warning-color)}.tier-card .tier-card-title,.tier-card .tier-card-stats{color:var(--dark-text)}.tier-cards.single-tier .tier-card{cursor:default}.tier-card:hover{transform:none;box-shadow:none}.tier-card-label{font-size:12px;color:var(--text-secondary)}.tier-card-title{font-size:12px;font-weight:600;color:var(--dark-text);margin:6px 0}.tier-card-stats{font-size:12px;line-height:1.6}.tier-card-verdict{font-size:11px}.tier-card-sev-pill{font-size:11px}.scope-strip{display:flex;flex-wrap:wrap;gap:8px 20px;font-size:11px;color:var(--text-secondary);padding:0 2px 16px}.scope-strip span+span{border-left:1px solid var(--border-color);padding-left:18px}+.run-metadata{border-top:1px solid var(--border-color);border-bottom:1px solid var(--border-color);padding:12px 2px;margin-bottom:28px;font-size:12px;scroll-margin-top:20px}.run-metadata summary{cursor:pointer;font-weight:600}.metadata-grid{display:grid;grid-template-columns:repeat(2,minmax(0,1fr));gap:15px 28px;margin-top:20px;padding-bottom:8px}.metadata-grid p,.metadata-grid .header-meta-item{font-size:11px;opacity:1;color:var(--text-secondary);text-align:left}.metadata-grid strong{display:block;color:var(--dark-text);font-size:12px;margin-bottom:5px}.metadata-grid code{font-size:12px;word-break:break-all}.header-target-link{color:var(--secondary-color)}.workspace-heading{margin:0 0 20px;scroll-margin-top:20px}.workspace-heading h2{font-size:19px;font-weight:650;margin-bottom:5px}.workspace-heading p{color:var(--text-secondary);font-size:12px}+.dashboard{display:flex;flex-wrap:wrap;gap:0;margin:18px 0 24px;border:1px solid var(--border-color);background:var(--bg);border-radius:8px;padding:14px 0}.dashboard-card{flex:1;min-width:130px;border:0;border-right:1px solid var(--border-color);border-radius:0;padding:0 18px;box-shadow:none;background:transparent}.dashboard-card:last-child{border-right:0}.dashboard-card:hover{box-shadow:none}.dashboard-card-title{font-size:11px;letter-spacing:.8px}.dashboard-card-value{font-size:21px}.dashboard-card-subtitle{font-size:12px}.dashboard .progress-bar{display:none}.severity-breakdown-label{font-size:11px}+.top-issues-section{border:1px solid #ead6d5;border-radius:8px;box-shadow:none;margin:0 0 18px;background:var(--bg);overflow:hidden}.top-issues-header{background:var(--badge-fail-bg);padding:13px 18px;color:var(--badge-fail-tx);border:0}.top-issues-header h3{font-size:13px;color:inherit;font-weight:650}.top-issues-content.active{max-height:none}.top-issues-table{font-size:11px}.top-issues-table th{background:var(--bg);font-size:11px;letter-spacing:.4px;text-transform:uppercase;color:var(--text-secondary);padding:12px 15px}.top-issues-table td{padding:12px 15px;vertical-align:top}.top-issues-table tbody tr{cursor:pointer}.skills-pills{gap:5px}.skill-pill,.quality-pill{font-size:11px}.count-badge,.metadata-tag,.severity-tag{font-size:11px;box-shadow:none}.contributor-section{margin-top:16px;border:1px solid var(--border-color);box-shadow:none}.contributor-header{background:var(--bg-secondary);padding:12px 16px}.contributor-header h3{font-size:12px}.contributor-content.active{max-height:none}+.report-searchbar{display:flex;align-items:center;gap:10px;flex-wrap:wrap;margin-bottom:15px}.report-searchbar label{font-size:11px;font-weight:600}.report-searchbar input{min-width:190px;flex:1;padding:10px 12px;border:1px solid var(--border-color);border-radius:6px;background:var(--bg);color:var(--dark-text);font-size:12px}.report-searchbar #report-search-count{font-size:12px;color:var(--text-secondary)}.report-no-matches{padding:24px;border:1px dashed var(--border-color);border-radius:8px;margin-bottom:18px;background:var(--bg);color:var(--text-secondary)}.view-toggle-section{background:none;border:0;padding:0;gap:6px}.view-toggle-btn,.filter-btn,.bulk-action-btn,.severity-filter-btn{font-size:11px;padding:7px 11px;min-height:32px;border-radius:5px;background:var(--bg);color:var(--dark-text);border:1px solid var(--border-color);box-shadow:none}.view-toggle-btn.active,.filter-btn.active,.filter-btn.success.active,.filter-btn.danger.active,.severity-filter-btn.active{background:var(--badge-pass-bg);border-color:var(--success-color);color:var(--badge-pass-tx)}.severity-filter-group{margin-left:auto}.skills-summary-bar{background:transparent;border:0;padding:10px 0;margin-bottom:12px;font-size:11px}.section-header{font-size:14px;letter-spacing:-.1px}.skill-card,.validator-card{border:1px solid var(--border-color);box-shadow:none;margin-bottom:10px;border-radius:8px;background:var(--bg)}.skill-header{padding:15px 18px;background:var(--bg);gap:14px}.skill-name{font-size:13px;overflow-wrap:anywhere;min-width:0;font-weight:600}.skill-checks-summary{font-size:12px;flex-wrap:wrap;gap:7px}.skill-checks-summary .status-badge{font-size:12px}.status-badge{font-size:12px;border-radius:4px;padding:3px 7px;text-transform:none}.skill-content.active{max-height:none;padding:18px;border-top:1px solid var(--border-color)}.validator-collapsible{padding:14px 18px;background:var(--bg);font-size:12px;gap:12px;text-align:left}.validator-content.active{max-height:none;padding:18px}.validator-name{font-size:13px}.check-finding{box-shadow:none;border-radius:5px}.quality-score-block{background:var(--bg-secondary)!important}.quality-score-block strong{font-size:14px!important}.finding-message{font-size:12px}.finding-suggestion{font-size:11px}.finding-suggestion.truncatable{max-height:none;overflow:visible;cursor:auto}.summary-table-wrapper,.top-issues-content,.contributor-content{overflow-x:auto}.skill-card,.validator-card{scroll-margin-top:20px}footer{margin-left:0;padding:20px 40px;border-top:1px solid var(--border-color);font-size:12px;background:var(--bg);text-align:center}#backToTop{background:var(--brand-deep);color:#fff;border-radius:6px;box-shadow:none}+@media screen{[data-theme="dark"]{--light-bg:#151a1f;--bg:#1e252c;--bg-secondary:#252d35;--bg-tertiary:#303b46;--dark-text:#e1e6eb;--text-secondary:#aab5bf;--border-color:#3b454f;--secondary-color:#b0d684;--accent-color:#b0d684;--success-color:#a2cf72;--danger-color:#ffadb1;--warning-color:#e7c778;--critical-color:#ffadb1;--high-color:#f7b27a;--medium-color:#e7c778;--low-color:#aab5bf;--info-color:#8ccff0;--on-accent:#151a1f;--on-success:#151a1f;--on-danger:#151a1f;--on-warning:#151a1f;--focus-ring:#b0d684;--chart-grid:#4b5661;--chart-tick:#aab5bf;--chart-label:#e1e6eb;--table-header-bg:#252d35;--table-row-hover:#303b46;--badge-pass-bg:#293e27;--badge-pass-tx:#c4e7a0;--badge-fail-bg:#412a2d;--badge-fail-tx:#ffc1c4;--badge-warn-bg:#403826;--badge-warn-tx:#f4d792;--sev-critical-bg:#412a2d;--sev-critical-tx:#ffc1c4;--sev-high-bg:#45301f;--sev-high-tx:#ffcd9e;--sev-medium-bg:#403826;--sev-medium-tx:#f4d792;--sev-low-bg:#303b46;--sev-low-tx:#d0d7de;--sev-info-bg:#1d3646;--sev-info-tx:#a9d8f5;--llm-tx:#c4b5fd;--sidebar-bg:#151a1f;--se-logo-panel:#152117;--se-logo-panel-edge:#304631;--se-logo-ring-outer:#39523a;--se-logo-ring-middle:#6f9264;--se-logo-ring-inner:#dbe8d6;--se-logo-core:#76b900;--se-logo-core-edge:#9bdb34}[data-theme="dark"] .filter-btn.active,[data-theme="dark"] .severity-filter-btn.active,[data-theme="dark"] .view-toggle-btn.active{background:#344a2e;color:#d4e9c2}}+@media(min-width:1600px){.container{padding-top:40px}}@media(max-width:1150px){body{padding-left:180px}.report-sidebar{width:180px}.report-sidebar .sidebar-brand{font-size:13px;gap:7px}.container{padding:25px}.header-content{padding:0 25px}.header-content{flex-wrap:wrap}.header-status-badge{display:inline-flex;white-space:normal}.dashboard-card{min-width:110px;padding:0 12px}.agent-hero-title{font-size:25px}.top-issues-table th,.top-issues-table td{padding:10px}.severity-filter-group{margin-left:0}}+@media(max-width:760px){body{padding-left:0}.report-sidebar{position:relative;width:100%;padding:13px 18px;flex-direction:row;align-items:center;gap:6px;flex-wrap:wrap}.report-sidebar .sidebar-brand{padding:0;margin-right:auto;font-size:14px}.sidebar-brand small,.sidebar-label,.sidebar-footer{display:none}.report-sidebar>a:not(.sidebar-brand){margin:0;padding:8px 10px;white-space:nowrap}.report-sidebar{overflow-x:auto;flex-wrap:nowrap}.main-header{padding:10px 0}.header-content{padding:0 18px;gap:8px}.header-brand .header-subtitle{display:none}.header-meta{width:100%;justify-content:space-between}.header-controls{width:100%;justify-content:flex-end}.header-controls button{min-height:36px}.container{padding:20px 16px 40px}.agent-hero{padding:20px}.agent-hero-top{display:block}.agent-hero-title{font-size:25px}.agent-hero-score.verdict-text,.agent-hero-score,.agent-hero-detail,.verdict-caption{text-align:left}.agent-hero-score.verdict-text{margin-top:22px;font-size:22px}.agent-hero-detail{margin-top:8px}.verdict-caption{margin-bottom:5px}.tier-cards{display:block}.tier-card{margin-top:8px;min-width:0}.tier-card-stats{display:flex;flex-wrap:wrap;gap:8px}.scope-strip{gap:6px 12px;font-size:12px}.scope-strip span+span{padding-left:0;border:0}.scope-strip span:first-child{width:100%}.metadata-grid{grid-template-columns:1fr}.dashboard{gap:16px 0}.dashboard-card{min-width:45%;padding-left:14px}.dashboard-card:nth-child(2n){border-right:0}.dashboard-card-value{font-size:20px}.report-searchbar input{width:100%;flex-basis:100%;order:2}.report-searchbar label{flex:1}.report-searchbar #report-search-count{order:3}.filter-group{gap:6px}.filter-group .severity-filter-group{width:100%;padding-top:8px}.skill-header{display:block;padding:14px}.skill-name{font-size:12px;word-break:break-word}.skill-checks-summary{margin-top:12px}.skill-content.active{padding:12px}.top-issues-table{min-width:620px}.quality-score-block>div:first-child{flex-wrap:wrap;gap:8px}.validator-collapsible{flex-wrap:wrap}.workspace-heading h2{font-size:18px}.workspace-heading p{font-size:11px}.skip-link{left:10px}footer{padding:18px;font-size:11px}}@media(prefers-reduced-motion:reduce){html{scroll-behavior:auto}*,*::before,*::after{transition:none!important;animation:none!important}}-@media print{body{padding-left:0;background:#fff;color:#111}.report-sidebar,.header-controls,.report-searchbar,.filter-group,.view-toggle-section,#backToTop,.report-no-matches{display:none!important}.container{max-width:none;padding:0}.tab-content,.results-view,.skill-card,.validator-card,.skill-content,.validator-content,.top-issues-content,.contributor-content,.summary-table-container{display:block!important;max-height:none!important;height:auto!important;overflow:visible!important}.skill-card,.validator-card,.top-issues-section{break-inside:auto}.skill-content,.validator-content{padding:12px!important}.scope-strip,.run-metadata{font-size:10px}.agent-hero{padding:15px}.run-metadata>.metadata-grid{display:grid!important}.finding-suggestion{max-height:none!important}footer{padding:12px}.dashboard-card{min-width:100px}.top-issues-table{min-width:0}.report-breadcrumb{margin-bottom:8px}a{color:inherit}}+@media print{body{padding-left:0;background:#fff;color:#111}.skip-link,.report-sidebar,.header-controls,.report-searchbar,.filter-group,.view-toggle-section,#backToTop,.report-no-matches{display:none!important}.container{max-width:none;padding:0}.tab-content,.results-view,.skill-card,.validator-card,.skill-content,.validator-content,.top-issues-content,.contributor-content,.summary-table-container{display:block!important;max-height:none!important;height:auto!important;overflow:visible!important}.skill-card,.validator-card,.top-issues-section{break-inside:auto}.skill-content,.validator-content{padding:12px!important}.scope-strip,.run-metadata{font-size:12px}.agent-hero{padding:15px}.run-metadata>.metadata-grid{display:grid!important}.finding-suggestion{max-height:none!important}footer{padding:12px}.dashboard-card{min-width:100px}.top-issues-table{min-width:0}.report-breadcrumb{margin-bottom:8px}a{color:inherit}}.quality-outcome-note{font-size:11px;color:var(--text-secondary);margin-bottom:12px}.export-btn{text-decoration:none}@@ -35,4 +36,48 @@ details.check-result>summary{cursor:pointer;list-style:none;margin-bottom:0}deta.gate-policy{display:flex;flex-wrap:wrap;gap:8px 18px;font-size:11px;color:var(--text-secondary);padding:0 2px 18px}.gate-policy strong{color:var(--dark-text);font-weight:600}-@media print{.main-header{display:block!important}.header-content{padding:0;flex-wrap:wrap}.header-status-badge{display:inline-flex!important}.header-meta{justify-content:flex-start}}++/* Severity tags: one token pair per level. The [data-theme] selectors outrank the base template's hard-coded dark colors. */+.severity-critical,[data-theme="dark"] .severity-critical{background:var(--sev-critical-bg);color:var(--sev-critical-tx)}.severity-high,[data-theme="dark"] .severity-high{background:var(--sev-high-bg);color:var(--sev-high-tx)}.severity-medium,[data-theme="dark"] .severity-medium{background:var(--sev-medium-bg);color:var(--sev-medium-tx)}.severity-low,[data-theme="dark"] .severity-low{background:var(--sev-low-bg);color:var(--sev-low-tx)}.severity-info,[data-theme="dark"] .severity-info{background:var(--sev-info-bg);color:var(--sev-info-tx)}+.severity-tag,.status-badge,.metadata-tag,.count-badge,.skill-pill,.quality-pill,.tier-card-verdict,.tier-card-sev-pill{white-space:nowrap}+/* Confidence badges: the type floor, and the palette's AA pairs in both themes. */+.confidence-badge{font-size:11px}.confidence-badge.high-conf,[data-theme="dark"] .confidence-badge.high-conf{background:var(--badge-fail-bg);color:var(--badge-fail-tx)}.confidence-badge.medium-conf,[data-theme="dark"] .confidence-badge.medium-conf{background:var(--badge-warn-bg);color:var(--badge-warn-tx)}.confidence-badge.low-conf,[data-theme="dark"] .confidence-badge.low-conf{background:var(--sev-low-bg);color:var(--sev-low-tx);border-color:var(--border-color)}+.count-badge.medium-count{background:var(--warning-color);color:var(--on-warning)}.count-badge.low-count{background:var(--low-color);color:var(--on-accent)}++/* Header, hero, executive summary and Fix first. */+.sr-only{position:absolute;width:1px;height:1px;padding:0;margin:-1px;overflow:hidden;clip:rect(0,0,0,0);white-space:nowrap;border:0}+.se-logo{flex:none;display:block}+/* The sidebar is dark in both themes, so its logo always uses the dark panel colors. */+.report-sidebar .se-logo{--se-logo-panel:#152117;--se-logo-panel-edge:#304631;--se-logo-ring-outer:#39523a;--se-logo-ring-middle:#6f9264;--se-logo-ring-inner:#dbe8d6;--se-logo-core-edge:#9bdb34}+/* One logo per view: the sidebar on wide screens, the header on narrow screens and in print. */+.header-brand{display:flex;align-items:center;gap:10px;min-width:0}.header-brand .se-logo,.header-brand .header-title{display:none}.header-brand .header-title{margin:0;font-size:15px;font-weight:650;line-height:1.2;letter-spacing:-.3px}.header-brand .header-subtitle{margin:0;font-size:12px;color:var(--text-secondary);opacity:1}+.header-content{display:flex;align-items:center;justify-content:space-between;gap:12px;margin:0 auto}.header-meta{align-items:center}.theme-toggle[aria-pressed="true"]{background:var(--bg-tertiary);border-color:var(--text-secondary)}+.verdict-badge{display:inline-flex;align-items:center;gap:7px;font-size:13px!important;font-weight:650;padding:6px 11px!important;border-radius:999px;white-space:nowrap}.verdict-dot{width:8px;height:8px;border-radius:50%;background:currentColor}+.verdict-chip{display:inline-block;font-size:15px;font-weight:700;padding:5px 12px;border-radius:999px;border:1px solid currentColor;white-space:nowrap}.verdict-chip.pass{background:var(--badge-pass-bg);color:var(--badge-pass-tx)}.verdict-chip.fail{background:var(--badge-fail-bg);color:var(--badge-fail-tx)}.verdict-chip.warn{background:var(--badge-warn-bg);color:var(--badge-warn-tx)}+.verdict-badge,.verdict-chip,.tier-card-verdict,.severity-tag,.status-badge,.sev-dot{print-color-adjust:exact;-webkit-print-color-adjust:exact}+.report-breadcrumb a{color:var(--text-secondary)}.agent-hero-title{color:var(--dark-text)!important}.hero-skills{margin:10px 0 0;font-size:12px;color:var(--text-secondary)}+.agent-hero-top{display:flex;justify-content:space-between;align-items:flex-start;gap:24px}.hero-side{display:flex;flex-direction:column;align-items:flex-end;gap:8px;text-align:right;max-width:380px;min-width:0}.hero-side .agent-hero-score,.hero-side .agent-hero-detail{text-align:right}+.hero-score-label{display:block;font-size:12px;font-weight:500;color:var(--text-secondary);margin-top:4px}.hero-reason{margin:0;font-size:12px;color:var(--text-secondary);max-width:340px}+.publication-line{margin:0;font-size:12px;color:var(--text-secondary);display:flex;flex-wrap:wrap;justify-content:flex-end;align-items:center;gap:4px 8px}.publication-label{font-weight:600;color:var(--dark-text)}.publication-reasons{flex-basis:100%;line-height:1.5}+.tier-card-title{margin:6px 0}/* One Tier 1 card is a scope line: the executive summary carries its verdict and numbers. */.tier-line{margin-top:16px;padding-top:14px}.tier-line .tier-card,.tier-line .tier-card.active{display:flex;flex-direction:row;flex-wrap:wrap;align-items:baseline;gap:4px 14px;border:0;box-shadow:none;background:none;padding:0;min-width:0;color:var(--text-secondary);opacity:1}.tier-line .tier-card::before{display:none}.tier-line .tier-card:hover{transform:none;box-shadow:none}.tier-line .tier-card-head,.tier-line .tier-card-stats{display:contents}.tier-line .tier-card-title{margin:0;font-size:13px}.tier-card-verdict{font-size:12px!important;font-weight:700;padding:2px 8px;border-radius:999px}+.exec-summary{border:1px solid var(--border-color);border-left:4px solid var(--border-color);border-radius:10px;background:var(--bg);padding:20px 22px;margin:0 0 24px}.exec-summary.fail{border-left-color:var(--danger-color)}.exec-summary.pass{border-left-color:var(--success-color)}.exec-summary.warn{border-left-color:var(--warning-color)}+.exec-verdict{display:flex;flex-wrap:wrap;align-items:center;gap:8px 14px}.exec-title{margin:0;font-size:inherit;line-height:1}.exec-reason{margin:0;font-size:14px;color:var(--dark-text);flex:1 1 320px}+.kpis{display:grid;grid-template-columns:repeat(auto-fit,minmax(170px,1fr));gap:12px;margin:18px 0 8px}.kpi{border:1px solid var(--border-color);border-radius:8px;padding:12px 14px;background:var(--bg-secondary);min-width:0}.kpi dt{font-size:12px;font-weight:600;color:var(--text-secondary)}.kpi dd{margin:4px 0 0;font-size:13px;color:var(--dark-text)}.kpi .kpi-value strong{font-size:24px;font-weight:650;letter-spacing:-.5px;font-variant-numeric:tabular-nums}.kpi .kpi-sub{font-size:12px;color:var(--text-secondary);margin-top:6px}+.sev-item{display:inline-block;white-space:nowrap;margin-right:6px}.sev-dot{display:inline-block;width:8px;height:8px;border-radius:50%;margin-right:4px}.sev-dot.critical{background:var(--critical-color)}.sev-dot.high{background:var(--high-color)}.sev-dot.medium{background:var(--medium-color)}.sev-dot.low{background:var(--low-color)}+.kpi-legend{font-size:12px;color:var(--text-secondary);margin:4px 0 0}+.fix-first{margin-top:16px;border-top:1px solid var(--border-color);padding-top:14px}.fix-first h3{font-size:14px;margin:0 0 8px}.fix-first ol{margin:0;padding-left:22px}.fix-first li{padding:6px 0;font-size:13px;line-height:1.5}.fix-first .severity-tag{margin-right:6px;font-size:11px}+.fix-link{font-weight:600;color:var(--dark-text);text-decoration:underline;text-decoration-color:var(--text-secondary);text-underline-offset:3px;overflow-wrap:anywhere}.fix-link:hover{text-decoration-color:currentColor}.fix-meta{display:block;font-size:12px;color:var(--text-secondary)}.fix-meta code{font-size:12px}.fix-more{font-size:12px;color:var(--text-secondary);margin:8px 0 0}+.severity-incomplete,[data-theme="dark"] .severity-incomplete{background:var(--badge-warn-bg);color:var(--badge-warn-tx)}+.check-id{font-size:11px;color:var(--text-secondary);background:var(--bg-tertiary);padding:1px 5px;border-radius:4px;font-weight:500}.validator-link{color:inherit;font-weight:600}+.check-finding:target,.check-result:target,.finding-flash{outline:3px solid var(--focus-ring);outline-offset:2px}/* Land below the sticky filter bar. */.check-finding,.check-result{scroll-margin-top:14rem}+.quality-pill{min-width:24px;min-height:24px;display:inline-flex;align-items:center;justify-content:center;padding:0 6px}+.raw-output{margin:0 0 10px;font-size:12px}.raw-output>summary{cursor:pointer;color:var(--text-secondary);font-weight:600;padding:4px 0}.raw-output .scan-diagnostic{margin-top:6px}+@media(max-width:1150px){.report-sidebar .sidebar-brand{gap:8px}}+@media(max-width:760px){.report-sidebar .sidebar-brand{display:none}.header-brand .se-logo,.header-brand .header-title{display:block}.sidebar-index-link{order:-1}.header-content{flex-direction:row;flex-wrap:wrap;align-items:center;justify-content:space-between}.header-meta{display:contents}.header-status-badge{margin-left:auto}.header-controls{width:100%;justify-content:flex-end}.agent-hero-top{display:block}.hero-side{align-items:flex-start;text-align:left;max-width:none;margin-top:16px}.hero-side .agent-hero-score,.hero-side .agent-hero-detail{text-align:left}.publication-line{justify-content:flex-start}.kpis{grid-template-columns:1fr 1fr}.exec-summary{padding:16px}.top-issues-table{min-width:0}}+@media(max-width:420px){.export-btn span:first-child{display:none}.theme-toggle #theme-icon{display:none}.kpis{grid-template-columns:1fr}}+@media print{.main-header{display:block!important;position:static;border-bottom:1px solid #ccc;padding:6px 0;background:#fff}.header-content{padding:0;flex-direction:row!important;justify-content:space-between!important;align-items:center!important;flex-wrap:wrap;text-align:left!important}.header-brand .se-logo,.header-brand .header-title,.header-brand .header-subtitle{display:block}.header-status-badge{display:inline-flex!important}.header-meta{justify-content:flex-start}.kpi,.fix-first li{break-inside:avoid}.kpi{background:#fff}.fix-link{color:#111}.tier-card{color:inherit}+/* One results view prints. The check table stays for its files scanned and checks performed counts. */+#results-by-validator>.filter-group,#results-by-validator>.section-header,#results-by-validator>.validator-card,#results-by-validator .toggle-icon{display:none!important}.raw-output:not([open]) .scan-diagnostic{display:none}}++/* Tier 3 pills: the type floor and AA text in both themes. The Tier 3 tab has its own later style block, hence the .tab-content prefix. */+.tab-content .t3-insight-source,.tab-content .t3-sug-cat{font-size:11px}.tab-content .insight-source-llm{color:var(--llm-tx)}diff --git a/src/skillevaluator/reporting/templates/report_ui.js.j2 b/src/skillevaluator/reporting/templates/report_ui.js.j2index e574e747..f030851d 100644--- a/src/skillevaluator/reporting/templates/report_ui.js.j2+++ b/src/skillevaluator/reporting/templates/report_ui.js.j2@@ -94,12 +94,36 @@ filterToAffectedSkills = function(row) {applyReportSearch();};// Printing includes supporting evidence while preserving the reader's disclosure state.+// Raw scanner output stays collapsed: it only repeats finding cards, which print.let reportPrintClosedDetails = [];window.addEventListener('beforeprint', function() {- reportPrintClosedDetails = Array.from(document.querySelectorAll('details:not([open])'));+ reportPrintClosedDetails = Array.from(document.querySelectorAll('details:not([open]):not(.raw-output)'));reportPrintClosedDetails.forEach(detail => { detail.open = true; });});window.addEventListener('afterprint', function() {reportPrintClosedDetails.forEach(detail => { detail.open = false; });reportPrintClosedDetails = [];});+// "Fix first" links open their check or finding: skill view, card expanded,+// filters cleared. The id goes in the address so the link can be shared.+function openFinding(id) {+ const target = document.getElementById(id);+ if (!target) return true;+ const card = target.closest('.skill-card');+ if (card) _openSkillCard(card.dataset.skillName);+ for (let node = target; node; node = node.parentElement) {+ if (node.tagName === 'DETAILS') node.open = true;+ }+ try { history.replaceState(null, '', '#' + id); } catch (err) { /* A file:// page may refuse; the jump still works. */ }+ const reduceMotion = window.matchMedia && window.matchMedia('(prefers-reduced-motion: reduce)').matches;+ target.scrollIntoView({ behavior: reduceMotion ? 'auto' : 'smooth', block: 'center' });+ target.setAttribute('tabindex', '-1');+ target.focus({ preventScroll: true });+ target.classList.add('finding-flash');+ setTimeout(function() { target.classList.remove('finding-flash'); }, 2000);+ return false;+}+// Run after load so the browser's own jump to the fragment does not override it.+window.addEventListener('load', function() {+ if (/^#[fc]-/.test(location.hash)) setTimeout(function() { openFinding(decodeURIComponent(location.hash.slice(1))); }, 0);+});diff --git a/tests/ci/test_evaluate_workflow.py b/tests/ci/test_evaluate_workflow.pyindex 3fd531d6..f8af3c55 100644--- a/tests/ci/test_evaluate_workflow.py+++ b/tests/ci/test_evaluate_workflow.py@@ -13,10 +13,12 @@ import yamlROOT = Path(__file__).parents[2]+def _workflow():+ return yaml.load((ROOT / ".github/workflows/evaluate-skills.yml").read_text(), Loader=yaml.BaseLoader)++def _job():- return yaml.load((ROOT / ".github/workflows/evaluate-skills.yml").read_text(), Loader=yaml.BaseLoader)["jobs"][- "tier1"- ]+ return _workflow()["jobs"]["tier1"]def test_pr_history_mode_and_attempt_artifact_wiring():@@ -24,15 +26,21 @@ def test_pr_history_mode_and_attempt_artifact_wiring():checkout = next(step for step in job["steps"] if step.get("uses", "").startswith("actions/checkout@"))scan = next(step for step in job["steps"] if step.get("name") == "Evaluate selected skills")env = scan["env"]- assert checkout["with"]["fetch-depth"] == "0"- assert checkout["with"]["ref"] == env["DIFF_HEAD_SHA"] == "${{ github.event.pull_request.head.sha || github.sha }}"- assert env["DIFF_BASE_SHA"] == "${{ github.event.pull_request.base.sha }}"+ # Steps inherit the job's scan mode, which also sizes the checkout.+ assert "SCAN_MODE" not in envassert (- env["SCAN_MODE"]+ job["env"]["SCAN_MODE"]== "${{ inputs.scan-mode == 'auto' && (github.event_name == 'pull_request' && 'changed' || 'all') || inputs.scan-mode }}")+ # Only changed mode fetches history, and it skips old file contents.+ # The numbers are ordered so that 0, which is falsy, is never the && operand.+ assert checkout["with"]["fetch-depth"] == "${{ env.SCAN_MODE != 'changed' && 1 || 0 }}"+ assert checkout["with"]["filter"] == "${{ env.SCAN_MODE == 'changed' && 'blob:none' || '' }}"+ assert checkout["with"]["ref"] == env["DIFF_HEAD_SHA"] == "${{ github.event.pull_request.head.sha || github.sha }}"+ assert env["DIFF_BASE_SHA"] == "${{ github.event.pull_request.base.sha }}"assert env["LLM_API_KEY"] == "${{ inputs.llm && secrets.LLM_API_KEY || '' }}"assert env["CALLER_WORKFLOW_SHA"] == "${{ github.workflow_sha }}"+ assert env["SCAN_WORKERS"] == "${{ inputs.scan-workers }}"upload = next(step for step in job["steps"] if step.get("uses", "").startswith("actions/upload-artifact@"))assert upload["with"]["name"] == "${{ inputs.artifact-name }}-attempt-${{ github.run_attempt }}"assert int(job["timeout-minutes"]) > 50@@ -69,6 +77,7 @@ def test_real_workflow_shell_preserves_container_boundary_and_secret_selection(tassert args[args.index("--cap-drop") + 1] == "ALL"assert args[args.index("--security-opt") + 1] == "no-new-privileges"assert any("dst=/workspace,readonly" in arg for arg in args)+ assert args[args.index("SCAN_WORKERS") - 1] == "--env"mounts = [args[index + 1] for index, arg in enumerate(args) if arg == "--tmpfs"]assert len(mounts) == 2 and all("noexec" in mount for mount in mounts)harness = (ROOT / "ci/smoke_image.py").read_text()@@ -96,3 +105,8 @@ def test_smoke_container_executes_with_the_same_hardening(tmp_path, monkeypatch)assert args[args.index("--security-opt") + 1] == "no-new-privileges"environment = [args[index + 1] for index, arg in enumerate(args) if arg == "--env"]assert {entry.split("=", 1)[0] for entry in environment}.issuperset({"SCAN_MODE", "DIFF_BASE_SHA", "DIFF_HEAD_SHA"})+++def test_scan_worker_input_defaults_to_four_parallel_scans():+ workers = _workflow()["on"]["workflow_call"]["inputs"]["scan-workers"]+ assert (workers["type"], workers["default"]) == ("number", "4")diff --git a/tests/ci/test_github_api.py b/tests/ci/test_github_api.pynew file mode 100644index 00000000..fd5f9398--- /dev/null+++ b/tests/ci/test_github_api.py@@ -0,0 +1,190 @@+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+# SPDX-License-Identifier: Apache-2.0++"""One GitHub client and one JSON policy for the trusted publisher and PR feedback."""++import importlib.util+import io+import json+from pathlib import Path++import pytest++ROOT = Path(__file__).parents[2]+++def load(name):+ spec = importlib.util.spec_from_file_location(name, ROOT / f"ci/{name}.py")+ module = importlib.util.module_from_spec(spec)+ spec.loader.exec_module(module)+ return module+++publisher = load("publish_reports")+feedback = load("comment_report")+render = load("render_templates")+TOOLS = [(publisher.GitHubAPI, publisher.PublicationError), (feedback.GitHubAPI, feedback.FeedbackError)]+++class Response(io.BytesIO):+ def __init__(self, body: bytes, headers=None):+ super().__init__(body)+ self.headers = headers or {}+++class Opener:+ def __init__(self, body=b"{}", headers=None):+ self.body, self.response_headers = body, headers+ self.requests = []++ def open(self, request, timeout):+ assert timeout == 60+ self.requests.append(request)+ return Response(self.body, self.response_headers)+++def client(cls, **opener):+ api = cls("owner/consumer", "private-token")+ api.opener = Opener(**opener)+ return api+++def test_both_tools_share_one_client_and_keep_their_own_diagnostics():+ for cls, error in TOOLS:+ assert cls.__mro__[1] is publisher.github_api.GitHubAPI+ assert cls.error is error+ assert publisher.github_api.GitHubAPI is feedback.github_api.GitHubAPI+ for module in (publisher, feedback):+ assert not hasattr(module, "_NoRedirect")++[email protected](("cls", "error"), TOOLS)+ "path",+ [+ "/repos/owner/consumer",+ "/repos/owner/consumer/pulls/1",+ f"/repos/owner/consumer/compare/{'a' * 40}...{'b' * 40}",+ ],+)+def test_this_repository_endpoints_are_allowed(cls, error, path):+ api = client(cls)+ assert api.get(path) == {}+ assert api.opener.requests[0].full_url == "https://api.github.com" + path++[email protected](("cls", "error"), TOOLS)+ "path",+ [+ "/repos/owner/consumer-fork/pulls/1",+ "/repos/owner/consumerx",+ "/repos/other/consumer/pulls/1",+ "/user",+ "/repos/owner/consumer/../other/pulls/1",+ "/repos/owner/consumer/pulls\\1",+ f"/repos/owner/consumer/compare/{'a' * 40}...{'b' * 40}/../other",+ f"/repos/owner/consumer/compare/{'a' * 40}..{'b' * 40}",+ f"/repos/owner/consumer-fork/compare/{'a' * 40}...{'b' * 40}",+ ],+)+def test_other_repositories_and_traversal_are_rejected_before_any_request(cls, error, path):+ api = client(cls)+ with pytest.raises(error, match="this repository"):+ api.get(path)+ assert api.opener.requests == []++[email protected](("cls", "error"), TOOLS)[email protected]("body", [b'{"id": 1, "id": 2}', b'{"value": NaN}', b"[Infinity]", b"\xff", b"{"])+def test_api_json_rejects_duplicates_non_finite_numbers_and_invalid_documents(cls, error, body):+ with pytest.raises(error, match="invalid"):+ client(cls, body=body).get("/repos/owner/consumer/pulls/1")++[email protected](("cls", "error"), TOOLS)+def test_declared_oversized_api_response_is_rejected_before_reading(cls, error, monkeypatch):+ monkeypatch.setattr(publisher.github_api, "MAX_API_BYTES", 8)+ with pytest.raises(error, match="size limit"):+ client(cls, body=b"{}", headers={"Content-Length": "9"}).get("/repos/owner/consumer")+ with pytest.raises(error, match="size limit"):+ client(cls, body=b'{"a": 1234}').get("/repos/owner/consumer")+ assert client(cls, body=b'{"a": 1}').get("/repos/owner/consumer") == {"a": 1}++[email protected](("cls", "error"), TOOLS)+def test_oversized_api_response_names_only_the_api_response(cls, error):+ with pytest.raises(error) as raised:+ client(cls, body=b"{}", headers={"Content-Length": str(16 * 1024 * 1024 + 1)}).get("/repos/owner/consumer")+ # The comment helper never downloads artifacts, so its diagnostic must not mention one.+ assert str(raised.value) == "The GitHub API response exceeds its size limit."++[email protected]("headers", [{"Content-Length": "9"}, {}])+def test_oversized_artifact_download_names_the_artifact(headers, monkeypatch):+ monkeypatch.setattr(publisher, "MAX_TOTAL_BYTES", 8)+ with pytest.raises(publisher.PublicationError) as raised:+ client(publisher.GitHubAPI, body=b"123456789", headers=headers).download_artifact(456)+ assert str(raised.value) == "The downloaded artifact exceeds its size limit."+++def test_publisher_api_responses_use_the_smaller_shared_cap():+ assert publisher.github_api.MAX_API_BYTES == 16 * 1024 * 1024 < publisher.MAX_FILE_BYTES+++def test_write_requests_send_strict_json_with_a_tool_user_agent():+ api = client(feedback.GitHubAPI, body=b'{"id": 7}')+ assert api.post("/repos/owner/consumer/issues/1/comments", {"body": "text"}) == {"id": 7}+ assert api.patch("/repos/owner/consumer/issues/comments/7", {"body": "new"}) == {"id": 7}+ post, patch = api.opener.requests+ assert (post.get_method(), patch.get_method()) == ("POST", "PATCH")+ assert json.loads(post.data) == {"body": "text"}+ assert post.get_header("Content-type") == "application/json"+ assert post.get_header("Authorization") == "Bearer private-token"+ assert post.get_header("User-agent") == "SkillEvaluator-PR-feedback"+ with pytest.raises(ValueError):+ api.post("/repos/owner/consumer/issues/1/comments", {"value": float("nan")})+ get = client(publisher.GitHubAPI)+ get.get("/repos/owner/consumer")+ assert get.opener.requests[0].get_method() == "GET"+ assert get.opener.requests[0].data is None+ assert get.opener.requests[0].get_header("User-agent") == "SkillEvaluator-report-publisher"+++def test_publication_metadata_uses_the_same_strict_json_policy(tmp_path):+ (tmp_path / "publication.json").write_text('{"pr_number": "1", "pr_number": "2"}', encoding="utf-8")+ (tmp_path / "pr-comment.md").write_text("ok", encoding="utf-8")+ with pytest.raises(feedback.FeedbackError, match="invalid JSON"):+ feedback.comment_report(object(), "owner/consumer", 123, tmp_path, "github-actions[bot]")+ (tmp_path / "publication.json").write_text('{"run_url": NaN}', encoding="utf-8")+ with pytest.raises(feedback.FeedbackError, match="invalid JSON"):+ feedback.comment_report(object(), "owner/consumer", 123, tmp_path, "github-actions[bot]")+++def test_release_rendering_uses_the_shared_identity_patterns(tmp_path):+ shared = publisher.github_api+ assert publisher.SHA_PATTERN is feedback.SHA_PATTERN is shared.COMMIT_SHA+ assert publisher.REPO_PATTERN is feedback.REPOSITORY_PATTERN is shared.REPOSITORY+ assert render.github_api.GHCR_DIGEST is shared.GHCR_DIGEST+ image = "ghcr.io/nvidia/scan@sha256:" + "a" * 64+ with pytest.raises(ValueError, match="commit SHA"):+ render.render_templates(tmp_path / "out", "B" * 40, image)+ with pytest.raises(ValueError, match="GHCR"):+ render.render_templates(tmp_path / "out", "b" * 40, image.upper())+++ ("workflow", "step", "patterns"),+ [+ (".github/workflows/publish-skill-reports.yml", "Validate trusted tool identity", ("COMMIT_SHA", "REPOSITORY")),+ (".github/workflows/evaluate-skills.yml", "Validate the immutable image reference", ("GHCR_DIGEST",)),+ ],+)+def test_workflow_copies_of_identity_patterns_match_the_shared_module(workflow, step, patterns):+ # These checks run before the tools exist on the runner, so they keep inline copies.+ import yaml++ jobs = yaml.load((ROOT / workflow).read_text(), Loader=yaml.BaseLoader)["jobs"]+ script = next(item["run"] for job in jobs.values() for item in job["steps"] if item.get("name") == step)+ for name in patterns:+ assert f'r"{getattr(publisher.github_api, name).pattern}"' in scriptdiff --git a/tests/ci/test_image_build.py b/tests/ci/test_image_build.pynew file mode 100644index 00000000..479017b2--- /dev/null+++ b/tests/ci/test_image_build.py@@ -0,0 +1,188 @@+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+# SPDX-License-Identifier: Apache-2.0+"""Guard the Tier 1 image layout and the workflow that builds it. Building the image is left to CI."""++import fnmatch+import re+import shlex+from pathlib import Path++import yaml++ROOT = Path(__file__).parents[2]+DOCKERFILE = ROOT / "ci/Dockerfile.tier1"+++def _instructions() -> list[tuple[str, str]]:+ """Dockerfile instructions as (keyword, arguments), with continuations joined and comments dropped."""+ text = re.sub(r"\\\n", " ", DOCKERFILE.read_text(encoding="utf-8"))+ lines = [line.strip() for line in text.splitlines()]+ return [tuple(line.split(None, 1)) for line in lines if line and not line.startswith("#")]+++def _stages() -> dict[str, list[tuple[str, str]]]:+ """Instructions per stage. The unnamed final stage is the shipped runtime."""+ stages: dict[str, list[tuple[str, str]]] = {}+ for keyword, arguments in _instructions():+ if keyword == "FROM":+ words = arguments.split()+ name = words[2] if len(words) == 3 and words[1].upper() == "AS" else "runtime"+ stages[name] = []+ stages[name].append((keyword, arguments))+ assert list(stages) == ["build", "project", "runtime"]+ return stages+++def _arguments(stage: list[tuple[str, str]], keyword: str) -> list[str]:+ return [arguments for name, arguments in stage if name == keyword]+++def _workflow() -> dict:+ path = ROOT / ".github/workflows/publish-tier1-image.yml"+ return yaml.load(path.read_text(encoding="utf-8"), Loader=yaml.BaseLoader)+++def _build_steps(job: dict) -> list[dict]:+ return [step for step in job["steps"] if step.get("uses", "").startswith("docker/build-push-action@")]+++def test_runtime_stage_ships_only_what_scans_run():+ runtime = _stages()["runtime"]+ assert _arguments(runtime, "COPY") == [+ "--from=build /opt/semgrep /opt/semgrep",+ "--from=build /opt/skillspector /opt/skillspector",+ "--from=build /usr/local/bin/gitleaks /usr/local/bin/gitleaks",+ "--from=build /usr/local/share/licenses/gitleaks /usr/local/share/licenses/gitleaks",+ "--from=build /opt/skillevaluator/.venv /opt/skillevaluator/.venv",+ "--from=project /overlay/ /",+ "LICENSE NOTICE THIRD_PARTY_NOTICES.md /opt/skillevaluator/",+ "ci /opt/skillevaluator/ci",+ ]+ commands = {word for command in _arguments(runtime, "RUN") for word in shlex.split(command)}+ assert {"uv", "curl", "pip"}.isdisjoint(commands)+ assert {"git", "ca-certificates"} <= commands+ assert _arguments(runtime, "USER") == ["evaluator"]+ assert _arguments(runtime, "ENTRYPOINT") == ['["python", "/opt/skillevaluator/ci/run_tier1.py"]']+++def test_build_and_runtime_stages_share_one_pinned_base():+ bases = {arguments.split()[0] for keyword, arguments in _instructions() if keyword == "FROM"}+ assert bases - {"build"} == {+ "python:3.12-slim-bookworm@sha256:62ed8e07e544e8555c5d93748d5275015136b7f5c52b5e8679b4de339c3779d3"+ }+++def test_bytecode_is_compiled_at_build_time_and_never_written_at_runtime():+ stages = _stages()+ build_environment = " ".join(_arguments(stages["build"], "ENV")).split()+ assert {"UV_COMPILE_BYTECODE=1", "PYC_INVALIDATION_MODE=UNCHECKED_HASH"} <= set(build_environment)+ runtime = stages["runtime"]+ assert "PYTHONDONTWRITEBYTECODE=1" in " ".join(_arguments(runtime, "ENV")).split()+ stdlib = [command for command in _arguments(runtime, "RUN") if command.endswith("/usr/local/lib/python3.12")]+ assert len(stdlib) == 1+ assert "-m compileall" in stdlib[0] and "--invalidation-mode unchecked-hash" in stdlib[0]+++def test_source_edits_rebuild_only_the_last_layers():+ stages = _stages()+ build = stages["build"]+ first_source = next(+ index+ for index, (keyword, arguments) in enumerate(build)+ if keyword == "COPY" and not arguments.startswith("ci/requirements/")+ )+ scanners = [+ index+ for index, (keyword, arguments) in enumerate(build)+ if keyword == "RUN" and re.search(r"/opt/semgrep|/opt/skillspector|gitleaks", arguments)+ ]+ assert len(scanners) == 3 and max(scanners) < first_source+ assert build[first_source] == ("COPY", "pyproject.toml uv.lock ./")+ assert _arguments(stages["project"], "COPY") == ["README.md LICENSE NOTICE THIRD_PARTY_NOTICES.md ./", "src ./src"]+ runtime = stages["runtime"]+ overlay = runtime.index(("COPY", "--from=project /overlay/ /"))+ checks = next(index for index, (_, arguments) in enumerate(runtime) if "semgrep --version" in arguments)+ shared = [index for index, (_, arguments) in enumerate(runtime) if arguments.startswith("--from=build ")]+ assert max([*shared, checks]) < overlay+++def _dockerignored(path: str) -> bool:+ lines = (ROOT / ".dockerignore").read_text(encoding="utf-8").splitlines()+ patterns = [line.strip().rstrip("/") for line in lines if line.strip() and not line.startswith("#")]+ parts = Path(path).parts+ prefixes = ["/".join(parts[:count]) for count in range(1, len(parts) + 1)]+ return any(fnmatch.fnmatchcase(prefix, pattern) for pattern in patterns for prefix in prefixes)+++def test_build_context_drops_unused_trees_but_keeps_image_inputs():+ copied = [+ source+ for keyword, arguments in _instructions()+ if keyword == "COPY" and not arguments.startswith("--from=")+ for source in arguments.split()[:-1]+ ]+ # The repository's root Dockerfile also builds the wheel from this context.+ package = ["pyproject.toml", "README.md", "src/skillevaluator/reporting/templates/report.html.j2"]+ for path in copied + package:+ assert (ROOT / path).exists(), path+ assert not _dockerignored(path), path+ unused = ["tests/ci/test_image_build.py", "docs/github-actions.mdx", "scripts", "fern", ".github", "CHANGELOG.md"]+ for path in unused:+ assert (ROOT / path).exists(), path+ assert _dockerignored(path), path+++def test_pull_requests_read_but_never_write_the_layer_cache():+ workflow = _workflow()+ assert workflow["on"]["push"]["branches"] == ["main"]+ assert workflow["on"]["push"]["paths"] == workflow["on"]["pull_request"]["paths"]+ (verify,) = _build_steps(workflow["jobs"]["verify"])+ assert verify["with"]["cache-from"] == "type=gha,scope=tier1-image"+ cache_to = verify["with"]["cache-to"]+ assert "github.event_name != 'pull_request'" in cache_to and "github.ref == 'refs/heads/main'" in cache_to+ assert "'type=gha,scope=tier1-image,mode=max,ignore-error=true' || ''" in cache_to+ (release,) = _build_steps(workflow["jobs"]["publish"])+ for step in (verify, release):+ assert step["with"]["context"] == "."+ assert step["with"]["file"] == "ci/Dockerfile.tier1"+ assert step["with"]["load"] == "true" and "push" not in step["with"]+ # These events run in main's cache scope on input that a pull request controls.+ for path in (ROOT / ".github/workflows").glob("*.yml"):+ triggers = yaml.load(path.read_text(encoding="utf-8"), Loader=yaml.BaseLoader)["on"]+ triggers = [triggers] if isinstance(triggers, str) else triggers+ assert {"pull_request_target", "workflow_run", "issue_comment"}.isdisjoint(triggers), path.name+++def test_release_builds_reuse_no_cached_layers():+ # A cached apt layer would ship the git and ca-certificates of whichever main build first wrote it.+ workflow = _workflow()+ (verify,) = _build_steps(workflow["jobs"]["verify"])+ (release,) = _build_steps(workflow["jobs"]["publish"])+ assert "cache-from" not in release["with"]+ assert release["with"]["no-cache"] == "true"+ # The release still refreshes the cache that later verify builds read.+ assert release["with"]["cache-to"] == "type=gha,scope=tier1-image,mode=max,ignore-error=true"+ assert verify["with"]["cache-from"] == "type=gha,scope=tier1-image"+++def test_release_builds_once_and_pushes_the_image_it_verified():+ jobs = _workflow()["jobs"]+ publish = jobs["publish"]+ # Exactly one of the two building jobs runs for any event.+ assert jobs["verify"]["if"] == "${{ !(" + publish["if"] + ") }}"+ assert "needs" not in publish+ (build,) = _build_steps(publish)+ assert build["with"]["tags"] == "${{ env.IMAGE }}:sha-${{ github.sha }}"+ assert build["with"]["labels"].splitlines() == [+ "org.opencontainers.image.source=https://github.com/NVIDIA/SkillEvaluator",+ "org.opencontainers.image.revision=${{ github.sha }}",+ "org.opencontainers.image.description=SkillEvaluator complete Tier 1 CI runtime",+ ]+ steps = publish["steps"]+ smoke = next(index for index, step in enumerate(steps) if "ci/smoke_image.py" in step.get("run", ""))+ release = next(index for index, step in enumerate(steps) if step.get("id") == "release")+ assert steps.index(build) < smoke < release+ assert '--image "$IMAGE:sha-$GITHUB_SHA"' in steps[smoke]["run"]+ assert 'docker push "$IMAGE:sha-$GITHUB_SHA"' in steps[release]["run"]+ runs = [step.get("run", "") for job in jobs.values() for step in job["steps"]]+ assert not any("docker build" in run for run in runs)diff --git a/tests/ci/test_native_reports.py b/tests/ci/test_native_reports.pyindex 86e02440..aa3ff7b9 100644--- a/tests/ci/test_native_reports.py+++ b/tests/ci/test_native_reports.py@@ -1,12 +1,16 @@# SPDX-License-Identifier: Apache-2.0"""Native CI rendering preserves retained evidence and fails closed."""+import base64+import hashlibimport json+import reimport shutil+from html import unescapefrom pathlib import Pathimport pytest-from ci.native_reports import render_native_reports+from ci.native_reports import _trusted_inline_code, constrain_native_html, render_native_reportsfrom ci.run_tier1 import _summaryfrom skillevaluator.models import Finding, ValidationResult@@ -52,6 +56,15 @@ def _write_fixture(tmp_path: Path, failed=False):return _summary([row]), raw+def _csp_hashes(document: str) -> set[str]:+ policy = re.search(r'http-equiv="Content-Security-Policy" content="([^"]+)"', document)[1]+ return set(re.findall(r"'(sha256-[^']+)'", unescape(policy)))+++def _sha256(code: str) -> str:+ return "sha256-" + base64.b64encode(hashlib.sha256(code.encode()).digest()).decode()++@pytest.mark.parametrize("failed", [False, True])def test_native_reports_overwrite_artifact_html_keep_json_and_link_details(tmp_path, failed):summary, raw = _write_fixture(tmp_path, failed)@@ -64,10 +77,27 @@ def test_native_reports_overwrite_artifact_html_keep_json_and_link_details(tmp_passert 'href="skill-001/skillevaluator-tier1.html"' in indexassert (tmp_path / "skill-001/skillevaluator-tier1.json").read_text() == rawassert "Content-Security-Policy" in native and "sha256-" in native and "unsafe-eval" not in native+ # The page links the canonical JSON instead of embedding an unused copy.+ assert 'id="report-data"' not in nativeassert "<img src=x onerror=alert(1)>" not in nativeassert "https://example.com/run/skill-001/skillevaluator-tier1.html" in (tmp_path / "details.md").read_text()+def test_omitting_embedded_report_json_keeps_the_csp_script_hashes(tmp_path):+ summary, raw = _write_fixture(tmp_path, failed=True)+ render_native_reports(tmp_path, summary, {})+ native = (tmp_path / "skill-001/skillevaluator-tier1.html").read_text(encoding="utf-8")+ page = re.sub(r'<meta http-equiv="Content-Security-Policy" content="[^"]+">', "", native, count=1)+ # Put back the block that standalone reports embed, escaped the way HTMLReporter escapes it.+ embedded = json.dumps(json.loads(raw)).replace("<", "\\u003c").replace(">", "\\u003e").replace("&", "\\u0026")+ block = f'<script id="report-data" type="application/json">{embedded}</script>'+ assert page.count("</body>") == 1+ standalone = page.replace("</body>", block + "</body>")+ hashes = _csp_hashes(native)+ assert _csp_hashes(constrain_native_html(standalone)) == _csp_hashes(constrain_native_html(page)) == hashes+ assert hashes and hashes <= {_sha256(code) for code in _trusted_inline_code()}++def test_malformed_retained_evidence_is_not_silently_published(tmp_path):summary, _ = _write_fixture(tmp_path)path = tmp_path / "skill-001/skillevaluator-tier1.json"@@ -128,8 +158,10 @@ def test_changed_selection_without_work_is_neutral_and_preserves_deleted_directo(tmp_path / "summary.json").write_text(raw)render_native_reports(tmp_path, summary, {"llm_enabled": True})page = (tmp_path / "index.html").read_text()- assert 'class="status ">No skills changed</span>' in page- assert "Validation did not run." in page+ # A run with nothing to scan uses the report vocabulary and explains itself once.+ assert '<span class="status neutral"><span class="sr-only">Verdict: </span>Skipped</span>' in page+ assert page.count("Validation did not run.") == 1+ assert "Quality is a content score" not in pageassert "Tier 1: Not run (no skills selected)" in pageassert "LLM: Not run" in pageassert "Deleted skill directories (not scanned)" in page@@ -152,7 +184,7 @@ def test_skipped_marker_with_inconsistent_evidence_cannot_claim_no_changes(tmp_pmutate(summary)render_native_reports(tmp_path, summary, {})page = (tmp_path / "index.html").read_text()- assert 'class="status ">No skills changed</span>' not in page+ assert "</span>Skipped</span>" not in pageassert "Validation did not run." not in page@@ -201,6 +233,23 @@ def test_csp_does_not_trust_code_found_in_final_render(tmp_path, injection):constrain_native_html(document.replace("</body>", injection + "</body>"))+def test_each_skill_report_is_read_once_for_all_views(tmp_path, monkeypatch):+ from ci import native_reports++ summary, _ = _write_fixture(tmp_path, failed=True)+ shutil.copytree(tmp_path / "skill-001", tmp_path / "skill-002")+ summary = _summary([summary["skills"][0], {**summary["skills"][0], "report_dir": "skill-002", "path": "skills/b"}])+ reads = []+ original = native_reports.views._read+ monkeypatch.setattr(+ native_reports.views, "_read", lambda directory: reads.append(directory.name) or original(directory)+ )+ render_native_reports(tmp_path, summary, {})+ assert sorted(reads) == ["skill-001", "skill-002"]+ assert "Bad name" in (tmp_path / "skill-002/skillevaluator-tier1.html").read_text()+ assert "2 skills · 0 passed · 2 failed · 0 incomplete" in (tmp_path / "details.md").read_text()++def test_failed_and_incomplete_counts_agree_across_all_surfaces(tmp_path):summary, _ = _write_fixture(tmp_path, failed=True)shutil.copytree(tmp_path / "skill-001", tmp_path / "skill-002")@@ -222,5 +271,67 @@ def test_failed_and_incomplete_counts_agree_across_all_surfaces(tmp_path):for filename in ("summary.md", "details.md", "pr-comment.md"):assert "2 skills · 0 passed · 1 failed · 1 incomplete" in (tmp_path / filename).read_text()page = (tmp_path / "index.html").read_text()- assert "<strong>1</strong><span>Failed</span>" in page+ assert "<strong>1</strong><span>Changes required</span>" in pageassert "<strong>1</strong><span>Incomplete</span>" in page+++def test_index_uses_the_report_vocabulary_logo_and_first_fix(tmp_path):+ summary, _ = _write_fixture(tmp_path, failed=True)+ render_native_reports(tmp_path, summary, {})+ page = (tmp_path / "index.html").read_text()+ report = (tmp_path / "skill-001/skillevaluator-tier1.html").read_text()++ assert '<span class="status fail"><span class="sr-only">Verdict: </span>Changes required</span>' in page+ assert '<span class="status fail">Changes required</span>' in page+ assert 'class="mark"' not in page+ assert re.search(r'<aside class="rail"><div class="brand"><svg class="se-logo"[^>]*aria-hidden="true"', page)+ assert "most severe first" not in page+ # "Needs attention" names the skill report's first fix and links to its card.+ anchor = re.search(r'href="#(f-[^"]+)" class="fix-link"', report).group(1)+ assert f'href="skill-001/skillevaluator-tier1.html#{anchor}"' in page+ assert f'id="{anchor}"' in report+++def test_repository_root_skill_lists_each_fix_first_location_once(tmp_path):+ """Code Risk Analysis records absolute scan paths; other scanners record skill-relative ones."""+ risk = ValidationResult(validator_name="Code Risk Analysis", validator_description="Static analysis")+ risk.add_finding(+ Finding(+ category="BANDIT",+ severity="high",+ check_name="B602",+ message="subprocess call with shell=True",+ file_path="/workspace/scripts/fetch.py",+ line_number=21,+ )+ )+ scan = ValidationResult(validator_name="Security Scan", validator_description="Detect vulnerabilities")+ scan.add_finding(+ Finding(+ category="SECURITY",+ severity="high",+ check_name="tool_misuse",+ message="Tool Misuse: subprocess.run(command, shell=True",+ file_path="scripts/fetch.py",+ line_number=21,+ )+ )+ summary, _ = _write_fixture(tmp_path)+ payload = json.loads(JSONReporter(include_timestamp=False).render_all([risk, scan]))+ (tmp_path / "skill-001/skillevaluator-tier1.json").write_text(json.dumps(payload))+ summary["skills"][0].update(path=".", passed=False, overall_status="failed")++ render_native_reports(tmp_path, summary, {})+ report = (tmp_path / "skill-001/skillevaluator-tier1.html").read_text()+ fix_first = report.split('class="fix-first"', 1)[1].split("</section>", 1)[0]++ assert re.findall(r"<code>([^<]+)</code>", fix_first) == ["scripts/fetch.py:21"]+ assert "/workspace/" not in fix_first+++def test_index_dark_theme_never_applies_to_print():+ from ci.native_reports import _STYLE++ assert "@media screen and (prefers-color-scheme:dark)" in _STYLE+ assert "@media(prefers-color-scheme:dark)" not in _STYLE+ assert all(int(size) >= 11 for size in re.findall(r"font-size:(\d+)px", _STYLE))diff --git a/tests/ci/test_publish_reports.py b/tests/ci/test_publish_reports.pyindex 2d5963a3..06bc7d5f 100644--- a/tests/ci/test_publish_reports.py+++ b/tests/ci/test_publish_reports.py@@ -6,14 +6,17 @@import importlib.utilimport ioimport json+import osimport statimport urllib.errorimport zipfilefrom pathlib import Pathimport pytest+import yaml-SPEC = importlib.util.spec_from_file_location("publish_reports", Path(__file__).parents[2] / "ci/publish_reports.py")+ROOT = Path(__file__).parents[2]+SPEC = importlib.util.spec_from_file_location("publish_reports", ROOT / "ci/publish_reports.py")publisher = importlib.util.module_from_spec(SPEC)SPEC.loader.exec_module(publisher)@@ -193,6 +196,36 @@ def test_no_target_configuration_error_can_be_published():assert publisher.read_artifact(archive([("summary.json", payload)]))[0] == payload+def test_site_writes_are_atomic_and_leave_no_partial_file(tmp_path):+ target = tmp_path / "site/runs/1/1/summary.json"+ publisher._write(target, "old")+ with pytest.raises(UnicodeEncodeError):+ publisher._write(target, "new \ud800")+ assert target.read_text(encoding="utf-8") == "old"+ assert [path.name for path in target.parent.iterdir()] == ["summary.json"]+ publisher._dump(target, {"new": True})+ assert json.loads(target.read_text(encoding="utf-8")) == {"new": True}+ assert [path.name for path in target.parent.iterdir()] == ["summary.json"]++[email protected]("linked", ["target", "parent"])+def test_site_writes_refuse_linked_targets_and_directories(tmp_path, linked):+ outside = tmp_path / "outside"+ outside.mkdir()+ (outside / "index.html").write_text("outside", encoding="utf-8")+ site = tmp_path / "site"+ site.mkdir()+ if linked == "target":+ (site / "index.html").symlink_to(outside / "index.html")+ target = site / "index.html"+ else:+ (site / "runs").symlink_to(outside, target_is_directory=True)+ target = site / "runs/index.html"+ with pytest.raises(publisher.PublicationError, match="symlink"):+ publisher._write(target, "replaced")+ assert (outside / "index.html").read_text(encoding="utf-8") == "outside"++def test_alias_order_is_run_then_attempt():assert publisher.is_newer({"run_id": 5, "run_attempt": 2}, {"run_id": 5, "run_attempt": 1})assert not publisher.is_newer({"run_id": 4, "run_attempt": 99}, {"run_id": 5, "run_attempt": 1})@@ -275,7 +308,7 @@ def test_pr_provenance_compares_recorded_revisions_through_real_github_client():def test_github_client_still_rejects_untrusted_paths(path):api = publisher.GitHubAPI("owner/consumer", "private-token")with pytest.raises(publisher.PublicationError):- api._request(path)+ api._request("GET", path)def test_github_client_accepts_repository_root_for_default_branch_lookup():@@ -363,7 +396,7 @@ class FakeAPI:"artifacts": [{"id": 456,- "name": "skill-evaluator-tier1",+ "name": "skill-evaluator-tier1-evidence-attempt-1","expired": False,"size_in_bytes": len(valid_archive()),"created_at": "2026-10-21T12:01:00Z",@@ -535,7 +568,7 @@ def test_requested_second_attempt_gets_only_its_own_artifact(tmp_path):{**result["artifacts"][0],"id": 789,- "name": "skill-evaluator-tier1-attempt-2",+ "name": "skill-evaluator-tier1-evidence-attempt-2","created_at": "2026-10-21T13:01:00Z",})@@ -563,13 +596,124 @@ def test_attempt_artifact_name_cannot_claim_another_attempt():class WrongNameAPI(FakeAPI):def get(self, path):result = super().get(path)- result["artifacts"][0]["name"] = "skill-evaluator-tier1-attempt-2"+ result["artifacts"][0]["name"] = "skill-evaluator-tier1-evidence-attempt-2"return result- with pytest.raises(publisher.PublicationError, match="no current report artifact"):+ with pytest.raises(publisher.PublicationError, match="no current report evidence artifact"):publisher._select_artifact(WrongNameAPI(), "owner/consumer", run_record())+def artifact_entry(name, identifier, size=100):+ return {+ "id": identifier,+ "name": name,+ "expired": False,+ "size_in_bytes": size,+ "created_at": "2026-10-21T12:01:00Z",+ "workflow_run": {"id": 123, "head_sha": "a" * 40},+ }+++class ListingAPI(FakeAPI):+ def __init__(self, artifacts):+ super().__init__()+ self.artifacts = artifacts++ def get(self, path):+ if path.endswith("/artifacts?per_page=100"):+ return {"total_count": len(self.artifacts), "artifacts": self.artifacts}+ return super().get(path)+++def test_publisher_selects_the_evidence_artifact_not_the_full_reports():+ artifacts = [+ artifact_entry("skill-evaluator-tier1-attempt-1", 455, size=publisher.MAX_TOTAL_BYTES + 1),+ artifact_entry("skill-evaluator-tier1-evidence-attempt-1", 456),+ ]+ assert publisher._select_artifact(ListingAPI(artifacts), "owner/consumer", run_record())["id"] == 456+++ "names",+ [+ # A failed or cancelled scan can stop before it writes any reports.+ [],+ ["skill-evaluator-tier1-attempt-1"],+ ["skill-evaluator-tier1"],+ ["skill-evaluator-tier1-evidence"],+ ],+)+def test_missing_evidence_artifact_fails_clearly_without_reading_full_reports(names):+ artifacts = [artifact_entry(name, 455) for name in names]+ with pytest.raises(publisher.PublicationError) as raised:+ publisher._select_artifact(ListingAPI(artifacts), "owner/consumer", run_record())+ # Name every common cause; a scan that uploaded nothing is not an evaluator revision problem.+ assert str(raised.value) == (+ "The selected run attempt has no current report evidence artifact named "+ "skill-evaluator-tier1-evidence-attempt-1. The scan may have stopped before writing reports, "+ "the artifact may have expired, or the scan used an evaluator revision that does not upload it."+ )+++ ("change", "message"),+ [+ ({"workflow_run": {"id": 999, "head_sha": "a" * 40}}, "does not match the selected run"),+ ({"workflow_run": {"id": 123, "head_sha": "f" * 40}}, "does not match the selected run"),+ ({"size_in_bytes": 64 * 1024 * 1024 + 1}, "size limit"),+ ({"id": 0}, "size limit"),+ ({"created_at": "2026-10-21T11:59:59Z"}, "no current report evidence"),+ ({"created_at": "2026-10-21T12:02:01Z"}, "no current report evidence"),+ ({"expired": True}, "no current report evidence"),+ ],+)+def test_evidence_artifact_keeps_provenance_attempt_and_size_checks(change, message):+ artifact = {**artifact_entry("skill-evaluator-tier1-evidence-attempt-1", 456), **change}+ with pytest.raises(publisher.PublicationError, match=message):+ publisher._select_artifact(ListingAPI([artifact]), "owner/consumer", run_record())+++def test_scan_evidence_upload_holds_exactly_what_the_publisher_reads(tmp_path):+ steps = yaml.load((ROOT / ".github/workflows/evaluate-skills.yml").read_text(), Loader=yaml.BaseLoader)["jobs"][+ "tier1"+ ]["steps"]+ uploads = [step for step in steps if step.get("uses", "").startswith("actions/upload-artifact@")]+ full, evidence = uploads+ assert evidence["uses"] == full["uses"] and evidence["if"] == full["if"] == "always()"+ assert evidence["with"]["name"] == "${{ inputs.artifact-name }}-evidence-attempt-${{ github.run_attempt }}"+ assert evidence["with"]["retention-days"] == full["with"]["retention-days"]+ assert f"{publisher.EVIDENCE_ARTIFACT}-attempt-1" == "skill-evaluator-tier1-evidence-attempt-1"+ reports = tmp_path / "skill-reports"+ payload = summary()+ for name in ("summary.md", "details.md", "pr-comment.md", "index.html"):+ (reports / name).parent.mkdir(parents=True, exist_ok=True)+ (reports / name).write_text("<p>not evidence</p>", encoding="utf-8")+ (reports / "summary.json").write_text(json.dumps(payload), encoding="utf-8")+ child = reports / "skill-001"+ (child / "logs").mkdir(parents=True)+ (child / "skillevaluator-tier1.json").write_text(json.dumps(report()), encoding="utf-8")+ for name in ("skillevaluator-tier1.html", "skillevaluator-tier1.md", "skillevaluator-tier1.sarif", "logs/a.log"):+ (child / name).write_text("not evidence", encoding="utf-8")+ lines = [line.strip() for line in evidence["with"]["path"].splitlines() if line.strip()]+ assert all(line.startswith("${{ runner.temp }}/") for line in lines)+ patterns = [line.removeprefix("${{ runner.temp }}/") for line in lines]+ # upload-artifact roots several paths at the common ancestor of their non-glob prefixes.+ search = [str(tmp_path / pattern.split("*", 1)[0].rsplit("/", 1)[0]) for pattern in patterns]+ root = Path(os.path.commonpath(search))+ members = sorted(path.relative_to(root).as_posix() for pattern in patterns for path in tmp_path.glob(pattern))+ assert root == reports+ assert members == ["skill-001/skillevaluator-tier1.json", "summary.json"]+ loaded, documents = publisher.read_artifact(archive([(name, (root / name).read_bytes()) for name in members]))+ assert loaded == payload+ assert list(documents) == ["skill-001"]+++def test_duplicate_evidence_artifacts_are_ambiguous():+ artifacts = [artifact_entry("skill-evaluator-tier1-evidence-attempt-1", number) for number in (456, 457)]+ with pytest.raises(publisher.PublicationError, match="ambiguous"):+ publisher._select_artifact(ListingAPI(artifacts), "owner/consumer", run_record())++@pytest.mark.parametrize("repo", [{"full_name": "attacker/fork"}, None])def test_fork_and_deleted_fork_from_commit_fallback_get_no_alias_or_comment(repo):class ForkAPI(FakeAPI):@@ -1158,3 +1302,24 @@ def test_published_report_preserves_recorded_llm_request(tmp_path, enabled):tmp_path / "out",render=capture,)+++def test_history_index_uses_the_report_vocabulary_and_logo(tmp_path):+ def record(run_id, status):+ return {+ "run_id": run_id,+ "run_attempt": 1,+ "head_sha": f"{run_id:040d}",+ "report_url": f"https://example.github.io/skills/runs/{run_id}/1/",+ "overall_status": status,+ }++ history = [record(1, "passed"), record(2, "failed"), record(3, "incomplete"), record(4, "skipped")]+ publisher._history_index(tmp_path, history)+ page = (tmp_path / "index.html").read_text()++ for run_id, word in ((1, "Passed"), (2, "Changes required"), (3, "Incomplete"), (4, "Skipped")):+ assert f"Run {run_id} / attempt 1 · {word} · " in page+ assert "— failed —" not in page+ assert 'class="mark"' not in page+ assert '<aside class="rail"><div class="brand"><svg class="se-logo"' in pagediff --git a/tests/ci/test_publisher_workflow.py b/tests/ci/test_publisher_workflow.pyindex b2214c3b..59cbb208 100644--- a/tests/ci/test_publisher_workflow.py+++ b/tests/ci/test_publisher_workflow.py@@ -3,8 +3,15 @@"""Workflow contracts for attempt-bound publication and queued PR feedback."""+import os+import re+import shutil+import subprocess+import sys+import tomllibfrom pathlib import Path+import pytestimport yamlROOT = Path(__file__).parents[2]@@ -65,5 +72,245 @@ def test_privileged_publisher_installs_the_reviewed_hash_locked_bootstrap():steps = load(".github/workflows/publish-skill-reports.yml")["jobs"]["prepare"]["steps"]install = next(step for step in steps if step.get("name") == "Install the trusted native report renderer")assert install["working-directory"] == "publisher"- assert "python3 -m pip install --require-hashes -r ci/requirements/uv.txt" in install["run"]- assert "uv sync --frozen --no-dev --no-editable" in install["run"]+ lines = [line.strip() for line in install["run"].replace("\\\n", " ").splitlines()]+ # uv comes from its hash-locked requirement, installed into a private environment.+ assert lines[0] == 'python3 -m venv "$RUNNER_TEMP/uv"'+ assert lines[1].startswith('"$RUNNER_TEMP/uv/bin/python" -m pip install ')+ for flag in ("--require-hashes", "--no-deps", "--only-binary :all:", "-r ci/requirements/uv.txt"):+ assert flag in lines[1]+ assert lines[2] == '"$RUNNER_TEMP/uv/bin/uv" sync --frozen --no-dev --no-editable'+ assert len(lines) == 3+ # The runner's system Python and pip are never modified.+ assert "python3 -m pip" not in install["run"]+ # This job can push to gh-pages, so it never restores a dependency cache that+ # other default-branch workflows could have written.+ for step in steps:+ assert "actions/cache" not in step.get("uses", "")+ assert "setup-uv" not in step.get("uses", "")+++def expression(text: str, context: dict) -> bool:+ """Evaluate the small GitHub expression subset these job conditions use."""+ tokens = re.findall(r"'[^']*'|==|!=|&&|\|\||!|\(|\)|always\(\)|[A-Za-z_][\w.-]*", text)+ assert "".join(tokens) == re.sub(r"\s+", "", text)+ translated = {"&&": " and ", "||": " or ", "!": " not ", "always()": "True"}+ python = []+ for token in tokens:+ if token.startswith("'"):+ python.append(repr(token[1:-1]))+ elif token in translated:+ python.append(translated[token])+ elif token in {"==", "!=", "(", ")"}:+ python.append(token)+ else:+ python.append(repr(context[token]))+ # The input is a workflow file from this repository, translated token by token.+ return eval("".join(python), {"__builtins__": {}})+++def job_context(committed, event="workflow_run", attempt="1", deploy="success", skipped=""):+ return {+ "needs.prepare.result": "success",+ "needs.prepare.outputs.skipped": skipped,+ "needs.prepare.outputs.committed": committed,+ "needs.prepare.outputs.pr_number": "1",+ "needs.prepare.outputs.is_current": "true",+ "needs.deploy.result": deploy,+ "github.event_name": event,+ "github.run_attempt": attempt,+ }+++ ("committed", "event", "attempt", "deploys"),+ [+ ("true", "workflow_run", "1", True),+ ("false", "workflow_run", "1", False),+ ("false", "workflow_dispatch", "1", True),+ ("false", "workflow_run", "2", True),+ ("true", "workflow_run", "3", True),+ ],+)+def test_deploy_is_skipped_only_when_an_automatic_first_attempt_committed_nothing(committed, event, attempt, deploys):+ deploy = load(".github/workflows/publish-skill-reports.yml")["jobs"]["deploy"]+ assert deploy["needs"] == "prepare"+ # No status function, so GitHub also requires prepare to succeed.+ assert "always()" not in deploy["if"]+ assert expression(deploy["if"], job_context(committed, event, attempt)) is deploys+++ ("committed", "deploy", "comments"),+ [+ ("true", "success", True),+ ("false", "success", True),+ # The run that pushed this content owns its deploy and its comment.+ ("false", "skipped", False),+ ("true", "skipped", False),+ ("true", "failure", False),+ ("false", "failure", False),+ ("true", "cancelled", False),+ ],+)+def test_comment_waits_for_this_runs_own_successful_deploy(committed, deploy, comments):+ job = load(".github/workflows/publish-skill-reports.yml")["jobs"]["comment"]+ assert expression(job["if"], job_context(committed, deploy=deploy)) is comments+ assert not expression(job["if"], {**job_context(committed, deploy=deploy), "needs.prepare.result": "failure"})++[email protected]("committed", ["true", "false"])+ ("event", "attempt"), [("workflow_run", "1"), ("workflow_dispatch", "1"), ("workflow_run", "2")]+)+def test_skipped_stale_publication_never_deploys_or_comments(committed, event, attempt):+ # A stale run can follow an earlier attempt that already wrote pr_number and is_current.+ jobs = load(".github/workflows/publish-skill-reports.yml")["jobs"]+ for deploy in ("success", "skipped"):+ context = job_context(committed, event, attempt, deploy, skipped="true")+ assert not expression(jobs["deploy"]["if"], context)+ assert not expression(jobs["comment"]["if"], context)+++def run_publish_loop(tmp_path, *, diffs, pushes):+ """Run the real publish step shell with stand-ins for git and the publisher."""+ step = next(+ step+ for step in load(".github/workflows/publish-skill-reports.yml")["jobs"]["prepare"]["steps"]+ if step.get("id") == "publish"+ )+ workspace, temp, bin_dir = tmp_path / "workspace", tmp_path / "temp", tmp_path / "bin"+ python = workspace / "publisher/.venv/bin/python"+ for directory in (workspace / "site", python.parent, temp, bin_dir):+ directory.mkdir(parents=True, exist_ok=True)+ python.write_text('#!/bin/sh\nmkdir -p "$OUTPUT_DIR"\necho summary > "$OUTPUT_DIR/summary.md"\n')+ git = bin_dir / "git"+ # Each `diff --cached --quiet` and `push` pops its exit code from a queue file.+ git.write_text(+ "#!/bin/sh\n"+ 'echo "$*" >> "$CALLS"\n'+ 'pop() { code=$(head -n 1 "$1"); tail -n +2 "$1" > "$1.next"; mv "$1.next" "$1"; exit "$code"; }\n'+ 'case "$3" in diff) pop "$DIFFS";; push) pop "$PUSHES";; esac\n'+ )+ for script in (python, git):+ script.chmod(0o755)+ (tmp_path / "diffs").write_text("".join(f"{code}\n" for code in diffs))+ (tmp_path / "pushes").write_text("".join(f"{code}\n" for code in pushes))+ output = tmp_path / "github-output"+ env = {+ "PATH": f"{bin_dir}{os.pathsep}{os.environ['PATH']}",+ "GITHUB_WORKSPACE": str(workspace),+ "RUNNER_TEMP": str(temp),+ "GITHUB_OUTPUT": str(output),+ "GITHUB_STEP_SUMMARY": str(tmp_path / "step-summary"),+ "REPORT_RUN_ID": "123",+ "CALLS": str(tmp_path / "calls"),+ "DIFFS": str(tmp_path / "diffs"),+ "PUSHES": str(tmp_path / "pushes"),+ }+ subprocess.run(+ ["bash", "--noprofile", "--norc", "-eo", "pipefail", "-c", step["run"]],+ cwd=workspace,+ env=env,+ check=True,+ capture_output=True,+ )+ calls = (tmp_path / "calls").read_text().splitlines()+ return output.read_text().splitlines(), calls+++ ("diffs", "pushes", "committed", "commits"),+ [+ ([0], [], "false", 0),+ ([1], [0], "true", 1),+ ([1, 1], [1, 0], "true", 2),+ # A concurrent writer already published this exact run attempt.+ ([1, 0], [1], "false", 1),+ ],+)+def test_publish_step_reports_whether_it_pushed_new_content(tmp_path, diffs, pushes, committed, commits):+ output, calls = run_publish_loop(tmp_path, diffs=diffs, pushes=pushes)+ assert output == [f"committed={committed}"]+ assert sum(call.startswith("-C site commit") for call in calls) == commits+ assert load(".github/workflows/publish-skill-reports.yml")["jobs"]["prepare"]["outputs"]["committed"] == (+ "${{ steps.publish.outputs.committed }}"+ )+++ ("event", "attempt", "deploys"),+ [("workflow_run", "1", False), ("workflow_dispatch", "1", True), ("workflow_run", "2", True)],+)+def test_losing_a_push_race_to_the_same_attempt_never_comments_before_a_deploy(tmp_path, event, attempt, deploys):+ # Attempt 1's push is rejected because another publisher run pushed this scan+ # attempt first. After the reset, attempt 2 has nothing left to commit.+ output, _calls = run_publish_loop(tmp_path, diffs=[1, 0], pushes=[1])+ assert output == ["committed=false"]+ jobs = load(".github/workflows/publish-skill-reports.yml")["jobs"]+ context = job_context("false", event, attempt)+ assert expression(jobs["deploy"]["if"], context) is deploys+ deploy = "success" if deploys else "skipped"+ assert expression(jobs["comment"]["if"], {**context, "needs.deploy.result": deploy}) is deploys+++def checkout(job: str) -> dict:+ steps = load(".github/workflows/publish-skill-reports.yml")["jobs"][job]["steps"]+ return next(+ step for step in steps if step.get("uses", "").startswith("actions/checkout@") and "repository" in step["with"]+ )+++def tracked_files() -> list[str]:+ listing = subprocess.run(["git", "ls-files", "-z"], cwd=ROOT, capture_output=True, check=True).stdout+ return [name for name in listing.decode().split("\0") if name and (ROOT / name).is_file()]+++def test_prepare_sparse_checkout_has_the_build_inputs_and_tools(tmp_path):+ options = checkout("prepare")["with"]+ assert options.get("sparse-checkout-cone-mode", "true") == "true"+ directories = options["sparse-checkout"].split()+ assert directories == ["ci", "src"]+ # Cone mode checks out every root file plus the listed directories.+ selected = [name for name in tracked_files() if "/" not in name or name.split("/", 1)[0] in directories]+ for name in selected:+ (tmp_path / name).parent.mkdir(parents=True, exist_ok=True)+ shutil.copy2(ROOT / name, tmp_path / name)+ project = tomllib.loads((tmp_path / "pyproject.toml").read_text())+ build_inputs = [project["project"]["readme"], *project["project"]["license-files"], "uv.lock"]+ build_inputs.append("ci/requirements/uv.txt")+ for name in build_inputs:+ assert (tmp_path / name).is_file(), name+ for package, data in project["tool"]["setuptools"]["package-data"].items():+ assert (tmp_path / "src" / package.replace(".", "/")).is_dir(), package+ assert data+ code = (+ "import sys; sys.path[:0] = [sys.argv[1] + '/src', sys.argv[1] + '/ci'];"+ "import publish_reports, native_reports, report_views, github_api, skillevaluator.reporting.retained as r;"+ "mods = (publish_reports, native_reports, report_views, github_api, r);"+ "assert all(m.__file__.startswith(sys.argv[1]) for m in mods), [m.__file__ for m in mods]"+ )+ subprocess.run([sys.executable, "-c", code, str(tmp_path)], check=True, capture_output=True, cwd=tmp_path)+++def test_comment_sparse_checkout_runs_with_only_the_standard_library(tmp_path):+ options = checkout("comment")["with"]+ assert options["sparse-checkout-cone-mode"] == "false"+ names = options["sparse-checkout"].split()+ assert names == ["ci/comment_report.py", "ci/github_api.py"]+ for name in names:+ (tmp_path / name).parent.mkdir(parents=True, exist_ok=True)+ shutil.copy2(ROOT / name, tmp_path / name)+ run = load(".github/workflows/publish-skill-reports.yml")["jobs"]["comment"]["steps"][-1]["run"]+ assert run == "python3 publisher/ci/comment_report.py"+ # No site-packages, user site or PYTHON* variables, like the runner's bare system Python.+ result = subprocess.run(+ [sys.executable, "-S", "-s", "-E", str(tmp_path / "ci/comment_report.py")],+ cwd=tmp_path,+ env={"REPORT_RUN_ID": "invalid"},+ capture_output=True,+ text=True,+ check=False,+ )+ assert result.returncode == 1+ assert "PR feedback rejected" in result.stderr+ assert "Traceback" not in result.stderrdiff --git a/tests/ci/test_report_views.py b/tests/ci/test_report_views.pyindex b225d37c..ba9ca54e 100644--- a/tests/ci/test_report_views.py+++ b/tests/ci/test_report_views.py@@ -393,6 +393,71 @@ def test_truncated_markdown_keeps_table_boundaries_intact():assert "| --- |" not in text+def test_markdown_budget_stops_only_after_the_bounded_output_is_fixed():+ import random++ generator = random.Random(238)+ for _ in range(300):+ lines = []+ for _ in range(generator.randrange(1, 60)):+ text = generator.choice("xé€|") * generator.randrange(0, 90)+ lines.append(f"| {text} |" if generator.random() < 0.4 else text)+ limit = generator.randrange(200, 3000)+ footer = "Full report: index.html"+ kept = []+ budget = views._Budget(kept, limit, footer)+ for line in lines:+ if budget.spent():+ break+ kept.append(line)+ assert views._bounded(kept, limit, footer) == views._bounded(lines, limit, footer)+++def test_details_markdown_does_not_escape_evidence_past_its_limit(tmp_path, monkeypatch):+ summary, data = example(tmp_path)+ data["results"][0]["success_details"] = [{"check": f"check-{index}", "message": "x" * 900} for index in range(700)]+ data["results"][0]["summary"]["checks_performed"] = 700+ rows = []+ for index in range(1, 4):+ directory = tmp_path / f"skill-{index:03d}"+ directory.mkdir(exist_ok=True)+ if index == 3:+ data["results"][0]["success_details"][0]["message"] = "LAST SKILL EVIDENCE"+ (directory / "skillevaluator-tier1.json").write_text(json.dumps(data))+ rows.append({**summary["skills"][0], "path": f"skills/sample-{index}", "report_dir": f"skill-{index:03d}"})+ summary.update(skills=rows, total=3, passed=3)+ escaped = []+ original = views._md+ monkeypatch.setattr(views, "_md", lambda value: escaped.append(value) or original(value))+ views.render_reports(tmp_path, summary, {})+ details = (tmp_path / "details.md").read_text()+ assert "Truncated for CI display limits" in details+ assert len(details.encode()) <= views.DETAIL_LIMIT+ assert "## skills/sample\\-2" not in details+ assert not any("LAST SKILL EVIDENCE" in str(value) for value in escaped)+ # Escaping stops inside the first skill's 700 evidence lines (two values each).+ assert len(escaped) < 1600+ assert "3 skills · 3 passed · 0 failed · 0 incomplete" in (tmp_path / "summary.md").read_text()+++def test_details_markdown_drops_a_lone_surrogate_past_its_limit(tmp_path):+ # JSON reports can carry lone surrogates as \udcXX escapes.+ summary, data = example(tmp_path)+ details = [{"check": f"check-{index}", "message": "x" * 180} for index in range(3000)]+ data["results"][0]["success_details"] = details+ path = tmp_path / "skill-001/skillevaluator-tier1.json"+ path.write_text(json.dumps(data), encoding="utf-8")+ views.render_reports(tmp_path, summary, {})+ expected = (tmp_path / "details.md").read_text(encoding="utf-8")+ kept = expected.count("**Evidence: check")+ assert "Truncated for CI display limits" in expected and kept < len(details)+ # _bounded() measures the first line it drops, so the surrogate goes one line later.+ details[kept + 1]["message"] = "bad \udcff name"+ path.write_text(json.dumps(data), encoding="utf-8")+ views.render_reports(tmp_path, summary, {})+ assert (tmp_path / "details.md").read_text(encoding="utf-8") == expected++def test_markdown_findings_render_readable_quotes_and_preserve_entity_text(tmp_path):from html.parser import HTMLParser@@ -452,6 +517,38 @@ def failed_result(}+def test_each_result_status_is_computed_once_for_all_views(tmp_path, monkeypatch):+ summary, data = example(tmp_path, result=failed_result())+ align_verdict(summary, data, "failed")+ (tmp_path / "skill-001/skillevaluator-tier1.json").write_text(json.dumps(data))+ calls = []+ original = views._status+ monkeypatch.setattr(views, "_status", lambda result: calls.append(result["validator"]) or original(result))+ views.render_reports(tmp_path, summary, {})+ assert calls == ["Schema"]+ assert "First blocking problem" in (tmp_path / "summary.md").read_text()+ assert "| Schema | FAIL |" in (tmp_path / "details.md").read_text()+++def test_given_models_are_rendered_without_rereading_or_changing_them(tmp_path, monkeypatch):+ summary, data = example(tmp_path, result=failed_result())+ align_verdict(summary, data, "failed")+ (tmp_path / "skill-001/skillevaluator-tier1.json").write_text(json.dumps(data))+ errors = {"skill-001": "Native report could not be rendered: example"}+ views.render_reports(tmp_path, summary, {}, render_errors=errors)+ expected = {name: (tmp_path / name).read_text() for name in ("summary.md", "details.md", "pr-comment.md")}+ models = views.read_skill_models(tmp_path, summary)+ before = copy.deepcopy(models)++ def unexpected_read(directory):+ raise AssertionError("render_reports re-read a report it was given")++ monkeypatch.setattr(views, "_read", unexpected_read)+ views.render_reports(tmp_path, summary, {}, render_errors=errors, models=models)+ assert {name: (tmp_path / name).read_text() for name in expected} == expected+ assert models == before++def test_compact_all_pass_is_concise_and_rich_evidence_stays_in_details(tmp_path):from markdown_it import MarkdownItdiff --git a/tests/ci/test_run_tier1.py b/tests/ci/test_run_tier1.pyindex fdd542a9..f4d30ba6 100644--- a/tests/ci/test_run_tier1.py+++ b/tests/ci/test_run_tier1.py@@ -3,12 +3,23 @@"""Test the CI adapter's boundaries; actual scanner coverage is tested by live CI."""+import contextlib+import errno+import gcimport importlib.utilimport ioimport jsonimport os+import reimport shutil+import signal+import statimport subprocess+import sys+import threading+import time+import types+import weakrefimport zipfilefrom pathlib import Pathfrom subprocess import CompletedProcess@@ -22,6 +33,13 @@ MODULE_PATH = Path(__file__).parents[2] / "ci" / "run_tier1.py"SPEC = importlib.util.spec_from_file_location("ci_run_tier1", MODULE_PATH)runner = importlib.util.module_from_spec(SPEC)SPEC.loader.exec_module(runner)+SELECTOR_SPEC = importlib.util.spec_from_file_location(+ "ci_selection", MODULE_PATH.with_name("select_changed_skills.py")+)+selector = importlib.util.module_from_spec(SELECTOR_SPEC)+SELECTOR_SPEC.loader.exec_module(selector)+# Upper bound for cross-thread waits; generous for loaded CI runners.+WAIT = 60def skill(workspace, path):@@ -31,7 +49,7 @@ def skill(workspace, path):return directory-def payload(passed=True, status="passed", incomplete=None):+def payload(passed=True, status="passed", incomplete=None, findings=1):results = [ValidationResult(validator_name=f"Tier 1 fixture validator {index + 1}",@@ -41,13 +59,13 @@ def payload(passed=True, status="passed", incomplete=None):]for result in results:result.metadata["gating"] = {"tier": 1, "blocking": True}- if not passed:+ for number in range(0 if passed else findings):results[0].add_finding(Finding(category="SCHEMA",severity="high",check_name="fixture_failure",- message="Fixture validation failed",+ message="Fixture validation failed" + (f" ({number + 1})" if number else ""),file_path="SKILL.md",))@@ -56,12 +74,20 @@ def payload(passed=True, status="passed", incomplete=None):return json.loads(JSONReporter(include_timestamp=False).render_all(results))+def report_dir(output: Path) -> str:+ """Name the skill report directory behind the private scanner directory the runner passes."""+ match = re.fullmatch(r"skillevaluator-private-(skill-[0-9]{3})-.+", output.name)+ assert match, output.name+ return match.group(1)++def fake_cli(responses, calls):+ # Responses follow skill order, not call order: skills may run in parallel.def invoke(argv, **kwargs):calls.append((argv, kwargs))- exit_code, report = responses[len(calls) - 1]- kwargs["stdout"].write(b"::error::untrusted scanner text\n")output = Path(argv[argv.index("-o") + 1])+ exit_code, report = responses[int(report_dir(output).removeprefix("skill-")) - 1]+ kwargs["stdout"].write(b"::error::untrusted scanner text\n")if report is not None:(output / "skillevaluator-tier1.json").write_text(json.dumps(report), encoding="utf-8")return CompletedProcess(argv, exit_code)@@ -151,6 +177,27 @@ def test_rejects_unsafe_roots(tmp_path, root):runner.discover_skills(tmp_path, root)+LONG_NAME = "x" * 170+++ "roots",+ [+ "skills/tab\tname",+ "skills/escape\x1bname",+ "\n".join(["skills"] * 101),+ "\n".join([f"skills/{LONG_NAME}"] * 100),+ ],+)+def test_full_scan_and_changed_selection_share_skill_root_rules(tmp_path, roots):+ for name in ("tab\tname", "escape\x1bname", LONG_NAME):+ skill(tmp_path, f"skills/{name}")+ with pytest.raises(runner.ConfigurationError):+ runner.discover_skills(tmp_path, roots)+ with pytest.raises(selector.SelectionError):+ selector.skill_roots(tmp_path, roots)++def test_empty_selection_and_missing_root_fail(tmp_path):for roots in ("", "skills"):with pytest.raises(runner.ConfigurationError):@@ -203,7 +250,12 @@ def test_all_skills_run_after_failure_and_scanner_stdout_is_not_printed(tmp_pathassert "::error::" not in capsys.readouterr().outassert "::error::" in (reports / "skill-001" / "stdout.log").read_text()assert "--no-llm" in calls[0][0]- assert calls[0][1]["env"]["HOME"] == "/tmp/evaluator-home"+ assert calls[0][1]["env"]["HOME"].startswith("/tmp/evaluator-home/worker-")+ # The CLI writes no HTML; the native renderer supplies each skill page.+ assert calls[0][0][calls[0][0].index("-r") + 1] == "json,markdown,sarif"+ for child in ("skill-001", "skill-002"):+ page = (reports / child / "skillevaluator-tier1.html").read_text()+ assert "Content-Security-Policy" in pagedef test_structured_failure_logs_and_annotations_are_escaped(capsys):@@ -356,8 +408,8 @@ def test_invalid_json_is_reported_and_later_skills_still_run(tmp_path):def invoke(argv, **kwargs):completed = original(argv, **kwargs)- if len(calls) == 1:- output = Path(argv[argv.index("-o") + 1])+ output = Path(argv[argv.index("-o") + 1])+ if report_dir(output) == "skill-001":(output / "skillevaluator-tier1.json").write_text("not json", encoding="utf-8")return completed@@ -368,6 +420,25 @@ def test_invalid_json_is_reported_and_later_skills_still_run(tmp_path):assert len(calls) == 2+def test_report_loading_validates_without_copying_metadata(tmp_path, monkeypatch):+ from skillevaluator.reporting import retained++ report = payload(passed=False)+ report["results"][0]["findings"][0]["metadata"] = {"rule": {"nested": ["evidence"]}}+ path = tmp_path / "skillevaluator-tier1.json"+ path.write_text(json.dumps(report), encoding="utf-8")++ def no_copies(value, memo=None):+ raise AssertionError("validation copied report metadata")++ monkeypatch.setattr(retained.copy, "deepcopy", no_copies)+ assert runner._load_report(path) == report+ report["results"][0]["passed"] = "false"+ path.write_text(json.dumps(report), encoding="utf-8")+ with pytest.raises(ValueError, match="passed must be boolean"):+ runner._load_report(path)++def test_process_start_error_does_not_skip_later_skills(tmp_path):workspace = tmp_path / "workspace"skill(workspace, "skills/a")@@ -377,14 +448,16 @@ def test_process_start_error_does_not_skip_later_skills(tmp_path):def invoke(argv, **kwargs):calls.append(argv)- if len(calls) == 1:- raise OSError("untrusted failure text")output = Path(argv[argv.index("-o") + 1])+ if report_dir(output) == "skill-001":+ raise OSError("untrusted failure text")(output / "skillevaluator-tier1.json").write_text(json.dumps(payload()), encoding="utf-8")return CompletedProcess(argv, 0)assert runner.run(workspace, reports, {"SKILL_PATHS": "skills"}, execute=invoke) == 1assert len(calls) == 2+ summary = json.loads((reports / "summary.json").read_text())+ assert [row["passed"] for row in summary["skills"]] == [False, True]def test_static_run_does_not_receive_ambient_model_credentials():@@ -593,6 +666,7 @@ def test_invalid_report_urls_fail_before_scanning_and_keep_diagnostics(tmp_path,def test_completed_results_survive_timeout_and_unstarted_skills_stay_incomplete(tmp_path):+ # One worker pins the sequential order: each result is saved before the next skill starts.workspace = tmp_path / "workspace"for name in ("a", "b", "c"):skill(workspace, f"skills/{name}")@@ -610,7 +684,7 @@ def test_completed_results_survive_timeout_and_unstarted_skills_stay_incomplete(assert "INCOMPLETE" in (reports / "summary.md").read_text()raise subprocess.TimeoutExpired(argv, kwargs["timeout"])- assert runner.run(workspace, reports, {}, execute=execute) == 1+ assert runner.run(workspace, reports, {"SCAN_WORKERS": "1"}, execute=execute) == 1summary = json.loads((reports / "summary.json").read_text())assert summary["passed"] == 1 and summary["incomplete"] == 2assert "time limit" in summary["skills"][1]["error"]@@ -654,3 +728,590 @@ def test_relative_annotation_location_is_confined_to_skill(capsys, path, expecteassert f"file={expected},line=7" in commandelse:assert "file=" not in command and "line=" not in command+++def parallel_cli(responses, calls, *, before=None):+ """Fake CLI for concurrent runs; ``before(name)`` runs inside the call."""+ lock = threading.Lock()++ def invoke(argv, **kwargs):+ output = Path(argv[argv.index("-o") + 1])+ name = report_dir(output)+ with lock:+ calls.append((name, dict(kwargs["env"])))+ if before:+ before(name)+ exit_code, report = responses[name]+ kwargs["stdout"].write(b"::error::untrusted scanner text\n")+ (output / "skillevaluator-tier1.json").write_text(json.dumps(report), encoding="utf-8")+ return CompletedProcess(argv, exit_code)++ return invoke+++def scan_outputs(reports):+ names = ["summary.json", "summary.md", "details.md", "pr-comment.md", "index.html"]+ names += sorted(path.relative_to(reports).as_posix() for path in reports.glob("skill-*/skillevaluator-tier1.html"))+ return {name: (reports / name).read_text() for name in names}+++def test_parallel_scan_matches_sequential_rows_logs_annotations_and_reports(tmp_path, capsys):+ workspace = tmp_path / "workspace"+ names = [f"skills/s{index}" for index in range(1, 7)]+ for name in names:+ skill(workspace, name)+ failing = {"skill-002", "skill-003", "skill-005"}+ # Three failing skills with four findings each exceed the 10-annotation budget.+ responses = {+ f"skill-{index:03d}": (1, payload(False, "failed", findings=4))+ if f"skill-{index:03d}" in failing+ else (0, payload())+ for index in range(1, 7)+ }+ environment = {"SKILL_PATHS": "skills", "GITHUB_ACTIONS": "true"}+ outputs, logs, finished = {}, {}, {}+ for workers in ("1", "4"):+ order = finished[workers] = []+ done = {name: threading.Event() for name in responses}++ def before(name, workers=workers, order=order, done=done):+ if workers != "1" and name == "skill-001":+ # Finish skill 1 last so completion order differs from skill order.+ assert done["skill-002"].wait(WAIT)+ order.append(name)+ done[name].set()++ reports = tmp_path / f"reports-{workers}"+ result = runner.run(+ workspace,+ reports,+ {**environment, "SCAN_WORKERS": workers},+ execute=parallel_cli(responses, [], before=before),+ )+ assert result == 1+ outputs[workers] = scan_outputs(reports)+ logs[workers] = [line for line in capsys.readouterr().out.splitlines() if "at a time" not in line]+ assert finished["1"] == sorted(responses)+ assert finished["4"].index("skill-002") < finished["4"].index("skill-001")+ assert logs["1"] == logs["4"]+ assert outputs["1"] == outputs["4"]+ annotations = [line for line in logs["4"] if line.startswith("::error")]+ assert len(annotations) == 10+ assert all("file=skills/s2/SKILL.md" in line or "file=skills/s3/SKILL.md" in line for line in annotations[:8])+ headers = [line for line in logs["4"] if line.startswith("SkillEvaluator: evaluating skill")]+ assert headers == [+ f"SkillEvaluator: evaluating skill {index} of 6: {json.dumps(name)}" for index, name in enumerate(names, 1)+ ]+++def test_default_scan_runs_four_skills_at_once(tmp_path, capsys):+ workspace = tmp_path / "workspace"+ for index in range(5):+ skill(workspace, f"skills/s{index}")+ responses = {f"skill-{index:03d}": (0, payload()) for index in range(1, 6)}+ barrier = threading.Barrier(4, timeout=WAIT)+ lock = threading.Lock()+ state = {"active": 0, "peak": 0, "calls": 0}++ def before(name):+ with lock:+ state["calls"] += 1+ state["active"] += 1+ state["peak"] = max(state["peak"], state["active"])+ first_wave = state["calls"] <= 4+ if first_wave:+ barrier.wait()+ with lock:+ state["active"] -= 1++ calls = []+ assert runner.run(workspace, tmp_path / "reports", {}, execute=parallel_cli(responses, calls, before=before)) == 0+ assert state["peak"] == 4 and len(calls) == 5+ assert "SkillEvaluator: scanning 5 skills, up to 4 at a time." in capsys.readouterr().out+++def test_one_worker_runs_skills_strictly_one_at_a_time(tmp_path):+ workspace = tmp_path / "workspace"+ for index in range(3):+ skill(workspace, f"skills/s{index}")+ reports = tmp_path / "reports"+ responses = {f"skill-{index:03d}": (0, payload()) for index in range(1, 4)}+ seen = []++ def before(name):+ # The previous skill's result is saved before this one starts.+ seen.append(json.loads((reports / "summary.json").read_text())["passed"])++ assert (+ runner.run(workspace, reports, {"SCAN_WORKERS": "1"}, execute=parallel_cli(responses, [], before=before)) == 0+ )+ assert seen == [0, 1, 2]+++def test_each_worker_gets_its_own_home_and_temporary_directory(tmp_path):+ workspace = tmp_path / "workspace"+ for index in range(4):+ skill(workspace, f"skills/s{index}")+ audit = tmp_path / "audit-tmp"+ audit.mkdir()+ responses = {f"skill-{index:03d}": (0, payload()) for index in range(1, 5)}+ barrier = threading.Barrier(2, timeout=WAIT)+ pairs = []++ def before(name):+ barrier.wait()+ pairs.append(name)++ calls = []+ environment = {"SCAN_WORKERS": "2", "TMPDIR": str(audit)}+ assert (+ runner.run(workspace, tmp_path / "reports", environment, execute=parallel_cli(responses, calls, before=before))+ == 0+ )+ assert len(calls) == 4+ homes = {env["HOME"] for _, env in calls}+ assert homes == {"/tmp/evaluator-home/worker-1", "/tmp/evaluator-home/worker-2"}+ for _, env in calls:+ worker = env["HOME"].rsplit("-", 1)[1]+ assert env["TMPDIR"] == str(audit / f"skillevaluator-worker-{worker}")+ assert Path(env["HOME"]).is_dir() and Path(env["TMPDIR"]).is_dir()+ # Calls 1-2 and 3-4 ran together, each pair on two different workers.+ assert calls[0][1]["HOME"] != calls[1][1]["HOME"] and calls[2][1]["HOME"] != calls[3][1]["HOME"]+++def test_concurrent_timeout_keeps_finished_results_and_starts_no_more_skills(tmp_path, capsys):+ workspace = tmp_path / "workspace"+ for name in "abcde":+ skill(workspace, f"skills/{name}")+ reports = tmp_path / "reports"+ now = [0.0]+ second_started, first_timed_out = threading.Event(), threading.Event()+ calls = []++ def execute(argv, **kwargs):+ output = Path(argv[argv.index("-o") + 1])+ calls.append(report_dir(output))+ if report_dir(output) == "skill-001":+ assert second_started.wait(WAIT)+ now[0] = runner.SCAN_TIMEOUT_SECONDS + 1+ first_timed_out.set()+ raise subprocess.TimeoutExpired(argv, kwargs["timeout"])+ second_started.set()+ # Skill 2 was already running when the budget ran out; its result counts.+ assert first_timed_out.wait(WAIT)+ (output / "skillevaluator-tier1.json").write_text(json.dumps(payload()), encoding="utf-8")+ return CompletedProcess(argv, 0)++ environment = {"SCAN_WORKERS": "2"}+ assert runner.run(workspace, reports, environment, execute=execute, clock=lambda: now[0]) == 1+ assert sorted(calls) == ["skill-001", "skill-002"]+ summary = json.loads((reports / "summary.json").read_text())+ assert (summary["passed"], summary["incomplete"]) == (1, 4)+ rows = summary["skills"]+ assert "time limit" in rows[0]["error"] and rows[1]["passed"] is True+ assert all("Not evaluated" in row["error"] for row in rows[2:])+ output = capsys.readouterr().out+ assert 'evaluating skill 2 of 5: "skills/b"' in output and "evaluating skill 3" not in output+ assert "Scan time limit reached; this skill did not finish." in output+ assert "3 skills were not started" in output+++def test_checkpoint_saves_a_later_skill_before_an_earlier_one_finishes(tmp_path):+ workspace = tmp_path / "workspace"+ skill(workspace, "skills/a")+ skill(workspace, "skills/b")+ reports = tmp_path / "reports"+ responses = {"skill-001": (0, payload()), "skill-002": (0, payload())}+ snapshots = []++ def before(name):+ if name != "skill-001":+ return+ deadline = time.monotonic() + WAIT+ while time.monotonic() < deadline:+ rows = json.loads((reports / "summary.json").read_text())["skills"]+ if rows[1]["passed"]:+ snapshots.append(rows)+ return+ time.sleep(0.01)++ assert (+ runner.run(workspace, reports, {"SCAN_WORKERS": "2"}, execute=parallel_cli(responses, [], before=before)) == 0+ )+ assert len(snapshots) == 1 and snapshots[0][0]["passed"] is False++[email protected]("value", ["0", "17", "four", "-1", "2.5", "٣"])+def test_invalid_scan_worker_count_fails_before_scanning(tmp_path, value):+ workspace = tmp_path / "workspace"+ skill(workspace, "skills/a")+ reports, calls = tmp_path / "reports", []+ assert runner.run(workspace, reports, {"SCAN_WORKERS": value}, execute=fake_cli([], calls)) == 1+ assert not calls+ assert "SCAN_WORKERS" in json.loads((reports / "summary.json").read_text())["error"]+++posix_only = pytest.mark.skipif(os.name != "posix", reason="uses a shell script as the evaluator")+++def scan_shim(tmp_path, *, fast=(), ignore_term=False):+ """Put a fake ``skillevaluator`` on a PATH and return that PATH.++ A scan records its PID and its background tool's PID in ``scan.pids``,+ then runs for five minutes. Skills named in ``fast`` pass at once instead.+ """+ tool = tmp_path / "bin"+ tool.mkdir()+ report = tmp_path / "passing-report.json"+ report.write_text(json.dumps(payload()), encoding="utf-8")+ cases = "".join(f'*/{name}) cp "{report}" "$out/skillevaluator-tier1.json"; exit 0 ;;\n' for name in fast)+ script = tool / "skillevaluator"+ script.write_text(+ "#!/bin/sh\n"+ + ("trap '' TERM\n" if ignore_term else "")+ + "for out; do :; done\n"+ + f'case "$2" in\n{cases}*) ;;\nesac\n'+ + "sleep 300 &\n"+ + 'echo "$$ $!" > "$out/scan.tmp" && mv "$out/scan.tmp" "$out/scan.pids"\n'+ + "exec sleep 300\n",+ encoding="utf-8",+ )+ script.chmod(0o755)+ return f"{tool}{os.pathsep}/usr/bin{os.pathsep}/bin"+++def scan_pids(output, deadline):+ pids = output / "scan.pids"+ while not pids.exists():+ assert time.monotonic() < deadline, "the scan did not start"+ time.sleep(0.02)+ return [int(pid) for pid in pids.read_text().split()]+++def private_scan_pids(temporary, name, deadline):+ """Read the PIDs of the scan for report directory ``name`` from its private directory under TMPDIR."""+ while not (found := sorted(temporary.rglob(f"skillevaluator-private-{name}-*/scan.pids"))):+ assert time.monotonic() < deadline, "the scan did not start"+ time.sleep(0.02)+ return scan_pids(found[0].parent, deadline)+++def still_running(pids):+ """Wait up to WAIT for the processes to end; return those still running."""+ deadline = time.monotonic() + WAIT+ running = [pid for pid in pids if not ended(pid)]+ while running and time.monotonic() < deadline:+ time.sleep(0.05)+ running = [pid for pid in running if not ended(pid)]+ return running+++def ended(pid):+ try:+ os.kill(pid, 0)+ except ProcessLookupError:+ return True+ return False+++def kill_quietly(pids):+ # Only for processes not seen to end: a PID that ended may be reused.+ for pid in pids:+ with contextlib.suppress(OSError):+ os.kill(pid, signal.SIGKILL)+++@posix_only+def test_scan_timeout_also_ends_the_scanner_tools(tmp_path, monkeypatch):+ path = scan_shim(tmp_path)+ output = tmp_path / "out"+ output.mkdir()+ clock = types.SimpleNamespace(offset=0.0)+ clock.monotonic = lambda: time.monotonic() + clock.offset+ monkeypatch.setattr(runner, "time", clock)+ outcome = {}++ def scan():+ try:+ runner.run_scan(+ ["skillevaluator", "tier1", "skills/a", "-o", str(output)],+ abort=threading.Event(),+ timeout=WAIT,+ env={"PATH": path},+ stdout=subprocess.DEVNULL,+ stderr=subprocess.DEVNULL,+ )+ except subprocess.TimeoutExpired as exc:+ outcome["error"] = exc++ thread = threading.Thread(target=scan, daemon=True)+ thread.start()+ pids = survivors = scan_pids(output, time.monotonic() + WAIT)+ try:+ clock.offset = WAIT+ thread.join(WAIT)+ assert not thread.is_alive() and "error" in outcome+ survivors = still_running(pids)+ assert not survivors+ finally:+ kill_quietly(survivors)+++@posix_only+def test_aborted_scan_is_killed_after_the_grace_period(tmp_path, monkeypatch):+ monkeypatch.setattr(runner, "STOP_GRACE_SECONDS", 0.5)+ path = scan_shim(tmp_path, ignore_term=True)+ output = tmp_path / "out"+ output.mkdir()+ abort, outcome = threading.Event(), {}++ def scan():+ outcome["process"] = runner.run_scan(+ ["skillevaluator", "tier1", "skills/a", "-o", str(output)],+ abort=abort,+ timeout=WAIT,+ env={"PATH": path},+ stdout=subprocess.DEVNULL,+ stderr=subprocess.DEVNULL,+ )++ thread = threading.Thread(target=scan, daemon=True)+ thread.start()+ pids = survivors = scan_pids(output, time.monotonic() + WAIT)+ try:+ abort.set()+ thread.join(WAIT)+ assert not thread.is_alive()+ # The scan and its tool ignore SIGTERM, so only SIGKILL ends them.+ assert outcome["process"].returncode == -signal.SIGKILL+ survivors = still_running(pids)+ assert not survivors+ finally:+ kill_quietly(survivors)+++@posix_only+def test_checkpoint_error_ends_running_scans_at_once(tmp_path, monkeypatch):+ workspace = tmp_path / "workspace"+ skill(workspace, "skills/fast")+ skill(workspace, "skills/slow")+ audit = tmp_path / "audit-tmp"+ audit.mkdir()+ reports = tmp_path / "reports"+ environment = {"PATH": scan_shim(tmp_path, fast=["fast"]), "TMPDIR": str(audit)}+ render, pids, outcome = runner.render_reports, [], {}+ survivors = pids++ def render_reports(*args):+ if (reports / "skill-001" / "skillevaluator-tier1.json").exists():+ # The slow scan is running when saving the fast result fails.+ pids.extend(private_scan_pids(audit, "skill-002", time.monotonic() + WAIT))+ raise OSError(errno.ENOSPC, "No space left on device")+ render(*args)++ monkeypatch.setattr(runner, "render_reports", render_reports)++ def scan():+ try:+ runner.run(workspace, reports, environment)+ except BaseException as exc:+ outcome["error"] = exc++ thread = threading.Thread(target=scan, daemon=True)+ thread.start()+ try:+ thread.join(WAIT)+ assert not thread.is_alive(), "the run waited for a running scan"+ assert isinstance(outcome["error"], OSError)+ survivors = still_running(pids)+ assert pids and not survivors+ finally:+ kill_quietly(survivors)+++def start_runner_with_endless_scans(tmp_path, *, ignore_term=False):+ """Start the runner alone in its session, as Docker runs it, and wait for two endless scans."""+ workspace = tmp_path / "workspace"+ for name in ("a", "b"):+ skill(workspace, f"skills/{name}")+ audit = tmp_path / "audit-tmp"+ audit.mkdir()+ reports = tmp_path / "reports"+ environment = {"PATH": scan_shim(tmp_path, ignore_term=ignore_term), "TMPDIR": str(audit)}+ environment.update(+ (name, os.environ[name]) for name in ("PYTHONPATH", "PYTHONDONTWRITEBYTECODE") if name in os.environ+ )+ with (tmp_path / "runner.log").open("wb") as log:+ process = subprocess.Popen(+ [sys.executable, str(MODULE_PATH), "--workspace", str(workspace), "--reports", str(reports)],+ env=environment,+ stdout=log,+ stderr=subprocess.STDOUT,+ start_new_session=True,+ )+ pids = []+ try:+ deadline = time.monotonic() + WAIT+ for name in ("skill-001", "skill-002"):+ pids += private_scan_pids(audit, name, deadline)+ except BaseException:+ process.kill()+ process.wait()+ kill_quietly(pids)+ raise+ return process, pids+++@posix_only+def test_ctrl_c_ends_running_scans_and_their_tools(tmp_path):+ process, pids = start_runner_with_endless_scans(tmp_path)+ survivors = pids+ try:+ # Docker sends Ctrl-C to the runner (PID 1) only, not to its scans.+ os.kill(process.pid, signal.SIGINT)+ assert process.wait(WAIT) == -signal.SIGINT+ survivors = still_running(pids)+ assert not survivors+ finally:+ if process.poll() is None:+ process.kill()+ process.wait()+ kill_quietly(survivors)+++@posix_only+def test_second_ctrl_c_does_not_leave_the_runner_waiting(tmp_path):+ # The scans ignore SIGTERM, so the second Ctrl-C arrives while the+ # runner is still waiting for them to end.+ process, pids = start_runner_with_endless_scans(tmp_path, ignore_term=True)+ survivors = pids+ try:+ os.kill(process.pid, signal.SIGINT)+ time.sleep(1)+ os.kill(process.pid, signal.SIGINT)+ assert process.wait(WAIT) == -signal.SIGINT+ survivors = still_running(pids)+ assert not survivors+ finally:+ if process.poll() is None:+ process.kill()+ process.wait()+ kill_quietly(survivors)+++def test_each_report_is_released_once_its_result_is_logged(tmp_path, monkeypatch):+ # Reports can be 32 MiB each; 100 of them must not stay in memory.+ workspace = tmp_path / "workspace"+ for index in range(5):+ skill(workspace, f"skills/s{index}")+ responses = {f"skill-{index:03d}": (0, payload()) for index in range(1, 6)}+ load, reports, alive = runner._load_report, [], []++ class Report(dict):+ pass++ def tracked(path):+ report = Report(load(path))+ reports.append(weakref.ref(report))+ return report++ def before(name):+ gc.collect()+ alive.append(sum(report() is not None for report in reports))++ monkeypatch.setattr(runner, "_load_report", tracked)+ environment = {"SCAN_WORKERS": "1"}+ assert (+ runner.run(workspace, tmp_path / "reports", environment, execute=parallel_cli(responses, [], before=before))+ == 0+ )+ # Only the latest result may still be referenced when the next scan starts.+ assert alive == [0, 1, 1, 1, 1]+++def worker_directory(env, name):+ return Path(env[name]) / ("worker-1" if name == "HOME" else "skillevaluator-worker-1")++[email protected]("name", ["HOME", "TMPDIR"])+def test_symlinked_worker_directory_is_rejected(tmp_path, name):+ env = {"HOME": str(tmp_path / "home"), "TMPDIR": str(tmp_path / "tmp")}+ directory = worker_directory(env, name)+ directory.parent.mkdir()+ (tmp_path / "elsewhere").mkdir(mode=0o700)+ directory.symlink_to(tmp_path / "elsewhere", target_is_directory=True)+ with pytest.raises(runner.ConfigurationError):+ runner.worker_environment(env, 1)+++@posix_only[email protected]("name", ["HOME", "TMPDIR"])+def test_existing_worker_directory_open_to_others_is_rejected(tmp_path, name):+ env = {"HOME": str(tmp_path / "home"), "TMPDIR": str(tmp_path / "tmp")}+ directory = worker_directory(env, name)+ directory.mkdir(parents=True)+ directory.chmod(0o777)+ with pytest.raises(runner.ConfigurationError):+ runner.worker_environment(env, 1)+ assert stat.S_IMODE(directory.stat().st_mode) == 0o777+++@posix_only+def test_worker_directory_owned_by_another_user_is_rejected(tmp_path, monkeypatch):+ env = {"HOME": str(tmp_path / "home"), "TMPDIR": str(tmp_path / "tmp")}+ uid = os.getuid()+ monkeypatch.setattr(runner.os, "getuid", lambda: uid + 1)+ with pytest.raises(runner.ConfigurationError):+ runner.worker_environment(env, 1)+++@posix_only+def test_existing_private_worker_directories_are_reused(tmp_path):+ env = {"HOME": str(tmp_path / "home"), "TMPDIR": str(tmp_path / "tmp")}+ for name in ("HOME", "TMPDIR"):+ worker_directory(env, name).mkdir(mode=0o700, parents=True)+ (worker_directory(env, name) / "kept").write_text("", encoding="utf-8")+ isolated = runner.worker_environment(env, 1)+ for name in ("HOME", "TMPDIR"):+ assert isolated[name] == str(worker_directory(env, name))+ assert (worker_directory(env, name) / "kept").is_file()+++def test_symlinked_worker_directory_fails_before_scanning(tmp_path):+ workspace = tmp_path / "workspace"+ skill(workspace, "skills/a")+ audit = tmp_path / "audit-tmp"+ audit.mkdir()+ (tmp_path / "elsewhere").mkdir(mode=0o700)+ (audit / "skillevaluator-worker-1").symlink_to(tmp_path / "elsewhere", target_is_directory=True)+ reports, calls = tmp_path / "reports", []+ assert runner.run(workspace, reports, {"TMPDIR": str(audit)}, execute=fake_cli([], calls)) == 1+ assert not calls+ summary = json.loads((reports / "summary.json").read_text())+ assert summary["overall_status"] == "error"+ assert summary["error"] == "Scan worker directories must be private directories owned by this user."++[email protected]("workers", ["1", "4"])+def test_unexpected_worker_error_is_a_row_error_and_later_skills_still_run(tmp_path, capsys, workers):+ workspace = tmp_path / "workspace"+ for name in ("a", "b", "c"):+ skill(workspace, f"skills/{name}")+ reports, calls = tmp_path / "reports", []+ passing = parallel_cli({"skill-002": (0, payload()), "skill-003": (0, payload())}, calls)++ def execute(argv, **kwargs):+ if report_dir(Path(argv[argv.index("-o") + 1])) == "skill-001":+ raise RuntimeError("untrusted worker text")+ return passing(argv, **kwargs)++ assert runner.run(workspace, reports, {"SCAN_WORKERS": workers}, execute=execute) == 1+ assert sorted(name for name, _ in calls) == ["skill-002", "skill-003"]+ rows = json.loads((reports / "summary.json").read_text())["skills"]+ assert rows[0]["error"] == "Evaluator execution failed unexpectedly; no result is claimed."+ assert [row["passed"] for row in rows] == [False, True, True]+ captured = capsys.readouterr()+ for name in ("a", "b", "c"):+ assert f'Skill result: "skills/{name}"' in captured.out+ assert "untrusted worker text" not in captured.out + captured.err+ assert all("untrusted worker text" not in text for text in scan_outputs(reports).values())diff --git a/tests/ci/test_select_changed_skills.py b/tests/ci/test_select_changed_skills.pyindex f8c7dd99..fd5d529b 100644--- a/tests/ci/test_select_changed_skills.py+++ b/tests/ci/test_select_changed_skills.py@@ -3,6 +3,7 @@import importlib.utilimport os+import randomimport shutilimport subprocessfrom pathlib import Path@@ -240,3 +241,116 @@ def test_unrelated_histories_do_not_fall_back_to_full_scan(repository):head = commit(repo)with pytest.raises(selector.SelectionError, match="common merge base"):select(repo, base, head)+++def _reference_within(path, directory):+ # The pairwise rule that _containing replaces.+ return directory in {".", path} or path.startswith(directory + "/")++[email protected]("seed", range(20))+def test_containing_set_matches_pairwise_directory_matching(seed):+ generator = random.Random(seed)+ names = ["a", "b", "ab", "a b", "SKILL.md", "[x]", "a.b"]++ def path():+ return "/".join(generator.choice(names) for _ in range(generator.randint(1, 4)))++ paths = sorted({path() for _ in range(generator.randint(0, 30))})+ directories = {".", *(path() for _ in range(40))}+ directories |= {item.rsplit("/", 1)[0] for item in paths if "/" in item}+ containing = selector._containing(paths)+ for directory in directories:+ assert (directory in containing) is any(_reference_within(item, directory) for item in paths), directory+++def test_commits_and_checkout_are_resolved_with_one_rev_parse(repository, monkeypatch):+ repo, base = repository+ (repo / "skills/second/SKILL.md").write_text("# Changed\n")+ head = commit(repo)+ calls = []+ real_git = selector._git++ def record(workspace, *arguments):+ calls.append(arguments)+ return real_git(workspace, *arguments)++ monkeypatch.setattr(selector, "_git", record)+ assert select(repo, base, head)["selected"] == ["skills/second"]+ assert [arguments[0] for arguments in calls].count("rev-parse") == 1+++def test_annotated_tag_object_is_not_accepted_as_a_commit(repository):+ repo, base = repository+ git(repo, "tag", "-a", "fixture-tag", "-m", "fixture tag")+ tag = git(repo, "rev-parse", "fixture-tag")+ assert tag != base+ with pytest.raises(selector.SelectionError, match="not tag objects"):+ select(repo, tag, base)+++def test_revision_shaped_file_name_is_not_read_as_a_path(repository):+ repo, head = repository+ missing = "1" * 40+ # An untracked file outside the skill roots, named like the requested revision.+ (repo / f"{missing}^{{commit}}").write_text("not a revision\n")+ with pytest.raises(selector.SelectionError, match="could not resolve"):+ select(repo, missing, head)+++def _clone(source, target, *options):+ subprocess.run(+ [+ "git",+ "-c",+ "protocol.file.allow=always",+ "clone",+ "-q",+ "--no-checkout",+ *options,+ source.as_uri(),+ str(target),+ ],+ check=True,+ env={**os.environ, "GIT_CONFIG_GLOBAL": os.devnull, "GIT_CONFIG_NOSYSTEM": "1"},+ capture_output=True,+ )+++def test_blobless_clone_selects_like_a_full_clone_without_fetching_blobs(repository, tmp_path):+ # The workflow checks out changed mode with filter: blob:none; selection must+ # need only commits, trees and the checked-out files.+ repo, base = repository+ (repo / "skills/first skill/SKILL.md").write_text("# Changed first skill\n")+ (repo / "skills/second/notes.md").write_text("new file\n")+ head = commit(repo)+ git(repo, "config", "uploadpack.allowFilter", "true")+ clone = tmp_path / "blobless"+ _clone(repo, clone, "--filter=blob:none")+ subprocess.run(+ ["git", "-C", str(clone), "-c", "protocol.file.allow=always", "checkout", "-q", "--detach", head],+ check=True,+ env={**os.environ, "GIT_CONFIG_GLOBAL": os.devnull, "GIT_CONFIG_NOSYSTEM": "1"},+ capture_output=True,+ )+ # Only the base commit has this manifest content, so a blobless clone lacks it.+ old_blob = git(repo, "rev-parse", f"{base}:skills/first skill/SKILL.md")+ missing = git(clone, "rev-list", "--objects", "--missing=print", base)+ assert f"?{old_blob}" in missing.splitlines()+ full = select(repo, base, head)+ partial = select(clone, base, head)+ assert partial["selected"] == full["selected"] == ["skills/first skill", "skills/second"]+ assert partial == full+ # Selection runs Git with protocol.allow=never, so the old blob stays missing.+ assert f"?{old_blob}" in git(clone, "rev-list", "--objects", "--missing=print", base).splitlines()+++def test_shallow_clone_fails_changed_selection_instead_of_scanning_everything(repository, tmp_path):+ repo, base = repository+ (repo / "skills/second/SKILL.md").write_text("# Changed\n")+ head = commit(repo)+ clone = tmp_path / "shallow"+ _clone(repo, clone, "--depth=1")+ git(clone, "checkout", "-q", "--detach", head)+ with pytest.raises(selector.SelectionError):+ select(clone, base, head)diff --git a/tests/reporting/test_advisory_skip_verdict.py b/tests/reporting/test_advisory_skip_verdict.pyindex bf2d5fa9..84dd519c 100644--- a/tests/reporting/test_advisory_skip_verdict.py+++ b/tests/reporting/test_advisory_skip_verdict.py@@ -409,8 +409,9 @@ def test_html_tier2_failure_stays_failed_when_cli_gate_is_nonblocking() -> None:html_data = _html_report_data(html)assert html_data["summary"]["status"] == "passed"+ # The tier still shows as failed, but a non-blocking gate requires no changes.assert re.search(- r'tier-card fail.*?tier-card-label">Tier 2</span>.*?tier-card-verdict">\s*FAIL\s*</span>',+ r'tier-card fail.*?tier-card-label">Tier 2</span>.*?tier-card-verdict">\s*Failed \(advisory\)\s*</span>',html,re.DOTALL,)@@ -425,7 +426,7 @@ def test_html_tier2_incomplete_evidence_is_not_rendered_as_success() -> None:tier2_tab = _html_tab(html, "tier2")assert re.search(- r'tier-card warn.*?tier-card-label">Tier 2</span>.*?tier-card-verdict">\s*INCOMPLETE\s*</span>',+ r'tier-card warn.*?tier-card-label">Tier 2</span>.*?tier-card-verdict">\s*Incomplete\s*</span>',html,re.DOTALL,)@@ -449,12 +450,12 @@ def test_html_malformed_tier3_result_is_incomplete_like_benchmark() -> None:assert "| Tier 3 | Live agent evaluation | **INCOMPLETE** |" in benchmarkassert re.search(- r'tier-card warn.*?tier-card-label">Tier 3</span>.*?tier-card-verdict">\s*INCOMPLETE\s*</span>',+ r'tier-card warn.*?tier-card-label">Tier 3</span>.*?tier-card-verdict">\s*Incomplete\s*</span>',html,re.DOTALL,)assert not re.search(- r'tier-card pass.*?tier-card-label">Tier 3</span>.*?tier-card-verdict">\s*PASS\s*</span>',+ r'tier-card pass.*?tier-card-label">Tier 3</span>.*?tier-card-verdict">\s*Passed\s*</span>',html,re.DOTALL,)@@ -478,12 +479,12 @@ def test_html_tier3_scan_incomplete_is_not_rendered_as_failure() -> None:assert "| Tier 3 | Live agent evaluation | **INCOMPLETE** |" in benchmarkassert html_data["summary"]["status"] == "incomplete"assert re.search(- r'tier-card warn.*?tier-card-label">Tier 3</span>.*?tier-card-verdict">\s*INCOMPLETE\s*</span>',+ r'tier-card warn.*?tier-card-label">Tier 3</span>.*?tier-card-verdict">\s*Incomplete\s*</span>',html,re.DOTALL,)assert not re.search(- r'tier-card fail.*?tier-card-label">Tier 3</span>.*?tier-card-verdict">\s*FAIL\s*</span>',+ r'tier-card fail.*?tier-card-label">Tier 3</span>.*?tier-card-verdict">\s*Changes required\s*</span>',html,re.DOTALL,)@@ -513,7 +514,7 @@ def test_html_payloadless_generic_tier3_skip_is_rendered_as_skipped() -> None:assert "| Tier 3 | Live agent evaluation | **INCOMPLETE** |" in benchmarkassert html_data["summary"]["status"] == "passed"assert re.search(- r'tier-card skipped.*?tier-card-label">Tier 3</span>.*?tier-card-verdict">\s*SKIPPED\s*</span>',+ r'tier-card skipped.*?tier-card-label">Tier 3</span>.*?tier-card-verdict">\s*Skipped\s*</span>',html,re.DOTALL,)@@ -629,12 +630,12 @@ def test_html_partial_tier3_payload_is_process_pass_but_publication_incomplete()assert html_data["summary"]["status"] == "passed"assert html_data["publication"]["status"] == "incomplete"assert re.search(- r'tier-card warn.*?tier-card-label">Tier 3</span>.*?tier-card-verdict">\s*INCOMPLETE\s*</span>',+ r'tier-card warn.*?tier-card-label">Tier 3</span>.*?tier-card-verdict">\s*Incomplete\s*</span>',html,re.DOTALL,)assert not re.search(- r'tier-card pass.*?tier-card-label">Tier 3</span>.*?tier-card-verdict">\s*PASS\s*</span>',+ r'tier-card pass.*?tier-card-label">Tier 3</span>.*?tier-card-verdict">\s*Passed\s*</span>',html,re.DOTALL,)@@ -669,7 +670,7 @@ def test_html_production_wrapped_partial_tier3_payload_is_incomplete() -> None:assert "| Tier 3 | Live agent evaluation | **INCOMPLETE** |" in benchmarkassert re.search(- r'tier-card warn.*?tier-card-label">Tier 3</span>.*?tier-card-verdict">\s*INCOMPLETE\s*</span>',+ r'tier-card warn.*?tier-card-label">Tier 3</span>.*?tier-card-verdict">\s*Incomplete\s*</span>',html,re.DOTALL,)@@ -763,7 +764,7 @@ def test_html_selects_executed_tier3_payload_independent_of_result_order() -> Nomarkdown = MarkdownReporter(include_timestamp=False).render_all(results)assert re.search(- r'tier-card pass.*?tier-card-label">Tier 3</span>.*?tier-card-verdict">\s*PASS\s*</span>',+ r'tier-card pass.*?tier-card-label">Tier 3</span>.*?tier-card-verdict">\s*Passed\s*</span>',html,re.DOTALL,)@@ -967,7 +968,7 @@ def test_failed_tier3_execution_status_overrides_pass_verdict_across_reporters()assert "| Tier 3 | Live agent evaluation | **FAIL** |" in benchmarkassert html_payload is not None and html_payload["verdict"] == "fail"assert 'data-publication-status="fail"' in html_output- assert re.search(r'tier-card-verdict">\s*FAIL\s*</span>', html_output)+ assert re.search(r'tier-card-verdict">\s*Changes required\s*</span>', html_output)assert json_payload["publication_status"] == "fail"assert "**Publication status:** ❌ FAIL" in markdownassert "**Verdict:** FAIL" in markdown@@ -1132,9 +1133,11 @@ def test_contradictory_tier3_truth_fields_fail_closed_across_reporters(("dimension_score", "expected_status"),[(0.45, "neutral"), (0.10, "fail")],)[email protected]("with_skill_effect", [False, True])def test_html_displays_effective_dimension_verdict(dimension_score: float,expected_status: str,+ with_skill_effect: bool,) -> None:tier1 = ValidationResult(validator_name="SCHEMA")tier1.add_success("schema", "Schema passed")@@ -1142,6 +1145,9 @@ def test_html_displays_effective_dimension_verdict(tier2.add_success("similarity_check", "Similarity scan completed")tier3 = _complete_tier3_result("DIMENSIONS", score=0.9, runtime_seconds=1.0)payload = tier3.metadata["agent_eval"]+ effect_message = "Per-case evidence identifies the skill effect."+ if with_skill_effect:+ payload["conclusions"] = [{"title": "Skill effect", "message": effect_message, "severity": "info"}]agent = payload["agents"]["codex"]baseline = agent["baseline"]agent["with_skill"] = dimension_score@@ -1162,11 +1168,19 @@ def test_html_displays_effective_dimension_verdict(markdown = MarkdownReporter(include_timestamp=False).render_all(results)expected_upper = expected_status.upper()+ # The HTML report uses one verdict vocabulary; the Markdown keeps PASS/FAIL.+ expected_words = {"pass": "Passed", "fail": "Changes required", "neutral": "Neutral"}[expected_status]assert f"Overall verdict: {expected_upper}" in benchmarkassert f"| Tier 3 | Live agent evaluation | **{expected_upper}** |" in benchmarkassert f'data-publication-status="{expected_status}"' in html_output- assert re.search(rf'tier-card-verdict">\s*{expected_upper}\s*</span>', html_output)- assert f"Verdict at a glance:</strong>\n {expected_upper}" in html_output+ assert re.search(rf'tier-card-verdict">\s*{expected_words}\s*</span>', html_output)+ assert f"Verdict at a glance:</strong>\n {expected_words}" in html_output+ if with_skill_effect:+ assert f"{expected_words} · {effect_message}" in html_output+ overview = html_output.split('class="t3-overview-lead"', 1)[1].split("</div>", 1)[0]+ assert "with an overall score of" not in overview+ else:+ assert f"{expected_words} with an overall score of" in html_outputassert json_payload["publication_status"] == expected_statusassert f"**Verdict:** {expected_upper}" in markdown@@ -1261,7 +1275,7 @@ def test_all_reporters_tolerate_non_mapping_agent_payload() -> None:markdown = MarkdownReporter(include_timestamp=False).render_all([malformed])assert "| Tier 3 | Live agent evaluation | **INCOMPLETE** |" in benchmark- assert re.search(r'tier-card-verdict">\s*INCOMPLETE\s*</span>', html)+ assert re.search(r'tier-card-verdict">\s*Incomplete\s*</span>', html)assert json_payload["tier3"]["agents"] == {"malformed": []}assert "## Tier 3: Agent Evaluation" in markdown@@ -1415,7 +1429,7 @@ def test_html_partial_tier3_payload_wrong_types_fail_closed_without_crashing() -payload = reporter._tier3_report_data([partial_tier3])assert re.search(- r'tier-card warn.*?tier-card-label">Tier 3</span>.*?tier-card-verdict">\s*INCOMPLETE\s*</span>',+ r'tier-card warn.*?tier-card-label">Tier 3</span>.*?tier-card-verdict">\s*Incomplete\s*</span>',html,re.DOTALL,)@@ -1828,7 +1842,14 @@ def test_publication_status_matrix_is_consistent_across_reporters(assert html_data["publication"]["status"] == expected_statusassert html_data["publication"]["eligible"] is eligibleassert f'data-publication-status="{expected_status}"' in html_output- assert f"Publication: {expected_status.title()}" in html_output+ # Eligibility sits next to the verdict in the report hero.+ publication_word = {"pass": "Eligible", "fail": "Not eligible", "neutral": "Neutral"}[expected_status]+ assert re.search(+ rf'data-publication-status="{expected_status}">.*?Publication eligibility</span>\s*'+ rf'<span class="status-badge [a-z]+">{publication_word}</span>',+ html_output,+ re.DOTALL,+ )assert "**Publication status:**" in markdown@@ -2397,9 +2418,7 @@ def test_advisory_tier3_skip_with_unusual_text_is_not_a_failure(tmp_path: Path,benchmark = BenchmarkReporter(skill_name="demo", include_timestamp=False).render_all(results)json_payload = json.loads(JSONReporter(include_timestamp=False, expected_skill_name="demo").render_all(results))- html_data = _html_report_data(- HTMLReporter(include_timestamp=False, expected_skill_name="demo").render_all(results)- )+ html_data = _html_report_data(HTMLReporter(include_timestamp=False, expected_skill_name="demo").render_all(results))markdown = MarkdownReporter(include_timestamp=False, expected_skill_name="demo").render_all(results)benchmark_path = tmp_path / "BENCHMARK.md"benchmark_path.write_text(benchmark, encoding="utf-8")@@ -3404,7 +3423,7 @@ def test_html_executed_tier3_payload_normalizes_malformed_display_fields() -> Nolossy = field in {"timing", "skill_name", "overall_score"}expected_class = "warn" if lossy else "pass"- expected_verdict = "INCOMPLETE" if lossy else "PASS"+ expected_verdict = "Incomplete" if lossy else "Passed"assert re.search(rf'tier-card {expected_class}.*?tier-card-label">Tier 3</span>.*?tier-card-verdict">\s*'rf"{expected_verdict}\s*</span>",@@ -3482,7 +3501,7 @@ def test_html_invalid_tier3_agent_scores_and_verdicts_fail_closed() -> None:assert "| Tier 3 | Live agent evaluation | **INCOMPLETE** |" in benchmarkassert re.search(- r'tier-card warn.*?tier-card-label">Tier 3</span>.*?tier-card-verdict">\s*INCOMPLETE\s*</span>',+ r'tier-card warn.*?tier-card-label">Tier 3</span>.*?tier-card-verdict">\s*Incomplete\s*</span>',html,re.DOTALL,)@@ -3529,7 +3548,7 @@ def test_html_payloadless_tier3_failure_is_order_independent() -> None:fallback = reporter._tier3_report_data(results)assert re.search(- r'tier-card fail.*?tier-card-label">Tier 3</span>.*?tier-card-verdict">\s*FAIL\s*</span>',+ r'tier-card fail.*?tier-card-label">Tier 3</span>.*?tier-card-verdict">\s*Changes required\s*</span>',html,re.DOTALL,)@@ -3579,8 +3598,9 @@ def test_html_tier1_dashboard_and_details_exclude_clean_skips() -> None:html = HTMLReporter(include_timestamp=False).render_all([completed, skipped])tier1 = _html_tab(html, "tier1")- assert re.search(r"Validators Run</h3>\s*<p[^>]*>1</p>", tier1)- assert re.search(r"Passed</h3>\s*<p[^>]*>1</p>", tier1)+ # A cleanly skipped check is counted as skipped, never as run or passed.+ assert re.search(r'Checks passed</dt>\s*<dd class="kpi-value"><strong>1</strong> of 1</dd>', tier1)+ assert '<dd class="kpi-sub">1 of 2 checks ran, 1 skipped</dd>' in tier1assert 'data-status="skipped"' in tier1assert re.search(r'status-badge warn">\s*Skipped\s*</span>', tier1)assert "Skipped: repository checkout unavailable" in tier1@@ -3641,9 +3661,9 @@ def test_advisory_skip_is_non_blocking_in_html_but_incomplete_for_benchmark_publassert html_data["results"][0]["status"] == "skipped"assert html_data["gating"]["would_block"] is Falsehtml = HTMLReporter(include_timestamp=False).render_all([result])- assert re.search(r'tier-card-verdict">\s*SKIPPED\s*</span>', html)+ assert re.search(r'tier-card-verdict">\s*Skipped\s*</span>', html)assert "Live evaluation skipped:</strong> No public provider key" in html- assert not re.search(r'tier-card-verdict">\s*NEUTRAL\s*</span>', html)+ assert not re.search(r'tier-card-verdict">\s*Neutral\s*</span>', html)assert "Overall verdict: INCOMPLETE" in benchmarkassert "Overall verdict: FAIL" not in benchmarkassert "Tier 3 live evaluation: SKIPPED — No public provider key" in benchmark@@ -3673,8 +3693,12 @@ def test_advisory_skip_does_not_appear_as_tier1_failure_in_html() -> None:assert html_data["summary"]["passed_count"] == 1assert html_data["summary"]["failed_count"] == 0assert html_data["summary"]["advisory_skipped_count"] == 1- assert re.search(r"Validators Run</h3>\s*<p[^>]*>1</p>", tier1)- assert re.search(r"Failed</h3>\s*<p[^>]*>0</p>", tier1)+ # The advisory Tier 3 skip is not a Tier 1 check: one check ran and none failed.+ assert re.search(+ r'Checks passed</dt>\s*<dd class="kpi-value"><strong>1</strong> of 1</dd>\s*'+ r'<dd class="kpi-sub">1 of 1 checks ran</dd>',+ tier1,+ )assert "AGENT_EVAL" not in tier1@@ -3690,7 +3714,7 @@ def test_real_agent_eval_failure_still_fails_required_validation() -> None:html = HTMLReporter(include_timestamp=False).render_all([result])tier3 = _html_tab(html, "tier3")- assert re.search(r'tier-card-verdict">\s*FAIL\s*</span>', html)+ assert re.search(r'tier-card-verdict">\s*Changes required\s*</span>', html)assert "Harbor execution failed" in tier3diff --git a/tests/reporting/test_incomplete_scans.py b/tests/reporting/test_incomplete_scans.pyindex be43d80e..404333a7 100644--- a/tests/reporting/test_incomplete_scans.py+++ b/tests/reporting/test_incomplete_scans.py@@ -23,6 +23,18 @@ def _plain(output: str) -> str:return " ".join(output.split())+def _assert_html_verdict_is_incomplete(output: str) -> None:+ """The header verdict and the summary verdict both say Incomplete, and nothing says Passed."""+ header = output.split('<header class="main-header">', 1)[1].split("</header>", 1)[0]+ summary = output.split('<section class="exec-summary', 1)[1].split("</section>", 1)[0]++ assert 'data-verdict="incomplete"' in header+ assert '<span class="sr-only">Verdict: </span>Incomplete</span>' in header+ assert '<span class="verdict-chip warn">Incomplete</span>' in summary+ assert 'data-verdict="pass"' not in output+ assert "verdict-chip pass" not in output++def _incomplete_result() -> ValidationResult:result = ValidationResult(validator_name="Security Scan",@@ -77,12 +89,24 @@ class TestIncompleteScanReporting:assert "skillspector timed out after 300 seconds" in outputassert "✅ PASSED" not in output- def test_html_marks_incomplete_and_never_all_passed(self):+ def test_html_marks_incomplete_and_never_claims_a_pass(self):output = HTMLReporter(include_timestamp=False).render_all([_incomplete_result()])- assert "Incomplete" in output+ _assert_html_verdict_is_incomplete(output)assert "skillspector" in output- assert "All Passed" not in output++ def test_html_incomplete_check_among_passing_checks_is_incomplete(self):+ passing = []+ for name in ("License Compliance", "PII Scan"):+ result = ValidationResult(validator_name=name, validator_description="d")+ result.add_success(check_name=name, message="No issues")+ passing.append(result)++ output = HTMLReporter(include_timestamp=False, explicit_skill_name="demo").render_all(+ [passing[0], _incomplete_result(), passing[1]]+ )++ _assert_html_verdict_is_incomplete(output)def test_benchmark_blocks_publication_on_incomplete_evidence(self):output = BenchmarkReporter(include_timestamp=False).render_all([_incomplete_result()])diff --git a/tests/reporting/test_report_logo.py b/tests/reporting/test_report_logo.pynew file mode 100644index 00000000..cd4d6cb6--- /dev/null+++ b/tests/reporting/test_report_logo.py@@ -0,0 +1,59 @@+# SPDX-License-Identifier: Apache-2.0+"""The inline SkillEvaluator logo used by the HTML report and the CI pages."""++import re+import xml.etree.ElementTree as ET+from pathlib import Path++from skillevaluator.reporting.html import render_logo++REPO_ROOT = Path(__file__).resolve().parents[2]+SVG = "{http://www.w3.org/2000/svg}"+GEOMETRY = ("x", "y", "width", "height", "rx", "cx", "cy", "r", "d", "stroke-width")+++def _shapes(svg: str) -> list[tuple]:+ root = ET.fromstring(svg)+ return [+ (element.tag.removeprefix(SVG), tuple((key, element.get(key)) for key in GEOMETRY if element.get(key)))+ for element in root.iter()+ if element.tag.removeprefix(SVG) in {"rect", "circle", "path"}+ ]+++def test_logo_matches_the_published_asset_geometry() -> None:+ asset = (REPO_ROOT / "docs" / "assets" / "skillevaluator-logo.svg").read_text(encoding="utf-8")+ assert _shapes(render_logo("se-logo-test")) == _shapes(asset)+++def test_labelled_logo_has_unique_prefixed_ids() -> None:+ logo = render_logo("se-logo-rail", 28, "SkillEvaluator logo")+ root = ET.fromstring(logo)+ assert root.get("role") == "img"+ assert root.get("aria-labelledby") == "se-logo-rail-title se-logo-rail-desc"+ assert re.findall(r'\bid="([^"]+)"', logo) == ["se-logo-rail-title", "se-logo-rail-desc"]+ assert root.get("width") == root.get("height") == "28"+++def test_decorative_logo_is_hidden_and_has_no_ids() -> None:+ logo = render_logo("se-logo-side", 28, "")+ root = ET.fromstring(logo)+ assert root.get("aria-hidden") == "true"+ assert root.get("role") is None+ assert " id=" not in logo+++def test_logo_colors_follow_theme_variables_not_the_os_setting() -> None:+ logo = render_logo("se-logo-theme")+ assert "prefers-color-scheme" not in logo+ assert "<style" not in logo+ colors = re.findall(r"(?:fill|stroke):([^;\"]+)", logo)+ assert colors+ assert all(color.startswith("var(--se-logo-") for color in colors)+++def test_logo_escapes_its_label_and_id_prefix() -> None:+ logo = render_logo('x"><script>', 28, "<b>name</b>")+ assert "<script>" not in logo+ assert "<b>" not in logo+ ET.fromstring(logo)diff --git a/tests/reporting/test_report_redesign.py b/tests/reporting/test_report_redesign.pynew file mode 100644index 00000000..78d73be6--- /dev/null+++ b/tests/reporting/test_report_redesign.py@@ -0,0 +1,506 @@+# SPDX-License-Identifier: Apache-2.0+"""HTML report layout: header, executive summary, Fix first, vocabulary, print and accessibility."""++import re+from html.parser import HTMLParser+from itertools import pairwise+from pathlib import Path++from skillevaluator.evaluation.tier3_report import build_agent_eval_payload+from skillevaluator.models import Finding, ValidationResult+from skillevaluator.reporting.html import HTMLReporter, fix_first_for_results, plain_check_name++TEMPLATES = Path(__file__).resolve().parents[2] / "src" / "skillevaluator" / "reporting" / "templates"+++def _finding(severity: str, message: str, file_path: str = "", line: int | None = None) -> Finding:+ return Finding(+ category="SECURITY",+ severity=severity,+ check_name="check",+ message=message,+ file_path=file_path,+ line_number=line,+ )+++def _results() -> list[ValidationResult]:+ """One skill: the same secret from two scanners, mixed severities, a failing quality score."""+ pii = ValidationResult(validator_name="PII Scan", validator_description="Detect PII")+ pii.add_finding(_finding("critical", "Hardcoded key in code", "scripts/fetch.py", 11))+ secrets = ValidationResult(validator_name="Secrets Detection", validator_description="Detect secrets")+ secrets.add_finding(_finding("critical", "API key found", "scripts/fetch.py", 11))+ risk = ValidationResult(validator_name="Code Risk Analysis", validator_description="Static analysis")+ risk.add_finding(_finding("medium", "Request without timeout", "scripts/fetch.py", 15))+ risk.add_finding(_finding("high", "Shell injection", "scripts/fetch.py", 21))+ lint = ValidationResult(validator_name="SCRIPT_LINT", validator_description="Lint scripts")+ lint.add_finding(_finding("low", "Line too long", "scripts/fetch.py", 3))+ quality = ValidationResult(validator_name="QUALITY", validator_description="Quality dimensions")+ quality.add_error("Quality score 66.2/100 is below the required 70/100")+ quality.metadata["quality_config"] = {"source": "packaged", "min_score": 70}+ quality.metadata["quality_scores"] = {+ "skill_name": "demo",+ "overall_score": 66.2,+ "grade": "D",+ "skill_type": "script-based",+ "dimensions": {"correctness": {"score": 66.2, "weight": 1.0, "issues_count": 0, "issues": []}},+ }+ clean = ValidationResult(validator_name="License Compliance", validator_description="Licenses")+ clean.add_success("license", "License found")+ return [pii, secrets, risk, lint, quality, clean]+++def _render(results: list[ValidationResult]) -> str:+ return HTMLReporter(include_timestamp=False, explicit_skill_name="demo").render_all(results)+++class _Outline(HTMLParser):+ def __init__(self) -> None:+ super().__init__()+ self.headings: list[int] = []+ self.ids: list[str] = []+ self.scripts: list[str] = []+ self._in_script = False++ def handle_starttag(self, tag, attrs):+ if re.fullmatch(r"h[1-6]", tag):+ self.headings.append(int(tag[1]))+ self.ids.extend(value for key, value in attrs if key == "id" and value)+ if tag == "script":+ self._in_script = True+ self.scripts.append("")++ def handle_endtag(self, tag):+ if tag == "script":+ self._in_script = False++ def handle_data(self, data):+ if self._in_script:+ self.scripts[-1] += data+++def _outline(html: str) -> _Outline:+ parser = _Outline()+ parser.feed(html)+ return parser+++def test_fix_first_ranks_by_severity_and_lists_each_location_once() -> None:+ items = fix_first_for_results(_results(), skill_name="demo")++ assert [item["severity"] for item in items] == ["critical", "high", "high", "medium", "low"]+ # The secret both scanners report at scripts/fetch.py:11 takes one slot.+ assert [item["location"] for item in items].count("scripts/fetch.py:11") == 1+ assert items[0]["check"] == "PII Scan"+ # A failed check without a located finding keeps its reason in the list.+ assert {"kind": "check", "check": "Quality score"}.items() <= items[2].items()+ assert items[2]["message"] == "Quality score 66.2/100 is below the required 70/100"+ assert items[-1]["blocking"] is False+++def test_fix_first_puts_an_unfinished_scan_first_when_incomplete() -> None:+ results = _results()+ secrets = results[1]+ secrets.mark_scan_incomplete("gitleaks")++ items = fix_first_for_results(results, skill_name="demo")++ assert items[0]["kind"] == "incomplete"+ # Without a diagnostic the report names the scanner and claims no cause.+ assert items[0]["message"] == "Secrets Detection did not finish (gitleaks)"+++def _timed_out_scan() -> list[ValidationResult]:+ """A scan that kept one partial finding, then timed out."""+ scan = ValidationResult(validator_name="Security Scan", validator_description="Detect security vulnerabilities")+ scan.add_finding(_finding("high", "Tool Misuse: subprocess.run(shell=True)", "scripts/run.py", 7))+ scan.mark_scan_incomplete("skillspector")+ scan.add_error("skillspector timed out after 300 seconds; partial results only")+ clean = ValidationResult(validator_name="License Compliance", validator_description="Licenses")+ clean.add_success("license", "License found")+ return [scan, clean]+++def test_incomplete_scan_with_partial_findings_keeps_its_diagnostic_visible() -> None:+ html = _render(_timed_out_scan())+ card = html.split('id="c-demo-security-scan"', 1)[1].split('<div class="check-finding', 1)[0]+ raw_output = re.findall(r'<details class="raw-output">.*?</details>', card, flags=re.DOTALL)+ outside_raw_output = re.sub(r'<details class="raw-output">.*?</details>', "", card, flags=re.DOTALL)++ # The reason the scan stopped is shown, and printed, outside the collapsed output.+ assert '<p class="scan-diagnostic">skillspector timed out after 300 seconds; partial results only</p>' in (+ outside_raw_output+ )+ # Only the line that repeats the finding card folds away.+ assert len(raw_output) == 1+ assert "[SECURITY-HIGH] Tool Misuse" in raw_output[0]+ assert "timed out" not in raw_output[0]+++def test_incomplete_scan_names_the_diagnostic_instead_of_a_missing_scanner() -> None:+ results = _timed_out_scan()+ html = _render(results)+ summary = html.split('<section class="exec-summary', 1)[1].split("</section>", 1)[0]++ items = fix_first_for_results(results, skill_name="demo")++ assert items[0]["message"] == (+ "Security Scan did not finish: skillspector timed out after 300 seconds; partial results only"+ )+ assert "Security Scan: skillspector timed out after 300 seconds; partial results only." in summary+ assert "unavailable" not in items[0]["message"]+ assert "Install or enable" not in summary+++def test_missing_scanner_asks_for_an_install() -> None:+ secrets = ValidationResult(validator_name="Secrets Detection", validator_description="Detect secrets")+ secrets.add_warning("gitleaks not installed - secrets scanning skipped. Install with: brew install gitleaks")+ secrets.mark_scan_incomplete("gitleaks")++ html = _render([secrets])+ summary = html.split('<section class="exec-summary', 1)[1].split("</section>", 1)[0]+ items = fix_first_for_results([secrets], skill_name="demo")++ assert items[0]["message"].startswith("Secrets Detection did not finish: gitleaks not installed")+ assert "Install or enable the scanner and run the evaluation again." in summary+++def test_fix_first_uses_the_scan_root_for_a_repository_root_skill() -> None:+ risk = ValidationResult(validator_name="Code Risk Analysis", validator_description="Static analysis")+ risk.add_finding(_finding("high", "subprocess call with shell=True", "/workspace/scripts/fetch.py", 21))+ scan = ValidationResult(validator_name="Security Scan", validator_description="Detect vulnerabilities")+ scan.add_finding(_finding("high", "Tool Misuse: subprocess.run(command, shell=True", "scripts/fetch.py", 21))++ items = fix_first_for_results([risk, scan], skill_name=".", expected_skill_name="workspace", scan_root="/workspace")+ html = HTMLReporter(include_timestamp=False, explicit_skill_name=".", scan_root="/workspace").render_all(+ [risk, scan]+ )+ fix_first = html.split('class="fix-first"', 1)[1].split("</section>", 1)[0]++ # Absolute and relative paths to one file and line take one slot, shown relative to the skill.+ assert [item["location"] for item in items] == ["scripts/fetch.py:21"]+ assert re.findall(r"<code>([^<]+)</code>", fix_first) == ["scripts/fetch.py:21"]+++def test_fix_first_links_open_existing_cards() -> None:+ html = _render(_results())+ ids = set(_outline(html).ids)+ summary = html.split('class="fix-first"', 1)[1].split("</section>", 1)[0]+ anchors = re.findall(r'href="#([^"]+)" class="fix-link"', summary)++ assert len(anchors) == 5+ assert set(anchors) <= ids+ assert "onclick=\"return openFinding(this.getAttribute('href').slice(1))\"" in summary+++def test_report_ids_are_unique() -> None:+ ids = _outline(_render(_results())).ids++ assert len(ids) == len(set(ids))+++def test_executive_summary_states_one_verdict_reason_and_four_kpis() -> None:+ html = _render(_results())+ summary = html.split('<section class="exec-summary', 1)[1].split("</section>", 1)[0]++ assert '<span class="verdict-chip fail">Changes required</span>' in summary+ assert re.search(r"4 of 6 checks failed: [^.]*PII Scan[^.]*\.", summary)+ assert "3 critical or high findings in those checks block the verdict." in summary+ assert re.findall(r"<dt>([^<]+)</dt>", summary) == [+ "Checks passed",+ "Findings",+ "Blocking critical + high",+ "Quality",+ ]+ assert "Minimum 70 to pass" in summary+ assert summary.count("A finding is one problem a check reports at one location") == 1+ assert "<h3>Fix first</h3>" in summary+++def test_passing_report_lists_advisory_findings_as_worth_a_look() -> None:+ lint = ValidationResult(validator_name="SCRIPT_LINT")+ lint.add_finding(_finding("low", "Line too long", "scripts/fetch.py", 3))++ html = _render([lint])++ assert '<span class="verdict-chip pass">Passed</span>' in html+ assert "<h3>Worth a look</h3>" in html+ assert "1 advisory finding does not block." in html+++def test_header_shows_one_verdict_and_skill_name_is_the_only_h1() -> None:+ html = _render(_results())+ header = html.split('<header class="main-header">', 1)[1].split("</header>", 1)[0]++ assert re.findall(r'class="header-status-badge verdict-badge (\w+)" data-verdict="(\w+)"', header) == [+ ("fail", "fail")+ ]+ assert "Changes required" in header+ assert "Download JSON" in header+ assert re.findall(r"<h1[^>]*>([^<]+)</h1>", html) == ["demo"]+++def _gated(results: list[ValidationResult], tier: int, *, blocking: bool = True) -> list[ValidationResult]:+ for result in results:+ result.metadata["gating"] = {"tier": tier, "blocking": blocking}+ return results+++def _header_verdict(html: str) -> list[tuple[str, str]]:+ header = html.split('<header class="main-header">', 1)[1].split("</header>", 1)[0]+ return re.findall(+ r'data-verdict="(\w+)">.*?<span class="sr-only">Verdict: </span>([^<]+)</span>', header, re.DOTALL+ )+++def _tier_card_verdicts(html: str) -> list[tuple[str, str]]:+ return re.findall(r'tier-card-label">(Tier \d)</span>\s*<span class="tier-card-verdict">([^<]+)</span>', html)+++def test_header_says_skipped_when_no_check_ran() -> None:+ skipped = []+ for name in ("Secrets Detection", "PII Scan"):+ result = ValidationResult(validator_name=name)+ result.metadata.update(skipped=True, skip_reason="Disabled by --checks")+ skipped.append(result)++ html = _render(_gated(skipped, 1))+ summary = html.split('<section class="exec-summary', 1)[1].split("</section>", 1)[0]++ # The header and the summary give the same verdict: nothing ran, so nothing passed.+ assert _header_verdict(html) == [("skipped", "Skipped")]+ assert '<span class="verdict-chip warn">Skipped</span>' in summary+ assert 'data-verdict="pass"' not in html+++def test_failing_advisory_tier_is_labelled_advisory() -> None:+ def results(blocking: bool) -> list[ValidationResult]:+ license_check = ValidationResult(validator_name="License Compliance")+ license_check.add_success("license", "License found")+ similarity = ValidationResult(validator_name="Similarity Check")+ similarity.add_error("Similarity provider failed")+ return [*_gated([license_check], 1), *_gated([similarity], 2, blocking=blocking)]++ advisory = _render(results(blocking=False))+ blocking = _render(results(blocking=True))++ # A failing tier whose gate does not block never says "Changes required".+ assert _header_verdict(advisory) == [("pass", "Passed")]+ assert _tier_card_verdicts(advisory) == [("Tier 1", "Passed"), ("Tier 2", "Failed (advisory)")]+ assert _header_verdict(blocking) == [("fail", "Changes required")]+ assert _tier_card_verdicts(blocking) == [("Tier 1", "Passed"), ("Tier 2", "Changes required")]+++def test_failing_advisory_tier3_is_labelled_advisory() -> None:+ agent_eval = ValidationResult(validator_name="AGENT_EVAL")+ agent_eval.add_error("Harbor execution failed")++ html = HTMLReporter(include_timestamp=False).render_all(_gated([agent_eval], 3, blocking=False))++ assert _tier_card_verdicts(html) == [("Tier 3", "Failed (advisory)")]+ assert '<span class="sr-only">Tier 3 verdict: </span>Failed (advisory)</span>' in html+ assert "Changes required" not in html.split("<main", 1)[1].split('id="tab-tier3"', 1)[0]+++def test_heading_levels_do_not_skip() -> None:+ for results in (_results(), [ValidationResult(validator_name="SCHEMA")]):+ levels = _outline(_render(results)).headings+ assert levels[0] == 1+ assert all(later <= earlier + 1 for earlier, later in pairwise(levels)), levels+++def test_publication_eligibility_sits_next_to_the_verdict() -> None:+ html = _render(_results())+ hero = html.split('<section class="agent-hero"', 1)[1].split("</section>", 1)[0]++ assert 'data-publication-status="fail"' in hero+ assert '<span class="status-badge fail">Not eligible</span>' in hero+++def test_folder_scan_keeps_its_skill_count_hint() -> None:+ html = HTMLReporter(include_timestamp=False).render_all([ValidationResult(validator_name="Security Scan")])+ hero = html.split('<section class="agent-hero"', 1)[1].split("</section>", 1)[0]++ assert re.search(r'<p class="hero-skills">Skills: \d+ of \d+ passed</p>', hero)+++def test_check_names_are_plain_with_the_raw_id_beside_them() -> None:+ html = _render(_results())++ assert plain_check_name("SCRIPT_LINT") == "Script lint"+ assert plain_check_name("NEW_SCANNER") == "New scanner"+ assert plain_check_name("Secrets Detection") == "Secrets Detection"+ assert 'Quality score <code class="check-id">QUALITY</code>' in html+ assert 'Script lint <code class="check-id">SCRIPT_LINT</code>' in html+++def test_finding_types_are_most_severe_first() -> None:+ html = _render(_results())+ table = html.split('class="top-issues-table"', 1)[1].split("</table>", 1)[0]+ severities = re.findall(r'class="severity-tag severity-(\w+)"', table)+ rank = {"critical": 4, "high": 3, "medium": 2, "low": 1, "info": 0}++ assert severities == sorted(severities, key=rank.__getitem__, reverse=True)+ # Only lists that really are sorted by severity say so.+ assert html.count("most severe first") == 1+ assert "most severe first</h3>" in html+++def test_check_table_rows_are_keyboard_reachable_links() -> None:+ html = _render(_results())+ table = html.split('class="summary-table"', 1)[1].split("</table>", 1)[0]+ links = re.findall(r'<a class="validator-link" href="#validator-(\d+)">', table)++ assert len(links) == len(_results())+ assert all(f'id="validator-{index}"' in html for index in links)+++def test_theme_toggle_has_a_fixed_name_and_reports_the_dark_state() -> None:+ html = _render(_results())+ button = html.split('class="theme-toggle"', 1)[1].split("</button>", 1)[0]++ assert 'aria-pressed="false"' in button+ assert '<span id="theme-text">Dark theme</span>' in button+ assert "toggle.setAttribute('aria-pressed', theme === 'dark' ? 'true' : 'false')" in html+ assert "getElementById('theme-text').textContent" not in html+ assert "matchMedia('(prefers-color-scheme: dark)')" in html+++def test_tier3_charts_are_redrawn_with_print_colors() -> None:+ result = ValidationResult(validator_name="AGENT_EVAL")+ result.metadata["agent_eval"] = build_agent_eval_payload(+ "example",+ {+ "agent": {+ "execution_status": "succeeded",+ "expected_attempts": 1,+ "scored_attempts": 1,+ "with_skill": {"security": 1.0},+ "rewards": [{"entry_id": "case-1", "security": 1.0}],+ }+ },+ use_llm_judge=False,+ )+ html = HTMLReporter(include_timestamp=False).render_all([result])+ script = next(script for script in _outline(html).scripts if "rebuildTier3Charts" in script)++ # A canvas keeps the colors it was drawn with, so printing redraws every+ # chart with the light (print) colors and restores the theme afterwards.+ assert 'window.addEventListener("beforeprint", function(){ redrawForPrint(true); });' in script+ assert 'window.addEventListener("afterprint", function(){ redrawForPrint(false); });' in script+ assert 'window.matchMedia("print")' in script+ assert 'if (printing && theme === "dark") root.setAttribute("data-theme", "light");' in script+ assert script.count("new Chart(") == script.count("options: chartOptions({") == 4+++def _rules(css: str):+ """Yield (media preludes, selector, declarations) for every style rule."""+ stack: list[str] = []+ prelude = ""+ depth_media: list[bool] = []+ index = 0+ while index < len(css):+ char = css[index]+ if css.startswith("/*", index):+ index = css.index("*/", index) + 2+ continue+ if char == "{":+ head = prelude.strip()+ if head.startswith("@"):+ stack.append(head)+ depth_media.append(True)+ prelude = ""+ else:+ end = css.index("}", index)+ yield tuple(stack), head, css[index + 1 : end]+ prelude = ""+ index = end + 1+ continue+ elif char == "}":+ if depth_media and depth_media.pop():+ stack.pop()+ prelude = ""+ else:+ prelude += char+ index += 1+++def test_dark_colors_never_reach_print() -> None:+ html = _render(_results())+ css = "\n".join(re.findall(r"<style>(.*?)</style>", html, flags=re.DOTALL))+ dark_rules = [(media, selector, body) for media, selector, body in _rules(css) if 'data-theme="dark"' in selector]++ assert dark_rules+ for media, selector, body in dark_rules:+ literal_colors = re.search(r"#[0-9a-fA-F]{3,8}\b|rgba?\(", body)+ assert not literal_colors or any(query.startswith("@media screen") for query in media), selector+++def test_print_shows_the_header_and_one_results_view() -> None:+ css = (TEMPLATES / "report_ui.css.j2").read_text(encoding="utf-8")+ print_rules = [(selector, body) for media, selector, body in _rules(css) if "@media print" in media]+ shown = " ".join(selector for selector, body in print_rules if "display:block" in body)+ hidden = " ".join(selector for selector, body in print_rules if "display:none" in body)++ assert ".main-header" in shown+ assert ".header-brand .se-logo" in shown+ assert ".header-controls" in hidden and ".skip-link" in hidden+ assert "#results-by-validator>.validator-card" in hidden+++def test_logo_is_in_the_sidebar_and_the_header_with_unique_ids() -> None:+ html = _render(_results())++ assert html.count('class="se-logo"') == 2+ assert 'class="brand-mark"' not in html+ # Both copies sit next to the visible name, so they are decorative and carry no ids.+ assert len(re.findall(r'<svg class="se-logo"[^>]*aria-hidden="true"', html)) == 2+ assert ' id="se-logo' not in html+++def test_no_workspace_font_size_is_below_eleven_pixels() -> None:+ css = (TEMPLATES / "report_ui.css.j2").read_text(encoding="utf-8")+ sizes = [float(size) for size in re.findall(r"font-size:(\d+(?:\.\d+)?)px", css)]++ assert sizes+ assert min(sizes) >= 11+++def _px(value: str, unit: str) -> float:+ return float(value) * (16 if unit == "rem" else 1)+++def test_template_text_under_eleven_pixels_is_overridden_or_unused() -> None:+ """Every small size in report.html.j2 is unused or raised to 11 px by the workspace stylesheet."""+ template = (TEMPLATES / "report.html.j2").read_text(encoding="utf-8")+ workspace = (TEMPLATES / "report_ui.css.j2").read_text(encoding="utf-8")+ styles = "\n".join(re.findall(r"<style[^>]*>(.*?)</style>", template, flags=re.DOTALL))+ markup = re.sub(r"<style[^>]*>.*?</style>", "", template, flags=re.DOTALL)+ size = r"font-size:\s*(\d*\.?\d+)(px|rem)\b"+ raised = {+ re.findall(r"\.([\w-]+)", part)[-1]+ for _media, selector, body in _rules(workspace)+ if any(_px(value, unit) >= 11 for value, unit in re.findall(size, body))+ for part in selector.split(",")+ if re.search(r"\.[\w-]+", part)+ }+ too_small = set()+ for _media, selector, body in _rules(styles):+ if not any(_px(value, unit) < 11 for value, unit in re.findall(size, body)):+ continue+ for part in selector.split(","):+ name = re.findall(r"\.([\w-]+)", part)[-1]+ if name not in raised and re.search(rf'class="[^"]*(?<![\w-]){re.escape(name)}(?![\w-])', markup):+ too_small.add(name)+ inline = [_px(value, unit) for value, unit in re.findall(r'style="[^"]*?' + size, markup)]++ assert not too_small+ assert inline+ assert min(inline) >= 11+++def test_short_table_cells_do_not_break_mid_word() -> None:+ css = (TEMPLATES / "report_ui.css.j2").read_text(encoding="utf-8")++ assert "td,th{overflow-wrap:anywhere}" not in css+ assert "td,th{overflow-wrap:break-word}th{white-space:nowrap}" in cssdiff --git a/tests/reporting/test_reporters.py b/tests/reporting/test_reporters.pyindex 356be561..1d6c11d9 100644--- a/tests/reporting/test_reporters.py+++ b/tests/reporting/test_reporters.py@@ -1124,7 +1124,7 @@ class TestGatingSplit:report_data = _html_report_data(output)assert reports_passed is Falseassert 'data-target-tab="tier2"' in output- assert "<strong>0</strong>/1 checks passed" in output+ assert "<strong>0</strong> of 1 checks passed" in outputassert report_data["gating"] == {"blocking_tiers": ["tier1", "tier2"],"advisory_tiers": ["tier3"],@@ -1217,7 +1217,7 @@ class TestGatingSplit:assert report_data["quality_scores"]["demo-skill"]["overall_score"] == 72.5assert 'data-quality-skill-name="demo-skill"' in output- assert "Quality Score: 72.5/100" in output+ assert "Quality score: 72.5 / 100" in outputassert "Type: script-based" in outputassert ">clarity</td>" in outputassert '<td class="warning-text">1</td>' in outputdiff --git a/tests/reporting/test_retained_html.py b/tests/reporting/test_retained_html.pyindex e84d7be9..3f3ae9a7 100644--- a/tests/reporting/test_retained_html.py+++ b/tests/reporting/test_retained_html.py@@ -148,6 +148,72 @@ def test_fixed_relative_ci_links_only():HTMLReporter(ci_context={key: value})+def test_validation_without_copies_matches_the_copying_rehydration(payload):+ payload["results"][0]["findings"][0]["metadata"] = {"rule": {"nested": ["evidence"]}}+ copied = retained_results(payload)+ shared = retained_results(payload, copy_metadata=False)+ assert shared == copied+ finding_metadata = payload["results"][0]["findings"][0]["metadata"]+ assert copied[0].findings[0].metadata is not finding_metadata+ assert shared[0].findings[0].metadata is finding_metadata+ payload["results"][0]["passed"] = "true"+ with pytest.raises(RetainedReportError, match="passed must be boolean"):+ retained_results(payload, copy_metadata=False)+++def test_reporters_share_one_environment_and_compile_each_template_once(payload, monkeypatch):+ from skillevaluator.reporting import html as html_module++ loads = []+ original = html_module.PackageLoader.get_source++ def counting(self, environment, template):+ loads.append(template)+ return original(self, environment, template)++ monkeypatch.setattr(html_module.PackageLoader, "get_source", counting)+ HTMLReporter._create_environment.cache_clear()+ results = retained_results(payload)+ plain = HTMLReporter(include_timestamp=False)+ ci = HTMLReporter(include_timestamp=False, title="Other", ci_context={"report_index_url": "../index.html"})+ assert plain._env is ci._env+ first = plain.render_all(results)+ ci.render_all(results)+ assert plain.render_all(results) == first+ # Pages built outside report.html.j2, such as the CI index, reuse the same compiled logo.+ assert html_module.render_logo("one") != html_module.render_logo("two")+ assert sorted(loads) == ["logo.svg.j2", "report.html.j2", "report_ui.css.j2", "report_ui.js.j2"]+++def test_shared_environment_renders_identically_from_many_threads(payload):+ from concurrent.futures import ThreadPoolExecutor++ HTMLReporter._create_environment.cache_clear()++ def render(index):+ context = {"report_index_url": "../index.html"} if index % 2 else None+ return render_retained_report(payload, skill_name="skill", context=context)++ expected = [render(0), render(1)]+ HTMLReporter._create_environment.cache_clear()+ with ThreadPoolExecutor(max_workers=8) as pool:+ documents = list(pool.map(render, range(32)))+ assert all(document == expected[index % 2] for index, document in enumerate(documents))+++def test_only_standalone_reports_embed_the_export_json(payload):+ standalone = render_retained_report(payload, skill_name="skill")+ embedded = standalone.split('<script id="report-data" type="application/json">', 1)[1].split("</script>", 1)[0]+ assert json.loads(embedded)["results"][0]["validator_name"] == payload["results"][0]["validator"]+ assert 'onclick="exportJSON()"' in standalone+ hosted = render_retained_report(+ payload, skill_name="skill", context={"canonical_json_url": "skillevaluator-tier1.json"}+ )+ assert 'id="report-data"' not in hosted+ assert 'href="skillevaluator-tier1.json" download' in hosted+ assert 'onclick="exportJSON()"' not in hosted++def test_not_applicable_and_incomplete_presentation_preserves_gate():lint = ValidationResult(validator_name="SCRIPT_LINT", validator_description="Scripts")lint.success_details.append(SuccessDetail(check_name="lint", message="No scripts/ or tools/ directory found"))