Skip to content

Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.

11.8 KB
Raw Blame
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
# ruff: noqa: INP001, T201 -- standalone trusted workflow entrypoint with fixed diagnostics.

"""Upsert verified current-head PR feedback from trusted scan evidence.

Only the configured bot's marked comments are eligible for update. This helper
does not check out PR code, execute artifacts, or print the comment body.
"""

from __future__ import annotations

import json
import os
import re
import sys
from pathlib import Path

try:
    import github_api
except ModuleNotFoundError:
    from ci import github_api

MARKER = "<!-- skillevaluator-tier1-report -->"
RUN_MARKER = re.compile(r"<!-- skillevaluator-tier1-run:([1-9][0-9]*):([1-9][0-9]*) -->")
REPOSITORY_PATTERN = github_api.REPOSITORY
SHA_PATTERN = github_api.COMMIT_SHA
BOT_PATTERN = re.compile(r"[A-Za-z0-9_-]+\[bot\]")
MAX_BODY_BYTES = 60000
MAX_METADATA_BYTES = 64 * 1024
MAX_PAGES = 10


class FeedbackError(ValueError):
    """A fixed diagnostic safe to print without feedback content or credentials."""


class GitHubAPI(github_api.GitHubAPI):
    error = FeedbackError
    json_error = "Feedback metadata or the GitHub API response is invalid JSON."
    user_agent = "SkillEvaluator-PR-feedback"


def _json(raw: bytes):
    return github_api.strict_json(raw, FeedbackError, GitHubAPI.json_error)


def _read(path: Path, limit: int) -> bytes:
    if path.is_symlink() or not path.is_file() or path.stat().st_size > limit:
        raise FeedbackError("A required feedback file is missing, linked, or oversized.")
    with path.open("rb") as handle:
        raw = handle.read(limit + 1)
    if len(raw) > limit:
        raise FeedbackError("A required feedback file exceeds its size limit.")
    return raw


def _positive(value) -> int:
    if not isinstance(value, str) or re.fullmatch(r"[1-9][0-9]{0,19}", value) is None:
        raise FeedbackError("A run, attempt, or pull request identity is invalid.")
    return int(value)


def run_marker(run_id: int, attempt: int) -> str:
    return f"<!-- skillevaluator-tier1-run:{run_id}:{attempt} -->"


def _verify_run(api, repository: str, run_id: int, attempt: int, sha: str | None = None) -> str | None:
    """Reject superseded attempts, including reruns started while rendering feedback."""
    run = api.get(f"/repos/{repository}/actions/runs/{run_id}")
    if (
        not isinstance(run, dict)
        or type(run.get("id")) is not int
        or run["id"] != run_id
        or not isinstance(run.get("repository"), dict)
        or run["repository"].get("full_name") != repository
        or not isinstance(run.get("head_sha"), str)
        or SHA_PATTERN.fullmatch(run["head_sha"]) is None
        or (sha is not None and run["head_sha"] != sha)
        or type(run.get("run_attempt")) is not int
        or run["run_attempt"] < attempt
        or run.get("status") not in {"completed", "queued", "in_progress", "waiting", "pending", "requested"}
    ):
        raise FeedbackError("The source run lacks trustworthy repository, revision, or attempt metadata.")
    if run["run_attempt"] != attempt:
        return "A newer scan attempt exists."
    if run["status"] != "completed":
        return "The scan attempt is not completed."
    return None


def _verify_pr(api, repository: str, number: int, sha: str) -> str | None:
    pr = api.get(f"/repos/{repository}/pulls/{number}")
    if not isinstance(pr, dict) or pr.get("state") not in {"open", "closed"}:
        raise FeedbackError("The pull request lacks a valid state.")
    if pr["state"] == "closed":
        return "The pull request is closed."
    head = pr.get("head")
    base = pr.get("base")
    if not isinstance(head, dict) or not isinstance(base, dict):
        raise FeedbackError("The pull request lacks trustworthy repository and revision metadata.")
    if not isinstance(head.get("sha"), str) or SHA_PATTERN.fullmatch(head["sha"]) is None:
        raise FeedbackError("The pull request lacks a valid head revision.")
    if (head.get("repo") or {}).get("full_name") != repository or (base.get("repo") or {}).get(
        "full_name"
    ) != repository:
        raise FeedbackError("The pull request belongs to another repository; feedback was not posted.")
    if head.get("sha") != sha:
        return "The pull request head changed."
    return None


def _owned(comment: dict, bot: str) -> bool:
    user = comment.get("user")
    return (
        isinstance(user, dict)
        and user.get("login") == bot
        and user.get("type") == "Bot"
        and isinstance(comment.get("body"), str)
        and MARKER in comment["body"]
    )


def _owned_comments(api, repository: str, number: int, bot: str) -> list[dict]:
    owned = []
    for page in range(1, MAX_PAGES + 1):
        comments = api.get(f"/repos/{repository}/issues/{number}/comments?per_page=100&page={page}")
        if (
            not isinstance(comments, list)
            or len(comments) > 100
            or any(not isinstance(item, dict) for item in comments)
        ):
            raise FeedbackError("The pull request comment listing is invalid.")
        for item in comments:
            if _owned(item, bot):
                if type(item.get("id")) is not int or item["id"] <= 0:
                    raise FeedbackError("An existing bot comment has an invalid identity.")
                owned.append(item)
        if len(comments) < 100:
            return owned
    raise FeedbackError("The pull request comment listing exceeded the pagination limit.")


def _comment_identity(comment: dict) -> tuple[int, int]:
    identities = RUN_MARKER.findall(comment["body"])
    if len(identities) != 1:
        raise FeedbackError("An owned report comment lacks a unique run receipt; it was not overwritten.")
    return tuple(int(part) for part in identities[0])


def _verified_comment(api, repository: str, number: int, identifier: int, bot: str, body: str) -> str:
    comment = api.get(f"/repos/{repository}/issues/comments/{identifier}")
    if (
        not isinstance(comment, dict)
        or comment.get("id") != identifier
        or not _owned(comment, bot)
        or comment.get("body") != body
    ):
        raise FeedbackError("GitHub did not return the exact expected bot comment on readback.")
    expected = f"https://github.com/{repository}/pull/{number}#issuecomment-{identifier}"
    if comment.get("html_url") != expected:
        raise FeedbackError("The comment readback URL does not match the expected pull request.")
    return expected


def comment_report(api, repository: str, run_id: int, feedback_dir: Path, actor_bot_login: str) -> dict[str, str]:
    """Upsert only a current, non-stale, verified owned PR comment."""
    if not REPOSITORY_PATTERN.fullmatch(repository) or type(run_id) is not int or run_id <= 0:
        raise FeedbackError("An explicit repository and positive report run ID are required.")
    if not BOT_PATTERN.fullmatch(actor_bot_login):
        raise FeedbackError("ACTOR_BOT_LOGIN must name the bot permitted to own report comments.")
    if feedback_dir.is_symlink() or not feedback_dir.is_dir():
        raise FeedbackError("FEEDBACK_DIR must be a regular feedback directory.")
    metadata = _json(_read(feedback_dir / "publication.json", MAX_METADATA_BYTES))
    if not isinstance(metadata, dict):
        raise FeedbackError("The publication metadata is invalid.")
    expected_run_url = f"https://github.com/{repository}/actions/runs/{run_id}"
    if metadata.get("run_url") != expected_run_url or metadata.get("is_current") != "true":
        raise FeedbackError("The feedback is not a current report from the expected repository and run.")
    number = _positive(metadata.get("pr_number"))
    attempt = _positive(metadata.get("run_attempt"))
    sha = metadata.get("source_head_sha")
    if not isinstance(sha, str) or SHA_PATTERN.fullmatch(sha) is None:
        raise FeedbackError("The feedback source revision is invalid.")
    try:
        message = _read(feedback_dir / "pr-comment.md", MAX_BODY_BYTES).decode("utf-8")
    except UnicodeError as exc:
        raise FeedbackError("The feedback body is not valid UTF-8.") from exc
    if not message.strip() or MARKER in message or RUN_MARKER.search(message):
        raise FeedbackError("The rendered feedback is empty or contains reserved report markers.")
    # Defense in depth even when the trusted renderer already neutralized mentions.
    message = message.replace("@", "@\u200b")
    body = (
        f"{MARKER}\n{run_marker(run_id, attempt)}\n"
        f"Source commit: `{sha}` · [Evaluation run]({expected_run_url})\n\n{message}"
    )
    if len(body.encode("utf-8")) > MAX_BODY_BYTES:
        raise FeedbackError("The complete feedback comment exceeds the 60000-byte limit.")
    reason = _verify_run(api, repository, run_id, attempt, sha) or _verify_pr(api, repository, number, sha)
    if reason:
        return {"comment_action": "skipped", "comment_reason": reason}
    owned = _owned_comments(api, repository, number, actor_bot_login)
    if any(_comment_identity(item) > (run_id, attempt) for item in owned):
        return {"comment_action": "skipped", "comment_reason": "A newer report comment already exists."}
    canonical = max(owned, key=lambda item: item["id"]) if owned else None
    # Recheck immediately before mutation after pagination and receipt validation.
    reason = _verify_run(api, repository, run_id, attempt, sha) or _verify_pr(api, repository, number, sha)
    if reason:
        return {"comment_action": "skipped", "comment_reason": reason}
    if canonical is not None and canonical["body"] == body:
        identifier = canonical["id"]
        action = "unchanged"
    elif canonical is not None:
        identifier = canonical["id"]
        response = api.patch(f"/repos/{repository}/issues/comments/{identifier}", {"body": body})
        if not isinstance(response, dict) or response.get("id") != identifier:
            raise FeedbackError("GitHub did not acknowledge the expected comment update.")
        action = "updated"
    else:
        response = api.post(f"/repos/{repository}/issues/{number}/comments", {"body": body})
        if not isinstance(response, dict) or type(response.get("id")) is not int or response["id"] <= 0:
            raise FeedbackError("GitHub did not acknowledge a valid new comment identity.")
        identifier = response["id"]
        action = "created"
    url = _verified_comment(api, repository, number, identifier, actor_bot_login, body)
    return {"comment_url": url, "comment_id": str(identifier), "comment_action": action}


def main() -> int:
    try:
        repository = os.environ.get("GITHUB_REPOSITORY", "")
        run_id = _positive(os.environ.get("REPORT_RUN_ID"))
        token = os.environ.get("GH_TOKEN", "").strip()
        if not token:
            raise FeedbackError("GH_TOKEN is required to publish the authorized PR feedback.")
        result = comment_report(
            GitHubAPI(repository, token),
            repository,
            run_id,
            Path(os.environ["FEEDBACK_DIR"]),
            os.environ.get("ACTOR_BOT_LOGIN", "github-actions[bot]"),
        )
        if os.environ.get("GITHUB_OUTPUT"):
            with Path(os.environ["GITHUB_OUTPUT"]).open("a", encoding="utf-8") as handle:
                handle.write("".join(f"{key}={value}\n" for key, value in result.items()))
        if result["comment_action"] == "skipped":
            print("SkillEvaluator PR feedback skipped: " + result["comment_reason"])
        else:
            print("SkillEvaluator PR feedback verified successfully.")
        return 0
    except FeedbackError as exc:
        print("SkillEvaluator PR feedback rejected: " + json.dumps(str(exc)[:1000]), file=sys.stderr)
        return 1
    except (OSError, KeyError, ValueError, TypeError, RecursionError):
        print("SkillEvaluator PR feedback failed; inspect the trusted workflow configuration.", file=sys.stderr)
        return 1


if __name__ == "__main__":
    raise SystemExit(main())