Skip to content

Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.

3.7 KB
Raw Blame
# SPDX-License-Identifier: Apache-2.0
"""Untrusted requirement files cannot invoke pip resolution or build backends."""

import json
from pathlib import Path

import pytest

from skillevaluator.utils.tool_runner import ToolResult, Tools
from skillevaluator.validators.dependencies import DependencySecurityValidator


def _audit(tmp_path, monkeypatch, text, *, returned=None):
    target = tmp_path / "requirements.txt"
    target.write_text(text)
    calls = []
    monkeypatch.setattr(type(Tools.pip_audit), "is_available", property(lambda _self: True))

    def run(args, **kwargs):
        assert "--no-deps" in args and "--disable-pip" in args
        path = Path(args[args.index("-r") + 1])
        assert path != target
        pins = path.read_text().splitlines()
        calls.extend(pins)
        dependencies = (
            returned
            if returned is not None
            else [{"name": pin.split("==")[0], "version": pin.split("==")[1], "vulns": []} for pin in pins]
        )
        return ToolResult(True, json.dumps({"dependencies": dependencies}), "", 0)

    monkeypatch.setattr(Tools.pip_audit, "run", run)
    result = DependencySecurityValidator(use_safety=False)._run_pip_audit_on_file(target)
    return result, calls


@pytest.mark.parametrize(
    "text",
    [
        "payload @ file:///workspace/payload",
        "./payload",
        "-e ./payload",
        "payload @ git+https://example.invalid/payload.git",
        "-r https://example.invalid/private.txt",
        "--index-url https://user:[email protected]",
        "requests>=2",
        "requests==2.*",
        "--find-links ./packages",
        "-c constraints.txt",
        "requests @ ${SECRET_URL}",
    ],
)
def test_unsupported_requirements_are_incomplete_without_invoking_scanner(tmp_path, monkeypatch, text):
    result, calls = _audit(tmp_path, monkeypatch, text)
    assert result.is_incomplete and not result.passed
    assert calls == []
    assert "secret" not in " ".join(result.warnings)


def test_supported_pins_are_audited_even_with_unsupported_declarations(tmp_path, monkeypatch):
    result, calls = _audit(tmp_path, monkeypatch, "idna==3.10\npayload @ file:///workspace/payload\n")
    assert calls == ["idna==3.10"]
    assert result.is_incomplete


def test_hashed_continuations_comments_and_markers_are_normalized(tmp_path, monkeypatch):
    text = (
        "idna==3.10 \\\n    --hash=sha256:"
        + "a" * 64
        + " \\\n    --hash=sha256:"
        + "b" * 64
        + "\n# comment\nrequests==2.32.4; python_version < '2'\n"
    )
    result, calls = _audit(tmp_path, monkeypatch, text)
    assert calls == ["idna==3.10"]
    assert result.passed and not result.is_incomplete


def test_missing_pin_evidence_is_incomplete(tmp_path, monkeypatch):
    result, calls = _audit(tmp_path, monkeypatch, "idna==3.10", returned=[])
    assert calls == ["idna==3.10"]
    assert result.is_incomplete
    assert not result.success_details


def test_empty_file_is_completed_without_installing_anything(tmp_path, monkeypatch):
    result, calls = _audit(tmp_path, monkeypatch, "# no Python dependencies\n")
    assert result.passed and calls == []


def test_symlink_is_not_followed(tmp_path, monkeypatch):
    target = tmp_path / "requirements.txt"
    target.symlink_to(tmp_path / "private")
    (tmp_path / "private").write_text("idna==3.10")
    result = DependencySecurityValidator(use_safety=False)._run_pip_audit_on_file(target)
    assert result.is_incomplete


@pytest.mark.parametrize("marker", ['"foo" in extras', '"foo" in dependency_groups'])
def test_unsupported_marker_context_is_incomplete_instead_of_crashing(tmp_path, monkeypatch, marker):
    result, calls = _audit(tmp_path, monkeypatch, f"demo==1; {marker}")
    assert result.is_incomplete and calls == []