Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.
Python agent-evaluationagent-pluginsagent-securityagent-skillsagentic-aibenchmarkci-templatesclaude-codecodexevaluateevaluationharnesspluginplugin-evaluationsecurity-scannerskill-evalskill-evaluationskill-evaluatorskill-gap-analysisskills
fix(api-caller): share one URL scheme check between scripts (#767)
Christopher Kevinab7f355afec8
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex cc5a9e59..29e1f0da 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -15,6 +15,14 @@ All notable changes to SkillEvaluator are documented in this file.existing skill its first two cases ask the agent to convert, under`evals/files/`, and each case declares `files` so only its own fixture isstaged. Previously those cases had no workspace input to work from.+- The bundled `api-caller` example no longer fails Tier 2 deduplication: its+ two scripts share one HTTP(S) URL check from `scripts/url_safety.py` instead+ of each defining `_validate_http_url()`, and `call_api.py` no longer repeats the+ SKILL.md usage section in its module docstring.+- Tier 3 dataset generation no longer names an import-only helper module, such+ as `url_safety.py`, as a case's `expected_script` when the skill has a script+ that does something when run. A skill with one script, or only helpers, keeps+ its previous choice.## 1.0.0 - 2026-11-17diff --git a/pyproject.toml b/pyproject.tomlindex 46eba6ef..a0f510e1 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -206,8 +206,9 @@ ignore = [# human-readable progress and results to stdout by design."src/skillevaluator/tier3/generate_dataset.py" = ["T201"]# Reference skill scripts are standalone executables run by the agent under-# Harbor (not imported); printing to stdout is their output contract.-"src/skillevaluator/tier3/reference_skills/**" = ["T201"]+# Harbor; printing to stdout is their output contract. Shared helpers beside+# them are imported from the script directory, not as a package (INP001).+"src/skillevaluator/tier3/reference_skills/**" = ["T201", "INP001"]# pytest test directories are implicit namespace packages by design (no# __init__.py anywhere under tests/)."tests/**" = ["INP001"]diff --git a/src/skillevaluator/tier3/generate_dataset.py b/src/skillevaluator/tier3/generate_dataset.pyindex 10ec8ba7..6fb30699 100644--- a/src/skillevaluator/tier3/generate_dataset.py+++ b/src/skillevaluator/tier3/generate_dataset.py@@ -45,6 +45,7 @@ Agent-refined mode (--refine):"""import argparse+import astimport asyncioimport hashlibimport json@@ -168,14 +169,14 @@ def _parse_skill(skill_path: Path, prompt_file: str | None = None) -> dict[str,description = str(frontmatter["description"]).strip()# Find scripts in scripts/ (historical) and tools/ (agentskills.io)- seen_scripts: set[str] = set()+ script_paths: dict[str, Path] = {}for dirname in EXECUTABLE_SKILL_DIRS:directory = skill_path / dirnameif not directory.is_dir():continuefor script_file in directory.glob("*.py"):- if script_file.name not in seen_scripts:- seen_scripts.add(script_file.name)+ if script_file.name not in script_paths:+ script_paths[script_file.name] = script_filescripts.append(script_file.name)# Detect interactive scripts from SKILL.md content@@ -217,24 +218,112 @@ def _parse_skill(skill_path: Path, prompt_file: str | None = None) -> dict[str,# Every script file `tier3 validate` lists, not only the Python ones in "scripts"."script_files": evals_spec._skill_script_names(skill_path),"interactive_scripts": interactive_scripts,+ # Scripts that only define code for other scripts to import; the agent never runs them.+ "import_only_scripts": {name for name, path in script_paths.items() if _is_import_only_script(path)},"content": content[:4000],"eval_prompt": eval_prompt,"eval_prompt_source": eval_prompt_source,}+# A script file larger than this is not read to decide whether it is import-only.+_MAX_SCRIPT_BYTES = 1_000_000++_ARGUMENT_PARSING_METHODS = frozenset({"parse_args", "parse_known_args"})+++def _is_main_guard(test: ast.expr) -> bool:+ """Whether an if-test is ``__name__ == "__main__"`` (either operand order)."""+ if not (isinstance(test, ast.Compare) and len(test.ops) == 1 and isinstance(test.ops[0], ast.Eq)):+ return False+ operands = [test.left, *test.comparators]+ return any(isinstance(node, ast.Name) and node.id == "__name__" for node in operands) and any(+ isinstance(node, ast.Constant) and node.value == "__main__" for node in operands+ )+++def _runs_script_code(statement: ast.stmt, local_functions: frozenset[str]) -> bool:+ """Whether a top-level assignment reads the command line or calls a function this file defines."""+ for node in ast.walk(statement):+ if isinstance(node, ast.Call):+ if isinstance(node.func, ast.Name) and node.func.id in local_functions:+ return True+ if isinstance(node.func, ast.Attribute) and node.func.attr in _ARGUMENT_PARSING_METHODS:+ return True+ if isinstance(node, ast.Attribute) and node.attr == "argv":+ return True+ return False+++def _only_definitions(body: Sequence[ast.stmt], local_functions: frozenset[str]) -> bool:+ for statement in body:+ if isinstance(+ statement,+ ast.Import | ast.ImportFrom | ast.FunctionDef | ast.AsyncFunctionDef | ast.ClassDef | ast.Pass,+ ):+ continue+ if isinstance(statement, ast.Expr) and isinstance(statement.value, ast.Constant):+ # A docstring or other bare constant.+ continue+ if isinstance(statement, ast.Assign | ast.AnnAssign | ast.AugAssign):+ if _runs_script_code(statement, local_functions):+ return False+ continue+ if isinstance(statement, ast.If):+ if _is_main_guard(statement.test) or not _only_definitions(+ [*statement.body, *statement.orelse], local_functions+ ):+ return False+ continue+ if isinstance(statement, ast.Try):+ blocks = [*statement.body, *statement.orelse, *statement.finalbody]+ for handler in statement.handlers:+ blocks.extend(handler.body)+ if not _only_definitions(blocks, local_functions):+ return False+ continue+ # A call, loop, with-block or anything else runs when the file is run.+ return False+ return True+++def _is_import_only_script(path: Path) -> bool:+ """Whether a Python file only defines things (imports, functions, classes, constants) and does nothing when run.++ Such a file is a helper that the skill's real scripts import, like a shared+ URL check. An agent following SKILL.md has no reason to run it, so it must+ not become a case's expected_script. A file that cannot be read or parsed is+ not called import-only: Tier 3 cannot tell what it does, so it stays a candidate.+ """+ try:+ if not path.is_file() or path.stat().st_size > _MAX_SCRIPT_BYTES:+ return False+ tree = ast.parse(path.read_text(encoding="utf-8").removeprefix("\ufeff"), filename=path.name)+ except (OSError, UnicodeDecodeError, SyntaxError, ValueError, RecursionError):+ return False+ local_functions = frozenset(+ node.name for node in tree.body if isinstance(node, ast.FunctionDef | ast.AsyncFunctionDef)+ )+ return _only_definitions(tree.body, local_functions)++def _pick_primary_script(skill: dict[str, Any]) -> str | None:- """Pick the best script for eval: prefer non-interactive scripts.+ """Pick the best script for eval: a script the agent runs, preferably a non-interactive one.Uses SKILL.md context to detect interactive scripts (e.g., "browser_login.pyis interactive because it opens a browser") rather than keyword-matching filenames.+ An import-only helper module (see `_is_import_only_script`) is passed over while+ the skill has a script that does something when run; if every script is a helper+ the first one is still returned."""scripts = skill.get("scripts", [])if not scripts:return None+ import_only = skill.get("import_only_scripts", set())+ runnable = [s for s in scripts if s not in import_only] or scriptsinteractive = skill.get("interactive_scripts", set())- non_interactive = [s for s in scripts if s not in interactive]- return non_interactive[0] if non_interactive else scripts[0]+ non_interactive = [s for s in runnable if s not in interactive]+ return non_interactive[0] if non_interactive else runnable[0]SECURITY_BEHAVIOR = (diff --git a/src/skillevaluator/tier3/reference_skills/api-caller/scripts/call_api.py b/src/skillevaluator/tier3/reference_skills/api-caller/scripts/call_api.pyindex 5b8a522e..1a435ed7 100644--- a/src/skillevaluator/tier3/reference_skills/api-caller/scripts/call_api.py+++ b/src/skillevaluator/tier3/reference_skills/api-caller/scripts/call_api.py@@ -2,41 +2,18 @@# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.# SPDX-License-Identifier: Apache-2.0-"""-Generic API Caller--Call any REST API endpoint directly or using an OpenAPI spec.--Usage:- # Direct call- python call_api.py --url "https://api.example.com/data" --method GET-- # With headers- python call_api.py --url "https://api.example.com/data" --headers '{"Authorization": "Bearer TOKEN"}'-- # POST with body- python call_api.py --url "https://api.example.com/items" --method POST --data '{"name": "test"}'-- # Using OpenAPI spec- python call_api.py --spec "https://api.weather.gov/openapi.json" --operation "point" --params '{"latitude": 37.7749}'-"""+"""Call a REST endpoint directly or through an OpenAPI operation. See SKILL.md for usage."""import argparseimport jsonimport sysfrom urllib.error import HTTPError, URLError-from urllib.parse import quote, urlencode, urlparse+from urllib.parse import quote, urlencodefrom urllib.request import Request, urlopen-_UNSET = object()-+from url_safety import validate_http_url-def _validate_http_url(url: str) -> str:- """Reject local-file and custom URL schemes before making a request."""- parsed = urlparse(url)- if parsed.scheme not in {"http", "https"} or not parsed.netloc:- raise ValueError("Only absolute HTTP or HTTPS URLs are supported")- return url+_UNSET = object()def make_request(@@ -55,7 +32,7 @@ def make_request(}try:- url = _validate_http_url(url)+ url = validate_http_url(url)# Prepare headersreq_headers = {"User-Agent": "skills-agent/1.0 (api-caller skill)",@@ -74,7 +51,7 @@ def make_request(req = Request(url, data=body, headers=req_headers, method=method)# Make request- # _validate_http_url() rejects non-HTTP schemes before this request.+ # validate_http_url() rejects non-HTTP schemes before this request.with urlopen(req, timeout=timeout) as response: # nosec B310result["status_code"] = response.statusresult["headers"] = dict(response.headers)diff --git a/src/skillevaluator/tier3/reference_skills/api-caller/scripts/parse_openapi.py b/src/skillevaluator/tier3/reference_skills/api-caller/scripts/parse_openapi.pyindex 4fe4713a..b56ca711 100644--- a/src/skillevaluator/tier3/reference_skills/api-caller/scripts/parse_openapi.py+++ b/src/skillevaluator/tier3/reference_skills/api-caller/scripts/parse_openapi.py@@ -11,22 +11,15 @@ import jsonimport sysimport urllib.errorimport urllib.request-from urllib.parse import urlparse--def _validate_http_url(url: str) -> str:- """Reject local-file and custom URL schemes before fetching a spec."""- parsed = urlparse(url)- if parsed.scheme not in {"http", "https"} or not parsed.netloc:- raise ValueError("Only absolute HTTP or HTTPS URLs are supported")- return url+from url_safety import validate_http_urldef fetch_spec(url: str) -> dict:"""Fetch OpenAPI spec from URL."""try:- req = urllib.request.Request(_validate_http_url(url), headers={"User-Agent": "api-caller/1.0"})- # _validate_http_url() rejects non-HTTP schemes before this request.+ req = urllib.request.Request(validate_http_url(url), headers={"User-Agent": "api-caller/1.0"})+ # validate_http_url() rejects non-HTTP schemes before this request.with urllib.request.urlopen(req, timeout=30) as response: # nosec B310content = response.read().decode("utf-8")return json.loads(content)diff --git a/src/skillevaluator/tier3/reference_skills/api-caller/scripts/url_safety.py b/src/skillevaluator/tier3/reference_skills/api-caller/scripts/url_safety.pynew file mode 100644index 00000000..6597b2d8--- /dev/null+++ b/src/skillevaluator/tier3/reference_skills/api-caller/scripts/url_safety.py@@ -0,0 +1,14 @@+# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+# SPDX-License-Identifier: Apache-2.0++"""URL checks shared by the api-caller scripts."""++from urllib.parse import urlparse+++def validate_http_url(url: str) -> str:+ """Reject local-file and custom URL schemes before making a request."""+ parsed = urlparse(url)+ if parsed.scheme not in {"http", "https"} or not parsed.netloc:+ raise ValueError("Only absolute HTTP or HTTPS URLs are supported")+ return urldiff --git a/tests/test_reference_api_caller.py b/tests/test_reference_api_caller.pyindex 28d7a60e..1fdef46b 100644--- a/tests/test_reference_api_caller.py+++ b/tests/test_reference_api_caller.py@@ -27,7 +27,11 @@ def _load_script(module_name: str):spec = importlib.util.spec_from_file_location(module_name, SCRIPTS_DIR / f"{module_name}.py")assert spec and spec.loadermodule = importlib.util.module_from_spec(spec)- spec.loader.exec_module(module)+ sys.path.insert(0, str(SCRIPTS_DIR))+ try:+ spec.loader.exec_module(module)+ finally:+ sys.path.remove(str(SCRIPTS_DIR))return module@@ -113,3 +117,11 @@ def test_openapi_parser_rejects_non_http_urls_without_opening_them() -> None:result = parser.fetch_spec("file:///etc/passwd")assert result == {"error": "Failed to fetch spec: Only absolute HTTP or HTTPS URLs are supported"}++[email protected]("script", ["call_api", "parse_openapi"])+def test_scripts_share_one_url_scheme_check(script: str) -> None:+ module = _load_script(script)++ assert module.validate_http_url.__module__ == "url_safety"+ assert "def validate_http_url" not in (SCRIPTS_DIR / f"{script}.py").read_text(encoding="utf-8")diff --git a/tests/test_reference_skill_scripts.py b/tests/test_reference_skill_scripts.pyindex 1634adb9..a0392c3a 100644--- a/tests/test_reference_skill_scripts.py+++ b/tests/test_reference_skill_scripts.py@@ -57,7 +57,12 @@ def _load_script(name: str) -> ModuleType:spec = importlib.util.spec_from_file_location(f"reference_script_{name}", SCRIPTS[name])assert spec and spec.loadermodule = importlib.util.module_from_spec(spec)- spec.loader.exec_module(module)+ scripts_dir = str(SCRIPTS[name].parent)+ sys.path.insert(0, scripts_dir)+ try:+ spec.loader.exec_module(module)+ finally:+ sys.path.remove(scripts_dir)return modulediff --git a/tests/tier3/test_dataset_primary_script.py b/tests/tier3/test_dataset_primary_script.pynew file mode 100644index 00000000..a0df639b--- /dev/null+++ b/tests/tier3/test_dataset_primary_script.py@@ -0,0 +1,183 @@+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+# SPDX-License-Identifier: Apache-2.0++"""The dataset generator never names an import-only helper module as the expected script."""++from __future__ import annotations++import itertools+from pathlib import Path+from typing import Any++import pytest++from skillevaluator.tier3 import generate_dataset++API_CALLER = Path(generate_dataset.__file__).parent / "reference_skills" / "api-caller"+ENTRY_POINTS = {"call_api.py", "parse_openapi.py"}++HELPER = '"""Shared checks."""\n\nfrom urllib.parse import urlparse\n\n\ndef check(url):\n return urlparse(url)\n'+GUARDED_ENTRY = "import sys\n\n\ndef main():\n return 0\n\n\nif __name__ == '__main__':\n sys.exit(main())\n"+++def _write_skill(tmp_path: Path, scripts: dict[str, str], skill_md_body: str = "# Skill\n") -> Path:+ skill = tmp_path / "demo-skill"+ skill.mkdir()+ (skill / "SKILL.md").write_text(+ f"---\nname: demo-skill\ndescription: Demo skill.\n---\n\n{skill_md_body}",+ encoding="utf-8",+ )+ for relative, source in scripts.items():+ path = skill / relative+ path.parent.mkdir(parents=True, exist_ok=True)+ path.write_text(source, encoding="utf-8")+ return skill+++def _parse_in_order(skill_path: Path, order: tuple[str, ...]) -> dict[str, Any]:+ """Parse a skill, then list its scripts in `order`, because the directory listing order is not defined."""+ parsed = generate_dataset._parse_skill(skill_path)+ assert sorted(parsed["scripts"]) == sorted(order)+ parsed["scripts"] = list(order)+ return parsed+++def test_api_caller_reference_skill_never_expects_the_url_helper() -> None:+ """Whatever order the file system lists scripts in, url_safety.py is not the expected script."""+ parsed = generate_dataset._parse_skill(API_CALLER)++ assert "url_safety.py" in parsed["scripts"]+ assert generate_dataset._pick_primary_script(parsed) in ENTRY_POINTS++[email protected]("order", list(itertools.permutations(["call_api.py", "parse_openapi.py", "url_safety.py"])))+def test_api_caller_helper_listed_first_is_skipped(order: tuple[str, ...]) -> None:+ parsed = _parse_in_order(API_CALLER, order)++ assert generate_dataset._pick_primary_script(parsed) == next(name for name in order if name in ENTRY_POINTS)++[email protected]("generate", [generate_dataset._generate_simple, generate_dataset._generate_full])+def test_generated_cases_expect_the_entry_point_not_the_helper(generate: Any) -> None:+ parsed = _parse_in_order(API_CALLER, ("url_safety.py", "parse_openapi.py", "call_api.py"))++ scripts = {case["expected_script"] for case in generate(parsed) if case["expected_skill"]}++ assert scripts == {"parse_openapi.py"}+++ "helper",+ [+ pytest.param(HELPER, id="functions-and-docstring"),+ pytest.param("", id="empty-init"),+ pytest.param("LIMIT = 10\nNAMES: list[str] = []\nNAMES += ['a']\n", id="constants-only"),+ pytest.param(+ "import logging\n\nlogger = logging.getLogger(__name__)\n\n\nclass Config:\n retries = 3\n",+ id="logger-and-class",+ ),+ pytest.param(+ "from typing import TYPE_CHECKING\n\nif TYPE_CHECKING:\n from pathlib import Path\n\n\n"+ "def name(path: 'Path') -> str:\n return path.name\n",+ id="type-checking-block",+ ),+ pytest.param(+ "try:\n import tomllib\nexcept ImportError:\n tomllib = None\n\n\ndef load():\n return tomllib\n",+ id="import-fallback",+ ),+ pytest.param(+ "import argparse\nimport sys\n\n\ndef parse():\n return argparse.ArgumentParser().parse_args(sys.argv[1:])\n",+ id="argument-parsing-only-inside-a-function",+ ),+ ],+)+def test_import_only_modules_are_skipped_in_any_position(tmp_path: Path, helper: str) -> None:+ skill = _write_skill(tmp_path, {"scripts/helper.py": helper, "scripts/run.py": GUARDED_ENTRY})++ for order in itertools.permutations(["helper.py", "run.py"]):+ assert generate_dataset._pick_primary_script(_parse_in_order(skill, order)) == "run.py"+++ "entry",+ [+ pytest.param(GUARDED_ENTRY, id="main-guard"),+ pytest.param("import sys\n\nif sys.argv[1:] and __name__ == '__main__':\n print('x')\n", id="guard-in-and"),+ pytest.param("def main():\n pass\n\n\nif '__main__' == __name__:\n main()\n", id="reversed-guard"),+ pytest.param("def main():\n print('hi')\n\n\nmain()\n", id="bare-top-level-call"),+ pytest.param("import sys\n\n\ndef main():\n return 0\n\n\nsys.exit(main())\n", id="exit-main"),+ pytest.param("def main():\n return 0\n\n\nresult = main()\n", id="assigned-main-call"),+ pytest.param("import sys\n\nname = sys.argv[1]\n", id="reads-argv-at-top-level"),+ pytest.param(+ "import argparse\n\nparser = argparse.ArgumentParser()\nargs = parser.parse_args()\n",+ id="top-level-argument-parsing",+ ),+ pytest.param("import json\n\nfor line in open('x'):\n print(json.loads(line))\n", id="top-level-loop"),+ pytest.param("print('hello')\n", id="print"),+ pytest.param("def broken(:\n", id="syntax-error-is-not-assumed-a-helper"),+ ],+)+def test_entry_points_are_kept_even_when_listed_after_a_helper(tmp_path: Path, entry: str) -> None:+ skill = _write_skill(tmp_path, {"scripts/helper.py": HELPER, "scripts/run.py": entry})++ assert generate_dataset._pick_primary_script(_parse_in_order(skill, ("helper.py", "run.py"))) == "run.py"+++def test_scripts_that_are_all_entry_points_keep_listing_order(tmp_path: Path) -> None:+ skill = _write_skill(tmp_path, {"scripts/a.py": GUARDED_ENTRY, "scripts/b.py": GUARDED_ENTRY})++ for order in (("a.py", "b.py"), ("b.py", "a.py")):+ assert generate_dataset._pick_primary_script(_parse_in_order(skill, order)) == order[0]+++def test_single_import_only_script_is_still_the_expected_script(tmp_path: Path) -> None:+ skill = _write_skill(tmp_path, {"scripts/helper.py": HELPER})++ assert generate_dataset._pick_primary_script(generate_dataset._parse_skill(skill)) == "helper.py"+++def test_all_import_only_scripts_fall_back_to_the_first(tmp_path: Path) -> None:+ skill = _write_skill(tmp_path, {"scripts/one.py": HELPER, "scripts/two.py": HELPER})++ for order in (("one.py", "two.py"), ("two.py", "one.py")):+ assert generate_dataset._pick_primary_script(_parse_in_order(skill, order)) == order[0]+++def test_helper_in_scripts_and_entry_point_in_tools(tmp_path: Path) -> None:+ skill = _write_skill(tmp_path, {"scripts/helper.py": HELPER, "tools/run.py": GUARDED_ENTRY})++ assert generate_dataset._pick_primary_script(_parse_in_order(skill, ("helper.py", "run.py"))) == "run.py"+++def test_interactive_entry_point_still_loses_to_a_non_interactive_one(tmp_path: Path) -> None:+ skill = _write_skill(+ tmp_path,+ {"scripts/helper.py": HELPER, "scripts/login.py": GUARDED_ENTRY, "scripts/fetch.py": GUARDED_ENTRY},+ skill_md_body="login.py is interactive and opens a browser.\n",+ )++ parsed = _parse_in_order(skill, ("helper.py", "login.py", "fetch.py"))++ assert generate_dataset._pick_primary_script(parsed) == "fetch.py"+++def test_interactive_entry_point_beats_a_non_interactive_helper(tmp_path: Path) -> None:+ """Only entry point is interactive: it still beats the helper, which the agent never runs."""+ skill = _write_skill(+ tmp_path,+ {"scripts/helper.py": HELPER, "scripts/login.py": GUARDED_ENTRY},+ skill_md_body="login.py is interactive and opens a browser.\n",+ )++ parsed = _parse_in_order(skill, ("helper.py", "login.py"))++ assert generate_dataset._pick_primary_script(parsed) == "login.py"+++def test_unreadable_script_is_not_assumed_to_be_a_helper(tmp_path: Path) -> None:+ skill = _write_skill(tmp_path, {"scripts/helper.py": HELPER})+ (skill / "scripts" / "binary.py").write_bytes(b"\xff\xfe\x00\x00 not utf-8 \x80")++ parsed = _parse_in_order(skill, ("helper.py", "binary.py"))++ assert generate_dataset._pick_primary_script(parsed) == "binary.py"