Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.
Python agent-evaluationagent-pluginsagent-securityagent-skillsagentic-aibenchmarkci-templatesclaude-codecodexevaluateevaluationharnesspluginplugin-evaluationsecurity-scannerskill-evalskill-evaluationskill-evaluatorskill-gap-analysisskills
fix(api-caller): encode scalar OpenAPI path values (#682)
PMcbda65c1aa3b
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 0115cd5a..dccc9a45 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -453,6 +453,7 @@ All notable changes to SkillEvaluator are documented in this file.### Fixed+- API caller now percent-encodes scalar OpenAPI path values as individual segments and rejects `.` and `..` values with a JSON input error.- Direct Tier 1 validates the target, selected checks and LLM configuration beforepreparing scanners, so configuration errors do not trigger downloads or stdout noise.- Documentation-only CI follows imports through collected test modules anddiff --git a/src/skillevaluator/tier3/reference_skills/api-caller/scripts/call_api.py b/src/skillevaluator/tier3/reference_skills/api-caller/scripts/call_api.pyindex a670923b..5b8a522e 100644--- a/src/skillevaluator/tier3/reference_skills/api-caller/scripts/call_api.py+++ b/src/skillevaluator/tier3/reference_skills/api-caller/scripts/call_api.py@@ -25,7 +25,7 @@ import argparseimport jsonimport sysfrom urllib.error import HTTPError, URLError-from urllib.parse import urlencode, urlparse+from urllib.parse import quote, urlencode, urlparsefrom urllib.request import Request, urlopen_UNSET = object()@@ -147,7 +147,10 @@ def build_url_from_spec(spec: dict, path: str, params: dict) -> str:for key, value in params.items():placeholder = "{" + key + "}"if placeholder in url_path:- url_path = url_path.replace(placeholder, str(value))+ path_value = str(value)+ if path_value in {".", ".."}:+ raise ValueError(f"Path parameter '{key}' must not be '.' or '..'")+ url_path = url_path.replace(placeholder, quote(path_value, safe=""))else:query_params[key] = value@@ -239,7 +242,11 @@ def main():)sys.exit(1)- url = build_url_from_spec(spec, path, params)+ try:+ url = build_url_from_spec(spec, path, params)+ except ValueError as exc:+ _print_json({"success": False, "error": f"Invalid input: {exc}"}, file=sys.stderr)+ sys.exit(1)method = spec_methodprint(f"Resolved OpenAPI operation: {method} {url}", file=sys.stderr)diff --git a/tests/test_reference_api_caller.py b/tests/test_reference_api_caller.pyindex 259bac8e..28d7a60e 100644--- a/tests/test_reference_api_caller.py+++ b/tests/test_reference_api_caller.py@@ -6,8 +6,12 @@from __future__ import annotationsimport importlib.util+import json+import sysfrom pathlib import Path+import pytest+SCRIPTS_DIR = (Path(__file__).resolve().parents[1]/ "src"@@ -27,6 +31,73 @@ def _load_script(module_name: str):return module+ ("value", "encoded"),+ [+ ("plain-id", "plain-id"),+ ("file.txt", "file.txt"),+ ("...", "..."),+ (7, "7"),+ ("red blue", "red%20blue"),+ ("café", "caf%C3%A9"),+ ("a/b?#c%", "a%2Fb%3F%23c%25"),+ ("a%2Fb", "a%252Fb"),+ ],+)+def test_openapi_path_values_are_encoded_as_segments(value: str | int, encoded: str) -> None:+ caller = _load_script("call_api")+ spec = {"servers": [{"url": "https://example.invalid/v1"}]}++ result = caller.build_url_from_spec(spec, "/items/{id}", {"id": value, "limit": 2})++ assert result == f"https://example.invalid/v1/items/{encoded}?limit=2"++[email protected]("value", [".", ".."])+def test_openapi_path_dot_segments_are_rejected(value: str) -> None:+ caller = _load_script("call_api")+ spec = {"servers": [{"url": "https://example.invalid/v1"}]}++ with pytest.raises(ValueError, match=r"Path parameter 'id' must not be '\.' or '\.\.'"):+ caller.build_url_from_spec(spec, "/items/{id}", {"id": value})++[email protected]("value", [".", ".."])+def test_openapi_path_dot_segments_report_json_errors(+ value: str, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]+) -> None:+ caller = _load_script("call_api")+ spec = {+ "servers": [{"url": "https://example.invalid/v1"}],+ "paths": {"/items/{id}": {"get": {"operationId": "getItem"}}},+ }+ monkeypatch.setattr(caller, "fetch_openapi_spec", lambda _url: spec)+ monkeypatch.setattr(+ sys,+ "argv",+ [+ "call_api.py",+ "--spec",+ "https://example.invalid/spec",+ "--operation",+ "getItem",+ "--params",+ json.dumps({"id": value}),+ ],+ )++ with pytest.raises(SystemExit) as exc:+ caller.main()++ captured = capsys.readouterr()+ assert exc.value.code == 1+ assert captured.out == ""+ assert json.loads(captured.err) == {+ "success": False,+ "error": "Invalid input: Path parameter 'id' must not be '.' or '..'",+ }++def test_api_caller_rejects_non_http_urls_without_opening_them() -> None:api_caller = _load_script("call_api")