Skip to content

Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.

fix(api-caller): encode scalar OpenAPI path values (#682)

PMcbda65c1aa3b
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 0115cd5a..dccc9a45 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -453,6 +453,7 @@ All notable changes to SkillEvaluator are documented in this file.
### Fixed
+- API caller now percent-encodes scalar OpenAPI path values as individual segments and rejects `.` and `..` values with a JSON input error.
- Direct Tier 1 validates the target, selected checks and LLM configuration before
preparing scanners, so configuration errors do not trigger downloads or stdout noise.
- Documentation-only CI follows imports through collected test modules and
diff --git a/src/skillevaluator/tier3/reference_skills/api-caller/scripts/call_api.py b/src/skillevaluator/tier3/reference_skills/api-caller/scripts/call_api.py
index a670923b..5b8a522e 100644
--- a/src/skillevaluator/tier3/reference_skills/api-caller/scripts/call_api.py
+++ b/src/skillevaluator/tier3/reference_skills/api-caller/scripts/call_api.py
@@ -25,7 +25,7 @@ import argparse
import json
import sys
from urllib.error import HTTPError, URLError
-from urllib.parse import urlencode, urlparse
+from urllib.parse import quote, urlencode, urlparse
from urllib.request import Request, urlopen
_UNSET = object()
@@ -147,7 +147,10 @@ def build_url_from_spec(spec: dict, path: str, params: dict) -> str:
for key, value in params.items():
placeholder = "{" + key + "}"
if placeholder in url_path:
- url_path = url_path.replace(placeholder, str(value))
+ path_value = str(value)
+ if path_value in {".", ".."}:
+ raise ValueError(f"Path parameter '{key}' must not be '.' or '..'")
+ url_path = url_path.replace(placeholder, quote(path_value, safe=""))
else:
query_params[key] = value
@@ -239,7 +242,11 @@ def main():
)
sys.exit(1)
- url = build_url_from_spec(spec, path, params)
+ try:
+ url = build_url_from_spec(spec, path, params)
+ except ValueError as exc:
+ _print_json({"success": False, "error": f"Invalid input: {exc}"}, file=sys.stderr)
+ sys.exit(1)
method = spec_method
print(f"Resolved OpenAPI operation: {method} {url}", file=sys.stderr)
diff --git a/tests/test_reference_api_caller.py b/tests/test_reference_api_caller.py
index 259bac8e..28d7a60e 100644
--- a/tests/test_reference_api_caller.py
+++ b/tests/test_reference_api_caller.py
@@ -6,8 +6,12 @@
from __future__ import annotations
import importlib.util
+import json
+import sys
from pathlib import Path
+import pytest
+
SCRIPTS_DIR = (
Path(__file__).resolve().parents[1]
/ "src"
@@ -27,6 +31,73 @@ def _load_script(module_name: str):
return module
+ ("value", "encoded"),
+ [
+ ("plain-id", "plain-id"),
+ ("file.txt", "file.txt"),
+ ("...", "..."),
+ (7, "7"),
+ ("red blue", "red%20blue"),
+ ("café", "caf%C3%A9"),
+ ("a/b?#c%", "a%2Fb%3F%23c%25"),
+ ("a%2Fb", "a%252Fb"),
+ ],
+)
+def test_openapi_path_values_are_encoded_as_segments(value: str | int, encoded: str) -> None:
+ caller = _load_script("call_api")
+ spec = {"servers": [{"url": "https://example.invalid/v1"}]}
+
+ result = caller.build_url_from_spec(spec, "/items/{id}", {"id": value, "limit": 2})
+
+ assert result == f"https://example.invalid/v1/items/{encoded}?limit=2"
+
+
[email protected]("value", [".", ".."])
+def test_openapi_path_dot_segments_are_rejected(value: str) -> None:
+ caller = _load_script("call_api")
+ spec = {"servers": [{"url": "https://example.invalid/v1"}]}
+
+ with pytest.raises(ValueError, match=r"Path parameter 'id' must not be '\.' or '\.\.'"):
+ caller.build_url_from_spec(spec, "/items/{id}", {"id": value})
+
+
[email protected]("value", [".", ".."])
+def test_openapi_path_dot_segments_report_json_errors(
+ value: str, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]
+) -> None:
+ caller = _load_script("call_api")
+ spec = {
+ "servers": [{"url": "https://example.invalid/v1"}],
+ "paths": {"/items/{id}": {"get": {"operationId": "getItem"}}},
+ }
+ monkeypatch.setattr(caller, "fetch_openapi_spec", lambda _url: spec)
+ monkeypatch.setattr(
+ sys,
+ "argv",
+ [
+ "call_api.py",
+ "--spec",
+ "https://example.invalid/spec",
+ "--operation",
+ "getItem",
+ "--params",
+ json.dumps({"id": value}),
+ ],
+ )
+
+ with pytest.raises(SystemExit) as exc:
+ caller.main()
+
+ captured = capsys.readouterr()
+ assert exc.value.code == 1
+ assert captured.out == ""
+ assert json.loads(captured.err) == {
+ "success": False,
+ "error": "Invalid input: Path parameter 'id' must not be '.' or '..'",
+ }
+
+
def test_api_caller_rejects_non_http_urls_without_opening_them() -> None:
api_caller = _load_script("call_api")