Create, Evaluate, and Connect AI Skills
Python ai-agentsai-skillsartificial-intelligenceknowledge-graphlarge-language-modelsnatural-language-processingopenkgskillnetskills
"""Routing and the two SDK contracts; all model and agent calls stay offline."""
import asyncio
import json
import sys
import threading
from collections.abc import AsyncIterator, Iterator
from contextlib import nullcontext
from pathlib import Path
from types import ModuleType, SimpleNamespace
import pytest
from pydantic import RootModel, SecretStr
from typer.testing import CliRunner
from skillnet_ai import Endpoint, RouteOptions, SkillNetClient
from skillnet_ai.core.config import ENVIRONMENT
from skillnet_ai.core.models import (
AnalyzedSkill,
Exploration,
GraphSnapshot,
Relation,
SkillProfile,
SkillSelection,
)
from skillnet_ai.interfaces.cli import app
from skillnet_ai.router import explorer as explorer_module
from skillnet_ai.router.explorer import ClaudeExplorer, CodexExplorer
ENDPOINT = Endpoint(api_key=SecretStr("test-key"), base_url="https://model.test/v1", model="test")
def selection(skill_id: str = "stats") -> SkillSelection:
"""A small SDK response; paths must still be resolved by SkillNet."""
return SkillSelection.model_validate(
{
"skills": [
{
"skill_id": skill_id,
"reason": "The user already has CSV.",
}
],
}
)
@pytest.fixture
def index(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path:
"""Persist three skill records without using analysis or any model."""
np = pytest.importorskip("numpy")
from skillnet_ai.router import router
from skillnet_ai.router.index import build_bm25
for variable in ENVIRONMENT.values():
monkeypatch.delenv(variable, raising=False)
monkeypatch.setenv("SKILLNET_CONFIG", str(tmp_path / "config.json"))
for prefix in ("EMBEDDING_", "SKILLNET_EXPLORER_"):
monkeypatch.setenv(prefix + "API_KEY", "test-key")
monkeypatch.setenv(prefix + "BASE_URL", ENDPOINT.base_url)
monkeypatch.setenv(prefix + "MODEL", ENDPOINT.model)
skills = []
for key, text in {
"stats": "Compute statistics from CSV.",
"extract": "Extract CSV from PDF.",
"alternative": "Extract CSV using another PDF engine.",
}.items():
profile = SkillProfile.model_validate(
{
"capability": {"text": text, "evidence": [{"line": 1}]},
"when_to_use": [],
"inputs": [],
"outputs": [],
"scenarios": [],
"constraints": [],
"tools": [],
}
)
skills.append(
AnalyzedSkill(
skill_id=key,
name=key,
path=str(tmp_path / "original" / key),
source=text,
content_hash=key,
profile=profile,
)
)
relations = [
Relation.model_validate(
{
"source": "extract",
"target": target,
"type": kind,
"contexts": [
{
"scenario": "CSV analysis",
"explanation": "A CSV handoff or alternative.",
"conditions": [],
"source_evidence": [{"line": 1}],
"target_evidence": [{"line": 1}],
}
],
}
)
for target, kind in (("stats", "compose_with"), ("alternative", "similar_to"))
]
graph = GraphSnapshot(
embedding_model=ENDPOINT.model,
embedding_base_url=ENDPOINT.base_url,
skills=skills,
relations=relations,
)
snapshot = tmp_path / "snapshot-test"
snapshot.mkdir()
(snapshot / "graph.json").write_text(graph.model_dump_json(), encoding="utf-8")
(tmp_path / "CURRENT").write_text(snapshot.name, encoding="utf-8")
np.savez(
snapshot / "embeddings.npz",
ids=[s.skill_id for s in skills],
vectors=[[1.0, 0.0], [0.0, 1.0], [0.0, 1.0]],
)
build_bm25(skills, snapshot / "bm25.sqlite")
monkeypatch.setattr(router, "embed", lambda *args, **kwargs: [[1.0, 0.0]])
return tmp_path
@pytest.mark.parametrize("supported", [True, False])
def test_route_selects_from_snapshot(
index: Path, monkeypatch: pytest.MonkeyPatch, supported: bool
) -> None:
"""Retrieval expands the graph, but does not force related skills or fill k."""
from skillnet_ai.router import router as explorer
def explore(
root: Path, query: str, k: int, endpoint: Endpoint, options: RouteOptions
) -> Exploration:
assert all(
(root / f"sources/{key}.md").is_file() for key in ("stats", "extract", "alternative")
)
assert json.loads((root / "relations.json").read_text())[0]["source"] == "extract"
answer = selection() if supported else SkillSelection(skills=[])
return Exploration(answer)
monkeypatch.setattr(explorer, "backend_for", lambda backend: SimpleNamespace(explore=explore))
result = SkillNetClient().route(
"statistics from existing CSV" if supported else "edit a video",
index_dir=index,
options=RouteOptions(seed_limit=1, candidate_limit=3),
)
assert [s.skill_id for s in result.skills] == (["stats"] if supported else [])
assert set(result.model_dump()) == {"skills", "usage"}
if supported:
assert result.skills[0].path == str(index / "original/stats")
assert set(result.skills[0].model_dump()) == {"skill_id", "name", "path", "reason"}
assert not Path(result.skills[0].path).exists() # Routing uses the source snapshot.
cli = CliRunner().invoke(app, ["route", "statistics CSV", "--index-dir", str(index), "--json"])
assert cli.exit_code == 0 and json.loads(cli.stdout)["ok"] is True
assert set(json.loads(cli.stdout)["data"]) == {"skills", "usage"}
assert "evidence" not in cli.stdout
@pytest.mark.parametrize("problem", ["unknown", "duplicate", "limit"])
def test_route_rejects_unsupported_selection(index: Path, problem: str) -> None:
"""Selection needs known, unique IDs within the requested limit."""
from skillnet_ai.router.index import load_snapshot
from skillnet_ai.router.router import validate_selection
answer = selection("missing" if problem == "unknown" else "stats")
if problem == "duplicate":
answer.skills *= 2
elif problem == "limit":
answer.skills.extend(selection("extract").skills)
with pytest.raises(ValueError):
validate_selection(
Exploration(answer),
load_snapshot(index)[1].skills,
1 if problem == "limit" else 5,
)
@pytest.mark.parametrize("problem", ["model", "dimension"])
def test_embedding_mismatch_fails(
index: Path, monkeypatch: pytest.MonkeyPatch, problem: str
) -> None:
"""An incompatible index must fail before starting an Explorer."""
from skillnet_ai.router import router
def unexpected(*args: object, **kwargs: object) -> None:
pytest.fail("An incompatible embedding must not start an Explorer")
monkeypatch.setattr(router, "explore", unexpected)
monkeypatch.setattr(router, "embed", lambda *args, **kwargs: [[1.0, 0.0, 0.0]])
embedding = ENDPOINT.model_copy(update={"model": "other"}) if problem == "model" else ENDPOINT
with pytest.raises(ValueError, match="endpoint/model" if problem == "model" else "dimension"):
SkillNetClient().route("task", index_dir=index, embedding=embedding)
@pytest.mark.parametrize("timeout", [False, True])
def test_claude_session(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, timeout: bool) -> None:
"""Exercise SDK options, tool permissions, structured output and timeout cleanup."""
sdk = ModuleType("claude_agent_sdk")
sdk.ClaudeAgentOptions = sdk.HookMatcher = SimpleNamespace
sdk.ResultMessage = type("ResultMessage", (SimpleNamespace,), {})
closed = threading.Event()
async def query(
*, prompt: AsyncIterator[dict], options: SimpleNamespace
) -> AsyncIterator[object]:
try:
assert [item async for item in prompt]
assert options.model == ENDPOINT.model and options.setting_sources == []
assert options.env["ANTHROPIC_BASE_URL"] == ENDPOINT.base_url
assert options.output_format["schema"]["additionalProperties"] is False
assert set(options.output_format["schema"]["properties"]) == {"skills"}
assert set(options.output_format["schema"]["$defs"]["Selection"]["properties"]) == {
"skill_id",
"reason",
}
if timeout:
await asyncio.Event().wait()
hook = options.hooks["PreToolUse"][0]
assert hook.matcher == "Read|LS|Glob|Grep"
denied = await hook.hooks[0](
{"tool_name": "Read", "tool_input": {"file_path": "../outside"}}, None, {}
)
assert denied["hookSpecificOutput"]["permissionDecision"] == "deny"
read = {"tool_name": "Read", "tool_input": {"file_path": "sources/stats.md"}}
allowed = await hook.hooks[0](read, None, {})
assert allowed["hookSpecificOutput"]["permissionDecision"] == "allow"
yield sdk.ResultMessage(
is_error=False,
structured_output=selection().model_dump(),
usage={"input_tokens": 5},
)
finally:
closed.set()
sdk.query = query
monkeypatch.setitem(sys.modules, "claude_agent_sdk", sdk)
if timeout:
with pytest.raises(TimeoutError):
ClaudeExplorer().explore(tmp_path, "task", 1, ENDPOINT, RouteOptions(timeout=0.05))
else:
result = ClaudeExplorer().explore(tmp_path, "task", 1, ENDPOINT, RouteOptions())
assert result.selection == selection()
assert result.usage == {"input_tokens": 5}
assert closed.is_set()
@pytest.mark.parametrize("failure", [None, "provider", "timeout", "outside", "tool", "windows"])
def test_codex_session(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch, failure: str | None
) -> None:
"""Exercise parsed SDK tool events and close the session on all exit paths."""
sdk, types = ModuleType("openai_codex"), ModuleType("openai_codex.types")
sdk.CodexConfig, types.ReasoningEffort = SimpleNamespace, str
sdk.ApprovalMode, sdk.Sandbox = (
SimpleNamespace(deny_all="never"),
SimpleNamespace(read_only="read-only"),
)
closed = threading.Event()
if failure == "windows":
monkeypatch.setattr(explorer_module, "sys", SimpleNamespace(platform="win32"))
def event(method: str, payload: dict[str, object]) -> SimpleNamespace:
return SimpleNamespace(method=method, payload=RootModel[dict](payload))
class Turn:
def stream(self) -> Iterator[SimpleNamespace]:
if failure == "timeout":
assert closed.wait(2)
return
if failure == "provider":
raise RuntimeError("provider failed")
item = {
"type": "commandExecution",
"cwd": str(tmp_path),
"exitCode": None,
"commandActions": [
{
"type": "unknown" if failure in {"tool", "windows"} else "search",
"path": "../outside" if failure == "outside" else "sources/stats.md",
"command": "python -V",
}
],
}
if failure == "windows":
item["commandActions"][0]["command"] = explorer_module.POWERSHELL_UTF8_PREFIX + (
"Get-Content -LiteralPath 'sources/stats.md' -Encoding UTF8"
)
yield event("item/started", {"item": item})
yield event("item/completed", {"item": {**item, "exitCode": 0}})
yield event(
"item/completed",
{
"item": {
"type": "agentMessage",
"phase": "final_answer",
"text": selection().model_dump_json(),
}
},
)
yield event("turn/completed", {"turn": {"status": "completed"}})
class Codex:
def __init__(self, *, config: SimpleNamespace) -> None:
assert config.env["OPENAI_API_KEY"] == "test-key"
assert {
"features.plugins=false",
"features.remote_plugin=false",
} <= set(config.config_overrides)
def __enter__(self) -> "Codex":
return self
def login_api_key(self, key: str) -> None:
assert key == "test-key"
def thread_start(self, **kwargs: object) -> SimpleNamespace:
assert kwargs["sandbox"] == "read-only" and kwargs["ephemeral"] is True
assert kwargs["config"]["model_providers"]["skillnet"]["base_url"] == ENDPOINT.base_url
return SimpleNamespace(turn=self.turn)
def turn(self, prompt: str, **kwargs: object) -> Turn:
assert kwargs["output_schema"]["additionalProperties"] is False
assert set(kwargs["output_schema"]["properties"]) == {"skills"}
assert set(kwargs["output_schema"]["$defs"]["Selection"]["properties"]) == {
"skill_id",
"reason",
}
return Turn()
def close(self) -> None:
closed.set()
sdk.Codex = Codex
monkeypatch.setitem(sys.modules, "openai_codex", sdk)
monkeypatch.setitem(sys.modules, "openai_codex.types", types)
options = RouteOptions(timeout=0.05 if failure == "timeout" else 300)
if failure and failure != "windows":
error = {
"provider": RuntimeError,
"timeout": TimeoutError,
"outside": ValueError,
"tool": ValueError,
}[failure]
with pytest.raises(error):
CodexExplorer().explore(tmp_path, "task", 1, ENDPOINT, options)
else:
result = CodexExplorer().explore(tmp_path, "task", 1, ENDPOINT, options)
assert result.selection == selection()
assert closed.is_set()
@pytest.mark.parametrize("prefix", ["", explorer_module.POWERSHELL_UTF8_PREFIX])
@pytest.mark.parametrize("platform", ["win32", "linux", "darwin"])
@pytest.mark.parametrize(
("command", "allowed"),
[
("Get-Content -LiteralPath 'sources/stats notes.md' -Encoding UTF8", True),
("Get-Content -LiteralPath 'sources/中文 notes.md' -Encoding UTF8", True),
("Get-ChildItem -LiteralPath '.' -Name", True),
("Get-Content -LiteralPath '../outside.md' -Encoding UTF8", False),
("Get-Content -LiteralPath 'Env:OPENAI_API_KEY' -Encoding UTF8", False),
("Get-Content -LiteralPath '~/outside.md' -Encoding UTF8", False),
("Get-ChildItem -LiteralPath '.' -Name; Remove-Item 'index.md'", False),
("Get-Content -LiteralPath 'index.md' -Encoding UTF8 | Invoke-Expression", False),
],
)
def test_powershell_wiki_commands(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
prefix: str,
platform: str,
command: str,
allowed: bool,
) -> None:
monkeypatch.setattr(explorer_module, "sys", SimpleNamespace(platform=platform))
item = {
"cwd": str(tmp_path),
"commandActions": [{"type": "unknown", "command": prefix + command}],
}
with nullcontext() if allowed and platform == "win32" else pytest.raises(ValueError):
explorer_module.validate_wiki_command(item, tmp_path)
@pytest.mark.parametrize("quote", ["\u2018", "\u2019", "\u201a", "\u201b"])
@pytest.mark.parametrize("prefix", ["", explorer_module.POWERSHELL_UTF8_PREFIX])
@pytest.mark.parametrize(
("cmdlet", "argument"), [("Get-Content", "-Encoding UTF8"), ("Get-ChildItem", "-Name")]
)
def test_powershell_rejects_typographic_quote_breakout(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
quote: str,
prefix: str,
cmdlet: str,
argument: str,
) -> None:
"""PowerShell treats typographic single quotes as string delimiters too."""
monkeypatch.setattr(explorer_module, "sys", SimpleNamespace(platform="win32"))
command = f"{prefix}{cmdlet} -LiteralPath 'index.md{quote}; Write-Output marker; #x' {argument}"
item = {"cwd": str(tmp_path), "commandActions": [{"type": "unknown", "command": command}]}
with pytest.raises(ValueError, match="literal PowerShell"):
explorer_module.validate_wiki_command(item, tmp_path)