Create, Evaluate, and Connect AI Skills
Python ai-agentsai-skillsartificial-intelligenceknowledge-graphlarge-language-modelsnatural-language-processingopenkgskillnetskills
import ast
import inspect
import hashlib
import json
import re
import logging
import os
import secrets
import shlex
import subprocess
import time
from concurrent.futures import ThreadPoolExecutor, as_completed, wait, FIRST_COMPLETED
from dataclasses import dataclass
from urllib.parse import urlsplit
from typing import Dict, Any, List, Optional, Tuple, Callable, Iterator
from openai import OpenAI
from json_repair import repair_json
from tqdm import tqdm
from skillnet_ai.downloader import SkillDownloader
from skillnet_ai.core.llm import chat_completion, error_details
from skillnet_ai.core.validation import validate_evaluation
from skillnet_ai.core.injection import (
InjectionContent,
InjectionReport,
InjectionScanIssue,
SkillInjectionScanner,
)
from skillnet_ai.core.prompts import EVALUATION_SYSTEM_PROMPT, SKILL_EVALUATION_PROMPT
logger = logging.getLogger(__name__)
# ==========================================================================
# Configuration and data models
# ==========================================================================
@dataclass
class EvaluatorConfig:
"""Configuration for the skill evaluator."""
api_key: str
base_url: str
model: str
max_workers: int = 5
temperature: float = 0.3
cache_dir: str = "./evaluate_cache_dir"
run_scripts: bool = False # Enable only for trusted local skill code.
script_timeout_sec: int = 8
max_script_runs: int = 5
script_python: str = "python"
include_script_results: bool = False
max_script_output_chars: int = 400
github_token: Optional[str] = None
scan_injection: bool = True
json_mode: str = "auto"
@dataclass
class Skill:
"""Unified representation of a skill."""
path: str # Local path to the skill root directory
name: str
description: Optional[str] = None
category: Optional[str] = None
url: Optional[str] = None # Original URL (when created from URL)
@classmethod
def from_url(
cls,
url: str,
downloader: 'SkillDownloader',
cache_dir: str,
max_retries: int = 3,
retry_delay: float = 2.0,
**kwargs
) -> Tuple[Optional['Skill'], Optional[str]]:
"""
Create a Skill from a GitHub URL.
download fails, it will retry, finally return (None, error_msg) instead of throwing an exception.
Returns:
(Skill, None) success; (None, error_msg) failure.
"""
normalized_url = cls._normalize_url(url)
if not normalized_url:
return None, f"Invalid GitHub URL: {url}"
# Namespace the owned cache by URL, so different repositories do not collide.
cache_dir = os.path.join(cache_dir, hashlib.sha256(normalized_url.encode()).hexdigest()[:16])
# Download to local cache (with retries in evaluator)
local_path = None
for attempt in range(max_retries):
try:
local_path = downloader.download(normalized_url, target_dir=cache_dir, overwrite=True, require_skill=True)
except Exception as exc:
return None, error_details(exc)["message"]
if local_path:
break
if attempt < max_retries - 1:
logger.warning(
"Download failed (attempt %d/%d). Retrying in %.1fs...",
attempt + 1, max_retries, retry_delay,
)
time.sleep(retry_delay)
if not local_path:
return None, f"Failed to download after {max_retries} retries: {url}"
return cls(
path=local_path,
name=kwargs.get("name") or os.path.basename(local_path),
url=url,
description=kwargs.get('description'),
category=kwargs.get('category')
), None
@classmethod
def from_path(cls, path: str, **kwargs) -> Tuple[Optional['Skill'], Optional[str]]:
"""Create a Skill from a local directory path.
Returns:
(Skill, None) success; (None, error_msg) failure.
"""
abs_path = os.path.abspath(path)
if not os.path.isdir(abs_path):
return None, f"Invalid skill path: {path}"
name = kwargs.get('name') or os.path.basename(abs_path)
return (
cls(
path=abs_path,
name=name,
description=kwargs.get('description'),
category=kwargs.get('category')
),
None,
)
@staticmethod
def _normalize_url(url: str) -> Optional[str]:
"""Keep blob semantics so the downloader resolves the whole skill package."""
if not url:
return None
parsed = urlsplit(url)
parts = parsed.path.strip("/").split("/")
if (parsed.scheme == "https" and parsed.netloc.lower() == "github.com"
and len(parts) >= 4 and parts[2] in {"tree", "blob"}):
return url
return None
@dataclass
class ScriptExecutionResult:
"""Result of executing a single script."""
path: str
status: str # success | compiled_only | failed | timeout | skipped
command: str
exit_code: Optional[int] = None
error: Optional[str] = None
duration_sec: Optional[float] = None
note: Optional[str] = None
def to_dict(self) -> Dict[str, Any]:
return {
"path": self.path,
"status": self.status,
"command": self.command,
"exit_code": self.exit_code,
"error": self.error,
"duration_sec": self.duration_sec,
"note": self.note,
}
class ScriptRunner:
"""Execute python scripts under scripts/ with safe defaults."""
PATH_LIKE_EXTS = {
".xml",
".json",
".yaml",
".yml",
".csv",
".tsv",
".txt",
".md",
".ini",
".toml",
".coverage",
".db",
".sqlite",
".sql",
".parquet",
}
def __init__(self, python_bin: str, timeout_sec: int, max_runs: int,
max_output_chars: int):
self.python_bin = python_bin
self.timeout_sec = timeout_sec
self.max_runs = max_runs
self.max_output_chars = max_output_chars
def run_for_skill(
self, skill_dir: str, skip_reason: Optional[str] = None
) -> List[ScriptExecutionResult]:
"""Execute discovered scripts, or report every script as skipped."""
scripts = self._discover_py_scripts(skill_dir)
if skip_reason:
return [
self._result_skipped(os.path.relpath(path, skill_dir), skip_reason)
for path in scripts
]
results: List[ScriptExecutionResult] = []
for script_path in scripts[: self.max_runs]:
results.append(self._run_script(skill_dir, script_path))
if len(scripts) > self.max_runs:
logger.info(
"Found %s runnable scripts, truncated to %s for execution",
len(scripts),
self.max_runs
)
return results
def _result_skipped(
self, rel_path: str, reason: str
) -> ScriptExecutionResult:
return ScriptExecutionResult(
path=rel_path,
status="skipped",
command="[not executed]",
note=reason,
)
def _discover_py_scripts(self, skill_dir: str) -> List[str]:
paths: List[str] = []
for root, _, files in os.walk(skill_dir):
if "scripts" not in root.split(os.sep):
continue
for filename in files:
if filename.lower().endswith(".py"):
paths.append(os.path.join(root, filename))
return sorted(paths)
def _run_script(self, skill_dir: str, script_path: str) -> ScriptExecutionResult:
rel_path = os.path.relpath(script_path, skill_dir)
usage_cmd = self._build_usage_command(script_path, rel_path)
if usage_cmd:
missing_inputs = self._detect_missing_inputs(usage_cmd, skill_dir)
if missing_inputs:
compile_result = self._run_command(
[self.python_bin, "-m", "py_compile", rel_path],
skill_dir
)
note = f"missing inputs: {', '.join(missing_inputs)}"
if compile_result["timed_out"]:
return self._result_timeout(rel_path, compile_result, note=note)
if compile_result["exit_code"] == 0:
return self._result_compiled_only(rel_path, compile_result, note=note)
return self._result_failed(rel_path, compile_result, note=note)
run_result = self._run_command(usage_cmd, skill_dir)
note = "usage-derived command"
if run_result["timed_out"]:
return self._result_timeout(rel_path, run_result, note=note)
if run_result["exit_code"] == 0:
return self._result_success(rel_path, run_result, note=note)
return self._result_failed(rel_path, run_result, note=note)
compile_result = self._run_command(
[self.python_bin, "-m", "py_compile", rel_path],
skill_dir
)
if compile_result["timed_out"]:
return self._result_timeout(rel_path, compile_result)
if compile_result["exit_code"] == 0:
return self._result_compiled_only(
rel_path,
compile_result,
note="no usage examples found; py_compile succeeded",
)
return self._result_failed(
rel_path,
compile_result,
note="no usage examples found",
)
def _result_timeout(self, rel_path: str, result: Dict[str, Any], note: Optional[str] = None) -> ScriptExecutionResult:
return ScriptExecutionResult(
path=rel_path,
status="timeout",
command=result["command"],
exit_code=None,
error=result.get("error"),
duration_sec=result["duration_sec"],
note=note,
)
def _result_success(self, rel_path: str, result: Dict[str, Any], note: Optional[str] = None) -> ScriptExecutionResult:
return ScriptExecutionResult(
path=rel_path,
status="success",
command=result["command"],
exit_code=result.get("exit_code"),
duration_sec=result["duration_sec"],
note=note,
)
def _result_compiled_only(self, rel_path: str, result: Dict[str, Any], note: Optional[str] = None) -> ScriptExecutionResult:
return ScriptExecutionResult(
path=rel_path,
status="compiled_only",
command=result["command"],
exit_code=result.get("exit_code"),
error=self._pick_error(result),
duration_sec=result["duration_sec"],
note=note,
)
def _result_failed(self, rel_path: str, result: Dict[str, Any], note: Optional[str] = None) -> ScriptExecutionResult:
return ScriptExecutionResult(
path=rel_path,
status="failed",
command=result["command"],
exit_code=result.get("exit_code"),
error=self._pick_error(result),
duration_sec=result["duration_sec"],
note=note,
)
def _build_usage_command(self, script_path: str,
rel_path: str) -> Optional[List[str]]:
script_name = os.path.basename(script_path)
usage_lines = self._extract_usage_lines(script_path, script_name)
if not usage_lines:
return None
candidates: List[List[str]] = []
for line in usage_lines:
cmd = self._parse_usage_line(line, rel_path, script_name)
if cmd:
candidates.append(cmd)
if not candidates:
return None
# Prefer runnable commands:
# - Commands containing placeholder tokens like "<file>", "[options]", "{path}" are
# often documentation examples and not directly runnable.
# - If placeholders exist, prefer a help-style command to at least verify the script
# starts up, otherwise fall back to compilation-only.
runnable = [cmd for cmd in candidates if not self._has_placeholder_tokens(cmd)]
for cmd in runnable:
if not self._is_help_command(cmd):
return cmd
for cmd in candidates:
if self._is_help_command(cmd):
return cmd
return None
def _extract_usage_lines(self, script_path: str,
script_name: str) -> List[str]:
try:
with open(script_path, "r", encoding="utf-8", errors="ignore") as f:
source = f.read()
except Exception as e:
logger.warning("Failed to read %s: %s", script_path, e)
return []
try:
tree = ast.parse(source)
doc = ast.get_docstring(tree) or ""
except Exception:
doc = ""
if not doc:
return []
lines = doc.splitlines()
usage_lines: List[str] = []
for idx, line in enumerate(lines):
if line.strip().lower().startswith("usage:"):
for follow in lines[idx + 1:]:
if not follow.strip():
break
usage_lines.append(follow.strip())
if usage_lines:
return usage_lines
for line in lines:
stripped = line.strip()
if not stripped:
continue
if stripped.startswith(("./", "python", "python3")):
usage_lines.append(stripped)
continue
if stripped.startswith(script_name):
usage_lines.append(stripped)
return usage_lines
def _parse_usage_line(self, line: str, rel_path: str,
script_name: str) -> Optional[List[str]]:
try:
tokens = shlex.split(line)
except ValueError:
return None
if not tokens:
return None
python_prefix = tokens[0].startswith("python")
if python_prefix:
tokens = [self.python_bin] + tokens[1:]
script_idx = None
for idx, token in enumerate(tokens):
token_base = os.path.basename(token)
if token_base == script_name:
script_idx = idx
break
if script_idx is None:
return None
tokens[script_idx] = rel_path
if not python_prefix:
tokens = [self.python_bin] + tokens[script_idx:]
return tokens
def _iter_non_flag_tokens(self, cmd: List[str]) -> Iterator[str]:
for token in cmd:
if not token or token.startswith("-"):
continue
if token == self.python_bin:
continue
yield token
def _is_help_command(self, cmd: List[str]) -> bool:
for token in cmd:
lowered = token.lower()
if lowered in {"--help", "-h", "help"}:
return True
return False
def _has_placeholder_tokens(self, cmd: List[str]) -> bool:
for token in self._iter_non_flag_tokens(cmd):
if self._is_placeholder_token(token):
return True
return False
@staticmethod
def _is_placeholder_token(token: str) -> bool:
# Common usage placeholders: <...>, [...], {...}
if any(ch in token for ch in ("<", ">", "[", "]", "{", "}")):
return True
lowered = token.strip().lower()
return lowered in {"options", "[options]", "<options>", "{options}"}
def _detect_missing_inputs(self, cmd: List[str], cwd: str) -> List[str]:
missing: List[str] = []
for token in self._iter_non_flag_tokens(cmd):
if self._is_placeholder_token(token):
continue
if not self._looks_like_path(token):
continue
path = token if os.path.isabs(token) else os.path.join(cwd, token)
if not os.path.exists(path):
missing.append(token)
return missing
def _looks_like_path(self, token: str) -> bool:
if "/" in token or token.startswith("."):
return True
_, ext = os.path.splitext(token)
return ext.lower() in self.PATH_LIKE_EXTS
def _run_command(self, cmd: List[str], cwd: str) -> Dict[str, Any]:
command_str = shlex.join(cmd)
start = time.time()
try:
completed = subprocess.run(
cmd,
cwd=cwd,
text=True,
capture_output=True,
timeout=self.timeout_sec
)
duration = time.time() - start
return {
"command": command_str,
"exit_code": completed.returncode,
"stdout": self._truncate(completed.stdout),
"stderr": self._truncate(completed.stderr),
"duration_sec": round(duration, 3),
"timed_out": False
}
except subprocess.TimeoutExpired:
duration = time.time() - start
return {
"command": command_str,
"exit_code": None,
"stdout": "",
"stderr": "",
"duration_sec": round(duration, 3),
"timed_out": True,
"error": f"Timeout after {self.timeout_sec}s"
}
except FileNotFoundError as e:
duration = time.time() - start
return {
"command": command_str,
"exit_code": None,
"stdout": "",
"stderr": str(e),
"duration_sec": round(duration, 3),
"timed_out": False,
"error": str(e)
}
def _truncate(self, text: str) -> str:
if not text:
return ""
if len(text) <= self.max_output_chars:
return text
return text[: self.max_output_chars] + "...[truncated]"
def _pick_error(self, *results: Dict[str, Any]) -> Optional[str]:
for result in results:
if not result:
continue
stderr = result.get("stderr") or ""
stdout = result.get("stdout") or ""
if stderr.strip():
return stderr.strip()
if stdout.strip():
return stdout.strip()
if result.get("error"):
return str(result.get("error"))
return None
# ==========================================================================
# Skill content loader
# ==========================================================================
class SkillLoader:
"""Load SKILL.md, scripts, and reference files for a skill."""
IGNORED_DIRECTORY_NAMES = {
"__pycache__",
"env",
"node_modules",
"venv",
}
MAX_WALK_ENTRIES = 10_000
REFERENCE_ALLOWED_EXTS = {
".md",
".txt",
".json",
".yaml",
".yml",
".ini",
".toml",
".cfg",
".csv",
".tsv",
}
@staticmethod
def _record_scan_issue(
scan_issues: Optional[List[InjectionScanIssue]], file: str, reason: str
) -> None:
if scan_issues is not None:
issue = InjectionScanIssue(file=file, reason=reason)
if issue not in scan_issues:
scan_issues.append(issue)
@staticmethod
def _walk_visible_files(
skill_dir: str,
scan_issues: Optional[List[InjectionScanIssue]] = None,
) -> Iterator[Tuple[str, List[str]]]:
entries_seen = 0
def record_walk_error(error: OSError) -> None:
failed_path = error.filename or skill_dir
rel_path = os.path.relpath(failed_path, skill_dir).replace(os.sep, "/")
SkillLoader._record_scan_issue(
scan_issues,
rel_path,
f"Failed to scan directory: {error}",
)
for root, dirs, files in os.walk(skill_dir, onerror=record_walk_error):
entries_seen += len(dirs) + len(files)
if entries_seen > SkillLoader.MAX_WALK_ENTRIES:
rel_root = os.path.relpath(root, skill_dir).replace(os.sep, "/")
SkillLoader._record_scan_issue(
scan_issues,
rel_root,
(
f"Directory entry limit {SkillLoader.MAX_WALK_ENTRIES} "
"exceeded; remaining files were not loaded or scanned."
),
)
return
dirs[:] = sorted(
directory
for directory in dirs
if not directory.startswith(".")
and directory.lower() not in SkillLoader.IGNORED_DIRECTORY_NAMES
)
visible_files = sorted(
filename for filename in files if not filename.startswith(".")
)
yield root, visible_files
@staticmethod
def _walk_and_load(
skill_dir: str,
max_files: int,
max_chars: int,
root_filter: Callable[[str], bool],
file_filter: Callable[[str], bool],
skip_skill_md: bool,
scan_issues: Optional[List[InjectionScanIssue]] = None,
) -> List[Dict[str, str]]:
items: List[Dict[str, str]] = []
for root, files in SkillLoader._walk_visible_files(skill_dir, scan_issues):
if not root_filter(root):
continue
for filename in files:
if skip_skill_md and filename.lower() == "skill.md":
continue
if not file_filter(filename):
continue
filepath = os.path.join(root, filename)
rel_path = os.path.relpath(filepath, skill_dir).replace(os.sep, "/")
if len(items) >= max_files:
SkillLoader._record_scan_issue(
scan_issues,
rel_path,
(
f"File limit {max_files} reached; this file and any "
"remaining eligible files were not loaded or scanned."
),
)
return items
try:
with open(filepath, "r", encoding="utf-8") as f:
content = f.read(max_chars + 1)
if len(content) > max_chars:
SkillLoader._record_scan_issue(
scan_issues,
rel_path,
f"Content was truncated at {max_chars} characters.",
)
content = content[:max_chars]
items.append({"path": rel_path, "content": content})
except (OSError, UnicodeError) as e:
logger.warning("Skip %s: %s", filepath, e)
SkillLoader._record_scan_issue(
scan_issues, rel_path, f"Failed to read content: {e}"
)
return items
@staticmethod
def load_skill_md(
skill_dir: str,
max_chars: int = 12000,
scan_issues: Optional[List[InjectionScanIssue]] = None,
) -> Optional[str]:
"""Load SKILL.md content with optional truncation."""
path = SkillLoader._find_file(
skill_dir, "skill.md", scan_issues=scan_issues
)
if not path:
logger.warning("SKILL.md not found in %s", skill_dir)
SkillLoader._record_scan_issue(
scan_issues, "SKILL.md", "SKILL.md was not found."
)
return None
rel_path = os.path.relpath(path, skill_dir).replace(os.sep, "/")
try:
with open(path, "r", encoding="utf-8") as f:
content = f.read(max_chars + 1)
except (OSError, UnicodeError) as e:
logger.warning("Skip %s: %s", path, e)
SkillLoader._record_scan_issue(
scan_issues, rel_path, f"Failed to read content: {e}"
)
return None
if len(content) > max_chars:
SkillLoader._record_scan_issue(
scan_issues,
rel_path,
f"Content was truncated at {max_chars} characters.",
)
content = content[:max_chars] + "\n\n...[truncated]..."
return content
@staticmethod
def load_scripts(
skill_dir: str,
max_files: int = 5,
max_chars: int = 12000,
scan_issues: Optional[List[InjectionScanIssue]] = None,
) -> List[Dict[str, str]]:
"""Read up to five scripts, with 12,000 characters per file by default.
Keep ordinary utility scripts complete while bounding model input.
Files beyond either limit remain reported as incomplete scan coverage.
"""
return SkillLoader._walk_and_load(
skill_dir,
max_files=max_files,
max_chars=max_chars,
root_filter=lambda root: "scripts" in root.split(os.sep),
file_filter=lambda _filename: True,
skip_skill_md=False,
scan_issues=scan_issues,
)
@staticmethod
def load_references(
skill_dir: str,
max_files: int = 10,
max_chars: int = 4000,
scan_issues: Optional[List[InjectionScanIssue]] = None,
) -> List[Dict[str, str]]:
"""
Load non-script reference files for a skill.
This is intended for files other than SKILL.md and scripts/,
e.g. README.md, references/, assets/, etc.
"""
def file_filter(filename: str) -> bool:
ext = os.path.splitext(filename)[1].lower()
return (not ext) or (ext in SkillLoader.REFERENCE_ALLOWED_EXTS)
return SkillLoader._walk_and_load(
skill_dir,
max_files=max_files,
max_chars=max_chars,
root_filter=lambda root: "scripts" not in root.split(os.sep),
file_filter=file_filter,
skip_skill_md=True,
scan_issues=scan_issues,
)
@staticmethod
def _find_file(
directory: str,
filename: str,
scan_issues: Optional[List[InjectionScanIssue]] = None,
) -> Optional[str]:
"""Recursively find a file in directory (case-insensitive)."""
for root, files in SkillLoader._walk_visible_files(directory, scan_issues):
for f in files:
if f.lower() == filename.lower():
return os.path.join(root, f)
return None
# ==========================================================================
# Prompt builder
# ==========================================================================
class PromptBuilder:
"""Build prompts for skill evaluation."""
@staticmethod
def _format_file_items(items: List[Dict[str, str]], empty_message: str) -> str:
formatted: List[str] = []
for item in items:
if not isinstance(item, dict):
continue
path = item.get("path") or "[unknown path]"
content = item.get("content") or ""
formatted.append(f"# {path}\n{content}\n")
return "\n".join(formatted) if formatted else empty_message
@staticmethod
def _fence(body: str, nonce: str) -> str:
"""Wrap untrusted content so it cannot terminate its own section."""
begin = f"<<<BEGIN UNTRUSTED {nonce}>>>"
end = f"<<<END UNTRUSTED {nonce}>>>"
escaped_body = body.replace(
begin, f"<<<ESCAPED BEGIN UNTRUSTED {nonce}>>>"
).replace(
end, f"<<<ESCAPED END UNTRUSTED {nonce}>>>"
)
return f"{begin}\n{escaped_body}\n{end}"
@staticmethod
def build(skill: Skill, skill_md: Optional[str],
scripts: List[Dict[str, str]],
references: Optional[List[Dict[str, str]]] = None,
script_exec_results: Optional[List[ScriptExecutionResult]] = None,
injection_report: Optional[InjectionReport] = None) -> str:
"""Build the evaluation prompt for a given skill."""
nonce = secrets.token_hex(16)
metadata_block = PromptBuilder._fence(
"\n".join(
(
f"- Name: {skill.name}",
f"- Description: {skill.description or 'N/A'}",
f"- Category: {skill.category or 'N/A'}",
)
),
nonce,
)
skill_md_block = skill_md or "[SKILL.md not found]"
if references:
references_block = PromptBuilder._format_file_items(
references,
"[No references or additional assets found]",
)
else:
references_block = "[No references or additional assets found]"
if scripts:
scripts_block = PromptBuilder._format_file_items(
scripts,
"[No scripts found]",
)
else:
scripts_block = "[No scripts found]"
if script_exec_results is None:
script_exec_block = "[Scripts not executed]"
elif not script_exec_results:
script_exec_block = "[No runnable python scripts found]"
else:
script_exec_block = "\n".join(
PromptBuilder._format_exec_result(r)
for r in script_exec_results
)
skill_md_block = PromptBuilder._fence(skill_md_block, nonce)
references_block = PromptBuilder._fence(references_block, nonce)
scripts_block = PromptBuilder._fence(scripts_block, nonce)
# Script stdout/stderr is attacker-influenced (a script can print forged
# section headers) just like the blocks above, so it gets the same fence.
script_exec_block = PromptBuilder._fence(script_exec_block, nonce)
injection_block = (
injection_report.to_prompt_block()
if injection_report is not None
else "[Pre-screen not run]"
)
# The pre-screen quotes matched excerpts of skill content verbatim, so it
# carries the same risk of forged headers/imperatives as the raw content
# above and must be fenced too rather than left as free text next to the
# output-format instructions.
injection_block = PromptBuilder._fence(injection_block, nonce)
return SKILL_EVALUATION_PROMPT.format(
metadata_block=metadata_block,
injection_block=injection_block,
skill_md_block=skill_md_block,
references_block=references_block,
scripts_block=scripts_block,
script_exec_block=script_exec_block
)
@staticmethod
def _format_exec_result(result: ScriptExecutionResult) -> str:
base = f"- {result.path}: {result.status}"
if result.exit_code is not None:
base += f" (exit={result.exit_code})"
base += f" | cmd: {result.command}"
if result.note:
base += f" | note: {result.note}"
if result.error:
clean_error = " ".join(result.error.splitlines())
base += f" | error: {clean_error}"
return base
# ==========================================================================
# LLM client
# ==========================================================================
class LLMClient:
"""Thin wrapper around the OpenAI client for evaluation calls."""
def __init__(self, config: EvaluatorConfig):
self.client = OpenAI(api_key=config.api_key, base_url=config.base_url, timeout=120, max_retries=0)
self.json_mode = config.json_mode
self.model = config.model
self.temperature = config.temperature
@staticmethod
def _parse_json_response(raw_response: Optional[str]) -> Dict[str, Any]:
"""Parse a JSON response from the LLM with multi-stage fallback.
Handles empty responses, markdown-wrapped JSON, trailing commas,
single-quoted keys, and other common malformations from
OpenAI-compatible providers that do not strictly honour
``response_format``.
"""
# Stage 1: empty / None check
cleaned = (raw_response or "").strip()
if not cleaned:
raise ValueError("LLM returned an empty response")
# Stage 2: strip markdown code fences (```json ... ``` or ``` ... ```)
cleaned = re.sub(r"^```(?:json)?\s*\n?", "", cleaned)
cleaned = re.sub(r"\n?```\s*$", "", cleaned)
cleaned = cleaned.strip()
# Stage 3: try standard json.loads first (fast path)
try:
return json.loads(cleaned)
except json.JSONDecodeError:
pass
# Stage 4: attempt repair via json-repair
logger.warning(
"JSON parsing failed, attempting repair.",
)
try:
repaired = repair_json(cleaned, return_objects=False)
return json.loads(repaired)
except Exception as exc:
raise ValueError(
f"Unable to parse LLM response as JSON even after repair: {exc}"
) from exc
def evaluate(self, prompt: str) -> Dict[str, Any]:
"""Call the LLM with the given prompt and parse JSON response."""
messages = [
{
"role": "system",
"content": (
EVALUATION_SYSTEM_PROMPT
)
},
{"role": "user", "content": prompt}
]
try:
response = chat_completion(self.client, model=self.model, messages=messages,
json_mode=self.json_mode, temperature=self.temperature)
choice = response.choices[0]
if choice.finish_reason != "stop":
raise ValueError("Model did not complete evaluation; retry with a suitable output budget/model.")
raw_response = choice.message.content
return validate_evaluation(self._parse_json_response(raw_response))
except Exception as e:
logger.error("LLM call failed: %s", error_details(e)["message"])
raise
# ==========================================================================
# Core evaluator
# ==========================================================================
class SkillEvaluator:
"""
Main entry point for evaluating AI skills.
Typical usage:
config = EvaluatorConfig(api_key="your-key")
evaluator = SkillEvaluator(config)
# Single skill from URL
skill, err = Skill.from_url("https://github.com/.../skill", evaluator.downloader, config.cache_dir)
if err:
result = evaluator._create_error_result(err)
else:
result = evaluator.evaluate(skill)
# Single skill from local path
skill = Skill.from_path("/path/to/skill")
result = evaluator.evaluate(skill)
# Batch evaluation
skills = [skill1, skill2, skill3]
results = evaluator.evaluate_batch(skills)
"""
def __init__(self, config: EvaluatorConfig):
"""Initialize the evaluator with configuration."""
if not config.api_key:
raise ValueError("API key is required")
self.config = config
self.downloader = SkillDownloader(api_token=config.github_token)
self.loader = SkillLoader()
self.prompt_builder = PromptBuilder()
self.llm_client = LLMClient(config)
self.script_runner = ScriptRunner(
python_bin=config.script_python,
timeout_sec=config.script_timeout_sec,
max_runs=config.max_script_runs,
max_output_chars=config.max_script_output_chars
)
@staticmethod
def _injection_contents(
skill: Skill,
skill_md: Optional[str],
scripts: List[Dict[str, str]],
references: List[Dict[str, str]],
) -> List[InjectionContent]:
contents = [
InjectionContent("metadata/name", str(skill.name)),
InjectionContent(
"metadata/description", str(skill.description or "N/A")
),
InjectionContent("metadata/category", str(skill.category or "N/A")),
InjectionContent(
"SKILL.md", str(skill_md or "[SKILL.md not found]")
),
]
for items, is_script in ((references, False), (scripts, True)):
for item in items:
if not isinstance(item, dict):
continue
path = str(item.get("path") or "[unknown path]")
content = str(item.get("content") or "")
contents.append(
InjectionContent(path, content, is_script=is_script)
)
return contents
@staticmethod
def _load_with_scan_issues(
loader_method: Callable[..., Any],
skill_path: str,
scan_issues: List[InjectionScanIssue],
) -> Any:
"""Call current and legacy loaders without breaking method overrides."""
try:
parameters = inspect.signature(loader_method).parameters.values()
except (TypeError, ValueError):
return loader_method(skill_path)
supports_scan_issues = any(
parameter.name == "scan_issues"
or parameter.kind == inspect.Parameter.VAR_KEYWORD
for parameter in parameters
)
if supports_scan_issues:
return loader_method(skill_path, scan_issues=scan_issues)
return loader_method(skill_path)
def evaluate(self, skill: Skill) -> Dict[str, Any]:
"""
Evaluate a single skill.
Args:
skill: A Skill instance to evaluate.
Returns:
A dict containing the evaluation result.
"""
try:
# Load content
load_issues: List[InjectionScanIssue] = []
skill_md = self._load_with_scan_issues(
self.loader.load_skill_md, skill.path, load_issues
)
scripts = self._load_with_scan_issues(
self.loader.load_scripts, skill.path, load_issues
)
references = self._load_with_scan_issues(
self.loader.load_references, skill.path, load_issues
)
injection_report: Optional[InjectionReport] = None
if self.config.scan_injection:
loaded_contents = self._injection_contents(
skill, skill_md, scripts, references
)
injection_report = SkillInjectionScanner().scan_contents(
loaded_contents
)
if load_issues:
injection_report.complete = False
injection_report.scan_issues.extend(load_issues)
# Optional script execution
script_exec_results: Optional[List[ScriptExecutionResult]] = None
if self.config.run_scripts:
skip_reason = None
if skill.url:
skip_reason = (
"Skipped: remote third-party skill scripts are never "
"executed, even when run_scripts=True."
)
script_exec_results = self.script_runner.run_for_skill(
skill.path,
skip_reason=skip_reason,
)
# Build prompt
prompt = self.prompt_builder.build(
skill,
skill_md,
scripts,
references=references,
script_exec_results=script_exec_results,
injection_report=injection_report,
)
# Call LLM
result = self.llm_client.evaluate(prompt)
if self.config.include_script_results and script_exec_results is not None:
result["script_execution"] = [
r.to_dict() for r in script_exec_results
]
if injection_report is not None:
result["prompt_injection_scan"] = injection_report.to_dict()
return result
except Exception as e:
skill_name = getattr(skill, "name", "[unknown skill]")
details = error_details(e)
logger.error("Evaluation failed for %s: %s", skill_name, details["message"])
result = self._create_error_result(details["message"])
result["error_details"] = details
return result
def evaluate_batch(self, skills: List[Skill]) -> List[Dict[str, Any]]:
"""
Evaluate multiple skills in parallel.
Args:
skills: List of Skill objects.
Returns:
List of evaluation results in the same order as input.
"""
results: List[Optional[Dict[str, Any]]] = [None] * len(skills)
max_workers = max(1, int(self.config.max_workers))
max_in_flight = max_workers * 2
with ThreadPoolExecutor(max_workers=max_workers) as executor:
it = iter(enumerate(skills))
in_flight: set = set()
future_to_idx: Dict[Any, int] = {}
def _submit_next() -> bool:
try:
idx, skill = next(it)
except StopIteration:
return False
fut = executor.submit(self.evaluate, skill)
in_flight.add(fut)
future_to_idx[fut] = idx
return True
for _ in range(min(max_in_flight, len(skills))):
_submit_next()
with tqdm(total=len(skills), desc="Evaluating skills") as pbar:
try:
while in_flight:
done, pending = wait(in_flight, return_when=FIRST_COMPLETED)
in_flight = set(pending)
for fut in done:
idx = future_to_idx.pop(fut)
results[idx] = fut.result()
pbar.update(1)
if len(in_flight) < max_in_flight:
_submit_next()
except KeyboardInterrupt:
for fut in in_flight:
fut.cancel()
executor.shutdown(wait=False, cancel_futures=True)
raise
return [r if r is not None else self._create_error_result("Missing result") for r in results]
def evaluate_from_url(self, url: str, **kwargs) -> Dict[str, Any]:
"""Convenience helper: create and evaluate a skill from a URL."""
skill, err = Skill.from_url(
url,
self.downloader,
self.config.cache_dir,
**kwargs
)
if err:
return self._create_error_result(err)
return self.evaluate(skill)
def evaluate_from_path(self, path: str, **kwargs) -> Dict[str, Any]:
"""Convenience helper: create and evaluate a skill from a local path."""
skill, err = Skill.from_path(
path, **kwargs
)
if err:
return self._create_error_result(err)
return self.evaluate(skill)
@staticmethod
def _create_error_result(error_msg: str) -> Dict[str, Any]:
"""Create a default error-shaped evaluation result."""
error_item = {"level": "Poor", "reason": f"Evaluation failed: {error_msg}"}
return {
"error": error_msg,
"safety": error_item,
"completeness": error_item,
"executability": error_item,
"maintainability": error_item,
"cost_awareness": error_item
}
# ==========================================================================
# CLI entry point
# ==========================================================================
if __name__ == '__main__':
"""Command line entry point for batch evaluation from JSONL."""
import argparse
parser = argparse.ArgumentParser(description='Evaluate AI Agent Skills')
parser.add_argument('--input', required=True, help='Input JSONL file')
parser.add_argument('--output', required=True, help='Output JSONL file')
parser.add_argument('--api-key', help='OpenAI API key')
parser.add_argument('--base-url', help='OpenAI API base URL')
parser.add_argument('--github-token', help='GitHub token for downloading private repos or avoiding rate limits')
parser.add_argument('--model', default='gpt-4o', help='Model name')
parser.add_argument('--max-workers', type=int, default=5, help='Max workers')
parser.add_argument('--cache-dir', default='skill_downloads', help='Cache directory')
parser.add_argument('--limit', type=int, default=-1, help='Limit number of input records to process (0 = all)')
parser.add_argument('--run-scripts', action='store_true',
help='Execute python scripts under scripts/')
parser.add_argument('--script-timeout', type=int, default=8,
help='Timeout seconds per script run')
parser.add_argument('--max-script-runs', type=int, default=5,
help='Max python scripts to execute per skill')
parser.add_argument('--script-python', default='python',
help='Python executable for running scripts')
parser.add_argument('--include-script-results', action='store_true',
help='Attach script execution results to evaluation output')
parser.add_argument('--max-script-output-chars', type=int, default=400,
help='Max chars of script stdout/stderr to keep')
args = parser.parse_args()
def _load_records(jsonl_path: str) -> List[Dict[str, Any]]:
with open(jsonl_path, 'r', encoding='utf-8') as f:
return [json.loads(line) for line in f if line.strip()]
def _evaluate_records_streaming(
records: List[Dict[str, Any]],
evaluator: 'SkillEvaluator',
config: EvaluatorConfig,
) -> List[Dict[str, Any]]:
results: List[Optional[Dict[str, Any]]] = [None] * len(records)
max_workers = max(1, int(config.max_workers))
max_in_flight = max_workers * 2
def _run_one(idx: int, rec: Dict[str, Any]) -> Dict[str, Any]:
if 'skill_url' in rec:
skill, err = Skill.from_url(
rec['skill_url'],
evaluator.downloader,
config.cache_dir,
name=rec.get('skill_name'),
description=rec.get('skill_description'),
category=rec.get('category'),
)
if err:
return evaluator._create_error_result(err)
return evaluator.evaluate(skill)
if 'skill_path' in rec:
skill, err = Skill.from_path(
rec['skill_path'],
name=rec.get('skill_name'),
description=rec.get('skill_description'),
category=rec.get('category'),
)
if err:
return evaluator._create_error_result(err)
return evaluator.evaluate(skill)
return evaluator._create_error_result("Record must have 'skill_url' or 'skill_path'")
with ThreadPoolExecutor(max_workers=max_workers) as executor:
it = iter(enumerate(records))
in_flight: set = set()
future_to_idx: Dict[Any, int] = {}
def _submit_next() -> bool:
try:
idx, rec = next(it)
except StopIteration:
return False
fut = executor.submit(_run_one, idx, rec)
in_flight.add(fut)
future_to_idx[fut] = idx
return True
for _ in range(min(max_in_flight, len(records))):
_submit_next()
with tqdm(total=len(records), desc="Downloading & evaluating") as pbar:
try:
while in_flight:
done, pending = wait(in_flight, return_when=FIRST_COMPLETED)
in_flight = set(pending)
for fut in done:
idx = future_to_idx.pop(fut)
results[idx] = fut.result()
pbar.update(1)
if len(in_flight) < max_in_flight:
_submit_next()
except KeyboardInterrupt:
for fut in in_flight:
fut.cancel()
executor.shutdown(wait=False, cancel_futures=True)
raise
return [r if r is not None else evaluator._create_error_result("Missing result") for r in results]
def _write_outputs(records: List[Dict[str, Any]], output_jsonl_path: str) -> str:
with open(output_jsonl_path, 'w', encoding='utf-8') as f:
for rec in records:
f.write(json.dumps(rec, ensure_ascii=False) + '\n')
json_path = output_jsonl_path.replace('.jsonl', '.json')
with open(json_path, 'w', encoding='utf-8') as f:
json.dump({str(i): rec for i, rec in enumerate(records)},
f, ensure_ascii=False, indent=2)
return json_path
config = EvaluatorConfig(
api_key=args.api_key or os.getenv('API_KEY'),
base_url=args.base_url or os.getenv('BASE_URL'),
model=args.model,
max_workers=args.max_workers,
cache_dir=args.cache_dir,
run_scripts=args.run_scripts,
script_timeout_sec=args.script_timeout,
max_script_runs=args.max_script_runs,
script_python=args.script_python,
include_script_results=args.include_script_results,
max_script_output_chars=args.max_script_output_chars,
github_token=args.github_token or os.getenv('GITHUB_TOKEN'),
)
evaluator = SkillEvaluator(config)
records = _load_records(args.input)
if args.limit and args.limit > 0:
records = records[: args.limit]
results = _evaluate_records_streaming(records, evaluator, config)
for rec, result in zip(records, results):
rec['evaluation'] = result
json_path = _write_outputs(records, args.output)
print(f"✓ Evaluated {len(results)} skills")
print(f"✓ Results saved to {args.output} and {json_path}")