Skip to content

Create, Evaluate, and Connect AI Skills

update evaluator

Xu Haominga0c54b8957c2
diff --git a/skillnet-ai/src/skillnet_ai/evaluator.py b/skillnet-ai/src/skillnet_ai/evaluator.py
index 37b0301..c16205e 100644
--- a/skillnet-ai/src/skillnet_ai/evaluator.py
+++ b/skillnet-ai/src/skillnet_ai/evaluator.py
@@ -7,7 +7,7 @@ import subprocess
import time
from concurrent.futures import ThreadPoolExecutor, as_completed
from dataclasses import dataclass
-from typing import Dict, Any, List, Optional
+from typing import Dict, Any, List, Optional, Tuple
from openai import OpenAI
from tqdm import tqdm
@@ -49,41 +49,71 @@ class Skill:
url: Optional[str] = None # Original URL (when created from URL)
@classmethod
- def from_url(cls, url: str, downloader: 'SkillDownloader',
- cache_dir: str, **kwargs) -> 'Skill':
- """Create a Skill from a GitHub URL."""
+ def from_url(
+ cls,
+ url: str,
+ downloader: 'SkillDownloader',
+ cache_dir: str,
+ max_retries: int = 3,
+ retry_delay: float = 2.0,
+ **kwargs
+ ) -> Tuple[Optional['Skill'], Optional[str]]:
+ """
+ Create a Skill from a GitHub URL.
+ 下载失败时会重试,最终失败返回 (None, error_msg) 而非抛出异常。
+
+ Returns:
+ (Skill, None) 成功;(None, error_msg) 失败。
+ """
normalized_url = cls._normalize_url(url)
if not normalized_url:
- raise ValueError(f"Invalid GitHub URL: {url}")
- # Download to local cache
- local_path = downloader.download(normalized_url, target_dir=cache_dir)
+ return None, f"Invalid GitHub URL: {url}"
+ # Download to local cache (with retries in evaluator)
+ local_path = None
+ for attempt in range(max_retries):
+ local_path = downloader.download(normalized_url, target_dir=cache_dir)
+ if local_path:
+ break
+ if attempt < max_retries - 1:
+ logger.warning(
+ "Download failed (attempt %d/%d). Retrying in %.1fs...",
+ attempt + 1, max_retries, retry_delay,
+ )
+ time.sleep(retry_delay)
if not local_path:
- raise RuntimeError(f"Failed to download: {url}")
+ return None, f"Failed to download after {max_retries} retries: {url}"
# Derive skill name from URL if not provided
name = kwargs.get('name') or normalized_url.rstrip('/').split('/')[-1]
-
+
return cls(
path=local_path,
name=name,
url=url,
description=kwargs.get('description'),
category=kwargs.get('category')
- )
+ ), None
@classmethod
def from_path(cls, path: str, **kwargs) -> 'Skill':
- """Create a Skill from a local directory path."""
+ """Create a Skill from a local directory path.
+
+ Returns:
+ (Skill, None) on success; (None, error_msg) on failure.
+ """
abs_path = os.path.abspath(path)
if not os.path.isdir(abs_path):
- raise ValueError(f"Invalid skill path: {path}")
-
+ return None, f"Invalid skill path: {path}"
+
name = kwargs.get('name') or os.path.basename(abs_path)
-
- return cls(
- path=abs_path,
- name=name,
- description=kwargs.get('description'),
- category=kwargs.get('category')
+
+ return (
+ cls(
+ path=abs_path,
+ name=name,
+ description=kwargs.get('description'),
+ category=kwargs.get('category')
+ ),
+ None,
)
@staticmethod
@@ -276,11 +306,21 @@ class ScriptRunner:
if not candidates:
return None
- for cmd in candidates:
+ # Prefer runnable commands:
+ # - Commands containing placeholder tokens like "<file>", "[options]", "{path}" are
+ # often documentation examples and not directly runnable.
+ # - If placeholders exist, prefer a help-style command to at least verify the script
+ # starts up, otherwise fall back to compilation-only.
+ runnable = [cmd for cmd in candidates if not self._has_placeholder_tokens(cmd)]
+ for cmd in runnable:
if not self._is_help_command(cmd):
return cmd
- return candidates[0]
+ for cmd in candidates:
+ if self._is_help_command(cmd):
+ return cmd
+
+ return None
def _extract_usage_lines(self, script_path: str,
script_name: str) -> List[str]:
@@ -362,6 +402,24 @@ class ScriptRunner:
return True
return False
+ def _has_placeholder_tokens(self, cmd: List[str]) -> bool:
+ for token in cmd:
+ if not token or token.startswith("-"):
+ continue
+ if token == self.python_bin:
+ continue
+ if self._is_placeholder_token(token):
+ return True
+ return False
+
+ @staticmethod
+ def _is_placeholder_token(token: str) -> bool:
+ # Common usage placeholders: <...>, [...], {...}
+ if any(ch in token for ch in ("<", ">", "[", "]", "{", "}")):
+ return True
+ lowered = token.strip().lower()
+ return lowered in {"options", "[options]", "<options>", "{options}"}
+
def _detect_missing_inputs(self, cmd: List[str], cwd: str) -> List[str]:
missing: List[str] = []
for token in cmd:
@@ -369,7 +427,7 @@ class ScriptRunner:
continue
if token == self.python_bin:
continue
- if "<" in token or ">" in token:
+ if self._is_placeholder_token(token):
continue
if not self._looks_like_path(token):
continue
@@ -599,16 +657,26 @@ class PromptBuilder:
skill_md_block = skill_md or "[SKILL.md not found]"
if references:
- references_block = "\n".join(
- f"# {r['path']}\n{r['content']}\n" for r in references
- )
+ formatted_refs: List[str] = []
+ for r in references:
+ if not isinstance(r, dict):
+ continue
+ path = r.get("path") or "[unknown path]"
+ content = r.get("content") or ""
+ formatted_refs.append(f"# {path}\n{content}\n")
+ references_block = "\n".join(formatted_refs) if formatted_refs else "[No references or additional assets found]"
else:
references_block = "[No references or additional assets found]"
if scripts:
- scripts_block = "\n".join([
- f"# {s['path']}\n{s['content']}\n" for s in scripts
- ])
+ formatted_scripts: List[str] = []
+ for s in scripts:
+ if not isinstance(s, dict):
+ continue
+ path = s.get("path") or "[unknown path]"
+ content = s.get("content") or ""
+ formatted_scripts.append(f"# {path}\n{content}\n")
+ scripts_block = "\n".join(formatted_scripts) if formatted_scripts else "[No scripts found]"
else:
scripts_block = "[No scripts found]"
@@ -701,8 +769,11 @@ class SkillEvaluator:
evaluator = SkillEvaluator(config)
# Single skill from URL
- skill = Skill.from_url("https://github.com/.../skill", evaluator.downloader, config.cache_dir)
- result = evaluator.evaluate(skill)
+ skill, err = Skill.from_url("https://github.com/.../skill", evaluator.downloader, config.cache_dir)
+ if err:
+ result = evaluator._create_error_result(err)
+ else:
+ result = evaluator.evaluate(skill)
# Single skill from local path
skill = Skill.from_path("/path/to/skill")
@@ -769,7 +840,8 @@ class SkillEvaluator:
return result
except Exception as e:
- logger.error(f"Evaluation failed for {skill.name}: {e}")
+ skill_name = getattr(skill, "name", "[unknown skill]")
+ logger.exception("Evaluation failed for %s: %s", skill_name, e)
return self._create_error_result(str(e))
def evaluate_batch(self, skills: List[Skill]) -> List[Dict[str, Any]]:
@@ -800,12 +872,18 @@ class SkillEvaluator:
def evaluate_from_url(self, url: str, **kwargs) -> Dict[str, Any]:
"""Convenience helper: create and evaluate a skill from a URL."""
- skill = Skill.from_url(url, self.downloader, self.config.cache_dir, **kwargs)
+ skill, err = Skill.from_url(
+ url, self.downloader, self.config.cache_dir, **kwargs
+ )
+ if err:
+ return self._create_error_result(err)
return self.evaluate(skill)
def evaluate_from_path(self, path: str, **kwargs) -> Dict[str, Any]:
"""Convenience helper: create and evaluate a skill from a local path."""
- skill = Skill.from_path(path, **kwargs)
+ skill, err = Skill.from_path(path, **kwargs)
+ if err:
+ return self._create_error_result(err)
return self.evaluate(skill)
@staticmethod
@@ -872,9 +950,10 @@ if __name__ == '__main__':
evaluator = SkillEvaluator(config)
skills = []
- for rec in records:
+ download_errors = {} # idx -> error_msg
+ for idx, rec in enumerate(records):
if 'skill_url' in rec:
- skill = Skill.from_url(
+ skill, err = Skill.from_url(
rec['skill_url'],
evaluator.downloader,
config.cache_dir,
@@ -882,18 +961,39 @@ if __name__ == '__main__':
description=rec.get('skill_description'),
category=rec.get('category')
)
+ if err:
+ download_errors[idx] = err
+ skills.append(None)
+ else:
+ skills.append(skill)
elif 'skill_path' in rec:
- skill = Skill.from_path(
+ skill, err = Skill.from_path(
rec['skill_path'],
name=rec.get('skill_name'),
description=rec.get('skill_description'),
category=rec.get('category')
)
+ if err:
+ download_errors[idx] = err
+ skills.append(None)
+ else:
+ skills.append(skill)
else:
raise ValueError("Record must have 'skill_url' or 'skill_path'")
- skills.append(skill)
-
- results = evaluator.evaluate_batch(skills)
+
+ # Use error results for failed downloads, and evaluate the rest in parallel
+ skills_to_eval = [(idx, s) for idx, s in enumerate(skills) if s is not None]
+ idx_to_result: Dict[int, Dict[str, Any]] = {}
+ if skills_to_eval:
+ indices, valid_skills = zip(*skills_to_eval)
+ batch_results = evaluator.evaluate_batch(list(valid_skills))
+ idx_to_result = dict(zip(indices, batch_results))
+ results = [
+ evaluator._create_error_result(download_errors[idx])
+ if idx in download_errors
+ else idx_to_result[idx]
+ for idx in range(len(records))
+ ]
for rec, result in zip(records, results):
rec['evaluation'] = result
diff --git a/skillnet-ai/src/skillnet_ai/prompts.py b/skillnet-ai/src/skillnet_ai/prompts.py
index 806a77b..e33d5ff 100644
--- a/skillnet-ai/src/skillnet_ai/prompts.py
+++ b/skillnet-ai/src/skillnet_ai/prompts.py
@@ -193,6 +193,7 @@ Evaluation dimensions and how to judge them (apply these rules even if the overa
- Concrete actions and artifacts (commands, files, parameters); minimal ambiguity.
- Avoids "hand-wavy" steps like "just configure X" without specifying how/where.
- If script execution results are provided, successful runs support a higher rating.
+ - **Instruction-only skills**: When the skill is designed to be executed purely through text instructions (e.g., guidelines, policies, brainstorming, design workflows) and does NOT require code execution, the absence of runnable scripts is acceptable. If SKILL.md provides clear, actionable guidance that an agent can follow using typical LLM tools (read files, apply guidelines, reason about content), rate executability as Good.
Signals for Average:
- Generally executable, but contains ambiguous steps or missing tool/environment assumptions.
Signals for Poor:
@@ -200,6 +201,8 @@ Evaluation dimensions and how to judge them (apply these rules even if the overa
- If script execution results show failures/timeouts/missing dependencies, reduce the rating accordingly.
- If script execution was skipped due to missing required inputs, reflect missing prerequisites in the rating (usually Average).
Additional guidance for Executability:
+ - **Do NOT rate Poor solely because "No runnable python scripts found"**. Many skills (security guidelines, ideation, policies, design workflows) are instruction-only: the agent reads SKILL.md and follows the guidance with typical tools. For such skills, if the instructions are clear and actionable, executability should be Good.
+ - If script execution fails due to an obvious documentation placeholder in an example command (e.g., tokens like "[options]", "<file>", "<pattern>", "{{path}}") or an argument parsing error caused by such placeholders, do NOT automatically set executability to Poor. Prefer Average and explain that the script likely needs real inputs or a concrete runnable example; only use Poor if there is additional evidence the workflow is not realistically executable.
- If you detect any CRITICAL CORRECTNESS ERROR in a core formula, algorithm, or code snippet (e.g., Python using "^" for exponentiation or other language-level mistakes that would produce wrong results or runtime failures), executability MUST be "Poor".
- If allowed_tools grants broader permissions than what the Skill clearly needs (e.g., allows "bash" or other powerful tools but the described workflow and examples do not require them), reduce executability by at least one level due to environment/permission ambiguity.
- When reading formulas and code snippets, audit them line-by-line in the context of their target language and typical runtime environment; if you find subtle traps or inconsistencies that would mislead an implementer or cause incorrect behavior, choose a lower (more conservative) executability rating.
@@ -284,7 +287,16 @@ Example 3: Well-scoped document summarizer Skill (mostly solid)
- cost_awareness: "Good"
- Reason: The Skill explicitly caps document size and describes a strategy (chunking) that avoids unbounded compute.
-Example 4: Overpowered deployment cleaner Skill (risky but technically executable)
+Example 4: Instruction-only guideline Skill (e.g., security standards)
+- Observations:
+ - SKILL.md provides guidance on how to handle global security when working on code. It points to an external standards document for details.
+ - There are no runnable Python scripts; script execution shows "No runnable python scripts found".
+ - The workflow is: agent reads SKILL.md, loads the referenced document, and applies the guidelines when editing code.
+- Expected ratings:
+ - executability: "Good"
+ - Reason: This is an instruction-only skill. The agent can execute the workflow by reading SKILL.md and the referenced file, then applying the guidelines with typical LLM tools. The absence of runnable scripts is acceptable because the skill does not require code execution.
+
+Example 5: Overpowered deployment cleaner Skill (risky but technically executable)
- Observations:
- SKILL.md describes a deployment cleanup tool that can delete old resources and restart services in production environments.
- It grants broad permissions via allowed_tools (e.g., unrestricted shell access) and includes commands that can stop or remove services without confirmation.