Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.
Python agent-evaluationagent-pluginsagent-securityagent-skillsagentic-aibenchmarkci-templatesclaude-codecodexevaluateevaluationharnesspluginplugin-evaluationsecurity-scannerskill-evalskill-evaluationskill-evaluatorskill-gap-analysisskills
feat(tier1): score tier 1 quality checks via a configurable YAML file (#213)
Weston Platter830fabe63889
diff --git a/src/skillevaluator/cli.py b/src/skillevaluator/cli.pyindex b906a7bb..fbdb766a 100644--- a/src/skillevaluator/cli.py+++ b/src/skillevaluator/cli.py@@ -1587,6 +1587,14 @@ def _print_run_banner(target_path: Path, content_type: str, profile: str | None)help_group=_TIER1_GROUP,help="Minimum quality score (0-100) required to pass when the 'quality' check runs.",)+ "--quality-config",+ type=click.Path(exists=True, dir_okay=False, path_type=Path),+ default=None,+ cls=GroupedOption,+ help_group=_TIER1_GROUP,+ help="Override the packaged quality-default.yaml with a custom YAML file.",+)@click.option("--profile",default=None,@@ -1838,6 +1846,7 @@ def validate(llm: bool,llm_verify: bool,min_score: int,+ quality_config: Path | None,profile: str | None,external: bool,policy_path: Path | None,@@ -2055,6 +2064,7 @@ def validate(use_llm=llm,llm_verify=llm_verify,min_score=min_score,+ quality_config=quality_config,previous_version=previous_version,policy=policy,content_type=resolved_type,@@ -2312,12 +2322,24 @@ validate.help_group_descriptions = {@cli.command("quality-check")@_target_argument@click.option("--min-score", type=int, default=70, show_default=True)+ "--quality-config",+ type=click.Path(exists=True, dir_okay=False, path_type=Path),+ default=None,+ help="Override the packaged quality-default.yaml with a custom YAML file.",+)@_report_options-def quality_check(target_path: Path, min_score: int, report_formats: tuple[str, ...], output_dir: Path) -> None:+def quality_check(+ target_path: Path,+ min_score: int,+ quality_config: Path | None,+ report_formats: tuple[str, ...],+ output_dir: Path,+) -> None:"""Score skill quality across correctness, discoverability, reliability, and efficiency."""output_dir = _resolve_file_report_output_location(target_path.resolve(), output_dir, report_formats)if not emit_reports(- run_quality_check(target_path, min_score=min_score),+ run_quality_check(target_path, min_score=min_score, quality_config=quality_config),report_formats=report_formats,output_dir=output_dir,basename=report_basename("quality"),diff --git a/src/skillevaluator/config/quality-default.yaml b/src/skillevaluator/config/quality-default.yamlnew file mode 100644index 00000000..b0ceded7--- /dev/null+++ b/src/skillevaluator/config/quality-default.yaml@@ -0,0 +1,4 @@+# Default deterministic quality-check configuration.+limitations:+ required: true+ heading: "## Limitations"diff --git a/src/skillevaluator/quality_config.py b/src/skillevaluator/quality_config.pynew file mode 100644index 00000000..cdf42a0c--- /dev/null+++ b/src/skillevaluator/quality_config.py@@ -0,0 +1,30 @@+"""Configuration for deterministic quality scoring."""+from __future__ import annotations++from importlib.resources import files+from pathlib import Path+from typing import Any++import yaml++DEFAULT_QUALITY_CONFIG = files("skillevaluator.config").joinpath("quality-default.yaml")+++class QualityConfigError(ValueError):+ """Raised when a quality configuration is invalid."""+++def load_quality_config(path: Path | None = None) -> dict[str, Any]:+ source = path.read_text(encoding="utf-8") if path else DEFAULT_QUALITY_CONFIG.read_text(encoding="utf-8")+ try:+ raw = yaml.safe_load(source)+ except yaml.YAMLError as exc:+ raise QualityConfigError(f"{path or 'packaged quality-default.yaml'}: invalid YAML: {exc}") from exc+ if not isinstance(raw, dict) or not isinstance(raw.get("limitations", {}), dict):+ raise QualityConfigError("quality config: top-level and limitations must be mappings")+ limitations = raw["limitations"]+ required = limitations.get("required", True)+ heading = limitations.get("heading", "## Limitations")+ if not isinstance(required, bool) or not isinstance(heading, str) or not heading.strip():+ raise QualityConfigError("quality config: limitations.required must be boolean and heading non-empty")+ return {"limitations": {"required": required, "heading": heading}}diff --git a/src/skillevaluator/tier1/commands.py b/src/skillevaluator/tier1/commands.pyindex ddc1b77b..68d4871d 100644--- a/src/skillevaluator/tier1/commands.py+++ b/src/skillevaluator/tier1/commands.py@@ -177,6 +177,7 @@ def run_validation(use_llm: bool = False,llm_verify: bool = False,min_score: int = 70,+ quality_config: Path | None = None,previous_version: str | None = None,policy: ValidationPolicy | None = None,content_type: str | None = None,@@ -246,7 +247,7 @@ def run_validation(return [_run_result("unicode", v.name, v.description, v.validate)]def _quality_results() -> list[ValidationResult]:- v = QualityScoreValidator(min_score=min_score)+ v = QualityScoreValidator(min_score=min_score, quality_config=quality_config)return [_run_result("quality", v.name, v.description, v.validate)]def _lint_results() -> list[ValidationResult]:@@ -330,8 +331,10 @@ def run_validation(return results-def run_quality_check(target_path: Path, *, min_score: int = 70) -> list[ValidationResult]:- validator = QualityScoreValidator(min_score=min_score)+def run_quality_check(+ target_path: Path, *, min_score: int = 70, quality_config: Path | None = None+) -> list[ValidationResult]:+ validator = QualityScoreValidator(min_score=min_score, quality_config=quality_config)return [_as_result(validator.name,diff --git a/src/skillevaluator/validators/quality_score.py b/src/skillevaluator/validators/quality_score.pyindex 30a89724..58d2947a 100644--- a/src/skillevaluator/validators/quality_score.py+++ b/src/skillevaluator/validators/quality_score.py@@ -36,6 +36,7 @@ from skillevaluator.constants import (from skillevaluator.logging_config import get_loggerfrom skillevaluator.models.quality import QualityScoreResultfrom skillevaluator.models.result import Finding, Severity, ValidationResult+from skillevaluator.quality_config import load_quality_configfrom skillevaluator.models.skill import XML_TAG_REfrom skillevaluator.validators.base import ValidatorBasefrom skillevaluator.validators.frontmatter_parser import FRONTMATTER_PATTERN@@ -129,8 +130,9 @@ class QualityScoreValidator(ValidatorBase):existing SkillEvaluator SchemaValidator checks."""- def __init__(self, min_score: int = 70) -> None:+ def __init__(self, min_score: int = 70, quality_config: Path | None = None) -> None:self.min_score = min_score+ self.quality_config = load_quality_config(quality_config)@propertydef name(self) -> str:@@ -676,7 +678,8 @@ class QualityScoreValidator(ValidatorBase):"No prerequisites/requirements documented","Document dependencies, API keys, or setup needed",)- if "## Limitations" not in content:+ limitations = self.quality_config["limitations"]+ if limitations["required"] and limitations["heading"] not in content:dim.deduct(5,"info",diff --git a/tests/golden/cli_surface.json b/tests/golden/cli_surface.jsonindex 36f93bf9..8d68336d 100644--- a/tests/golden/cli_surface.json+++ b/tests/golden/cli_surface.json@@ -980,6 +980,14 @@"param_type": "option","type": "integer"},+ {+ "name": "quality_config",+ "opts": [+ "--quality-config"+ ],+ "param_type": "option",+ "type": "file"+ },{"choices": ["cli",@@ -1364,6 +1372,14 @@"param_type": "option","type": "integer"},+ {+ "name": "quality_config",+ "opts": [+ "--quality-config"+ ],+ "param_type": "option",+ "type": "file"+ },{"choices": ["cli",@@ -3145,6 +3161,14 @@"param_type": "option","type": "integer"},+ {+ "name": "quality_config",+ "opts": [+ "--quality-config"+ ],+ "param_type": "option",+ "type": "file"+ },{"name": "profile","opts": [diff --git a/tests/test_quality_config.py b/tests/test_quality_config.pynew file mode 100644index 00000000..ab1f6965--- /dev/null+++ b/tests/test_quality_config.py@@ -0,0 +1,27 @@+from pathlib import Path++from skillevaluator.quality_config import load_quality_config+++def test_packaged_default_quality_config_is_available() -> None:+ config = load_quality_config()++ assert config["limitations"] == {+ "required": True,+ "heading": "## Limitations",+ }+++def test_quality_config_can_be_overridden(tmp_path: Path) -> None:+ config_path = tmp_path / "quality.yaml"+ config_path.write_text(+ 'limitations:\n required: true\n heading: "## Failure Constellations"\n',+ encoding="utf-8",+ )++ config = load_quality_config(config_path)++ assert config["limitations"] == {+ "required": True,+ "heading": "## Failure Constellations",+ }