Skip to content

Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.

65.4 KB
Raw Blame
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0

"""Tests for schema validator.

Based on SkillEvaluator HOW_TO_CONTRIBUTE_SKILLS.md specification.
"""

from pathlib import Path

import pytest

from skillevaluator.models.result import Severity
from skillevaluator.models.skill import SkillMetadata
from skillevaluator.validators.schema import SchemaValidator


class TestSchemaValidator:
    """Test cases for SchemaValidator based on SkillEvaluator spec."""

    @pytest.mark.parametrize("key", ["123", "true", "false", "null", "2026-01-01"])
    @pytest.mark.parametrize("nested", [False, True])
    def test_non_string_key_returns_validation_error(self, sample_skill_dir, key, nested):
        manifest = sample_skill_dir / "SKILL.md"
        content = manifest.read_text()
        marker = "metadata:\n" if nested else "---\n"
        indentation = "  " if nested else ""
        manifest.write_text(content.replace(marker, f"{marker}{indentation}{key}: extra value\n", 1))

        result = SchemaValidator().validate(sample_skill_dir)

        assert not result.passed
        field = f"metadata.{key}" if nested else key
        findings = [finding for finding in result.findings if finding.check_name == "frontmatter_field"]
        assert any(finding.message == f"Field '{field}': Keys should be strings" for finding in findings)
        assert any(finding.metadata == {"field": field, "error_type": "invalid_key"} for finding in findings)

    @pytest.mark.parametrize("key", ["123", "true", "false", "null", "2026-01-01"])
    def test_quoted_extra_keys_remain_strings(self, sample_skill_dir, key):
        manifest = sample_skill_dir / "SKILL.md"
        content = manifest.read_text().replace("---\n", f'---\n"{key}": top-level\n', 1)
        manifest.write_text(content.replace("metadata:\n", f'metadata:\n  "{key}": nested\n', 1))

        result = SchemaValidator().validate(sample_skill_dir)

        assert result.passed, result.errors
        assert result.metadata["frontmatter"].model_extra[key] == "top-level"
        assert result.metadata["frontmatter"].metadata.model_extra[key] == "nested"

    def test_valid_skill(self, sample_skill_dir: Path):
        """Test validation passes for valid skill with proper SkillEvaluator structure."""
        validator = SchemaValidator()
        result = validator.validate(sample_skill_dir)

        assert result.passed, f"Expected valid skill to pass. Errors: {result.errors}"
        assert len(result.errors) == 0

    def test_valid_team_skill(self, team_skill_dir: Path):
        """Test validation passes for valid team-specific skill."""
        validator = SchemaValidator()
        result = validator.validate(team_skill_dir)

        assert result.passed, f"Expected team skill to pass. Errors: {result.errors}"

    def test_invalid_skill_name(self, invalid_skill_dir: Path):
        """Test validation fails for invalid skill name (not kebab-case)."""
        validator = SchemaValidator()
        result = validator.validate(invalid_skill_dir)

        assert not result.passed
        # Should have error about name format
        assert any("kebab-case" in err.lower() or "name" in err.lower() for err in result.errors)

    def test_missing_skill_md(self, tmp_path: Path):
        """Test validation fails when SKILL.md is missing."""
        empty_dir = tmp_path / "empty-skill"
        empty_dir.mkdir()

        validator = SchemaValidator()
        result = validator.validate(empty_dir)

        assert not result.passed
        assert any("SKILL.md" in err or "not found" in err or "No skills found" in err for err in result.errors)

    def test_missing_required_fields(self, tmp_path: Path):
        """Test validation fails for missing required fields (description)."""
        skill_dir = tmp_path / "incomplete-skill"
        skill_dir.mkdir()

        skill_md = skill_dir / "SKILL.md"
        skill_md.write_text("""---
name: incomplete-skill
---

# Incomplete
""")

        validator = SchemaValidator()
        result = validator.validate(skill_dir)

        assert not result.passed
        # Should fail for missing description
        assert any("description" in err.lower() for err in result.errors)

    def test_forbidden_fields_rejected(self, skill_with_forbidden_fields: Path):
        """Test validation fails when forbidden fields (alwaysApply, globs) are present."""
        validator = SchemaValidator()
        result = validator.validate(skill_with_forbidden_fields)

        assert not result.passed
        assert any("forbidden" in err.lower() or "alwaysapply" in err.lower() for err in result.errors)

    def test_name_must_match_directory(self, skill_with_name_mismatch: Path):
        """Test validation fails when frontmatter name doesn't match directory name."""
        validator = SchemaValidator()
        result = validator.validate(skill_with_name_mismatch)

        assert not result.passed
        assert any("match" in err.lower() or "directory" in err.lower() for err in result.errors)

    def test_skill_md_case_insensitive(self, tmp_path: Path):
        """Test that skill.md (lowercase) is found but produces a naming warning."""
        skill_dir = tmp_path / "lowercase-skill"
        skill_dir.mkdir()

        skill_md = skill_dir / "skill.md"  # lowercase
        skill_md.write_text("""---
name: lowercase-skill
description: A skill with lowercase skill.md for testing case insensitivity
---

# Lowercase Skill

## Instructions

1. Use this skill

## Examples

Example usage here.
""")

        validator = SchemaValidator()
        result = validator.validate(skill_dir)

        naming_findings = [f for f in result.findings if f.check_name == "manifest_naming"]
        assert len(naming_findings) == 1, "Expected a HIGH finding for lowercase skill.md naming"
        assert "SKILL.md" in naming_findings[0].message
        assert "agentskills.io" in naming_findings[0].message

    def test_skill_md_uppercase_no_naming_warning(self, tmp_path: Path):
        """Test that SKILL.md (uppercase) does NOT produce a naming warning."""
        skill_dir = tmp_path / "uppercase-skill"
        skill_dir.mkdir()

        skill_md = skill_dir / "SKILL.md"
        skill_md.write_text("""---
name: uppercase-skill
description: A skill with proper SKILL.md naming
---

# Uppercase Skill

## Instructions

1. Use this skill

## Examples

Example usage here.
""")

        validator = SchemaValidator()
        result = validator.validate(skill_dir)

        naming_findings = [f for f in result.findings if f.check_name == "manifest_naming"]
        assert len(naming_findings) == 0, "SKILL.md (uppercase) should not produce a manifest_naming finding"

    def test_name_length_constraints(self, tmp_path: Path):
        """Test validation enforces name length constraints (1-64 chars)."""
        skill_dir = tmp_path / ("a" * 65 + "-skill")  # 65+ chars
        skill_dir.mkdir()

        skill_md = skill_dir / "SKILL.md"
        skill_md.write_text(f"""---
name: {"a" * 65}-skill
description: A skill with name exceeding 64 character limit
---

# Long Name Skill
""")

        validator = SchemaValidator()
        result = validator.validate(skill_dir)

        assert not result.passed
        assert any("64" in err or "character" in err.lower() or "length" in err.lower() for err in result.errors)

    def test_consecutive_hyphens_rejected(self, tmp_path: Path):
        """Test validation fails for names with consecutive hyphens."""
        skill_dir = tmp_path / "bad--name"
        skill_dir.mkdir()

        skill_md = skill_dir / "SKILL.md"
        skill_md.write_text("""---
name: bad--name
description: A skill with consecutive hyphens in name
---

# Bad Name Skill
""")

        validator = SchemaValidator()
        result = validator.validate(skill_dir)

        assert not result.passed
        assert any("consecutive" in err.lower() or "hyphen" in err.lower() for err in result.errors)

    def test_description_length_constraints(self, tmp_path: Path):
        """Test validation enforces description length constraints (1-1024 chars)."""
        skill_dir = tmp_path / "long-description"
        skill_dir.mkdir()

        # Create description over 1024 chars
        long_desc = "A" * 1030

        skill_md = skill_dir / "SKILL.md"
        skill_md.write_text(f"""---
name: long-description
description: {long_desc}
---

# Long Description Skill
""")

        validator = SchemaValidator()
        result = validator.validate(skill_dir)

        assert not result.passed
        assert any("1024" in err or "description" in err.lower() for err in result.errors)

    def test_metadata_author_validation(self, tmp_path: Path):
        """A name without an email is advisory attribution under the default profile."""
        from skillevaluator.models.result import Severity

        skill_dir = tmp_path / "no-email-author"
        skill_dir.mkdir()

        skill_md = skill_dir / "SKILL.md"
        skill_md.write_text("""---
name: no-email-author
description: A skill with malformed author metadata
metadata:
  author: John Doe
---

# Author Test Skill

## Instructions

1. Test the author field

## Examples

Example usage.
""")

        validator = SchemaValidator()
        result = validator.validate(skill_dir)

        assert result.passed, result.errors
        assert not [f for f in result.findings if f.check_name == "author_format"]
        author_findings = [f for f in result.findings if f.check_name == "author_without_email"]
        assert len(author_findings) == 1
        assert author_findings[0].severity == Severity.INFO
        assert author_findings[0].message == "Author has no email address: John Doe"
        assert author_findings[0].metadata.get("shape_ok") is False

    def test_metadata_author_missing_is_advisory(self, tmp_path: Path):
        """A missing metadata.author is a LOW advisory under the default profile."""
        from skillevaluator.models.result import Severity

        skill_dir = tmp_path / "missing-author"
        skill_dir.mkdir()

        skill_md = skill_dir / "SKILL.md"
        skill_md.write_text("""---
name: missing-author
description: A skill with no metadata author for testing
---

# Missing Author Skill

## Instructions

1. Test that the author field is required

## Examples

Example usage.
""")

        validator = SchemaValidator()
        result = validator.validate(skill_dir)

        assert result.passed, result.errors
        author_findings = [f for f in result.findings if f.check_name == "author_missing"]
        assert len(author_findings) == 1
        assert author_findings[0].severity == Severity.LOW
        assert "Author not specified" in author_findings[0].message

    def test_valid_metadata_structure(self, tmp_path: Path):
        """Test validation passes with complete valid metadata structure."""
        skill_dir = tmp_path / "complete-metadata"
        skill_dir.mkdir()

        skill_md = skill_dir / "SKILL.md"
        skill_md.write_text("""---
name: complete-metadata
description: A skill with complete metadata structure per SkillEvaluator spec
license: MIT
compatibility: Requires Python 3.11+
metadata:
  author: Test User <[email protected]>
  tags:
    - testing
    - metadata
  languages:
    - python
  frameworks:
    - pytest
  domain:
    - testing
    - quality
---

# Complete Metadata Skill

## Instructions

1. Check metadata fields
2. Verify author format

## Examples

```yaml
metadata:
  author: Test User <[email protected]>
```
""")

        validator = SchemaValidator()
        result = validator.validate(skill_dir)

        assert result.passed, f"Expected complete metadata skill to pass. Errors: {result.errors}"
        assert result.metadata["frontmatter"].metadata.domain == ["testing", "quality"]

    @pytest.mark.parametrize(
        ("metadata", "expected"),
        [
            pytest.param({"domain": "backend"}, ["backend"], id="scalar"),
            pytest.param({"domain": ["backend", "ai-ml"]}, ["backend", "ai-ml"], id="list"),
            pytest.param(
                {"domain": [" backend ", "ai-ml", "backend"]},
                ["backend", "ai-ml"],
                id="strip-and-stable-dedupe",
            ),
            pytest.param({}, [], id="missing"),
            pytest.param({"domain": None}, [], id="null"),
            pytest.param({"domain": []}, [], id="empty"),
            pytest.param({"domain": ""}, [], id="blank-scalar"),
            pytest.param({"domain": "   "}, [], id="whitespace-scalar"),
        ],
    )
    def test_metadata_domain_normalization(self, metadata: dict[str, object], expected: list[str]):
        assert SkillMetadata(**metadata).domain == expected

    @pytest.mark.parametrize(
        ("domain", "message"),
        [
            pytest.param(["backend", 1], "domain values must be strings", id="non-string"),
            pytest.param(["backend", "   "], "domain values must not be blank", id="blank-list-entry"),
            pytest.param(["backend"] * 17, "domain must contain at most 16 values", id="too-many-raw-items"),
            pytest.param("x" * 129, "domain values must be at most 128 characters", id="too-long"),
            pytest.param({"name": "backend"}, "domain must be a string or list of strings", id="mapping"),
        ],
    )
    def test_metadata_domain_rejects_invalid_values(self, domain: object, message: str):
        with pytest.raises(ValueError, match=message):
            SkillMetadata(domain=domain)

    @pytest.mark.parametrize(
        ("domain_yaml", "message"),
        [
            pytest.param(
                "  domain:\n    - backend\n    - 42\n",
                "domain values must be strings",
                id="non-string-list-entry",
            ),
            pytest.param(
                f"  domain: {'x' * 129}\n",
                "domain values must be at most 128 characters",
                id="over-long-single-string",
            ),
        ],
    )
    def test_invalid_metadata_domain_reports_field_finding(self, tmp_path: Path, domain_yaml: str, message: str):
        """An invalid domain, including an over-long single string, is a HIGH metadata.domain finding."""
        from skillevaluator.models.result import Severity

        skill_dir = tmp_path / "bad-domain"
        skill_dir.mkdir()
        (skill_dir / "SKILL.md").write_text(
            "---\n"
            "name: bad-domain\n"
            "description: A skill with an invalid domain\n"
            "metadata:\n"
            "  author: Test User <[email protected]>\n"
            f"{domain_yaml}"
            "---\n\n# Bad Domain\n",
            encoding="utf-8",
        )

        result = SchemaValidator().validate(skill_dir)

        assert not result.passed
        findings = [f for f in result.findings if f.check_name == "frontmatter_field"]
        assert len(findings) == 1
        assert findings[0].severity == Severity.HIGH
        assert findings[0].metadata["field"] == "metadata.domain"
        assert message in findings[0].message

    # =========================================================================
    # agentskills.io spec + validate_skill_md.py CI checks
    # =========================================================================

    def test_reserved_word_in_name_rejected(self, tmp_path: Path):
        """Test validation fails when name contains reserved words ('anthropic', 'claude')."""
        for word in ("anthropic", "claude"):
            skill_dir = tmp_path / f"my-{word}-skill"
            skill_dir.mkdir(exist_ok=True)

            skill_md = skill_dir / "SKILL.md"
            skill_md.write_text(f"""---
name: my-{word}-skill
description: A skill with a reserved word in its name
---

# Reserved Word Skill

## Instructions

1. Should not pass

## Examples

Example.
""")

            validator = SchemaValidator()
            result = validator.validate(skill_dir)

            assert not result.passed, f"Name containing '{word}' should be rejected"
            assert any("reserved" in err.lower() for err in result.errors)

    def test_reserved_word_substring_in_name_is_allowed(self, tmp_path: Path):
        """Reserved providers only match complete hyphen-delimited name segments."""
        skill_dir = tmp_path / "philanthropic-grant-matcher"
        skill_dir.mkdir()
        (skill_dir / "SKILL.md").write_text(
            "---\n"
            "name: philanthropic-grant-matcher\n"
            "description: Use when matching philanthropic grants to eligible programs.\n"
            "metadata:\n"
            "  author: Test User <[email protected]>\n"
            "---\n\n"
            "# Philanthropic Grant Matcher\n\n"
            "## Instructions\n\n1. Match each grant to an eligible program.\n\n"
            "## Examples\n\nMatch the example grant.\n"
        )

        result = SchemaValidator().validate(skill_dir)

        assert result.passed, result.errors

    def test_xml_tags_in_name_rejected(self, tmp_path: Path):
        """Test validation fails when name contains XML tags."""
        skill_dir = tmp_path / "xml-name"
        skill_dir.mkdir()

        skill_md = skill_dir / "SKILL.md"
        skill_md.write_text("""---
name: "<script>alert</script>"
description: A skill with XML tags in name
---

# XML Name Skill

## Instructions

1. Should fail

## Examples

Example.
""")

        validator = SchemaValidator()
        result = validator.validate(skill_dir)

        assert not result.passed

    def test_xml_tags_in_description_rejected(self, tmp_path: Path):
        """Test validation fails when description contains XML tags."""
        skill_dir = tmp_path / "xml-desc"
        skill_dir.mkdir()

        skill_md = skill_dir / "SKILL.md"
        skill_md.write_text("""---
name: xml-desc
description: "A skill <script>alert('xss')</script> with injected tags"
---

# XML Description Skill

## Instructions

1. Should fail

## Examples

Example.
""")

        validator = SchemaValidator()
        result = validator.validate(skill_dir)

        assert not result.passed
        assert any("xml" in err.lower() or "description" in err.lower() for err in result.errors)

    def test_unclosed_xml_tag_in_description_rejected(self, tmp_path: Path):
        """Test validation fails when description contains an unclosed tag-like value."""
        skill_dir = tmp_path / "unclosed-xml-desc"
        skill_dir.mkdir()

        skill_md = skill_dir / "SKILL.md"
        skill_md.write_text("""---
name: unclosed-xml-desc
description: "A skill with an unclosed <script foo tag in the description"
---

# Unclosed XML Description Skill

## Instructions

1. Should fail

## Examples

Example.
""")

        validator = SchemaValidator()
        result = validator.validate(skill_dir)

        assert not result.passed
        assert any("xml" in err.lower() or "description" in err.lower() for err in result.errors)

    def test_closing_xml_tag_in_description_rejected(self, tmp_path: Path):
        """Opening and closing XML tags use the same schema and quality definition."""
        skill_dir = tmp_path / "closing-xml-desc"
        skill_dir.mkdir()
        (skill_dir / "SKILL.md").write_text(
            "---\n"
            "name: closing-xml-desc\n"
            'description: "Use when validating a closing </tool> tag."\n'
            "metadata:\n"
            "  author: Test User <[email protected]>\n"
            "---\n\n"
            "# Closing XML Description\n\n"
            "## Instructions\n\n1. Run.\n\n"
            "## Examples\n\nExample.\n"
        )

        result = SchemaValidator().validate(skill_dir)

        assert not result.passed
        assert any("xml" in err.lower() for err in result.errors)

    def test_self_closing_xml_tag_in_description_rejected(self, tmp_path: Path):
        """Self-closing XML tags remain forbidden in frontmatter descriptions."""
        skill_dir = tmp_path / "self-closing-xml-desc"
        skill_dir.mkdir()
        (skill_dir / "SKILL.md").write_text(
            "---\n"
            "name: self-closing-xml-desc\n"
            'description: "Use when validating an injected <tool/> tag."\n'
            "metadata:\n"
            "  author: Test User <[email protected]>\n"
            "---\n\n"
            "# Self-closing XML Description\n\n"
            "## Instructions\n\n1. Run.\n\n"
            "## Examples\n\nExample.\n"
        )

        result = SchemaValidator().validate(skill_dir)

        assert not result.passed
        assert any("xml" in err.lower() for err in result.errors)

    def test_comparison_operators_in_description_allowed(self, tmp_path: Path):
        """Test threshold comparators in descriptions are not treated as XML tags."""
        skill_dir = tmp_path / "threshold-monitor"
        skill_dir.mkdir()

        skill_md = skill_dir / "SKILL.md"
        skill_md.write_text("""---
name: threshold-monitor
description: "Use this skill when free disk space < 10GB and CPU usage > 90% to trigger cleanup"
metadata:
  author: Test User <[email protected]>
---

# Threshold Monitor

## Instructions

1. Check threshold values.

## Examples

Use this skill when cleanup thresholds are exceeded.
""")

        validator = SchemaValidator()
        result = validator.validate(skill_dir)

        assert result.passed, f"Expected comparator description to pass: {result.errors}"

    def test_missing_top_level_heading_rejected(self, tmp_path: Path):
        """Test validation fails when body has no top-level heading (# Title)."""
        skill_dir = tmp_path / "no-heading"
        skill_dir.mkdir()

        skill_md = skill_dir / "SKILL.md"
        skill_md.write_text("""---
name: no-heading
description: A skill missing top-level heading in body
---

This body has no heading at all.

## Instructions

1. Do things

## Examples

Example.
""")

        validator = SchemaValidator()
        result = validator.validate(skill_dir)

        assert not result.passed
        assert any("heading" in err.lower() for err in result.errors)

    def test_missing_examples_section_is_medium(self, tmp_path: Path):
        """Missing ## Examples produces a MEDIUM finding (recommended, not required)."""
        from skillevaluator.models.result import Severity

        skill_dir = tmp_path / "no-examples"
        skill_dir.mkdir()

        skill_md = skill_dir / "SKILL.md"
        skill_md.write_text("""---
name: no-examples
description: A skill missing the Examples section
---

# No Examples

## Instructions

1. Do something
""")

        validator = SchemaValidator()
        result = validator.validate(skill_dir)

        rec_findings = [f for f in result.findings if f.check_name == "body_recommended_section"]
        assert len(rec_findings) == 1
        assert rec_findings[0].severity == Severity.MEDIUM
        assert "Examples" in rec_findings[0].message

    def test_usage_section_accepted_as_instructions_alternative(self, tmp_path: Path):
        """## Usage satisfies the recommended ## Instructions section nudge."""
        skill_dir = tmp_path / "usage-skill"
        skill_dir.mkdir()

        skill_md = skill_dir / "SKILL.md"
        skill_md.write_text("""---
name: usage-skill
description: A skill that uses Usage instead of Instructions
---

# Usage Skill

## Usage

1. Run the tool
2. Check results

## Examples

```bash
run-tool --help
```
""")

        validator = SchemaValidator()
        result = validator.validate(skill_dir)

        instructions_findings = [
            f for f in result.findings if f.check_name == "body_recommended_section" and "Instructions" in f.message
        ]
        assert len(instructions_findings) == 0, (
            f"## Usage should satisfy the Instructions nudge. Findings: {instructions_findings}"
        )

    def test_allowed_tools_field_accepted(self, tmp_path: Path):
        """Test that the optional 'allowed-tools' field is accepted per agentskills.io spec."""
        skill_dir = tmp_path / "with-tools"
        skill_dir.mkdir()

        skill_md = skill_dir / "SKILL.md"
        skill_md.write_text("""---
name: with-tools
description: A skill that declares allowed tools per agentskills.io spec
allowed-tools: Bash(git:*) Bash(jq:*) Read
metadata:
  author: Tool User <[email protected]>
---

# Skill With Allowed Tools

## Instructions

1. Use git commands
2. Use jq for JSON processing

## Examples

```bash
git status
jq '.name' package.json
```
""")

        validator = SchemaValidator()
        result = validator.validate(skill_dir)

        assert result.passed, f"Skill with allowed-tools should pass. Errors: {result.errors}"

    def test_all_required_sections_present_passes(self, tmp_path: Path):
        """Test validation passes when all body requirements are met."""
        skill_dir = tmp_path / "complete-body"
        skill_dir.mkdir()

        skill_md = skill_dir / "SKILL.md"
        skill_md.write_text("""---
name: complete-body
description: A skill with all required body sections
metadata:
  author: Body User <[email protected]>
---

# Complete Body Skill

## Instructions

1. Follow step one
2. Follow step two

## Examples

```python
print("hello")
```
""")

        validator = SchemaValidator()
        result = validator.validate(skill_dir)

        assert result.passed, f"Skill with complete body should pass. Errors: {result.errors}"

    def test_utf8_bom_skill_md_passes_schema(self, tmp_path: Path):
        """A valid SKILL.md that only adds a UTF-8 BOM must still pass schema (#91)."""
        skill_dir = tmp_path / "bom-skill"
        skill_dir.mkdir()
        body = """---
name: bom-skill
description: Valid skill whose SKILL.md starts with a UTF-8 BOM
metadata:
  author: Bom User <[email protected]>
---

# BOM Skill

## Instructions

1. Open the file in an editor that writes a BOM.

## Examples

Example usage.
"""
        (skill_dir / "SKILL.md").write_bytes(b"\xef\xbb\xbf" + body.encode("utf-8"))

        result = SchemaValidator().validate(skill_dir)

        assert result.passed, f"BOM-only difference should still pass schema. Errors: {result.errors}"
        assert all(f.check_name != "frontmatter_format" for f in result.findings)

    def test_canonical_support_dirs_accepted(self, tmp_path: Path):
        """Canonical public skill support directories must not be flagged.

        Skills may carry agent metadata, local verification, executable helpers,
        and runtime configuration without producing an ``unexpected_file`` finding.
        """
        skill_dir = tmp_path / "spec-layout-skill"
        skill_dir.mkdir()
        (skill_dir / "tools").mkdir()
        (skill_dir / "tools" / "run.py").write_text("print('hi')\n")
        (skill_dir / "config").mkdir()
        (skill_dir / "config" / "settings.json").write_text('{"k": "v"}\n')
        (skill_dir / "agents").mkdir()
        (skill_dir / "agents" / "openai.yaml").write_text("name: spec-layout-skill\n")
        (skill_dir / "tests").mkdir()
        (skill_dir / "tests" / "test_skill.py").write_text("def test_skill():\n    assert True\n")

        (skill_dir / "SKILL.md").write_text("""---
name: spec-layout-skill
description: A skill using the agentskills.io tools/ layout plus a config/ dir.
metadata:
  author: Spec User <[email protected]>
---

# Spec Layout Skill

## Instructions

1. Use the tools/ directory for executables.

## Examples

Example usage.
""")

        result = SchemaValidator().validate(skill_dir)

        unexpected = [f.message for f in result.findings if f.check_name == "unexpected_file"]
        assert not unexpected, f"Canonical support directories should be allowed, got: {unexpected}"

    def test_allowed_dirs_extended_via_env(self, tmp_path: Path, monkeypatch):
        """``SKILLEVALUATOR_SCHEMA_ALLOWED_DIRS`` extends the allowed skill-root dirs.

        A consumer with a non-canonical directory (e.g. ``data/``) should be able
        to clear the otherwise unavoidable LOW ``unexpected_file`` finding per-repo
        without forking the validator.
        """
        skill_dir = tmp_path / "extra-dirs-skill"
        skill_dir.mkdir()
        (skill_dir / "data").mkdir()
        (skill_dir / "data" / "table.json").write_text('{"k": "v"}\n')
        (skill_dir / "SKILL.md").write_text("""---
name: extra-dirs-skill
description: A skill that ships a data/ directory extended via the schema env knob.
metadata:
  author: Spec User <[email protected]>
---

# Extra Dirs Skill

## Instructions

1. Load runtime tables from data/.

## Examples

Example usage.
""")

        def _unexpected(result):
            return [f.message for f in result.findings if f.check_name == "unexpected_file"]

        monkeypatch.delenv("SKILLEVALUATOR_SCHEMA_ALLOWED_DIRS", raising=False)
        flagged = _unexpected(SchemaValidator().validate(skill_dir))
        assert any("data" in m for m in flagged), "data/ should be flagged without the env knob"

        monkeypatch.setenv("SKILLEVALUATOR_SCHEMA_ALLOWED_DIRS", "data, fixtures")
        cleared = _unexpected(SchemaValidator().validate(skill_dir))
        assert not cleared, f"data/ should be allowed via env knob, got: {cleared}"

    def test_missing_instructions_section_is_medium(self, tmp_path: Path):
        """Missing ## Instructions produces a MEDIUM finding (recommended, not required).

        Per agentskills.io spec the body has no format restrictions, so skills
        produced by tools like Codex's skill-creator (which don't emit a
        ## Instructions heading) should not be hard-failed.
        """
        from skillevaluator.models.result import Severity

        skill_dir = tmp_path / "no-instructions"
        skill_dir.mkdir()

        skill_md = skill_dir / "SKILL.md"
        skill_md.write_text("""---
name: no-instructions
description: A skill missing the Instructions section
metadata:
  author: Body User <[email protected]>
---

# No Instructions

## Examples

Example usage.
""")

        validator = SchemaValidator()
        result = validator.validate(skill_dir)

        rec_findings = [
            f for f in result.findings if f.check_name == "body_recommended_section" and "Instructions" in f.message
        ]
        assert len(rec_findings) == 1
        assert rec_findings[0].severity == Severity.MEDIUM

        body_required_findings = [f for f in result.findings if f.check_name == "body_required_section"]
        assert body_required_findings == [], (
            f"## Instructions must not produce a body_required_section finding anymore. Got: {body_required_findings}"
        )


class TestSchemaValidatorProfiles:
    """Profile-aware author validation for the external public policy."""

    @staticmethod
    def _write_skill(
        tmp_path: Path,
        *,
        name: str,
        author_line: str | None,
    ) -> Path:
        """Write a minimal SKILL.md with the given author (or no metadata block)."""
        skill_dir = tmp_path / name
        skill_dir.mkdir()
        metadata_block = ""
        if author_line is not None:
            metadata_block = f"metadata:\n  author: {author_line}\n"
        (skill_dir / "SKILL.md").write_text(
            f"""---
name: {name}
description: A profile-aware skill used by the test suite for testing severity overrides
{metadata_block}---

# {name}

## Instructions

1. Run.

## Examples

Example.
"""
        )
        return skill_dir

    def test_external_profile_accepts_well_formed_author(self, tmp_path: Path):
        """A well-formed author is accepted under the external profile."""
        from skillevaluator.validators.policy import load_profile

        skill_dir = self._write_skill(
            tmp_path,
            name="external-author-ok",
            author_line="Jane Doe <[email protected]>",
        )

        result = SchemaValidator(policy=load_profile("external")).validate(skill_dir)

        assert result.passed, f"External profile should accept a well-formed author. Errors: {result.errors}"
        assert not [f for f in result.findings if f.check_name == "author_format"]

    def test_external_profile_reports_missing_author_as_advisory(self, tmp_path: Path):
        """Missing author is reported but does not block under the public profile."""
        from skillevaluator.models.result import Severity
        from skillevaluator.validators.policy import load_profile

        skill_dir = self._write_skill(
            tmp_path,
            name="external-no-author",
            author_line=None,
        )

        result = SchemaValidator(policy=load_profile("external")).validate(skill_dir)

        author_findings = [f for f in result.findings if f.check_name == "author_missing"]
        assert len(author_findings) == 1
        assert author_findings[0].severity == Severity.LOW
        assert result.passed, result.errors

    def test_external_profile_rejects_malformed_author_shape(self, tmp_path: Path):
        """A bare email fails the shape check under the external profile."""
        from skillevaluator.models.result import Severity
        from skillevaluator.validators.policy import load_profile

        skill_dir = self._write_skill(
            tmp_path,
            name="external-malformed-author",
            author_line="[email protected]",
        )
        result = SchemaValidator(policy=load_profile("external")).validate(skill_dir)

        author_findings = [f for f in result.findings if f.check_name == "author_format"]
        assert len(author_findings) == 1, (
            f"Expected exactly one author_format finding for malformed shape, got {len(author_findings)}"
        )
        assert author_findings[0].severity == Severity.HIGH
        assert author_findings[0].metadata.get("shape_ok") is False
        assert not result.passed

    @pytest.mark.parametrize(
        "author",
        [
            "NVIDIA cuOpt Team",
            "Example Corporation and Affiliates",
            "Jane Doe (@jdoe)",
            "Équipe Données",
        ],
    )
    def test_default_profile_accepts_name_only_attribution(self, tmp_path: Path, author: str):
        """Person, team, and organization names without an email are advisory, not blocking."""
        from skillevaluator.models.result import Severity
        from skillevaluator.validators.policy import load_profile

        skill_dir = self._write_skill(tmp_path, name="team-author", author_line=f'"{author}"')

        result = SchemaValidator(policy=load_profile("default")).validate(skill_dir)

        assert result.passed, result.errors
        findings = [f for f in result.findings if f.check_name.startswith("author")]
        assert [(f.check_name, f.severity) for f in findings] == [("author_without_email", Severity.INFO)]

    @pytest.mark.parametrize(
        ("author", "reason"),
        [
            ("[email protected]", "contains an email address"),
            ("Jane Doe [email protected]", "contains an email address"),
            # An intranet host has no dot, but the value is still only an email.
            ("root@buildhost", "contains an email address"),
            ("jane@example", "contains an email address"),
            ("Jos\u00e9@example.com", "contains an email address"),
            ("Jane Doe <jane>", "angle brackets"),
            ("Jane Doe <[email protected]> and team", "angle brackets"),
            ("Jane\u202eDoe", "control, private-use, or bidirectional formatting"),
            ("Jane\tDoe", "control, private-use, or bidirectional formatting"),
            ("Jane\u2066Doe", "control, private-use, or bidirectional formatting"),
            ("Jane\ue000Doe", "control, private-use, or bidirectional formatting"),
            ("---", "no letters or digits"),
            ("J" * 257, "longer than 256 characters"),
        ],
    )
    def test_malformed_or_unsafe_author_blocks_in_every_profile(self, tmp_path: Path, author: str, reason: str):
        """Malformed or unsafe values stay HIGH under both bundled profiles."""
        from skillevaluator.models.result import Severity
        from skillevaluator.validators.policy import load_profile

        skill_dir = self._write_skill(tmp_path, name="bad-author", author_line=f'"{author}"')

        for profile in ("default", "registry"):
            result = SchemaValidator(policy=load_profile(profile)).validate(skill_dir)
            findings = [f for f in result.findings if f.check_name == "author_format"]
            assert len(findings) == 1, (profile, [f.check_name for f in result.findings])
            assert findings[0].severity == Severity.HIGH
            assert reason in findings[0].message
            assert len(findings[0].line_content) <= len("author: ") + 256
            assert not result.passed

    def test_registry_profile_requires_contact_author(self, tmp_path: Path):
        """The registry profile blocks a missing author and a name without an email."""
        from skillevaluator.models.result import Severity
        from skillevaluator.validators.policy import load_profile

        policy = load_profile("registry")
        missing = self._write_skill(tmp_path, name="registry-missing", author_line=None)
        team = self._write_skill(tmp_path, name="registry-team", author_line="Example Team")

        missing_result = SchemaValidator(policy=policy).validate(missing)
        team_result = SchemaValidator(policy=policy).validate(team)

        assert not missing_result.passed
        assert not team_result.passed
        assert {(f.check_name, f.severity) for f in missing_result.findings if f.check_name.startswith("author")} == {
            ("author_missing", Severity.HIGH)
        }
        assert {(f.check_name, f.severity) for f in team_result.findings if f.check_name.startswith("author")} == {
            ("author_without_email", Severity.HIGH)
        }

    def test_domain_restriction_rejects_name_without_email(self, tmp_path: Path):
        """An author_email_regex needs an email in that domain, so a bare name fails it."""
        import re

        from skillevaluator.models.result import Severity
        from skillevaluator.validators.policy import ValidationPolicy, default_policy

        policy = ValidationPolicy(
            profile="org",
            author_email_regex=re.compile(r"@example\.org>$"),
            severity_overrides=dict(default_policy().severity_overrides),
        )
        team = self._write_skill(tmp_path, name="org-team", author_line="Example Team")
        member = self._write_skill(tmp_path, name="org-member", author_line="Jane Doe <[email protected]>")

        team_result = SchemaValidator(policy=policy).validate(team)
        findings = [f for f in team_result.findings if f.check_name == "author_format"]
        assert len(findings) == 1
        assert findings[0].severity == Severity.HIGH
        assert "has no email address matching the required domain" in findings[0].message
        assert findings[0].metadata.get("shape_ok") is False
        assert SchemaValidator(policy=policy).validate(member).passed

    def test_domain_restriction_rejects_well_formed_author_outside_the_domain(self, tmp_path: Path):
        """The other half of the domain branch: a correct shape in the wrong domain still blocks.

        The domain check is the only blocking author rule an organization
        policy keeps, so a regression that accepted any domain has to fail a
        test rather than only the bare-name case above.
        """
        import re

        from skillevaluator.models.result import Severity
        from skillevaluator.validators.policy import ValidationPolicy, default_policy

        policy = ValidationPolicy(
            profile="org",
            author_email_regex=re.compile(r"@example\.org>$"),
            severity_overrides=dict(default_policy().severity_overrides),
        )
        outsider = self._write_skill(tmp_path, name="org-outsider", author_line="Jane Doe <[email protected]>")

        result = SchemaValidator(policy=policy).validate(outsider)

        findings = [f for f in result.findings if f.check_name == "author_format"]
        assert len(findings) == 1
        assert findings[0].severity == Severity.HIGH
        assert "Author email does not match the required domain" in findings[0].message
        assert findings[0].metadata.get("shape_ok") is True
        assert findings[0].metadata.get("domain_ok") is False
        assert not result.passed

    @pytest.mark.parametrize(
        "author",
        [
            "Jane\u200cDoe",  # ZWNJ: required by Persian and Devanagari names
            "Jane\u200dDoe",  # ZWJ
            "Jane\u00adDoe",  # soft hyphen
            "Jane\u0530Doe",  # unassigned in Unicode 16, assigned in some later version
        ],
    )
    def test_joiners_and_unassigned_code_points_are_accepted_in_an_author(self, tmp_path: Path, author: str):
        """The character guard rejects spoofing characters, not every Unicode category C.

        ZWNJ/ZWJ are needed to render Persian, Arabic and Devanagari names, and
        an unassigned code point would make the verdict depend on the running
        interpreter's Unicode data rather than on the value.
        """
        from skillevaluator.validators.policy import load_profile

        skill_dir = self._write_skill(tmp_path, name="joiner-author", author_line=f'"{author} <[email protected]>"')

        result = SchemaValidator(policy=load_profile("default")).validate(skill_dir)

        assert not [f for f in result.findings if f.check_name.startswith("author")], result.errors
        assert result.passed, result.errors

    @pytest.mark.parametrize(
        "author",
        ["Jane Doe (@jdoe)", "Team (@org-handle)", "@jdoe", "Platform Team @ Example"],
    )
    def test_social_handles_are_not_mistaken_for_a_bare_email(self, tmp_path: Path, author: str):
        """Widening the bare-email pattern must not turn a documented handle into a HIGH finding."""
        from skillevaluator.models.result import Severity
        from skillevaluator.validators.policy import load_profile

        skill_dir = self._write_skill(tmp_path, name="handle-author", author_line=f'"{author}"')

        result = SchemaValidator(policy=load_profile("default")).validate(skill_dir)

        assert result.passed, result.errors
        findings = [f for f in result.findings if f.check_name.startswith("author")]
        assert [(f.check_name, f.severity) for f in findings] == [("author_without_email", Severity.INFO)]

    def test_default_validator_uses_public_profile(self, tmp_path: Path):
        """Constructing SchemaValidator() without a policy accepts public domains."""

        skill_dir = self._write_skill(
            tmp_path,
            name="default-uses-public-profile",
            author_line="Jane Doe <[email protected]>",
        )

        result = SchemaValidator().validate(skill_dir)

        assert result.passed


class TestMetadataIdContract:
    """Optional ``metadata.id``: validated when present, required only by policy."""

    @staticmethod
    def _write_skill(root: Path, name: str, id_line: str | None) -> Path:
        skill_dir = root / name
        skill_dir.mkdir(parents=True)
        id_block = f"  id: {id_line}\n" if id_line is not None else ""
        (skill_dir / "SKILL.md").write_text(
            f"""---
name: {name}
description: Use when testing the metadata.id contract. A skill used by the test suite to check the contract
metadata:
  author: Jane Doe <[email protected]>
{id_block}---

# {name}

## Instructions

1. Run.

## Examples

Example.
""",
            encoding="utf-8",
        )
        return skill_dir

    @staticmethod
    def _id_findings(result) -> list[tuple[str, str]]:
        return [(f.check_name, f.severity.value) for f in result.findings if f.check_name.startswith("metadata_id")]

    def test_valid_id_passes_and_does_not_claim_uniqueness(self, tmp_path: Path):
        from skillevaluator.validators.policy import load_profile

        skill_dir = self._write_skill(tmp_path, "id-ok", "id-ok-2")

        for profile in ("default", "registry"):
            result = SchemaValidator(policy=load_profile(profile)).validate(skill_dir)
            assert result.passed, result.errors
            assert self._id_findings(result) == []
            detail = next(d for d in result.success_details if d.check_name == "metadata_id_format")
            assert "Uniqueness is checked only across skills validated together" in detail.message
            assert result.metadata["metadata_id"] == "id-ok-2"

    def test_missing_id_follows_the_profile_requirement(self, tmp_path: Path):
        from skillevaluator.validators.policy import ValidationPolicy, load_profile

        skill_dir = self._write_skill(tmp_path, "no-id", None)

        default_result = SchemaValidator(policy=load_profile("default")).validate(skill_dir)
        recommend_result = SchemaValidator(policy=ValidationPolicy(metadata_id_requirement="recommend")).validate(
            skill_dir
        )
        registry_result = SchemaValidator(policy=load_profile("registry")).validate(skill_dir)

        assert default_result.passed
        assert self._id_findings(default_result) == []
        assert recommend_result.passed
        assert self._id_findings(recommend_result) == [("metadata_id_missing", "low")]
        assert not registry_result.passed
        assert self._id_findings(registry_result) == [("metadata_id_missing", "high")]
        assert "profile 'registry' requires" in registry_result.findings[-1].message

    @pytest.mark.parametrize(
        ("id_line", "reason"),
        [
            ('"My-Skill"', "lowercase ASCII"),
            ("my_skill", "lowercase ASCII"),
            ("-my-skill", "lowercase ASCII"),
            ("my--skill", "lowercase ASCII"),
            ('" my-skill"', "no surrounding whitespace"),
            ('"skill-é"', "lowercase ASCII"),
            ('"' + "a" * 65 + '"', "at most 64 characters (got 65)"),
            ('""', "must not be empty"),
            ("123", "must be a string (got int)"),
            ("true", "must be a string (got bool)"),
            ("null", "must be a string (got NoneType)"),
            ("[a, b]", "must be a string (got list)"),
        ],
    )
    def test_invalid_id_is_advisory_by_default_and_blocks_in_registry(self, tmp_path: Path, id_line: str, reason: str):
        from skillevaluator.validators.policy import load_profile

        skill_dir = self._write_skill(tmp_path, "bad-id", id_line)

        default_result = SchemaValidator(policy=load_profile("default")).validate(skill_dir)
        registry_result = SchemaValidator(policy=load_profile("registry")).validate(skill_dir)

        assert default_result.passed, default_result.errors
        assert self._id_findings(default_result) == [("metadata_id_format", "medium")]
        assert not registry_result.passed
        assert self._id_findings(registry_result) == [("metadata_id_format", "high")]
        finding = next(f for f in registry_result.findings if f.check_name == "metadata_id_format")
        assert reason in finding.message
        assert len(finding.line_content) <= len("id: ") + 81
        assert "metadata_id" not in registry_result.metadata

    def test_collection_reports_duplicate_ids(self, tmp_path: Path):
        from skillevaluator.validators.policy import load_profile

        skills = tmp_path / "skills"
        self._write_skill(skills, "first-skill", "shared-id")
        self._write_skill(skills, "second-skill", "shared-id")
        self._write_skill(skills, "third-skill", "third-skill")
        self._write_skill(skills, "fourth-skill", None)

        default_result = SchemaValidator(policy=load_profile("default")).validate(tmp_path)
        registry_result = SchemaValidator(policy=load_profile("registry")).validate(tmp_path)

        duplicates = [f for f in default_result.findings if f.check_name == "metadata_id_duplicate"]
        assert [(f.severity.value, f.metadata["skills"]) for f in duplicates] == [
            ("medium", ["skills/first-skill", "skills/second-skill"])
        ]
        assert "metadata.id 'shared-id' is declared by 2 skills" in duplicates[0].message
        assert ("metadata_id_duplicate", "high") in [(f.check_name, f.severity.value) for f in registry_result.findings]
        assert not registry_result.passed

    def test_collection_with_unique_ids_reports_scope(self, tmp_path: Path):
        skills = tmp_path / "skills"
        self._write_skill(skills, "first-skill", "first-skill")
        self._write_skill(skills, "second-skill", "second-skill")

        result = SchemaValidator().validate(tmp_path)

        assert result.passed, result.errors
        assert not [f for f in result.findings if f.check_name == "metadata_id_duplicate"]
        detail = next(d for d in result.success_details if d.check_name == "metadata_id_unique")
        assert detail.message == "2 declared metadata.id value(s) are unique within this collection"

    @pytest.mark.parametrize("include_failure", [False, True])
    def test_folder_api_preserves_child_advisories_once(self, tmp_path: Path, include_failure: bool):
        from skillevaluator import tier1

        skills = tmp_path / "skills"
        team = self._write_skill(skills, "team-skill", "INVALID_ID")
        manifest = team / "SKILL.md"
        manifest.write_text(manifest.read_text().replace("Jane Doe <[email protected]>", "Example Team"))
        missing = self._write_skill(skills, "missing-author", "missing-author")
        manifest = missing / "SKILL.md"
        manifest.write_text(manifest.read_text().replace("  author: Jane Doe <[email protected]>\n", ""))
        if include_failure:
            broken = self._write_skill(skills, "broken-author", "INVALID_ID")
            manifest = broken / "SKILL.md"
            manifest.write_text(manifest.read_text().replace("Jane Doe <[email protected]>", "[email protected]"))

        (result,) = tier1.run_validation(tmp_path, checks="schema")

        expected = [
            ("author_missing", "low"),
            ("author_without_email", "info"),
            ("metadata_id_format", "medium"),
        ]
        if include_failure:
            expected += [("author_format", "high"), ("metadata_id_format", "medium")]
        assert sorted((finding.check_name, finding.severity.value) for finding in result.findings) == sorted(expected)
        assert result.passed is not include_failure
        assert result.summary.warnings == len(result.warnings) == (4 if include_failure else 3)
        assert result.summary.errors == len(result.errors) == int(include_failure)
        assert result.summary.low_count == 1
        assert result.summary.medium_count == (2 if include_failure else 1)
        assert result.summary.high_count == int(include_failure)
        for name in ("team-skill", "missing-author"):
            summaries = [detail for detail in result.success_details if detail.check_name == name]
            assert len(summaries) == 1
            checks = summaries[0].metadata["checks"]
            assert len(checks) == summaries[0].metadata["total_checks"]
            assert len({(check["name"], check["description"]) for check in checks}) == len(checks)
            assert not any(detail.check_name.startswith(f"[{name}]") for detail in result.success_details)

    def test_folder_api_preserves_recommended_id_advisory(self, tmp_path: Path):
        from skillevaluator import tier1
        from skillevaluator.validators.policy import ValidationPolicy

        self._write_skill(tmp_path / "skills", "no-id", None)

        (result,) = tier1.run_validation(
            tmp_path, checks="schema", policy=ValidationPolicy(metadata_id_requirement="recommend")
        )

        assert result.passed
        assert self._id_findings(result) == [("metadata_id_missing", "low")]
        assert result.summary.warnings == len(result.warnings) == 1
        assert result.summary.low_count == 1


class TestCollectionAdvisoryIdentity:
    """A preserved advisory must say which skill in the collection produced it.

    Collection validation keeps the advisories of the skills that passed, but
    a consumer reading the collection result also has to be able to attribute
    each one, which a finding carried over unchanged does not allow.
    """

    @staticmethod
    def _write_skill(root: Path, name: str, *, author_line: str, id_line: str | None) -> Path:
        skill_dir = root / name
        skill_dir.mkdir(parents=True)
        id_block = f"  id: {id_line}\n" if id_line is not None else ""
        (skill_dir / "SKILL.md").write_text(
            f"""---
name: {name}
description: Use when testing collection advisories. A skill used by the test suite to check collection-level advisory handling
metadata:
  author: {author_line}
{id_block}---

# {name}

## Instructions

1. Run.

## Examples

Example.
""",
            encoding="utf-8",
        )
        return skill_dir

    def test_carried_advisory_names_its_skill(self, tmp_path: Path):
        from skillevaluator.models.result import Severity
        from skillevaluator.validators.policy import load_profile

        skills = tmp_path / "skills"
        self._write_skill(skills, "advisory-skill", author_line="Example Team", id_line="INVALID_ID")

        skill_result = SchemaValidator(policy=load_profile("default")).validate(skills / "advisory-skill")
        collection = SchemaValidator(policy=load_profile("default")).validate(tmp_path)

        assert skill_result.passed, skill_result.errors
        carried = {f.check_name: f for f in collection.findings if f.metadata.get("collection_skill")}
        assert carried.keys() >= {"author_without_email", "metadata_id_format"}
        assert carried["author_without_email"].severity == Severity.INFO
        assert carried["metadata_id_format"].severity == Severity.MEDIUM
        for finding in carried.values():
            assert finding.metadata["collection_skill"] == "advisory-skill"
            assert finding.file_path.startswith("[advisory-skill] ")
        assert any("[advisory-skill] " in warning for warning in collection.warnings)
        # An advisory from a passing child stays advisory: it cannot fail the collection.
        assert collection.passed, collection.errors
        assert not collection.errors
        assert not [f for f in collection.findings if f.severity.is_error()]
        # The roll-up must not claim everything passed while advisories exist.
        rollup = next(d for d in collection.success_details if d.check_name == "advisory-skill")
        assert rollup.metadata["advisory_findings"] == len(skill_result.findings)
        assert "advisory finding(s)" in rollup.message

    def test_collection_of_clean_skills_carries_no_advisories(self, tmp_path: Path):
        """Negative case: nothing is invented for a skill with no findings of its own."""
        skills = tmp_path / "skills"
        self._write_skill(skills, "clean-skill", author_line="Jane Doe <[email protected]>", id_line="clean-skill")

        collection = SchemaValidator().validate(tmp_path)

        assert collection.passed, collection.errors
        assert not [f for f in collection.findings if f.metadata.get("collection_skill")]
        rollup = next(d for d in collection.success_details if d.check_name == "clean-skill")
        assert rollup.message == "All schema and structure checks passed"
        assert rollup.metadata["advisory_findings"] == 0


class TestMetadataIdPreviewBounds:
    """A rejected ``metadata.id`` must never be rendered by expanding its value."""

    @staticmethod
    def _alias_bomb(levels: int) -> str:
        lines = ["a0: &a0 [x, x]"]
        lines += [f"a{i}: &a{i} [*a{i - 1}, *a{i - 1}]" for i in range(1, levels + 1)]
        lines.append(f"id: *a{levels}")
        return "\n".join(lines)

    def test_non_string_id_is_described_by_type_not_rendered(self):
        from skillevaluator.validators.schema import _display_metadata_id

        assert _display_metadata_id([1, 2, 3]) == "<list>"
        assert _display_metadata_id(123) == "<int>"
        assert _display_metadata_id(None) == "<NoneType>"
        assert _display_metadata_id(True) == "<bool>"

    def test_yaml_alias_graph_does_not_expand_into_the_finding(self):
        import yaml

        from skillevaluator.validators.schema import _display_metadata_id

        # 30 doubling aliases: repr() of this value is ~2**30 elements, which
        # would exhaust memory. The preview must not build it.
        value = yaml.safe_load(self._alias_bomb(30))["id"]

        preview = _display_metadata_id(value)

        assert preview == "<list>"
        assert len(preview) <= 16

    def test_alias_graph_id_is_rejected_with_a_bounded_finding(self, tmp_path: Path):
        skill_dir = tmp_path / "alias-id"
        skill_dir.mkdir()
        (skill_dir / "SKILL.md").write_text(
            "---\nname: alias-id\ndescription: A skill whose metadata.id is a YAML alias graph\nmetadata:\n"
            + "\n".join(f"  {line}" for line in self._alias_bomb(20).splitlines())
            + "\n---\n\n# Alias\n\n## Instructions\n\n1. Run.\n",
            encoding="utf-8",
        )

        result = SchemaValidator().validate(skill_dir)

        findings = [f for f in result.findings if f.check_name == "metadata_id_format"]
        assert len(findings) == 1
        assert "must be a string (got list)" in findings[0].message
        assert len(findings[0].line_content) <= 32

    @pytest.mark.parametrize("id_value", ["&recursive [*recursive]", "&recursive {self: *recursive}"])
    def test_recursive_id_is_rejected_with_bounded_evidence(self, tmp_path: Path, id_value: str):
        skill = TestMetadataIdContract._write_skill(tmp_path, "recursive-id", id_value)

        result = SchemaValidator().validate(skill)

        assert result.passed
        (finding,) = [finding for finding in result.findings if finding.check_name == "metadata_id_format"]
        assert "must be a string" in finding.message
        assert len(finding.line_content) <= 32


def _write_skill(
    root: Path, frontmatter: str, body: str = "# Portable\n\n## Instructions\n\n1. Step\n\n## Examples\n\nExample.\n"
) -> Path:
    skill_dir = root / "skills" / "portable-skill"
    skill_dir.mkdir(parents=True)
    (skill_dir / "SKILL.md").write_text(f"---\n{frontmatter}---\n\n{body}", encoding="utf-8")
    return skill_dir


_BASE_FM = (
    "name: portable-skill\n"
    "description: Use when checking harness portability of a skill.\n"
    "metadata:\n  author: Test User <[email protected]>\n"
)


def _checks(result, name: str) -> list:
    return [finding for finding in result.findings if finding.check_name == name]


class TestSchemaPortabilityChecks:
    """Spec-valid patterns that agent harnesses load differently (Tier 1, harness-neutral)."""

    def test_clean_skill_has_no_portability_findings(self, tmp_path: Path):
        result = SchemaValidator().validate(_write_skill(tmp_path, _BASE_FM))

        assert result.passed, result.errors
        names = {
            "duplicate_key",
            "manifest_bom",
            "manifest_symlink",
            "manifest_size",
            "nested_manifest",
            "description_utf16_length",
            "description_trigger",
            "unknown_field",
        }
        assert not [f for f in result.findings if f.check_name in names]

    def test_duplicate_top_level_key_is_high(self, tmp_path: Path):
        result = SchemaValidator().validate(_write_skill(tmp_path, _BASE_FM + "description: Second value.\n"))

        findings = _checks(result, "duplicate_key")
        assert not result.passed
        assert [f.metadata["key"] for f in findings] == ["description"]
        assert findings[0].severity == "high"
        assert findings[0].line_number == 6

    def test_duplicate_nested_key_is_reported_with_path(self, tmp_path: Path):
        fm = _BASE_FM + "  author: Other User <[email protected]>\n"
        result = SchemaValidator().validate(_write_skill(tmp_path, fm))

        assert [f.metadata["key"] for f in _checks(result, "duplicate_key")] == ["metadata.author"]

    def test_same_key_in_different_mappings_is_not_duplicate(self, tmp_path: Path):
        fm = _BASE_FM + "  tags:\n    - name: a\n    - name: b\n"
        result = SchemaValidator().validate(_write_skill(tmp_path, fm))

        assert not _checks(result, "duplicate_key")

    def test_bom_is_medium_and_does_not_fail(self, tmp_path: Path):
        skill_dir = _write_skill(tmp_path, _BASE_FM)
        manifest = skill_dir / "SKILL.md"
        manifest.write_bytes(b"\xef\xbb\xbf" + manifest.read_bytes())

        result = SchemaValidator().validate(skill_dir)

        assert result.passed, result.errors
        assert [f.severity for f in _checks(result, "manifest_bom")] == ["medium"]

    def test_symlinked_manifest_is_medium(self, tmp_path: Path):
        skill_dir = _write_skill(tmp_path, _BASE_FM)
        target = tmp_path / "real.md"
        (skill_dir / "SKILL.md").rename(target)
        (skill_dir / "SKILL.md").symlink_to(target)

        result = SchemaValidator().validate(skill_dir)

        assert result.passed, result.errors
        assert len(_checks(result, "manifest_symlink")) == 1

    @pytest.mark.parametrize(
        ("size", "severity"),
        [(32 * 1024 + 1, "low"), (50 * 1024 + 1, "medium"), (256_001, "high")],
    )
    def test_manifest_size_thresholds(self, tmp_path: Path, size: int, severity: str):
        skill_dir = _write_skill(tmp_path, _BASE_FM)
        manifest = skill_dir / "SKILL.md"
        padding = size - len(manifest.read_bytes())
        manifest.write_text(manifest.read_text() + "x" * padding)

        findings = _checks(SchemaValidator().validate(skill_dir), "manifest_size")

        assert [f.severity for f in findings] == [severity]
        assert findings[0].metadata["size_bytes"] == size

    def test_manifest_at_low_threshold_has_no_size_finding(self, tmp_path: Path):
        skill_dir = _write_skill(tmp_path, _BASE_FM)
        manifest = skill_dir / "SKILL.md"
        manifest.write_text(manifest.read_text() + "x" * (32 * 1024 - len(manifest.read_bytes())))

        assert not _checks(SchemaValidator().validate(skill_dir), "manifest_size")

    def test_nested_manifest_is_medium(self, tmp_path: Path):
        skill_dir = _write_skill(tmp_path, _BASE_FM)
        (skill_dir / "references" / "inner").mkdir(parents=True)
        (skill_dir / "references" / "inner" / "SKILL.md").write_text("---\nname: inner\n---\n")

        findings = _checks(SchemaValidator().validate(skill_dir), "nested_manifest")

        assert len(findings) == 1
        assert "references/inner/SKILL.md" in findings[0].message

    def test_nested_lowercase_manifest_is_reported_once_with_its_own_name(self, tmp_path: Path):
        """On macOS and Windows one nested file answers to both SKILL.md and skill.md."""
        skill_dir = _write_skill(tmp_path, _BASE_FM)
        (skill_dir / "references" / "inner").mkdir(parents=True)
        (skill_dir / "references" / "inner" / "skill.md").write_text("---\nname: inner\n---\n")

        findings = _checks(SchemaValidator().validate(skill_dir), "nested_manifest")

        assert [f.file_path for f in findings] == [str(skill_dir / "references" / "inner" / "skill.md")]

    def test_nested_manifest_under_evals_is_reported(self, tmp_path: Path):
        skill_dir = _write_skill(tmp_path, _BASE_FM)
        (skill_dir / "evals" / "snapshot").mkdir(parents=True)
        (skill_dir / "evals" / "snapshot" / "SKILL.md").write_text("---\nname: snapshot\n---\n")

        findings = _checks(SchemaValidator().validate(skill_dir), "nested_manifest")

        assert [f.file_path for f in findings] == [str(skill_dir / "evals" / "snapshot" / "SKILL.md")]

    def test_nested_manifest_under_dependency_dir_is_ignored(self, tmp_path: Path):
        skill_dir = _write_skill(tmp_path, _BASE_FM)
        (skill_dir / "node_modules" / "pkg").mkdir(parents=True)
        (skill_dir / "node_modules" / "pkg" / "SKILL.md").write_text("---\nname: pkg\n---\n")

        assert not _checks(SchemaValidator().validate(skill_dir), "nested_manifest")

    def test_description_over_utf16_limit_is_medium(self, tmp_path: Path):
        description = "Use when testing. " + "\U0001f600" * 600
        fm = _BASE_FM.replace("Use when checking harness portability of a skill.", description)
        result = SchemaValidator().validate(_write_skill(tmp_path, fm))

        findings = _checks(result, "description_utf16_length")
        assert result.passed, result.errors
        assert [f.metadata["utf16_units"] for f in findings] == [len(description.encode("utf-16-le")) // 2]

    def test_bmp_description_at_limit_has_no_utf16_finding(self, tmp_path: Path):
        fm = _BASE_FM.replace("Use when checking harness portability of a skill.", "Use when x. " + "a" * 1012)
        result = SchemaValidator().validate(_write_skill(tmp_path, fm))

        assert not _checks(result, "description_utf16_length")

    @pytest.mark.parametrize(
        ("description", "expected"),
        [
            ("Formats tables. Use when the user asks for a table.", False),
            ("Use this skill to format tables in reports.", False),
            ("Formats tables, charts and other structured report content nicely. Use when needed.", True),
            ("Formats tables in reports.", True),
            ("Formats tables, charts and structured report sections. Use when needed.", True),
        ],
    )
    def test_description_trigger_window(self, tmp_path: Path, description: str, expected: bool):
        fm = _BASE_FM.replace("Use when checking harness portability of a skill.", description)
        findings = _checks(SchemaValidator().validate(_write_skill(tmp_path, fm)), "description_trigger")

        assert bool(findings) is expected
        assert all(f.severity == "low" for f in findings)

    def test_unknown_top_level_fields_are_low(self, tmp_path: Path):
        fm = _BASE_FM + "owner: team-x\nversion: 1.0.0\nlicense: MIT\nallowed-tools: Read\n"
        result = SchemaValidator().validate(_write_skill(tmp_path, fm))

        findings = _checks(result, "unknown_field")
        assert result.passed, result.errors
        assert [f.metadata["fields"] for f in findings] == [["owner", "version"]]

    def test_portability_severity_can_be_overridden_by_policy(self, tmp_path: Path):
        from skillevaluator.validators.policy import ValidationPolicy

        skill_dir = _write_skill(tmp_path, _BASE_FM)
        manifest = skill_dir / "SKILL.md"
        manifest.write_bytes(b"\xef\xbb\xbf" + manifest.read_bytes())
        policy = ValidationPolicy(severity_overrides={"SCHEMA.manifest_bom": Severity.HIGH})
        result = SchemaValidator(policy=policy).validate(skill_dir)

        assert not result.passed
        assert [f.severity for f in _checks(result, "manifest_bom")] == ["high"]