Skip to content

armgit / dify

Public Imported from GitHub
0

Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.

15.1 KB
Raw Blame
"""Unit tests for the Skill package validator/normalizer (ENG-370)."""

from __future__ import annotations

import hashlib
import io
import zipfile
import zlib

import pytest

from services.agent.skill_package_service import NormalizedSkillPackage, SkillPackageError, SkillPackageService
from tests.helpers.unit.config_override import apply_config_overrides

_SKILL_MD = """---
name: pdf-toolkit
description: Tools for working with PDF files.
---

# PDF Toolkit

Do things with PDFs.
"""


def _zip(members: dict[str, bytes], *, compression: int = zipfile.ZIP_DEFLATED) -> bytes:
    buffer = io.BytesIO()
    with zipfile.ZipFile(buffer, "w", compression=compression) as archive:
        for name, data in members.items():
            archive.writestr(name, data)
    return buffer.getvalue()


def _normalize(members: dict[str, bytes], *, filename: str = "skill.zip") -> NormalizedSkillPackage:
    return SkillPackageService().validate_and_normalize(content=_zip(members), filename=filename)


def test_normalize_replaces_invalid_utf8_member_names() -> None:
    original = _zip(
        {
            "SKILL.md": _SKILL_MD.encode(),
            "scripts/\u00e9.py": b"print('keep payload')\n",
            "assets/\u00f6.txt": b"valid unicode name",
            "assets/X.txt": b"legacy encoded name",
        }
    )
    damaged = original.replace(b"\xc3\xa9.py", b"\xffa.py").replace(b"assets/X.txt", b"assets/\x82.txt")
    service = SkillPackageService()
    assert service.inspect(content=damaged, filename="skill.zip").name == "pdf-toolkit"
    normalized = service.validate_and_normalize(content=damaged, filename="skill.zip")
    with zipfile.ZipFile(io.BytesIO(normalized.archive_bytes)) as archive:
        assert archive.read("scripts/\ufffda.py") == b"print('keep payload')\n"
        assert archive.read("assets/\u00f6.txt") == b"valid unicode name"
        assert archive.read("assets/\u00e9.txt") == b"legacy encoded name"
        assert archive.testzip() is None


def test_normalize_recovers_zip64_skill_names(monkeypatch: pytest.MonkeyPatch) -> None:
    with monkeypatch.context() as scoped:
        scoped.setattr(zipfile, "ZIP64_LIMIT", 0)
        original = _zip({"SKILL.md": _SKILL_MD.encode(), "scripts/\u00e9.py": b"print('ok')\n"})
    damaged = original.replace(b"\xc3\xa9.py", b"\xffa.py")
    service = SkillPackageService()
    assert service.inspect(content=damaged, filename="skill.zip").name == "pdf-toolkit"
    normalized = service.validate_and_normalize(content=damaged, filename="skill.zip")
    with zipfile.ZipFile(io.BytesIO(normalized.archive_bytes)) as archive:
        assert archive.read("scripts/\ufffda.py") == b"print('ok')\n"


def test_replacement_member_names_must_remain_unique() -> None:
    original = _zip(
        {
            "SKILL.md": _SKILL_MD.encode(),
            "scripts/\u00e9.py": b"one",
            "scripts/\u00f6.py": b"two",
        }
    )
    damaged = original.replace(b"\xc3\xa9.py", b"\xffa.py").replace(b"\xc3\xb6.py", b"\xfea.py")
    with pytest.raises(SkillPackageError) as error:
        SkillPackageService().validate_and_normalize(content=damaged, filename="skill.zip")
    assert error.value.code == "duplicate_member_path"


def test_replacement_names_do_not_hide_local_header_mismatch() -> None:
    original = _zip({"SKILL.md": _SKILL_MD.encode(), "\u00e9.py": b"payload"})
    # Damage only the central name, leaving the local header unchanged.
    central = original.index(b"PK\x01\x02")
    damaged = original[:central] + original[central:].replace(b"\xc3\xa9.py", b"\xffa.py")
    with pytest.raises(SkillPackageError) as error:
        SkillPackageService().validate_and_normalize(content=damaged, filename="skill.zip")
    assert error.value.code == "invalid_archive"


def test_normalized_highly_compressible_skill_passes_reinspection() -> None:
    payload = b"x" * (1024 * 1024)
    content = _zip({"SKILL.md": _SKILL_MD.encode(), "data.bin": payload}, compression=zipfile.ZIP_STORED)
    service = SkillPackageService()
    normalized = service.validate_and_normalize(content=content, filename="skill.zip")
    inspection = service.inspect(content=normalized.archive_bytes, filename="skill.zip")
    assert inspection.name == normalized.manifest.name
    with zipfile.ZipFile(io.BytesIO(normalized.archive_bytes)) as archive:
        assert archive.getinfo("data.bin").compress_type == zipfile.ZIP_DEFLATED
        assert archive.read("data.bin") == payload


def test_inspect_reports_uncompressed_size_without_rebuilding(monkeypatch: pytest.MonkeyPatch) -> None:
    members = {"SKILL.md": _SKILL_MD.encode(), "scripts/run.py": b"print('hi')\n"}

    def fail_if_called(*_args: object, **_kwargs: object) -> bytes:
        raise AssertionError("inspection must not rebuild the archive")

    monkeypatch.setattr(SkillPackageService, "_build_normalized_archive", fail_if_called)

    inspection = SkillPackageService().inspect(content=_zip(members), filename="skill.zip")

    assert inspection.name == "pdf-toolkit"
    assert inspection.uncompressed_size == sum(len(content) for content in members.values())


@pytest.mark.parametrize("prefix", ["", "pdf-toolkit/"])
@pytest.mark.parametrize("crlf", [False, True])
def test_inspection_and_normalization_keep_distinct_size_meanings(prefix: str, crlf: bool) -> None:
    skill_md = _SKILL_MD.replace("\n", "\r\n") if crlf else _SKILL_MD
    script = b"print('hi')\n"
    members = {f"{prefix}SKILL.md": skill_md.encode(), f"{prefix}scripts/run.py": script}
    if prefix:
        members["README.md"] = b"ignored outside the selected root"
    content = _zip(members)
    service = SkillPackageService()
    inspection = service.inspect(content=content, filename="skill.zip")
    normalized = service.validate_and_normalize(content=content, filename="skill.zip")
    assert inspection.uncompressed_size == sum(len(value) for value in members.values())
    expected_size = len(_SKILL_MD.encode()) + len(script)
    assert normalized.manifest.size == expected_size
    assert normalized.manifest.name == "pdf-toolkit"
    assert normalized.manifest.description == "Tools for working with PDF files."
    with zipfile.ZipFile(io.BytesIO(normalized.archive_bytes)) as archive:
        assert sum(info.file_size for info in archive.infolist()) == expected_size
        assert archive.read("SKILL.md") == _SKILL_MD.encode()
    if prefix or crlf:
        assert inspection.uncompressed_size > normalized.manifest.size


def test_shallowest_skill_md_preferred_during_normalization() -> None:
    manifest = _normalize({"SKILL.md": _SKILL_MD.encode(), "nested/SKILL.md": _SKILL_MD.encode()}).manifest
    assert manifest.entry_path == "SKILL.md"
    assert manifest.files == ["SKILL.md", "nested/SKILL.md"]


@pytest.mark.parametrize(
    ("prefix", "extra_members"),
    [
        pytest.param("", {}, id="root"),
        pytest.param("pdf-toolkit/", {}, id="single-folder"),
        pytest.param("pdf-toolkit/", {"README.md": b"bundle notes\n"}, id="root-outsider"),
        pytest.param("pdf-toolkit/", {".": b"x"}, id="dot-outsider"),
        pytest.param("pdf-toolkit/", {"bundle/other.txt": b"x"}, id="nested-outsider"),
        pytest.param("bundle/pdf-toolkit/", {}, id="deep-root"),
    ],
)
def test_normalization_preserves_selected_skill_contents(prefix: str, extra_members: dict[str, bytes]) -> None:
    expected = {"SKILL.md": _SKILL_MD.encode(), "scripts/run.py": b"print('hi')\n"}
    original = _zip({**{prefix + path: content for path, content in expected.items()}, **extra_members})
    package = SkillPackageService().validate_and_normalize(content=original, filename="skill.zip")

    assert package.manifest.name == "pdf-toolkit"
    assert package.manifest.description == "Tools for working with PDF files."
    assert package.manifest.size == sum(len(content) for content in expected.values())
    assert package.manifest.entry_path == "SKILL.md"
    assert package.manifest.files == sorted(expected)
    with zipfile.ZipFile(io.BytesIO(package.archive_bytes)) as archive:
        assert archive.namelist() == sorted(expected)
        assert {path: archive.read(path) for path in archive.namelist()} == expected
    assert package.manifest.hash == hashlib.sha256(package.archive_bytes).hexdigest()
    if prefix:
        assert package.manifest.hash != hashlib.sha256(original).hexdigest()


@pytest.mark.parametrize(
    ("members", "filename", "code"),
    [
        ({"README.md": b"x"}, "skill.zip", "missing_skill_md"),
        ({"SKILL.md": _SKILL_MD.encode(), ".": b"x"}, "skill.zip", "unsafe_path"),
        ({"SKILL.md": _SKILL_MD.encode()}, "skill.tar", "unsupported_extension"),
        ({"SKILL.md": b""}, "skill.zip", "empty_skill_md"),
        ({"SKILL.md": b"---\ndescription: valid\n---\n# no name here"}, "skill.zip", "missing_skill_name"),
        ({"SKILL.md": b"---\nname: pdf-toolkit\n---\n# no description"}, "skill.zip", "missing_skill_description"),
        (
            {"SKILL.md": b"---\nname: PDF Toolkit\ndescription: valid\n---\n# invalid name"},
            "skill.zip",
            "invalid_skill_name",
        ),
        (
            {"SKILL.md": f"---\nname: pdf-toolkit\ndescription: {'x' * 1025}\n---\n# long".encode()},
            "skill.zip",
            "invalid_skill_description",
        ),
        ({"SKILL.md": b"\xff\xfenot utf8"}, "skill.zip", "skill_md_not_utf8"),
        pytest.param({"SKILL.md": b"# heading-name\n\nbody"}, "skill.zip", "missing_skill_name", id="no-frontmatter"),
        pytest.param(
            {
                "pdf-toolkit/SKILL.md": _SKILL_MD.encode(),
                "pdf-toolkit/scripts/run.py": b"print('hi')\n",
                "other-tool/SKILL.md": _SKILL_MD.encode(),
            },
            "skill.zip",
            "files_outside_skill_root",
            id="multiple-skill-roots",
        ),
        pytest.param({"../evil.txt": b"x", "SKILL.md": _SKILL_MD.encode()}, "skill.zip", "unsafe_path", id="zip-slip"),
        pytest.param(
            {"SKILL.md": b"---\n: : : not yaml\n---\n# x\n"},
            "skill.zip",
            "invalid_frontmatter",
            id="invalid-frontmatter",
        ),
        pytest.param(
            {"SKILL.md": b"---\n# heading-wins\nbody"}, "skill.zip", "missing_skill_name", id="unterminated-frontmatter"
        ),
        pytest.param(
            {"bundle/pdf-toolkit/SKILL.md": _SKILL_MD.encode(), "README.md": b"x"},
            "skill.zip",
            "files_outside_skill_root",
            id="outside-deep-root",
        ),
        pytest.param(
            {
                "pdf-toolkit/SKILL.md": _SKILL_MD.encode(),
                "pdf-toolkit/scripts/run.py": b"print('x')\n",
                "pdf-toolkit/scripts/./run.py": b"print('y')\n",
            },
            "skill.zip",
            "duplicate_member_path",
            id="duplicate-normalized-path",
        ),
    ],
)
def test_invalid_packages_rejected(members: dict[str, bytes], filename: str, code: str) -> None:
    with pytest.raises(SkillPackageError) as exc_info:
        _normalize(members, filename=filename)
    assert exc_info.value.code == code
    assert exc_info.value.status_code == 400


@pytest.mark.parametrize(
    "description_line",
    [
        'description: "Compare before---after reports."',
        "description: Compare before---after reports.",
    ],
)
def test_description_keeps_literal_dashes_inside_frontmatter(description_line: str) -> None:
    skill_md = f"---\nname: toolkit\n{description_line}\n---\n# Instructions\nCompare reports.\n"
    package = _normalize({"SKILL.md": skill_md.encode()})
    assert package.manifest.name == "toolkit"
    assert package.manifest.description == "Compare before---after reports."


def test_opening_delimiter_with_trailing_text_is_not_frontmatter() -> None:
    skill_md = b"---junk\nname: toolkit\ndescription: valid\n---\n# body\n"
    with pytest.raises(SkillPackageError) as exc_info:
        _normalize({"SKILL.md": skill_md})
    assert exc_info.value.code == "missing_skill_name"


def test_non_zip_content_rejected() -> None:
    with pytest.raises(SkillPackageError) as exc_info:
        SkillPackageService().validate_and_normalize(content=b"not a zip", filename="skill.zip")
    assert exc_info.value.code == "invalid_archive"


def test_empty_archive_rejected() -> None:
    with pytest.raises(SkillPackageError) as exc_info:
        SkillPackageService().validate_and_normalize(content=b"", filename="skill.zip")
    assert exc_info.value.code == "empty_archive"


def test_validate_and_normalize_rejects_skill_md_too_large(monkeypatch: pytest.MonkeyPatch) -> None:
    apply_config_overrides(monkeypatch, SKILL_PACKAGE_MAX_SKILL_MD_BYTES=8)

    with pytest.raises(SkillPackageError) as exc_info:
        _normalize({"SKILL.md": _SKILL_MD.encode()})
    assert exc_info.value.code == "skill_md_too_large"


def test_validate_and_normalize_rejects_too_many_entries(monkeypatch: pytest.MonkeyPatch) -> None:
    apply_config_overrides(monkeypatch, SKILL_PACKAGE_MAX_ENTRIES=1)

    with pytest.raises(SkillPackageError) as exc_info:
        _normalize({"SKILL.md": _SKILL_MD.encode(), "assets/": b""})
    assert exc_info.value.code == "too_many_entries"


def test_validate_and_normalize_rejects_archive_too_large_uncompressed(monkeypatch: pytest.MonkeyPatch) -> None:
    apply_config_overrides(monkeypatch, SKILL_PACKAGE_MAX_UNCOMPRESSED_BYTES=32)

    with pytest.raises(SkillPackageError) as exc_info:
        _normalize({"SKILL.md": _SKILL_MD.encode(), "scripts/run.py": b"x" * 33})
    assert exc_info.value.code == "archive_too_large"


def test_validate_and_normalize_rejects_archive_too_large_uploaded_bytes(monkeypatch: pytest.MonkeyPatch) -> None:
    apply_config_overrides(monkeypatch, UPLOAD_SKILL_FILE_SIZE_LIMIT=1)

    with pytest.raises(SkillPackageError) as exc_info:
        SkillPackageService().validate_and_normalize(content=b"x" * (1024 * 1024 + 1), filename="skill.zip")
    assert exc_info.value.code == "archive_too_large"


def test_high_compression_ratio_uses_absolute_expansion_limit(monkeypatch: pytest.MonkeyPatch) -> None:
    content = _zip({"SKILL.md": _SKILL_MD.encode(), "data.bin": b"x" * (1024 * 1024)})
    service = SkillPackageService()
    assert service.inspect(content=content, filename="skill.zip").name == "pdf-toolkit"
    apply_config_overrides(monkeypatch, SKILL_PACKAGE_MAX_UNCOMPRESSED_BYTES=1024)
    with pytest.raises(SkillPackageError) as exc_info:
        service.inspect(content=content, filename="skill.zip")
    assert exc_info.value.code == "archive_too_large"


def test_validate_and_normalize_maps_member_decompression_failures_to_invalid_archive(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    original_read = zipfile.ZipFile.read

    def corrupted_read(self: zipfile.ZipFile, member: str | zipfile.ZipInfo, pwd: bytes | None = None) -> bytes:
        filename = member.filename if isinstance(member, zipfile.ZipInfo) else member
        if filename == "scripts/run.py":
            raise zlib.error("invalid distance too far back")
        return original_read(self, member, pwd=pwd)

    monkeypatch.setattr(zipfile.ZipFile, "read", corrupted_read)

    with pytest.raises(SkillPackageError) as exc_info:
        _normalize({"SKILL.md": _SKILL_MD.encode(), "scripts/run.py": b"print('x')\n"})
    assert exc_info.value.code == "invalid_archive"
    assert exc_info.value.message == "skill archive is not a valid zip"