Skip to content

armgit / dify

Public Imported from GitHub
0

Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.

17.1 KB
Raw Blame
"""Validate and normalize uploaded Skill packages.

A Skill is a ``.zip`` / ``.skill`` archive that must contain a ``SKILL.md`` entry
file (Anthropic Skills convention: YAML frontmatter with ``name`` + ``description``,
followed by markdown instructions). This service validates the archive (extension,
size, zip integrity, zip-slip safety, SKILL.md presence/encoding/fields),
normalizes retained member paths relative to the selected skill root, rebuilds
canonical archive bytes, and returns normalized metadata together with the
archive-root ``SKILL.md`` bytes.

It does NOT execute or load the skill — the agent backend owns execution. It also
does not persist anything into Agent Soul or bind anything to config versions;
``ConfigSkillNormalizeService`` consumes the normalized package for Agent config.
"""

from __future__ import annotations

import hashlib
import io
import posixpath
import zipfile
import zlib
from dataclasses import dataclass

import yaml
from pydantic import BaseModel, Field, ValidationError, field_validator

from configs import dify_config
from libs.zip_filename_recovery import open_zip_with_replacement_names

_ALLOWED_EXTENSIONS = (".zip", ".skill")
_SKILL_MD_NAME = "SKILL.md"
_SKILL_NAME_PATTERN = r"^[a-z0-9]+(?:-[a-z0-9]+)*$"
_MAX_SKILL_DESCRIPTION_LENGTH = 1024


class SkillPackageError(Exception):
    """A skill-package validation failure mapped to an HTTP status by the controller."""

    code: str
    message: str
    status_code: int

    def __init__(self, code: str, message: str, *, status_code: int = 400) -> None:
        super().__init__(message)
        self.code = code
        self.message = message
        self.status_code = status_code


class _SkillMetadata(BaseModel):
    name: str = Field(min_length=1, max_length=64, pattern=_SKILL_NAME_PATTERN)
    description: str = Field(min_length=1, max_length=_MAX_SKILL_DESCRIPTION_LENGTH)

    @field_validator("name", "description", mode="before")
    @classmethod
    def _strip_required_string(cls, value: object) -> object:
        return value.strip() if isinstance(value, str) else value


class SkillPackageInspection(_SkillMetadata):
    """Metadata read without rebuilding or fully extracting a Skill archive."""

    uncompressed_size: int = Field(ge=0)


class SkillManifest(_SkillMetadata):
    """Validated metadata extracted from a Skill package."""

    entry_path: str  # path of SKILL.md inside the archive
    files: list[str]  # all (safe) file paths inside the archive
    size: int  # total uncompressed bytes
    hash: str  # sha256 of the archive bytes


class NormalizedSkillPackage(BaseModel):
    """Canonical skill package bytes and metadata ready to store as Agent config."""

    manifest: SkillManifest
    archive_bytes: bytes


@dataclass(frozen=True)
class _InspectedSkillArchive:
    metadata: SkillPackageInspection
    normalized_members: dict[str, zipfile.ZipInfo]
    skill_md_bytes: bytes


class SkillPackageService:
    """Validate Skill archives and produce a normalized package."""

    def inspect(self, *, content: bytes, filename: str) -> SkillPackageInspection:
        """Validate archive metadata and SKILL.md without rebuilding the archive."""
        archive = self._open_archive(content=content, filename=filename)
        with archive:
            return self._inspect_archive(archive).metadata

    def validate_and_normalize(self, *, content: bytes, filename: str) -> NormalizedSkillPackage:
        """Return the canonical package for an uploaded skill archive.

        The shallowest ``SKILL.md`` defines the skill root. When exactly one
        depth-2 ``<folder>/SKILL.md`` exists, normalization strips that top-level
        folder and silently discards all members outside it, including nested
        foreign paths. When that unique depth-2 condition does not apply, files
        outside the selected skill root still raise ``files_outside_skill_root``.
        The returned manifest is normalized to archive-root ``SKILL.md`` and its
        hash describes the rebuilt archive bytes. Member read/decompression
        failures while consuming the archive are mapped to ``invalid_archive``.
        Malformed UTF-8 member names are decoded with replacement characters;
        recovered path collisions still fail the normal duplicate-path check.
        """
        archive = self._open_archive(content=content, filename=filename)
        with archive:
            inspection = self._inspect_archive(archive)
            normalized_archive_bytes = self._build_normalized_archive(
                archive=archive,
                normalized_members=inspection.normalized_members,
                skill_md_bytes=inspection.skill_md_bytes,
            )

        if len(normalized_archive_bytes) > dify_config.UPLOAD_SKILL_FILE_SIZE_LIMIT * 1024 * 1024:
            raise SkillPackageError("archive_too_large", "normalized skill archive exceeds size limit", status_code=400)
        try:
            manifest = SkillManifest(
                name=inspection.metadata.name,
                description=inspection.metadata.description,
                entry_path=_SKILL_MD_NAME,
                files=sorted(inspection.normalized_members),
                size=sum(
                    len(inspection.skill_md_bytes) if path == _SKILL_MD_NAME else info.file_size
                    for path, info in inspection.normalized_members.items()
                ),
                hash=hashlib.sha256(normalized_archive_bytes).hexdigest(),
            )
        except ValidationError as exc:
            raise self._manifest_validation_error(exc) from exc
        return NormalizedSkillPackage(
            manifest=manifest,
            archive_bytes=normalized_archive_bytes,
        )

    def _inspect_archive(self, archive: zipfile.ZipFile) -> _InspectedSkillArchive:
        members = self._collect_file_members(archive)
        member_paths = [safe_path for _, safe_path in members]
        entry_path = self._find_skill_md(member_paths)
        strip_prefix = self._skill_root_prefix(entry_path)
        normalized_members = self._normalize_members(
            members=members,
            skill_root_prefix=strip_prefix,
            ignore_outside_selected_root=self._can_strip_single_top_level_folder(
                paths=member_paths, entry_path=entry_path
            ),
        )
        skill_md_member = normalized_members[_SKILL_MD_NAME]
        self._validate_skill_md_size(skill_md_member)
        raw_skill_md_bytes = self._read_member_bytes_from_archive(archive, member_info=skill_md_member)
        skill_md = self._decode_skill_md(raw_skill_md_bytes)
        skill_md_bytes = skill_md.encode("utf-8")
        name, description = self._parse_skill_md(skill_md)
        try:
            metadata = SkillPackageInspection(
                name=name,
                description=description,
                uncompressed_size=sum(info.file_size for info, _ in members),
            )
        except ValidationError as exc:
            raise self._manifest_validation_error(exc) from exc
        return _InspectedSkillArchive(
            metadata=metadata,
            normalized_members=normalized_members,
            skill_md_bytes=skill_md_bytes,
        )

    @staticmethod
    def _manifest_validation_error(exc: ValidationError) -> SkillPackageError:
        first_error = exc.errors()[0]
        loc = first_error["loc"]
        field = loc[0] if loc else "manifest"
        error_type = first_error["type"]
        if field == "name":
            code = "missing_skill_name" if error_type == "string_too_short" else "invalid_skill_name"
            message = (
                "SKILL.md frontmatter name is required"
                if code == "missing_skill_name"
                else "SKILL.md frontmatter name must be lowercase letters, numbers, and hyphens only, "
                "must not start or end with a hyphen, and must be at most 64 characters"
            )
            return SkillPackageError(code, message, status_code=400)
        if field == "description":
            code = "missing_skill_description" if error_type == "string_too_short" else "invalid_skill_description"
            message = (
                "SKILL.md frontmatter description is required"
                if code == "missing_skill_description"
                else f"SKILL.md frontmatter description must be at most {_MAX_SKILL_DESCRIPTION_LENGTH} characters"
            )
            return SkillPackageError(code, message, status_code=400)
        return SkillPackageError("invalid_skill_manifest", "SKILL.md frontmatter is invalid", status_code=400)

    def _open_archive(self, *, content: bytes, filename: str) -> zipfile.ZipFile:
        self._check_extension(filename)
        if not content:
            raise SkillPackageError("empty_archive", "skill archive is empty", status_code=400)
        max_archive_bytes = dify_config.UPLOAD_SKILL_FILE_SIZE_LIMIT * 1024 * 1024
        if len(content) > max_archive_bytes:
            raise SkillPackageError("archive_too_large", "skill archive exceeds size limit", status_code=400)

        try:
            return open_zip_with_replacement_names(content)
        except zipfile.BadZipFile as exc:
            raise SkillPackageError("invalid_archive", "skill archive is not a valid zip", status_code=400) from exc

    def _collect_file_members(self, archive: zipfile.ZipFile) -> list[tuple[zipfile.ZipInfo, str]]:
        archive_infos = archive.infolist()
        if len(archive_infos) > dify_config.SKILL_PACKAGE_MAX_ENTRIES:
            raise SkillPackageError("too_many_entries", "skill archive has too many files", status_code=400)
        infos = [info for info in archive_infos if not info.is_dir()]

        members: list[tuple[zipfile.ZipInfo, str]] = []
        total_uncompressed = 0
        for info in infos:
            members.append((info, self._safe_member_path(info.filename)))
            total_uncompressed += info.file_size
            if info.file_size and info.compress_size == 0:
                raise SkillPackageError("invalid_archive", "skill package has invalid ZIP metadata", status_code=400)
        if total_uncompressed > dify_config.SKILL_PACKAGE_MAX_UNCOMPRESSED_BYTES:
            raise SkillPackageError(
                "archive_too_large",
                "skill archive uncompressed size exceeds limit",
                status_code=400,
            )
        return members

    @staticmethod
    def _skill_root_prefix(entry_path: str) -> str | None:
        skill_root = posixpath.dirname(entry_path)
        if not skill_root:
            return None
        return f"{skill_root}/"

    def _normalize_members(
        self,
        *,
        members: list[tuple[zipfile.ZipInfo, str]],
        skill_root_prefix: str | None,
        ignore_outside_selected_root: bool = False,
    ) -> dict[str, zipfile.ZipInfo]:
        normalized_members: dict[str, zipfile.ZipInfo] = {}
        for info, safe_path in members:
            if skill_root_prefix is not None:
                if not safe_path.startswith(skill_root_prefix):
                    if ignore_outside_selected_root:
                        continue
                    raise SkillPackageError(
                        "files_outside_skill_root",
                        (
                            "skill package must contain exactly one skill; "
                            "multiple skill folders in one archive are not supported"
                        ),
                        status_code=400,
                    )
                normalized_path = safe_path.removeprefix(skill_root_prefix)
            else:
                normalized_path = safe_path

            # Paths are already normalized and safe; "." is not a file.
            if normalized_path == ".":
                raise SkillPackageError("unsafe_path", "skill archive contains an unsafe path", status_code=400)
            if normalized_path in normalized_members:
                raise SkillPackageError(
                    "duplicate_member_path",
                    "skill archive contains duplicate normalized paths",
                    status_code=400,
                )
            normalized_members[normalized_path] = info

        if _SKILL_MD_NAME not in normalized_members:
            raise SkillPackageError("missing_skill_md", "skill archive must contain a SKILL.md", status_code=400)
        return normalized_members

    def _build_normalized_archive(
        self,
        *,
        archive: zipfile.ZipFile,
        normalized_members: dict[str, zipfile.ZipInfo],
        skill_md_bytes: bytes,
    ) -> bytes:
        output = io.BytesIO()
        with zipfile.ZipFile(output, "w", compression=zipfile.ZIP_DEFLATED) as normalized_archive:
            for normalized_path in sorted(normalized_members):
                payload = (
                    skill_md_bytes
                    if normalized_path == _SKILL_MD_NAME
                    else self._read_member_bytes_from_archive(archive, member_info=normalized_members[normalized_path])
                )
                normalized_archive.writestr(normalized_path, payload)
        return output.getvalue()

    @staticmethod
    def _check_extension(filename: str) -> None:
        lowered = (filename or "").lower()
        if not lowered.endswith(_ALLOWED_EXTENSIONS):
            raise SkillPackageError(
                "unsupported_extension",
                f"skill must be one of {', '.join(_ALLOWED_EXTENSIONS)}",
                status_code=400,
            )

    @staticmethod
    def _safe_member_path(name: str) -> str:
        """Reject zip-slip and normalize the archive member path."""
        if "\x00" in name or "\\" in name:
            raise SkillPackageError("unsafe_path", "skill archive contains an unsafe path", status_code=400)
        normalized = posixpath.normpath(name)
        if normalized.startswith("/") or normalized == ".." or normalized.startswith("../"):
            raise SkillPackageError("unsafe_path", "skill archive contains an unsafe path", status_code=400)
        return normalized

    @staticmethod
    def _find_skill_md(paths: list[str]) -> str:
        candidates = [p for p in paths if posixpath.basename(p) == _SKILL_MD_NAME]
        if not candidates:
            raise SkillPackageError("missing_skill_md", "skill archive must contain a SKILL.md", status_code=400)
        # Prefer the shallowest SKILL.md (skill root).
        return min(candidates, key=lambda p: (p.count("/"), len(p)))

    @staticmethod
    def _can_strip_single_top_level_folder(*, paths: list[str], entry_path: str) -> bool:
        if entry_path.count("/") != 1:
            return False
        candidates = [path for path in paths if path.count("/") == 1 and posixpath.basename(path) == _SKILL_MD_NAME]
        return len(candidates) == 1 and candidates[0] == entry_path

    @staticmethod
    def _read_member_bytes_from_archive(archive: zipfile.ZipFile, *, member_info: zipfile.ZipInfo) -> bytes:
        try:
            return archive.read(member_info)
        except (zipfile.BadZipFile, EOFError, OSError, RuntimeError, ValueError, zlib.error) as exc:
            raise SkillPackageError("invalid_archive", "skill archive is not a valid zip", status_code=400) from exc

    @staticmethod
    def _validate_skill_md_size(member_info: zipfile.ZipInfo) -> None:
        if member_info.file_size > dify_config.SKILL_PACKAGE_MAX_SKILL_MD_BYTES:
            raise SkillPackageError("skill_md_too_large", "SKILL.md exceeds size limit", status_code=400)

    @staticmethod
    def _decode_skill_md(raw: bytes) -> str:
        try:
            decoded = raw.decode("utf-8")
        except UnicodeDecodeError as exc:
            raise SkillPackageError("skill_md_not_utf8", "SKILL.md must be UTF-8 encoded", status_code=400) from exc
        # Windows editors often save SKILL.md with CRLF; normalize before frontmatter parse.
        return decoded.replace("\r\n", "\n").replace("\r", "\n")

    @classmethod
    def _parse_skill_md(cls, content: str) -> tuple[str, str]:
        if not content.strip():
            raise SkillPackageError("empty_skill_md", "SKILL.md is empty", status_code=400)
        frontmatter = cls._parse_frontmatter(content)
        name = str(frontmatter.get("name") or "").strip()
        description = str(frontmatter.get("description") or "").strip()
        return name, description

    @staticmethod
    def _parse_frontmatter(content: str) -> dict[str, object]:
        # Delimiters are standalone lines. A literal "---" inside a YAML value must not end frontmatter.
        lines = content.split("\n")
        if not lines or lines[0].strip() != "---":
            return {}
        closing_index = None
        for index, line in enumerate(lines[1:], start=1):
            if line.strip() == "---":
                closing_index = index
                break
        if closing_index is None:
            return {}
        try:
            loaded = yaml.safe_load("\n".join(lines[1:closing_index]))
        except yaml.YAMLError as exc:
            raise SkillPackageError(
                "invalid_frontmatter", "SKILL.md frontmatter is not valid YAML", status_code=400
            ) from exc
        return loaded if isinstance(loaded, dict) else {}


__all__ = [
    "NormalizedSkillPackage",
    "SkillManifest",
    "SkillPackageError",
    "SkillPackageInspection",
    "SkillPackageService",
]