Skip to content

Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.

22.8 KB
Raw Blame
"""Validation and readiness helpers for manifest-declared Hugging Face sources."""

from __future__ import annotations

import math
import re
import unicodedata
from pathlib import Path
from typing import Any


_SAFE_ID = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]*$")
_WINDOWS_DEVICE = re.compile(
    r"^(?:con|prn|aux|nul|com[1-9]|lpt[1-9])(?:\..*)?$", re.IGNORECASE
)
_WINDOWS_UNSAFE = re.compile(r'[<>"|?*\x00-\x1f]')


def _portable_segment(value: str, field: str) -> str:
    if (
        not value
        or value in {".", ".."}
        or value.endswith((".", " "))
        or ":" in value
        or _WINDOWS_UNSAFE.search(value)
        or _WINDOWS_DEVICE.fullmatch(value)
    ):
        raise ValueError(f'{field} contains unsafe path segment "{value}"')
    return value


def safe_source_id(value: Any, field: str = "model source id") -> str:
    if (
        not isinstance(value, str)
        or not value
        or value != value.strip()
        or _SAFE_ID.fullmatch(value) is None
    ):
        raise ValueError(f"{field} must be a safe non-empty identifier")
    return _portable_segment(value, field)


def safe_relative_path(value: Any, field: str, *, allow_dot: bool = False) -> str:
    if not isinstance(value, str) or not value or value != value.strip():
        raise ValueError(f"{field} must be a non-empty relative path")
    if allow_dot and value == ".":
        return value
    if value == "." or value.startswith("/") or "\\" in value:
        raise ValueError(f"{field} must be a safe relative POSIX path")
    for part in value.split("/"):
        _portable_segment(part, field)
    return value


def _safe_prefix(value: Any, field: str) -> str:
    path = value[:-1] if isinstance(value, str) and value.endswith("/") else value
    safe_relative_path(path, field)
    return value


def _prefixes(value: Any, field: str) -> list[str] | None:
    if not isinstance(value, list):
        raise ValueError(f"{field} must be an array")
    return [_safe_prefix(entry, f"{field}[{index}]") for index, entry in enumerate(value)]


def _safe_repo_id(value: Any, field: str) -> str:
    if not isinstance(value, str) or not value or value != value.strip() or "\\" in value:
        raise ValueError(f"{field} must be a non-empty Hugging Face repository id")
    parts = value.split("/")
    if len(parts) > 2 or any(
        part in {"", ".", ".."} or _SAFE_ID.fullmatch(part) is None for part in parts
    ):
        raise ValueError(f"{field} is not a safe Hugging Face repository id")
    return value


def _safe_revision(value: Any, field: str) -> str | None:
    if value is None:
        return None
    if (
        not isinstance(value, str)
        or not value
        or value != value.strip()
        or value.startswith("/")
        or "\\" in value
        or "\0" in value
        or any(part in {"", ".", ".."} for part in value.split("/"))
    ):
        raise ValueError(f"{field} must be a safe non-empty revision")
    return value


def normalize_model_sources(
    node: dict[str, Any], *, field_name: str = "model_sources"
) -> list[dict[str, Any]] | None:
    """Validate only the new contract; legacy fields remain untouched."""
    if "model_sources" not in node:
        return None
    raw_sources = node["model_sources"]
    if not isinstance(raw_sources, list) or not raw_sources:
        raise ValueError(f"{field_name} must be a non-empty array")

    aliases: dict[str, str] = {}
    sources: list[dict[str, Any]] = []
    for index, raw in enumerate(raw_sources):
        field = f"{field_name}[{index}]"
        if not isinstance(raw, dict):
            raise ValueError(f"{field} must be an object")
        source_id = safe_source_id(raw.get("id"), f"{field}.id")
        alias = unicodedata.normalize("NFC", source_id).casefold()
        if alias in aliases:
            raise ValueError(
                f'model source ids "{aliases[alias]}" and "{source_id}" are not portable-unique'
            )
        aliases[alias] = source_id
        if raw.get("provider") != "huggingface":
            raise ValueError(f'{field}.provider must be "huggingface"')
        checks = raw.get("checks")
        if not isinstance(checks, list) or not checks:
            raise ValueError(f"{field}.checks must be a non-empty array")

        source: dict[str, Any] = {
            "id": source_id,
            "provider": "huggingface",
            "repo_id": _safe_repo_id(raw.get("repo_id"), f"{field}.repo_id"),
            "destination": safe_relative_path(
                raw.get("destination"), f"{field}.destination", allow_dot=True
            ),
            "checks": [
                safe_relative_path(check, f"{field}.checks[{check_index}]")
                for check_index, check in enumerate(checks)
            ],
        }
        revision = (
            _safe_revision(raw["revision"], f"{field}.revision")
            if "revision" in raw
            else None
        )
        if "revision" in raw and revision is None:
            raise ValueError(f"{field}.revision must be a safe non-empty revision")
        include = (
            _prefixes(raw["include_prefixes"], f"{field}.include_prefixes")
            if "include_prefixes" in raw
            else None
        )
        skip = (
            _prefixes(raw["skip_prefixes"], f"{field}.skip_prefixes")
            if "skip_prefixes" in raw
            else None
        )
        if revision is not None:
            source["revision"] = revision
        if include is not None:
            source["include_prefixes"] = include
        if skip is not None:
            source["skip_prefixes"] = skip
        sources.append(source)
    return sources


def validate_model_node_ids(nodes: list[dict[str, Any]]) -> None:
    """Managed node roots must be unique on case-insensitive filesystems too."""
    seen: set[str] = set()
    for node in nodes:
        if not isinstance(node, dict):
            raise ValueError("model node must be an object")
        if isinstance(node.get("id"), str) and node["id"].casefold() == "_shared":
            raise ValueError('model node id "_shared" is reserved')
        node_id = safe_source_id(node.get("id"), "model node id")
        alias = node_id.casefold()
        if alias in seen:
            raise ValueError(f'model node id "{node_id}" is not portable-unique')
        seen.add(alias)


def normalize_weight_groups(manifest: dict[str, Any]) -> list[dict[str, Any]] | None:
    if "weight_groups" not in manifest:
        return None
    raw_groups = manifest["weight_groups"]
    if not isinstance(raw_groups, list) or not raw_groups:
        raise ValueError("weight_groups must be a non-empty array")

    aliases: dict[str, str] = {}
    groups: list[dict[str, Any]] = []
    for index, raw in enumerate(raw_groups):
        field = f"weight_groups[{index}]"
        if not isinstance(raw, dict):
            raise ValueError(f"{field} must be an object")
        raw_group_id = raw.get("id")
        if isinstance(raw_group_id, str) and raw_group_id.casefold() == "_shared":
            raise ValueError(f'{field}.id uses the reserved identifier "_shared"')
        group_id = safe_source_id(raw_group_id, f"{field}.id")
        alias = unicodedata.normalize("NFC", group_id).casefold()
        if alias in aliases:
            raise ValueError(
                f'weight group ids "{aliases[alias]}" and "{group_id}" '
                "are not portable-unique"
            )
        aliases[alias] = group_id
        sources = normalize_model_sources(
            {"model_sources": raw.get("model_sources")},
            field_name=f"{field}.model_sources",
        )
        groups.append({"id": group_id, "model_sources": sources})
    return groups


def normalize_weight_group_references(
    node: dict[str, Any],
    groups: list[dict[str, Any]] | None,
    *,
    field_name: str = "weight_groups",
) -> list[str] | None:
    if "weight_groups" not in node:
        return None
    raw_refs = node["weight_groups"]
    if not isinstance(raw_refs, list) or not raw_refs:
        raise ValueError(f"{field_name} must be a non-empty array of weight group ids")

    available = {
        unicodedata.normalize("NFC", group["id"]).casefold(): group["id"]
        for group in groups or []
    }
    aliases: dict[str, str] = {}
    refs: list[str] = []
    for index, raw in enumerate(raw_refs):
        group_id = safe_source_id(raw, f"{field_name}[{index}]")
        alias = unicodedata.normalize("NFC", group_id).casefold()
        if alias in aliases:
            raise ValueError(
                f'weight group references "{aliases[alias]}" and "{group_id}" '
                "are not portable-unique"
            )
        aliases[alias] = group_id
        canonical = available.get(alias)
        if canonical is None:
            raise ValueError(
                f'{field_name}[{index}] references unknown weight group "{group_id}"'
            )
        refs.append(canonical)
    return refs


def _path_has_symlink(root: Path, candidate: Path) -> bool:
    root = root.absolute()
    candidate = candidate.absolute()
    try:
        relative = candidate.relative_to(root)
    except ValueError:
        return True
    current = root
    if current.exists() and current.is_symlink():
        return True
    for part in relative.parts:
        current /= part
        if current.exists() and current.is_symlink():
            return True
    return False


def resolve_model_root(models_dir: Path, model_id: str) -> Path:
    if not isinstance(model_id, str):
        raise ValueError("Model id must be a string")
    parts = model_id.split("/")
    if len(parts) != 2:
        raise ValueError("Model id must identify one extension node")
    extension_id = safe_source_id(parts[0], "extension id")
    if parts[1].casefold() == "_shared":
        raise ValueError('Model node id "_shared" is reserved')
    node_id = safe_source_id(parts[1], "model node id")
    root = models_dir.absolute()
    candidate = root / extension_id / node_id
    if _path_has_symlink(root, candidate):
        raise ValueError("Model path resolves through a symlink")
    try:
        candidate.resolve().relative_to(root.resolve())
    except ValueError as exc:
        raise ValueError("Model path escapes the models directory") from exc
    return candidate


def resolve_weight_group_root(models_dir: Path, extension_id: str, group_id: str) -> Path:
    safe_extension_id = safe_source_id(extension_id, "extension id")
    if isinstance(group_id, str) and group_id.casefold() == "_shared":
        raise ValueError('Weight group id "_shared" is reserved')
    safe_group_id = safe_source_id(group_id, "weight group id")
    root = models_dir.absolute()
    candidate = root / safe_extension_id / "_shared" / safe_group_id
    if _path_has_symlink(root, candidate):
        raise ValueError("Weight group path resolves through a symlink")
    try:
        candidate.resolve().relative_to(root.resolve())
    except ValueError as exc:
        raise ValueError("Weight group path escapes the models directory") from exc
    return candidate


def resolve_weight_storage_root(models_dir: Path, target_id: str) -> Path:
    if not isinstance(target_id, str):
        raise ValueError("Weight target id must be a string")
    parts = target_id.split("/")
    if len(parts) == 2:
        return resolve_model_root(models_dir, target_id)
    if len(parts) == 3 and parts[1] == "_shared":
        return resolve_weight_group_root(models_dir, parts[0], parts[2])
    raise ValueError(
        "Weight target id must identify one model node or extension weight group"
    )


def resolve_source_destination(models_dir: Path, model_id: str, destination: str) -> Path:
    model_root = resolve_model_root(models_dir, model_id)
    safe_destination = safe_relative_path(destination, "destination", allow_dot=True)
    candidate = model_root if safe_destination == "." else model_root.joinpath(*safe_destination.split("/"))
    if _path_has_symlink(model_root, candidate):
        raise ValueError("Source destination resolves through a symlink")
    return candidate


def resolve_source_destination_at_root(model_root: Path, destination: str) -> Path:
    safe_destination = safe_relative_path(destination, "destination", allow_dot=True)
    candidate = (
        model_root
        if safe_destination == "."
        else model_root.joinpath(*safe_destination.split("/"))
    )
    if _path_has_symlink(model_root, candidate):
        raise ValueError("Source destination resolves through a symlink")
    return candidate


def resolve_download_path(destination: Path, filename: str) -> Path:
    safe_filename = safe_relative_path(filename, "Hugging Face repository file")
    candidate = destination.joinpath(*safe_filename.split("/"))
    if _path_has_symlink(destination, candidate):
        raise ValueError("Download target resolves through a symlink")
    return candidate


def model_sources_are_downloaded(
    models_dir: Path, model_id: str, sources: list[dict[str, Any]]
) -> bool:
    try:
        model_root = resolve_model_root(models_dir, model_id)
        return model_sources_are_downloaded_at_root(model_root, sources)
    except (KeyError, OSError, TypeError, ValueError):
        return False


def model_sources_are_downloaded_at_root(
    model_root: Path, sources: list[dict[str, Any]]
) -> bool:
    try:
        if not model_root.is_dir():
            return False
        for source in sources:
            destination = resolve_source_destination_at_root(
                model_root, source["destination"]
            )
            if not destination.is_dir():
                return False
            for check in source["checks"]:
                candidate = resolve_download_path(destination, check)
                if (
                    not candidate.is_file()
                    or candidate.stat().st_size <= 0
                    or _path_has_symlink(model_root, candidate)
                ):
                    return False
        return bool(sources)
    except (KeyError, OSError, TypeError, ValueError):
        return False


def weight_group_sources_are_downloaded(
    models_dir: Path, extension_id: str, group: dict[str, Any]
) -> bool:
    try:
        return model_sources_are_downloaded_at_root(
            resolve_weight_group_root(models_dir, extension_id, group["id"]),
            group["model_sources"],
        )
    except (KeyError, OSError, TypeError, ValueError):
        return False


def validate_source_file_plan(
    sources: list[dict[str, Any]], files_by_source: dict[str, list[str]]
) -> None:
    """Reject cross-source aliases before the first file is written."""
    aliases: dict[str, tuple[str, str]] = {}
    for source in sources:
        source_id = source["id"]
        destination = source["destination"]
        source_files = {
            safe_relative_path(filename, f'model source "{source_id}" file')
            for filename in files_by_source[source_id]
        }
        missing_checks = [check for check in source["checks"] if check not in source_files]
        if missing_checks:
            raise ValueError(
                f'Model source "{source_id}" checks files excluded from its download plan: '
                + ", ".join(missing_checks)
            )
        for safe_filename in source_files:
            target = safe_filename if destination == "." else f"{destination}/{safe_filename}"
            for value in (target, f"{target}.part"):
                alias = unicodedata.normalize("NFC", value).casefold()
                for previous_alias, (previous_source, previous_target) in aliases.items():
                    if previous_source == source_id:
                        continue
                    if (
                        alias == previous_alias
                        or alias.startswith(f"{previous_alias}/")
                        or previous_alias.startswith(f"{alias}/")
                    ):
                        raise ValueError(
                            "Model sources have a portable target collision: "
                            f'"{previous_source}:{previous_target}" and "{source_id}:{value}"'
                        )
                aliases[alias] = (source_id, value)


def _prefixes_overlap(left: list[str], right: list[str]) -> bool:
    return any(
        a.lower().startswith(b.lower()) or b.lower().startswith(a.lower())
        for a in left
        for b in right
    )


def _assert_param_offers_variants(param: str, params_schema: Any, ids: list[str]) -> None:
    """Variant ids must be selectable, so the param they key has to exist and offer them."""
    entry = next(
        (p for p in params_schema if isinstance(p, dict) and p.get("id") == param),
        None,
    ) if isinstance(params_schema, list) else None
    if entry is None:
        raise ValueError(f'weight_variants.param must name a params_schema entry ("{param}")')
    options = entry.get("options")
    if not isinstance(options, list):
        return
    values = {
        str(option.get("value")) if isinstance(option, dict) else str(option)
        for option in options
    }
    missing = [variant_id for variant_id in ids if variant_id not in values]
    if missing:
        raise ValueError(
            f'the "{param}" param must offer every weight variant id '
            f'(missing: {", ".join(missing)})'
        )


def normalize_weight_variants(
    node: dict[str, Any], params_schema: Any = None
) -> dict[str, Any] | None:
    """Validate a node's separately installable weight variants (e.g. quantizations)."""
    if "weight_variants" not in node:
        return None
    if "model_sources" in node:
        raise ValueError("weight_variants cannot be combined with model_sources")
    if "weight_groups" in node:
        raise ValueError("weight_variants cannot be combined with weight_groups")
    if not isinstance(node.get("hf_repo"), str) or not node["hf_repo"]:
        raise ValueError("weight_variants requires hf_repo on the same node")
    raw = node["weight_variants"]
    if not isinstance(raw, dict):
        raise ValueError("weight_variants must be an object")
    param = safe_source_id(raw.get("param"), "weight_variants.param")
    raw_options = raw.get("options")
    if not isinstance(raw_options, list) or not raw_options:
        raise ValueError("weight_variants.options must be a non-empty array")

    aliases: dict[str, str] = {}
    options: list[dict[str, Any]] = []
    for index, raw_option in enumerate(raw_options):
        field = f"weight_variants.options[{index}]"
        if not isinstance(raw_option, dict):
            raise ValueError(f"{field} must be an object")
        variant_id = safe_source_id(raw_option.get("id"), f"{field}.id")
        alias = unicodedata.normalize("NFC", variant_id).casefold()
        if alias in aliases:
            raise ValueError(
                f'weight variant ids "{aliases[alias]}" and "{variant_id}" are not portable-unique'
            )
        aliases[alias] = variant_id
        label = raw_option.get("label", variant_id)
        if not isinstance(label, str) or not label.strip():
            raise ValueError(f"{field}.label must be a non-empty string")
        size_gb = raw_option.get("size_gb")
        if size_gb is not None and (
            isinstance(size_gb, bool)
            or not isinstance(size_gb, (int, float))
            or not math.isfinite(size_gb)
            or size_gb <= 0
        ):
            raise ValueError(f"{field}.size_gb must be a positive number")
        vram_gb = raw_option.get("vram_gb")
        if vram_gb is not None and (
            isinstance(vram_gb, bool)
            or not isinstance(vram_gb, (int, float))
            or not math.isfinite(vram_gb)
            or vram_gb <= 0
        ):
            raise ValueError(f"{field}.vram_gb must be a positive number")
        include = _prefixes(raw_option.get("include_prefixes"), f"{field}.include_prefixes")
        if not include:
            raise ValueError(f"{field}.include_prefixes must be a non-empty array")
        checks = raw_option.get("checks")
        if not isinstance(checks, list) or not checks:
            raise ValueError(f"{field}.checks must be a non-empty array")
        safe_checks: list[str] = []
        for check_index, check in enumerate(checks):
            path = safe_relative_path(check, f"{field}.checks[{check_index}]")
            if not any(path.startswith(prefix) for prefix in include):
                raise ValueError(
                    f"{field}.checks[{check_index}] is not covered by its include_prefixes"
                )
            safe_checks.append(path)
        option: dict[str, Any] = {
            "id": variant_id,
            "label": label,
            "include_prefixes": include,
            "checks": safe_checks,
        }
        if size_gb is not None:
            option["size_gb"] = size_gb
        if vram_gb is not None:
            option["vram_gb"] = vram_gb
        options.append(option)

    for index, option in enumerate(options):
        for other in options[index + 1:]:
            if _prefixes_overlap(option["include_prefixes"], other["include_prefixes"]):
                raise ValueError(
                    f'weight variants "{option["id"]}" and "{other["id"]}" share files'
                )
    download_check = node.get("download_check")
    if isinstance(download_check, str) and any(
        _prefixes_overlap([download_check], option["include_prefixes"]) for option in options
    ):
        raise ValueError("download_check must name a file outside every weight variant")
    default = raw.get("default", options[0]["id"])
    if not any(option["id"] == default for option in options):
        raise ValueError("weight_variants.default must name one of its options")
    _assert_param_offers_variants(
        param,
        node.get("params_schema") if params_schema is None else params_schema,
        [option["id"] for option in options],
    )
    return {"param": param, "default": default, "options": options}


def installed_weight_variants(
    models_dir: Path, model_id: str, variants: dict[str, Any]
) -> list[str]:
    try:
        model_root = resolve_model_root(models_dir, model_id)
    except ValueError:
        return []

    def _present(check: str) -> bool:
        try:
            candidate = resolve_download_path(model_root, check)
            return candidate.is_file() and candidate.stat().st_size > 0
        except (OSError, ValueError):
            return False

    return [
        option["id"]
        for option in variants["options"]
        if all(_present(check) for check in option["checks"])
    ]


def missing_weight_variant(
    models_dir: Path,
    model_id: str,
    variants: dict[str, Any] | None,
    params: dict[str, Any],
) -> dict[str, Any] | None:
    """The variant selected by params when its files are not installed, else None."""
    if not variants:
        return None
    selected = params.get(variants["param"])
    selected_id = variants["default"] if selected is None else str(selected)
    option = next((o for o in variants["options"] if o["id"] == selected_id), None)
    if option is None or option["id"] in installed_weight_variants(models_dir, model_id, variants):
        return None
    return option