AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.
feat(backend): store, copy and delete a skill as its whole package (#14549)
Reinier van der Leer54d1e8f212bb
diff --git a/autogpt_platform/backend/backend/api/features/v1.py b/autogpt_platform/backend/backend/api/features/v1.pyindex 221c84f5b3..c87393fdc9 100644--- a/autogpt_platform/backend/backend/api/features/v1.py+++ b/autogpt_platform/backend/backend/api/features/v1.py@@ -2866,6 +2866,7 @@ async def upload_copilot_skill(body=parsed.body,triggers=list(parsed.triggers),version=parsed.version,+ extra=parsed.extra,)except SkillLimitError as exc:raise HTTPException(status_code=409, detail=str(exc))diff --git a/autogpt_platform/backend/backend/api/features/v1_test.py b/autogpt_platform/backend/backend/api/features/v1_test.pyindex cefcea78a1..49260b2e1e 100644--- a/autogpt_platform/backend/backend/api/features/v1_test.py+++ b/autogpt_platform/backend/backend/api/features/v1_test.py@@ -1575,6 +1575,24 @@ def test_upload_copilot_skill_creates_skill(assert store_mock.await_args.kwargs["name"] == "oauth_flow"+def test_upload_copilot_skill_keeps_frontmatter_the_platform_ignores(+ mocker: pytest_mock.MockFixture,+) -> None:+ """An uploaded package's licence line is the author's, not ours — storing+ the file without it rewrites what they uploaded."""+ store_mock = AsyncMock(+ return_value=ParsedSkill(name="oauth_flow", description="d", body="b")+ )+ mocker.patch("backend.api.features.v1.store_user_skill", store_mock)++ response = client.post(+ "/skills",+ json={"content": _VALID_SKILL_MD.replace("---\n\n", "license: MIT\n---\n\n")},+ )+ assert response.status_code == 201+ assert store_mock.await_args.kwargs["extra"] == {"license": "MIT"}++def test_upload_copilot_skill_rejects_malformed_markdown() -> None:"""A file without valid frontmatter returns 400 before touching storage."""response = client.post(diff --git a/autogpt_platform/backend/backend/copilot/tools/skills.py b/autogpt_platform/backend/backend/copilot/tools/skills.pyindex e4678b2b58..e047913e67 100644--- a/autogpt_platform/backend/backend/copilot/tools/skills.py+++ b/autogpt_platform/backend/backend/copilot/tools/skills.py@@ -28,9 +28,10 @@ import loggingimport posixpathimport reimport uuid-from dataclasses import dataclass+from collections.abc import Mapping+from dataclasses import dataclass, fieldfrom pathlib import Path-from typing import Any+from typing import Any, NamedTupleimport yamlfrom pydantic import BaseModel@@ -54,10 +55,10 @@ from backend.util.workspace import WorkspaceManagerfrom .base import BaseToolfrom .models import ErrorResponse, ResponseType, ToolResponseBasefrom .workdir import (- make_executable,read_workdir_bytes,remove_from_workdir,save_to_workdir,+ set_executable,workdir_root,)@@ -77,7 +78,9 @@ logger = logging.getLogger(__name__)MAX_USER_SKILLS = 50MAX_NAME_CHARS = 64MAX_DESCRIPTION_CHARS = 1024-MAX_BODY_CHARS = 20_000+# Loaded only on activation, so it costs nothing per turn; 50k clears+# skill-creator's 33 KB SKILL.md, the package authors are told to copy.+MAX_BODY_CHARS = 50_000# Triggers appear inline in the per-turn ``<available_skills>`` index,# so an unbounded list (or one huge trigger) would balloon the prefix# the model parses every turn. Cap both the count and the per-entry@@ -90,6 +93,18 @@ MAX_TRIGGER_CHARS = 64# bounded. Enumeration fetches one more than this so a folder that exceeds# it is reported rather than silently truncated.MAX_PACKAGE_FILES = 100+# Largest public package file is 237 KB and package 5.4 MB, so these leave+# room for a font while keeping a hire-time install bounded.+MAX_PACKAGE_FILE_BYTES = 2 * 1024 * 1024+MAX_PACKAGE_BYTES = 20 * 1024 * 1024+# The spec keeps references one level deep; 8 stops a path of directories.+MAX_PACKAGE_PATH_DEPTH = 8+# An E2B write is a 200 ms round trip and a workspace read a blob fetch, so+# 60 of either in series is seconds of a turn. Bounded, not unlimited.+_COPY_CONCURRENCY = 16+# Passes delete_user_skill will make over a folder, each one page deep.+# Bounded so a file that cannot be deleted can never spin the loop.+_DELETE_PASSES = 20SKILL_FOLDER = "/skills"@@ -122,6 +137,9 @@ _META_KIND_VALUE = "copilot_skill"_META_DESCRIPTION = "description"_META_TRIGGERS = "triggers"_META_VERSION = "version"+# The workspace has no mode bits, so a script's executable bit survives+# store → copy → sandbox as this flag.+_META_EXECUTABLE = "executable"# Skill names are slug-like — lowercase letters, digits, dashes, underscores.# Must start and end with [a-z0-9] (no trailing/leading punctuation) so the@@ -185,6 +203,11 @@ _DEFAULT_SKILLS_BY_NAME: dict[str, _DefaultSkill] = {s.name: s for s in DEFAULT__FRONTMATTER_RE = re.compile(r"^---\n(.*?)\n---\n?(.*)$", re.DOTALL)+# Spec frontmatter the platform has no use for. Dropping it rewrites the+# author's SKILL.md on every store, so it rides through parse and render.+_CARRIED_FRONTMATTER_KEYS = ("license", "compatibility", "allowed-tools", "metadata")++@dataclass(frozen=True)class ParsedSkill:"""A SKILL.md decoded into its frontmatter metadata + markdown body."""@@ -194,6 +217,7 @@ class ParsedSkill:body: strtriggers: tuple[str, ...] = ()version: str | None = None+ extra: Mapping[str, Any] = field(default_factory=dict)def parse_skill_markdown(text: str, fallback_name: str = "") -> ParsedSkill | None:@@ -230,6 +254,7 @@ def parse_skill_markdown(text: str, fallback_name: str = "") -> ParsedSkill | Nobody=body.lstrip("\n"),triggers=triggers,version=str(version) if version is not None else None,+ extra={k: meta[k] for k in _CARRIED_FRONTMATTER_KEYS if k in meta},)@@ -241,6 +266,9 @@ def render_skill_markdown(skill: ParsedSkill) -> str:metadata the model wrote."""meta: dict[str, Any] = {"name": skill.name, "description": skill.description}+ for key in _CARRIED_FRONTMATTER_KEYS:+ if key in skill.extra:+ meta[key] = skill.extra[key]if skill.triggers:meta["triggers"] = list(skill.triggers)if skill.version:@@ -368,6 +396,128 @@ def get_default_skill_with_body(name: str) -> ParsedSkill | None:)+# ---------------------------------------------------------------------------+# The package boundary — a skill as its whole directory. Everything that+# writes a package (``store_user_skill``, the copy to an expert, and the+# upload/install paths above this layer) goes through these types, so the+# caps and the path rules are stated once.+# ---------------------------------------------------------------------------++_PACKAGE_SEGMENT_RE = re.compile(r"^[A-Za-z0-9_-][A-Za-z0-9._-]*$")+_ROOT_SKILL_MD = "SKILL.md"+++class SkillPackageError(ValueError):+ """A package that breaks a cap or a path rule.++ ``over_limit`` separates a size or count refusal — 413 at the REST edge —+ from a malformed one, which is 400. A ``ValueError`` subclass so the+ handlers that already map validation failures keep working.+ """++ def __init__(self, message: str, *, over_limit: bool = False):+ super().__init__(message)+ self.over_limit = over_limit+++class SkillFile(BaseModel):+ """One file beside a package's ``SKILL.md``, by its path relative to the+ skill folder (``scripts/with_server.py``, ``references/REFERENCE.md``)."""++ relative_path: str+ # pydantic encodes a ``str`` here as utf-8, so a caller may pass either.+ content: bytes+ is_executable: bool = False++ @property+ def size_bytes(self) -> int:+ """Derived, never stored: a size that could disagree with the content+ would make every cap below lie."""+ return len(self.content)+++class SkillPackage(BaseModel):+ """A whole skill: its ``SKILL.md`` text plus every sibling file."""++ skill_md: str+ files: list[SkillFile] = []++ @property+ def size_bytes(self) -> int:+ return len(self.skill_md.encode("utf-8")) + sum(+ f.size_bytes for f in self.files+ )+++def validate_package(package: SkillPackage) -> None:+ """Refuse a package that breaks a cap or carries an unsafe path.++ Runs over the whole package so a caller can write nothing on failure;+ every message names the field and the number it broke.+ """+ if len(package.files) > MAX_PACKAGE_FILES:+ raise SkillPackageError(+ f"package has {len(package.files)} files; the limit is "+ f"{MAX_PACKAGE_FILES}",+ over_limit=True,+ )+ seen: set[str] = set()+ for entry in package.files:+ path_error = _package_path_error(entry.relative_path)+ if path_error:+ raise SkillPackageError(path_error)+ if entry.relative_path in seen:+ raise SkillPackageError(+ f"file '{entry.relative_path}' appears twice in the package"+ )+ seen.add(entry.relative_path)+ if entry.size_bytes > MAX_PACKAGE_FILE_BYTES:+ raise SkillPackageError(+ f"file '{entry.relative_path}' is {entry.size_bytes} bytes; "+ f"the limit is {MAX_PACKAGE_FILE_BYTES}",+ over_limit=True,+ )+ if package.size_bytes > MAX_PACKAGE_BYTES:+ raise SkillPackageError(+ f"package is {package.size_bytes} bytes; the limit is "+ f"{MAX_PACKAGE_BYTES}",+ over_limit=True,+ )+++def _package_path_error(path: str) -> str | None:+ """Why *path* may not be written into a skill folder, or ``None``.++ Rejects what a zip member or an API caller can smuggle in: an escape out+ of the folder, a hidden file, a backslash or NUL a later consumer would+ read as a separator or a terminator.+ """+ named = f"file path '{path[:120]}'"+ if not path or path != path.strip():+ return f"{named} is empty or padded with whitespace"+ if path.startswith("/"):+ return f"{named} must be relative to the skill folder"+ if "\\" in path or "\x00" in path:+ return f"{named} may not contain a backslash or a NUL byte"+ if posixpath.normpath(path) != path:+ return f"{named} is not normalised (no '.', '..' or repeated '/')"+ if path == _ROOT_SKILL_MD:+ return f"{named} is the package root; pass it as skill_md, not a file"+ segments = path.split("/")+ if len(segments) > MAX_PACKAGE_PATH_DEPTH:+ return (+ f"{named} is {len(segments)} segments deep; the limit is "+ f"{MAX_PACKAGE_PATH_DEPTH}"+ )+ for segment in segments:+ if not _PACKAGE_SEGMENT_RE.match(segment):+ return (+ f"{named} has an unusable segment '{segment[:40]}' — use "+ "letters, digits, '.', '_' or '-', not starting with '.'"+ )+ return None++class SkillNotFoundError(Exception):"""Raised by :func:`delete_user_skill` when the skill is missing."""@@ -390,7 +540,9 @@ async def delete_user_skill("""Delete a user-distilled skill folder by slug from *expert_id*'s folder(personal Otto's when ``None``).- Returns the normalised slug on success so callers can echo it back.+ The whole tree goes, however many files it holds — a package written+ before the cap existed still has to be removable. Returns the normalised+ slug on success so callers can echo it back.Raises :class:`BuiltInSkillError` for default skills,:class:`SkillNotFoundError` if the skill does not exist, and``ValueError`` if ``name`` is blank. Sibling-file cleanup is@@ -479,6 +631,8 @@ async def store_user_skill(body: str,triggers: list[str] | None = None,version: str | None = None,+ extra: Mapping[str, Any] | None = None,+ files: list[SkillFile] | None = None,expert_id: str | None = None,scope: WorkspaceScope | None = None,) -> ParsedSkill:@@ -491,6 +645,12 @@ async def store_user_skill(failure, :class:`SkillLimitError` when the per-user cap is reached, andpropagates ``VirusDetectedError`` / ``VirusScanError`` (and any otherworkspace write error) to the caller.++ *files* is the whole package: it replaces the folder's contents, so a+ file the caller leaves out is deleted. ``None`` — every single-file+ caller — leaves the existing siblings alone, which is what keeps the+ model's own ``store_skill`` from wiping a package it only rewrote the+ body of."""name = name.strip().lower()# Strip any server-injected XML tags (``<available_skills>``,@@ -532,6 +692,20 @@ async def store_user_skill(f"trigger '{oversized_trigger[:32]}…' exceeds {MAX_TRIGGER_CHARS} chars")+ parsed = ParsedSkill(+ name=name,+ description=description,+ body=body,+ triggers=tuple(triggers),+ version=version,+ extra=dict(extra or {}),+ )+ rendered = render_skill_markdown(parsed)+ if files is not None:+ # Whole-package validation before the first write, so a package that+ # breaks a cap leaves the stored skill exactly as it was.+ validate_package(SkillPackage(skill_md=rendered, files=files))+# Serialise the count-then-write critical section per-user so two# concurrent writers cannot both pass the MAX_USER_SKILLS check.# ``AsyncClusterLock.try_acquire`` is non-blocking, so poll for up to@@ -588,14 +762,6 @@ async def store_user_skill("Delete an unused skill first.")- parsed = ParsedSkill(- name=name,- description=description,- body=body,- triggers=tuple(triggers),- version=version,- )- rendered = render_skill_markdown(parsed)metadata: dict[str, Any] = {_META_KIND: _META_KIND_VALUE,_META_DESCRIPTION: description,@@ -603,6 +769,38 @@ async def store_user_skill(}if version:metadata[_META_VERSION] = version+ folder = skill_folder(expert_id)+ stale = (+ await _list_package_files(manager, folder, name, cap=None)+ if files is not None+ else []+ )+ # The root is what indexes the skill, so it goes last: a new skill+ # that fails part-way is never indexed. An upsert cannot be made+ # atomic here — the old bytes are gone once overwritten. Serial+ # because ``write_file``'s quota check is read-then-write.+ existing_paths = {f.path for f in stale}+ written: set[str] = set()+ try:+ for entry in files or []:+ path = f"{folder}/{name}/{entry.relative_path}"+ written.add(path)+ await manager.write_file(+ content=entry.content,+ filename=entry.relative_path.rsplit("/", 1)[-1],+ path=path,+ mime_type=None,+ overwrite=True,+ metadata=(+ {_META_EXECUTABLE: True} if entry.is_executable else None+ ),+ )+ except Exception:+ # Undo only what this call created: a file that was already+ # there has lost its old bytes either way, and deleting it would+ # turn a failed write into a lost file.+ await _delete_paths(manager, written - existing_paths)+ raiseawait manager.write_file(content=rendered.encode("utf-8"),filename="SKILL.md",@@ -611,6 +809,9 @@ async def store_user_skill(overwrite=True,metadata=metadata,)+ await _delete_paths(+ manager, {f.path for f in stale if f.path not in written}, stale+ )await invalidate_skills_index_cache(user_id, expert_id)if expert_id is not None:await experts_db().add_expert_skill_name(user_id, expert_id, name)@@ -627,6 +828,28 @@ async def store_user_skill()+async def _delete_paths(+ manager: WorkspaceManager,+ paths: set[str],+ known: list[SkillFileInfo] | None = None,+) -> None:+ """Delete workspace files by path, best-effort: a file that will not go+ is logged, never raised, because every caller here has already done the+ thing the user asked for."""+ ids = {f.path: f.file_id for f in known or []}+ for path in paths:+ try:+ file_id = ids.get(path)+ if file_id is None:+ info = await manager.get_file_info_by_path(path)+ if info is None:+ continue+ file_id = info.id+ await manager.delete_file(file_id)+ except Exception:+ logger.warning("[skills] failed to delete %s", path, exc_info=True)++async def _parse_skill_from_workspace(manager: WorkspaceManager, file_path: str) -> ParsedSkill | None:@@ -1029,10 +1252,12 @@ async def find_user_skill_slugs(user_id: str, names: list[str]) -> dict[str, strasync def copy_skill_to_expert(user_id: str, expert_id: str, name: str) -> str | None:"""Give *expert_id* its own copy of one of personal Otto's skills.- Returns the stored slug, or ``None`` when Otto has no such skill.+ Returns the stored slug, or ``None`` when Otto has no such skill, and+ raises when the source package cannot be read whole — a partial copy would+ be permanent, since the next call sees the root and returns early.Idempotent: an expert that already owns the slug keeps its copy. The- SKILL.md is re-validated through :func:`store_user_skill`; sibling files- are copied best-effort afterwards.+ whole package goes through :func:`store_user_skill`, so the copy is+ validated, capped and written siblings-first exactly like any other."""slug = name.strip().lower()if not slug:@@ -1052,23 +1277,57 @@ async def copy_skill_to_expert(user_id: str, expert_id: str, name: str) -> str |body=source.body,triggers=list(source.triggers),version=source.version,+ extra=source.extra,+ files=await _read_package_files(manager, SKILL_FOLDER, slug),expert_id=expert_id,)- for path in await list_user_skill_sibling_paths(user_id, slug):- relative = path[len(f"{SKILL_FOLDER}/{slug}/") :]- try:- await manager.write_file(- content=await manager.read_file(path),- filename=relative.rsplit("/", 1)[-1],- path=f"{skill_folder(expert_id)}/{slug}/{relative}",- mime_type=None,- overwrite=True,- )- except Exception:- logger.warning("[skills] failed to copy %s for expert", path, exc_info=True)return stored.name+async def _read_package_files(+ manager: WorkspaceManager, folder: str, slug: str+) -> list[SkillFile]:+ """Load a stored package's siblings into memory, ready to be written+ somewhere else. Reads run concurrently — each is a blob fetch, and a+ 60-file package read one at a time is a hire the user waits through.++ A file that cannot be read raises, because the caller's copy is idempotent+ on the root alone: a package written without it would never be repaired.+ """+ prefix = f"{folder}/{slug}/"+ infos = await _list_package_files(manager, folder, slug)+ if len(infos) > MAX_PACKAGE_FILES:+ # Written before the cap existed, or by hand. Validating it whole+ # would fail and take the hire down, so truncate as read_skill does.+ logger.warning(+ "[skills] package %s has more than %s files; copying the first %s",+ slug,+ MAX_PACKAGE_FILES,+ MAX_PACKAGE_FILES,+ )+ infos = infos[:MAX_PACKAGE_FILES]+ limit = asyncio.Semaphore(_COPY_CONCURRENCY)++ async def load(info: SkillFileInfo) -> SkillFile:+ async with limit:+ content = await manager.read_file(info.path)+ return SkillFile(+ relative_path=info.path[len(prefix) :],+ content=content,+ is_executable=info.is_executable,+ )++ # ``return_exceptions`` so one failure does not leave its siblings' reads+ # unawaited; the first is re-raised once they have all settled.+ loaded = await asyncio.gather(+ *(load(info) for info in infos), return_exceptions=True+ )+ failure = next((r for r in loaded if isinstance(r, BaseException)), None)+ if failure is not None:+ raise failure+ return [f for f in loaded if isinstance(f, SkillFile)]++def get_default_skills_for_index() -> list[ParsedSkill]:"""Return the default seeded skills with **empty bodies** — suitablefor rendering the per-turn ``<available_skills>`` index, which only@@ -1226,6 +1485,10 @@ class SkillFileInfo(BaseModel):path: strfile_id: strsize_bytes: int = 0+ is_executable: bool = False+ # The workspace hashes every write with the same sha256 the manifest+ # records, so a match means the copy is current without reading the blob.+ checksum: str | None = Noneclass ReadSkillResponse(ToolResponseBase):@@ -1783,33 +2046,43 @@ def _root_skill_slug(path: str, folder: str) -> str | None:async def _list_package_files(- manager: WorkspaceManager, folder: str, slug: str+ manager: WorkspaceManager,+ folder: str,+ slug: str,+ *,+ cap: int | None = MAX_PACKAGE_FILES,) -> list[SkillFileInfo]:- """Every file in a skill's folder except its own SKILL.md, at most- ``MAX_PACKAGE_FILES + 1`` so a caller can tell a full package from an- oversized one.+ """Every file in a skill's folder except its own SKILL.md, stopping at+ ``cap`` + 1 so a caller can tell a full package from an oversized one.+ ``cap=None`` drains the folder in one read, which the orphan prune needs:+ a file it cannot see is one it would leave behind.Nested paths are kept: a package's ``scripts/`` and ``references/`` arewhat make it more than one file."""prefix = f"{folder}/{slug}/"root = f"{prefix}SKILL.md"- page = MAX_PACKAGE_FILES + 1+ page = (cap or MAX_PACKAGE_FILES) + 1files: list[SkillFileInfo] = []offset = 0- while len(files) <= MAX_PACKAGE_FILES:+ while cap is None or len(files) <= cap:rows = await manager.list_files(path=prefix, limit=page, offset=offset, include_all_sessions=True)for row in rows:if row.path == root:continue+ meta = row.metadata if isinstance(row.metadata, dict) else {}files.append(SkillFileInfo(- path=row.path, file_id=row.id, size_bytes=row.size_bytes or 0+ path=row.path,+ file_id=row.id,+ size_bytes=row.size_bytes or 0,+ is_executable=bool(meta.get(_META_EXECUTABLE)),+ checksum=getattr(row, "checksum", None),))- if len(files) > MAX_PACKAGE_FILES:+ if cap is not None and len(files) > cap:breakif len(rows) < page:break@@ -1826,7 +2099,23 @@ async def _list_package_files(# package is copied into: a skill shipping its own ``.package.json`` would# otherwise be overwritten by it, and the digest would then match forever._MANIFEST_DIR = ".skill-packages"+_MANIFEST_SHA = "sha256"+_MANIFEST_EXEC = "executable"+# A package with no bits to give still gets ``scripts/`` marked, because that+# is where the spec puts its runnables._EXECUTABLE_PREFIX = "scripts/"+++class _CopiedFile(NamedTuple):+ """One file settled in the working directory. ``target`` is its path there+ when this pass wrote it, and ``None`` when the manifest already matched."""++ relative: str+ digest: str+ executable: bool+ target: str | None++# One E2B ``files.write`` is an HTTP round trip and measured 200 ms, so a# 60-file package copied serially would cost 12 s of the turn (0.35 s# concurrent). Bounded, because each copy also reads a blob.@@ -1859,13 +2148,20 @@ async def _sync_skill_package(manifest = await _read_package_manifest(manifest_path, session_id)limit = asyncio.Semaphore(_COPY_CONCURRENCY)- async def copy(info: SkillFileInfo) -> tuple[str, str, str | None] | None:- """``(relative path, digest, path to mark executable)`` once the file- is in place, or ``None`` when it could not be put there."""+ async def copy(info: SkillFileInfo) -> _CopiedFile | None:+ """The file's manifest entry once it is in place, or ``None`` when it+ could not be put there."""relative = info.path[len(prefix) :] if info.path.startswith(prefix) else ""if not _is_safe_relative(relative):logger.warning("[skills] skipping odd package path %s", info.path)return None+ executable = info.is_executable or relative.startswith(_EXECUTABLE_PREFIX)+ settled = {_MANIFEST_SHA: info.checksum, _MANIFEST_EXEC: executable}+ # The row's checksum is recomputed on every write, so it describes the+ # bytes as stored; matching it means the copy on disk is current and the+ # blob does not have to be fetched to find that out.+ if info.checksum and manifest.get(relative) == settled:+ return _CopiedFile(relative, info.checksum, executable, None)async with limit:try:content = await manager.read_file(info.path)@@ -1873,8 +2169,13 @@ async def _sync_skill_package(logger.warning("[skills] failed to read %s", info.path, exc_info=True)return Nonedigest = hashlib.sha256(content).hexdigest()- if manifest.get(relative) == digest:- return relative, digest, None+ # The mode is part of what was materialised: a file whose bit flips+ # without its bytes changing still has to be re-chmodded.+ if manifest.get(relative) == {+ _MANIFEST_SHA: digest,+ _MANIFEST_EXEC: executable,+ }:+ return _CopiedFile(relative, digest, executable, None)target = await save_to_workdir(f"{package_dir}/{relative}", content, session_id)@@ -1883,17 +2184,30 @@ async def _sync_skill_package("[skills] failed to materialise %s: %s", info.path, target.message)return None- return (- relative,- digest,- target if relative.startswith(_EXECUTABLE_PREFIX) else None,- )+ return _CopiedFile(relative, digest, executable, target)- copied = await asyncio.gather(*(copy(info) for info in files))- written = {relative: digest for relative, digest, _ in filter(None, copied)}- executables = [path for _, _, path in filter(None, copied) if path]+ copied = [c for c in await asyncio.gather(*(copy(info) for info in files)) if c]+ written = {+ c.relative: {_MANIFEST_SHA: c.digest, _MANIFEST_EXEC: c.executable}+ for c in copied+ }- await make_executable(executables, session_id)+ # The manifest is what a later activation trusts instead of re-doing work,+ # so it records only what actually happened. A mode that would not apply or+ # a file that would not go is left out of it, and the next activation tries+ # again; committing them would make a transient failure permanent.+ settled = dict(written)++ # Only files this pass wrote carry a ``target``; a manifest hit needs no+ # chmod, because its recorded mode already matches.+ unset = await set_executable(+ [c.target for c in copied if c.target and c.executable], True, session_id+ ) + await set_executable(+ [c.target for c in copied if c.target and not c.executable], False, session_id+ )+ for c in copied:+ if c.target in unset:+ settled.pop(c.relative, None)# A file the skill no longer has must not stay where bash_exec can run it,# and delete_skill followed by store_skill on the same slug is exactly that# case. Prune by what the package HOLDS, not by what was copied: a copy@@ -1904,22 +2218,26 @@ async def _sync_skill_package(# reach the manifest by a hand edit; excluding it keeps the prune from# acting on a name that does not belong to it.current = {info.path[len(prefix) :] for info in files} | {"SKILL.md"}- removed = set(manifest) - current- await remove_from_workdir(- [f"{package_dir}/{relative}" for relative in sorted(removed)], session_id+ stale = sorted(set(manifest) - current)+ left = set(+ await remove_from_workdir(+ [f"{package_dir}/{relative}" for relative in stale], session_id+ ))-- # RESOLVING A CONFLICT HERE: the manifest may only lose an entry for a file- # we know is gone — one we removed, or one a listing we know was complete- # did not contain. Several branches narrow `removed` or hold an entry back- # for their own reason; each is that one rule applied to a different way of- # not knowing. Keep every reason rather than taking one side of the diff.+ removed = {r for r in stale if f"{package_dir}/{r}" not in left}++ # THE INVARIANT, and why this statement keeps collecting edits: the manifest+ # may only lose an entry for a file we KNOW is gone — one we removed, or one+ # a listing we know was complete did not contain. Every branch that touches+ # it narrows `removed` or `settled` for its own way of not knowing: a delete+ # that failed, a chmod that did not apply, a listing that was truncated.+ # Narrow further if you must; never widen by taking one side of the diff.next_manifest = {- relative: digest- for relative, digest in manifest.items()- if relative not in written and relative not in removed+ relative: entry+ for relative, entry in manifest.items()+ if relative not in settled and relative not in removed}- next_manifest.update(written)+ next_manifest.update(settled)# A single-file skill must not leave an empty package directory behind, so# the manifest is written only when there is, or was, something to track.if next_manifest or manifest:@@ -1954,9 +2272,19 @@ def _is_safe_relative(path: str) -> bool:)-async def _read_package_manifest(manifest_path: str, session_id: str) -> dict[str, str]:- """Hashes written by the last activation, empty on a first run or any- unreadable manifest — a re-copy is cheap, a stale skip is not."""+async def _read_package_manifest(+ manifest_path: str, session_id: str+) -> dict[str, dict[str, Any]]:+ """What the last activation wrote, per path — empty on a first run or any+ unreadable manifest, because a re-copy is cheap and a stale skip is not.++ A bare digest string is the pre-executable-tracking format; reading it as+ non-executable is the safe direction, since the worst it costs is one+ redundant chmod.++ The manifest lives in the model's own working directory, so its keys are+ untrusted input to a later rm: only paths we would have written survive.+ """raw = await read_workdir_bytes(manifest_path, session_id)if not raw:return {}@@ -1966,17 +2294,22 @@ async def _read_package_manifest(manifest_path: str, session_id: str) -> dict[streturn {}if not isinstance(loaded, dict):return {}- # The manifest lives in the model's own working directory, so its keys are- # untrusted input to a later rm: keep only paths we would have written.- return {- k: v- for k, v in loaded.items()- if isinstance(k, str) and isinstance(v, str) and _is_safe_relative(k)- }+ entries: dict[str, dict[str, Any]] = {}+ for path, value in loaded.items():+ if not isinstance(path, str) or not _is_safe_relative(path):+ continue+ if isinstance(value, str):+ entries[path] = {_MANIFEST_SHA: value, _MANIFEST_EXEC: False}+ elif isinstance(value, dict) and isinstance(value.get(_MANIFEST_SHA), str):+ entries[path] = {+ _MANIFEST_SHA: value[_MANIFEST_SHA],+ _MANIFEST_EXEC: bool(value.get(_MANIFEST_EXEC)),+ }+ return entriesasync def _write_package_manifest(- manifest_path: str, hashes: dict[str, str], session_id: str+ manifest_path: str, hashes: dict[str, dict[str, Any]], session_id: str) -> None:result = await save_to_workdir(manifest_path, json.dumps(hashes).encode(), session_iddiff --git a/autogpt_platform/backend/backend/copilot/tools/skills_isolation_test.py b/autogpt_platform/backend/backend/copilot/tools/skills_isolation_test.pyindex 2e352a0229..bbe11c96a8 100644--- a/autogpt_platform/backend/backend/copilot/tools/skills_isolation_test.py+++ b/autogpt_platform/backend/backend/copilot/tools/skills_isolation_test.py@@ -412,3 +412,40 @@ async def test_copy_gives_expert_its_own_copy_of_an_autopilot_skill(world):assert AUTOPILOT in fake.filesexperts.add_expert_skill_name.assert_awaited_once_with("user-1", "expert-a", "mine")assert await copy_skill_to_expert("user-1", "expert-a", "nope") is None+++async def test_copy_gives_the_expert_the_whole_package(world):+ """A hire used to get the first 50 files of a package and no way to tell.+ Sixty is what the public ``docx``/``pptx`` packages ship."""+ fake, _ = world+ fake.files["/skills/mine/scripts/run.py"] = b"#!/usr/bin/env python3\n"+ fake.metadata["/skills/mine/scripts/run.py"] = {"executable": True}+ for i in range(59):+ fake.files[f"/skills/mine/references/r{i:03d}.md"] = f"ref {i}".encode()++ assert await copy_skill_to_expert("user-1", "expert-a", "mine") == "mine"++ copied = {+ path[len("/experts/expert-a/skills/mine/") :]+ for path in fake.files+ if path.startswith("/experts/expert-a/skills/mine/")+ }+ assert len(copied) == 61 # 60 package files + the SKILL.md+ assert fake.files["/experts/expert-a/skills/mine/references/r058.md"] == b"ref 58"+ assert fake.metadata["/experts/expert-a/skills/mine/scripts/run.py"] == {+ "executable": True+ }+++async def test_copy_carries_frontmatter_the_platform_has_no_use_for(world):+ """An expert's copy of a marketplace package must keep the author's+ licence line; re-rendering it away rewrites their file."""+ fake, _ = world+ fake.files[AUTOPILOT] = (+ "---\nname: mine\ndescription: d\nlicense: Apache-2.0\n---\n\nsteps\n"+ ).encode()++ assert await copy_skill_to_expert("user-1", "expert-a", "mine") == "mine"++ copy = fake.files["/experts/expert-a/skills/mine/SKILL.md"].decode()+ assert "license: Apache-2.0" in copydiff --git a/autogpt_platform/backend/backend/copilot/tools/skills_test.py b/autogpt_platform/backend/backend/copilot/tools/skills_test.pyindex b18be16ff0..f5d1619df2 100644--- a/autogpt_platform/backend/backend/copilot/tools/skills_test.py+++ b/autogpt_platform/backend/backend/copilot/tools/skills_test.py@@ -1,6 +1,7 @@"""Tests for the skill registry (frontmatter parsing + rendering + the``<available_skills>`` index builder)."""+import hashlibimport jsonimport osimport shutil@@ -9,6 +10,7 @@ from unittest.mock import AsyncMock, MagicMock, patchimport pytest+import backend.copilot.tools.skills as skillsfrom backend.copilot.model import ChatSessionfrom backend.copilot.tools.models import ErrorResponsefrom backend.copilot.tools.skills import (@@ -16,6 +18,8 @@ from backend.copilot.tools.skills import (MAX_BODY_CHARS,MAX_DESCRIPTION_CHARS,MAX_NAME_CHARS,+ MAX_PACKAGE_BYTES,+ MAX_PACKAGE_FILE_BYTES,MAX_PACKAGE_FILES,MAX_TRIGGER_CHARS,MAX_TRIGGERS,@@ -28,13 +32,17 @@ from backend.copilot.tools.skills import (ParsedSkill,ReadSkillResponse,ReadSkillTool,+ SkillFile,SkillNotFoundError,+ SkillPackage,+ SkillPackageError,StoreSkillResponse,StoreSkillTool,_is_safe_relative,_list_user_skills_from_workspace,_validate_name,build_skills_context,+ copy_skill_to_expert,delete_user_skill,find_user_skill_slugs,get_default_skills,@@ -44,6 +52,7 @@ from backend.copilot.tools.skills import (render_skill_markdown,render_skills_index,store_user_skill,+ validate_package,)# ---------------------------------------------------------------------------@@ -218,6 +227,7 @@ class _FakeWorkspaceManager:def __init__(self):self.files: dict[str, bytes] = {}self.metadata: dict[str, dict] = {}+ self.reads: list[str] = []async def write_file(self, *, content, filename, path, mime_type, overwrite, metadata=None@@ -228,6 +238,7 @@ class _FakeWorkspaceManager:async def read_file(self, path: str) -> bytes:if path not in self.files:raise FileNotFoundError(path)+ self.reads.append(path)return self.files[path]async def list_files(@@ -254,6 +265,9 @@ class _FakeWorkspaceManager:info.id = f"id-{p}"info.size_bytes = len(self.files[p])info.metadata = self.metadata.get(p, {})+ # write_file recomputes this on every write, so the row's checksum+ # always describes the bytes that are stored right now.+ info.checksum = hashlib.sha256(self.files[p]).hexdigest()result.append(info)result = result[offset:]return result if limit is None else result[:limit]@@ -274,6 +288,22 @@ class _FakeWorkspaceManager:return+class _FailingWorkspaceManager(_FakeWorkspaceManager):+ """Fails the *fail_on*-th ``write_file``, so a package that dies+ part-way can be checked for what it left behind."""++ def __init__(self, fail_on: int):+ super().__init__()+ self.fail_on = fail_on+ self.writes = 0++ async def write_file(self, **kwargs):+ self.writes += 1+ if self.writes == self.fail_on:+ raise RuntimeError("storage unavailable")+ return await super().write_file(**kwargs)++class _patch_skills_path:"""Context manager that patches the workspace lookup, the Redisclient, the AsyncClusterLock used by ``StoreSkillTool``, and the@@ -1483,6 +1513,323 @@ async def test_read_skill_says_when_a_package_exceeds_the_cap():assert f"first {MAX_PACKAGE_FILES} package files" in result.message+# ---------------------------------------------------------------------------+# The package boundary: frontmatter carry-through, caps, whole-tree writes+# ---------------------------------------------------------------------------++# Verbatim frontmatter from ``anthropics/skills`` at 34040c9c56 —+# ``skills/webapp-testing/SKILL.md`` and ``skills/pdf/SKILL.md``, the two+# reference packages the ecosystem points authors at. Both carry a+# ``license`` the platform has no use for and must not eat.+UPSTREAM_SKILL_MD = {+ "webapp-testing": (+ "---\n"+ "name: webapp-testing\n"+ "description: Toolkit for interacting with and testing local web"+ " applications using Playwright. Supports verifying frontend"+ " functionality, debugging UI behavior, capturing browser screenshots,"+ " and viewing browser logs.\n"+ "license: Complete terms in LICENSE.txt\n"+ "---\n\n"+ "# Web Application Testing\n"+ ),+ "pdf": (+ "---\n"+ "name: pdf\n"+ "description: Use this skill whenever the user wants to do anything"+ " with PDF files. This includes reading or extracting text/tables from"+ " PDFs, combining or merging multiple PDFs into one, splitting PDFs"+ " apart, rotating pages, adding watermarks, creating new PDFs, filling"+ " PDF forms, encrypting/decrypting PDFs, extracting images, and OCR on"+ " scanned PDFs to make them searchable. If the user mentions a .pdf"+ " file or asks to produce one, use this skill.\n"+ "license: Proprietary. LICENSE.txt has complete terms\n"+ "---\n\n"+ "# PDF Processing Guide\n"+ ),+}++[email protected]("slug", sorted(UPSTREAM_SKILL_MD))+def test_upstream_frontmatter_survives_a_round_trip(slug):+ """A real package's SKILL.md must come back out of the platform the way+ it went in. ``license`` is dropped by a parser that only knows our own+ fields, which silently rewrites the author's file on every store."""+ original = parse_skill_markdown(UPSTREAM_SKILL_MD[slug])+ assert original is not None+ assert original.extra["license"]++ rendered = render_skill_markdown(original)+ reparsed = parse_skill_markdown(rendered)+ assert reparsed is not None+ assert reparsed.extra == original.extra+ assert reparsed.description == original.description+ # Byte-stable from the first render on: a second pass changes nothing.+ assert render_skill_markdown(reparsed) == rendered+++def test_every_spec_frontmatter_field_is_carried():+ raw = (+ "---\n"+ "name: kitchen-sink\n"+ "description: all four spec fields\n"+ "license: Apache-2.0\n"+ "compatibility: claude-code >=2.0\n"+ "allowed-tools:\n"+ " - Bash\n"+ " - Read\n"+ "metadata:\n"+ " author: someone\n"+ "---\n"+ "body\n"+ )+ parsed = parse_skill_markdown(raw)+ assert parsed is not None+ assert parsed.extra == {+ "license": "Apache-2.0",+ "compatibility": "claude-code >=2.0",+ "allowed-tools": ["Bash", "Read"],+ "metadata": {"author": "someone"},+ }+ assert parse_skill_markdown(render_skill_markdown(parsed)).extra == parsed.extra+++def test_a_body_the_reference_package_needs_is_accepted():+ """skill-creator's SKILL.md is 33,168 bytes and is what the ecosystem+ tells authors to copy; the old 20,000-char cap refused it."""+ assert MAX_BODY_CHARS >= 33_168+++ "relative_path",+ [+ "../escape.md",+ "/absolute.md",+ "scripts/../../escape.md",+ "scripts//double.md",+ "./here.md",+ ".hidden/secret.md",+ "scripts/.hidden",+ "windows\\path.md",+ "nul\x00byte.md",+ "SKILL.md",+ "a/b/c/d/e/f/g/h/i.md",+ "spaces are out.md",+ ],+)+def test_validate_package_refuses_an_unsafe_path(relative_path):+ with pytest.raises(SkillPackageError) as exc:+ validate_package(+ SkillPackage(+ skill_md="---\nname: a\ndescription: b\n---\nbody\n",+ files=[SkillFile(relative_path=relative_path, content=b"x")],+ )+ )+ assert relative_path[:120] in str(exc.value)+ assert not exc.value.over_limit+++def test_validate_package_refuses_a_duplicate_path():+ with pytest.raises(SkillPackageError, match="appears twice"):+ validate_package(+ SkillPackage(+ skill_md="---\nname: a\ndescription: b\n---\nbody\n",+ files=[+ SkillFile(relative_path="a.md", content=b"1"),+ SkillFile(relative_path="a.md", content=b"2"),+ ],+ )+ )+++def test_validate_package_accepts_a_nested_skill_md_as_a_file():+ """The spec lets a package ship an example SKILL.md. Only the root one+ is special, and it travels as ``skill_md``."""+ validate_package(+ SkillPackage(+ skill_md="---\nname: a\ndescription: b\n---\nbody\n",+ files=[+ SkillFile(relative_path="references/examples/SKILL.md", content=b"x"),+ SkillFile(relative_path="scripts/with_server.py", content="print(1)"),+ ],+ )+ )+++ "files, expected",+ [+ (+ [SkillFile(relative_path=f"r{i}.md", content=b"x") for i in range(101)],+ f"package has 101 files; the limit is {MAX_PACKAGE_FILES}",+ ),+ (+ [+ SkillFile(+ relative_path="big.bin", content=b"x" * (MAX_PACKAGE_FILE_BYTES + 1)+ )+ ],+ (+ f"file 'big.bin' is {MAX_PACKAGE_FILE_BYTES + 1} bytes; "+ f"the limit is {MAX_PACKAGE_FILE_BYTES}"+ ),+ ),+ (+ [+ SkillFile(relative_path=f"b{i}.bin", content=b"x" * 2 * 1024 * 1024)+ for i in range(10)+ ],+ f"the limit is {MAX_PACKAGE_BYTES}",+ ),+ ],+ ids=["file-count", "file-size", "package-size"],+)+def test_validate_package_names_the_field_and_the_number(files, expected):+ with pytest.raises(SkillPackageError) as exc:+ validate_package(+ SkillPackage(+ skill_md="---\nname: a\ndescription: b\n---\nbody\n", files=files+ )+ )+ assert expected in str(exc.value)+ assert exc.value.over_limit+++async def test_an_over_cap_package_writes_nothing():+ """Validation runs over the whole package before the first write, so a+ refusal leaves the workspace exactly as it was."""+ fake = _FakeWorkspaceManager()+ with _patch_skills_path(fake):+ with pytest.raises(SkillPackageError, match="the limit is 100"):+ await store_user_skill(+ "user-1",+ name="big",+ description="d",+ body="b",+ files=[+ SkillFile(relative_path=f"r{i}.md", content=b"x")+ for i in range(MAX_PACKAGE_FILES + 1)+ ],+ )+ assert fake.files == {}+++async def test_a_package_writes_its_files_before_its_skill_md():+ """The SKILL.md is what puts a skill in <available_skills>, so it goes+ last: the model never reads a skill whose resources are not there yet."""+ fake = _FakeWorkspaceManager()+ with _patch_skills_path(fake):+ await store_user_skill(+ "user-1",+ name="pkg",+ description="d",+ body="b",+ files=[+ SkillFile(relative_path="references/a.md", content="alpha"),+ SkillFile(+ relative_path="scripts/run.py", content=b"x", is_executable=True+ ),+ ],+ )+ assert list(fake.files) == [+ "/skills/pkg/references/a.md",+ "/skills/pkg/scripts/run.py",+ "/skills/pkg/SKILL.md",+ ]+ assert fake.files["/skills/pkg/references/a.md"] == b"alpha"+ assert fake.metadata["/skills/pkg/scripts/run.py"] == {"executable": True}+++async def test_a_failed_file_write_leaves_no_skill_and_no_tree():+ fake = _FailingWorkspaceManager(fail_on=3)+ with _patch_skills_path(fake):+ with pytest.raises(RuntimeError, match="storage unavailable"):+ await store_user_skill(+ "user-1",+ name="pkg",+ description="d",+ body="b",+ files=[+ SkillFile(relative_path=f"r{i}.md", content=b"x") for i in range(4)+ ],+ )+ assert await _list_user_skills_from_workspace("user-1") == []+ assert fake.files == {}+++async def test_re_storing_with_fewer_files_removes_the_orphans():+ fake = _FakeWorkspaceManager()+ with _patch_skills_path(fake):+ await store_user_skill(+ "user-1",+ name="pkg",+ description="d",+ body="b",+ files=[+ SkillFile(relative_path="keep.md", content=b"1"),+ SkillFile(relative_path="scripts/gone.py", content=b"2"),+ ],+ )+ await store_user_skill(+ "user-1",+ name="pkg",+ description="d",+ body="b2",+ files=[SkillFile(relative_path="keep.md", content=b"3")],+ )+ assert set(fake.files) == {"/skills/pkg/keep.md", "/skills/pkg/SKILL.md"}+ assert fake.files["/skills/pkg/keep.md"] == b"3"+++async def test_a_single_file_store_leaves_an_existing_package_alone():+ """``store_skill`` — the model's own tool — passes no files. It must+ rewrite the body without deleting the package it belongs to."""+ fake = _FakeWorkspaceManager()+ with _patch_skills_path(fake):+ await store_user_skill(+ "user-1",+ name="pkg",+ description="d",+ body="b",+ files=[SkillFile(relative_path="scripts/run.py", content=b"1")],+ )+ await store_user_skill("user-1", name="pkg", description="d", body="new body")+ assert "/skills/pkg/scripts/run.py" in fake.files+ assert b"new body" in fake.files["/skills/pkg/SKILL.md"]+++async def test_a_stored_executable_flag_reaches_the_sandbox_copy():+ """A package file outside ``scripts/`` is only runnable if the bit it+ arrived with survives the store."""+ fake = _FakeWorkspaceManager()+ with _patch_skills_path(fake) as patched:+ await store_user_skill(+ "user-1",+ name="pkg",+ description="d",+ body="b",+ files=[+ SkillFile(+ relative_path="bin/tool", content=b"#!/bin/sh\n", is_executable=True+ ),+ SkillFile(relative_path="bin/data.txt", content=b"plain"),+ ],+ )+ result = await ReadSkillTool()._execute(+ user_id="user-1", session=_make_session(), name="pkg"+ )+ assert isinstance(result, ReadSkillResponse)+ package_dir = os.path.join(patched.workdir, "skills", "pkg")+ assert os.stat(os.path.join(package_dir, "bin", "tool")).st_mode & 0o111+ assert not os.stat(os.path.join(package_dir, "bin", "data.txt")).st_mode & 0o111++@pytest.mark.asyncioasync def test_a_file_dropped_from_a_package_leaves_the_workdir():"""delete_skill then store_skill under the same slug is reachable in one@@ -1540,6 +1887,99 @@ async def test_delete_drains_a_folder_bigger_than_one_page():@pytest.mark.asyncio+async def test_a_mode_only_change_is_re_applied_in_the_sandbox():+ """A file whose executable bit flips without its bytes changing hashes the+ same, so a digest-only manifest skips it and leaves a tool unrunnable."""+ fake = _FakeWorkspaceManager()+ with _patch_skills_path(fake) as patched:+ for executable in (False, True, False):+ await store_user_skill(+ "user-1",+ name="pkg",+ description="d",+ body="b",+ files=[+ SkillFile(+ relative_path="bin/tool",+ content=b"#!/bin/sh\n",+ is_executable=executable,+ )+ ],+ )+ result = await ReadSkillTool()._execute(+ user_id="user-1", session=_make_session(), name="pkg"+ )+ assert isinstance(result, ReadSkillResponse)+ tool = os.path.join(patched.workdir, "skills", "pkg", "bin", "tool")+ assert bool(os.stat(tool).st_mode & 0o111) is executable+++async def test_a_manifest_from_before_executable_tracking_is_still_honoured():+ """T233.1's manifest maps a path to a bare digest. Discarding it would+ re-copy every file of every package once, so it is read as+ non-executable — the safe direction, costing at most a redundant chmod.++ The skip is what is asserted: a sentinel written over the materialised+ file survives only if the manifest hit stopped the re-copy."""+ fake = _FakeWorkspaceManager()+ with _patch_skills_path(fake) as patched:+ await store_user_skill(+ "user-1",+ name="pkg",+ description="d",+ body="b",+ files=[SkillFile(relative_path="references/a.md", content=b"alpha")],+ )+ package_dir = os.path.join(patched.workdir, "skills", "pkg")+ os.makedirs(os.path.join(package_dir, "references"), exist_ok=True)+ digest = hashlib.sha256(b"alpha").hexdigest()+ manifest_path = os.path.join(patched.workdir, ".skill-packages", "pkg.json")+ os.makedirs(os.path.dirname(manifest_path), exist_ok=True)+ with open(manifest_path, "w") as f:+ json.dump({"references/a.md": digest}, f)+ materialised = os.path.join(package_dir, "references", "a.md")+ with open(materialised, "wb") as f:+ f.write(b"sentinel")++ result = await ReadSkillTool()._execute(+ user_id="user-1", session=_make_session(), name="pkg"+ )+ assert isinstance(result, ReadSkillResponse)+ with open(materialised, "rb") as f:+ assert f.read() == b"sentinel"+ with open(manifest_path) as f:+ assert json.load(f) == {+ "references/a.md": {"sha256": digest, "executable": False}+ }+++async def test_a_copy_that_cannot_read_a_file_writes_nothing():+ """The copy is idempotent on the root, so an expert given a package with a+ file missing would never get the missing one — the next call sees the root+ and returns early. Refusing the copy is what keeps that repairable."""+ fake = _FakeWorkspaceManager()+ fake.files["/skills/mine/SKILL.md"] = render_skill_markdown(+ ParsedSkill(name="mine", description="d", body="steps")+ ).encode()+ fake.files["/skills/mine/references/a.md"] = b"alpha"+ fake.files["/skills/mine/references/b.md"] = b"beta"++ real_read = fake.read_file++ async def read(path: str) -> bytes:+ if path.endswith("references/b.md"):+ raise RuntimeError("blob store unavailable")+ return await real_read(path)++ fake.read_file = read+ with _patch_skills_path(fake):+ with pytest.raises(RuntimeError, match="blob store unavailable"):+ await copy_skill_to_expert("user-1", "expert-a", "mine")+ assert not [p for p in fake.files if p.startswith("/experts/")]++async def test_a_manifest_path_with_a_parent_segment_is_dropped():"""The manifest is read back out of the working directory, which themodel's own shell can write, so its keys drive a delete only after the@@ -1726,3 +2166,278 @@ async def test_a_package_file_named_like_the_manifest_survives():)with open(theirs, "rb") as f:assert f.read() == b'{"mine": true}'+++async def test_one_activation_prunes_a_dropped_file_and_re_chmods_a_kept_one():+ """The two halves of this layer meet here: pruning keys off the manifest+ (#14544) and carrying a mode in its values. A digest-only manifest leaves+ the kept file's bit stale; a prune that reads the manifest's values as+ strings cannot run at all."""+ fake = _FakeWorkspaceManager()+ with _patch_skills_path(fake) as patched:+ await store_user_skill(+ "user-1",+ name="pkg",+ description="d",+ body="b",+ files=[+ SkillFile(relative_path="bin/tool", content=b"#!/bin/sh\n"),+ SkillFile(relative_path="scripts/gone.sh", content=b"echo stale\n"),+ ],+ )+ await ReadSkillTool()._execute(+ user_id="user-1", session=_make_session(), name="pkg"+ )+ package_dir = os.path.join(patched.workdir, "skills", "pkg")+ dropped = os.path.join(package_dir, "scripts", "gone.sh")+ kept = os.path.join(package_dir, "bin", "tool")+ assert os.path.exists(dropped)+ assert not os.stat(kept).st_mode & 0o111++ # One re-store: the package loses a file and flips the bit on the file+ # it keeps, with that file's bytes unchanged.+ await store_user_skill(+ "user-1",+ name="pkg",+ description="d",+ body="b",+ files=[+ SkillFile(+ relative_path="bin/tool",+ content=b"#!/bin/sh\n",+ is_executable=True,+ )+ ],+ )+ await ReadSkillTool()._execute(+ user_id="user-1", session=_make_session(), name="pkg"+ )+ assert not os.path.exists(dropped)+ assert os.stat(kept).st_mode & 0o111+++async def test_the_prune_never_reads_a_root_skill_md_as_a_stale_sibling():+ """`_package_path_error` refuses a root SKILL.md as a package file while+ `_is_safe_relative` accepts it, so the manifest — which the model's own+ shell can write — is filtered by the stricter of the two."""+ fake = _FakeWorkspaceManager()+ with _patch_skills_path(fake) as patched:+ await store_user_skill(+ "user-1",+ name="pkg",+ description="d",+ body="b",+ files=[SkillFile(relative_path="references/a.md", content=b"alpha")],+ )+ await ReadSkillTool()._execute(+ user_id="user-1", session=_make_session(), name="pkg"+ )+ package_dir = os.path.join(patched.workdir, "skills", "pkg")+ decoy = os.path.join(package_dir, "SKILL.md")+ with open(decoy, "w") as f:+ f.write("the model's own note")+ escape = os.path.join(patched.workdir, "outside.txt")+ with open(escape, "w") as f:+ f.write("not the package's")+ manifest_path = os.path.join(patched.workdir, ".skill-packages", "pkg.json")+ with open(manifest_path, "w") as f:+ json.dump(+ {+ "SKILL.md": {"sha256": "dead", "executable": False},+ "../../outside.txt": {"sha256": "beef", "executable": False},+ "references/a.md": {+ "sha256": hashlib.sha256(b"alpha").hexdigest(),+ "executable": False,+ },+ },+ f,+ )++ await ReadSkillTool()._execute(+ user_id="user-1", session=_make_session(), name="pkg"+ )+ assert os.path.exists(decoy)+ assert os.path.exists(escape)+++async def test_a_chmod_that_failed_is_retried_next_activation():+ """The manifest is what a later activation trusts instead of re-doing the+ work, so recording a mode that never applied makes a transient sandbox+ failure permanent: the file is skipped from then on and stays unrunnable."""+ fake = _FakeWorkspaceManager()+ with _patch_skills_path(fake) as patched:+ await store_user_skill(+ "user-1",+ name="pkg",+ description="d",+ body="b",+ files=[+ SkillFile(+ relative_path="bin/tool",+ content=b"#!/bin/sh\n",+ is_executable=True,+ )+ ],+ )+ real = skills.set_executable+ calls: list[list[str]] = []++ async def flaky(paths, executable, session_id):+ calls.append(list(paths))+ if len(calls) <= 2:+ return list(paths)+ return await real(paths, executable, session_id)++ tool = os.path.join(patched.workdir, "skills", "pkg", "bin", "tool")+ with patch.object(skills, "set_executable", flaky):+ await ReadSkillTool()._execute(+ user_id="user-1", session=_make_session(), name="pkg"+ )+ assert not os.stat(tool).st_mode & 0o111+ await ReadSkillTool()._execute(+ user_id="user-1", session=_make_session(), name="pkg"+ )+ assert os.stat(tool).st_mode & 0o111+++async def test_a_stale_file_that_would_not_go_stays_in_the_manifest():+ """Dropping it from the manifest is what stops the next activation from+ trying again, so a file the package no longer has stays where bash_exec+ can run it for good."""+ fake = _FakeWorkspaceManager()+ with _patch_skills_path(fake) as patched:+ await store_user_skill(+ "user-1",+ name="pkg",+ description="d",+ body="b",+ files=[+ SkillFile(relative_path="a.md", content=b"a"),+ SkillFile(relative_path="gone.md", content=b"g"),+ ],+ )+ await ReadSkillTool()._execute(+ user_id="user-1", session=_make_session(), name="pkg"+ )+ await store_user_skill(+ "user-1",+ name="pkg",+ description="d",+ body="b",+ files=[SkillFile(relative_path="a.md", content=b"a")],+ )+ asked: list[list[str]] = []++ async def dead_rm(paths, session_id):+ asked.append(list(paths))+ return list(paths)++ with patch.object(skills, "remove_from_workdir", dead_rm):+ await ReadSkillTool()._execute(+ user_id="user-1", session=_make_session(), name="pkg"+ )+ await ReadSkillTool()._execute(+ user_id="user-1", session=_make_session(), name="pkg"+ )+ gone = os.path.join(patched.workdir, "skills", "pkg", "gone.md")+ # Asked both times, not just the first.+ assert [c for c in asked if c] == [[gone], [gone]]+ with open(os.path.join(patched.workdir, ".skill-packages", "pkg.json")) as f:+ assert "gone.md" in json.load(f)+++async def test_an_emptied_package_still_records_a_deletion_that_failed():+ """The manifest write is guarded so a single-file skill leaves no empty+ package directory behind. When every file goes and the deletion fails,+ nothing this pass copied is left to carry the retry — the entry has to+ survive inside the guarded value, not around it."""+ fake = _FakeWorkspaceManager()+ with _patch_skills_path(fake) as patched:+ await store_user_skill(+ "user-1",+ name="pkg",+ description="d",+ body="b",+ files=[SkillFile(relative_path="gone.md", content=b"g")],+ )+ await ReadSkillTool()._execute(+ user_id="user-1", session=_make_session(), name="pkg"+ )+ # The package now has no files at all.+ await store_user_skill(+ "user-1", name="pkg", description="d", body="b", files=[]+ )++ async def dead_rm(paths, session_id):+ return list(paths)++ with patch.object(skills, "remove_from_workdir", dead_rm):+ await ReadSkillTool()._execute(+ user_id="user-1", session=_make_session(), name="pkg"+ )+ with open(os.path.join(patched.workdir, ".skill-packages", "pkg.json")) as f:+ assert "gone.md" in json.load(f)+++async def test_an_unchanged_package_is_not_re_read_on_the_next_activation():+ """The manifest fast-path used to sit after the blob read, so it saved the+ write and never the fetch: every activation re-read every file to compute a+ hash the workspace had already stored."""+ fake = _FakeWorkspaceManager()+ with _patch_skills_path(fake):+ await store_user_skill(+ "user-1",+ name="pkg",+ description="d",+ body="b",+ files=[+ SkillFile(relative_path=f"references/r{i}.md", content=f"c{i}".encode())+ for i in range(20)+ ],+ )+ await ReadSkillTool()._execute(+ user_id="user-1", session=_make_session(), name="pkg"+ )+ fake.reads.clear()+ await ReadSkillTool()._execute(+ user_id="user-1", session=_make_session(), name="pkg"+ )+ # The SKILL.md itself is still read; none of the twenty package files are.+ assert [r for r in fake.reads if r.startswith("/skills/pkg/references/")] == []+++async def test_a_file_changed_in_place_is_still_picked_up():+ """The one way a read-skipping fast-path can be wrong: trusting a stale+ hash. The bytes change without the size changing, so nothing but the+ checksum distinguishes the new content from the old."""+ fake = _FakeWorkspaceManager()+ with _patch_skills_path(fake) as patched:+ await store_user_skill(+ "user-1",+ name="pkg",+ description="d",+ body="b",+ files=[SkillFile(relative_path="references/a.md", content=b"before")],+ )+ await ReadSkillTool()._execute(+ user_id="user-1", session=_make_session(), name="pkg"+ )+ materialised = os.path.join(+ patched.workdir, "skills", "pkg", "references", "a.md"+ )+ with open(materialised, "rb") as f:+ assert f.read() == b"before"++ fake.files["/skills/pkg/references/a.md"] = b"after!" # same length+ await ReadSkillTool()._execute(+ user_id="user-1", session=_make_session(), name="pkg"+ )+ with open(materialised, "rb") as f:+ assert f.read() == b"after!"diff --git a/autogpt_platform/backend/backend/copilot/tools/workdir.py b/autogpt_platform/backend/backend/copilot/tools/workdir.pyindex f3ff384050..966046a756 100644--- a/autogpt_platform/backend/backend/copilot/tools/workdir.py+++ b/autogpt_platform/backend/backend/copilot/tools/workdir.py@@ -93,23 +93,22 @@ async def read_workdir_bytes(path: str, session_id: str) -> bytes | None:return None-async def remove_from_workdir(paths: list[str], session_id: str) -> None:- """Best-effort delete. A file left behind is one the model can still read- or run after it has left the package, so a failure here is logged loudly- rather than swallowed."""+async def remove_from_workdir(paths: list[str], session_id: str) -> list[str]:+ """Best-effort delete, returning the paths that are still there.++ A file left behind is one the model can still read or run after it has+ left the package, so the caller needs to know which ones to try again+ rather than record as gone. The sandbox runs one command for the batch,+ so a failure there is reported against all of them.+ """if not paths:- return+ return []sandbox = get_current_sandbox()try:if sandbox is not None:quoted = " ".join(shlex.quote(p) for p in paths)await sandbox.commands.run(f"rm -f {quoted}")- return- for path in paths:- try:- os.remove(path)- except FileNotFoundError:- pass+ return []except Exception:logger.warning("[workdir] failed to remove %d stale file(s) in session %s",@@ -117,29 +116,68 @@ async def remove_from_workdir(paths: list[str], session_id: str) -> None:session_id,exc_info=True,)+ return list(paths)+ failed: list[str] = []+ for path in paths:+ try:+ os.remove(path)+ except FileNotFoundError:+ pass+ except Exception:+ logger.warning(+ "[workdir] failed to remove %s in session %s",+ path,+ session_id,+ exc_info=True,+ )+ failed.append(path)+ return failed-async def make_executable(paths: list[str], session_id: str) -> None:- """Best-effort ``chmod +x``. A package whose scripts are not executable is- still usable through ``python script.py``, so a failure here is logged and- never surfaced."""+async def set_executable(+ paths: list[str], executable: bool, session_id: str+) -> list[str]:+ """Best-effort ``chmod +x`` / ``-x``, returning the paths it could not set.++ Rewriting a file does not change an existing mode, so clearing the bit+ needs its own call. A package whose scripts are not executable is still+ usable through ``python script.py``, so this never raises — but the caller+ must not record a mode that was not applied, or the retry never comes.+ The sandbox runs one command for the batch, so a failure there is reported+ against all of them.+ """if not paths:- return+ return []sandbox = get_current_sandbox()try:if sandbox is not None:quoted = " ".join(shlex.quote(p) for p in paths)- await sandbox.commands.run(f"chmod +x {quoted}")- return- for path in paths:- os.chmod(path, os.stat(path).st_mode | 0o111)+ await sandbox.commands.run(f"chmod {'+' if executable else '-'}x {quoted}")+ return []except Exception:logger.warning(- "[workdir] failed to mark %d file(s) executable in session %s",+ "[workdir] failed to set executable=%s on %d file(s) in session %s",+ executable,len(paths),session_id,exc_info=True,)+ return list(paths)+ failed: list[str] = []+ for path in paths:+ try:+ mode = os.stat(path).st_mode+ os.chmod(path, mode | 0o111 if executable else mode & ~0o111)+ except Exception:+ logger.warning(+ "[workdir] failed to set executable=%s on %s in session %s",+ executable,+ path,+ session_id,+ exc_info=True,+ )+ failed.append(path)+ return faileddef resolve_sandbox_path_or_error(