Skip to content

AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.

fix(backend): save a skill package as one change (#15599)

Nicholas Tindleccb658aa3977
diff --git a/autogpt_platform/backend/backend/copilot/tools/skill_package_versions_test.py b/autogpt_platform/backend/backend/copilot/tools/skill_package_versions_test.py
index a1a84ddc93..da6570c632 100644
--- a/autogpt_platform/backend/backend/copilot/tools/skill_package_versions_test.py
+++ b/autogpt_platform/backend/backend/copilot/tools/skill_package_versions_test.py
@@ -96,7 +96,7 @@ async def test_untracked_script_edit_is_not_attributed_to_the_old_version(
@pytest.mark.asyncio
-async def test_failed_file_removal_keeps_publication_pending_until_retry(
+async def test_a_failed_package_swap_keeps_publication_pending_until_retry(
package_workspace, fake_learning_store, monkeypatch
):
for target in ("skill_publication_db", "skill_versions_db"):
@@ -125,10 +125,15 @@ async def test_failed_file_removal_keeps_publication_pending_until_retry(
),
expected_current_version=head.current_version,
)
- delete = package_workspace.delete_file
- monkeypatch.setattr(
- package_workspace, "delete_file", AsyncMock(side_effect=OSError("unavailable"))
- )
+ write = package_workspace.write_file
+
+ async def fail_the_root(**kwargs):
+ # The last step of the package swap, after the stale file is gone.
+ if kwargs["path"].endswith("/SKILL.md"):
+ raise OSError("unavailable")
+ return await write(**kwargs)
+
+ monkeypatch.setattr(package_workspace, "write_file", fail_the_root)
failed = await publish.write_committed_version("user-1", pending.version, None)
assert failed.status == "write_failed"
assert (
@@ -138,7 +143,8 @@ async def test_failed_file_removal_keeps_publication_pending_until_retry(
"user-1", ChatSession.new("user-1", dry_run=False), name="checks"
)
assert isinstance(loaded, ErrorResponse)
- monkeypatch.setattr(package_workspace, "delete_file", delete)
+ assert "/skills/checks/scripts/removed.py" in package_workspace.files
+ monkeypatch.setattr(package_workspace, "write_file", write)
retried = await publish.write_committed_version("user-1", pending.version, None)
assert retried.status == "applied"
assert "/skills/checks/scripts/removed.py" not in package_workspace.files
diff --git a/autogpt_platform/backend/backend/copilot/tools/skill_update_recovery_test.py b/autogpt_platform/backend/backend/copilot/tools/skill_update_recovery_test.py
index 92f83e7458..ed5a750755 100644
--- a/autogpt_platform/backend/backend/copilot/tools/skill_update_recovery_test.py
+++ b/autogpt_platform/backend/backend/copilot/tools/skill_update_recovery_test.py
@@ -9,7 +9,7 @@ from backend.copilot.tools.skills_test import _FakeWorkspaceManager, _patch_skil
@pytest.mark.asyncio
@pytest.mark.parametrize("change_asset", [False, True])
-async def test_marketplace_cleanup_failure_preserves_baseline_and_retries(
+async def test_a_failed_marketplace_update_preserves_baseline_and_retries(
monkeypatch, fake_learning_store, change_asset
):
manager = _FakeWorkspaceManager()
@@ -32,10 +32,15 @@ async def test_marketplace_cleanup_failure_preserves_baseline_and_retries(
remember = AsyncMock()
monkeypatch.setattr(skills, "invalidate_skills_index_cache", invalidate)
monkeypatch.setattr(skills, "_remember_updated", remember)
- delete = manager.delete_file
- monkeypatch.setattr(
- manager, "delete_file", AsyncMock(side_effect=OSError("unavailable"))
- )
+ write = manager.write_file
+
+ async def fail_the_root(**kwargs):
+ # The last step of the package swap, after the stale file is gone.
+ if kwargs["path"].endswith("/SKILL.md"):
+ raise OSError("unavailable")
+ return await write(**kwargs)
+
+ monkeypatch.setattr(manager, "write_file", fail_the_root)
[failed] = await skills.list_user_skills("user-1")
assert failed.update == skills.UPDATE_AVAILABLE
@@ -54,7 +59,7 @@ async def test_marketplace_cleanup_failure_preserves_baseline_and_retries(
invalidate.assert_not_awaited()
remember.assert_not_awaited()
- monkeypatch.setattr(manager, "delete_file", delete)
+ monkeypatch.setattr(manager, "write_file", write)
[recovered] = await skills.list_user_skills("user-1")
assert recovered.baseline is not None
assert recovered.baseline.version_id == v2.version_id
diff --git a/autogpt_platform/backend/backend/copilot/tools/skills.py b/autogpt_platform/backend/backend/copilot/tools/skills.py
index 47de6bce77..1250b0e6d4 100644
--- a/autogpt_platform/backend/backend/copilot/tools/skills.py
+++ b/autogpt_platform/backend/backend/copilot/tools/skills.py
@@ -98,7 +98,7 @@ from backend.data.workspace_scope import (
from backend.executor.cluster_lock import AsyncClusterLock
from backend.util.exceptions import ConflictError
from backend.util.feature_flag import Flag, is_feature_enabled
-from backend.util.workspace import WorkspaceManager
+from backend.util.workspace import SkillPackageFile, WorkspaceManager
from backend.util.workspace_storage import compute_file_checksum
if TYPE_CHECKING:
@@ -1555,61 +1555,45 @@ async def _write_skill(
metadata.update(_baseline_metadata(baseline))
folder = skill_folder(expert_id)
stale = await _stale_files(manager, folder, name, skill)
- # The root is what indexes the skill, so it goes last: a new skill
- # that fails part-way is never indexed. An upsert cannot be made
- # atomic here — the old bytes are gone once overwritten. Serial
- # because ``write_file``'s quota check is read-then-write.
- existing_paths = {f.path for f in stale}
current = {f.path: f for f in stale}
written: set[str] = set()
checksums: set[str] = set()
- try:
- for entry in skill.files or []:
- path = f"{folder}/{name}/{entry.relative_path}"
- written.add(path)
- digest = compute_file_checksum(entry.content)
- kept = current.get(path)
- if (
- kept is not None
- and kept.checksum == digest
- and kept.is_executable == entry.is_executable
- ):
- # Same bytes already there: a delta write leaves it alone.
- checksums.add(digest)
- continue
- await manager.write_file(
- content=entry.content,
- filename=entry.relative_path.rsplit("/", 1)[-1],
+ changed: list[SkillPackageFile] = []
+ for entry in skill.files or []:
+ path = f"{folder}/{name}/{entry.relative_path}"
+ written.add(path)
+ digest = compute_file_checksum(entry.content)
+ checksums.add(digest)
+ kept = current.get(path)
+ if (
+ kept is not None
+ and kept.checksum == digest
+ and kept.is_executable == entry.is_executable
+ ):
+ # Same bytes already there: a delta write leaves it alone.
+ continue
+ changed.append(
+ SkillPackageFile(
path=path,
- mime_type=None,
- overwrite=True,
- metadata=({_META_EXECUTABLE: True} if entry.is_executable else None),
- scanned_checksums=skill.scanned_checksums,
+ content=entry.content,
+ metadata={_META_EXECUTABLE: True} if entry.is_executable else None,
)
- checksums.add(digest)
- await _delete_paths(
- manager,
- {f.path for f in stale if f.path not in written},
- stale,
- strict=True,
)
- skill_md = skill.rendered.encode("utf-8")
- await manager.write_file(
- content=skill_md,
- filename="SKILL.md",
+ skill_md = skill.rendered.encode("utf-8")
+ # One change for the whole package: a failure leaves the previous
+ # package as it was, never new files beside an old SKILL.md.
+ await manager.write_skill_package(
+ root=SkillPackageFile(
path=_skill_md_path(name, expert_id),
+ content=skill_md,
mime_type="text/markdown",
- overwrite=True,
metadata=metadata,
- scanned_checksums=skill.scanned_checksums,
- )
- checksums.add(compute_file_checksum(skill_md))
- except Exception:
- # Not a rollback: a file already here keeps the new bytes, so an
- # upsert can fail mixed. Undo only what this call created — deleting
- # the rest would turn a failed write into a lost file.
- await _delete_paths(manager, written - existing_paths)
- raise
+ ),
+ files=changed,
+ removed_paths=sorted(f.path for f in stale if f.path not in written),
+ scanned_checksums=skill.scanned_checksums,
+ )
+ checksums.add(compute_file_checksum(skill_md))
return frozenset(checksums)
@@ -1637,30 +1621,6 @@ async def _stale_files(
return [info for path, info in listed.items() if path != SKILL_MD]
-async def _delete_paths(
- manager: WorkspaceManager,
- paths: set[str],
- known: list[SkillFileInfo] | None = None,
- *,
- strict: bool = False,
-) -> None:
- """Require stale-file removal for exact replacements; cleanup is best-effort."""
- ids = {f.path: f.file_id for f in known or []}
- for path in paths:
- try:
- file_id = ids.get(path)
- if file_id is None:
- info = await manager.get_file_info_by_path(path)
- if info is None:
- continue
- file_id = info.id
- await manager.delete_file(file_id)
- except Exception:
- if strict:
- raise
- logger.warning("[skills] failed to delete %s", path, exc_info=True)
-
-
async def _parse_skill_from_workspace(
manager: WorkspaceManager, file_path: str
) -> ParsedSkill | None:
diff --git a/autogpt_platform/backend/backend/copilot/tools/skills_test.py b/autogpt_platform/backend/backend/copilot/tools/skills_test.py
index 96747e6088..59c761a3b2 100644
--- a/autogpt_platform/backend/backend/copilot/tools/skills_test.py
+++ b/autogpt_platform/backend/backend/copilot/tools/skills_test.py
@@ -280,6 +280,34 @@ class _FakeWorkspaceManager:
self.metadata[path] = metadata or {}
self.scanned[path] = frozenset(scanned_checksums)
+ async def write_skill_package(
+ self, *, root, files, removed_paths, scanned_checksums=()
+ ):
+ """All or nothing, as the real manager's single transaction is: a
+ failure part-way puts every file back as it was."""
+ before = (dict(self.files), dict(self.metadata), dict(self.scanned))
+ try:
+ for item in files:
+ await self._write_package_item(item, scanned_checksums)
+ for path in removed_paths:
+ self.files.pop(path, None)
+ self.metadata.pop(path, None)
+ await self._write_package_item(root, scanned_checksums)
+ except Exception:
+ self.files, self.metadata, self.scanned = before
+ raise
+
+ async def _write_package_item(self, item, scanned_checksums):
+ await self.write_file(
+ content=item.content,
+ filename=item.path.rsplit("/", 1)[-1],
+ path=item.path,
+ mime_type=item.mime_type,
+ overwrite=True,
+ metadata=item.metadata,
+ scanned_checksums=scanned_checksums,
+ )
+
async def read_file(self, path: str) -> bytes:
if path not in self.files:
raise FileNotFoundError(path)
@@ -2570,14 +2598,11 @@ async def test_a_rejected_root_publication_removes_new_package_files():
@pytest.mark.asyncio
-async def test_a_failed_re_store_undoes_only_what_it_wrote():
- """The rollback deletes what this call created, never what was already
- there: a file that existed has lost its old bytes to the overwrite either
- way, so deleting it would turn a failed write into a lost file."""
- fake = _FailingWorkspaceManager(fail_on=6)
+async def test_a_failed_re_store_leaves_the_previous_package_as_it_was():
+ """A save that fails part-way changes nothing: the old SKILL.md never
+ ends up beside new bytes, or without a file it still names."""
+ fake = _FailingWorkspaceManager(fail_on=0)
with _patch_skills_path(fake):
- # Writes 1-2 the siblings, 3 the SKILL.md.
- fake.fail_on = 0
await store_user_skill(
"user-1",
name="pkg",
@@ -2588,8 +2613,9 @@ async def test_a_failed_re_store_undoes_only_what_it_wrote():
SkillFile(relative_path="untouched.md", content=b"old too"),
],
)
- # Writes 4 kept.md (already there), 5 added.md (new), 6 fails.
- fake.fail_on = 6
+ before = dict(fake.files)
+ # kept.md changes, added.md is new, and doomed.md's write fails.
+ fake.fail_on = fake.writes + 3
with pytest.raises(RuntimeError, match="storage unavailable"):
await store_user_skill(
"user-1",
@@ -2603,12 +2629,7 @@ async def test_a_failed_re_store_undoes_only_what_it_wrote():
],
)
- assert "/skills/pkg/kept.md" in fake.files, "an existing file was deleted"
- assert "/skills/pkg/untouched.md" in fake.files
- assert "/skills/pkg/added.md" not in fake.files
- assert "/skills/pkg/doomed.md" not in fake.files
- # The skill itself survives: the root was never rewritten.
- assert "/skills/pkg/SKILL.md" in fake.files
+ assert fake.files == before
@pytest.mark.asyncio
diff --git a/autogpt_platform/backend/backend/data/workspace_skill.py b/autogpt_platform/backend/backend/data/workspace_skill.py
index 15c728e281..d3226ff577 100644
--- a/autogpt_platform/backend/backend/data/workspace_skill.py
+++ b/autogpt_platform/backend/backend/data/workspace_skill.py
@@ -7,7 +7,7 @@ from typing import Literal
from prisma import Prisma
from prisma.models import UserWorkspaceFile
-from pydantic import BaseModel
+from pydantic import BaseModel, Field
from backend.data import db
from backend.data.db import transaction
@@ -18,6 +18,7 @@ from backend.data.skill_capacity import (
SKILL_ORIGIN_METADATA_KEY,
SKILL_ORIGIN_USER,
SkillLimitError,
+ SkillOwnedError,
skill_origin,
skill_owner_folder,
)
@@ -25,6 +26,19 @@ from backend.data.workspace import WorkspaceFile
from backend.util.json import SafeJson
+class WorkspaceSkillFile(BaseModel):
+ """A package file other than SKILL.md, already stored at ``storage_path``."""
+
+ file_id: str
+ name: str
+ path: str
+ storage_path: str
+ mime_type: str
+ size_bytes: int
+ checksum: str | None = None
+ metadata: dict | None = None
+
+
class WorkspaceSkillWrite(BaseModel):
workspace_id: str
file_id: str
@@ -36,6 +50,15 @@ class WorkspaceSkillWrite(BaseModel):
overwrite: bool
checksum: str | None = None
metadata: dict | None = None
+ # The rest of the package, swapped in with the root: files that changed,
+ # and paths the package no longer has.
+ files: list[WorkspaceSkillFile] = Field(default_factory=list)
+ removed_paths: list[str] = Field(default_factory=list)
+
+
+class RetiredSkillFile(BaseModel):
+ file_id: str
+ storage_path: str
class WorkspaceSkillPublication(BaseModel):
@@ -43,14 +66,42 @@ class WorkspaceSkillPublication(BaseModel):
file: WorkspaceFile | None = None
replaced_storage_path: str | None = None
replaced_file_id: str | None = None
+ retired_files: list[RetiredSkillFile] = Field(default_factory=list)
+
+
+def publication_refusal(
+ publication: WorkspaceSkillPublication, write: WorkspaceSkillWrite
+) -> Exception | None:
+ """The error a refused publication raises; ``None`` once it is stored."""
+ if publication.status == "capacity":
+ origin = skill_origin(write.metadata) or SKILL_ORIGIN_USER
+ return SkillLimitError(
+ f"Skill limit reached ({MAX_SKILLS_PER_EXPERT} {SKILL_ORIGIN_LABELS[origin]} "
+ "skills). Delete an unused skill first."
+ )
+ if publication.status == "owned":
+ return SkillOwnedError(
+ f"'{write.path.rsplit('/', 2)[1]}' is one of the owner's own skills; "
+ "rename or delete it before installing a skill by that name."
+ )
+ if publication.status == "exists":
+ return ValueError(f"File already exists at path: {write.path}")
+ return None
async def publish_workspace_skill_file(
write: WorkspaceSkillWrite,
) -> WorkspaceSkillPublication:
+ """Publish a skill root, and with it any package files, as one change.
+
+ Every blob is stored before this runs, so readers see either the whole
+ previous package or the whole new one: a refused or failed publication
+ leaves every row as it was.
+ """
folder = skill_owner_folder(write.path)
if folder is None:
raise ValueError("Skill publication requires a canonical SKILL.md path")
+ _require_package_paths(write)
async with _publication_transaction(write.workspace_id, folder) as tx:
# Live rows only: since paths are unique among live files, a path holds
# at most one, and retired roots left behind at it are history rather
@@ -82,6 +133,7 @@ async def publish_workspace_skill_file(
>= MAX_SKILLS_PER_EXPERT
):
return WorkspaceSkillPublication(status="capacity")
+ retired = await _swap_package_files(tx, write)
record = (
await _replace_root(tx, existing.id, write)
if existing
@@ -92,6 +144,7 @@ async def publish_workspace_skill_file(
file=WorkspaceFile.from_db(record),
replaced_storage_path=existing.storagePath if existing else None,
replaced_file_id=existing.id if existing else None,
+ retired_files=retired,
)
@@ -123,6 +176,53 @@ def _takes_a_slot(existing: UserWorkspaceFile | None, origin: str) -> bool:
return (existing_origin or SKILL_ORIGIN_USER) != origin
+def _require_package_paths(write: WorkspaceSkillWrite) -> None:
+ package = posixpath.dirname(write.path) + "/"
+ paths = [f.path for f in write.files] + write.removed_paths
+ if any(
+ not path.startswith(package) or path == write.path or ".." in path.split("/")
+ for path in paths
+ ):
+ raise ValueError("Skill package files must sit inside the package folder")
+ if len(set(paths)) != len(paths):
+ raise ValueError("Skill package paths must be distinct")
+
+
+async def _swap_package_files(
+ tx: Prisma, write: WorkspaceSkillWrite
+) -> list[RetiredSkillFile]:
+ """Retire each live row the package replaces or drops, then add the new
+ ones. Paths are unique among live rows only, so a retired row frees its
+ path for the replacement in the same transaction."""
+ retired: list[RetiredSkillFile] = []
+ for path in [f.path for f in write.files] + write.removed_paths:
+ live = await tx.userworkspacefile.find_first(
+ where={"workspaceId": write.workspace_id, "path": path, "isDeleted": False}
+ )
+ if live is None:
+ continue
+ await tx.userworkspacefile.update(
+ where={"id": live.id},
+ data={"isDeleted": True, "deletedAt": datetime.now(timezone.utc)},
+ )
+ retired.append(RetiredSkillFile(file_id=live.id, storage_path=live.storagePath))
+ for file in write.files:
+ await tx.userworkspacefile.create(
+ data={
+ "id": file.file_id,
+ "workspaceId": write.workspace_id,
+ "name": file.name,
+ "path": file.path,
+ "storagePath": file.storage_path,
+ "mimeType": file.mime_type,
+ "sizeBytes": file.size_bytes,
+ "checksum": file.checksum,
+ "metadata": SafeJson(file.metadata or {}),
+ }
+ )
+ return retired
+
+
@asynccontextmanager
async def _publication_transaction(
workspace_id: str, folder: str
diff --git a/autogpt_platform/backend/backend/data/workspace_skill_test.py b/autogpt_platform/backend/backend/data/workspace_skill_test.py
index 6def03f531..e9e1d2b63d 100644
--- a/autogpt_platform/backend/backend/data/workspace_skill_test.py
+++ b/autogpt_platform/backend/backend/data/workspace_skill_test.py
@@ -14,11 +14,12 @@ from backend.data import db
from backend.data.skill_capacity import MAX_SKILLS_PER_EXPERT, SkillLimitError
from backend.data.workspace import WorkspaceFile, rename_workspace_file
from backend.data.workspace_skill import (
+ WorkspaceSkillFile,
WorkspaceSkillWrite,
publish_workspace_skill_file,
skill_root_has_room,
)
-from backend.util.workspace import WorkspaceManager
+from backend.util.workspace import SkillPackageFile, WorkspaceManager
from backend.util.workspace_storage import LocalWorkspaceStorage
@@ -404,6 +405,160 @@ async def test_rename_and_new_publication_share_last_slot(workspace_id: str):
)
+def sibling(workspace_id: str, path: str) -> WorkspaceSkillFile:
+ file_id = str(uuid.uuid4())
+ return WorkspaceSkillFile(
+ file_id=file_id,
+ name=path.rsplit("/", 1)[-1],
+ path=path,
+ storage_path=f"test://{file_id}",
+ mime_type="text/markdown",
+ size_bytes=3,
+ )
+
+
+async def live_paths(workspace_id: str, prefix: str) -> dict[str, str]:
+ rows = await db.prisma.userworkspacefile.find_many(
+ where={"workspaceId": workspace_id, "isDeleted": False}
+ )
+ return {row.path: row.storagePath for row in rows if row.path.startswith(prefix)}
+
+
[email protected](loop_scope="session")
+ "path",
+ [
+ "/skills/other/a.md",
+ "/skills/pkg/SKILL.md",
+ "/skills/pkg/../other/a.md",
+ "/skills/pkgx/a.md",
+ ],
+)
+async def test_package_files_must_sit_inside_their_package(
+ workspace_id: str, path: str
+):
+ with pytest.raises(ValueError, match="inside the package"):
+ await publish_workspace_skill_file(
+ write(workspace_id, "pkg").model_copy(
+ update={"files": [sibling(workspace_id, path)]}
+ )
+ )
+ assert await live_paths(workspace_id, "/skills/") == {}
+
+
[email protected](loop_scope="session")
+async def test_a_refused_root_leaves_its_package_files_alone(workspace_id: str):
+ await seed(workspace_id, MAX_SKILLS_PER_EXPERT)
+ await db.prisma.userworkspacefile.create(
+ data={
+ "workspaceId": workspace_id,
+ "name": "a.md",
+ "path": "/skills/new/a.md",
+ "storagePath": "test://old-a",
+ "mimeType": "text/markdown",
+ "sizeBytes": 3,
+ }
+ )
+ result = await publish_workspace_skill_file(
+ write(workspace_id, "new").model_copy(
+ update={
+ "files": [sibling(workspace_id, "/skills/new/a.md")],
+ "removed_paths": ["/skills/new/b.md"],
+ }
+ )
+ )
+ assert result.status == "capacity"
+ assert await live_paths(workspace_id, "/skills/new/") == {
+ "/skills/new/a.md": "test://old-a"
+ }
+
+
[email protected](loop_scope="session")
+async def test_a_package_write_swaps_every_file_or_none(
+ workspace_id: str, tmp_path: Path
+):
+ """Readers see the whole old package or the whole new one: a failed
+ blob or a failed transaction leaves every file, and no stray blob."""
+ workspace = await db.prisma.userworkspace.find_unique(where={"id": workspace_id})
+ assert workspace is not None
+ manager = WorkspaceManager(workspace.userId, workspace_id)
+ storage = LocalWorkspaceStorage(base_dir=str(tmp_path))
+
+ def package(root: bytes, files: dict[str, bytes], removed: list[str]):
+ return dict(
+ root=SkillPackageFile(
+ path="/skills/pkg/SKILL.md", content=root, mime_type="text/markdown"
+ ),
+ files=[
+ SkillPackageFile(path=f"/skills/pkg/{name}", content=content)
+ for name, content in files.items()
+ ],
+ removed_paths=[f"/skills/pkg/{name}" for name in removed],
+ )
+
+ async def contents() -> dict[str, bytes]:
+ paths = await live_paths(workspace_id, "/skills/pkg/")
+ return {path: await manager.read_file(path) for path in sorted(paths)}
+
+ def blobs() -> int:
+ return sum(1 for p in tmp_path.rglob("*") if p.is_file())
+
+ with (
+ patch(
+ "backend.util.workspace.get_workspace_storage",
+ AsyncMock(return_value=storage),
+ ),
+ patch(
+ "backend.util.workspace.get_workspace_storage_limit_bytes",
+ AsyncMock(return_value=1_000_000),
+ ),
+ patch("backend.util.workspace.scan_content_safe", AsyncMock()),
+ patch(
+ "backend.api.features.workspace.embeddings.schedule_workspace_file_embedding"
+ ),
+ patch(
+ "backend.api.features.workspace.embeddings.delete_workspace_file_embedding",
+ AsyncMock(),
+ ),
+ ):
+ await manager.write_skill_package(
+ **package(b"v1", {"a.md": b"a1", "b.md": b"b1"}, [])
+ )
+ before, stored = await contents(), blobs()
+ update = package(b"v2", {"a.md": b"a2", "c.md": b"c2"}, ["b.md"])
+
+ real_store = storage.store
+ calls = 0
+
+ async def fail_second_blob(**kwargs):
+ nonlocal calls
+ calls += 1
+ if calls == 2:
+ raise RuntimeError("storage unavailable")
+ return await real_store(**kwargs)
+
+ with patch.object(storage, "store", side_effect=fail_second_blob):
+ with pytest.raises(RuntimeError, match="storage unavailable"):
+ await manager.write_skill_package(**update)
+ assert (await contents(), blobs()) == (before, stored)
+
+ with patch(
+ "backend.data.workspace_skill._replace_root",
+ AsyncMock(side_effect=RuntimeError("database unavailable")),
+ ):
+ with pytest.raises(RuntimeError, match="database unavailable"):
+ await manager.write_skill_package(**update)
+ assert (await contents(), blobs()) == (before, stored)
+
+ await manager.write_skill_package(**update)
+ assert await contents() == {
+ "/skills/pkg/SKILL.md": b"v2",
+ "/skills/pkg/a.md": b"a2",
+ "/skills/pkg/c.md": b"c2",
+ }
+ assert blobs() == 3
+
+
@pytest.mark.asyncio(loop_scope="session")
@pytest.mark.parametrize(
"saved, unreadable, target_origin, origin",
diff --git a/autogpt_platform/backend/backend/util/workspace.py b/autogpt_platform/backend/backend/util/workspace.py
index 80bffaacf0..89efcc1d94 100644
--- a/autogpt_platform/backend/backend/util/workspace.py
+++ b/autogpt_platform/backend/backend/util/workspace.py
@@ -8,11 +8,13 @@ combining the storage backend and database layer.
import asyncio
import logging
import mimetypes
+import posixpath
import uuid
-from collections.abc import Collection
+from collections.abc import Collection, Sequence
from typing import TYPE_CHECKING, Optional
from prisma.errors import UniqueViolationError
+from pydantic import BaseModel
from backend.copilot.rate_limit import get_workspace_storage_limit_bytes
from backend.data.db_accessors import (
@@ -20,15 +22,7 @@ from backend.data.db_accessors import (
workspace_folder_db,
workspace_skill_db,
)
-from backend.data.skill_capacity import (
- MAX_SKILLS_PER_EXPERT,
- SKILL_ORIGIN_LABELS,
- SKILL_ORIGIN_USER,
- SkillLimitError,
- SkillOwnedError,
- skill_origin,
- skill_owner_folder,
-)
+from backend.data.skill_capacity import skill_owner_folder
from backend.data.workspace import WorkspaceFile
from backend.data.workspace_scope import (
EXPERT_FILE_ACCESS_DENIED,
@@ -36,10 +30,19 @@ from backend.data.workspace_scope import (
WorkspaceAccessDeniedError,
WorkspaceScope,
)
-from backend.data.workspace_skill import WorkspaceSkillWrite
+from backend.data.workspace_skill import (
+ WorkspaceSkillFile,
+ WorkspaceSkillPublication,
+ WorkspaceSkillWrite,
+ publication_refusal,
+)
from backend.util.settings import Config
from backend.util.virus_scanner import scan_content_safe
-from backend.util.workspace_storage import compute_file_checksum, get_workspace_storage
+from backend.util.workspace_storage import (
+ WorkspaceStorageBackend,
+ compute_file_checksum,
+ get_workspace_storage,
+)
if TYPE_CHECKING:
from backend.data.workspace_folder import WorkspaceFolder
@@ -62,6 +65,15 @@ def format_bytes(n: int) -> str:
logger = logging.getLogger(__name__)
+class SkillPackageFile(BaseModel):
+ """One file of a skill package written by :meth:`WorkspaceManager.write_skill_package`."""
+
+ path: str
+ content: bytes
+ mime_type: str | None = None
+ metadata: dict | None = None
+
+
class WorkspaceManager:
"""
Manages workspace file operations.
@@ -394,33 +406,24 @@ class WorkspaceManager:
replaced_file_id: str | None = None
try:
if skill_owner_folder(path) is not None:
+ write = WorkspaceSkillWrite(
+ workspace_id=self.workspace_id,
+ file_id=file_id,
+ name=filename,
+ path=path,
+ storage_path=storage_path,
+ mime_type=mime_type,
+ size_bytes=len(content),
+ overwrite=overwrite,
+ checksum=checksum,
+ metadata=metadata,
+ )
publication = await workspace_skill_db().publish_workspace_skill_file(
- WorkspaceSkillWrite(
- workspace_id=self.workspace_id,
- file_id=file_id,
- name=filename,
- path=path,
- storage_path=storage_path,
- mime_type=mime_type,
- size_bytes=len(content),
- overwrite=overwrite,
- checksum=checksum,
- metadata=metadata,
- )
+ write
)
- if publication.status == "capacity":
- origin = skill_origin(metadata) or SKILL_ORIGIN_USER
- raise SkillLimitError(
- f"Skill limit reached ({MAX_SKILLS_PER_EXPERT} {SKILL_ORIGIN_LABELS[origin]} "
- "skills). Delete an unused skill first."
- )
- if publication.status == "owned":
- raise SkillOwnedError(
- f"'{path.rsplit('/', 2)[1]}' is one of the owner's own skills; "
- "rename or delete it before installing a skill by that name."
- )
- if publication.status == "exists":
- raise ValueError(f"File already exists at path: {path}")
+ refusal = publication_refusal(publication, write)
+ if refusal is not None:
+ raise refusal
assert publication.file is not None
file = publication.file
replaced_storage_path = publication.replaced_storage_path
@@ -476,6 +479,188 @@ class WorkspaceManager:
return file
+ async def write_skill_package(
+ self,
+ root: SkillPackageFile,
+ files: Sequence[SkillPackageFile],
+ removed_paths: Collection[str],
+ scanned_checksums: Collection[str] = (),
+ ) -> WorkspaceFile:
+ """Write a skill package as one change: ``root`` is its SKILL.md,
+ ``files`` the other files that changed, and ``removed_paths`` the
+ files it no longer has.
+
+ Every blob is checked, scanned and stored before one transaction
+ swaps the rows, so a failure anywhere leaves the previous package
+ exactly as it was, rather than new files beside an old SKILL.md.
+ """
+ package = [
+ item.model_copy(update={"path": self._resolve_path(item.path)})
+ for item in [root, *files]
+ ]
+ removed = [self._resolve_path(path) for path in removed_paths]
+ for path in [*(item.path for item in package), *removed]:
+ self._authorize_path(path, write=True)
+ await self._require_package_room(package, removed)
+ for item in package:
+ if compute_file_checksum(item.content) not in scanned_checksums:
+ await scan_content_safe(
+ item.content, filename=posixpath.basename(item.path)
+ )
+ storage = await get_workspace_storage()
+ stored: list[WorkspaceSkillFile] = []
+ try:
+ for item in package:
+ stored.append(await self._store_package_blob(storage, item))
+ write = WorkspaceSkillWrite(
+ workspace_id=self.workspace_id,
+ overwrite=True,
+ files=stored[1:],
+ removed_paths=removed,
+ **stored[0].model_dump(),
+ )
+ publication = await workspace_skill_db().publish_workspace_skill_file(write)
+ except BaseException:
+ # Cancellation included: a save stopped part-way must not leave
+ # blobs no row points at. Shielded so a second cancel cannot
+ # interrupt the cleanup itself.
+ await asyncio.shield(self._discard_unpublished(storage, stored))
+ raise
+ refusal = publication_refusal(publication, write)
+ if refusal is not None:
+ await asyncio.shield(self._discard_unpublished(storage, stored))
+ raise refusal
+ assert publication.file is not None
+ await self._retire_replaced(storage, publication)
+ logger.info(
+ f"Wrote skill package {publication.file.path} ({len(stored)} files, "
+ f"{len(removed)} removed) to workspace {self.workspace_id}"
+ )
+ for blob in stored:
+ self._schedule_embedding(blob.file_id, blob.name, blob.path)
+ return publication.file
+
+ async def _discard_unpublished(
+ self, storage: WorkspaceStorageBackend, stored: list[WorkspaceSkillFile]
+ ) -> None:
+ """Delete blobs a failed package write staged. A publication whose
+ reply was lost may have committed anyway; its root row then exists
+ and every blob it names must stay."""
+ if not stored:
+ return
+ try:
+ if await workspace_db().get_workspace_file(
+ stored[0].file_id, self.workspace_id
+ ):
+ return
+ except Exception:
+ logger.warning(
+ "Could not tell whether a skill package was published; "
+ "keeping its blobs",
+ exc_info=True,
+ )
+ return
+ for blob in stored:
+ try:
+ await storage.delete(blob.storage_path)
+ except Exception:
+ logger.warning("Failed to clean up unpublished skill blob")
+
+ async def _require_package_room(
+ self, package: list[SkillPackageFile], removed: list[str]
+ ) -> None:
+ max_file_size = Config().max_file_size_mb * 1024 * 1024
+ for item in package:
+ if len(item.content) > max_file_size:
+ raise ValueError(
+ f"File too large: {len(item.content)} bytes exceeds "
+ f"{Config().max_file_size_mb}MB limit"
+ )
+ db = workspace_db()
+ storage_limit, current_usage = await asyncio.gather(
+ get_workspace_storage_limit_bytes(self.user_id),
+ db.get_workspace_total_size(self.workspace_id),
+ )
+ for path in {item.path for item in package} | set(removed):
+ existing = await db.get_workspace_file_by_path(self.workspace_id, path)
+ if existing is not None:
+ current_usage -= existing.size_bytes
+ current_usage = max(0, current_usage)
+ projected = current_usage + sum(len(item.content) for item in package)
+ if storage_limit > 0 and projected > storage_limit:
+ raise ValueError(
+ f"Storage limit exceeded. "
+ f"You've used {format_bytes(current_usage)} of your "
+ f"{format_bytes(storage_limit)} quota. "
+ f"Delete some files or upgrade your plan for more storage."
+ )
+
+ async def _store_package_blob(
+ self, storage: WorkspaceStorageBackend, item: SkillPackageFile
+ ) -> WorkspaceSkillFile:
+ file_id = str(uuid.uuid4())
+ name = posixpath.basename(item.path)
+ mime_type = item.mime_type or (
+ mimetypes.guess_type(name)[0] or "application/octet-stream"
+ )
+ storage_path = await storage.store(
+ workspace_id=self.workspace_id,
+ file_id=file_id,
+ filename=name,
+ content=item.content,
+ )
+ return WorkspaceSkillFile(
+ file_id=file_id,
+ name=name,
+ path=item.path,
+ storage_path=storage_path,
+ mime_type=mime_type,
+ size_bytes=len(item.content),
+ checksum=compute_file_checksum(item.content),
+ metadata=item.metadata,
+ )
+
+ async def _retire_replaced(
+ self, storage: WorkspaceStorageBackend, publication: WorkspaceSkillPublication
+ ) -> None:
+ """Best-effort cleanup once the new package is live: the blobs and
+ search entries of every row it replaced."""
+ retired = [(f.file_id, f.storage_path) for f in publication.retired_files]
+ if publication.replaced_file_id and publication.replaced_storage_path:
+ retired.append(
+ (publication.replaced_file_id, publication.replaced_storage_path)
+ )
+ for file_id, storage_path in retired:
+ try:
+ await storage.delete(storage_path)
+ except Exception:
+ logger.warning("Failed to clean up replaced skill blob", exc_info=True)
+ try:
+ from backend.api.features.workspace.embeddings import (
+ delete_workspace_file_embedding,
+ )
+
+ await delete_workspace_file_embedding(
+ file_id=file_id, user_id=self.user_id
+ )
+ except Exception:
+ logger.warning(
+ "Failed to clean up replaced skill embedding", exc_info=True
+ )
+
+ def _schedule_embedding(self, file_id: str, name: str, path: str) -> None:
+ try:
+ from backend.api.features.workspace.embeddings import (
+ schedule_workspace_file_embedding,
+ )
+
+ schedule_workspace_file_embedding(
+ file_id=file_id, user_id=self.user_id, name=name, path=path
+ )
+ except Exception as e:
+ # Embedding is purely a search-quality concern — never block writes.
+ logger.warning(f"Failed to schedule file embedding for {file_id}: {e}")
+
async def list_folders(self) -> list["WorkspaceFolder"]:
"""The folder tree; empty for a scope without the owner's files, whose
folders these are."""
diff --git a/autogpt_platform/backend/backend/util/workspace_skill_test.py b/autogpt_platform/backend/backend/util/workspace_skill_test.py
index 467ce75527..ad7edf8e68 100644
--- a/autogpt_platform/backend/backend/util/workspace_skill_test.py
+++ b/autogpt_platform/backend/backend/util/workspace_skill_test.py
@@ -1,3 +1,4 @@
+import asyncio
from datetime import datetime, timezone
from unittest.mock import AsyncMock, patch
@@ -6,7 +7,7 @@ import pytest
from backend.data.skill_capacity import SkillLimitError, SkillOwnedError
from backend.data.workspace import WorkspaceFile
from backend.data.workspace_skill import WorkspaceSkillPublication
-from backend.util.workspace import WorkspaceManager
+from backend.util.workspace import SkillPackageFile, WorkspaceManager
def stored_file():
@@ -95,3 +96,123 @@ async def test_skill_publication_cleans_only_the_appropriate_blob(status: str):
delete_embedding.assert_not_awaited()
database.create_workspace_file.assert_not_awaited()
database.soft_delete_workspace_file.assert_not_awaited()
+
+
+def package_manager(database, publisher, storage):
+ return (
+ patch("backend.util.workspace.workspace_db", return_value=database),
+ patch("backend.util.workspace.workspace_skill_db", return_value=publisher),
+ patch(
+ "backend.util.workspace.get_workspace_storage",
+ AsyncMock(return_value=storage),
+ ),
+ patch(
+ "backend.util.workspace.get_workspace_storage_limit_bytes",
+ AsyncMock(return_value=1_000_000),
+ ),
+ patch("backend.util.workspace.scan_content_safe", AsyncMock()),
+ )
+
+
+def package_write():
+ return dict(
+ root=SkillPackageFile(path="/skills/example/SKILL.md", content=b"root"),
+ files=[
+ SkillPackageFile(path="/skills/example/a.md", content=b"a"),
+ SkillPackageFile(path="/skills/example/b.md", content=b"b"),
+ ],
+ removed_paths=[],
+ )
+
+
+ "failure, published, error",
+ [
+ ("cancelled", False, asyncio.CancelledError),
+ ("refused", False, SkillLimitError),
+ ("rolled_back", False, RuntimeError),
+ ("reply_lost", True, RuntimeError),
+ ],
+)
+async def test_a_failed_package_write_drops_only_unpublished_blobs(
+ failure: str, published: bool, error: type[BaseException]
+):
+ """A save that stops part-way, cancelled included, deletes the blobs it
+ staged; one whose publication committed but whose reply was lost keeps
+ them, since its rows now name them."""
+ database = AsyncMock()
+ database.get_workspace_total_size.return_value = 0
+ database.get_workspace_file_by_path.return_value = None
+ database.get_workspace_file.return_value = stored_file() if published else None
+ publisher = AsyncMock()
+ storage = AsyncMock()
+ blobs = iter(["test://root", "test://a", "test://b"])
+
+ async def store(**kwargs):
+ if failure == "cancelled" and kwargs["filename"] == "b.md":
+ raise asyncio.CancelledError
+ return next(blobs)
+
+ storage.store.side_effect = store
+ if failure == "refused":
+ publisher.publish_workspace_skill_file.return_value = WorkspaceSkillPublication(
+ status="capacity"
+ )
+ else:
+ publisher.publish_workspace_skill_file.side_effect = RuntimeError(
+ "DB unavailable"
+ )
+ patches = package_manager(database, publisher, storage)
+ with patches[0], patches[1], patches[2], patches[3], patches[4]:
+ with pytest.raises(error):
+ await WorkspaceManager("user", "workspace").write_skill_package(
+ **package_write()
+ )
+
+ deleted = {call.args[0] for call in storage.delete.await_args_list}
+ if failure == "cancelled":
+ publisher.publish_workspace_skill_file.assert_not_awaited()
+ assert deleted == {"test://root", "test://a"}
+ elif published:
+ assert deleted == set()
+ else:
+ assert deleted == {"test://root", "test://a", "test://b"}
+
+
+async def test_a_cancel_during_refusal_cleanup_still_drops_the_blobs():
+ """A refused package whose save is cancelled while it cleans up still
+ deletes everything it staged."""
+ checking, release = asyncio.Event(), asyncio.Event()
+
+ async def held_check(*_args):
+ checking.set()
+ await release.wait()
+
+ database = AsyncMock()
+ database.get_workspace_total_size.return_value = 0
+ database.get_workspace_file_by_path.return_value = None
+ database.get_workspace_file.side_effect = held_check
+ publisher = AsyncMock()
+ publisher.publish_workspace_skill_file.return_value = WorkspaceSkillPublication(
+ status="capacity"
+ )
+ storage = AsyncMock()
+ blobs = iter(["test://root", "test://a", "test://b"])
+ storage.store.side_effect = lambda **_kwargs: next(blobs)
+ patches = package_manager(database, publisher, storage)
+ with patches[0], patches[1], patches[2], patches[3], patches[4]:
+ save = asyncio.create_task(
+ WorkspaceManager("user", "workspace").write_skill_package(**package_write())
+ )
+ await asyncio.wait_for(checking.wait(), 5)
+ save.cancel()
+ with pytest.raises(asyncio.CancelledError):
+ await save
+ release.set()
+ for _ in range(10):
+ await asyncio.sleep(0)
+
+ deleted = {call.args[0] for call in storage.delete.await_args_list}
+ assert deleted == {"test://root", "test://a", "test://b"}
diff --git a/docs/platform/skill-learning.md b/docs/platform/skill-learning.md
index 98b957f506..a1d6beef73 100644
--- a/docs/platform/skill-learning.md
+++ b/docs/platform/skill-learning.md
@@ -201,10 +201,11 @@ or improving a version preserves only retained approved occurrences.
New, duplicated, relocated, and seeded values still require the applicable
checks.
-Ordinary registry package writes use an error-and-retry contract. An
-interrupted ordinary write can leave partial unversioned file changes;
-retry the complete package until the write succeeds. Versioned pending
-writes stay unavailable until recovery completes. Repeating source exclusion
+Every registry package write replaces the whole package in one change:
+every file is stored first, then the package's files are swapped in a
+single database transaction. A failed or interrupted write leaves the
+previous package exactly as it was; retry it to apply the change. Versioned
+pending writes stay unavailable until recovery completes. Repeating source exclusion
also resumes any interrupted head repair, while preserving an owner use-pause.
## Not in the first release