AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.
fix(backend): speed up expert hires by batching skill installs and skipping repeat embeds and scans (#14867)
Nicholas Tindleb613cae22554
diff --git a/autogpt_platform/backend/backend/api/features/experts/experts_db.py b/autogpt_platform/backend/backend/api/features/experts/experts_db.pyindex 71bab5d08b..25cb46c5cc 100644--- a/autogpt_platform/backend/backend/api/features/experts/experts_db.py+++ b/autogpt_platform/backend/backend/api/features/experts/experts_db.py@@ -1391,6 +1391,13 @@ async def _detach_expert_skill(user_id: str, expert_id: str, name: str) -> None:async def add_expert_skill_name(user_id: str, expert_id: str, name: str) -> None:"""Record a skill the expert now owns; idempotent, and a display name andits slug count as one name."""+ await add_expert_skill_names(user_id, expert_id, [name])+++async def add_expert_skill_names(+ user_id: str, expert_id: str, names: list[str]+) -> None:+ """:func:`add_expert_skill_name` for several names in one row write."""await _rewrite_skill_names({"id": expert_id,@@ -1398,14 +1405,20 @@ async def add_expert_skill_name(user_id: str, expert_id: str, name: str) -> None"isTemplate": False,"isArchived": False,},- lambda names: (- names- if skill_name_key(name) in {skill_name_key(n) for n in names}- else [*names, name]- ),+ lambda current: _with_names(current, names),)+def _with_names(current: list[str], names: list[str]) -> list[str]:+ merged = list(current)+ keys = {skill_name_key(n) for n in merged}+ for name in names:+ if skill_name_key(name) not in keys:+ keys.add(skill_name_key(name))+ merged.append(name)+ return merged++async def remove_expert_skill_name(user_id: str, expert_id: str, name: str) -> None:"""Forget a skill the expert no longer owns, in any spelling of its name."""await _rewrite_skill_names(@@ -1792,14 +1805,22 @@ async def _install_bundled_skills(leaves no name, so the hire never lists a skill it does not have."""bundled = await _live_bundled_skills(user_id, [template_id])- for skill in bundled.get(template_id, []):- try:- await skill_db.install_marketplace_skill(- user_id, skill.slug, expert_id=expert_id+ skills = bundled.get(template_id, [])+ if not skills:+ return+ try:+ outcomes: list[object] = list(+ await skill_db.install_marketplace_skills(+ user_id, [s.slug for s in skills], expert_id=expert_id)- except Exception:- logger.exception(- f"Failed to install bundled skill {skill.slug!r} on expert #{expert_id}"+ )+ except Exception as e:+ outcomes = [e] * len(skills)+ for skill, outcome in zip(skills, outcomes):+ if isinstance(outcome, Exception):+ logger.error(+ f"Failed to install bundled skill {skill.slug!r} on expert #{expert_id}",+ exc_info=outcome,)diff --git a/autogpt_platform/backend/backend/api/features/experts/experts_db_test.py b/autogpt_platform/backend/backend/api/features/experts/experts_db_test.pyindex 49a47779f8..8a1d55d8a6 100644--- a/autogpt_platform/backend/backend/api/features/experts/experts_db_test.py+++ b/autogpt_platform/backend/backend/api/features/experts/experts_db_test.py@@ -805,7 +805,7 @@ async def test_a_failed_bundled_skill_install_does_not_fail_the_hire(server: SpinTestServer, test_user, hub_listing, skills_hub_on, monkeypatch):install = AsyncMock(side_effect=RuntimeError("storage down"))- monkeypatch.setattr(experts_db.skill_db, "install_marketplace_skill", install)+ _patch_install(monkeypatch, install)template = await _seed_template(name="Maria", preload_listings=[], bundled=[hub_listing.id])@@ -822,7 +822,7 @@ async def test_hire_installs_nothing_while_the_hub_is_off():monkeypatch.setattr(experts_db, "is_feature_enabled", AsyncMock(return_value=False))install = AsyncMock()- monkeypatch.setattr(experts_db.skill_db, "install_marketplace_skill", install)+ _patch_install(monkeypatch, install)template = await _seed_template(name="Maria", preload_listings=[], bundled=[hub_listing.id])@@ -832,6 +832,21 @@ async def test_hire_installs_nothing_while_the_hub_is_off(install.assert_not_awaited()+def _patch_install(monkeypatch, install: AsyncMock) -> None:+ """Serve the hire's batch install from a per-slug *install* double."""++ async def batch(user_id, slugs, *, expert_id):+ outcomes = []+ for slug in slugs:+ try:+ outcomes.append(await install(user_id, slug, expert_id=expert_id))+ except Exception as e:+ outcomes.append(e)+ return outcomes++ monkeypatch.setattr(experts_db.skill_db, "install_marketplace_skills", batch)++@pytest.mark.asyncio(loop_scope="session")async def test_hire_expert_is_idempotent_at_active_cap(server: SpinTestServer):owner = await _create_seed_user()@@ -5551,6 +5566,17 @@ async def test_expert_skill_names_add_and_remove_atomically(assert row is not Noneassert "alpha" not in {s.lower() for s in row.skills}+ await experts_db.add_expert_skill_names(+ test_user.id, expert_id, ["gamma", "Beta", "Gamma", "delta"]+ )+ row = await prisma.models.Expert.prisma().find_unique(where={"id": expert_id})+ assert row is not None+ assert [s for s in row.skills if s.lower() in {"beta", "gamma", "delta"}] == [+ "beta",+ "gamma",+ "delta",+ ]+@pytest.mark.asyncio(loop_scope="session")@pytest.mark.parametrize("operation", ["add", "remove"])diff --git a/autogpt_platform/backend/backend/api/features/store/skill_db.py b/autogpt_platform/backend/backend/api/features/store/skill_db.pyindex e73bbb203c..9d5b1630b8 100644--- a/autogpt_platform/backend/backend/api/features/store/skill_db.py+++ b/autogpt_platform/backend/backend/api/features/store/skill_db.py@@ -9,6 +9,8 @@ owner (personal library or expert): a re-install overwrites that copy and isnot counted again."""+import logging+import prisma.enumsimport prisma.modelsimport prisma.types@@ -16,8 +18,9 @@ import prisma.typesfrom backend.copilot.tools.skills import (SKILL_ORIGIN_MARKETPLACE,SkillFile,- list_user_skills,- store_user_skill,+ SkillWrite,+ StoredSkill,+ store_user_skills,)from backend.data.db import query_raw_with_schemafrom backend.util.exceptions import NotFoundError@@ -26,6 +29,8 @@ from backend.util.models import Paginationfrom . import skill_modelfrom .categories import category_filter_values+logger = logging.getLogger(__name__)+_LISTING_INCLUDE: prisma.types.SkillListingInclude = {"ActiveVersion": True,"CreatorProfile": True,@@ -180,57 +185,151 @@ async def install_marketplace_skill(idempotent and a re-install picks up a newer approved version. The wholepackage is passed, so a file the new version dropped is removed too."""- listing = await _find_live_listing(slug)- active = skill_model.active_version(listing)- # A re-install overwrites the existing copy, so counting it again would- # report installs rather than installers. Counting only: no healing.- owned = await list_user_skills(user_id, expert_id, heal_missing=False)- is_new = all(s.name != listing.slug for s in owned)- await store_user_skill(- user_id,- name=listing.slug,- description=active.description,- body=active.body,- triggers=list(active.triggers),- version=str(active.version),- extra={- key: value- for key, value in (- ("license", active.license),- ("source", active.sourceRepo),- ("source_url", active.sourceUrl),- )- if value is not None- },- # `[]`, never `None` — which means "leave the folder alone" and would- # keep a sibling only the previously installed version had.- files=await _read_version_files(active.id),- expert_id=expert_id,- origin=SKILL_ORIGIN_MARKETPLACE,+ [outcome] = await install_marketplace_skills(user_id, [slug], expert_id=expert_id)+ if isinstance(outcome, Exception):+ raise outcome+ return outcome+++async def install_marketplace_skills(+ user_id: str, slugs: list[str], *, expert_id: str | None = None+) -> list[skill_model.InstalledSkill | Exception]:+ """:func:`install_marketplace_skill` for several listings: one listing+ query, one file query and one locked write, with each slug's outcome+ returned in order (``NotFoundError`` for one that is not live)."""+ listings = {+ listing.slug: listing+ for listing in await prisma.models.SkillListing.prisma().find_many(+ where=_live_listing_where({"slug": {"in": slugs}}),+ include=_LISTING_INCLUDE,+ )+ }+ live = [listings[slug] for slug in slugs if slug in listings]+ files = await _read_versions_files(+ [skill_model.active_version(listing).id for listing in live])- if is_new:- await prisma.models.SkillListing.prisma().update(- where={"id": listing.id}, data={"installCount": {"increment": 1}}+ writes: list[SkillWrite] = []+ for listing in live:+ active = skill_model.active_version(listing)+ writes.append(+ SkillWrite(+ name=listing.slug,+ description=active.description,+ body=active.body,+ triggers=list(active.triggers),+ version=str(active.version),+ extra={+ key: value+ for key, value in (+ ("license", active.license),+ ("source", active.sourceRepo),+ ("source_url", active.sourceUrl),+ )+ if value is not None+ },+ # `[]`, never `None` — which means "leave the folder alone"+ # and would keep a sibling only the previous version had.+ files=files.get(active.id, []),+ scanned_checksums=frozenset(active.scannedSha256),+ ))- return skill_model.InstalledSkill(- name=listing.slug, required_providers=list(active.requiredProviders)+ stored = (+ dict(+ zip(+ (listing.slug for listing in live),+ await store_user_skills(+ user_id,+ writes,+ expert_id=expert_id,+ origin=SKILL_ORIGIN_MARKETPLACE,+ ),+ )+ )+ if writes+ else {})+ # A re-install overwrites the existing copy, so counting it again would+ # report installs rather than installers.+ new_ids = [+ listings[slug].id+ for slug, outcome in stored.items()+ if isinstance(outcome, StoredSkill) and outcome.is_new+ ]+ if new_ids:+ await prisma.models.SkillListing.prisma().update_many(+ where={"id": {"in": new_ids}}, data={"installCount": {"increment": 1}}+ )+ await _record_scanned(live, stored)+ outcomes: list[skill_model.InstalledSkill | Exception] = []+ for slug in slugs:+ outcome = stored.get(slug)+ if outcome is None:+ outcomes.append(NotFoundError(f"Skill '{slug}' not found"))+ elif isinstance(outcome, Exception):+ outcomes.append(outcome)+ else:+ outcomes.append(+ skill_model.InstalledSkill(+ name=slug,+ required_providers=list(+ skill_model.active_version(listings[slug]).requiredProviders+ ),+ )+ )+ return outcomes+++async def _record_scanned(+ live: list[prisma.models.SkillListing],+ stored: dict[str, StoredSkill | Exception],+) -> None:+ """Remember the bytes each install wrote past the scan, so the next+ install of the same version writing the same bytes need not scan them."""+ for listing in live:+ outcome = stored.get(listing.slug)+ active = skill_model.active_version(listing)+ if not isinstance(outcome, StoredSkill) or outcome.checksums <= set(+ active.scannedSha256+ ):+ continue+ try:+ await prisma.models.SkillListingVersion.prisma().update(+ where={"id": active.id},+ data={+ "scannedSha256": sorted(+ outcome.checksums | set(active.scannedSha256)+ )+ },+ )+ except Exception:+ # Only a cache of scan results: failing to record costs a rescan+ # next time, never this install.+ logger.warning(+ f"Could not record scanned checksums for skill '{listing.slug}'",+ exc_info=True,+ )-async def _read_version_files(skill_listing_version_id: str) -> list[SkillFile]:- """The published package's files, contents included, ready to install."""+async def _read_versions_files(+ version_ids: list[str],+) -> dict[str, list[SkillFile]]:+ """Each version's published files, contents included, in one query."""+ if not version_ids:+ return {}rows = await prisma.models.SkillListingFile.prisma().find_many(- where={"skillListingVersionId": skill_listing_version_id},+ where={"skillListingVersionId": {"in": version_ids}},order={"relativePath": "asc"},)- return [- SkillFile(- relative_path=row.relativePath,- content=row.content.decode(),- is_executable=row.isExecutable,+ by_version: dict[str, list[SkillFile]] = {}+ for row in rows:+ by_version.setdefault(row.skillListingVersionId, []).append(+ SkillFile(+ relative_path=row.relativePath,+ content=row.content.decode(),+ is_executable=row.isExecutable,+ ))- for row in rows- ]+ return by_versionasync def _list_version_file_meta(diff --git a/autogpt_platform/backend/backend/api/features/store/skill_db_test.py b/autogpt_platform/backend/backend/api/features/store/skill_db_test.pyindex 94ed854f6d..bb6b4eb529 100644--- a/autogpt_platform/backend/backend/api/features/store/skill_db_test.py+++ b/autogpt_platform/backend/backend/api/features/store/skill_db_test.py@@ -1,13 +1,15 @@import datetime-from types import SimpleNamespace+import hashlibfrom unittest.mock import AsyncMock, MagicMockimport prisma+import prisma.actionsimport prisma.enumsimport prisma.modelsimport pytest-from backend.copilot.tools.skills import SkillPackageError+from backend.copilot.tools import skills as skills_module+from backend.copilot.tools.skills import ParsedSkill, SkillPackageError, StoredSkillfrom backend.copilot.tools.skills_test import _FakeWorkspaceManager, _patch_skills_pathfrom backend.util.exceptions import NotFoundErrorfrom backend.util.test import SpinTestServer@@ -196,19 +198,16 @@ async def test_install_stores_under_the_listing_slug_and_counts(mocker, expert_ilisting = await _make_listing(f"install-one-{expert_id or 'library'}", body="# do this\n")- stored = mocker.patch.object(skill_db, "store_user_skill")- listed = mocker.patch.object(skill_db, "list_user_skills", return_value=[])+ stored = _patch_store(mocker)result = await skill_db.install_marketplace_skill("user-1", listing.slug, expert_id=expert_id)- # The count checks the folder being written: the expert's or the library.- listed.assert_awaited_once_with("user-1", expert_id, heal_missing=False)stored.assert_awaited_once()- assert stored.await_args.kwargs["name"] == listing.slug+ assert _written(stored).name == listing.slugassert stored.await_args.kwargs["expert_id"] == expert_id- assert stored.await_args.kwargs["body"] == "# do this\n"+ assert _written(stored).body == "# do this\n"# An install fills the platform's budget on that owner, not the owner's.assert stored.await_args.kwargs["origin"] == "marketplace"assert result.name == listing.slug@@ -234,7 +233,7 @@ async def test_install_writes_into_the_target_owners_folder_and_no_other(listing = await _make_listing(f"folder-one-{expert_id or 'library'}")workspace = _FakeWorkspaceManager()experts = MagicMock()- experts.add_expert_skill_name = AsyncMock()+ experts.add_expert_skill_names = AsyncMock()mocker.patch("backend.copilot.tools.skills.experts_db", return_value=experts)with _patch_skills_path(workspace):@@ -254,23 +253,146 @@ async def test_install_writes_into_the_target_owners_folder_and_no_other(assert refreshed is not Noneassert refreshed.installCount == 1if expert_id is None:- experts.add_expert_skill_name.assert_not_awaited()+ experts.add_expert_skill_names.assert_not_awaited()else:- experts.add_expert_skill_name.assert_awaited_with(- "user-1", expert_id, listing.slug+ experts.add_expert_skill_names.assert_awaited_with(+ "user-1", expert_id, [listing.slug])+async def test_a_batch_install_looks_up_listings_and_the_owners_skills_once(mocker):+ listings = [await _make_listing(f"batch-one-{i}") for i in range(3)]+ workspace = _FakeWorkspaceManager()+ experts = MagicMock()+ experts.add_expert_skill_names = AsyncMock()+ mocker.patch("backend.copilot.tools.skills.experts_db", return_value=experts)+ listed = mocker.spy(skills_module, "_list_user_skills_from_workspace")+ queried = mocker.spy(prisma.actions.SkillListingActions, "find_many")++ with _patch_skills_path(workspace):+ locked = mocker.spy(skills_module, "AsyncClusterLock")+ outcomes = await skill_db.install_marketplace_skills(+ "user-1",+ [listing.slug for listing in listings] + ["missing"],+ expert_id="expert-a",+ )++ assert [type(o).__name__ for o in outcomes] == ["InstalledSkill"] * 3 + [+ "NotFoundError"+ ]+ # Kills: a batch that stores (lists, locks, looks up) once per skill.+ assert (listed.call_count, locked.call_count, queried.call_count) == (1, 1, 1)+ # Kills: recording each name on the expert's row in its own write.+ experts.add_expert_skill_names.assert_awaited_once_with(+ "user-1", "expert-a", [listing.slug for listing in listings]+ )+ assert sorted(workspace.files) == sorted(+ f"/experts/expert-a/skills/{listing.slug}/SKILL.md" for listing in listings+ )+ counts = await prisma.models.SkillListing.prisma().find_many(+ where={"id": {"in": [listing.id for listing in listings]}}+ )+ assert [c.installCount for c in counts] == [1, 1, 1]+++async def test_a_failed_name_write_fails_only_the_skills_it_could_not_record(mocker):+ listings = [await _make_listing(f"names-one-{i}") for i in range(3)]+ lost = listings[1].slug+ workspace = _FakeWorkspaceManager()+ experts = MagicMock()+ experts.add_expert_skill_names = AsyncMock(side_effect=RuntimeError("row busy"))++ async def add_one(user_id, expert_id, name):+ if name == lost:+ raise RuntimeError("row busy")++ experts.add_expert_skill_name = AsyncMock(side_effect=add_one)+ mocker.patch("backend.copilot.tools.skills.experts_db", return_value=experts)++ with _patch_skills_path(workspace):+ outcomes = await skill_db.install_marketplace_skills(+ "user-1", [listing.slug for listing in listings], expert_id="expert-a"+ )++ # Kills: failing every skill in the batch when the one-write record fails.+ assert [type(o).__name__ for o in outcomes] == [+ "InstalledSkill",+ "RuntimeError",+ "InstalledSkill",+ ]+ assert [c.args[2] for c in experts.add_expert_skill_name.await_args_list] == [+ listing.slug for listing in listings+ ]+++async def test_an_install_skips_the_scan_only_for_bytes_an_earlier_install_scanned(+ mocker,+):+ listing = await _make_listing("scan-once-one", body="# v1\n")+ assert listing.activeVersionId is not None+ workspace = _FakeWorkspaceManager()+ experts = MagicMock()+ experts.add_expert_skill_names = AsyncMock()+ mocker.patch("backend.copilot.tools.skills.experts_db", return_value=experts)++ def install_and_read(expert_id: str) -> tuple[str, frozenset[str]]:+ path = f"/experts/{expert_id}/skills/{listing.slug}/SKILL.md"+ written = hashlib.sha256(workspace.files[path]).hexdigest()+ return written, workspace.scanned[path]++ with _patch_skills_path(workspace):+ await skill_db.install_marketplace_skill("u", listing.slug, expert_id="a")+ first, first_skip = install_and_read("a")+ await skill_db.install_marketplace_skill("u", listing.slug, expert_id="b")+ second, second_skip = install_and_read("b")+ await prisma.models.SkillListingVersion.prisma().update(+ where={"id": listing.activeVersionId}, data={"body": "# v2\n"}+ )+ await skill_db.install_marketplace_skill("u", listing.slug, expert_id="c")+ changed, changed_skip = install_and_read("c")++ version = await prisma.models.SkillListingVersion.prisma().find_unique(+ where={"id": listing.activeVersionId}+ )+ assert version is not None+ # Kills: recording nothing, or recording anything but the bytes written.+ assert first not in first_skip+ assert second == first and second in second_skip+ assert changed != first and changed not in changed_skip+ assert set(version.scannedSha256) == {first, changed}+++async def test_a_failed_scan_record_does_not_fail_the_install(mocker):+ listing = await _make_listing("scan-once-record-fails")+ workspace = _FakeWorkspaceManager()+ experts = MagicMock()+ experts.add_expert_skill_names = AsyncMock()+ mocker.patch("backend.copilot.tools.skills.experts_db", return_value=experts)+ mocker.patch.object(+ prisma.actions.SkillListingVersionActions,+ "update",+ AsyncMock(side_effect=RuntimeError("db down")),+ )++ with _patch_skills_path(workspace):+ result = await skill_db.install_marketplace_skill(+ "u", listing.slug, expert_id="a"+ )++ # Kills: letting the scan cache's write fail the install it follows.+ assert result.name == listing.slug+ assert f"/experts/a/skills/{listing.slug}/SKILL.md" in workspace.files++async def test_install_of_a_listing_with_no_files_passes_an_empty_package(mocker):"""`files=None` means "leave the folder alone", so a single-file listinginstalled over a package would leave the old package's files in place."""listing = await _make_listing("no-files-one")- stored = mocker.patch.object(skill_db, "store_user_skill")- mocker.patch.object(skill_db, "list_user_skills", return_value=[])+ stored = _patch_store(mocker)await skill_db.install_marketplace_skill("user-1", listing.slug)- assert stored.await_args.kwargs["files"] == []+ assert _written(stored).files == []async def test_install_carries_the_versions_files_and_their_executable_bits(mocker):@@ -295,12 +417,11 @@ async def test_install_carries_the_versions_files_and_their_executable_bits(mock},])- stored = mocker.patch.object(skill_db, "store_user_skill")- mocker.patch.object(skill_db, "list_user_skills", return_value=[])+ stored = _patch_store(mocker)await skill_db.install_marketplace_skill("user-1", listing.slug)- files = stored.await_args.kwargs["files"]+ files = _written(stored).filesassert [(f.relative_path, f.content, f.is_executable) for f in files] == [("references/a.md", b"ref", False),("scripts/run.py", b"print", True),@@ -327,12 +448,7 @@ async def test_detail_lists_the_package_files_without_their_bytes(mocker):async def test_reinstalling_stores_again_but_does_not_count_again(mocker):listing = await _make_listing("install-twice", body="# do this\n")- stored = mocker.patch.object(skill_db, "store_user_skill")- mocker.patch.object(- skill_db,- "list_user_skills",- return_value=[SimpleNamespace(name="install-twice")],- )+ stored = _patch_store(mocker, is_new=False)await skill_db.install_marketplace_skill("user-1", "install-twice")@@ -351,7 +467,7 @@ async def test_install_of_an_off_shelf_listing_never_reaches_the_library(await _make_listing("off-shelf-one", approved=False, has_approved_version=has_approved_version)- stored = mocker.patch.object(skill_db, "store_user_skill")+ stored = _patch_store(mocker)with pytest.raises(NotFoundError):await skill_db.install_marketplace_skill("user-1", "off-shelf-one")@@ -578,23 +694,22 @@ async def test_seeded_skills_carry_their_attribution(tmp_path):async def test_seeded_skills_install_with_their_package_files(mocker, tmp_path):await skill_seed.seed_catalog_skills(_write_catalog(tmp_path))- stored = mocker.patch.object(skill_db, "store_user_skill")- mocker.patch.object(skill_db, "list_user_skills", return_value=[])+ stored = _patch_store(mocker)result = await skill_db.install_marketplace_skill("user-1", "cold-email")assert result.name == "cold-email"- assert stored.await_args.kwargs["name"] == "cold-email"- assert stored.await_args.kwargs["extra"] == {+ assert _written(stored).name == "cold-email"+ assert _written(stored).extra == {"license": "MIT","source": "acme/marketing-skills","source_url": ("https://github.com/acme/marketing-skills/tree/abc/skills/cold-email"),}- assert [- (f.relative_path, f.content) for f in stored.await_args.kwargs["files"]- ] == [("references/frameworks.md", b"# Frameworks\n")]+ assert [(f.relative_path, f.content) for f in _written(stored).files] == [+ ("references/frameworks.md", b"# Frameworks\n")+ ]async def test_a_bad_catalog_entry_writes_nothing(tmp_path):@@ -605,3 +720,25 @@ async def test_a_bad_catalog_entry_writes_nothing(tmp_path):await skill_seed.seed_catalog_skills(catalog)assert await prisma.models.SkillListing.prisma().count() == 0+++def _patch_store(mocker, *, is_new: bool = True) -> AsyncMock:+ """Stand in for the locked write, reporting every skill stored."""++ async def store(user_id, writes, **kwargs):+ return [+ StoredSkill(+ ParsedSkill(name=w.name, description=w.description, body=w.body),+ is_new,+ )+ for w in writes+ ]++ return mocker.patch.object(+ skill_db, "store_user_skills", AsyncMock(side_effect=store)+ )+++def _written(stored: AsyncMock):+ [write] = stored.await_args.args[1]+ return writediff --git a/autogpt_platform/backend/backend/api/features/workspace/embeddings.py b/autogpt_platform/backend/backend/api/features/workspace/embeddings.pyindex da43cef46a..a73fd4cf5e 100644--- a/autogpt_platform/backend/backend/api/features/workspace/embeddings.py+++ b/autogpt_platform/backend/backend/api/features/workspace/embeddings.py@@ -13,14 +13,19 @@ from __future__ import annotationsimport asyncioimport logging+import threading+import time+from collections import OrderedDictfrom prisma.enums import ContentTypefrom backend.api.features.search.content_handlers import build_workspace_file_textfrom backend.api.features.search.embeddings import (+ EMBEDDING_MODEL,delete_content_embedding,- ensure_content_embedding,+ generate_embedding,get_content_embedding,+ store_content_embedding,)logger = logging.getLogger(__name__)@@ -28,6 +33,19 @@ logger = logging.getLogger(__name__)# See ``library/embeddings.py`` for why we hold a strong ref to the task._background_tasks: set[asyncio.Task[None]] = set()+_EMBED_TTL_SECONDS = 86_400+_EMBED_CACHE_SIZE = 256+# Finished vectors, oldest first. Keyed by the text, which is all that is+# embedded: every installed skill is "SKILL.md SKILL", so a hire of 8 skills+# asked for one vector 8 times.+_embedded: OrderedDict[tuple[str, str], tuple[float, list[float]]] = OrderedDict()+_embedded_lock = threading.Lock()+# Requests in flight, per event loop: callers wanting the same text share one+# request, and a request for other text never waits behind it.+_in_flight: dict[+ tuple[asyncio.AbstractEventLoop, str, str], asyncio.Future[list[float]]+] = {}+async def _run_embedding(file_id: str, user_id: str, name: str, path: str) -> None:try:@@ -43,13 +61,13 @@ async def _run_embedding(file_id: str, user_id: str, name: str, path: str) -> No)if existing and existing.get("searchableText") == searchable_text:return- await ensure_content_embedding(+ await store_content_embedding(content_type=ContentType.WORKSPACE_FILE,content_id=file_id,+ embedding=await _embed(EMBEDDING_MODEL, searchable_text),searchable_text=searchable_text,metadata={"name": name, "path": path},user_id=user_id,- force=True,)except Exception as e:logger.warning(@@ -57,6 +75,33 @@ async def _run_embedding(file_id: str, user_id: str, name: str, path: str) -> No)+async def _embed(model: str, text: str) -> list[float]:+ """*text*'s vector, requested at most once per process a day."""+ with _embedded_lock:+ hit = _embedded.get((model, text))+ if hit is not None and time.monotonic() - hit[0] < _EMBED_TTL_SECONDS:+ return hit[1]+ key = (asyncio.get_running_loop(), model, text)+ request = _in_flight.get(key)+ if request is None:+ request = asyncio.ensure_future(_request_embedding(model, text))+ _in_flight[key] = request+ request.add_done_callback(+ lambda done: _in_flight.pop(key) if _in_flight.get(key) is done else None+ )+ return await asyncio.shield(request)+++async def _request_embedding(model: str, text: str) -> list[float]:+ vector = await generate_embedding(text)+ with _embedded_lock:+ _embedded[(model, text)] = (time.monotonic(), vector)+ _embedded.move_to_end((model, text))+ while len(_embedded) > _EMBED_CACHE_SIZE:+ _embedded.popitem(last=False)+ return vector++def schedule_workspace_file_embedding(file_id: str, user_id: str, name: str, path: str) -> asyncio.Task[None]:diff --git a/autogpt_platform/backend/backend/api/features/workspace/embeddings_test.py b/autogpt_platform/backend/backend/api/features/workspace/embeddings_test.pynew file mode 100644index 0000000000..77f4adefb6--- /dev/null+++ b/autogpt_platform/backend/backend/api/features/workspace/embeddings_test.py@@ -0,0 +1,79 @@+import asyncio+from unittest.mock import AsyncMock++import pytest++from backend.api.features.workspace import embeddings+++def embedding_calls(mocker):+ embeddings._embedded.clear()+ mocker.patch.object(+ embeddings, "get_content_embedding", AsyncMock(return_value=None)+ )+ store = mocker.patch.object(+ embeddings, "store_content_embedding", AsyncMock(return_value=True)+ )+ generate = mocker.patch.object(+ embeddings, "generate_embedding", AsyncMock(return_value=[0.5, 0.25])+ )+ yield generate, store+ embeddings._embedded.clear()+++async def test_files_with_the_same_text_share_one_embedding_call(embedding_calls):+ generate, store = embedding_calls++ # Three hires' copies of one skill, then a file with other text.+ for i in range(3):+ await embeddings._run_embedding(+ f"file-{i}", f"user-{i}", "SKILL.md", f"/experts/e{i}/skills/s/SKILL.md"+ )+ await embeddings._run_embedding("file-9", "user-9", "notes.md", "/notes.md")++ # Kills: embedding each file through its own OpenAI call.+ assert generate.await_count == 2+ assert [c.kwargs["content_id"] for c in store.await_args_list] == [+ "file-0",+ "file-1",+ "file-2",+ "file-9",+ ]+ assert all(c.kwargs["embedding"] == [0.5, 0.25] for c in store.await_args_list)+ assert store.await_args_list[0].kwargs["searchable_text"] == "SKILL.md SKILL"+++async def test_one_text_is_requested_once_and_other_text_does_not_wait(+ embedding_calls,+):+ generate, _ = embedding_calls+ started, release = asyncio.Event(), asyncio.Event()++ async def respond(text: str) -> list[float]:+ if text == "slow":+ started.set()+ await release.wait()+ return [float(len(text))]++ generate.side_effect = respond+ first = asyncio.create_task(embeddings._embed("m", "slow"))+ second = asyncio.create_task(embeddings._embed("m", "slow"))+ await started.wait()++ # Kills: one lock across every text, which parks this behind "slow".+ assert await asyncio.wait_for(embeddings._embed("m", "fast"), timeout=2) == [4.0]+ release.set()+ assert await first == await second == [4.0]+ # Kills: a second request for text that is already in flight.+ assert sorted(c.args[0] for c in generate.await_args_list) == ["fast", "slow"]+++async def test_a_failed_request_is_retried_rather_than_cached(embedding_calls):+ generate, _ = embedding_calls+ generate.side_effect = [RuntimeError("rate limited"), [1.0]]++ with pytest.raises(RuntimeError):+ await embeddings._embed("m", "text")+ # Kills: remembering the failure for a day.+ assert await embeddings._embed("m", "text") == [1.0]diff --git a/autogpt_platform/backend/backend/copilot/tools/skills.py b/autogpt_platform/backend/backend/copilot/tools/skills.pyindex bb4037c49a..0425ccea6a 100644--- a/autogpt_platform/backend/backend/copilot/tools/skills.py+++ b/autogpt_platform/backend/backend/copilot/tools/skills.py@@ -31,7 +31,7 @@ import uuidfrom collections.abc import Iterable, Mappingfrom dataclasses import dataclass, fieldfrom pathlib import Path-from typing import Any, NamedTuple+from typing import Any, NamedTuple, castimport yamlfrom pydantic import BaseModel@@ -59,6 +59,7 @@ from backend.executor.cluster_lock import AsyncClusterLockfrom backend.util.exceptions import ConflictErrorfrom backend.util.feature_flag import Flag, is_feature_enabledfrom backend.util.workspace import WorkspaceManager+from backend.util.workspace_storage import compute_file_checksumfrom .base import BaseToolfrom .models import ErrorResponse, ResponseType, ToolResponseBase@@ -711,41 +712,72 @@ async def store_user_skill(model's own ``store_skill`` from wiping a package it only rewrote thebody of."""- name = name.strip().lower()- # Strip any server-injected XML tags (``<available_skills>``,- # ``<env_context>``, etc.) from the persisted fields *before* storage —- # when the skill is later loaded that text lands in conversation history- # and could otherwise appear alongside the real server-injected versions.- description = strip_server_injected_tags(description.strip())- body = strip_server_injected_tags(body.strip())- triggers = [- strip_server_injected_tags(t.strip())- for t in (triggers or [])- if str(t).strip()- ]- triggers = [t for t in triggers if t]-- name_err = _validate_name(name)- if name_err:- raise ValueError(name_err)- if origin not in _SKILL_ORIGINS:- raise ValueError(f"origin must be one of {', '.join(sorted(_SKILL_ORIGINS))}")- validate_skill_content(description, body, triggers)-- parsed = ParsedSkill(- name=name,- description=description,- body=body,- triggers=tuple(triggers),- version=version,- extra=dict(extra or {}),+ [outcome] = await store_user_skills(+ user_id,+ [+ SkillWrite(+ name=name,+ description=description,+ body=body,+ triggers=triggers,+ version=version,+ extra=extra,+ files=files,+ )+ ],+ expert_id=expert_id,+ scope=scope,origin=origin,)- rendered = render_skill_markdown(parsed)- if files is not None:- # Whole-package validation before the first write, so a package that- # breaks a cap leaves the stored skill exactly as it was.- validate_package(SkillPackage(skill_md=rendered, files=files))+ if isinstance(outcome, Exception):+ raise outcome+ return outcome.skill+++class SkillWrite(NamedTuple):+ """One skill for :func:`store_user_skills`; fields as ``store_user_skill``."""++ name: str+ description: str+ body: str+ triggers: list[str] | None = None+ version: str | None = None+ extra: Mapping[str, Any] | None = None+ files: list[SkillFile] | None = None+ # Server-recorded SHA-256s of bytes already scanned clean (never a+ # client's); a file hashing to one skips the virus scan.+ scanned_checksums: frozenset[str] = frozenset()+++class StoredSkill(NamedTuple):+ skill: ParsedSkill+ # False when the write replaced a copy the owner already had.+ is_new: bool+ # SHA-256 of every file written, each of which passed the scan or matched.+ checksums: frozenset[str] = frozenset()+++async def store_user_skills(+ user_id: str,+ writes: list[SkillWrite],+ *,+ expert_id: str | None = None,+ scope: WorkspaceScope | None = None,+ origin: str = SKILL_ORIGIN_USER,+) -> list[StoredSkill | Exception]:+ """:func:`store_user_skill` for several skills under one write lock and+ one listing of the owner's folder; returns each write's outcome in order,+ so one bad skill fails alone."""+ outcomes: list[StoredSkill | Exception | None] = []+ prepared: list[tuple[int, _PreparedSkill]] = []+ for write in writes:+ try:+ prepared.append((len(outcomes), _prepare_skill(write, origin)))+ outcomes.append(None)+ except Exception as e:+ outcomes.append(e)+ if not prepared:+ return cast(list[StoredSkill | Exception], outcomes)# Coordinate ordinary package writes; the database owns the hard cap.lock: AsyncClusterLock | None = None@@ -773,76 +805,35 @@ async def store_user_skill(manager = await _get_user_skill_manager(user_id, scope)existing = await _list_user_skills_from_workspace(user_id, expert_id, scope)same_origin = {s.name for s in existing if budget_origin(s) == origin}- if origin == SKILL_ORIGIN_MARKETPLACE and any(- s.name == name and s.origin == SKILL_ORIGIN_USER for s in existing- ):- raise SkillOwnedError(- f"'{name}' is one of the owner's own skills; rename or delete "- "it before installing a skill by that name."- )- at_cap = len(same_origin) >= MAX_SKILLS_PER_EXPERT- is_new = name not in same_origin- if at_cap and is_new:- raise SkillLimitError(- f"Skill limit reached ({MAX_SKILLS_PER_EXPERT} {_ORIGIN_LABELS[origin]} "- "skills). Delete an unused skill first."- )-- metadata: dict[str, Any] = {- _META_KIND: _META_KIND_VALUE,- _META_DESCRIPTION: description,- _META_TRIGGERS: list(triggers),- _META_SKILL_ORIGIN: origin,- }- if version:- metadata[_META_VERSION] = version- folder = skill_folder(expert_id)- stale = (- await _list_package_files(manager, folder, name, cap=None)- if files is not None- else []- )- # The root is what indexes the skill, so it goes last: a new skill- # that fails part-way is never indexed. An upsert cannot be made- # atomic here — the old bytes are gone once overwritten. Serial- # because ``write_file``'s quota check is read-then-write.- existing_paths = {f.path for f in stale}- written: set[str] = set()- try:- for entry in files or []:- path = f"{folder}/{name}/{entry.relative_path}"- written.add(path)- await manager.write_file(- content=entry.content,- filename=entry.relative_path.rsplit("/", 1)[-1],- path=path,- mime_type=None,- overwrite=True,- metadata=(- {_META_EXECUTABLE: True} if entry.is_executable else None- ),+ owners = {s.name for s in existing if s.origin == SKILL_ORIGIN_USER}+ stored: list[tuple[int, str]] = []+ for index, skill in prepared:+ # A batch outlasts one lease, so renew it before each skill. Once+ # it is lost another writer may hold the key: finish best-effort,+ # as when it could not be acquired, and never release it.+ if lock is not None and lock_held and not await lock.refresh():+ lock_held = False+ logger.warning(+ "[skills] lost the write lock for user %s mid-batch — "+ "continuing as an unlocked best-effort write",+ user_id,)- await manager.write_file(- content=rendered.encode("utf-8"),- filename="SKILL.md",- path=_skill_md_path(name, expert_id),- mime_type="text/markdown",- overwrite=True,- metadata=metadata,- )- except Exception:- # Not a rollback: a file already here keeps the new bytes, so an- # upsert can fail mixed. Undo only what this call created — deleting- # the rest would turn a failed write into a lost file.- await _delete_paths(manager, written - existing_paths)- raise- await _delete_paths(- manager, {f.path for f in stale if f.path not in written}, stale- )- await invalidate_skills_index_cache(user_id, expert_id)- if expert_id is not None:- await experts_db().add_expert_skill_name(user_id, expert_id, name)- return parsed+ try:+ is_new = skill.parsed.name not in same_origin+ checksums = await _write_skill(+ manager, skill, expert_id, origin, same_origin, owners+ )+ except Exception as e:+ outcomes[index] = e+ continue+ same_origin.add(skill.parsed.name)+ stored.append((index, skill.parsed.name))+ outcomes[index] = StoredSkill(skill.parsed, is_new, checksums)+ if stored:+ await invalidate_skills_index_cache(user_id, expert_id)+ if expert_id is not None and stored:+ await _record_skill_names(user_id, expert_id, stored, outcomes)+ return cast(list[StoredSkill | Exception], outcomes)finally:if lock is not None and lock_held:try:@@ -855,6 +846,168 @@ async def store_user_skill()+async def _record_skill_names(+ user_id: str,+ expert_id: str,+ stored: list[tuple[int, str]],+ outcomes: list[StoredSkill | Exception | None],+) -> None:+ """Record the stored skills' names on the expert's row in one write.++ When a batch's write fails, retry name by name, so a skill is reported as+ failed only when its own name cannot be recorded. Its files are written+ either way, and a re-install records the name again.+ """+ try:+ await experts_db().add_expert_skill_names(+ user_id, expert_id, [name for _, name in stored]+ )+ return+ except Exception as e:+ if len(stored) == 1:+ outcomes[stored[0][0]] = e+ return+ logger.warning(+ "[skills] recording %d skill names on expert %s failed (%s); "+ "retrying one at a time",+ len(stored),+ expert_id,+ e,+ )+ for index, name in stored:+ try:+ await experts_db().add_expert_skill_name(user_id, expert_id, name)+ except Exception as e:+ outcomes[index] = e+++class _PreparedSkill(NamedTuple):+ parsed: ParsedSkill+ rendered: str+ files: list[SkillFile] | None+ scanned_checksums: frozenset[str]+++def _prepare_skill(write: SkillWrite, origin: str) -> _PreparedSkill:+ """Normalise and validate one write before anything is locked or stored."""+ name = write.name.strip().lower()+ # Strip any server-injected XML tags (``<available_skills>``,+ # ``<env_context>``, etc.) from the persisted fields *before* storage —+ # when the skill is later loaded that text lands in conversation history+ # and could otherwise appear alongside the real server-injected versions.+ description = strip_server_injected_tags(write.description.strip())+ body = strip_server_injected_tags(write.body.strip())+ triggers = [+ strip_server_injected_tags(t.strip())+ for t in (write.triggers or [])+ if str(t).strip()+ ]+ triggers = [t for t in triggers if t]++ name_err = _validate_name(name)+ if name_err:+ raise ValueError(name_err)+ if origin not in _SKILL_ORIGINS:+ raise ValueError(f"origin must be one of {', '.join(sorted(_SKILL_ORIGINS))}")+ validate_skill_content(description, body, triggers)++ parsed = ParsedSkill(+ name=name,+ description=description,+ body=body,+ triggers=tuple(triggers),+ version=write.version,+ extra=dict(write.extra or {}),+ origin=origin,+ )+ rendered = render_skill_markdown(parsed)+ if write.files is not None:+ # Whole-package validation before the first write, so a package that+ # breaks a cap leaves the stored skill exactly as it was.+ validate_package(SkillPackage(skill_md=rendered, files=write.files))+ return _PreparedSkill(parsed, rendered, write.files, write.scanned_checksums)+++async def _write_skill(+ manager: WorkspaceManager,+ skill: _PreparedSkill,+ expert_id: str | None,+ origin: str,+ same_origin: set[str],+ owners: set[str],+) -> frozenset[str]:+ """Write one prepared package; return the SHA-256 of every file written."""+ name = skill.parsed.name+ if origin == SKILL_ORIGIN_MARKETPLACE and name in owners:+ raise SkillOwnedError(+ f"'{name}' is one of the owner's own skills; rename or delete "+ "it before installing a skill by that name."+ )+ at_cap = len(same_origin) >= MAX_SKILLS_PER_EXPERT+ if at_cap and name not in same_origin:+ raise SkillLimitError(+ f"Skill limit reached ({MAX_SKILLS_PER_EXPERT} {_ORIGIN_LABELS[origin]} "+ "skills). Delete an unused skill first."+ )++ metadata: dict[str, Any] = {+ _META_KIND: _META_KIND_VALUE,+ _META_DESCRIPTION: skill.parsed.description,+ _META_TRIGGERS: list(skill.parsed.triggers),+ _META_SKILL_ORIGIN: origin,+ }+ if skill.parsed.version:+ metadata[_META_VERSION] = skill.parsed.version+ folder = skill_folder(expert_id)+ stale = (+ await _list_package_files(manager, folder, name, cap=None)+ if skill.files is not None+ else []+ )+ # The root is what indexes the skill, so it goes last: a new skill+ # that fails part-way is never indexed. An upsert cannot be made+ # atomic here — the old bytes are gone once overwritten. Serial+ # because ``write_file``'s quota check is read-then-write.+ existing_paths = {f.path for f in stale}+ written: set[str] = set()+ checksums: set[str] = set()+ try:+ for entry in skill.files or []:+ path = f"{folder}/{name}/{entry.relative_path}"+ written.add(path)+ await manager.write_file(+ content=entry.content,+ filename=entry.relative_path.rsplit("/", 1)[-1],+ path=path,+ mime_type=None,+ overwrite=True,+ metadata=({_META_EXECUTABLE: True} if entry.is_executable else None),+ scanned_checksums=skill.scanned_checksums,+ )+ checksums.add(compute_file_checksum(entry.content))+ skill_md = skill.rendered.encode("utf-8")+ await manager.write_file(+ content=skill_md,+ filename="SKILL.md",+ path=_skill_md_path(name, expert_id),+ mime_type="text/markdown",+ overwrite=True,+ metadata=metadata,+ scanned_checksums=skill.scanned_checksums,+ )+ checksums.add(compute_file_checksum(skill_md))+ except Exception:+ # Not a rollback: a file already here keeps the new bytes, so an+ # upsert can fail mixed. Undo only what this call created — deleting+ # the rest would turn a failed write into a lost file.+ await _delete_paths(manager, written - existing_paths)+ raise+ await _delete_paths(+ manager, {f.path for f in stale if f.path not in written}, stale+ )+ return frozenset(checksums)++async def _delete_paths(manager: WorkspaceManager,paths: set[str],diff --git a/autogpt_platform/backend/backend/copilot/tools/skills_isolation_test.py b/autogpt_platform/backend/backend/copilot/tools/skills_isolation_test.pyindex bbe11c96a8..2a23371a50 100644--- a/autogpt_platform/backend/backend/copilot/tools/skills_isolation_test.py+++ b/autogpt_platform/backend/backend/copilot/tools/skills_isolation_test.py@@ -61,6 +61,7 @@ def world():))experts.add_expert_skill_name = AsyncMock()+ experts.add_expert_skill_names = AsyncMock()experts.remove_expert_skill_name = AsyncMock()with (_patch_skills_path(fake),@@ -225,7 +226,7 @@ async def test_a_row_conflict_reaches_the_expert_as_a_retryable_error(world, toopasses its message through and must not log it as an exception."""_, experts = worldconflict = ConflictError("Changed at the same time. Try again.")- experts.add_expert_skill_name = AsyncMock(side_effect=conflict)+ experts.add_expert_skill_names = AsyncMock(side_effect=conflict)experts.remove_expert_skill_name = AsyncMock(side_effect=conflict)with patch.object(skills.logger, "exception") as logged:@@ -359,8 +360,8 @@ async def test_expert_creates_and_deletes_skills_in_its_own_folder(world):)assert isinstance(stored, StoreSkillResponse) and stored.expert_id == "expert-a"assert "/experts/expert-a/skills/learned/SKILL.md" in fake.files- experts.add_expert_skill_name.assert_awaited_once_with(- "user-1", "expert-a", "learned"+ experts.add_expert_skill_names.assert_awaited_once_with(+ "user-1", "expert-a", ["learned"])deleted = await DeleteSkillTool()._execute(@@ -410,7 +411,9 @@ async def test_copy_gives_expert_its_own_copy_of_an_autopilot_skill(world):assert "/experts/expert-a/skills/mine/SKILL.md" in fake.filesassert fake.files["/experts/expert-a/skills/mine/references/notes.md"] == b"notes"assert AUTOPILOT in fake.files- experts.add_expert_skill_name.assert_awaited_once_with("user-1", "expert-a", "mine")+ experts.add_expert_skill_names.assert_awaited_once_with(+ "user-1", "expert-a", ["mine"]+ )assert await copy_skill_to_expert("user-1", "expert-a", "nope") is Nonediff --git a/autogpt_platform/backend/backend/copilot/tools/skills_test.py b/autogpt_platform/backend/backend/copilot/tools/skills_test.pyindex f0270567d3..948f7a8ee9 100644--- a/autogpt_platform/backend/backend/copilot/tools/skills_test.py+++ b/autogpt_platform/backend/backend/copilot/tools/skills_test.py@@ -44,6 +44,8 @@ from backend.copilot.tools.skills import (SkillOwnedError,SkillPackage,SkillPackageError,+ SkillWrite,+ StoredSkill,StoreSkillResponse,StoreSkillTool,_is_safe_relative,@@ -66,6 +68,7 @@ from backend.copilot.tools.skills import (render_skill_markdown,render_skills_index,store_user_skill,+ store_user_skills,validate_package,)from backend.util.exceptions import ConflictError@@ -252,12 +255,23 @@ class _FakeWorkspaceManager:self.files: dict[str, bytes] = {}self.metadata: dict[str, dict] = {}self.reads: list[str] = []+ # Per path, the hashes the write was told need no scan.+ self.scanned: dict[str, frozenset[str]] = {}async def write_file(- self, *, content, filename, path, mime_type, overwrite, metadata=None+ self,+ *,+ content,+ filename,+ path,+ mime_type,+ overwrite,+ metadata=None,+ scanned_checksums=(),):self.files[path] = contentself.metadata[path] = metadata or {}+ self.scanned[path] = frozenset(scanned_checksums)async def read_file(self, path: str) -> bytes:if path not in self.files:@@ -345,7 +359,9 @@ class _patch_skills_path:fake_lock = MagicMock()fake_lock.owner_id = "test-owner"fake_lock.try_acquire = AsyncMock(return_value="test-owner")+ fake_lock.refresh = AsyncMock(return_value=True)fake_lock.release = AsyncMock()+ self.lock = fake_lockself.workdir = tempfile.mkdtemp(prefix="copilot-skills-test-")self._patches = [patch(@@ -766,6 +782,30 @@ async def test_store_user_skill_rejects_an_unknown_origin():)+async def test_a_batch_renews_its_write_lock_before_each_skill():+ writes = [+ SkillWrite(name=f"batch-{i}", description="d", body="b") for i in range(3)+ ]+ with _patch_skills_path(_FakeWorkspaceManager()) as patched:+ outcomes = await store_user_skills("user-1", writes)+ assert all(isinstance(o, StoredSkill) for o in outcomes)+ # Kills: holding one 30 s lease across a whole batch.+ assert patched.lock.refresh.await_count == 3+ patched.lock.release.assert_awaited_once()+++async def test_a_batch_that_lost_its_lock_finishes_without_releasing_it():+ writes = [SkillWrite(name=f"lost-{i}", description="d", body="b") for i in range(2)]+ with _patch_skills_path(_FakeWorkspaceManager()) as patched:+ patched.lock.refresh.return_value = False+ outcomes = await store_user_skills("user-1", writes)+ assert all(isinstance(o, StoredSkill) for o in outcomes)+ # Kills: releasing a key another writer may hold by now.+ patched.lock.release.assert_not_awaited()++@pytest.mark.asyncioasync def test_copy_to_expert_keeps_the_installed_origin():"""A bundled skill healed into an expert's folder stays a bundled one@@ -774,15 +814,15 @@ async def test_copy_to_expert_keeps_the_installed_origin():_seed_skill(fake, "bundled", origin=SKILL_ORIGIN_MARKETPLACE)# Storing into an expert's folder records the name on the expert's row.experts = MagicMock()- experts.add_expert_skill_name = AsyncMock()+ experts.add_expert_skill_names = AsyncMock()with (_patch_skills_path(fake),patch("backend.copilot.tools.skills.experts_db", return_value=experts),):slug = await copy_skill_to_expert("user-1", "expert-1", "bundled")assert slug == "bundled"- experts.add_expert_skill_name.assert_awaited_once_with(- "user-1", "expert-1", "bundled"+ experts.add_expert_skill_names.assert_awaited_once_with(+ "user-1", "expert-1", ["bundled"])copied = fake.metadata["/experts/expert-1/skills/bundled/SKILL.md"]assert copied["skill_origin"] == "marketplace"diff --git a/autogpt_platform/backend/backend/data/db_manager.py b/autogpt_platform/backend/backend/data/db_manager.pyindex 82de3ffe69..4e92ccd475 100644--- a/autogpt_platform/backend/backend/data/db_manager.py+++ b/autogpt_platform/backend/backend/data/db_manager.py@@ -560,6 +560,7 @@ class DatabaseManager(AppService):update_soul_fields = _(experts_db.update_soul_fields)update_soul_fields_if_current = _(experts_db.update_soul_fields_if_current)add_expert_skill_name = _(experts_db.add_expert_skill_name)+ add_expert_skill_names = _(experts_db.add_expert_skill_names)remove_expert_skill_name = _(experts_db.remove_expert_skill_name)install_workflow = _(experts_db.install_workflow)remove_workflow = _(experts_db.remove_workflow)@@ -967,6 +968,7 @@ class DatabaseManagerAsyncClient(AppServiceClient):update_soul_fields = d.update_soul_fieldsupdate_soul_fields_if_current = d.update_soul_fields_if_currentadd_expert_skill_name = d.add_expert_skill_name+ add_expert_skill_names = d.add_expert_skill_namesremove_expert_skill_name = d.remove_expert_skill_nameinstall_workflow = d.install_workflowremove_workflow = d.remove_workflowdiff --git a/autogpt_platform/backend/backend/util/workspace.py b/autogpt_platform/backend/backend/util/workspace.pyindex 1738c8da66..80bffaacf0 100644--- a/autogpt_platform/backend/backend/util/workspace.py+++ b/autogpt_platform/backend/backend/util/workspace.py@@ -9,6 +9,7 @@ import asyncioimport loggingimport mimetypesimport uuid+from collections.abc import Collectionfrom typing import TYPE_CHECKING, Optionalfrom prisma.errors import UniqueViolationError@@ -246,6 +247,7 @@ class WorkspaceManager:mime_type: Optional[str] = None,overwrite: bool = False,metadata: Optional[dict] = None,+ scanned_checksums: Collection[str] = (),) -> WorkspaceFile:"""Write file to workspace.@@ -260,6 +262,9 @@ class WorkspaceManager:mime_type: MIME type (auto-detected if not provided)overwrite: Whether to overwrite existing file at pathmetadata: Optional metadata dict (e.g., origin tracking)+ scanned_checksums: SHA-256s of bytes the server has already+ scanned clean; content hashing to one of them is not scanned+ again. Never pass hashes a client supplied.Returns:Created WorkspaceFile instance@@ -327,16 +332,15 @@ class WorkspaceManager:# Scan here — callers must NOT duplicate this scan.# WorkspaceManager owns virus scanning for all persisted files.- await scan_content_safe(content, filename=filename)+ checksum = compute_file_checksum(content)+ if checksum not in scanned_checksums:+ await scan_content_safe(content, filename=filename)# Auto-detect MIME type if not providedif mime_type is None:mime_type, _ = mimetypes.guess_type(filename)mime_type = mime_type or "application/octet-stream"- # Compute checksum- checksum = compute_file_checksum(content)-# Generate unique file ID for storagefile_id = str(uuid.uuid4())diff --git a/autogpt_platform/backend/backend/util/workspace_test.py b/autogpt_platform/backend/backend/util/workspace_test.pyindex 0e8124a459..b3cc31929c 100644--- a/autogpt_platform/backend/backend/util/workspace_test.py+++ b/autogpt_platform/backend/backend/util/workspace_test.py@@ -2,6 +2,7 @@Tests for WorkspaceManager.write_file UniqueViolationError handling."""+import hashlibfrom datetime import datetime, timezonefrom unittest.mock import AsyncMock, MagicMock, patch@@ -263,6 +264,36 @@ async def test_write_file_overwrite_not_double_counted(manager, mock_storage, moassert result == created_file+ "scanned, expect_scan",+ [((), True), (("0" * 64,), True), ((hashlib.sha256(b"hello").hexdigest(),), False)],+ ids=["none-recorded", "other-bytes", "same-bytes"],+)+async def test_write_file_skips_the_scan_only_for_bytes_already_scanned(+ manager, mock_storage, mock_db, scanned, expect_scan+):+ mock_db.get_workspace_file_by_path.return_value = None+ mock_db.create_workspace_file.return_value = _make_workspace_file()+ with (+ patch(+ "backend.util.workspace.get_workspace_storage", return_value=mock_storage+ ),+ patch("backend.util.workspace.workspace_db", return_value=mock_db),+ patch(+ "backend.util.workspace.scan_content_safe", new_callable=AsyncMock+ ) as scan,+ patch(+ "backend.util.workspace.get_workspace_storage_limit_bytes",+ return_value=1_000,+ ),+ ):+ await manager.write_file(+ filename="test.txt", content=b"hello", scanned_checksums=scanned+ )+ # Kills: skipping on any recorded hash, or never skipping.+ assert scan.await_count == (1 if expect_scan else 0)++@pytest.mark.asyncioasync def test_write_file_storage_check_routes_through_workspace_db_accessor(manager, mock_storage, mock_dbdiff --git a/autogpt_platform/backend/migrations/20260923220000_add_skill_version_scanned_sha256/migration.sql b/autogpt_platform/backend/migrations/20260923220000_add_skill_version_scanned_sha256/migration.sqlnew file mode 100644index 0000000000..e6a707df71--- /dev/null+++ b/autogpt_platform/backend/migrations/20260923220000_add_skill_version_scanned_sha256/migration.sql@@ -0,0 +1,2 @@+-- AlterTable+ALTER TABLE "SkillListingVersion" ADD COLUMN "scannedSha256" TEXT[] DEFAULT ARRAY[]::TEXT[];diff --git a/autogpt_platform/backend/schema.prisma b/autogpt_platform/backend/schema.prismaindex 2665c2fb08..0ff8778010 100644--- a/autogpt_platform/backend/schema.prisma+++ b/autogpt_platform/backend/schema.prisma@@ -2289,6 +2289,10 @@ model SkillListingVersion {// provider names as used by the credentials system.requiredProviders String[] @default([])+ // SHA-256 of each exact file an install wrote after it passed the virus+ // scan; a later install writing the same bytes skips the scan.+ scannedSha256 String[] @default([])+// The creator's own library skill slug this was published from, so a// re-publish can find its listing again. Null once that skill is deleted.sourceSkillSlug String?