AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.
fix(backend): check the skill limit before saving a new learned skill (#15598)
Nicholas Tindleba81e7b8df39
diff --git a/autogpt_platform/backend/backend/copilot/learning/conftest.py b/autogpt_platform/backend/backend/copilot/learning/conftest.pyindex 71668ade73..bb7dd18519 100644--- a/autogpt_platform/backend/backend/copilot/learning/conftest.py+++ b/autogpt_platform/backend/backend/copilot/learning/conftest.py@@ -38,6 +38,10 @@ def fake_store(monkeypatch):"backend.copilot.learning.nightly.load_reviewed_packages",AsyncMock(return_value={}),)+ monkeypatch.setattr(+ "backend.copilot.learning.publish.saved_skill_limit_reached",+ AsyncMock(return_value=False),+ )modules = ("retrieval","history",diff --git a/autogpt_platform/backend/backend/copilot/learning/publication_scanner_test.py b/autogpt_platform/backend/backend/copilot/learning/publication_scanner_test.pyindex cc642c43c9..a6ffb1d7bc 100644--- a/autogpt_platform/backend/backend/copilot/learning/publication_scanner_test.py+++ b/autogpt_platform/backend/backend/copilot/learning/publication_scanner_test.py@@ -48,21 +48,21 @@ async def test_virus_rejection_abandons_the_snapshot_without_repeat_review(@pytest.mark.asyncio@pytest.mark.parametrize(- "error",+ "error, disposition",[- VirusScanError("scanner unavailable"),- StorageUploadError("upload unavailable"),- SkillLimitError("skill capacity reached"),- ValueError("storage quota exceeded"),+ (VirusScanError("scanner unavailable"), "write_failed"),+ (StorageUploadError("upload unavailable"), "write_failed"),+ (SkillLimitError("skill capacity reached"), "paused"),+ (ValueError("storage quota exceeded"), "write_failed"),],ids=["scanner", "storage", "capacity", "quota"],)async def test_recoverable_publication_failure_replays_without_model_calls(- fake_store, adapter, boundaries, error+ fake_store, adapter, boundaries, error, disposition):source = await _source(fake_store)boundaries["write"].side_effect = error- assert (await _run()).dispositions == {"write_failed": 1}+ assert (await _run()).dispositions == {disposition: 1}version = (await fake_store.list_pending_publications(USER))[0]assert (await fake_store.get_source(USER, source.id)).has_unprocessed_revisionboundaries["write"].side_effect = Nonediff --git a/autogpt_platform/backend/backend/copilot/learning/publish.py b/autogpt_platform/backend/backend/copilot/learning/publish.pyindex 8570468e80..81e848323a 100644--- a/autogpt_platform/backend/backend/copilot/learning/publish.py+++ b/autogpt_platform/backend/backend/copilot/learning/publish.py@@ -34,6 +34,7 @@ from backend.copilot.tools.skills import (read_user_skill_files,read_user_skill_markdown,render_skill_markdown,+ saved_skill_limit_reached,store_user_skill,validate_package,)@@ -42,6 +43,7 @@ from backend.data.db_accessors import (skill_use_db,skill_versions_db,)+from backend.data.skill_capacity import MAX_SKILLS_PER_EXPERT, SkillLimitErrorfrom backend.data.skill_content_approvals import SkillContentApprovalfrom backend.data.skill_learning import owner_key_forfrom backend.data.skill_publication import ReviewStamp, VersionDraft@@ -80,6 +82,11 @@ from .publication_validation import (logger = logging.getLogger(__name__)+SKILL_LIMIT_REASON = (+ f"the skill limit is reached ({MAX_SKILLS_PER_EXPERT} saved skills); "+ "delete a skill you no longer use to make room"+)+# ---------------------------------------------------------------------------# Automated publication (nightly / requested)# ---------------------------------------------------------------------------@@ -196,6 +203,14 @@ async def publish_learned_version(request: PublishRequest) -> PublishOutcome:blocked = await _check_bundle(request, rendered, files, approvals)if blocked is not None:return blocked+ if await saved_skill_limit_reached(+ request.user_id, request.expert_id, request.skill_name+ ):+ # Checked before anything is written, so a skill that would need a+ # saved slot the owner no longer has (a new one, or an installed copy+ # an edit moves into the saved budget) leaves nothing behind. Nightly+ # keeps the paid review for its next retry.+ return PublishOutcome(status="paused", reason=SKILL_LIMIT_REASON)versions = skill_versions_db()head = await versions.ensure_head(@@ -589,6 +604,10 @@ async def write_committed_version(user_id, version_id=version.id, review_id=review_id, reason=reason)return PublishOutcome(status="conflict", reason=reason)+ except SkillLimitError:+ # The limit was reached after the precheck; the version stays pending+ # and is written once the owner frees a slot.+ return PublishOutcome(status="paused", reason=SKILL_LIMIT_REASON)except Exception as exc:logger.warning("Skill workspace write failed for %s v%s (user %s); left pending",diff --git a/autogpt_platform/backend/backend/copilot/learning/skill_limit_test.py b/autogpt_platform/backend/backend/copilot/learning/skill_limit_test.pynew file mode 100644index 0000000000..a72db26c45--- /dev/null+++ b/autogpt_platform/backend/backend/copilot/learning/skill_limit_test.py@@ -0,0 +1,113 @@+"""Learning at the owner's saved-skill limit."""++from unittest.mock import AsyncMock++import pytest++from backend.copilot.learning import nightly, nightly_test, publish+from backend.copilot.learning.nightly_test import EXPERT, USER, _run, _source+from backend.copilot.tools.skills import parse_skill_markdown++adapter = nightly_test.adapter+boundaries = nightly_test.boundaries++NAME = "csv-import-checks"+++def at_limit(monkeypatch):+ limit = AsyncMock(return_value=True)+ monkeypatch.setattr(publish, "saved_skill_limit_reached", limit)+ return limit+++async def test_a_new_skill_waits_for_room_and_is_published_from_its_kept_review(+ fake_store, adapter, boundaries, at_limit+):+ """A new skill at the limit leaves no head or pending write that could+ publish unprompted later; its paid review is kept and reused."""+ source = await _source(fake_store)++ waiting = await _run()++ assert waiting.dispositions == {"paused": 1}+ at_limit.assert_awaited_with(USER, EXPERT, NAME)+ assert await fake_store.get_head(USER, EXPERT, NAME) is None+ assert await fake_store.list_pending_publications(USER) == []+ boundaries["write"].assert_not_awaited()+ ledger = await fake_store.get_review_for_revision(+ USER, source.id, source.revision, nightly.POLICY_VERSION+ )+ assert ledger.replay is not None+ assert ledger.reason == publish.SKILL_LIMIT_REASON+ assert (await fake_store.get_source(USER, source.id)).processed_revision is None++ at_limit.return_value = False+ published = await _run()++ assert published.applied == 1 and published.model_calls == 0+ boundaries["review"].assert_awaited_once()+ head = await fake_store.get_head(USER, EXPERT, NAME)+ assert head is not None and head.current_version_id is not None+++async def _publish_an_update(fake_store, monkeypatch):+ """Publish a skill, then propose an update to it through the publisher."""+ await _source(fake_store)+ assert (await _run()).applied == 1+ head = await fake_store.get_head(USER, EXPERT, NAME)+ current = await fake_store.get_version(USER, head.current_version_id)+ saved = parse_skill_markdown(current.content)+ monkeypatch.setattr(+ publish, "read_user_skill_markdown", AsyncMock(return_value=current.content)+ )+ request = publish.PublishRequest(+ user_id=USER,+ expert_id=EXPERT,+ skill_name=NAME,+ description=saved.description,+ triggers=list(saved.triggers),+ body=saved.body + "\n4. Log the rows that were skipped.\n",+ summary="Logged the rows that were skipped.",+ origin="saved_overnight",+ )+ return head, request+++async def test_an_update_that_needs_no_new_slot_applies_at_the_limit(+ fake_store, adapter, boundaries, at_limit, monkeypatch+):+ """Every learned publication asks; a skill that already fills its slot+ gets room, so it keeps improving."""+ at_limit.return_value = False+ _, request = await _publish_an_update(fake_store, monkeypatch)+ at_limit.reset_mock()++ outcome = await publish.publish_learned_version(request)++ assert outcome.status == "applied"+ at_limit.assert_awaited_once_with(USER, EXPERT, NAME)+ assert (await fake_store.get_head(USER, EXPERT, NAME)).current_version == 2+++async def test_an_update_that_would_take_a_new_slot_waits_like_a_new_skill(+ fake_store, adapter, boundaries, at_limit, monkeypatch+):+ """An existing root can still need a saved slot, for example an+ installed copy with no install record that an edit moves into the+ owner's budget. At the limit it waits too, before any version."""+ at_limit.return_value = False+ head, request = await _publish_an_update(fake_store, monkeypatch)+ writes = boundaries["write"].await_count+ at_limit.return_value = True++ outcome = await publish.publish_learned_version(request)++ assert (outcome.status, outcome.reason) == ("paused", publish.SKILL_LIMIT_REASON)+ assert await fake_store.get_head(USER, EXPERT, NAME) == head+ assert len(await fake_store.list_versions(USER, EXPERT, NAME)) == 1+ assert await fake_store.list_pending_publications(USER) == []+ assert boundaries["write"].await_count == writesdiff --git a/autogpt_platform/backend/backend/copilot/tools/skills.py b/autogpt_platform/backend/backend/copilot/tools/skills.pyindex afe41ec274..47de6bce77 100644--- a/autogpt_platform/backend/backend/copilot/tools/skills.py+++ b/autogpt_platform/backend/backend/copilot/tools/skills.py@@ -65,6 +65,7 @@ from backend.data.db_accessors import (skill_deletion_db,skill_versions_db,workspace_db,+ workspace_skill_db,)from backend.data.redis_client import get_redis_asyncfrom backend.data.skill_capacity import MAX_SKILLS_PER_EXPERT@@ -356,6 +357,14 @@ def budget_origin(skill: ParsedSkill) -> str:return skill.origin or SKILL_ORIGIN_USER+def _keeps_install(origin: str | None, baseline: SkillBaseline | None) -> bool:+ """An installed copy that records its install stays installed when its+ owner edits it: the write keeps its marketplace origin and the baseline a+ later update merges into. :func:`_write_skill` and+ :func:`saved_skill_limit_reached` both decide by this."""+ return origin == SKILL_ORIGIN_MARKETPLACE and baseline is not None++def parse_skill_markdown(text: str, fallback_name: str = "") -> ParsedSkill | None:"""Parse a ``SKILL.md`` blob (YAML frontmatter + markdown body).@@ -1167,11 +1176,7 @@ async def store_user_skills(existing = await _list_user_skills_from_workspace(user_id, expert_id, scope)same_origin = {s.name for s in existing if budget_origin(s) == origin}owners = {s.name for s in existing if s.origin == SKILL_ORIGIN_USER}- copies = {- s.name: s- for s in existing- if s.origin == SKILL_ORIGIN_MARKETPLACE and s.baseline is not None- }+ copies = {s.name: s for s in existing if _keeps_install(s.origin, s.baseline)}stored: list[tuple[int, str]] = []for index, skill in prepared:# A batch outlasts one lease, so renew it before each skill. Once@@ -1898,6 +1903,28 @@ async def invalidate_skills_index_cache(pass+async def saved_skill_limit_reached(+ user_id: str, expert_id: str | None, name: str+) -> bool:+ """Whether saving ``name`` as the owner's skill would need a saved-skill+ slot the owner no longer has, by the same count of live roots the+ workspace enforces. A marketplace copy that records its install stays+ installed when edited (:func:`_keeps_install`) and needs no saved slot;+ any other root moves into, or already fills, the saved budget."""+ workspace = await workspace_db().get_or_create_workspace(user_id)+ path = _skill_md_path(name, expert_id)+ root = await workspace_db().get_workspace_file_by_path(workspace.id, path)+ meta = root.metadata if root is not None and root.metadata else {}+ origin = (+ SKILL_ORIGIN_MARKETPLACE+ if _keeps_install(_skill_origin(meta), _baseline_from_metadata(meta))+ else SKILL_ORIGIN_USER+ )+ return not await workspace_skill_db().skill_root_has_room(+ workspace.id, path, origin+ )++async def list_user_skills(user_id: str,expert_id: str | None = None,diff --git a/autogpt_platform/backend/backend/copilot/tools/skills_test.py b/autogpt_platform/backend/backend/copilot/tools/skills_test.pyindex 2ec2dcee90..96747e6088 100644--- a/autogpt_platform/backend/backend/copilot/tools/skills_test.py+++ b/autogpt_platform/backend/backend/copilot/tools/skills_test.py@@ -15,6 +15,7 @@ from backend.copilot.baseline.service import _prepend_skills_notice_to_current_mfrom backend.copilot.learning.fingerprint import behavior_fingerprintfrom backend.copilot.model import ChatMessage, ChatSessionfrom backend.copilot.sdk.service import _maybe_prepend_skills_update+from backend.copilot.tools import skills as skills_modulefrom backend.copilot.tools.models import ErrorResponsefrom backend.copilot.tools.skills import (_MAX_ROOTS_PER_FOLDER,@@ -70,6 +71,7 @@ from backend.copilot.tools.skills import (read_user_skill_package,render_skill_markdown,render_skills_index,+ saved_skill_limit_reached,store_user_skill,store_user_skills,validate_package,@@ -674,6 +676,55 @@ async def test_installed_skills_do_not_take_the_owners_saved_slots():assert fake.metadata["/skills/mine/SKILL.md"]["skill_origin"] == "user"+ "expert_id, path",+ [+ (None, "/skills/new-skill/SKILL.md"),+ ("expert-1", "/experts/expert-1/skills/new-skill/SKILL.md"),+ ],+)+ "root_metadata, origin",+ [+ (None, SKILL_ORIGIN_USER),+ ({"skill_origin": "user"}, SKILL_ORIGIN_USER),+ ({"skill_origin": "marketplace"}, SKILL_ORIGIN_USER),+ (+ {+ "skill_origin": "marketplace",+ "listing_id": "listing-1",+ "listing_version_id": "listing-1-v1",+ },+ SKILL_ORIGIN_MARKETPLACE,+ ),+ ],+ ids=["new", "saved", "installed-without-record", "installed-copy"],+)+async def test_the_saved_skill_limit_asks_the_workspace_about_the_root(+ monkeypatch, expert_id, path, root_metadata, origin+):+ """Learning checks the same live-root count the workspace enforces when+ the root is published, in the budget the write will use: an installed+ copy that records its install stays installed when edited."""+ root = None if root_metadata is None else MagicMock(metadata=root_metadata)+ workspaces = MagicMock(+ get_or_create_workspace=AsyncMock(return_value=MagicMock(id="workspace-1")),+ get_workspace_file_by_path=AsyncMock(return_value=root),+ )+ monkeypatch.setattr(skills_module, "workspace_db", lambda: workspaces)+ room = AsyncMock(return_value=False)+ monkeypatch.setattr(+ skills_module,+ "workspace_skill_db",+ lambda: MagicMock(skill_root_has_room=room),+ )++ assert await saved_skill_limit_reached("user-1", expert_id, "new-skill")+ workspaces.get_workspace_file_by_path.assert_awaited_once_with("workspace-1", path)+ room.assert_awaited_once_with("workspace-1", path, origin)++@pytest.mark.asyncioasync def test_saved_skills_do_not_take_the_installed_slots():"""The mirror: an owner at their own cap can still be given a bundleddiff --git a/autogpt_platform/backend/backend/data/db_manager.py b/autogpt_platform/backend/backend/data/db_manager.pyindex 0f2d81c560..e5dbf6aed1 100644--- a/autogpt_platform/backend/backend/data/db_manager.py+++ b/autogpt_platform/backend/backend/data/db_manager.py@@ -214,7 +214,10 @@ from backend.data.workspace import (soft_delete_workspace_file,)from backend.data.workspace_folder import list_workspace_folders-from backend.data.workspace_skill import publish_workspace_skill_file+from backend.data.workspace_skill import (+ publish_workspace_skill_file,+ skill_root_has_room,+)from backend.platform_linking import db as platform_linking_dbfrom backend.util.service import (AppService,@@ -512,6 +515,7 @@ class DatabaseManager(AppService):count_workspace_files = _(count_workspace_files)create_workspace_file = _(create_workspace_file)publish_workspace_skill_file = _(publish_workspace_skill_file)+ skill_root_has_room = _(skill_root_has_room)get_or_create_workspace = _(get_or_create_workspace)get_workspace_file = _(get_workspace_file)get_workspace_file_by_path = _(get_workspace_file_by_path)@@ -1023,6 +1027,7 @@ class DatabaseManagerAsyncClient(AppServiceClient):count_workspace_files = d.count_workspace_filescreate_workspace_file = d.create_workspace_filepublish_workspace_skill_file = d.publish_workspace_skill_file+ skill_root_has_room = d.skill_root_has_roomget_or_create_workspace = d.get_or_create_workspaceget_workspace_file = d.get_workspace_fileget_workspace_file_by_path = d.get_workspace_file_by_pathdiff --git a/autogpt_platform/backend/backend/data/workspace_skill.py b/autogpt_platform/backend/backend/data/workspace_skill.pyindex fe5e26aeb4..15c728e281 100644--- a/autogpt_platform/backend/backend/data/workspace_skill.py+++ b/autogpt_platform/backend/backend/data/workspace_skill.py@@ -9,6 +9,7 @@ from prisma import Prismafrom prisma.models import UserWorkspaceFilefrom pydantic import BaseModel+from backend.data import dbfrom backend.data.db import transactionfrom backend.data.skill_capacity import (MAX_SKILLS_PER_EXPERT,@@ -76,7 +77,7 @@ async def publish_workspace_skill_file(and existing_origin == SKILL_ORIGIN_USER):return WorkspaceSkillPublication(status="owned")- if (not active or (existing_origin or SKILL_ORIGIN_USER) != origin) and (+ if _takes_a_slot(existing, origin) and (await _root_count(tx, write.workspace_id, folder, origin)>= MAX_SKILLS_PER_EXPERT):@@ -94,6 +95,34 @@ async def publish_workspace_skill_file()+async def skill_root_has_room(workspace_id: str, path: str, origin: str) -> bool:+ """Whether a root at ``path`` would fit the owner's budget for ``origin``:+ the capacity decision :func:`publish_workspace_skill_file` makes, read+ without taking its lock, so a concurrent write can still win the slot."""+ folder = skill_owner_folder(path)+ if folder is None:+ raise ValueError("Skill capacity is counted for canonical SKILL.md paths")+ existing = await UserWorkspaceFile.prisma().find_first(+ where={"workspaceId": workspace_id, "path": path, "isDeleted": False}+ )+ if not _takes_a_slot(existing, origin):+ return True+ return (+ await _root_count(db.prisma, workspace_id, folder, origin)+ < MAX_SKILLS_PER_EXPERT+ )+++def _takes_a_slot(existing: UserWorkspaceFile | None, origin: str) -> bool:+ """A new root, or one changing budgets, needs a free slot in its own."""+ if existing is None:+ return True+ existing_origin = skill_origin(+ existing.metadata if isinstance(existing.metadata, dict) else None+ )+ return (existing_origin or SKILL_ORIGIN_USER) != origin++@asynccontextmanagerasync def _publication_transaction(workspace_id: str, folder: strdiff --git a/autogpt_platform/backend/backend/data/workspace_skill_test.py b/autogpt_platform/backend/backend/data/workspace_skill_test.pyindex 12eb845051..6def03f531 100644--- a/autogpt_platform/backend/backend/data/workspace_skill_test.py+++ b/autogpt_platform/backend/backend/data/workspace_skill_test.py@@ -16,6 +16,7 @@ from backend.data.workspace import WorkspaceFile, rename_workspace_filefrom backend.data.workspace_skill import (WorkspaceSkillWrite,publish_workspace_skill_file,+ skill_root_has_room,)from backend.util.workspace import WorkspaceManagerfrom backend.util.workspace_storage import LocalWorkspaceStorage@@ -401,3 +402,59 @@ async def test_rename_and_new_publication_share_last_slot(workspace_id: str):)== MAX_SKILLS_PER_EXPERT)++[email protected](loop_scope="session")+ "saved, unreadable, target_origin, origin",+ [+ (MAX_SKILLS_PER_EXPERT - 1, 0, None, "user"),+ (MAX_SKILLS_PER_EXPERT - 1, 1, None, "user"),+ (MAX_SKILLS_PER_EXPERT, 0, "user", "user"),+ (MAX_SKILLS_PER_EXPERT, 0, "marketplace", "user"),+ (MAX_SKILLS_PER_EXPERT, 0, None, "marketplace"),+ ],+ ids=[+ "one-left",+ "unreadable-root-fills-it",+ "update",+ "budget-move",+ "other-budget",+ ],+)+async def test_room_is_what_the_publication_decides(+ workspace_id: str, saved: int, unreadable: int, target_origin, origin: str+):+ """Learning asks before it saves a new skill; the answer must match the+ publication's own capacity decision, unreadable roots included."""+ await seed(workspace_id, saved - (1 if target_origin else 0), origin="user")+ for i in range(unreadable):+ await db.prisma.userworkspacefile.create(+ data={+ "workspaceId": workspace_id,+ "name": "SKILL.md",+ "path": f"/skills/unreadable-{i}/SKILL.md",+ "storagePath": f"test://unreadable-{i}",+ "mimeType": "text/markdown",+ "sizeBytes": 1,+ }+ )+ if target_origin:+ await db.prisma.userworkspacefile.create(+ data={+ "workspaceId": workspace_id,+ "name": "SKILL.md",+ "path": "/skills/target/SKILL.md",+ "storagePath": "test://target",+ "mimeType": "text/markdown",+ "sizeBytes": 1,+ "metadata": Json({"skill_origin": target_origin}),+ }+ )+ room = await skill_root_has_room(workspace_id, "/skills/target/SKILL.md", origin)+ published = await publish_workspace_skill_file(+ write(workspace_id, "target").model_copy(+ update={"metadata": {"skill_origin": origin}}+ )+ )+ assert room == (published.status == "stored")diff --git a/docs/platform/skill-learning.md b/docs/platform/skill-learning.mdindex b2c18ced03..98b957f506 100644--- a/docs/platform/skill-learning.md+++ b/docs/platform/skill-learning.md@@ -186,6 +186,15 @@ and policy. After that limit the revision is skipped; new verified evidencecan be considered in a later revision. A reservation does not prove thatthe provider completed a call.+At the saved-skill limit (150 for the account or for each Expert), learning+saves nothing that would need another saved slot: a new skill, or an edit+that would move an installed copy into the saved budget. The revision is+recorded as paused before any version exists, and its paid review is kept+for the retry above, so it is saved once there is room. If another save+takes the last slot between that check and the write, the new version waits+as a pending write and is saved, without another review, once there is+room. Improvements to skills already saved continue.+Owner-approved content-check exceptions are stored with the immutableversion as exact match hashes, file paths, and occurrence counts. Restoringor improving a version preserves only retained approved occurrences.