Skip to content

AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.

fix(backend): update pre-catalog copies and harden the marketplace copy reconcile (#14954)

Nicholas Tindlefebfb867f136
diff --git a/.github/workflows/platform-autogpt-deploy-dev.yaml b/.github/workflows/platform-autogpt-deploy-dev.yaml
index a3dabfdf13..c078e9e12d 100644
--- a/.github/workflows/platform-autogpt-deploy-dev.yaml
+++ b/.github/workflows/platform-autogpt-deploy-dev.yaml
@@ -64,6 +64,38 @@ jobs:
DATABASE_URL: ${{ secrets.BACKEND_DATABASE_URL }}
DIRECT_URL: ${{ secrets.BACKEND_DATABASE_URL }}
+ # The catalog's main is what this environment serves from now on: new and
+ # changed packages get immutable versions, the expert roster is upserted by
+ # key, and copies users hold catch up lazily on their next turn (unedited
+ # copies replaced, edited ones merged). Idempotent: a second run against
+ # the same commit is a no-op.
+ #
+ # After the migrations and never in the deploy's way: ``trigger`` waits on
+ # ``migrate`` only, so a catalog that cannot be fetched or published fails
+ # this job visibly while the backend keeps serving the previous catalog,
+ # until the next deploy or a manual run of this workflow publishes it.
+ publish-skills:
+ needs: migrate
+ environment: develop
+ name: Publish skills catalog
+ runs-on: ubuntu-latest
+ # One publish at a time per environment: the skills go under a database
+ # lock, the roster does not.
+ concurrency:
+ group: publish-skills-develop
+ cancel-in-progress: false
+
+ steps:
+ - name: Checkout code
+ uses: actions/checkout@v6
+ with:
+ ref: ${{ github.event.inputs.git_ref || github.ref_name }}
+
+ - name: Set up Python
+ uses: actions/setup-python@v5
+ with:
+ python-version: '3.11'
+
- name: Generate ephemeral secrets
# backend module imports read settings.secrets, so ENCRYPTION_KEY has to
# exist. Generated per run: a value committed here would be published.
@@ -87,11 +119,6 @@ jobs:
poetry install --only main --no-interaction --no-ansi
poetry run prisma generate
- # The catalog's main is what this environment serves from now on: new
- # and changed packages get immutable versions, the expert roster is
- # upserted by key, and copies users hold catch up lazily on their next
- # turn (unedited copies replaced, edited ones merged). Idempotent, and
- # a second deploy against the same commit is a no-op.
- name: Publish skills catalog
working-directory: ./autogpt_platform/backend
run: poetry run publish-skills-catalog
diff --git a/.github/workflows/platform-autogpt-deploy-prod.yml b/.github/workflows/platform-autogpt-deploy-prod.yml
index f186e80ad0..df5126470b 100644
--- a/.github/workflows/platform-autogpt-deploy-prod.yml
+++ b/.github/workflows/platform-autogpt-deploy-prod.yml
@@ -39,6 +39,38 @@ jobs:
DATABASE_URL: ${{ secrets.BACKEND_DATABASE_URL }}
DIRECT_URL: ${{ secrets.BACKEND_DATABASE_URL }}
+ # The catalog's main is what this environment serves from now on: new and
+ # changed packages get immutable versions, the expert roster is upserted by
+ # key, and copies users hold catch up lazily on their next turn (unedited
+ # copies replaced, edited ones merged). Idempotent: a second run against
+ # the same commit is a no-op.
+ #
+ # After the migrations and never in the deploy's way: ``trigger`` waits on
+ # ``migrate`` only, so a catalog that cannot be fetched or published fails
+ # this job visibly while the backend keeps serving the previous catalog,
+ # until the next deploy or a manual run of this workflow publishes it.
+ publish-skills:
+ needs: migrate
+ environment: production
+ name: Publish skills catalog
+ runs-on: ubuntu-latest
+ # One publish at a time per environment: the skills go under a database
+ # lock, the roster does not.
+ concurrency:
+ group: publish-skills-production
+ cancel-in-progress: false
+
+ steps:
+ - name: Checkout code
+ uses: actions/checkout@v6
+ with:
+ ref: ${{ github.ref_name || 'master' }}
+
+ - name: Set up Python
+ uses: actions/setup-python@v5
+ with:
+ python-version: '3.11'
+
- name: Generate ephemeral secrets
# backend module imports read settings.secrets, so ENCRYPTION_KEY has to
# exist. Generated per run: a value committed here would be published.
@@ -62,11 +94,6 @@ jobs:
poetry install --only main --no-interaction --no-ansi
poetry run prisma generate
- # The catalog's main is what this environment serves from now on: new
- # and changed packages get immutable versions, the expert roster is
- # upserted by key, and copies users hold catch up lazily on their next
- # turn (unedited copies replaced, edited ones merged). Idempotent, and
- # a second deploy against the same commit is a no-op.
- name: Publish skills catalog
working-directory: ./autogpt_platform/backend
run: poetry run publish-skills-catalog
@@ -75,7 +102,6 @@ jobs:
DIRECT_URL: ${{ secrets.BACKEND_DATABASE_URL }}
SKILLS_CATALOG_TOKEN: ${{ github.token }}
-
trigger:
needs: migrate
runs-on: ubuntu-latest
diff --git a/autogpt_platform/backend/backend/api/features/store/skill_db.py b/autogpt_platform/backend/backend/api/features/store/skill_db.py
index 863c13ca65..51ccfd45b2 100644
--- a/autogpt_platform/backend/backend/api/features/store/skill_db.py
+++ b/autogpt_platform/backend/backend/api/features/store/skill_db.py
@@ -208,6 +208,37 @@ async def get_version_packages(
return packages
+async def find_version_by_hash(listing_id: str, package_sha256: str) -> str | None:
+ """The id of the listing's version whose package hashes to
+ *package_sha256*, any version, not only the active one.
+
+ A copy installed before baselines were recorded is matched through
+ this: its files still hash to the version it came from when nobody
+ edited it, which proves it unedited and names what to fast-forward from.
+ A version with no stored hash is hashed as its installs rendered it,
+ from its files' recorded hashes, never their bytes. The newest match
+ wins when two versions hash alike.
+ """
+ rows = await prisma.models.SkillListingVersion.prisma().find_many(
+ where={"skillListingId": listing_id},
+ include={"SkillListing": True},
+ order={"version": "desc"},
+ )
+ file_hashes = await _file_hashes(
+ [row.id for row in rows if row.packageSha256 is None]
+ )
+ for row in rows:
+ if row.SkillListing is None:
+ continue
+ digest = row.packageSha256 or skill_model.package_sha256_of(
+ skill_model.legacy_skill_markdown(row, row.SkillListing.slug),
+ file_hashes.get(row.id, []),
+ )
+ if digest == package_sha256:
+ return row.id
+ return None
+
+
async def get_marketplace_skills(
*,
category: str | None = None,
diff --git a/autogpt_platform/backend/backend/api/features/store/skill_db_test.py b/autogpt_platform/backend/backend/api/features/store/skill_db_test.py
index 3ef393681a..a75e7e7159 100644
--- a/autogpt_platform/backend/backend/api/features/store/skill_db_test.py
+++ b/autogpt_platform/backend/backend/api/features/store/skill_db_test.py
@@ -622,6 +622,31 @@ async def test_version_packages_serve_any_version_verbatim(tmp_path):
assert packages[versions[1].id].package_sha256 == versions[1].packageSha256
+async def test_find_version_by_hash_matches_hashed_and_legacy_versions(tmp_path):
+ """A copy installed before baselines were recorded is matched to the
+ version its files still hash to: a published version by its stored hash,
+ a pre-publisher version by the hash of what its install rendered."""
+ legacy = await _make_listing("old-hand", body="# Old hand\n")
+ assert legacy.ActiveVersion is not None
+ legacy_hash = skill_model.legacy_package_sha256(legacy.ActiveVersion, "old-hand")
+ await _publish(_write_catalog(tmp_path))
+ published = await prisma.models.SkillListing.prisma().find_unique(
+ where={"slug": "cold-email"}, include={"ActiveVersion": True}
+ )
+ assert published is not None and published.ActiveVersion is not None
+ assert published.ActiveVersion.packageSha256 is not None
+
+ found_legacy = await skill_db.find_version_by_hash(legacy.id, legacy_hash)
+ found_published = await skill_db.find_version_by_hash(
+ published.id, published.ActiveVersion.packageSha256
+ )
+
+ assert found_legacy == legacy.ActiveVersion.id
+ assert found_published == published.ActiveVersion.id
+ assert await skill_db.find_version_by_hash(legacy.id, "0" * 64) is None
+ assert await skill_db.find_version_by_hash(published.id, legacy_hash) is None
+
+
def _patch_store(mocker, *, is_new: bool = True) -> AsyncMock:
"""Stand in for the locked write, reporting every skill stored."""
diff --git a/autogpt_platform/backend/backend/api/features/store/skill_model.py b/autogpt_platform/backend/backend/api/features/store/skill_model.py
index 17bfe475ba..802ddc9662 100644
--- a/autogpt_platform/backend/backend/api/features/store/skill_model.py
+++ b/autogpt_platform/backend/backend/api/features/store/skill_model.py
@@ -7,6 +7,7 @@ takes once installed, so the two can never drift. ``name`` is that slug again,
never a display string: the title on the card is :func:`skill_title`.
"""
+import base64
import datetime
import re
@@ -140,6 +141,39 @@ class SkillVersionPackage(pydantic.BaseModel):
# SHA-256s an earlier install already scanned clean; skips the rescan.
scanned_checksums: list[str] = []
+ # The copilot executor has no database connection, so a package reaches
+ # it over the DB manager RPC as JSON, and JSON has no bytes: the default
+ # encoding tries UTF-8 and a font, a PNG or a spreadsheet in a package
+ # fails it. Base64 carries any file, and only over the wire.
+ @pydantic.field_serializer("files", when_used="json")
+ def _files_over_the_wire(self, files: list[SkillFile]) -> list[dict[str, object]]:
+ return [
+ {
+ "relative_path": entry.relative_path,
+ "content_b64": base64.b64encode(entry.content).decode("ascii"),
+ "is_executable": entry.is_executable,
+ }
+ for entry in files
+ ]
+
+ @pydantic.field_validator("files", mode="before")
+ @classmethod
+ def _files_from_the_wire(cls, value: object) -> object:
+ if not isinstance(value, list):
+ return value
+ return [
+ (
+ SkillFile(
+ relative_path=str(item["relative_path"]),
+ content=base64.b64decode(item["content_b64"]),
+ is_executable=bool(item.get("is_executable", False)),
+ )
+ if isinstance(item, dict) and "content_b64" in item
+ else item
+ )
+ for item in value
+ ]
+
def legacy_skill_markdown(version: prisma.models.SkillListingVersion, slug: str) -> str:
"""Exactly what an install wrote for a version with no ``skillMarkdown``:
diff --git a/autogpt_platform/backend/backend/api/features/store/skill_model_test.py b/autogpt_platform/backend/backend/api/features/store/skill_model_test.py
index e562b38769..33485545ed 100644
--- a/autogpt_platform/backend/backend/api/features/store/skill_model_test.py
+++ b/autogpt_platform/backend/backend/api/features/store/skill_model_test.py
@@ -1,8 +1,11 @@
-"""Tests for the display title a listing shows on the marketplace."""
+"""Tests for the display title a listing shows on the marketplace, and for
+the package model that crosses the DB manager RPC."""
import pytest
-from backend.api.features.store.skill_model import skill_title
+from backend.api.features.store.skill_model import SkillVersionPackage, skill_title
+from backend.copilot.tools.skills import SkillFile
+from backend.util.json import dumps, loads
@pytest.mark.parametrize(
@@ -45,3 +48,34 @@ def test_a_body_that_does_not_open_with_a_heading_falls_back_to_the_slug(name, b
def test_a_nameless_listing_keeps_its_name_rather_than_rendering_empty():
assert skill_title("", "") == ""
+
+
+def test_a_package_with_binary_files_survives_the_rpc_encoding():
+ """The copilot executor gets packages over the DB manager RPC as JSON,
+ the way FastAPI encodes a response and the client validates it back; a
+ font or an image in a package has to come back byte for byte."""
+ png = b"\x89PNG\r\n\x1a\n\xff\x00"
+ package = SkillVersionPackage(
+ version_id="v1",
+ listing_id="l1",
+ slug="demo",
+ version=1,
+ package_sha256=None,
+ skill_markdown="---\nname: demo\ndescription: d\n---\n",
+ files=[
+ SkillFile(relative_path="assets/logo.png", content=png),
+ SkillFile(
+ relative_path="scripts/run.sh",
+ content=b"#!/bin/sh\n",
+ is_executable=True,
+ ),
+ ],
+ )
+
+ back = SkillVersionPackage.model_validate(loads(dumps({"v1": package}))["v1"])
+
+ assert [(f.relative_path, f.content, f.is_executable) for f in back.files] == [
+ ("assets/logo.png", png, False),
+ ("scripts/run.sh", b"#!/bin/sh\n", True),
+ ]
+ assert back.model_dump() == package.model_dump()
diff --git a/autogpt_platform/backend/backend/copilot/tools/skill_merge.py b/autogpt_platform/backend/backend/copilot/tools/skill_merge.py
new file mode 100644
index 0000000000..1be6cb449b
--- /dev/null
+++ b/autogpt_platform/backend/backend/copilot/tools/skill_merge.py
@@ -0,0 +1,122 @@
+"""Merging a ``SKILL.md``: frontmatter by field, body by line.
+
+A copy the owner edited through ``store_skill`` has its frontmatter
+re-rendered by the runtime, so line by line it differs from the catalog's
+text even when no field changed. Merged as text, every later catalog change
+to a field would collide with a "change" the owner never made, and the
+owner's stale field would win. The frontmatter is a mapping, so it merges as
+one: field by field, the owner's value winning a field both sides changed.
+The body is prose and merges by line like any other file.
+"""
+
+import re
+from typing import Any
+
+import yaml
+from pydantic import BaseModel
+
+from backend.data.skill_package import MergedText, merge_text
+
+# The closing fence's newline is required so the text goes back together
+# byte for byte; a file without one merges as plain text.
+_FRONTMATTER_RE = re.compile(r"^---\n(.*?)\n---\n(.*)$", re.DOTALL)
+_MISSING = object()
+
+
+def merge_skill_markdown(base: str, ours: str, theirs: str) -> MergedText:
+ """Three-way merge of a ``SKILL.md``: *ours* is the owner's copy, *theirs*
+ the update, *base* the version the copy was installed from. The owner's
+ side wins any field or hunk both sides changed differently."""
+ if ours == theirs:
+ return MergedText(text=ours, conflicted=False)
+ if base == ours:
+ return MergedText(text=theirs, conflicted=False)
+ if base == theirs:
+ return MergedText(text=ours, conflicted=False)
+ b, o, t = _split(base), _split(ours), _split(theirs)
+ if b is None or o is None or t is None:
+ return merge_text(base, ours, theirs)
+ frontmatter = _merge_frontmatter(b.frontmatter, o.frontmatter, t.frontmatter)
+ body = merge_text(b.body, o.body, t.body)
+ return MergedText(
+ text=f"---\n{frontmatter.text}\n---\n{body.text}",
+ conflicted=frontmatter.conflicted or body.conflicted,
+ )
+
+
+class _Split(BaseModel):
+ frontmatter: str
+ body: str
+
+
+def _split(text: str) -> _Split | None:
+ match = _FRONTMATTER_RE.match(text)
+ if match is None:
+ return None
+ return _Split(frontmatter=match.group(1), body=match.group(2))
+
+
+def _merge_frontmatter(base: str, ours: str, theirs: str) -> MergedText:
+ """Field by field. The text comes back untouched from whichever side the
+ result equals, so a frontmatter nobody changed keeps the catalog's bytes;
+ only a real three-way combination is re-rendered."""
+ if ours == theirs or base == theirs:
+ return MergedText(text=ours, conflicted=False)
+ if base == ours:
+ return MergedText(text=theirs, conflicted=False)
+ b, o, t = _mapping(base), _mapping(ours), _mapping(theirs)
+ if b is None or o is None or t is None:
+ return merge_text(base, ours, theirs)
+ merged: dict[str, Any] = {}
+ conflicted = False
+ for key in [
+ *t,
+ *(k for k in o if k not in t),
+ *(k for k in b if k not in t and k not in o),
+ ]:
+ bv, ov, tv = b.get(key, _MISSING), o.get(key, _MISSING), t.get(key, _MISSING)
+ if _same(ov, tv):
+ value = ov
+ elif _same(ov, bv):
+ value = tv
+ elif _same(tv, bv):
+ value = ov
+ else:
+ conflicted = True
+ value = ov
+ if value is not _MISSING:
+ merged[key] = value
+ if _same_mapping(merged, t):
+ return MergedText(text=theirs, conflicted=conflicted)
+ if _same_mapping(merged, o):
+ return MergedText(text=ours, conflicted=conflicted)
+ rendered = yaml.safe_dump(merged, sort_keys=False).strip()
+ return MergedText(text=rendered, conflicted=conflicted)
+
+
+def _mapping(text: str) -> dict[str, Any] | None:
+ try:
+ loaded = yaml.safe_load(text)
+ except yaml.YAMLError:
+ return None
+ if not isinstance(loaded, dict):
+ return None
+ return {str(key): value for key, value in loaded.items()}
+
+
+def _same(a: object, b: object) -> bool:
+ return _scalar(a) == _scalar(b)
+
+
+def _same_mapping(a: dict[str, Any], b: dict[str, Any]) -> bool:
+ return a.keys() == b.keys() and all(_same(a[key], b[key]) for key in a)
+
+
+def _scalar(value: object) -> object:
+ # ``version: 1`` in the catalog and ``version: '1'`` after the runtime
+ # re-rendered it are the same field value, not an edit.
+ if isinstance(value, bool):
+ return value
+ if isinstance(value, (int, float)):
+ return str(value)
+ return value
diff --git a/autogpt_platform/backend/backend/copilot/tools/skill_merge_test.py b/autogpt_platform/backend/backend/copilot/tools/skill_merge_test.py
new file mode 100644
index 0000000000..728345016e
--- /dev/null
+++ b/autogpt_platform/backend/backend/copilot/tools/skill_merge_test.py
@@ -0,0 +1,93 @@
+"""A SKILL.md merges frontmatter by field and body by line, so an owner's
+edit (which re-renders the frontmatter) never masks a catalog change to a
+field, and a field nobody touched keeps the catalog's bytes."""
+
+import yaml
+
+from backend.copilot.tools.skill_merge import merge_skill_markdown
+from backend.copilot.tools.skills import ParsedSkill, render_skill_markdown
+
+CATALOG_V1 = "---\nname: demo\ndescription: First description\nversion: 1\n---\n\n# Demo\n\nintro\nkeep\n"
+CATALOG_V2 = "---\nname: demo\ndescription: Second description\nversion: 2\n---\n\n# Demo\n\nnew intro\nkeep\n"
+
+
+def _rerendered(text: str, body: str) -> str:
+ """What ``store_skill`` writes when the owner edits the body: the same
+ fields, re-rendered by the runtime, so the frontmatter text differs."""
+ from backend.copilot.tools.skills import parse_skill_markdown
+
+ parsed = parse_skill_markdown(text)
+ assert parsed is not None
+ return render_skill_markdown(ParsedSkill(**{**parsed.__dict__, "body": body}))
+
+
+def test_a_catalog_field_change_reaches_a_body_edited_copy_without_conflict():
+ ours = _rerendered(CATALOG_V1, "# Demo\n\nintro\nkeep\nmy addition\n")
+ assert ours.splitlines()[1:4] != CATALOG_V1.splitlines()[1:4] # re-rendered
+
+ merged = merge_skill_markdown(CATALOG_V1, ours, CATALOG_V2)
+
+ assert not merged.conflicted
+ front, body = merged.text.split("\n---\n", 1)
+ assert yaml.safe_load(front[4:]) == {
+ "name": "demo",
+ "description": "Second description",
+ "version": 2,
+ }
+ assert "new intro\nkeep\nmy addition" in body
+
+
+def test_a_field_both_sides_changed_keeps_the_owners_value_and_says_so():
+ ours = CATALOG_V1.replace("First description", "My description")
+
+ merged = merge_skill_markdown(CATALOG_V1, ours, CATALOG_V2)
+
+ assert merged.conflicted
+ assert "description: My description" in merged.text
+ assert "version: 2" in merged.text # the field only the catalog changed
+
+
+def test_an_untouched_frontmatter_keeps_the_catalogs_bytes():
+ ours = CATALOG_V1.replace("keep\n", "keep\nmine\n")
+
+ merged = merge_skill_markdown(CATALOG_V1, ours, CATALOG_V2)
+
+ assert not merged.conflicted
+ assert merged.text.startswith(CATALOG_V2.split("\n---\n")[0] + "\n---\n")
+ assert merged.text.endswith("# Demo\n\nnew intro\nkeep\nmine\n")
+
+
+def test_an_edit_next_to_the_catalogs_change_is_a_conflict_the_owner_wins():
+ ours = CATALOG_V1.replace("intro\n", "intro\nmine\n")
+
+ merged = merge_skill_markdown(CATALOG_V1, ours, CATALOG_V2)
+
+ assert merged.conflicted
+ assert "intro\nmine\nkeep" in merged.text
+ assert "new intro" not in merged.text
+
+
+def test_a_field_the_owner_added_survives_the_update():
+ ours = CATALOG_V1.replace("version: 1\n", "version: 1\nallowed-tools: bash\n")
+
+ merged = merge_skill_markdown(CATALOG_V1, ours, CATALOG_V2)
+
+ front = yaml.safe_load(merged.text.split("\n---\n", 1)[0][4:])
+ assert front["allowed-tools"] == "bash"
+ assert front["description"] == "Second description"
+
+
+def test_a_number_the_runtime_quoted_is_not_an_edit():
+ ours = CATALOG_V1.replace("version: 1", "version: '1'")
+
+ merged = merge_skill_markdown(CATALOG_V1, ours, CATALOG_V2)
+
+ assert not merged.conflicted
+ assert merged.text == CATALOG_V2
+
+
+def test_text_without_frontmatter_merges_as_plain_text():
+ merged = merge_skill_markdown("a\nb\n", "a\nb\nmine\n", "theirs\nb\n")
+
+ assert merged.text == "theirs\nb\nmine\n"
+ assert not merged.conflicted
diff --git a/autogpt_platform/backend/backend/copilot/tools/skill_updates_test.py b/autogpt_platform/backend/backend/copilot/tools/skill_updates_test.py
index ebbc1eb6c9..00eb94d201 100644
--- a/autogpt_platform/backend/backend/copilot/tools/skill_updates_test.py
+++ b/autogpt_platform/backend/backend/copilot/tools/skill_updates_test.py
@@ -15,17 +15,21 @@ from backend.api.features.store.skill_model import (
from backend.copilot.tools import skills as skills_module
from backend.copilot.tools.skills import (
_RECONCILE_WRITE_BUDGET,
+ MAX_PACKAGE_FILES,
SKILL_ORIGIN_MARKETPLACE,
SKILL_ORIGIN_USER,
UPDATE_AVAILABLE,
UPDATE_MERGED,
UPDATE_RETIRED,
SkillBaseline,
+ SkillChangedError,
SkillFile,
+ SkillWrite,
build_skills_update_notice,
list_user_skills,
render_skills_index,
store_user_skill,
+ store_user_skills,
)
from backend.copilot.tools.skills_test import _FakeWorkspaceManager, _patch_skills_path
from backend.data.skill_package import package_tree_sha256
@@ -42,10 +46,18 @@ def _package(
version_id: str,
body: str,
files: dict[str, bytes] | None = None,
+) -> SkillVersionPackage:
+ return _package_text(slug, version_id, _skill_md(slug, body), files)
+
+
+def _package_text(
+ slug: str,
+ version_id: str,
+ text: str,
+ files: dict[str, bytes] | None = None,
) -> SkillVersionPackage:
"""*version_id* is "v<n>"; the id stored is unique per slug, as real
version rows are, so several skills can share a version number."""
- text = _skill_md(slug, body)
siblings = [
SkillFile(relative_path=path, content=content)
for path, content in sorted((files or {}).items())
@@ -381,10 +393,14 @@ def test_the_index_names_each_update_state():
async def test_the_next_turn_is_told_which_bodies_changed():
fake = _FakeWorkspaceManager()
v1 = _package("cold-email", "v1", "# Cold\n")
- with _patch_skills_path(fake), patch.object(
- skills_module, "_consume_updated", AsyncMock(return_value=["cold-email"])
- ), patch.object(
- skills_module, "is_skills_feature_enabled", AsyncMock(return_value=True)
+ with (
+ _patch_skills_path(fake),
+ patch.object(
+ skills_module, "_consume_updated", AsyncMock(return_value=["cold-email"])
+ ),
+ patch.object(
+ skills_module, "is_skills_feature_enabled", AsyncMock(return_value=True)
+ ),
):
await _install(v1)
prior = [
@@ -411,3 +427,197 @@ async def test_an_owner_skill_is_never_touched_by_the_reconcile():
assert skill.update is None
assert fake.files["/skills/cold-email/SKILL.md"].decode().endswith("# Mine\n")
patched.skill_db.get_active_versions.assert_not_called()
+
+
+async def test_a_copy_from_before_baselines_is_matched_to_its_version_and_moved():
+ """A pre-catalog install rendered its SKILL.md from the row's fields, so
+ its bytes never equal the catalog's. The publisher hashed that rendering
+ onto the old version; a copy that still hashes to it is unedited, and
+ takes the update like any other."""
+ fake = _FakeWorkspaceManager()
+ v1 = _package("cold-email", "v1", "# Cold\n", {"references/a.md": b"a\n"})
+ v2 = _package(
+ "cold-email", "v2", "# Cold, from the catalog\n", {"references/a.md": b"a2\n"}
+ )
+ with _patch_skills_path(fake) as patched:
+ await store_user_skill(
+ USER,
+ name="cold-email",
+ description="",
+ body="",
+ files=list(v1.files),
+ origin=SKILL_ORIGIN_MARKETPLACE,
+ skill_markdown=v1.skill_markdown,
+ )
+ _marketplace(patched, v1, v2)
+ patched.skill_db.find_version_by_hash = AsyncMock(
+ side_effect=lambda listing_id, sha: (
+ v1.version_id
+ if (listing_id, sha) == (v1.listing_id, v1.package_sha256)
+ else None
+ )
+ )
+ [skill] = await list_user_skills(USER)
+
+ assert fake.files["/skills/cold-email/SKILL.md"] == v2.skill_markdown.encode()
+ assert fake.files["/skills/cold-email/references/a.md"] == b"a2\n"
+ assert skill.baseline == _baseline(v2)
+ assert skill.update is None
+
+
+async def test_a_copy_that_cannot_take_a_version_is_not_tried_again_for_it():
+ fake = _FakeWorkspaceManager()
+ v1 = _package("cold-email", "v1", "# Cold\n")
+ v2 = _package("cold-email", "v2", "# Cold v2\n")
+ noted: dict[str, str] = {}
+
+ async def read(user_id, expert_id):
+ return dict(noted)
+
+ async def write(user_id, expert_id, entries):
+ noted.update(entries)
+
+ with (
+ _patch_skills_path(fake) as patched,
+ patch.object(skills_module, "_read_update_backoff", read),
+ patch.object(skills_module, "_set_update_backoff", write),
+ ):
+ await _install(v1)
+ _marketplace(patched, v1, v2)
+ # The update cannot be fetched: nothing to fast-forward to.
+ patched.skill_db.get_version_packages = AsyncMock(return_value={})
+ [first] = await list_user_skills(USER)
+ [second] = await list_user_skills(USER)
+
+ assert (first.update, second.update) == (UPDATE_AVAILABLE, UPDATE_AVAILABLE)
+ assert noted == {"cold-email": v2.version_id}
+ patched.skill_db.get_version_packages.assert_called_once()
+
+
+async def test_a_newer_publish_retries_a_copy_in_backoff_at_once():
+ fake = _FakeWorkspaceManager()
+ v1 = _package("cold-email", "v1", "# Cold\n")
+ v2 = _package("cold-email", "v2", "# Cold v2\n")
+ v3 = _package("cold-email", "v3", "# Cold v3\n")
+ # The copy could not take v2 on an earlier turn; v3 has since been published.
+ noted = {"cold-email": v2.version_id}
+
+ async def read(user_id, expert_id):
+ return dict(noted)
+
+ async def write(user_id, expert_id, entries):
+ noted.update(entries)
+
+ with (
+ _patch_skills_path(fake) as patched,
+ patch.object(skills_module, "_read_update_backoff", read),
+ patch.object(skills_module, "_set_update_backoff", write),
+ ):
+ await _install(v1)
+ _marketplace(patched, v1, v3)
+ [skill] = await list_user_skills(USER)
+
+ assert skill.baseline == _baseline(v3)
+ assert skill.update is None
+ assert fake.files["/skills/cold-email/SKILL.md"] == v3.skill_markdown.encode()
+ patched.skill_db.get_version_packages.assert_called_once()
+
+
+async def test_failed_attempts_spend_the_turns_budget_too():
+ fake = _FakeWorkspaceManager()
+ count = _RECONCILE_WRITE_BUDGET + 2
+ olds = [_package(f"skill-{i}", "v1", f"# {i}\n") for i in range(count)]
+ news = [_package(f"skill-{i}", "v2", f"# {i} v2\n") for i in range(count)]
+ with _patch_skills_path(fake) as patched:
+ for old in olds:
+ await _install(old)
+ _marketplace(patched, *news)
+ patched.skill_db.get_version_packages = AsyncMock(return_value={})
+ skills = await list_user_skills(USER)
+
+ assert all(s.update == UPDATE_AVAILABLE for s in skills)
+ assert patched.skill_db.get_version_packages.call_count == _RECONCILE_WRITE_BUDGET
+
+
+async def test_a_write_decided_on_a_copy_that_has_since_changed_is_refused():
+ """The reconcile decides outside the write lock; the write itself checks
+ the folder still hashes to what the decision saw, or an edit landed in
+ between would be put back and a file just added would be pruned."""
+ fake = _FakeWorkspaceManager()
+ v1 = _package("cold-email", "v1", "# Cold\n", {"references/a.md": b"a\n"})
+ v2 = _package("cold-email", "v2", "# Cold v2\n", {"references/a.md": b"a2\n"})
+ with _patch_skills_path(fake):
+ await _install(v1)
+ [outcome] = await store_user_skills(
+ USER,
+ [
+ SkillWrite(
+ name="cold-email",
+ description="",
+ body="",
+ skill_markdown=v2.skill_markdown,
+ files=list(v2.files),
+ baseline=_baseline(v2),
+ expected_package_sha256="not-what-the-folder-hashes-to",
+ )
+ ],
+ origin=SKILL_ORIGIN_MARKETPLACE,
+ )
+
+ assert isinstance(outcome, SkillChangedError)
+ assert fake.files["/skills/cold-email/SKILL.md"] == v1.skill_markdown.encode()
+ assert fake.files["/skills/cold-email/references/a.md"] == b"a\n"
+
+
+async def test_a_copy_over_the_files_cap_is_left_alone():
+ fake = _FakeWorkspaceManager()
+ v1 = _package("cold-email", "v1", "# Cold\n")
+ v2 = _package("cold-email", "v2", "# Cold v2\n")
+ with _patch_skills_path(fake) as patched:
+ await _install(v1)
+ for i in range(MAX_PACKAGE_FILES + 1):
+ fake.files[f"/skills/cold-email/references/{i}.md"] = b"by hand\n"
+ _marketplace(patched, v1, v2)
+ [skill] = await list_user_skills(USER)
+
+ assert skill.update == UPDATE_AVAILABLE
+ assert fake.files["/skills/cold-email/SKILL.md"] == v1.skill_markdown.encode()
+ by_hand = [p for p in fake.files if p.startswith("/skills/cold-email/references/")]
+ assert len(by_hand) == MAX_PACKAGE_FILES + 1
+
+
+async def test_a_catalog_frontmatter_change_reaches_a_copy_the_owner_edited():
+ """An owner's edit re-renders the frontmatter; merged line by line, the
+ catalog's new description would collide with that and be lost."""
+ fake = _FakeWorkspaceManager()
+ v1 = _package("cold-email", "v1", "# Cold\n\nintro\n")
+ v2 = _package_text(
+ "cold-email",
+ "v2",
+ "---\nname: cold-email\ndescription: A better description\n---\n\n"
+ "# Cold\n\nintro\n",
+ )
+ with _patch_skills_path(fake) as patched:
+ await _install(v1)
+ await store_user_skill(
+ USER,
+ name="cold-email",
+ description="cold-email description",
+ body="# Cold\n\nintro\nmy addition\n",
+ )
+ _marketplace(patched, v1, v2)
+ [skill] = await list_user_skills(USER)
+
+ text = fake.files["/skills/cold-email/SKILL.md"].decode()
+ assert "description: A better description" in text
+ assert "intro\nmy addition" in text
+ assert skill.description == "A better description"
+ assert skill.update == UPDATE_MERGED
+ assert skill.baseline is not None and skill.baseline.merge_conflicts == ()
diff --git a/autogpt_platform/backend/backend/copilot/tools/skills.py b/autogpt_platform/backend/backend/copilot/tools/skills.py
index 28b02f71e2..7601d601fc 100644
--- a/autogpt_platform/backend/backend/copilot/tools/skills.py
+++ b/autogpt_platform/backend/backend/copilot/tools/skills.py
@@ -51,6 +51,7 @@ from backend.data.skill_capacity import SkillLimitError, SkillOwnedError
from backend.data.skill_capacity import normalize_skill_origin as _normalize_origin
from backend.data.skill_package import (
SKILL_MD,
+ MergedPackage,
PackageFile,
merge_packages,
package_tree_sha256,
@@ -76,6 +77,7 @@ if TYPE_CHECKING:
from .base import BaseTool
from .models import ErrorResponse, ResponseType, ToolResponseBase
+from .skill_merge import merge_skill_markdown
from .workdir import (
read_workdir_bytes,
remove_from_workdir,
@@ -190,9 +192,24 @@ UPDATE_RETIRED = "retired"
# tell the model the bodies it may have read are stale.
SKILLS_UPDATED_KEY = "copilot:skills_updated:{user_id}:{owner}"
SKILLS_UPDATED_TTL_S = 7 * 24 * 3600
-# Copies one reconcile will rewrite. A cold turn right after a publish could
+# Names one notice carries; the rest wait for the next. More than the
+# owner's skill cap, so in practice one pop takes them all.
+_UPDATED_NOTICE_MAX = 500
+# Copies one reconcile will try. A cold turn right after a publish could
# otherwise spend seconds fast-forwarding a whole bundle; the rest go next time.
_RECONCILE_WRITE_BUDGET = 5
+# Copies that could not take the version they were last tried against, so a
+# cold turn does not fetch the same package and fail on it again: slug → the
+# version id it failed to take. A newer publish retries at once.
+SKILLS_UPDATE_BACKOFF_KEY = "copilot:skills_update_backoff:{user_id}:{owner}"
+SKILLS_UPDATE_BACKOFF_TTL_S = 6 * 3600
+
+
+class SkillChangedError(RuntimeError):
+ """A reconcile's write found the copy changed since it decided what to
+ write. The write is refused; the next cold turn compares again."""
+
+
# Where a skill in an owner's folder came from. Kept in the row's metadata
# (server-written; the frontmatter is the author's to edit) so the per-owner
# cap counts what the owner saved apart from what the platform installed.
@@ -814,6 +831,9 @@ class SkillWrite(NamedTuple):
skill_markdown: str | None = None
# For a marketplace install: what the copy is being installed from.
baseline: SkillBaseline | None = None
+ # For a reconcile: the hash the folder had when this write was decided.
+ # The write is refused if the folder has moved since (an edit landed).
+ expected_package_sha256: str | None = None
class StoredSkill(NamedTuple):
@@ -959,6 +979,7 @@ class _PreparedSkill(NamedTuple):
files: list[SkillFile] | None
scanned_checksums: frozenset[str]
baseline: SkillBaseline | None = None
+ expected_package_sha256: str | None = None
def _prepare_skill(write: SkillWrite, origin: str) -> _PreparedSkill:
@@ -1030,6 +1051,7 @@ def _prepare_exact_package(write: SkillWrite, origin: str) -> _PreparedSkill:
write.files or [],
write.scanned_checksums,
write.baseline,
+ write.expected_package_sha256,
)
@@ -1076,11 +1098,7 @@ async def _write_skill(
if baseline is not None:
metadata.update(_baseline_metadata(baseline))
folder = skill_folder(expert_id)
- stale = (
- await _list_package_files(manager, folder, name, cap=None)
- if skill.files is not None
- else []
- )
+ stale = await _stale_files(manager, folder, name, skill)
# The root is what indexes the skill, so it goes last: a new skill
# that fails part-way is never indexed. An upsert cannot be made
# atomic here — the old bytes are gone once overwritten. Serial
@@ -1136,6 +1154,30 @@ async def _write_skill(
return frozenset(checksums)
+async def _stale_files(
+ manager: WorkspaceManager, folder: str, name: str, skill: _PreparedSkill
+) -> list[SkillFileInfo]:
+ """The folder's files a whole-package write replaces, listed under the
+ lock. A reconcile also names the hash it decided on: the folder having
+ moved since means an edit landed in between, and the write would put it
+ back or prune a file the owner just added, so it is refused instead."""
+ if skill.files is None:
+ return []
+ if skill.expected_package_sha256 is None:
+ return await _list_package_files(manager, folder, name, cap=None)
+ listed = await _working_files(manager, folder, name)
+ if listed is None:
+ raise SkillChangedError(f"'{name}' was deleted while its update was prepared")
+ found = package_tree_sha256(
+ (path, info.checksum or "", info.is_executable) for path, info in listed.items()
+ )
+ if found != skill.expected_package_sha256:
+ raise SkillChangedError(
+ f"'{name}' changed while its update was prepared; it is compared again"
+ )
+ return [info for path, info in listed.items() if path != SKILL_MD]
+
+
async def _delete_paths(
manager: WorkspaceManager,
paths: set[str],
@@ -1553,8 +1595,10 @@ async def _reconcile_marketplace_copies(
)
return skills
manager = await _get_user_skill_manager(user_id, scope)
+ backoff = await _read_update_backoff(user_id, expert_id)
flags: dict[str, str] = {}
rewritten: list[str] = []
+ failed: dict[str, str] = {}
relist = False
budget = _RECONCILE_WRITE_BUDGET
for skill in copies:
@@ -1570,25 +1614,37 @@ async def _reconcile_marketplace_copies(
if baseline.merged_from is not None:
flags[skill.name] = UPDATE_MERGED
continue
- if budget <= 0:
+ if backoff.get(skill.name) == current.version_id or budget <= 0:
+ # Already shown not to take this version, or this turn's share
+ # of attempts is spent: the copy waits, and the index says so.
flags[skill.name] = UPDATE_AVAILABLE
continue
+ # An attempt spends budget whether or not it lands, so copies that
+ # cannot be updated bound the turn like the ones that can.
+ budget -= 1
try:
outcome = await _update_copy(
manager, user_id, expert_id, scope, skill, current
)
+ except SkillChangedError:
+ # Written under the update; the next cold turn compares again.
+ flags[skill.name] = UPDATE_AVAILABLE
+ continue
except Exception:
logger.exception("[skills] could not update copy %r", skill.name)
- outcome = None
+ flags[skill.name] = UPDATE_AVAILABLE
+ continue
if outcome is None:
+ failed[skill.name] = current.version_id
flags[skill.name] = UPDATE_AVAILABLE
continue
- budget -= 1
relist = True
if outcome != _STAMPED:
rewritten.append(skill.name)
if outcome == UPDATE_MERGED:
flags[skill.name] = UPDATE_MERGED
+ if failed:
+ await _set_update_backoff(user_id, expert_id, {**backoff, **failed})
if rewritten:
await _remember_updated(user_id, expert_id, rewritten)
if relist:
@@ -1609,29 +1665,36 @@ async def _update_copy(
current: "ActiveSkillVersion",
) -> str | None:
"""Move one copy to the listing's active version. Returns what happened,
- or None when the copy has to be left alone (edited with no baseline to
- merge against, or the package could not be fetched)."""
+ or None when the copy has to be left alone: edited with no version to
+ merge against, over the files cap, or its package could not be fetched."""
folder = skill_folder(expert_id)
listed = await _working_files(manager, folder, skill.name)
if listed is None:
return None
+ if len(listed) - 1 > MAX_PACKAGE_FILES:
+ # Written before the cap, or by hand. A merge reads the copy through
+ # the capped reader, so the files past the cap would look deleted
+ # and the write would prune them: leave the whole copy alone.
+ logger.warning(
+ "[skills] copy %r has more than %s files; not updating it",
+ skill.name,
+ MAX_PACKAGE_FILES,
+ )
+ return None
working_hash = package_tree_sha256(
(path, info.checksum or "", info.is_executable) for path, info in listed.items()
)
baseline = skill.baseline
if baseline is None:
- # Installed before baselines were recorded. Unedited if it still
- # matches the version the marketplace serves now; stamp it so the
- # next publish can fast-forward it. Otherwise nothing can tell an
- # edit from an old version, so the copy stays as it is.
- if working_hash != current.package_sha256:
+ baseline = await _prove_baseline(working_hash, current)
+ if baseline is None:
return None
- ours = await _read_working_package(manager, folder, skill.name)
- proven = SkillBaseline(
- current.listing_id, current.version_id, current.package_sha256
- )
- await _write_copy(user_id, expert_id, scope, skill.name, ours, proven)
- return _STAMPED
+ if baseline.version_id == current.version_id:
+ ours = await _read_working_package(manager, folder, skill.name)
+ await _write_copy(
+ user_id, expert_id, scope, skill.name, ours, baseline, working_hash
+ )
+ return _STAMPED
wanted = [current.version_id]
edited = working_hash != baseline.package_sha256
if edited:
@@ -1650,6 +1713,7 @@ async def _update_copy(
skill.name,
theirs,
fresh,
+ working_hash,
target.scanned_checksums,
)
return _FAST_FORWARDED
@@ -1657,9 +1721,9 @@ async def _update_copy(
if base is None:
return None
ours = await _read_working_package(manager, folder, skill.name)
- merged = merge_packages(_as_package(base), ours, theirs)
- if SKILL_MD not in merged.files:
- return None
+ # Line alignment is CPU work, and this loop serves every chat session on
+ # the process: off the loop, so a long file stalls one update, not all.
+ merged = await asyncio.to_thread(_merge_copy, _as_package(base), ours, theirs)
stamped = fresh._replace(
merged_from=baseline.version_id, merge_conflicts=tuple(merged.conflicts)
)
@@ -1670,11 +1734,58 @@ async def _update_copy(
skill.name,
merged.files,
stamped,
+ working_hash,
target.scanned_checksums,
)
return UPDATE_MERGED
+async def _prove_baseline(
+ working_hash: str, current: "ActiveSkillVersion"
+) -> SkillBaseline | None:
+ """The baseline of a copy installed before baselines were recorded.
+
+ Its files still hash to the version it came from if nobody edited it:
+ the active version, or an older one, which the publisher hashed after
+ the fact (a pre-catalog install rendered its SKILL.md from the row's
+ fields, so the catalog's bytes never match it). A match proves the copy
+ unedited and names what to fast-forward from. No match is an edit that
+ nothing can tell from an old version, so the copy stays as it is.
+ """
+ if working_hash == current.package_sha256:
+ return SkillBaseline(
+ current.listing_id, current.version_id, current.package_sha256
+ )
+ version_id = await skill_db().find_version_by_hash(current.listing_id, working_hash)
+ if version_id is None:
+ return None
+ return SkillBaseline(current.listing_id, version_id, working_hash)
+
+
+def _merge_copy(
+ base: Mapping[str, PackageFile],
+ ours: Mapping[str, PackageFile],
+ theirs: Mapping[str, PackageFile],
+) -> MergedPackage:
+ """The siblings merge file by file; the SKILL.md merges frontmatter by
+ field and body by line (:mod:`skill_merge`), so a frontmatter the runtime
+ re-rendered on an edit never masks a catalog change to a field."""
+ merged = merge_packages(_siblings(base), _siblings(ours), _siblings(theirs))
+ root = merge_skill_markdown(_root_text(base), _root_text(ours), _root_text(theirs))
+ merged.files[SKILL_MD] = PackageFile(content=root.text.encode("utf-8"))
+ if root.conflicted:
+ merged.conflicts = sorted([*merged.conflicts, SKILL_MD])
+ return merged
+
+
+def _siblings(package: Mapping[str, PackageFile]) -> dict[str, PackageFile]:
+ return {path: entry for path, entry in package.items() if path != SKILL_MD}
+
+
+def _root_text(package: Mapping[str, PackageFile]) -> str:
+ return package[SKILL_MD].content.decode("utf-8")
+
+
async def _working_files(
manager: WorkspaceManager, folder: str, slug: str
) -> dict[str, SkillFileInfo] | None:
@@ -1696,10 +1807,11 @@ async def _working_files(
async def _read_working_package(
manager: WorkspaceManager, folder: str, slug: str
) -> dict[str, PackageFile]:
- """The copy's files with their bytes, for a merge or a stamp."""
+ """The copy's files with their bytes, for a merge or a stamp. Whole or
+ not at all: a truncated read would merge as if the rest were deleted."""
root = await manager.read_file(f"{folder}/{slug}/{SKILL_MD}")
package = {SKILL_MD: PackageFile(content=root)}
- for entry in await _read_package_files(manager, folder, slug):
+ for entry in await _read_package_files(manager, folder, slug, complete=True):
package[entry.relative_path] = PackageFile(
content=entry.content, executable=entry.is_executable
)
@@ -1722,10 +1834,12 @@ async def _write_copy(
slug: str,
package: Mapping[str, PackageFile],
baseline: SkillBaseline,
+ expected_package_sha256: str,
scanned_checksums: list[str] | None = None,
) -> None:
"""Write a whole package over the copy through the ordinary install path,
- so it is validated, capped, locked and delta-written like any install."""
+ so it is validated, capped, locked and delta-written like any install,
+ and only if the copy still hashes to what the decision was made on."""
[outcome] = await store_user_skills(
user_id,
[
@@ -1745,6 +1859,7 @@ async def _write_copy(
],
scanned_checksums=frozenset(scanned_checksums or ()),
baseline=baseline,
+ expected_package_sha256=expected_package_sha256,
)
],
expert_id=expert_id,
@@ -1777,9 +1892,9 @@ async def _consume_updated(user_id: str, expert_id: str | None) -> list[str]:
try:
redis = await get_redis_async()
key = _updated_key(user_id, expert_id)
- names = await redis.smembers(key)
- if names:
- await redis.delete(key)
+ # One atomic pop, so a name noted between a read and a delete is
+ # never lost; anything past the count is picked up next turn.
+ names = await redis.spop(key, _UPDATED_NOTICE_MAX) or []
return sorted(
n.decode("utf-8") if isinstance(n, bytes) else str(n) for n in names
)
@@ -1787,6 +1902,44 @@ async def _consume_updated(user_id: str, expert_id: str | None) -> list[str]:
return []
+def _update_backoff_key(user_id: str, expert_id: str | None) -> str:
+ return SKILLS_UPDATE_BACKOFF_KEY.format(
+ user_id=user_id, owner=expert_id or "autopilot"
+ )
+
+
+async def _read_update_backoff(user_id: str, expert_id: str | None) -> dict[str, str]:
+ """Copies that could not take the version they were last tried against,
+ slug → version id. Empty when Redis is away: the cost is a retry."""
+ try:
+ redis = await get_redis_async()
+ raw = await redis.get(_update_backoff_key(user_id, expert_id))
+ loaded = json.loads(raw) if raw else {}
+ except Exception:
+ return {}
+ if not isinstance(loaded, dict):
+ return {}
+ return {
+ slug: version_id
+ for slug, version_id in loaded.items()
+ if isinstance(slug, str) and isinstance(version_id, str)
+ }
+
+
+async def _set_update_backoff(
+ user_id: str, expert_id: str | None, entries: dict[str, str]
+) -> None:
+ try:
+ redis = await get_redis_async()
+ await redis.set(
+ _update_backoff_key(user_id, expert_id),
+ json.dumps(entries),
+ ex=SKILLS_UPDATE_BACKOFF_TTL_S,
+ )
+ except Exception:
+ pass
+
+
async def read_user_skill_with_body(
user_id: str,
name: str,
diff --git a/autogpt_platform/backend/backend/copilot/tools/skills_test.py b/autogpt_platform/backend/backend/copilot/tools/skills_test.py
index 29097009fd..df99c5d791 100644
--- a/autogpt_platform/backend/backend/copilot/tools/skills_test.py
+++ b/autogpt_platform/backend/backend/copilot/tools/skills_test.py
@@ -368,6 +368,7 @@ class _patch_skills_path:
self.skill_db = MagicMock()
self.skill_db.get_active_versions = AsyncMock(return_value={})
self.skill_db.get_version_packages = AsyncMock(return_value={})
+ self.skill_db.find_version_by_hash = AsyncMock(return_value=None)
self.workdir = tempfile.mkdtemp(prefix="copilot-skills-test-")
self._patches = [
patch(
diff --git a/autogpt_platform/backend/backend/data/db_manager.py b/autogpt_platform/backend/backend/data/db_manager.py
index 8f478932fd..3765606811 100644
--- a/autogpt_platform/backend/backend/data/db_manager.py
+++ b/autogpt_platform/backend/backend/data/db_manager.py
@@ -591,6 +591,7 @@ class DatabaseManager(AppService):
# fast-forward or merge them.
get_active_versions = _(marketplace_skill_db.get_active_versions)
get_version_packages = _(marketplace_skill_db.get_version_packages)
+ find_version_by_hash = _(marketplace_skill_db.find_version_by_hash)
# ============ CoPilot Chat Sessions ============ #
# NOTE: no eager-load `get_chat_session` here — callers go through
@@ -887,6 +888,7 @@ class DatabaseManagerAsyncClient(AppServiceClient):
# ============ Marketplace skills ============ #
get_active_versions = d.get_active_versions
get_version_packages = d.get_version_packages
+ find_version_by_hash = d.find_version_by_hash
# ============ Chat Sharing ============ #
link_new_execution_to_chat_share = d.link_new_execution_to_chat_share
diff --git a/autogpt_platform/backend/backend/data/skill_package.py b/autogpt_platform/backend/backend/data/skill_package.py
index 8a5061e248..6496a1aaa9 100644
--- a/autogpt_platform/backend/backend/data/skill_package.py
+++ b/autogpt_platform/backend/backend/data/skill_package.py
@@ -23,6 +23,10 @@ from pydantic import BaseModel, ConfigDict
SKILL_MD = "SKILL.md"
_NEWLINE = "\n"
+# Above this many lines on any side the line alignment is not worth its CPU
+# (quadratic at worst, and it runs where every chat session on the process
+# waits): the file counts as a conflict and the user's copy stays as it is.
+MAX_MERGE_LINES = 5_000
def file_sha256(content: bytes) -> str:
@@ -99,6 +103,8 @@ def merge_text(base: str, ours: str, theirs: str) -> MergedText:
base_lines = base.split(_NEWLINE)
our_lines = ours.split(_NEWLINE)
their_lines = theirs.split(_NEWLINE)
+ if max(len(base_lines), len(our_lines), len(their_lines)) > MAX_MERGE_LINES:
+ return MergedText(text=ours, conflicted=True)
out: list[str] = []
conflicted = False
for chunk in _diff3_chunks(base_lines, our_lines, their_lines):
diff --git a/autogpt_platform/backend/backend/data/skill_package_test.py b/autogpt_platform/backend/backend/data/skill_package_test.py
index 91a9a5a946..06cd7277f2 100644
--- a/autogpt_platform/backend/backend/data/skill_package_test.py
+++ b/autogpt_platform/backend/backend/data/skill_package_test.py
@@ -1,4 +1,5 @@
from backend.data.skill_package import (
+ MAX_MERGE_LINES,
PackageFile,
merge_packages,
merge_text,
@@ -143,3 +144,16 @@ def test_merge_packages_executable_bit_follows_the_changed_side():
ours = {"s.py": PackageFile(content=b"x\n", executable=False)}
theirs = {"s.py": PackageFile(content=b"x\n", executable=True)}
assert merge_packages(base, ours, theirs).files["s.py"].executable is True
+
+
+def test_merge_text_over_the_line_cap_keeps_ours_as_a_conflict():
+ """The alignment is quadratic at worst and runs on a shared event loop,
+ so a huge file is not merged at all: the user's copy stays, flagged."""
+ base = "\n".join(str(i) for i in range(MAX_MERGE_LINES + 1))
+ ours = base + "\nmine"
+ theirs = "changed" + base[1:]
+
+ merged = merge_text(base, ours, theirs)
+
+ assert merged.text == ours
+ assert merged.conflicted