AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.
fix(backend/copilot): index whole descriptions and link save/store/write in capability search (#14778)
Nicholas Tindle187bb9ed1ed5
diff --git a/autogpt_platform/backend/backend/copilot/capabilities/eval/benchmark.py b/autogpt_platform/backend/backend/copilot/capabilities/eval/benchmark.pyindex 8cbfcf5844..ae1754006e 100644--- a/autogpt_platform/backend/backend/copilot/capabilities/eval/benchmark.py+++ b/autogpt_platform/backend/backend/copilot/capabilities/eval/benchmark.py@@ -35,6 +35,9 @@ NAMED_CASES: dict[str, tuple[str, set[str]]] = {"send email": ("direct", {"GmailSendBlock", "SendEmailBlock"}),"google sheets add row": ("direct", {"GoogleSheetsAppendRowBlock"}),"OrchestratorBlock": ("graph", {"OrchestratorBlock"}),+ # #14646: "save" reached neither the block that saves to the workspace+ # nor the tool that writes to it.+ "save file output": ("direct", {"FileStoreBlock", "write_workspace_file"}),}diff --git a/autogpt_platform/backend/backend/copilot/capabilities/index.py b/autogpt_platform/backend/backend/copilot/capabilities/index.pyindex 6154cbe4e7..9425645e2c 100644--- a/autogpt_platform/backend/backend/copilot/capabilities/index.py+++ b/autogpt_platform/backend/backend/copilot/capabilities/index.py@@ -276,7 +276,7 @@ def _coverage(groups: list[list[str]], doc: frozenset[str]) -> float:def _document(entry: CapabilityEntry) -> list[str]:tokens = tokenize(entry.name) * _NAME_WEIGHT- tokens += tokenize(entry.purpose)+ tokens += tokenize(entry.description or entry.purpose)for tag in entry.tags:tokens += tokenize(tag) or [tag.lower()]tokens += tokenize(" ".join(entry.argument_names))diff --git a/autogpt_platform/backend/backend/copilot/capabilities/index_test.py b/autogpt_platform/backend/backend/copilot/capabilities/index_test.pyindex 49f3421599..c3235edb6c 100644--- a/autogpt_platform/backend/backend/copilot/capabilities/index_test.py+++ b/autogpt_platform/backend/backend/copilot/capabilities/index_test.py@@ -12,10 +12,29 @@ LINEAR_ID = "11111111-1111-1111-1111-111111111111"HTTP_ID = "22222222-2222-2222-2222-222222222222"GITHUB_ID = "33333333-3333-3333-3333-333333333333"INPUT_ID = "44444444-4444-4444-4444-444444444444"+FILE_ID = "55555555-5555-5555-5555-555555555555"++# Four sentences, of which ``clip_purpose`` keeps the first two: the one that+# says what the block does in CoPilot is the one a "save" query needs.+FILE_STORE_DESCRIPTION = (+ "Downloads and stores a file from a URL, data URI, or local path. "+ "Use this to fetch images, documents, or other files for processing. "+ "In CoPilot: saves to workspace (use list_workspace_files to see it). "+ "In graphs: outputs a data URI to pass to other blocks."+)def _block(- block_id, name, purpose, *, provider=None, klass="service", context="both", args=()+ block_id,+ name,+ purpose,+ *,+ provider=None,+ klass="service",+ context="both",+ args=(),+ description="",+ tags=("issue tracking",),):connection = (Connection(required=True, key_type="provider", key=provider)@@ -28,7 +47,8 @@ def _block(klass=klass,name=name,purpose=purpose,- tags=[*([provider] if provider else []), "issue tracking"],+ description=description,+ tags=[*([provider] if provider else []), *tags],context=context,implementations=[Implementation(kind="block", ref=block_id)],connection=connection,@@ -62,6 +82,33 @@ def index() -> CapabilityIndex:args=("url", "method", "body"),),_block(INPUT_ID, "AgentInputBlock", "Graph input.", context="graph"),+ _block(+ FILE_ID,+ "FileStoreBlock",+ "Downloads and stores a file from a URL, data URI, or local path.",+ klass="primitive",+ args=("file_in",),+ description=FILE_STORE_DESCRIPTION,+ tags=("file",),+ ),+ CapabilityEntry(+ id="tool:read_workspace_file",+ kind="tool",+ name="read_workspace_file",+ purpose="Read a file from persistent workspace.",+ description=(+ "Read a file from persistent workspace. Use save_to_path "+ "to copy to working dir for processing."+ ),+ tags=["file", "read", "workspace"],+ context="direct",+ implementations=[+ Implementation(+ kind="tool", ref="read_workspace_file", context="direct"+ )+ ],+ argument_names=["path", "save_to_path"],+ ),CapabilityEntry(id="tool:web_search",kind="tool",@@ -163,6 +210,23 @@ def test_empty_and_nonsense_queries(index):assert index.search("zzzz qqqq").hits == []+def test_description_past_the_listed_purpose_is_indexed_but_not_listed(index):+ """ "save file output" names FileStoreBlock only through the sentence+ ``clip_purpose`` drops; indexing the whole description finds it, and+ the listing the model reads stays the clipped purpose."""+ result = index.search("save file output")+ assert result.names[0] == "FileStoreBlock"+ assert "description" not in result.hits[0].entry.listing()+ assert result.hits[0].entry.listing()["purpose"] != FILE_STORE_DESCRIPTION+++def test_save_query_reaches_a_store_entry_through_the_synonym(index):+ hits = index.search("save a file").hits+ store = next(h for h in hits if h.entry.name == "FileStoreBlock")+ assert store.coverage == 2.0 # "save" as written (description), "file"+ assert store.entry in [h.entry for h in hits[:2]]++def test_eager_tools_are_flagged_in_listing(index):hit = index.search("web search").hits[0]assert hit.entry.listing()["eager"] is Truediff --git a/autogpt_platform/backend/backend/copilot/capabilities/models.py b/autogpt_platform/backend/backend/copilot/capabilities/models.pyindex 02def85738..4389daa9f8 100644--- a/autogpt_platform/backend/backend/copilot/capabilities/models.py+++ b/autogpt_platform/backend/backend/copilot/capabilities/models.py@@ -2,8 +2,8 @@One :class:`CapabilityEntry` describes one thing the copilot can do: aplatform tool, a block, an MCP server from the catalog, or (phase 2) a-library agent. The registry indexes entries by name, purpose and tags;-argument schemas are fetched on demand and never indexed.+library agent. The registry indexes entries by name, description and+tags; argument schemas are fetched on demand and never indexed."""from __future__ import annotations@@ -18,6 +18,10 @@ CapabilityContext = Literal["direct", "graph", "both"]ConnectionKeyType = Literal["provider", "server_url", "host", "none"]PURPOSE_MAX_CHARS = 160+# Bounds the index-only description: room for any real block, tool or catalog+# text (the longest block description is under 1,000 chars), and a ceiling on+# what a runaway one can add to the index and its memory.+DESCRIPTION_MAX_CHARS = 4000class Implementation(BaseModel):@@ -55,6 +59,11 @@ class CapabilityEntry(BaseModel):klass: CapabilityClass = "service"name: strpurpose: str = Field(max_length=PURPOSE_MAX_CHARS)+ # Everything the source says about the capability, for the index only:+ # ``purpose`` is the first sentence or two of it, sized for a listing,+ # and the sentence that names what a block does in CoPilot ("saves to+ # workspace") is often the one clipped away. Never listed.+ description: str = ""tags: list[str] = Field(default_factory=list)context: CapabilityContext = "both"implementations: list[Implementation] = Field(default_factory=list)@@ -90,9 +99,21 @@ class CapabilityEntry(BaseModel):return self.context == "both" or context == "both" or self.context == context+def normalize_text(text: str | None, limit: int = DESCRIPTION_MAX_CHARS) -> str:+ """*text* with its whitespace collapsed and cut at *limit* on a word+ boundary, for the index."""+ text = " ".join((text or "").split())+ if len(text) <= limit:+ return text+ # One past the limit, so a word that ends exactly there is kept whole.+ cut = text[: limit + 1]+ idx = cut.rfind(" ")+ return (cut[:idx] if idx >= limit // 2 else cut[:limit]).rstrip()++def clip_purpose(text: str | None, limit: int = PURPOSE_MAX_CHARS) -> str:"""First sentence-ish of *text*, at most *limit* characters."""- text = " ".join((text or "").split())+ text = normalize_text(text)if len(text) <= limit:return textcut = text[: limit - 1]diff --git a/autogpt_platform/backend/backend/copilot/capabilities/models_test.py b/autogpt_platform/backend/backend/copilot/capabilities/models_test.pyindex d256870c17..c440f90751 100644--- a/autogpt_platform/backend/backend/copilot/capabilities/models_test.py+++ b/autogpt_platform/backend/backend/copilot/capabilities/models_test.py@@ -1,4 +1,4 @@-from .models import CapabilityEntry, Connection, clip_purpose+from .models import CapabilityEntry, Connection, clip_purpose, normalize_textdef test_clip_purpose_keeps_short_text_verbatim():@@ -16,9 +16,21 @@ def test_clip_purpose_falls_back_to_word_boundary_with_ellipsis():assert clipped.endswith("…") and len(clipped) <= 160+def test_normalize_text_collapses_whitespace_and_bounds_the_index_text():+ assert normalize_text(" Send an\nemail. ") == "Send an email."+ text = "word " * 3000+ bounded = normalize_text(text)+ assert len(bounded) <= 4000 and bounded.endswith("word")+ assert normalize_text("alpha beta gamma", limit=10) == "alpha beta"++def test_listing_is_compact_and_only_shows_connection_when_required():entry = CapabilityEntry(- id="tool:web_search", kind="tool", name="web_search", purpose="Search."+ id="tool:web_search",+ kind="tool",+ name="web_search",+ purpose="Search.",+ description="Search. Returns the top results for a query.",)assert entry.listing() == {"id": "tool:web_search",diff --git a/autogpt_platform/backend/backend/copilot/capabilities/registry.py b/autogpt_platform/backend/backend/copilot/capabilities/registry.pyindex 7f1bc87cee..6deb4407b7 100644--- a/autogpt_platform/backend/backend/copilot/capabilities/registry.py+++ b/autogpt_platform/backend/backend/copilot/capabilities/registry.py@@ -105,6 +105,9 @@ def _merge(entries: list[CapabilityEntry]) -> list[CapabilityEntry]:continuetool.implementations += block.implementationstool.context = "both"+ tool.description = " ".join(+ text for text in (tool.description, block.description) if text+ )tool.tags = sorted(set(tool.tags) | set(block.tags))tool.argument_names = list(dict.fromkeys(tool.argument_names + block.argument_names)diff --git a/autogpt_platform/backend/backend/copilot/capabilities/registry_test.py b/autogpt_platform/backend/backend/copilot/capabilities/registry_test.pyindex fcaeefa810..a419f1ae2f 100644--- a/autogpt_platform/backend/backend/copilot/capabilities/registry_test.py+++ b/autogpt_platform/backend/backend/copilot/capabilities/registry_test.py@@ -4,6 +4,7 @@ import jsonimport pytest+from backend.blocks import get_blocksfrom backend.copilot.tools import TOOL_GROUPS, TOOL_REGISTRYfrom backend.integrations.mcp_catalog import get_mcp_catalog@@ -11,6 +12,7 @@ from .index import CapabilityIndexfrom .registry import MERGED_IMPLEMENTATIONS, build_entriesfrom .resolve import resolve_entryfrom .sources import RETIRED_TOOLS+from .sources.blocks import _block_entryfrom .sources.mcp_catalog import setup_hintEXPLICIT_PRIMITIVES = {@@ -52,6 +54,10 @@ def test_bash_exec_and_execute_code_block_are_one_capability(index):folded = MERGED_IMPLEMENTATIONS["tool:bash_exec"]assert all(e.name != folded for e in index.entries)assert index.search("execute python code").names[0] == "bash_exec"+ # The folded block's description joins the tool's, so a query phrased+ # for the block ("sandbox") still lands on the one capability.+ assert "sandbox" in entry.description+ assert index.search("code sandbox").names[0] == "bash_exec"def test_block_capability_kind_drives_class(index):@@ -66,6 +72,21 @@ def test_block_capability_kind_drives_class(index):assert by_name["SendAuthenticatedWebRequestBlock"].connection.key_type == "host"+def test_optimized_description_is_indexed_alongside_the_source_text():+ """``optimized_description`` is curated for retrieval and loaded from the+ database at runtime, so no block carries one in CI; indexing only the+ source description would drop it from the index in production."""+ block = next(+ cls() for cls in get_blocks().values() if cls.__name__ == "FileStoreBlock"+ )+ block.optimized_description = "Curated: keeps a download in the workspace."+ entry = _block_entry(block)+ assert entry.purpose.startswith("Curated:")+ assert (+ "Curated" in entry.description and "Downloads and stores" in entry.description+ )++def test_graph_only_blocks_keep_graph_context(index):assert "AgentInputBlock" not in index.search("AgentInputBlock").namesassert (diff --git a/autogpt_platform/backend/backend/copilot/capabilities/sources/blocks.py b/autogpt_platform/backend/backend/copilot/capabilities/sources/blocks.pyindex ce5d4e6183..a9e3b6bee2 100644--- a/autogpt_platform/backend/backend/copilot/capabilities/sources/blocks.py+++ b/autogpt_platform/backend/backend/copilot/capabilities/sources/blocks.py@@ -19,6 +19,7 @@ from backend.copilot.capabilities.models import (Connection,Implementation,clip_purpose,+ normalize_text,)from backend.copilot.capabilities.text import tokenizefrom backend.data.model import CredentialsFieldInfo@@ -69,6 +70,15 @@ def _block_entry(block: AnyBlockSchema) -> CapabilityEntry:klass=block.capability_kind,name=block.name,purpose=clip_purpose(block.optimized_description or block.description),+ # The optimized description is curated for retrieval (it is loaded from+ # the database at runtime), so the index sees it as well as the source.+ description=normalize_text(+ " ".join(+ text+ for text in (block.optimized_description, block.description)+ if text+ )+ ),tags=tags,context="graph" if graph_only else "both",implementations=[Implementation(kind="block", ref=block.id, name=block.name)],diff --git a/autogpt_platform/backend/backend/copilot/capabilities/sources/mcp_catalog.py b/autogpt_platform/backend/backend/copilot/capabilities/sources/mcp_catalog.pyindex 2ba7f41ca5..baddedfd7b 100644--- a/autogpt_platform/backend/backend/copilot/capabilities/sources/mcp_catalog.py+++ b/autogpt_platform/backend/backend/copilot/capabilities/sources/mcp_catalog.py@@ -14,6 +14,7 @@ from backend.copilot.capabilities.models import (Connection,Implementation,clip_purpose,+ normalize_text,)from backend.copilot.capabilities.text import tokenizefrom backend.integrations.mcp_catalog import MCPCatalogEntry, get_mcp_catalog@@ -57,6 +58,7 @@ def _catalog_entry(preset: MCPCatalogEntry, keyed_by_host: bool) -> CapabilityEnklass="service",name=preset.display_name,purpose=clip_purpose(preset.description),+ description=normalize_text(preset.description),tags=tags,context="direct",implementations=[diff --git a/autogpt_platform/backend/backend/copilot/capabilities/sources/static_tools.py b/autogpt_platform/backend/backend/copilot/capabilities/sources/static_tools.pyindex b9ff026189..5716930c0c 100644--- a/autogpt_platform/backend/backend/copilot/capabilities/sources/static_tools.py+++ b/autogpt_platform/backend/backend/copilot/capabilities/sources/static_tools.py@@ -12,6 +12,7 @@ from backend.copilot.capabilities.models import (CapabilityEntry,Implementation,clip_purpose,+ normalize_text,)from backend.copilot.capabilities.text import tokenize@@ -88,6 +89,7 @@ def _tool_entry(name: str, tool: "BaseTool", group: str | None) -> CapabilityEntklass="service",name=name,purpose=clip_purpose(tool.description),+ description=normalize_text(tool.description),tags=tags,context="direct",implementations=[diff --git a/autogpt_platform/backend/backend/copilot/capabilities/text.py b/autogpt_platform/backend/backend/copilot/capabilities/text.pyindex 49c7131b26..efb50dc631 100644--- a/autogpt_platform/backend/backend/copilot/capabilities/text.py+++ b/autogpt_platform/backend/backend/copilot/capabilities/text.py@@ -63,6 +63,12 @@ SYNONYMS: dict[str, tuple[str, ...]] = {"stringify": ("json", "encode"),"count": ("length",),"fetch": ("get", "read"),+ # Saving, storing and writing are one intent spread over three entries+ # (FileStoreBlock, write_workspace_file, memory_store); without the link+ # "save a file" reached neither of the first two.+ "save": ("store", "write"),+ "store": ("save", "write"),+ "write": ("save", "store"),"scrape": ("extract", "crawl"),"webpage": ("website", "web", "page"),"http": ("web",),diff --git a/autogpt_platform/backend/backend/copilot/capabilities/text_test.py b/autogpt_platform/backend/backend/copilot/capabilities/text_test.pyindex af1167ab9d..f398aa799c 100644--- a/autogpt_platform/backend/backend/copilot/capabilities/text_test.py+++ b/autogpt_platform/backend/backend/copilot/capabilities/text_test.py@@ -36,5 +36,15 @@ def test_query_groups_expand_synonyms_once_per_concept():assert groups[2] == [stem("code")]+def test_query_groups_link_save_store_and_write():+ """Saving, storing and writing are one intent across three entries+ (FileStoreBlock, write_workspace_file, memory_store); a query using any+ of the verbs must reach the entries described with the others."""+ for verb in ("save", "store", "write"):+ (group,) = query_groups(verb)+ assert group[0] == stem(verb)+ assert {stem("save"), stem("store"), stem("write")} <= set(group)++def test_normalize_name():assert normalize_name("Orchestrator Block") == "orchestratorblock"