AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.
refactor(backend): make builder search faster (#13290)
Reinier van der Leer76ec6be7e107
diff --git a/autogpt_platform/backend/backend/api/features/builder/db.py b/autogpt_platform/backend/backend/api/features/builder/db.pyindex 3ca1d34671..8a39e18390 100644--- a/autogpt_platform/backend/backend/api/features/builder/db.py+++ b/autogpt_platform/backend/backend/api/features/builder/db.py@@ -1,7 +1,8 @@+import asyncioimport loggingfrom dataclasses import dataclassfrom difflib import SequenceMatcher-from typing import Any, Sequence, get_args, get_origin+from typing import Any, Awaitable, Sequence, get_args, get_originimport prismafrom prisma.models import mv_suggested_blocks@@ -45,6 +46,9 @@ MIN_SCORE_FOR_FILTERED_RESULTS = 10.0# Boost blocks over marketplace agents in search resultsBLOCK_SCORE_BOOST = 50.0+# Bonus when a block exposes an LlmModel field and the query names an LLM model+LLM_MODEL_MATCH_BONUS = 20.0+# Block IDs to exclude from search resultsEXCLUDED_BLOCK_IDS = frozenset({@@ -69,6 +73,20 @@ class _SearchCacheEntry:total_items: dict[FilterType, int]+@dataclass(frozen=True)+class _BlockIndexEntry:+ block_info: BlockInfo+ normalized_name: str # split_camelcase(name).lower(); query-path sort_key+ name_sort_key: str # block_info.name.lower(); no-query-path sort_key+ searchable_text: str+ is_integration: bool+ has_llm_model: bool+++# (scored items, per-filter result counts) returned by a single search branch.+_SearchBranchResult = tuple[list[_ScoredItem], dict[FilterType, int]]++def get_block_categories(category_blocks: int = 3) -> list[BlockCategoryResponse]:categories: dict[BlockCategory, BlockCategoryResponse] = {}@@ -260,7 +278,6 @@ async def _build_cached_search_results(include_library_agents = "my_agents" in filtersinclude_marketplace_agents = "marketplace_agents" in filters- scored_items: list[_ScoredItem] = []total_items: dict[FilterType, int] = {"blocks": 0,"integrations": 0,@@ -268,59 +285,32 @@ async def _build_cached_search_results("my_agents": 0,}- # Use hybrid search when query is present, otherwise list all blocks- if (include_blocks or include_integrations) and normalized_query:- block_results, block_total, integration_total = await _text_search_blocks(- query=search_query,- include_blocks=include_blocks,- include_integrations=include_integrations,- )- scored_items.extend(block_results)- total_items["blocks"] = block_total- total_items["integrations"] = integration_total- elif include_blocks or include_integrations:- # No query - list all blocks using in-memory approach- block_results, block_total, integration_total = _collect_block_results(- include_blocks=include_blocks,- include_integrations=include_integrations,- )- scored_items.extend(block_results)- total_items["blocks"] = block_total- total_items["integrations"] = integration_total-- if include_library_agents:- library_response = await library_db.list_library_agents(- user_id=user_id,- search_term=search_query or None,- page=1,- page_size=MAX_LIBRARY_AGENT_RESULTS,- # Hide trigger agents — they're parent-coupled, not- # generally reusable as a sub-agent block.- is_hidden=False,- )- total_items["my_agents"] = library_response.pagination.total_items- scored_items.extend(- _build_library_items(- agents=library_response.agents,- normalized_query=normalized_query,+ # Run the independent sub-searches concurrently: block scoring is CPU-bound+ # (offloaded to a thread so it doesn't block the event loop), while library+ # and marketplace are independent DB queries.+ branches: list[Awaitable[_SearchBranchResult]] = []+ if include_blocks or include_integrations:+ branches.append(+ asyncio.to_thread(+ _search_blocks,+ search_query,+ normalized_query,+ include_blocks,+ include_integrations,))-+ if include_library_agents:+ branches.append(_search_library(user_id, search_query, normalized_query))if include_marketplace_agents:- marketplace_response = await store_db.get_store_agents(- creators=list(by_creator) or None,- search_query=search_query or None,- page=1,- page_size=MAX_MARKETPLACE_AGENT_RESULTS,- )- total_items["marketplace_agents"] = marketplace_response.pagination.total_items- scored_items.extend(- _build_marketplace_items(- agents=marketplace_response.agents,- normalized_query=normalized_query,- )+ branches.append(+ _search_marketplace(list(by_creator), search_query, normalized_query))+ scored_items: list[_ScoredItem] = []+ for items, totals in await asyncio.gather(*branches):+ scored_items.extend(items)+ total_items.update(totals)+sorted_items = sorted(scored_items,key=lambda entry: (-entry.score, entry.sort_key, entry.filter_type),@@ -332,129 +322,198 @@ async def _build_cached_search_results()-def _collect_block_results(- *,+def _search_blocks(+ search_query: str,+ normalized_query: str,include_blocks: bool,include_integrations: bool,-) -> tuple[list[_ScoredItem], int, int]:+) -> _SearchBranchResult:"""- Collect all blocks for listing (no search query).-- All blocks get BLOCK_SCORE_BOOST to prioritize them over marketplace agents.+ Block/integration sub-search: text search when a query is present,+ otherwise the full block listing. Returns the scored items and the+ per-filter ("blocks"/"integrations") result counts."""- results: list[_ScoredItem] = []- block_count = 0- integration_count = 0+ if normalized_query:+ results, block_total, integration_total = _text_search_blocks(+ query=search_query,+ include_blocks=include_blocks,+ include_integrations=include_integrations,+ )+ else:+ results, block_total, integration_total = _collect_block_results(+ include_blocks=include_blocks,+ include_integrations=include_integrations,+ )+ return results, {"blocks": block_total, "integrations": integration_total}- if not include_blocks and not include_integrations:- return results, block_count, integration_count- for block_type in load_all_blocks().values():- block: AnyBlockSchema = block_type()- if block.disabled:- continue+async def _search_library(+ user_id: str,+ search_query: str,+ normalized_query: str,+) -> _SearchBranchResult:+ library_response = await library_db.list_library_agents(+ user_id=user_id,+ search_term=search_query or None,+ page=1,+ page_size=MAX_LIBRARY_AGENT_RESULTS,+ # Hide trigger agents — they're parent-coupled, not+ # generally reusable as a sub-agent block.+ is_hidden=False,+ )+ items = _build_library_items(+ agents=library_response.agents,+ normalized_query=normalized_query,+ )+ return items, {"my_agents": library_response.pagination.total_items}- # Skip excluded blocks- if block.id in EXCLUDED_BLOCK_IDS:- continue- block_info = block.get_info()- credentials = list(block.input_schema.get_credentials_fields().values())- is_integration = len(credentials) > 0+async def _search_marketplace(+ by_creator: list[str],+ search_query: str,+ normalized_query: str,+) -> _SearchBranchResult:+ marketplace_response = await store_db.get_store_agents(+ creators=by_creator or None,+ search_query=search_query or None,+ page=1,+ page_size=MAX_MARKETPLACE_AGENT_RESULTS,+ )+ items = _build_marketplace_items(+ agents=marketplace_response.agents,+ normalized_query=normalized_query,+ )+ return items, {"marketplace_agents": marketplace_response.pagination.total_items}- if is_integration and not include_integrations:- continue- if not is_integration and not include_blocks:- continue- filter_type: FilterType = "integrations" if is_integration else "blocks"- if is_integration:- integration_count += 1- else:- block_count += 1+def _collect_block_results(+ *,+ include_blocks: bool,+ include_integrations: bool,+) -> tuple[list[_ScoredItem], int, int]:+ """+ Collect all blocks for listing (no search query).- results.append(- _ScoredItem(- item=block_info,- filter_type=filter_type,- score=BLOCK_SCORE_BOOST,- sort_key=block_info.name.lower(),- )+ All blocks get BLOCK_SCORE_BOOST to prioritize them over marketplace agents.+ """+ if not include_blocks and not include_integrations:+ return [], 0, 0++ results = [+ _ScoredItem(+ item=entry.block_info,+ filter_type="integrations" if entry.is_integration else "blocks",+ score=BLOCK_SCORE_BOOST,+ sort_key=entry.name_sort_key,)-+ for entry in _get_block_search_index()+ if _entry_included(entry, include_blocks, include_integrations)+ ]+ block_count = sum(1 for r in results if r.filter_type == "blocks")+ integration_count = sum(1 for r in results if r.filter_type == "integrations")return results, block_count, integration_count-async def _text_search_blocks(+def _text_search_blocks(*,query: str,include_blocks: bool,include_integrations: bool,) -> tuple[list[_ScoredItem], int, int]:"""- Search blocks using in-memory text matching over the block registry.-- All blocks are already loaded in memory, so this is fast and reliable- regardless of whether OpenAI embeddings are available.+ Search blocks using in-memory text matching over the cached block index.Scoring:- Base: text relevance via _score_primary_fields, plus BLOCK_SCORE_BOOSTto prioritize blocks over marketplace agents in combined results- +20 if the block has an LlmModel field and the query matches an LLM model name"""- results: list[_ScoredItem] = []-if not include_blocks and not include_integrations:- return results, 0, 0+ return [], 0, 0normalized_query = query.strip().lower()+ results = [+ scored+ for entry in _get_block_search_index()+ if _entry_included(entry, include_blocks, include_integrations)+ if (scored := _score_block_entry(entry, normalized_query)) is not None+ ]+ block_count = sum(1 for r in results if r.filter_type == "blocks")+ integration_count = sum(1 for r in results if r.filter_type == "integrations")+ return results, block_count, integration_count+- all_results, _, _ = _collect_block_results(- include_blocks=include_blocks,- include_integrations=include_integrations,+def _score_block_entry(+ entry: _BlockIndexEntry, normalized_query: str+) -> _ScoredItem | None:+ score = _score_primary_fields(+ entry.normalized_name, entry.searchable_text, normalized_query+ )+ # Add LLM model match bonus+ if entry.has_llm_model and _query_matches_llm_model(normalized_query):+ score += LLM_MODEL_MATCH_BONUS++ if score < MIN_SCORE_FOR_FILTERED_RESULTS:+ return None++ return _ScoredItem(+ item=entry.block_info,+ filter_type="integrations" if entry.is_integration else "blocks",+ score=score + BLOCK_SCORE_BOOST,+ sort_key=entry.normalized_name,)- all_blocks = load_all_blocks()-- for item in all_results:- block_info = item.item- assert isinstance(block_info, BlockInfo)- name = split_camelcase(block_info.name).lower()-- # Build rich description including input field descriptions,- # matching the searchable text that the embedding pipeline uses- desc_parts = [block_info.description or ""]- block_cls = all_blocks.get(block_info.id)- if block_cls is not None:- block: AnyBlockSchema = block_cls()- desc_parts += [- f"{f}: {info.description}"- for f, info in block.input_schema.model_fields.items()- if info.description- ]- description = " ".join(desc_parts).lower()-- score = _score_primary_fields(name, description, normalized_query)-- # Add LLM model match bonus- if block_cls is not None and _matches_llm_model(- block_cls().input_schema, normalized_query- ):- score += 20-- if score >= MIN_SCORE_FOR_FILTERED_RESULTS:- results.append(- _ScoredItem(- item=block_info,- filter_type=item.filter_type,- score=score + BLOCK_SCORE_BOOST,- sort_key=name,- )- )- block_count = sum(1 for r in results if r.filter_type == "blocks")- integration_count = sum(1 for r in results if r.filter_type == "integrations")- return results, block_count, integration_count+@cached(ttl_seconds=3600)+def _get_block_search_index() -> tuple[_BlockIndexEntry, ...]:+ """+ Build a lightweight, in-memory index of all searchable blocks once per hour.++ Each block is instantiated a single time here; search requests then score+ against the precomputed fields without re-instantiating blocks or rebuilding+ BlockInfo per request.+ """+ blocks = (block_type() for block_type in load_all_blocks().values())+ entries = [+ _build_block_index_entry(block)+ for block in blocks+ if not block.disabled and block.id not in EXCLUDED_BLOCK_IDS+ ]+ return tuple(entries)+++def _build_block_index_entry(block: AnyBlockSchema) -> _BlockIndexEntry:+ block_info = block.get_info()+ return _BlockIndexEntry(+ block_info=block_info,+ normalized_name=split_camelcase(block_info.name).lower(),+ name_sort_key=block_info.name.lower(),+ searchable_text=_block_searchable_text(block),+ is_integration=bool(block.input_schema.get_credentials_fields()),+ has_llm_model=_schema_has_llm_model(block.input_schema),+ )+++def _entry_included(+ entry: _BlockIndexEntry, include_blocks: bool, include_integrations: bool+) -> bool:+ if entry.is_integration:+ return include_integrations+ return include_blocks+++def _block_searchable_text(block: AnyBlockSchema) -> str:+ """+ Build searchable text for a block: its description plus the descriptions of+ its input fields, matching the text the embedding pipeline indexes.+ """+ parts = [block.description or ""]+ parts += [+ f"{field}: {info.description}"+ for field, info in block.input_schema.model_fields.items()+ if info.description+ ]+ return " ".join(parts).lower()def _build_library_items(@@ -613,13 +672,15 @@ def _contains_type(annotation: Any, target: type) -> bool:return any(_contains_type(arg, target) for arg in get_args(annotation))-def _matches_llm_model(schema_cls: type[BlockSchema], query: str) -> bool:- for field in schema_cls.model_fields.values():- if _contains_type(field.annotation, LlmModel):- # Check if query matches any value in llm_models- if any(query in name for name in llm_models):- return True- return False+def _schema_has_llm_model(schema_cls: type[BlockSchema]) -> bool:+ return any(+ _contains_type(field.annotation, LlmModel)+ for field in schema_cls.model_fields.values()+ )+++def _query_matches_llm_model(query: str) -> bool:+ return any(query in name for name in llm_models)def _score_library_agent(diff --git a/autogpt_platform/backend/backend/api/features/builder/db_test.py b/autogpt_platform/backend/backend/api/features/builder/db_test.pynew file mode 100644index 0000000000..20cde07898--- /dev/null+++ b/autogpt_platform/backend/backend/api/features/builder/db_test.py@@ -0,0 +1,282 @@+"""Unit tests for builder search.++Covers the block scoring/index logic that the /builder/search endpoint+depends on. The behavioural tests patch the in-memory block index with a+small set of fake entries so they're deterministic and don't depend on the+real ~400-block registry; one test exercises the real index builder.+"""++from types import SimpleNamespace+from typing import Any++import pytest++import backend.api.features.builder.db as db+from backend.blocks._base import BlockInfo+from backend.util.text import split_camelcase+++def _entry(+ name: str,+ normalized_name: str,+ searchable_text: str,+ *,+ is_integration: bool = False,+ has_llm_model: bool = False,+) -> db._BlockIndexEntry:+ return db._BlockIndexEntry(+ block_info=BlockInfo.model_construct(id=name, name=name),+ normalized_name=normalized_name,+ name_sort_key=name.lower(),+ searchable_text=searchable_text,+ is_integration=is_integration,+ has_llm_model=has_llm_model,+ )+++def _patch_index(mocker, entries: list[db._BlockIndexEntry]) -> None:+ mocker.patch.object(+ db,+ "_get_block_search_index",+ return_value=tuple(entries),+ )+++# ============================================================================+# Scoring formula — locked so "ranking identical" can't silently regress+# ============================================================================+++def test_score_primary_fields_regression():+ # name.startswith(query) -> +90, plus SequenceMatcher(name, query) contribution+ assert (+ db._score_primary_fields("youtube transcript", "get the transcript", "youtube")+ == 120.0+ )+ # unrelated name/description -> small fuzzy-only score, below threshold+ assert db._score_primary_fields("foo bar", "baz", "xyzzy") == 6.25+ # exact name match is the top of the scale+ assert db._score_primary_fields("exact", "some desc", "exact") == pytest.approx(+ 177.14285714285714+ )+++# ============================================================================+# Block text search — matching, filtering, thresholds+# ============================================================================+++def test_text_search_matches_expected_block(mocker):+ _patch_index(+ mocker,+ [+ _entry("YoutubeTranscript", "youtube transcript", "get the transcript"),+ _entry("WeatherForecast", "weather forecast", "current weather conditions"),+ _entry("SendEmail", "send email", "send an email message"),+ ],+ )++ results, blocks, integrations = db._text_search_blocks(+ query="youtube", include_blocks=True, include_integrations=True+ )++ assert [r.item.name for r in results] == ["YoutubeTranscript"]+ assert (blocks, integrations) == (1, 0)+++def test_integration_filter(mocker):+ _patch_index(+ mocker,+ [+ _entry("PlainBlock", "plain block", "does things", is_integration=False),+ _entry("ApiBlock", "api block", "calls an api", is_integration=True),+ ],+ )++ blocks_only, b_count, i_count = db._collect_block_results(+ include_blocks=True, include_integrations=False+ )+ assert [r.item.name for r in blocks_only] == ["PlainBlock"]+ assert (b_count, i_count) == (1, 0)++ integrations_only, b_count, i_count = db._collect_block_results(+ include_blocks=False, include_integrations=True+ )+ assert [r.item.name for r in integrations_only] == ["ApiBlock"]+ assert (b_count, i_count) == (0, 1)+++def test_threshold_filtering(mocker):+ # A query that only weakly fuzzy-matches scores below+ # MIN_SCORE_FOR_FILTERED_RESULTS and is dropped.+ _patch_index(+ mocker,+ [+ _entry("Calculator", "calculator", "add two numbers together"),+ _entry("Note", "note", "a sticky note"),+ ],+ )+ below, _, _ = db._text_search_blocks(+ query="xyzzy", include_blocks=True, include_integrations=True+ )+ assert below == []++ # An exact-name query clears the threshold.+ _patch_index(mocker, [_entry("ExactName", "exact", "some desc")])+ above, _, _ = db._text_search_blocks(+ query="exact", include_blocks=True, include_integrations=True+ )+ assert [r.item.name for r in above] == ["ExactName"]+++def test_query_results_are_subset_of_full_listing(mocker):+ entries = [+ _entry("YoutubeTranscript", "youtube transcript", "get the transcript"),+ _entry("WeatherForecast", "weather forecast", "current weather conditions"),+ ]+ _patch_index(mocker, entries)++ listed, _, _ = db._collect_block_results(+ include_blocks=True, include_integrations=True+ )+ matched, _, _ = db._text_search_blocks(+ query="youtube", include_blocks=True, include_integrations=True+ )++ assert {r.item.name for r in listed} == {"YoutubeTranscript", "WeatherForecast"}+ assert {r.item.name for r in matched} == {"YoutubeTranscript"}+ assert len(matched) < len(listed)+++# ============================================================================+# LLM-model bonus — the build-time/query-time split must preserve the +20+# ============================================================================+++def test_query_matches_llm_model():+ a_model_name = db.llm_models[0]+ assert db._query_matches_llm_model(a_model_name) is True+ assert db._query_matches_llm_model("definitely-not-a-model-name") is False+++def test_llm_model_bonus_adds_twenty():+ # Use a real model name as both the block name and the query so the base+ # score clears the threshold and the only difference is the LLM bonus.+ model_query = db.llm_models[0]+ with_llm = _entry("M", model_query, "", has_llm_model=True)+ without_llm = _entry("M", model_query, "", has_llm_model=False)++ scored_with = db._score_block_entry(with_llm, model_query)+ scored_without = db._score_block_entry(without_llm, model_query)++ assert scored_with is not None and scored_without is not None+ assert scored_with.score - scored_without.score == 20+++# ============================================================================+# Index builder — exercised against the real block registry+# ============================================================================+++def test_index_excludes_disabled_and_excluded_blocks():+ db._get_block_search_index.cache_clear()+ entries = db._get_block_search_index()+ db._get_block_search_index.cache_clear()++ assert len(entries) > 0+ entry_ids = {entry.block_info.id for entry in entries}+ assert entry_ids.isdisjoint(db.EXCLUDED_BLOCK_IDS)++ sample = entries[0]+ assert sample.normalized_name+ assert sample.block_info.name+ assert sample.normalized_name == split_camelcase(sample.block_info.name).lower()+++# ============================================================================+# Orchestration — concurrent branches are all merged (gather regression guard)+# ============================================================================+++async def test_build_cached_search_results_merges_all_branches(mocker):+ _patch_index(+ mocker,+ [_entry("YoutubeTranscript", "youtube transcript", "get the transcript")],+ )+ library_item = db._ScoredItem(+ item=BlockInfo.model_construct(id="lib", name="LibAgent"),+ filter_type="my_agents",+ score=100.0,+ sort_key="libagent",+ )+ marketplace_item = db._ScoredItem(+ item=BlockInfo.model_construct(id="mkt", name="StoreAgent"),+ filter_type="marketplace_agents",+ score=80.0,+ sort_key="storeagent",+ )+ mock_library = mocker.patch.object(+ db, "_search_library", return_value=([library_item], {"my_agents": 3})+ )+ mock_marketplace = mocker.patch.object(+ db,+ "_search_marketplace",+ return_value=([marketplace_item], {"marketplace_agents": 9}),+ )++ # Call the undecorated function to bypass the Redis-backed @cached wrapper.+ result = await db._build_cached_search_results.__wrapped__(+ "user-1",+ "youtube",+ ("blocks", "integrations", "marketplace_agents", "my_agents"),+ (),+ )++ # Items are merged and ordered by descending score (170 > 100 > 80).+ assert [item.name for item in result.items] == [+ "YoutubeTranscript",+ "LibAgent",+ "StoreAgent",+ ]+ assert result.total_items == {+ "blocks": 1,+ "integrations": 0,+ "marketplace_agents": 9,+ "my_agents": 3,+ }+ # Each branch is awaited exactly once with the expected arguments.+ mock_library.assert_awaited_once_with("user-1", "youtube", "youtube")+ mock_marketplace.assert_awaited_once_with([], "youtube", "youtube")+++def test_search_blocks_routes_query_vs_listing(mocker):+ _patch_index(+ mocker,+ [+ _entry("YoutubeTranscript", "youtube transcript", "get the transcript"),+ _entry("WeatherForecast", "weather forecast", "current weather"),+ ],+ )++ # Query present -> text search returns only matching blocks.+ items, totals = db._search_blocks("youtube", "youtube", True, True)+ assert [i.item.name for i in items] == ["YoutubeTranscript"]+ assert totals == {"blocks": 1, "integrations": 0}++ # No query -> full block listing.+ items, totals = db._search_blocks("", "", True, True)+ assert {i.item.name for i in items} == {"YoutubeTranscript", "WeatherForecast"}+ assert totals == {"blocks": 2, "integrations": 0}+++def test_block_searchable_text_includes_field_descriptions():+ block: Any = SimpleNamespace(+ description="My Block",+ input_schema=SimpleNamespace(+ model_fields={+ "keyword": SimpleNamespace(description="A searchable keyword"),+ "plain": SimpleNamespace(description=""), # no description -> skipped+ }+ ),+ )+ assert db._block_searchable_text(block) == "my block keyword: a searchable keyword"