AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.
dx(backend): Make Builder and Marketplace search work without embeddings (#12479)
Otto943a1df815f3
diff --git a/.github/workflows/platform-fullstack-ci.yml b/.github/workflows/platform-fullstack-ci.ymlindex a67f80e414..fc772171b1 100644--- a/.github/workflows/platform-fullstack-ci.yml+++ b/.github/workflows/platform-fullstack-ci.yml@@ -294,7 +294,7 @@ jobs:uses: actions/upload-artifact@v4with:name: playwright-report- path: playwright-report+ path: autogpt_platform/frontend/playwright-reportif-no-files-found: ignoreretention-days: 3@@ -303,7 +303,7 @@ jobs:uses: actions/upload-artifact@v4with:name: playwright-test-results- path: test-results+ path: autogpt_platform/frontend/test-resultsif-no-files-found: ignoreretention-days: 3diff --git a/autogpt_platform/backend/backend/api/features/builder/db.py b/autogpt_platform/backend/backend/api/features/builder/db.pyindex d573b2e1b2..6f6a0cd34c 100644--- a/autogpt_platform/backend/backend/api/features/builder/db.py+++ b/autogpt_platform/backend/backend/api/features/builder/db.py@@ -4,14 +4,12 @@ from difflib import SequenceMatcherfrom typing import Any, Sequence, get_args, get_originimport prisma-from prisma.enums import ContentTypefrom prisma.models import mv_suggested_blocksimport backend.api.features.library.db as library_dbimport backend.api.features.library.model as library_modelimport backend.api.features.store.db as store_dbimport backend.api.features.store.model as store_model-from backend.api.features.store.hybrid_search import unified_hybrid_searchfrom backend.blocks import load_all_blocksfrom backend.blocks._base import (AnyBlockSchema,@@ -24,6 +22,7 @@ from backend.blocks.llm import LlmModelfrom backend.integrations.providers import ProviderNamefrom backend.util.cache import cachedfrom backend.util.models import Pagination+from backend.util.text import split_camelcasefrom .model import (BlockCategoryResponse,@@ -271,7 +270,7 @@ async def _build_cached_search_results(# Use hybrid search when query is present, otherwise list all blocksif (include_blocks or include_integrations) and normalized_query:- block_results, block_total, integration_total = await _hybrid_search_blocks(+ block_results, block_total, integration_total = await _text_search_blocks(query=search_query,include_blocks=include_blocks,include_integrations=include_integrations,@@ -383,117 +382,75 @@ def _collect_block_results(return results, block_count, integration_count-async def _hybrid_search_blocks(+async def _text_search_blocks(*,query: str,include_blocks: bool,include_integrations: bool,) -> tuple[list[_ScoredItem], int, int]:"""- Search blocks using hybrid search with builder-specific filtering.+ Search blocks using in-memory text matching over the block registry.- Uses unified_hybrid_search for semantic + lexical search, then applies- post-filtering for block/integration types and scoring adjustments.+ All blocks are already loaded in memory, so this is fast and reliable+ regardless of whether OpenAI embeddings are available.Scoring:- - Base: hybrid relevance score (0-1) scaled to 0-100, plus BLOCK_SCORE_BOOST+ - Base: text relevance via _score_primary_fields, plus BLOCK_SCORE_BOOSTto prioritize blocks over marketplace agents in combined results- - +30 for exact name match, +15 for prefix name match- +20 if the block has an LlmModel field and the query matches an LLM model name-- Args:- query: The search query string- include_blocks: Whether to include regular blocks- include_integrations: Whether to include integration blocks-- Returns:- Tuple of (scored_items, block_count, integration_count)"""results: list[_ScoredItem] = []- block_count = 0- integration_count = 0if not include_blocks and not include_integrations:- return results, block_count, integration_count+ return results, 0, 0normalized_query = query.strip().lower()- # Fetch more results to account for post-filtering- search_results, _ = await unified_hybrid_search(- query=query,- content_types=[ContentType.BLOCK],- page=1,- page_size=150,- min_score=0.10,+ all_results, _, _ = _collect_block_results(+ include_blocks=include_blocks,+ include_integrations=include_integrations,)- # Load all blocks for getting BlockInfoall_blocks = load_all_blocks()- for result in search_results:- block_id = result["content_id"]-- # Skip excluded blocks- if block_id in EXCLUDED_BLOCK_IDS:- continue-- metadata = result.get("metadata", {})- hybrid_score = result.get("relevance", 0.0)-- # Get the actual block class- if block_id not in all_blocks:- continue-- block_cls = all_blocks[block_id]- block: AnyBlockSchema = block_cls()-- if block.disabled:- continue-- # Check block/integration filter using metadata- is_integration = metadata.get("is_integration", False)-- if is_integration and not include_integrations:- continue- if not is_integration and not include_blocks:- continue-- # Get block info- block_info = block.get_info()-- # Calculate final score: scale hybrid score and add builder-specific bonuses- # Hybrid scores are 0-1, builder scores were 0-200+- # Add BLOCK_SCORE_BOOST to prioritize blocks over marketplace agents- final_score = hybrid_score * 100 + BLOCK_SCORE_BOOST+ for item in all_results:+ block_info = item.item+ assert isinstance(block_info, BlockInfo)+ name = split_camelcase(block_info.name).lower()++ # Build rich description including input field descriptions,+ # matching the searchable text that the embedding pipeline uses+ desc_parts = [block_info.description or ""]+ block_cls = all_blocks.get(block_info.id)+ if block_cls is not None:+ block: AnyBlockSchema = block_cls()+ desc_parts += [+ f"{f}: {info.description}"+ for f, info in block.input_schema.model_fields.items()+ if info.description+ ]+ description = " ".join(desc_parts).lower()++ score = _score_primary_fields(name, description, normalized_query)# Add LLM model match bonus- has_llm_field = metadata.get("has_llm_model_field", False)- if has_llm_field and _matches_llm_model(block.input_schema, normalized_query):- final_score += 20-- # Add exact/prefix match bonus for deterministic tie-breaking- name = block_info.name.lower()- if name == normalized_query:- final_score += 30- elif name.startswith(normalized_query):- final_score += 15-- # Track counts- filter_type: FilterType = "integrations" if is_integration else "blocks"- if is_integration:- integration_count += 1- else:- block_count += 1-- results.append(- _ScoredItem(- item=block_info,- filter_type=filter_type,- score=final_score,- sort_key=name,+ if block_cls is not None and _matches_llm_model(+ block_cls().input_schema, normalized_query+ ):+ score += 20++ if score >= MIN_SCORE_FOR_FILTERED_RESULTS:+ results.append(+ _ScoredItem(+ item=block_info,+ filter_type=item.filter_type,+ score=score + BLOCK_SCORE_BOOST,+ sort_key=name,+ ))- )+ block_count = sum(1 for r in results if r.filter_type == "blocks")+ integration_count = sum(1 for r in results if r.filter_type == "integrations")return results, block_count, integration_countdiff --git a/autogpt_platform/backend/backend/api/features/store/db.py b/autogpt_platform/backend/backend/api/features/store/db.pyindex 2b4d8b12ec..4ae7f561a0 100644--- a/autogpt_platform/backend/backend/api/features/store/db.py+++ b/autogpt_platform/backend/backend/api/features/store/db.py@@ -9,7 +9,7 @@ import prisma.errorsimport prisma.modelsimport prisma.types-from backend.data.db import transaction+from backend.data.db import query_raw_with_schema, transactionfrom backend.data.graph import (GraphModel,GraphModelWithoutNodes,@@ -104,7 +104,8 @@ async def get_store_agents(# search_used_hybrid remains False, will use fallback path below# Convert hybrid search results (dict format) if hybrid succeeded- if search_used_hybrid:+ # Fall through to direct DB search if hybrid returned nothing+ if search_used_hybrid and agents:total_pages = (total + page_size - 1) // page_sizestore_agents: list[store_model.StoreAgent] = []for agent in agents:@@ -130,52 +131,20 @@ async def get_store_agents()continue- if not search_used_hybrid:- # Fallback path - use basic search or no search- where_clause: prisma.types.StoreAgentWhereInput = {"is_available": True}- if featured:- where_clause["featured"] = featured- if creators:- where_clause["creator_username"] = {"in": creators}- if category:- where_clause["categories"] = {"has": category}-- # Add basic text search if search_query provided but hybrid failed- if search_query:- where_clause["OR"] = [- {"agent_name": {"contains": search_query, "mode": "insensitive"}},- {"sub_heading": {"contains": search_query, "mode": "insensitive"}},- {"description": {"contains": search_query, "mode": "insensitive"}},- ]-- order_by = []- if sorted_by == StoreAgentsSortOptions.RATING:- order_by.append({"rating": "desc"})- elif sorted_by == StoreAgentsSortOptions.RUNS:- order_by.append({"runs": "desc"})- elif sorted_by == StoreAgentsSortOptions.NAME:- order_by.append({"agent_name": "asc"})- elif sorted_by == StoreAgentsSortOptions.UPDATED_AT:- order_by.append({"updated_at": "desc"})-- db_agents = await prisma.models.StoreAgent.prisma().find_many(- where=where_clause,- order=order_by,- skip=(page - 1) * page_size,- take=page_size,+ if not search_used_hybrid or not agents:+ # Fallback path: direct DB query with optional tsvector search.+ # This mirrors the original pre-hybrid-search implementation.+ store_agents, total = await _fallback_store_agent_search(+ search_query=search_query,+ featured=featured,+ creators=creators,+ category=category,+ sorted_by=sorted_by,+ page=page,+ page_size=page_size,)-- total = await prisma.models.StoreAgent.prisma().count(where=where_clause)total_pages = (total + page_size - 1) // page_size- store_agents: list[store_model.StoreAgent] = []- for agent in db_agents:- try:- store_agents.append(store_model.StoreAgent.from_db(agent))- except Exception as e:- logger.error(f"Error parsing StoreAgent from db: {e}")- continue-logger.debug(f"Found {len(store_agents)} agents")return store_model.StoreAgentsResponse(agents=store_agents,@@ -195,6 +164,126 @@ async def get_store_agents(# await log_search_term(search_query=search_term)+async def _fallback_store_agent_search(+ *,+ search_query: str | None,+ featured: bool,+ creators: list[str] | None,+ category: str | None,+ sorted_by: StoreAgentsSortOptions | None,+ page: int,+ page_size: int,+) -> tuple[list[store_model.StoreAgent], int]:+ """Direct DB search fallback when hybrid search is unavailable or empty.++ Uses ad-hoc to_tsvector/plainto_tsquery with ts_rank_cd for text search,+ matching the quality of the original pre-hybrid-search implementation.+ Falls back to simple listing when no search query is provided.+ """+ if not search_query:+ # No search query — use Prisma for simple filtered listing+ where_clause: prisma.types.StoreAgentWhereInput = {"is_available": True}+ if featured:+ where_clause["featured"] = featured+ if creators:+ where_clause["creator_username"] = {"in": creators}+ if category:+ where_clause["categories"] = {"has": category}++ order_by = []+ if sorted_by == StoreAgentsSortOptions.RATING:+ order_by.append({"rating": "desc"})+ elif sorted_by == StoreAgentsSortOptions.RUNS:+ order_by.append({"runs": "desc"})+ elif sorted_by == StoreAgentsSortOptions.NAME:+ order_by.append({"agent_name": "asc"})+ elif sorted_by == StoreAgentsSortOptions.UPDATED_AT:+ order_by.append({"updated_at": "desc"})++ db_agents = await prisma.models.StoreAgent.prisma().find_many(+ where=where_clause,+ order=order_by,+ skip=(page - 1) * page_size,+ take=page_size,+ )+ total = await prisma.models.StoreAgent.prisma().count(where=where_clause)+ return [store_model.StoreAgent.from_db(a) for a in db_agents], total++ # Text search using ad-hoc tsvector on StoreAgent view fields+ params: list[Any] = [search_query]+ filters = ["sa.is_available = true"]+ param_idx = 2++ if featured:+ filters.append("sa.featured = true")+ if creators:+ params.append(creators)+ filters.append(f"sa.creator_username = ANY(${param_idx})")+ param_idx += 1+ if category:+ params.append(category)+ filters.append(f"${param_idx} = ANY(sa.categories)")+ param_idx += 1++ where_sql = " AND ".join(filters)++ params.extend([page_size, (page - 1) * page_size])+ limit_param = f"${param_idx}"+ param_idx += 1+ offset_param = f"${param_idx}"++ sql = f"""+ WITH ranked AS (+ SELECT sa.*,+ ts_rank_cd(+ to_tsvector('english',+ COALESCE(sa.agent_name, '') || ' ' ||+ COALESCE(sa.sub_heading, '') || ' ' ||+ COALESCE(sa.description, '')+ ),+ plainto_tsquery('english', $1)+ ) AS rank,+ COUNT(*) OVER () AS total_count+ FROM {{schema_prefix}}"StoreAgent" sa+ WHERE {where_sql}+ AND to_tsvector('english',+ COALESCE(sa.agent_name, '') || ' ' ||+ COALESCE(sa.sub_heading, '') || ' ' ||+ COALESCE(sa.description, '')+ ) @@ plainto_tsquery('english', $1)+ )+ SELECT * FROM ranked+ ORDER BY rank DESC+ LIMIT {limit_param} OFFSET {offset_param}+ """++ results = await query_raw_with_schema(sql, *params)+ total = results[0]["total_count"] if results else 0++ store_agents = []+ for row in results:+ try:+ store_agents.append(+ store_model.StoreAgent(+ slug=row["slug"],+ agent_name=row["agent_name"],+ agent_image=row["agent_image"][0] if row["agent_image"] else "",+ creator=row["creator_username"] or "Needs Profile",+ creator_avatar=row["creator_avatar"] or "",+ sub_heading=row["sub_heading"],+ description=row["description"],+ runs=row["runs"],+ rating=row["rating"],+ agent_graph_id=row.get("graph_id", ""),+ )+ )+ except Exception as e:+ logger.error(f"Error parsing StoreAgent from fallback search: {e}")+ continue++ return store_agents, total++async def log_search_term(search_query: str):"""Log a search term to the database"""@@ -1139,16 +1228,21 @@ async def review_store_submission(},)- # Generate embedding for approved listing (blocking - admin operation)- # Inside transaction: if embedding fails, entire transaction rolls back- await ensure_embedding(- version_id=store_listing_version_id,- name=submission.name,- description=submission.description,- sub_heading=submission.subHeading,- categories=submission.categories,- tx=tx,- )+ # Generate embedding for approved listing (best-effort)+ try:+ await ensure_embedding(+ version_id=store_listing_version_id,+ name=submission.name,+ description=submission.description,+ sub_heading=submission.subHeading,+ categories=submission.categories,+ tx=tx,+ )+ except Exception as emb_err:+ logger.warning(+ f"Could not generate embedding for listing "+ f"{store_listing_version_id}: {emb_err}"+ )await prisma.models.StoreListing.prisma(tx).update(where={"id": submission.storeListingId},