Skip to content

AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.

feat(platform): global search endpoint + bucketed command-palette modal (#13217)

Abhimanyu Yadavb3607cd0efc3
diff --git a/autogpt_platform/backend/backend/api/features/library/embeddings.py b/autogpt_platform/backend/backend/api/features/library/embeddings.py
index f9b45eabab..3686ff1eef 100644
--- a/autogpt_platform/backend/backend/api/features/library/embeddings.py
+++ b/autogpt_platform/backend/backend/api/features/library/embeddings.py
@@ -14,7 +14,7 @@ import logging
from prisma.enums import ContentType
-from backend.api.features.store.embeddings import (
+from backend.api.features.search.embeddings import (
ensure_content_embedding,
get_content_embedding,
)
diff --git a/autogpt_platform/backend/backend/api/features/library/search.py b/autogpt_platform/backend/backend/api/features/library/search.py
index 17cce284b3..d7c94564c6 100644
--- a/autogpt_platform/backend/backend/api/features/library/search.py
+++ b/autogpt_platform/backend/backend/api/features/library/search.py
@@ -14,11 +14,11 @@ from __future__ import annotations
import logging
import re
-from typing import Any
+from typing import Any, cast
from prisma.enums import ContentType
-from backend.api.features.store.hybrid_search import UnifiedSearchWeights
+from backend.api.features.search.hybrid_search import UnifiedSearchWeights
from backend.data.db_accessors import search
logger = logging.getLogger(__name__)
@@ -192,4 +192,8 @@ async def hybrid_search_library_agents(
min_score,
[round(r.get("combined_score") or 0.0, 3) for r in raw_results[:5]],
)
- return [r for r in raw_results if (r.get("combined_score") or 0.0) >= min_score]
+ return [
+ cast(dict[str, Any], r)
+ for r in raw_results
+ if (r.get("combined_score") or 0.0) >= min_score
+ ]
diff --git a/autogpt_platform/backend/backend/api/features/search/README.md b/autogpt_platform/backend/backend/api/features/search/README.md
new file mode 100644
index 0000000000..dfe9b369f5
--- /dev/null
+++ b/autogpt_platform/backend/backend/api/features/search/README.md
@@ -0,0 +1,46 @@
+# search
+
+Shared search infrastructure for the platform. Anything that searches or
+embeds *user/store/library/workspace/chat* content imports from here.
+
+## Modules
+
+- **`embeddings.py`** — generic embedding service. OpenAI wrapper
+ (`generate_embedding`, `embed_query`), `UnifiedContentEmbedding` CRUD
+ (`store_content_embedding`, `get_content_embedding`,
+ `ensure_content_embedding`, `delete_content_embedding`),
+ `semantic_search`, and admin ops (`backfill_all_content_types`,
+ `cleanup_orphaned_embeddings`, `get_embedding_stats`).
+- **`hybrid_search.py`** — `unified_hybrid_search` engine + BM25
+ reranking (`tokenize`, `bm25_rerank`). Searches the
+ `UnifiedContentEmbedding` table across every content type with
+ combined semantic + lexical + recency + category scoring.
+- **`content_handlers.py`** — pluggable per-`ContentType` handlers
+ (store agent, block, doc, library agent, workspace file, chat
+ session). Used by the embedding backfill/cleanup loops.
+- **`service.py` / `routes.py`** — the `/api/search/global` endpoint
+ used by the global search modal.
+
+## Where things live
+
+| Concern | Lives in |
+|---|---|
+| Generic engine (above) | `search/` |
+| Store-listing wrappers (`store_embedding`, `hybrid_search` over store agents) | `store/embeddings.py`, `store/hybrid_search.py` |
+| Per-feature background indexing | `library/embeddings.py`, `workspace/embeddings.py`, `copilot/chat_session_embeddings.py` |
+
+**Rule:** anything used by more than one feature lives here. Feature-
+specific wrappers (a SQL join that pulls store-agent metadata, the
+fire-and-forget task that embeds a `LibraryAgent` row) stay in the
+feature.
+
+## Adding a new searchable content type
+
+1. Add the enum value to `ContentType` (Prisma schema).
+2. Add a `ContentHandler` subclass in `content_handlers.py` and
+ register it in `CONTENT_HANDLERS`.
+3. In the owning feature, add a small background task that calls
+ `ensure_content_embedding` on write — see `library/embeddings.py`
+ for the pattern.
+4. The new type is now picked up by `unified_hybrid_search` and the
+ global search endpoint automatically.
diff --git a/autogpt_platform/backend/backend/api/features/search/__init__.py b/autogpt_platform/backend/backend/api/features/search/__init__.py
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/autogpt_platform/backend/backend/api/features/store/content_handlers.py b/autogpt_platform/backend/backend/api/features/search/content_handlers.py
similarity index 70%
rename from autogpt_platform/backend/backend/api/features/store/content_handlers.py
rename to autogpt_platform/backend/backend/api/features/search/content_handlers.py
index 76bc2f2333..0f7cffb480 100644
--- a/autogpt_platform/backend/backend/api/features/store/content_handlers.py
+++ b/autogpt_platform/backend/backend/api/features/search/content_handlers.py
@@ -82,6 +82,20 @@ class ContentHandler(ABC):
"""
pass
+ @abstractmethod
+ async def get_valid_content_ids(self) -> set[str]:
+ """Return the set of ``contentId`` values that should currently
+ have an embedding for this content type.
+
+ Used by :func:`cleanup_orphaned_embeddings` to compute the
+ difference against ``UnifiedContentEmbedding`` and delete rows
+ whose source content has been removed / hidden / untitled. The
+ definition of "valid" must match the corresponding handler's
+ :meth:`get_missing_items` filter so the embedder isn't fighting
+ the orphan-cleaner.
+ """
+ pass
+
class StoreAgentHandler(ContentHandler):
"""Handler for marketplace store agent listings."""
@@ -92,7 +106,7 @@ class StoreAgentHandler(ContentHandler):
async def get_missing_items(self, batch_size: int) -> list[ContentItem]:
"""Fetch approved store listings without embeddings."""
- from backend.api.features.store.embeddings import build_searchable_text
+ from backend.api.features.search.embeddings import build_searchable_text
missing = await query_raw_with_schema(
"""
@@ -101,12 +115,19 @@ class StoreAgentHandler(ContentHandler):
slv.name,
slv.description,
slv."subHeading",
- slv.categories
+ slv.categories,
+ sl.slug,
+ p.username AS creator
FROM {schema_prefix}"StoreListingVersion" slv
+ JOIN {schema_prefix}"StoreListing" sl
+ ON slv."storeListingId" = sl.id
+ JOIN {schema_prefix}"Profile" p
+ ON p."userId" = sl."owningUserId"
LEFT JOIN {schema_prefix}"UnifiedContentEmbedding" uce
ON slv.id = uce."contentId" AND uce."contentType" = 'STORE_AGENT'::{schema_prefix}"ContentType"
WHERE slv."submissionStatus" = 'APPROVED'
AND slv."isDeleted" = false
+ AND sl."isDeleted" = false
AND uce."contentId" IS NULL
LIMIT $1
""",
@@ -123,9 +144,16 @@ class StoreAgentHandler(ContentHandler):
sub_heading=row["subHeading"],
categories=row["categories"] or [],
),
+ # ``creator`` (Profile.username) + ``slug`` (StoreListing.slug)
+ # build the marketplace URL — the FE search-result click
+ # handler in ChatSidebar.tsx routes to
+ # ``/marketplace/agent/{creator}/{slug}`` and silently no-ops
+ # without them.
metadata={
"name": row["name"],
"categories": row["categories"] or [],
+ "creator": row["creator"],
+ "slug": row["slug"],
},
user_id=None, # Store agents are public
)
@@ -163,6 +191,22 @@ class StoreAgentHandler(ContentHandler):
"without_embeddings": total_approved - with_embeddings,
}
+ async def get_valid_content_ids(self) -> set[str]:
+ # Approved StoreListingVersion rows from non-deleted listings
+ # mirror ``get_missing_items``: rejected / soft-deleted listings
+ # should have their embeddings cleaned up.
+ rows = await query_raw_with_schema(
+ """
+ SELECT slv.id
+ FROM {schema_prefix}"StoreListingVersion" slv
+ JOIN {schema_prefix}"StoreListing" sl ON slv."storeListingId" = sl.id
+ WHERE slv."submissionStatus" = 'APPROVED'
+ AND slv."isDeleted" = false
+ AND sl."isDeleted" = false
+ """,
+ )
+ return {row["id"] for row in rows}
+
@functools.lru_cache(maxsize=1)
def _get_enabled_blocks() -> dict[str, AnyBlockSchema]:
@@ -316,6 +360,13 @@ class BlockHandler(ContentHandler):
"without_embeddings": total_blocks - with_embeddings,
}
+ async def get_valid_content_ids(self) -> set[str]:
+ # The enabled-block registry is the source of truth — a block
+ # that's been removed from the registry or disabled is no
+ # longer a valid embedding target.
+ enabled = await asyncio.to_thread(_get_enabled_blocks)
+ return set(enabled.keys())
+
@dataclass
class MarkdownSection:
@@ -643,6 +694,16 @@ class DocumentationHandler(ContentHandler):
"without_embeddings": total_sections - with_embeddings,
}
+ async def get_valid_content_ids(self) -> set[str]:
+ # Section IDs are derived from on-disk markdown — a deleted file
+ # or removed section drops from this set and its embedding gets
+ # cleaned up.
+ docs_root = self._get_docs_root()
+ if not docs_root.exists():
+ return set()
+ # Filesystem walk is sync; keep it off the event loop.
+ return await asyncio.to_thread(self._get_all_section_content_ids, docs_root)
+
class LibraryAgentHandler(ContentHandler):
"""Handler for user-scoped library agents.
@@ -729,6 +790,236 @@ class LibraryAgentHandler(ContentHandler):
"without_embeddings": total - with_embeddings,
}
+ async def get_valid_content_ids(self) -> set[str]:
+ # Mirrors ``get_missing_items``: only non-deleted, non-hidden
+ # LibraryAgent rows are considered valid embedding targets.
+ rows = await query_raw_with_schema(
+ """
+ SELECT id
+ FROM {schema_prefix}"LibraryAgent"
+ WHERE "isDeleted" = false AND "isHidden" = false
+ """,
+ )
+ return {row["id"] for row in rows}
+
+
+def build_workspace_file_text(name: str, path: str) -> str:
+ """Build the embedding-searchable text for a workspace file.
+
+ Shared between :class:`WorkspaceFileHandler` (batch backfill) and
+ the per-write indexer in ``workspace/embeddings.py`` so both encode
+ the file under the same ``searchableText`` representation. If they
+ drift, the "skip if unchanged" guard in the indexer keeps re-
+ embedding the same row forever.
+ """
+ # Strip the leading slash and extension from the path stem so the
+ # token "report" from "/documents/report.pdf" reinforces a rename
+ # to "Final.pdf" without polluting the text with directory noise.
+ from pathlib import PurePosixPath
+
+ stem = PurePosixPath(path).stem if path else ""
+ parts = [name or "", stem]
+ # Dedup while preserving order: a freshly-created file usually has
+ # ``name == stem`` and we don't want the same token weighted twice.
+ seen: set[str] = set()
+ out: list[str] = []
+ for p in parts:
+ p = p.strip()
+ if p and p.lower() not in seen:
+ seen.add(p.lower())
+ out.append(p)
+ return " ".join(out)
+
+
+class WorkspaceFileHandler(ContentHandler):
+ """Handler for user-scoped workspace files.
+
+ Files are private per workspace owner. ``searchable_text`` is the
+ user-visible ``name`` (plus the path stem when it adds extra signal —
+ e.g. files renamed to a generic name still match against their path)
+ so search results surface meaningful matches without us having to
+ embed file contents.
+ """
+
+ @property
+ def content_type(self) -> ContentType:
+ return ContentType.WORKSPACE_FILE
+
+ async def get_missing_items(self, batch_size: int) -> list[ContentItem]:
+ missing = await query_raw_with_schema(
+ """
+ SELECT
+ uwf.id,
+ uwf.name,
+ uwf.path,
+ uwf."mimeType",
+ uw."userId"
+ FROM {schema_prefix}"UserWorkspaceFile" uwf
+ JOIN {schema_prefix}"UserWorkspace" uw ON uw.id = uwf."workspaceId"
+ LEFT JOIN {schema_prefix}"UnifiedContentEmbedding" uce
+ ON uce."contentId" = uwf.id
+ AND uce."contentType" = 'WORKSPACE_FILE'::{schema_prefix}"ContentType"
+ AND uce."userId" = uw."userId"
+ WHERE uwf."isDeleted" = false
+ AND uce."contentId" IS NULL
+ LIMIT $1
+ """,
+ batch_size,
+ )
+
+ items: list[ContentItem] = []
+ for row in missing:
+ text = build_workspace_file_text(row["name"] or "", row["path"] or "")
+ if not text:
+ continue
+ items.append(
+ ContentItem(
+ content_id=row["id"],
+ content_type=ContentType.WORKSPACE_FILE,
+ searchable_text=text,
+ metadata={
+ "name": row["name"] or "",
+ "path": row["path"] or "",
+ "mime_type": row["mimeType"] or "",
+ },
+ user_id=row["userId"],
+ )
+ )
+ return items
+
+ async def get_stats(self) -> dict[str, int]:
+ total_result = await query_raw_with_schema(
+ """
+ SELECT COUNT(*) as count
+ FROM {schema_prefix}"UserWorkspaceFile"
+ WHERE "isDeleted" = false
+ """
+ )
+ total = total_result[0]["count"] if total_result else 0
+
+ with_result = await query_raw_with_schema(
+ """
+ SELECT COUNT(*) as count
+ FROM {schema_prefix}"UserWorkspaceFile" uwf
+ JOIN {schema_prefix}"UserWorkspace" uw ON uw.id = uwf."workspaceId"
+ JOIN {schema_prefix}"UnifiedContentEmbedding" uce
+ ON uce."contentId" = uwf.id
+ AND uce."contentType" = 'WORKSPACE_FILE'::{schema_prefix}"ContentType"
+ AND uce."userId" = uw."userId"
+ WHERE uwf."isDeleted" = false
+ """
+ )
+ with_embeddings = with_result[0]["count"] if with_result else 0
+
+ return {
+ "total": total,
+ "with_embeddings": with_embeddings,
+ "without_embeddings": total - with_embeddings,
+ }
+
+ async def get_valid_content_ids(self) -> set[str]:
+ # Mirrors ``get_missing_items``: a soft-deleted file's embedding
+ # is orphaned and should be cleaned up.
+ rows = await query_raw_with_schema(
+ """
+ SELECT id
+ FROM {schema_prefix}"UserWorkspaceFile"
+ WHERE "isDeleted" = false
+ """,
+ )
+ return {row["id"] for row in rows}
+
+
+class ChatSessionHandler(ContentHandler):
+ """Handler for user-scoped chat sessions.
+
+ Only sessions with a non-empty ``title`` are embedded — untitled
+ sessions are noise for search and CoPilot auto-fills the title within
+ the first turn anyway.
+ """
+
+ @property
+ def content_type(self) -> ContentType:
+ return ContentType.CHAT_SESSION
+
+ async def get_missing_items(self, batch_size: int) -> list[ContentItem]:
+ missing = await query_raw_with_schema(
+ """
+ SELECT
+ cs.id,
+ cs."userId",
+ cs.title
+ FROM {schema_prefix}"ChatSession" cs
+ LEFT JOIN {schema_prefix}"UnifiedContentEmbedding" uce
+ ON uce."contentId" = cs.id
+ AND uce."contentType" = 'CHAT_SESSION'::{schema_prefix}"ContentType"
+ AND uce."userId" = cs."userId"
+ WHERE cs.title IS NOT NULL
+ AND btrim(cs.title) <> ''
+ AND uce."contentId" IS NULL
+ LIMIT $1
+ """,
+ batch_size,
+ )
+
+ items: list[ContentItem] = []
+ for row in missing:
+ title = (row["title"] or "").strip()
+ if not title:
+ continue
+ items.append(
+ ContentItem(
+ content_id=row["id"],
+ content_type=ContentType.CHAT_SESSION,
+ searchable_text=title,
+ metadata={"title": title},
+ user_id=row["userId"],
+ )
+ )
+ return items
+
+ async def get_stats(self) -> dict[str, int]:
+ total_result = await query_raw_with_schema(
+ """
+ SELECT COUNT(*) as count
+ FROM {schema_prefix}"ChatSession"
+ WHERE title IS NOT NULL AND btrim(title) <> ''
+ """
+ )
+ total = total_result[0]["count"] if total_result else 0
+
+ with_result = await query_raw_with_schema(
+ """
+ SELECT COUNT(*) as count
+ FROM {schema_prefix}"ChatSession" cs
+ JOIN {schema_prefix}"UnifiedContentEmbedding" uce
+ ON uce."contentId" = cs.id
+ AND uce."contentType" = 'CHAT_SESSION'::{schema_prefix}"ContentType"
+ AND uce."userId" = cs."userId"
+ WHERE cs.title IS NOT NULL AND btrim(cs.title) <> ''
+ """
+ )
+ with_embeddings = with_result[0]["count"] if with_result else 0
+
+ return {
+ "total": total,
+ "with_embeddings": with_embeddings,
+ "without_embeddings": total - with_embeddings,
+ }
+
+ async def get_valid_content_ids(self) -> set[str]:
+ # Mirrors ``get_missing_items``: sessions without a non-empty
+ # title aren't embedded; a session that was deleted or had its
+ # title cleared should have its embedding cleaned up.
+ rows = await query_raw_with_schema(
+ """
+ SELECT id
+ FROM {schema_prefix}"ChatSession"
+ WHERE title IS NOT NULL AND btrim(title) <> ''
+ """,
+ )
+ return {row["id"] for row in rows}
+
# Content handler registry
CONTENT_HANDLERS: dict[ContentType, ContentHandler] = {
@@ -736,4 +1027,6 @@ CONTENT_HANDLERS: dict[ContentType, ContentHandler] = {
ContentType.BLOCK: BlockHandler(),
ContentType.DOCUMENTATION: DocumentationHandler(),
ContentType.LIBRARY_AGENT: LibraryAgentHandler(),
+ ContentType.WORKSPACE_FILE: WorkspaceFileHandler(),
+ ContentType.CHAT_SESSION: ChatSessionHandler(),
}
diff --git a/autogpt_platform/backend/backend/api/features/store/content_handlers_integration_test.py b/autogpt_platform/backend/backend/api/features/search/content_handlers_integration_test.py
similarity index 94%
rename from autogpt_platform/backend/backend/api/features/store/content_handlers_integration_test.py
rename to autogpt_platform/backend/backend/api/features/search/content_handlers_integration_test.py
index b53a9e80b0..b7ac888d23 100644
--- a/autogpt_platform/backend/backend/api/features/store/content_handlers_integration_test.py
+++ b/autogpt_platform/backend/backend/api/features/search/content_handlers_integration_test.py
@@ -1,7 +1,7 @@
"""
Integration tests for content handlers using real DB.
-Run with: poetry run pytest backend/api/features/store/content_handlers_integration_test.py -xvs
+Run with: poetry run pytest backend/api/features/search/content_handlers_integration_test.py -xvs
These tests use the real database but mock OpenAI calls.
"""
@@ -10,13 +10,13 @@ from unittest.mock import patch
import pytest
-from backend.api.features.store.content_handlers import (
+from backend.api.features.search.content_handlers import (
CONTENT_HANDLERS,
BlockHandler,
DocumentationHandler,
StoreAgentHandler,
)
-from backend.api.features.store.embeddings import (
+from backend.api.features.search.embeddings import (
EMBEDDING_DIM,
backfill_all_content_types,
ensure_content_embedding,
@@ -138,7 +138,7 @@ async def test_get_embedding_stats_all_types():
@pytest.mark.asyncio(loop_scope="session")
-@patch("backend.api.features.store.embeddings.generate_embedding")
+@patch("backend.api.features.search.embeddings.generate_embedding")
async def test_ensure_content_embedding_blocks(mock_generate):
"""Test creating embeddings for blocks (mocked OpenAI)."""
# Mock OpenAI to return fake embedding
@@ -168,7 +168,7 @@ async def test_ensure_content_embedding_blocks(mock_generate):
@pytest.mark.asyncio(loop_scope="session")
-@patch("backend.api.features.store.embeddings.generate_embedding")
+@patch("backend.api.features.search.embeddings.generate_embedding")
async def test_backfill_all_content_types_dry_run(mock_generate):
"""Test backfill_all_content_types processes all handlers in order."""
# Mock OpenAI to return fake embedding
diff --git a/autogpt_platform/backend/backend/api/features/store/content_handlers_test.py b/autogpt_platform/backend/backend/api/features/search/content_handlers_test.py
similarity index 88%
rename from autogpt_platform/backend/backend/api/features/store/content_handlers_test.py
rename to autogpt_platform/backend/backend/api/features/search/content_handlers_test.py
index 0dfd9b5a86..982ca15027 100644
--- a/autogpt_platform/backend/backend/api/features/store/content_handlers_test.py
+++ b/autogpt_platform/backend/backend/api/features/search/content_handlers_test.py
@@ -8,7 +8,7 @@ from unittest.mock import MagicMock, patch
import pytest
from prisma.enums import ContentType
-from backend.api.features.store.content_handlers import (
+from backend.api.features.search.content_handlers import (
CONTENT_HANDLERS,
BlockHandler,
DocumentationHandler,
@@ -70,7 +70,7 @@ def test_get_enabled_blocks_filters_disabled():
"disabled": _make_block_class(name="D", disabled=True),
}
with patch(
- "backend.api.features.store.content_handlers.get_blocks", return_value=blocks
+ "backend.api.features.search.content_handlers.get_blocks", return_value=blocks
):
result = _get_enabled_blocks()
assert list(result.keys()) == ["enabled"]
@@ -83,7 +83,7 @@ def test_get_enabled_blocks_skips_broken():
"bad": _make_block_class(raise_on_init=RuntimeError("boom")),
}
with patch(
- "backend.api.features.store.content_handlers.get_blocks", return_value=blocks
+ "backend.api.features.search.content_handlers.get_blocks", return_value=blocks
):
result = _get_enabled_blocks()
assert list(result.keys()) == ["good"]
@@ -93,7 +93,7 @@ def test_get_enabled_blocks_cached():
"""_get_enabled_blocks() calls get_blocks() only once across multiple calls."""
blocks = {"b1": _make_block_class(name="B1")}
with patch(
- "backend.api.features.store.content_handlers.get_blocks", return_value=blocks
+ "backend.api.features.search.content_handlers.get_blocks", return_value=blocks
) as mock_get_blocks:
result1 = _get_enabled_blocks()
result2 = _get_enabled_blocks()
@@ -118,11 +118,16 @@ async def test_store_agent_handler_get_missing_items(mocker):
"description": "A test agent",
"subHeading": "Test heading",
"categories": ["AI", "Testing"],
+ # SELECT now joins StoreListing + Profile so the embedding
+ # row carries the marketplace URL fields the FE click
+ # handler needs (see service._enrich_store_agent_metadata).
+ "slug": "test-agent",
+ "creator": "alice",
}
]
with patch(
- "backend.api.features.store.content_handlers.query_raw_with_schema",
+ "backend.api.features.search.content_handlers.query_raw_with_schema",
return_value=mock_missing,
):
items = await handler.get_missing_items(batch_size=10)
@@ -133,6 +138,8 @@ async def test_store_agent_handler_get_missing_items(mocker):
assert "Test Agent" in items[0].searchable_text
assert "A test agent" in items[0].searchable_text
assert items[0].metadata["name"] == "Test Agent"
+ assert items[0].metadata["creator"] == "alice"
+ assert items[0].metadata["slug"] == "test-agent"
assert items[0].user_id is None
@@ -145,7 +152,7 @@ async def test_store_agent_handler_get_stats(mocker):
mock_embedded = [{"count": 30}]
with patch(
- "backend.api.features.store.content_handlers.query_raw_with_schema",
+ "backend.api.features.search.content_handlers.query_raw_with_schema",
side_effect=[mock_approved, mock_embedded],
):
stats = await handler.get_stats()
@@ -175,10 +182,10 @@ async def test_block_handler_get_missing_items():
}
with patch(
- "backend.api.features.store.content_handlers.get_blocks", return_value=blocks
+ "backend.api.features.search.content_handlers.get_blocks", return_value=blocks
):
with patch(
- "backend.api.features.store.content_handlers.query_raw_with_schema",
+ "backend.api.features.search.content_handlers.query_raw_with_schema",
return_value=[],
):
items = await handler.get_missing_items(batch_size=10)
@@ -205,10 +212,10 @@ async def test_block_handler_get_missing_items_splits_camelcase():
}
with patch(
- "backend.api.features.store.content_handlers.get_blocks", return_value=blocks
+ "backend.api.features.search.content_handlers.get_blocks", return_value=blocks
):
with patch(
- "backend.api.features.store.content_handlers.query_raw_with_schema",
+ "backend.api.features.search.content_handlers.query_raw_with_schema",
return_value=[],
):
items = await handler.get_missing_items(batch_size=10)
@@ -225,10 +232,10 @@ async def test_block_handler_get_missing_items_batch_size_zero():
blocks = {"b1": _make_block_class(name="B1")}
with patch(
- "backend.api.features.store.content_handlers.get_blocks", return_value=blocks
+ "backend.api.features.search.content_handlers.get_blocks", return_value=blocks
):
with patch(
- "backend.api.features.store.content_handlers.query_raw_with_schema",
+ "backend.api.features.search.content_handlers.query_raw_with_schema",
return_value=[],
) as mock_query:
items = await handler.get_missing_items(batch_size=0)
@@ -252,10 +259,10 @@ async def test_block_handler_disabled_dont_exhaust_batch():
}
with patch(
- "backend.api.features.store.content_handlers.get_blocks", return_value=blocks
+ "backend.api.features.search.content_handlers.get_blocks", return_value=blocks
):
with patch(
- "backend.api.features.store.content_handlers.query_raw_with_schema",
+ "backend.api.features.search.content_handlers.query_raw_with_schema",
return_value=[],
):
items = await handler.get_missing_items(batch_size=2)
@@ -278,10 +285,10 @@ async def test_block_handler_get_stats():
mock_embedded = [{"count": 2}]
with patch(
- "backend.api.features.store.content_handlers.get_blocks", return_value=blocks
+ "backend.api.features.search.content_handlers.get_blocks", return_value=blocks
):
with patch(
- "backend.api.features.store.content_handlers.query_raw_with_schema",
+ "backend.api.features.search.content_handlers.query_raw_with_schema",
return_value=mock_embedded,
):
stats = await handler.get_stats()
@@ -304,10 +311,10 @@ async def test_block_handler_get_stats_skips_broken():
mock_embedded = [{"count": 1}]
with patch(
- "backend.api.features.store.content_handlers.get_blocks", return_value=blocks
+ "backend.api.features.search.content_handlers.get_blocks", return_value=blocks
):
with patch(
- "backend.api.features.store.content_handlers.query_raw_with_schema",
+ "backend.api.features.search.content_handlers.query_raw_with_schema",
return_value=mock_embedded,
):
stats = await handler.get_stats()
@@ -332,10 +339,10 @@ async def test_block_handler_handles_none_name():
blocks["none-name-block"].return_value.name = None
with patch(
- "backend.api.features.store.content_handlers.get_blocks", return_value=blocks
+ "backend.api.features.search.content_handlers.get_blocks", return_value=blocks
):
with patch(
- "backend.api.features.store.content_handlers.query_raw_with_schema",
+ "backend.api.features.search.content_handlers.query_raw_with_schema",
return_value=[],
):
items = await handler.get_missing_items(batch_size=10)
@@ -357,10 +364,10 @@ async def test_block_handler_handles_empty_attributes():
blocks = {"block-minimal": _make_block_class(name="Minimal Block")}
with patch(
- "backend.api.features.store.content_handlers.get_blocks", return_value=blocks
+ "backend.api.features.search.content_handlers.get_blocks", return_value=blocks
):
with patch(
- "backend.api.features.store.content_handlers.query_raw_with_schema",
+ "backend.api.features.search.content_handlers.query_raw_with_schema",
return_value=[],
):
items = await handler.get_missing_items(batch_size=10)
@@ -380,10 +387,10 @@ async def test_block_handler_skips_failed_blocks():
}
with patch(
- "backend.api.features.store.content_handlers.get_blocks", return_value=blocks
+ "backend.api.features.search.content_handlers.get_blocks", return_value=blocks
):
with patch(
- "backend.api.features.store.content_handlers.query_raw_with_schema",
+ "backend.api.features.search.content_handlers.query_raw_with_schema",
return_value=[],
):
items = await handler.get_missing_items(batch_size=10)
@@ -409,7 +416,7 @@ async def test_documentation_handler_get_missing_items(tmp_path, mocker):
with patch.object(handler, "_get_docs_root", return_value=docs_root):
with patch(
- "backend.api.features.store.content_handlers.query_raw_with_schema",
+ "backend.api.features.search.content_handlers.query_raw_with_schema",
return_value=[],
):
items = await handler.get_missing_items(batch_size=10)
@@ -448,7 +455,7 @@ async def test_documentation_handler_get_stats(tmp_path, mocker):
with patch.object(handler, "_get_docs_root", return_value=docs_root):
with patch(
- "backend.api.features.store.content_handlers.query_raw_with_schema",
+ "backend.api.features.search.content_handlers.query_raw_with_schema",
return_value=mock_embedded,
):
stats = await handler.get_stats()
@@ -585,7 +592,7 @@ async def test_library_agent_handler_emits_user_scoped_items():
},
]
with patch(
- "backend.api.features.store.content_handlers.query_raw_with_schema",
+ "backend.api.features.search.content_handlers.query_raw_with_schema",
return_value=rows,
):
handler = LibraryAgentHandler()
@@ -614,7 +621,7 @@ async def test_library_agent_handler_stats():
return [{"count": 10}]
with patch(
- "backend.api.features.store.content_handlers.query_raw_with_schema",
+ "backend.api.features.search.content_handlers.query_raw_with_schema",
side_effect=fake_query_raw,
):
stats = await LibraryAgentHandler().get_stats()
diff --git a/autogpt_platform/backend/backend/api/features/search/embeddings.py b/autogpt_platform/backend/backend/api/features/search/embeddings.py
new file mode 100644
index 0000000000..c50882a4f5
--- /dev/null
+++ b/autogpt_platform/backend/backend/api/features/search/embeddings.py
@@ -0,0 +1,814 @@
+"""
+Unified Content Embeddings — shared across features.
+
+Generic OpenAI embedding generation + ``UnifiedContentEmbedding`` row CRUD
+that every feature (store, library, workspace, copilot) uses.
+
+Feature-specific wrappers live next to the feature they belong to:
+- ``backend.api.features.store.embeddings`` — store listing wrappers
+- ``backend.api.features.library.embeddings`` — library agent bg task
+- ``backend.api.features.workspace.embeddings`` — workspace file bg task
+- ``backend.copilot.chat_session_embeddings`` — chat session bg task
+"""
+
+import asyncio
+import logging
+import time
+from typing import Any
+
+import prisma
+from prisma.enums import ContentType
+from tiktoken import encoding_for_model
+
+from backend.api.features.search.content_handlers import CONTENT_HANDLERS
+from backend.data.db import execute_raw_with_schema, query_raw_with_schema
+from backend.util.clients import get_openai_client
+from backend.util.json import dumps
+
+logger = logging.getLogger(__name__)
+
+# OpenAI embedding model configuration
+EMBEDDING_MODEL = "text-embedding-3-small"
+# Embedding dimension for the model above
+# text-embedding-3-small: 1536, text-embedding-3-large: 3072
+EMBEDDING_DIM = 1536
+# OpenAI embedding token limit (8,191 with 1 token buffer for safety)
+EMBEDDING_MAX_TOKENS = 8191
+
+# Single source of truth for the content types the maintenance jobs
+# (backfill + orphan cleanup) walk. BLOCK is first so foundational content
+# is searchable soonest. Keep in sync with the handlers in
+# ``content_handlers.CONTENT_HANDLERS``.
+SEARCHABLE_EMBEDDING_TYPES = [
+ ContentType.BLOCK,
+ ContentType.STORE_AGENT,
+ ContentType.DOCUMENTATION,
+ ContentType.LIBRARY_AGENT,
+ ContentType.WORKSPACE_FILE,
+ ContentType.CHAT_SESSION,
+]
+
+
+def build_searchable_text(
+ name: str,
+ description: str,
+ sub_heading: str,
+ categories: list[str],
+) -> str:
+ """
+ Build searchable text from listing version fields.
+
+ Combines relevant fields into a single string for embedding.
+ """
+ parts = []
+
+ # Name is important - include it
+ if name:
+ parts.append(name)
+
+ # Sub-heading provides context
+ if sub_heading:
+ parts.append(sub_heading)
+
+ # Description is the main content
+ if description:
+ parts.append(description)
+
+ # Categories help with semantic matching
+ if categories:
+ parts.append(" ".join(categories))
+
+ return " ".join(parts)
+
+
+# Cached at module load: ``encoding_for_model`` loads a tokenizer file
+# from disk on every call (~10–20 ms). At backfill scale we run this on
+# every row, so amortising it module-level is a meaningful win.
+_ENCODER = encoding_for_model(EMBEDDING_MODEL)
+
+
+async def generate_embedding(text: str) -> list[float]:
+ """
+ Generate embedding for text using OpenAI API.
+
+ Raises exceptions on failure - caller should handle.
+ """
+ client = get_openai_client()
+ if not client:
+ raise RuntimeError("openai_internal_api_key not set, cannot generate embedding")
+
+ # Truncate text to token limit using tiktoken
+ # Character-based truncation is insufficient because token ratios vary by content type
+ tokens = _ENCODER.encode(text)
+ original_token_count = len(tokens)
+ if original_token_count > EMBEDDING_MAX_TOKENS:
+ tokens = tokens[:EMBEDDING_MAX_TOKENS]
+ truncated_text = _ENCODER.decode(tokens)
+ logger.info(
+ f"Truncated text from {original_token_count} to {len(tokens)} tokens"
+ )
+ else:
+ truncated_text = text
+
+ start_time = time.time()
+ response = await client.embeddings.create(
+ model=EMBEDDING_MODEL,
+ input=truncated_text,
+ )
+ latency_ms = (time.time() - start_time) * 1000
+
+ embedding = response.data[0].embedding
+ logger.info(
+ f"Generated embedding: {len(embedding)} dims, "
+ f"{len(tokens)} tokens, {latency_ms:.0f}ms"
+ )
+ return embedding
+
+
+async def embed_query(query: str) -> list[float]:
+ """
+ Generate embedding for a search query.
+
+ Same as generate_embedding but with clearer intent.
+ Raises exceptions on failure - caller should handle.
+ """
+ return await generate_embedding(query)
+
+
+def embedding_to_vector_string(embedding: list[float]) -> str:
+ """Convert embedding list to PostgreSQL vector string format."""
+ return "[" + ",".join(str(x) for x in embedding) + "]"
+
+
+async def store_content_embedding(
+ content_type: ContentType,
+ content_id: str,
+ embedding: list[float],
+ searchable_text: str,
+ metadata: dict | None = None,
+ user_id: str | None = None,
+ tx: prisma.Prisma | None = None,
+) -> bool:
+ """
+ Store embedding in the unified content embeddings table.
+
+ New function for unified content embedding storage.
+ Uses raw SQL since Prisma doesn't natively support pgvector.
+
+ Raises exceptions on failure - caller should handle.
+ """
+ client = tx if tx else prisma.get_client()
+
+ # Convert embedding to PostgreSQL vector format
+ embedding_str = embedding_to_vector_string(embedding)
+ metadata_json = dumps(metadata or {})
+
+ # Upsert the embedding
+ # WHERE clause in DO UPDATE prevents PostgreSQL 15 bug with NULLS NOT DISTINCT
+ # Use unqualified ::vector - pgvector is in search_path on all environments
+ await execute_raw_with_schema(
+ """
+ INSERT INTO {schema_prefix}"UnifiedContentEmbedding" (
+ "id", "contentType", "contentId", "userId", "embedding", "searchableText", "metadata", "createdAt", "updatedAt"
+ )
+ VALUES (gen_random_uuid()::text, $1::{schema_prefix}"ContentType", $2, $3, $4::vector, $5, $6::jsonb, NOW(), NOW())
+ ON CONFLICT ("contentType", "contentId", "userId")
+ DO UPDATE SET
+ "embedding" = $4::vector,
+ "searchableText" = $5,
+ "metadata" = $6::jsonb,
+ "updatedAt" = NOW()
+ WHERE {schema_prefix}"UnifiedContentEmbedding"."contentType" = $1::{schema_prefix}"ContentType"
+ AND {schema_prefix}"UnifiedContentEmbedding"."contentId" = $2
+ AND ({schema_prefix}"UnifiedContentEmbedding"."userId" = $3 OR ($3 IS NULL AND {schema_prefix}"UnifiedContentEmbedding"."userId" IS NULL))
+ """,
+ content_type,
+ content_id,
+ user_id,
+ embedding_str,
+ searchable_text,
+ metadata_json,
+ client=client,
+ )
+
+ logger.info(f"Stored embedding for {content_type}:{content_id}")
+ return True
+
+
+async def get_content_embedding(
+ content_type: ContentType, content_id: str, user_id: str | None = None
+) -> dict[str, Any] | None:
+ """
+ Retrieve embedding record for any content type.
+
+ New function for unified content embedding retrieval.
+ Returns dict with contentType, contentId, embedding, timestamps or None if not found.
+
+ Raises exceptions on failure - caller should handle.
+ """
+ result = await query_raw_with_schema(
+ """
+ SELECT
+ "contentType",
+ "contentId",
+ "userId",
+ "embedding"::text as "embedding",
+ "searchableText",
+ "metadata",
+ "createdAt",
+ "updatedAt"
+ FROM {schema_prefix}"UnifiedContentEmbedding"
+ WHERE "contentType" = $1::{schema_prefix}"ContentType" AND "contentId" = $2 AND ("userId" = $3 OR ($3 IS NULL AND "userId" IS NULL))
+ """,
+ content_type,
+ content_id,
+ user_id,
+ )
+
+ if result and len(result) > 0:
+ return result[0]
+ return None
+
+
+async def ensure_content_embedding(
+ content_type: ContentType,
+ content_id: str,
+ searchable_text: str,
+ metadata: dict | None = None,
+ user_id: str | None = None,
+ force: bool = False,
+ tx: prisma.Prisma | None = None,
+) -> bool:
+ """
+ Ensure an embedding exists for any content type.
+
+ Generic function for creating embeddings for store agents, blocks, docs, etc.
+
+ Args:
+ content_type: ContentType enum value (STORE_AGENT, BLOCK, etc.)
+ content_id: Unique identifier for the content
+ searchable_text: Combined text for embedding generation
+ metadata: Optional metadata to store with embedding
+ force: Force regeneration even if embedding exists
+ tx: Optional transaction client
+
+ Returns:
+ True if embedding exists/was created
+
+ Raises exceptions on failure - caller should handle.
+ """
+ # Check if embedding already exists
+ if not force:
+ existing = await get_content_embedding(content_type, content_id, user_id)
+ if existing and existing.get("embedding"):
+ logger.debug(f"Embedding for {content_type}:{content_id} already exists")
+ return True
+
+ # Generate new embedding
+ embedding = await generate_embedding(searchable_text)
+
+ # Store the embedding
+ return await store_content_embedding(
+ content_type=content_type,
+ content_id=content_id,
+ embedding=embedding,
+ searchable_text=searchable_text,
+ metadata=metadata or {},
+ user_id=user_id,
+ tx=tx,
+ )
+
+
+async def delete_content_embedding(
+ content_type: ContentType, content_id: str, user_id: str | None = None
+) -> bool:
+ """
+ Delete embedding for any content type.
+
+ New function for unified content embedding deletion.
+ Note: This is usually handled automatically by CASCADE delete,
+ but provided for manual cleanup if needed.
+
+ Args:
+ content_type: The type of content (STORE_AGENT, LIBRARY_AGENT, etc.)
+ content_id: The unique identifier for the content
+ user_id: Optional user ID. For public content (STORE_AGENT, BLOCK), pass None.
+ For user-scoped content (LIBRARY_AGENT), pass the user's ID to avoid
+ deleting embeddings belonging to other users.
+
+ Returns:
+ True if deletion succeeded, False otherwise
+ """
+ try:
+ client = prisma.get_client()
+
+ await execute_raw_with_schema(
+ """
+ DELETE FROM {schema_prefix}"UnifiedContentEmbedding"
+ WHERE "contentType" = $1::{schema_prefix}"ContentType"
+ AND "contentId" = $2
+ AND ("userId" = $3 OR ($3 IS NULL AND "userId" IS NULL))
+ """,
+ content_type,
+ content_id,
+ user_id,
+ client=client,
+ )
+
+ user_str = f" (user: {user_id})" if user_id else ""
+ logger.info(f"Deleted embedding for {content_type}:{content_id}{user_str}")
+ return True
+
+ except Exception as e:
+ logger.error(f"Failed to delete embedding for {content_type}:{content_id}: {e}")
+ return False
+
+
+async def get_embedding_stats() -> dict[str, Any]:
+ """
+ Get statistics about embedding coverage for all content types.
+
+ Returns stats per content type and overall totals.
+ """
+ try:
+ stats_by_type = {}
+ total_items = 0
+ total_with_embeddings = 0
+ total_without_embeddings = 0
+
+ # Aggregate stats from all handlers
+ for content_type, handler in CONTENT_HANDLERS.items():
+ try:
+ stats = await handler.get_stats()
+ stats_by_type[content_type.value] = {
+ "total": stats["total"],
+ "with_embeddings": stats["with_embeddings"],
+ "without_embeddings": stats["without_embeddings"],
+ "coverage_percent": (
+ round(stats["with_embeddings"] / stats["total"] * 100, 1)
+ if stats["total"] > 0
+ else 0
+ ),
+ }
+
+ total_items += stats["total"]
+ total_with_embeddings += stats["with_embeddings"]
+ total_without_embeddings += stats["without_embeddings"]
+
+ except Exception as e:
+ logger.error(f"Failed to get stats for {content_type.value}: {e}")
+ stats_by_type[content_type.value] = {
+ "total": 0,
+ "with_embeddings": 0,
+ "without_embeddings": 0,
+ "coverage_percent": 0,
+ "error": str(e),
+ }
+
+ return {
+ "by_type": stats_by_type,
+ "totals": {
+ "total": total_items,
+ "with_embeddings": total_with_embeddings,
+ "without_embeddings": total_without_embeddings,
+ "coverage_percent": (
+ round(total_with_embeddings / total_items * 100, 1)
+ if total_items > 0
+ else 0
+ ),
+ },
+ }
+
+ except Exception as e:
+ logger.error(f"Failed to get embedding stats: {e}")
+ return {
+ "by_type": {},
+ "totals": {
+ "total": 0,
+ "with_embeddings": 0,
+ "without_embeddings": 0,
+ "coverage_percent": 0,
+ },
+ "error": str(e),
+ }
+
+
+async def backfill_all_content_types(batch_size: int = 10) -> dict[str, Any]:
+ """
+ Generate embeddings for all content types using registered handlers.
+
+ Walks ``SEARCHABLE_EMBEDDING_TYPES`` in order (BLOCK first) so
+ foundational content is searchable soonest.
+
+ Args:
+ batch_size: Number of embeddings to generate per content type
+
+ Returns:
+ Dict with stats per content type and overall totals
+ """
+ results_by_type = {}
+ total_processed = 0
+ total_success = 0
+ total_failed = 0
+ all_errors: dict[str, int] = {} # Aggregate errors across all content types
+
+ # Process content types in explicit order
+ processing_order = SEARCHABLE_EMBEDDING_TYPES
+
+ for content_type in processing_order:
+ handler = CONTENT_HANDLERS.get(content_type)
+ if not handler:
+ logger.warning(f"No handler registered for {content_type.value}")
+ continue
+ try:
+ logger.info(f"Processing {content_type.value} content type...")
+
+ # Get missing items from handler
+ missing_items = await handler.get_missing_items(batch_size)
+
+ if not missing_items:
+ results_by_type[content_type.value] = {
+ "processed": 0,
+ "success": 0,
+ "failed": 0,
+ "message": "No missing embeddings",
+ }
+ continue
+
+ # Process embeddings concurrently for better performance
+ embedding_tasks = [
+ ensure_content_embedding(
+ content_type=item.content_type,
+ content_id=item.content_id,
+ searchable_text=item.searchable_text,
+ metadata=item.metadata,
+ user_id=item.user_id,
+ )
+ for item in missing_items
+ ]
+
+ results = await asyncio.gather(*embedding_tasks, return_exceptions=True)
+
+ success = sum(1 for result in results if result is True)
+ failed = len(results) - success
+
+ # Aggregate errors across all content types
+ if failed > 0:
+ for result in results:
+ if isinstance(result, Exception):
+ error_key = f"{type(result).__name__}: {str(result)}"
+ all_errors[error_key] = all_errors.get(error_key, 0) + 1
+
+ results_by_type[content_type.value] = {
+ "processed": len(missing_items),
+ "success": success,
+ "failed": failed,
+ "message": f"Backfilled {success} embeddings, {failed} failed",
+ }
+
+ total_processed += len(missing_items)
+ total_success += success
+ total_failed += failed
+
+ logger.info(
+ f"{content_type.value}: processed {len(missing_items)}, "
+ f"success {success}, failed {failed}"
+ )
+
+ except Exception as e:
+ logger.error(f"Failed to process {content_type.value}: {e}")
+ results_by_type[content_type.value] = {
+ "processed": 0,
+ "success": 0,
+ "failed": 0,
+ "error": str(e),
+ }
+
+ # Log aggregated errors once at the end
+ if all_errors:
+ error_details = ", ".join(
+ f"{error} ({count}x)" for error, count in all_errors.items()
+ )
+ logger.error(f"Embedding backfill errors: {error_details}")
+
+ return {
+ "by_type": results_by_type,
+ "totals": {
+ "processed": total_processed,
+ "success": total_success,
+ "failed": total_failed,
+ "message": f"Overall: {total_success} succeeded, {total_failed} failed",
+ },
+ }
+
+
+async def cleanup_orphaned_embeddings() -> dict[str, Any]:
+ """
+ Clean up embeddings for content that no longer exists or is no longer valid.
+
+ Compares current content with embeddings in database and removes orphaned records:
+ - STORE_AGENT: Removes embeddings for rejected/deleted store listings
+ - BLOCK: Removes embeddings for blocks no longer registered
+ - DOCUMENTATION: Removes embeddings for deleted doc files
+ - LIBRARY_AGENT: Removes embeddings for soft-deleted/hidden library agents
+ (matches the LibraryAgentHandler.get_missing_items filter)
+ - WORKSPACE_FILE: Removes embeddings for soft-deleted workspace files
+ - CHAT_SESSION: Removes embeddings for deleted/untitled chat sessions
+
+ Returns:
+ Dict with cleanup statistics per content type
+ """
+ results_by_type = {}
+ total_deleted = 0
+
+ # Cleanup orphaned embeddings for all content types
+ cleanup_types = SEARCHABLE_EMBEDDING_TYPES
+
+ for content_type in cleanup_types:
+ try:
+ handler = CONTENT_HANDLERS.get(content_type)
+ if not handler:
+ logger.warning(f"No handler registered for {content_type}")
+ results_by_type[content_type.value] = {
+ "deleted": 0,
+ "error": "No handler registered",
+ }
+ continue
+
+ # Each handler owns the definition of "what is currently
+ # valid for embedding". The same domain knowledge lives in
+ # its ``get_missing_items`` filter — keeping both behind one
+ # method means the orphan-cleaner and the embedder can't
+ # drift out of sync.
+ current_ids = await handler.get_valid_content_ids()
+
+ # Get all embedding IDs from database
+ db_embeddings = await query_raw_with_schema(
+ """
+ SELECT "contentId"
+ FROM {schema_prefix}"UnifiedContentEmbedding"
+ WHERE "contentType" = $1::{schema_prefix}"ContentType"
+ """,
+ content_type,
+ )
+
+ db_ids = {row["contentId"] for row in db_embeddings}
+
+ # Find orphaned embeddings (in DB but not in current content)
+ orphaned_ids = db_ids - current_ids
+
+ if not orphaned_ids:
+ logger.info(f"{content_type.value}: No orphaned embeddings found")
+ results_by_type[content_type.value] = {
+ "deleted": 0,
+ "message": "No orphaned embeddings",
+ }
+ continue
+
+ # Delete orphaned embeddings in batch for better performance
+ orphaned_list = list(orphaned_ids)
+ try:
+ await execute_raw_with_schema(
+ """
+ DELETE FROM {schema_prefix}"UnifiedContentEmbedding"
+ WHERE "contentType" = $1::{schema_prefix}"ContentType"
+ AND "contentId" = ANY($2::text[])
+ """,
+ content_type,
+ orphaned_list,
+ )
+ deleted = len(orphaned_list)
+ except Exception as e:
+ logger.error(f"Failed to batch delete orphaned embeddings: {e}")
+ deleted = 0
+
+ logger.info(
+ f"{content_type.value}: Deleted {deleted}/{len(orphaned_ids)} orphaned embeddings"
+ )
+ results_by_type[content_type.value] = {
+ "deleted": deleted,
+ "orphaned": len(orphaned_ids),
+ "message": f"Deleted {deleted} orphaned embeddings",
+ }
+
+ total_deleted += deleted
+
+ except Exception as e:
+ logger.error(f"Failed to cleanup {content_type.value}: {e}")
+ results_by_type[content_type.value] = {
+ "deleted": 0,
+ "error": str(e),
+ }
+
+ return {
+ "by_type": results_by_type,
+ "totals": {
+ "deleted": total_deleted,
+ "message": f"Deleted {total_deleted} orphaned embeddings",
+ },
+ }
+
+
+async def semantic_search(
+ query: str,
+ content_types: list[ContentType] | None = None,
+ user_id: str | None = None,
+ limit: int = 20,
+ min_similarity: float = 0.5,
+) -> list[dict[str, Any]]:
+ """
+ Semantic search across content types using embeddings.
+
+ Performs vector similarity search on UnifiedContentEmbedding table.
+ Used directly for blocks/docs/library agents, or as the semantic component
+ within hybrid_search for store agents.
+
+ If embedding generation fails, falls back to lexical search on searchableText.
+
+ Args:
+ query: Search query string
+ content_types: List of ContentType to search. Defaults to [BLOCK, STORE_AGENT, DOCUMENTATION]
+ user_id: Optional user ID for searching private content (library agents)
+ limit: Maximum number of results to return (default: 20)
+ min_similarity: Minimum cosine similarity threshold (0-1, default: 0.5)
+
+ Returns:
+ List of search results with the following structure:
+ [
+ {
+ "content_id": str,
+ "content_type": str, # "BLOCK", "STORE_AGENT", "DOCUMENTATION", or "LIBRARY_AGENT"
+ "searchable_text": str,
+ "metadata": dict,
+ "similarity": float, # Cosine similarity score (0-1)
+ },
+ ...
+ ]
+
+ Examples:
+ # Search blocks only
+ results = await semantic_search("calculate", content_types=[ContentType.BLOCK])
+
+ # Search blocks and documentation
+ results = await semantic_search(
+ "how to use API",
+ content_types=[ContentType.BLOCK, ContentType.DOCUMENTATION]
+ )
+
+ # Search all public content (default)
+ results = await semantic_search("AI agent")
+
+ # Search user's library agents
+ results = await semantic_search(
+ "my custom agent",
+ content_types=[ContentType.LIBRARY_AGENT],
+ user_id="user123"
+ )
+ """
+ # Default to searching all public content types
+ if content_types is None:
+ content_types = [
+ ContentType.BLOCK,
+ ContentType.STORE_AGENT,
+ ContentType.DOCUMENTATION,
+ ]
+
+ # Validate inputs
+ if not content_types:
+ return [] # Empty content_types would cause invalid SQL (IN ())
+
+ query = query.strip()
+ if not query:
+ return []
+
+ if limit < 1:
+ limit = 1
+ if limit > 100:
+ limit = 100
+
+ # Generate query embedding
+ try:
+ query_embedding = await embed_query(query)
+ # Semantic search with embeddings
+ embedding_str = embedding_to_vector_string(query_embedding)
+
+ # Build params in order: limit, then user_id (if provided), then content types
+ params: list[Any] = [limit]
+ user_filter = ""
+ if user_id is not None:
+ user_filter = 'AND "userId" = ${}'.format(len(params) + 1)
+ params.append(user_id)
+
+ # Add content type parameters and build placeholders dynamically
+ content_type_start_idx = len(params) + 1
+ content_type_placeholders = ", ".join(
+ "$" + str(content_type_start_idx + i) + '::{schema_prefix}"ContentType"'
+ for i in range(len(content_types))
+ )
+ params.extend([ct.value for ct in content_types])
+
+ # Build min_similarity param index before appending
+ min_similarity_idx = len(params) + 1
+ params.append(min_similarity)
+
+ # Use unqualified ::vector and <=> operator - pgvector is in search_path on all environments
+ sql = (
+ """
+ SELECT
+ "contentId" as content_id,
+ "contentType" as content_type,
+ "searchableText" as searchable_text,
+ metadata,
+ 1 - (embedding <=> '"""
+ + embedding_str
+ + """'::vector) as similarity
+ FROM {schema_prefix}"UnifiedContentEmbedding"
+ WHERE "contentType" IN ("""
+ + content_type_placeholders
+ + """)
+ """
+ + user_filter
+ + """
+ AND 1 - (embedding <=> '"""
+ + embedding_str
+ + """'::vector) >= $"""
+ + str(min_similarity_idx)
+ + """
+ ORDER BY similarity DESC
+ LIMIT $1
+ """
+ )
+
+ results = await query_raw_with_schema(sql, *params)
+ return [
+ {
+ "content_id": row["content_id"],
+ "content_type": row["content_type"],
+ "searchable_text": row["searchable_text"],
+ "metadata": row["metadata"],
+ "similarity": float(row["similarity"]),
+ }
+ for row in results
+ ]
+ except Exception as e:
+ logger.warning(f"Semantic search failed, falling back to lexical search: {e}")
+
+ # Fallback to lexical search if embeddings unavailable
+
+ params_lexical: list[Any] = [limit]
+ user_filter = ""
+ if user_id is not None:
+ user_filter = 'AND "userId" = ${}'.format(len(params_lexical) + 1)
+ params_lexical.append(user_id)
+
+ # Add content type parameters and build placeholders dynamically
+ content_type_start_idx = len(params_lexical) + 1
+ content_type_placeholders_lexical = ", ".join(
+ "$" + str(content_type_start_idx + i) + '::{schema_prefix}"ContentType"'
+ for i in range(len(content_types))
+ )
+ params_lexical.extend([ct.value for ct in content_types])
+
+ # Build query param index before appending
+ query_param_idx = len(params_lexical) + 1
+ params_lexical.append(f"%{query}%")
+
+ # Use regular string (not f-string) for template to preserve {schema_prefix} placeholders
+ sql_lexical = (
+ """
+ SELECT
+ "contentId" as content_id,
+ "contentType" as content_type,
+ "searchableText" as searchable_text,
+ metadata,
+ 0.0 as similarity
+ FROM {schema_prefix}"UnifiedContentEmbedding"
+ WHERE "contentType" IN ("""
+ + content_type_placeholders_lexical
+ + """)
+ """
+ + user_filter
+ + """
+ AND "searchableText" ILIKE $"""
+ + str(query_param_idx)
+ + """
+ ORDER BY "updatedAt" DESC
+ LIMIT $1
+ """
+ )
+
+ try:
+ results = await query_raw_with_schema(sql_lexical, *params_lexical)
+ return [
+ {
+ "content_id": row["content_id"],
+ "content_type": row["content_type"],
+ "searchable_text": row["searchable_text"],
+ "metadata": row["metadata"],
+ "similarity": 0.0, # Lexical search doesn't provide similarity
+ }
+ for row in results
+ ]
+ except Exception as e:
+ logger.error(f"Lexical search failed: {e}")
+ return []
diff --git a/autogpt_platform/backend/backend/api/features/store/embeddings_e2e_test.py b/autogpt_platform/backend/backend/api/features/search/embeddings_e2e_test.py
similarity index 98%
rename from autogpt_platform/backend/backend/api/features/store/embeddings_e2e_test.py
rename to autogpt_platform/backend/backend/api/features/search/embeddings_e2e_test.py
index 86af457f50..1f39224878 100644
--- a/autogpt_platform/backend/backend/api/features/store/embeddings_e2e_test.py
+++ b/autogpt_platform/backend/backend/api/features/search/embeddings_e2e_test.py
@@ -16,12 +16,10 @@ from typing import AsyncGenerator
import pytest
from prisma.enums import ContentType
-from backend.api.features.store import embeddings
-from backend.api.features.store.embeddings import EMBEDDING_DIM
-from backend.api.features.store.hybrid_search import (
- hybrid_search,
- unified_hybrid_search,
-)
+from backend.api.features.search import embeddings
+from backend.api.features.search.embeddings import EMBEDDING_DIM
+from backend.api.features.search.hybrid_search import unified_hybrid_search
+from backend.api.features.store.hybrid_search import hybrid_search
# ============================================================================
# Test Fixtures
diff --git a/autogpt_platform/backend/backend/api/features/store/embeddings_schema_test.py b/autogpt_platform/backend/backend/api/features/search/embeddings_schema_test.py
similarity index 72%
rename from autogpt_platform/backend/backend/api/features/store/embeddings_schema_test.py
rename to autogpt_platform/backend/backend/api/features/search/embeddings_schema_test.py
index 5aa13b4d23..3f878e6362 100644
--- a/autogpt_platform/backend/backend/api/features/store/embeddings_schema_test.py
+++ b/autogpt_platform/backend/backend/api/features/search/embeddings_schema_test.py
@@ -9,8 +9,8 @@ from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from prisma.enums import ContentType
-from backend.api.features.store import embeddings
-from backend.api.features.store.embeddings import EMBEDDING_DIM
+from backend.api.features.search import embeddings
+from backend.api.features.search.embeddings import EMBEDDING_DIM
# Schema prefix tests removed - functionality moved to db.raw_with_schema() helper
@@ -138,7 +138,7 @@ async def test_get_embedding_stats_with_schema():
)
with patch(
- "backend.api.features.store.embeddings.CONTENT_HANDLERS",
+ "backend.api.features.search.embeddings.CONTENT_HANDLERS",
{ContentType.STORE_AGENT: mock_handler},
):
result = await embeddings.get_embedding_stats()
@@ -157,11 +157,10 @@ async def test_get_embedding_stats_with_schema():
@pytest.mark.asyncio(loop_scope="session")
@pytest.mark.integration
-async def test_backfill_missing_embeddings_with_schema():
- """Test backfilling embeddings via content handlers."""
- from backend.api.features.store.content_handlers import ContentItem
+async def test_backfill_all_content_types_with_schema():
+ """Backfill loops over CONTENT_HANDLERS and aggregates per-type stats."""
+ from backend.api.features.search.content_handlers import ContentItem
- # Create mock content item
mock_item = ContentItem(
content_id="version-1",
content_type=ContentType.STORE_AGENT,
@@ -169,31 +168,30 @@ async def test_backfill_missing_embeddings_with_schema():
metadata={"name": "Test Agent"},
)
- # Mock handler
mock_handler = MagicMock()
mock_handler.get_missing_items = AsyncMock(return_value=[mock_item])
with patch(
- "backend.api.features.store.embeddings.CONTENT_HANDLERS",
+ "backend.api.features.search.embeddings.CONTENT_HANDLERS",
{ContentType.STORE_AGENT: mock_handler},
):
with patch(
- "backend.api.features.store.embeddings.generate_embedding",
- return_value=[0.1] * EMBEDDING_DIM,
+ "backend.api.features.search.embeddings.ensure_content_embedding",
+ new=AsyncMock(return_value=True),
):
- with patch(
- "backend.api.features.store.embeddings.store_content_embedding",
- return_value=True,
- ):
- result = await embeddings.backfill_missing_embeddings(batch_size=10)
+ result = await embeddings.backfill_all_content_types(batch_size=10)
+
+ mock_handler.get_missing_items.assert_called_once_with(10)
- # Verify handler was called
- mock_handler.get_missing_items.assert_called_once_with(10)
+ by_type = result["by_type"]
+ assert by_type[ContentType.STORE_AGENT.value]["processed"] == 1
+ assert by_type[ContentType.STORE_AGENT.value]["success"] == 1
+ assert by_type[ContentType.STORE_AGENT.value]["failed"] == 0
- # Verify results
- assert result["processed"] == 1
- assert result["success"] == 1
- assert result["failed"] == 0
+ totals = result["totals"]
+ assert totals["processed"] == 1
+ assert totals["success"] == 1
+ assert totals["failed"] == 0
@pytest.mark.asyncio(loop_scope="session")
@@ -204,18 +202,18 @@ async def test_ensure_content_embedding_with_schema():
mock_schema.return_value = "platform"
with patch(
- "backend.api.features.store.embeddings.get_content_embedding"
+ "backend.api.features.search.embeddings.get_content_embedding"
) as mock_get:
# Simulate no existing embedding
mock_get.return_value = None
with patch(
- "backend.api.features.store.embeddings.generate_embedding"
+ "backend.api.features.search.embeddings.generate_embedding"
) as mock_generate:
mock_generate.return_value = [0.1] * EMBEDDING_DIM
with patch(
- "backend.api.features.store.embeddings.store_content_embedding"
+ "backend.api.features.search.embeddings.store_content_embedding"
) as mock_store:
mock_store.return_value = True
@@ -235,57 +233,6 @@ async def test_ensure_content_embedding_with_schema():
assert result is True
[email protected](loop_scope="session")
-async def test_backward_compatibility_store_embedding():
- """Test backward compatibility wrapper for store_embedding."""
- with patch(
- "backend.api.features.store.embeddings.store_content_embedding"
- ) as mock_store:
- mock_store.return_value = True
-
- result = await embeddings.store_embedding(
- version_id="test-version-id",
- embedding=[0.1] * EMBEDDING_DIM,
- tx=None,
- )
-
- # Verify it calls the new function with correct parameters
- assert mock_store.called
- call_args = mock_store.call_args
-
- assert call_args[1]["content_type"] == ContentType.STORE_AGENT
- assert call_args[1]["content_id"] == "test-version-id"
- assert call_args[1]["user_id"] is None
- assert result is True
-
-
[email protected](loop_scope="session")
-async def test_backward_compatibility_get_embedding():
- """Test backward compatibility wrapper for get_embedding."""
- with patch(
- "backend.api.features.store.embeddings.get_content_embedding"
- ) as mock_get:
- mock_get.return_value = {
- "contentType": "STORE_AGENT",
- "contentId": "test-version-id",
- "embedding": "[0.1, 0.2]",
- "createdAt": "2024-01-01",
- "updatedAt": "2024-01-01",
- }
-
- result = await embeddings.get_embedding("test-version-id")
-
- # Verify it calls the new function
- assert mock_get.called
-
- # Verify it transforms to old format
- assert result is not None
- assert result["storeListingVersionId"] == "test-version-id"
- assert "embedding" in result
-
-
@pytest.mark.asyncio(loop_scope="session")
@pytest.mark.integration
async def test_schema_handling_error_cases():
diff --git a/autogpt_platform/backend/backend/api/features/search/embeddings_test.py b/autogpt_platform/backend/backend/api/features/search/embeddings_test.py
new file mode 100644
index 0000000000..6a7acb19a0
--- /dev/null
+++ b/autogpt_platform/backend/backend/api/features/search/embeddings_test.py
@@ -0,0 +1,178 @@
+"""Tests for the generic embedding service in search/embeddings.py."""
+
+from unittest.mock import AsyncMock, MagicMock, patch
+
+import prisma
+import pytest
+from prisma import Prisma
+from prisma.enums import ContentType
+
+from backend.api.features.search import embeddings
+
+
[email protected](autouse=True)
+async def setup_prisma():
+ """Setup Prisma client for tests."""
+ try:
+ Prisma()
+ except prisma.errors.ClientAlreadyRegisteredError:
+ pass
+ yield
+
+
[email protected](loop_scope="session")
+async def test_build_searchable_text():
+ """Test searchable text building from listing fields."""
+ result = embeddings.build_searchable_text(
+ name="AI Assistant",
+ description="A helpful AI assistant for productivity",
+ sub_heading="Boost your productivity",
+ categories=["AI", "Productivity"],
+ )
+
+ expected = "AI Assistant Boost your productivity A helpful AI assistant for productivity AI Productivity"
+ assert result == expected
+
+
[email protected](loop_scope="session")
+async def test_build_searchable_text_empty_fields():
+ """Test searchable text building with empty fields."""
+ result = embeddings.build_searchable_text(
+ name="", description="Test description", sub_heading="", categories=[]
+ )
+
+ assert result == "Test description"
+
+
[email protected](loop_scope="session")
+async def test_generate_embedding_success():
+ """Test successful embedding generation."""
+ mock_client = MagicMock()
+ mock_response = MagicMock()
+ mock_response.data = [MagicMock()]
+ mock_response.data[0].embedding = [0.1, 0.2, 0.3] * 512 # 1536 dimensions
+
+ mock_client.embeddings.create = AsyncMock(return_value=mock_response)
+
+ with patch(
+ "backend.api.features.search.embeddings.get_openai_client"
+ ) as mock_get_client:
+ mock_get_client.return_value = mock_client
+
+ result = await embeddings.generate_embedding("test text")
+
+ assert result is not None
+ assert len(result) == embeddings.EMBEDDING_DIM
+ assert result[0] == 0.1
+
+ mock_client.embeddings.create.assert_called_once_with(
+ model="text-embedding-3-small", input="test text"
+ )
+
+
[email protected](loop_scope="session")
+async def test_generate_embedding_no_api_key():
+ """Test embedding generation without API key."""
+ with patch(
+ "backend.api.features.search.embeddings.get_openai_client"
+ ) as mock_get_client:
+ mock_get_client.return_value = None
+
+ with pytest.raises(RuntimeError, match="openai_internal_api_key not set"):
+ await embeddings.generate_embedding("test text")
+
+
[email protected](loop_scope="session")
+async def test_generate_embedding_api_error():
+ """Test embedding generation with API error."""
+ mock_client = MagicMock()
+ mock_client.embeddings.create = AsyncMock(side_effect=Exception("API Error"))
+
+ with patch(
+ "backend.api.features.search.embeddings.get_openai_client"
+ ) as mock_get_client:
+ mock_get_client.return_value = mock_client
+
+ with pytest.raises(Exception, match="API Error"):
+ await embeddings.generate_embedding("test text")
+
+
[email protected](loop_scope="session")
+async def test_generate_embedding_text_truncation():
+ """Test that long text is properly truncated using tiktoken."""
+ from tiktoken import encoding_for_model
+
+ mock_client = MagicMock()
+ mock_response = MagicMock()
+ mock_response.data = [MagicMock()]
+ mock_response.data[0].embedding = [0.1] * embeddings.EMBEDDING_DIM
+
+ mock_client.embeddings.create = AsyncMock(return_value=mock_response)
+
+ with patch(
+ "backend.api.features.search.embeddings.get_openai_client"
+ ) as mock_get_client:
+ mock_get_client.return_value = mock_client
+
+ # Create text that will exceed 8191 tokens
+ words = [f"word{i}" for i in range(10000)]
+ long_text = " ".join(words) # ~10000 tokens
+
+ await embeddings.generate_embedding(long_text)
+
+ call_args = mock_client.embeddings.create.call_args
+ truncated_text = call_args.kwargs["input"]
+
+ enc = encoding_for_model("text-embedding-3-small")
+ actual_tokens = len(enc.encode(truncated_text))
+
+ assert actual_tokens <= 8191
+ assert actual_tokens >= 8100
+
+
[email protected](loop_scope="session")
+async def test_get_embedding_stats():
+ """Test embedding statistics retrieval."""
+ mock_handler = MagicMock()
+ mock_handler.get_stats = AsyncMock(
+ return_value={
+ "total": 100,
+ "with_embeddings": 75,
+ "without_embeddings": 25,
+ }
+ )
+
+ with patch(
+ "backend.api.features.search.embeddings.CONTENT_HANDLERS",
+ {ContentType.STORE_AGENT: mock_handler},
+ ):
+ result = await embeddings.get_embedding_stats()
+
+ assert "by_type" in result
+ assert "totals" in result
+ assert result["totals"]["total"] == 100
+ assert result["totals"]["with_embeddings"] == 75
+ assert result["totals"]["without_embeddings"] == 25
+ assert result["totals"]["coverage_percent"] == 75.0
+
+
[email protected](loop_scope="session")
+async def test_embedding_to_vector_string():
+ """Test embedding to PostgreSQL vector string conversion."""
+ embedding = [0.1, 0.2, 0.3, -0.4]
+ result = embeddings.embedding_to_vector_string(embedding)
+ assert result == "[0.1,0.2,0.3,-0.4]"
+
+
[email protected](loop_scope="session")
+async def test_embed_query():
+ """Test embed_query function (alias for generate_embedding)."""
+ with patch(
+ "backend.api.features.search.embeddings.generate_embedding"
+ ) as mock_generate:
+ mock_generate.return_value = [0.1, 0.2, 0.3]
+
+ result = await embeddings.embed_query("test query")
+
+ assert result == [0.1, 0.2, 0.3]
+ mock_generate.assert_called_once_with("test query")
diff --git a/autogpt_platform/backend/backend/api/features/search/hybrid_search.py b/autogpt_platform/backend/backend/api/features/search/hybrid_search.py
new file mode 100644
index 0000000000..43bf31d5b7
--- /dev/null
+++ b/autogpt_platform/backend/backend/api/features/search/hybrid_search.py
@@ -0,0 +1,573 @@
+"""
+Unified Hybrid Search — shared engine across features.
+
+Combines semantic (embedding) search with lexical (tsvector) search plus
+BM25 reranking. Searches the ``UnifiedContentEmbedding`` table across all
+content types (store agents, library agents, workspace files, chat
+sessions, blocks, docs).
+
+Feature-specific search functions (e.g. ``store.hybrid_search`` which
+joins to ``StoreAgent`` for popularity/featured metadata) live next to
+the feature they belong to and call into the helpers here.
+"""
+
+import logging
+import re
+import time
+from dataclasses import dataclass
+from datetime import datetime
+from typing import Any, NotRequired, TypedDict, cast
+
+from prisma.enums import ContentType
+from rank_bm25 import BM25Okapi
+
+from backend.api.features.search.embeddings import (
+ EMBEDDING_DIM,
+ embed_query,
+ embedding_to_vector_string,
+)
+from backend.data.db import query_raw_with_schema
+
+
+class HybridSearchRow(TypedDict):
+ """One row from :func:`unified_hybrid_search`.
+
+ The SQL CTE always populates the core columns
+ (``content_id``/``content_type``/``searchable_text``/``metadata``/
+ ``updated_at`` and the score breakdown). Optional fields are either
+ attached by post-processing (``bm25_rerank``) or stripped before
+ the row leaves this module (``total_count``).
+ """
+
+ content_type: ContentType | str
+ content_id: str
+ searchable_text: str
+ metadata: dict[str, Any]
+ updated_at: datetime | None
+ semantic_score: float
+ lexical_score: float
+ lexical_raw: float
+ category_score: float
+ recency_score: float
+ combined_score: float
+ # Window-function side car, identical across all rows in one
+ # response. Popped before the row leaves ``unified_hybrid_search``.
+ total_count: NotRequired[int]
+ # Added by ``bm25_rerank`` after the SQL roundtrip — absent when
+ # rerank short-circuits (empty corpus / empty query tokens).
+ bm25_score: NotRequired[float]
+ final_score: NotRequired[float]
+ relevance: NotRequired[float]
+
+
+logger = logging.getLogger(__name__)
+
+
+# ============================================================================
+# BM25 Reranking
+# ============================================================================
+
+
+def tokenize(text: str) -> list[str]:
+ """Tokenize text for BM25."""
+ if not text:
+ return []
+ return re.findall(r"\b\w+\b", text.lower())
+
+
+def bm25_rerank(
+ query: str,
+ results: list[HybridSearchRow],
+ text_field: str = "searchable_text",
+ bm25_weight: float = 0.3,
+ original_score_field: str = "combined_score",
+) -> list[HybridSearchRow]:
+ """
+ Rerank search results using BM25.
+
+ Combines the original combined_score with BM25 score for improved
+ lexical relevance, especially for exact term matches.
+
+ Args:
+ query: The search query
+ results: List of result dicts with text_field and original_score_field
+ text_field: Field name containing the text to score
+ bm25_weight: Weight for BM25 score (0-1). Original score gets (1 - bm25_weight)
+ original_score_field: Field name containing the original score
+
+ Returns:
+ Results list sorted by combined score (BM25 + original)
+ """
+ if not results or not query:
+ return results
+
+ # Extract texts and tokenize
+ corpus = [tokenize(r.get(text_field, "") or "") for r in results]
+
+ # Handle edge case where all documents are empty
+ if all(len(doc) == 0 for doc in corpus):
+ return results
+
+ # Build BM25 index
+ bm25 = BM25Okapi(corpus)
+
+ # Score query against corpus
+ query_tokens = tokenize(query)
+ if not query_tokens:
+ return results
+
+ bm25_scores = bm25.get_scores(query_tokens)
+
+ # Normalize BM25 scores to 0-1 range
+ max_bm25 = max(bm25_scores) if max(bm25_scores) > 0 else 1.0
+ normalized_bm25 = [s / max_bm25 for s in bm25_scores]
+
+ # Combine scores
+ original_weight = 1.0 - bm25_weight
+ for i, result in enumerate(results):
+ original_score = result.get(original_score_field, 0) or 0
+ result["bm25_score"] = normalized_bm25[i]
+ final_score = (
+ original_weight * original_score + bm25_weight * normalized_bm25[i]
+ )
+ result["final_score"] = final_score
+ result["relevance"] = final_score
+
+ # Sort by relevance descending
+ results.sort(key=lambda x: x.get("relevance", 0), reverse=True)
+
+ return results
+
+
+@dataclass
+class UnifiedSearchWeights:
+ """Weights for unified search (no popularity signal)."""
+
+ semantic: float = 0.40 # Embedding cosine similarity
+ lexical: float = 0.40 # tsvector ts_rank_cd score
+ category: float = 0.10 # Category match boost (for types that have categories)
+ recency: float = 0.10 # Newer content ranked higher
+
+ def __post_init__(self):
+ """Validate weights are non-negative and sum to approximately 1.0."""
+ total = self.semantic + self.lexical + self.category + self.recency
+
+ if any(
+ w < 0 for w in [self.semantic, self.lexical, self.category, self.recency]
+ ):
+ raise ValueError("All weights must be non-negative")
+
+ if not (0.99 <= total <= 1.01):
+ raise ValueError(f"Weights must sum to ~1.0, got {total:.3f}")
+
+
+# Default weights for unified search
+DEFAULT_UNIFIED_WEIGHTS = UnifiedSearchWeights()
+
+# Minimum relevance score thresholds
+DEFAULT_MIN_SCORE = 0.15 # For unified search (more permissive)
+DEFAULT_STORE_AGENT_MIN_SCORE = 0.20 # For store agent search (original threshold)
+
+
+async def unified_hybrid_search(
+ query: str,
+ content_types: list[ContentType] | None = None,
+ category: str | None = None,
+ page: int = 1,
+ page_size: int = 20,
+ weights: UnifiedSearchWeights | None = None,
+ min_score: float | None = None,
+ user_id: str | None = None,
+ lexical_query: str | None = None,
+ prefix_match: bool = False,
+) -> tuple[list[HybridSearchRow], int]:
+ """
+ Unified hybrid search across all content types.
+
+ Searches UnifiedContentEmbedding using both semantic (vector) and lexical (tsvector) signals.
+
+ Args:
+ query: Search query string (used for embedding + lexical unless
+ ``lexical_query`` is provided).
+ content_types: List of content types to search. Defaults to all public types.
+ category: Filter by category (for content types that support it)
+ page: Page number (1-indexed)
+ page_size: Results per page
+ weights: Custom weights for search signals
+ min_score: Minimum relevance score threshold (0-1)
+ user_id: User ID for searching private content (library agents)
+ lexical_query: Optional override for the tsvector ``@@`` candidate
+ selection and ``ts_rank_cd`` scoring. Use when the natural-
+ language query is too long for ``plainto_tsquery``'s
+ AND-of-terms semantics — typically a keyword-extracted form
+ of ``query`` (e.g. ``"youtube summarize video"`` for a goal
+ like ``"summarize a YouTube video..."``). Defaults to
+ ``query`` when None.
+
+ Returns:
+ Tuple of (results list, total count)
+ """
+ # Validate inputs
+ query = query.strip()
+ if not query:
+ return [], 0
+ # Default the lexical query to the semantic query for backwards-
+ # compatibility; callers (e.g. library similarity search) override
+ # when ``plainto_tsquery``'s AND semantics over a long natural goal
+ # would zero out every match.
+ lexical_query = (lexical_query or query).strip() or query
+
+ # Lexical matching mode. ``plainto_tsquery`` matches whole words only,
+ # which is wrong for search-as-you-type callers (e.g. /search/global):
+ # the last token is usually a partial word ("se", "fir"). With
+ # ``prefix_match`` we build a prefix ``to_tsquery`` (``term:*``) from the
+ # query's word tokens so partial words still match. ``to_tsquery`` drops
+ # stopwords gracefully, so an all-stopword fragment simply yields no
+ # lexical hits and falls back to the semantic signal.
+ lexical_fn = "plainto_tsquery"
+ lexical_tsquery = lexical_query
+ if prefix_match:
+ tokens = tokenize(lexical_query)
+ if tokens:
+ lexical_fn = "to_tsquery"
+ lexical_tsquery = " & ".join(f"{t}:*" for t in tokens)
+
+ if page < 1:
+ page = 1
+ if page_size < 1:
+ page_size = 1
+ if page_size > 100:
+ page_size = 100
+
+ if content_types is None:
+ content_types = [
+ ContentType.STORE_AGENT,
+ ContentType.BLOCK,
+ ContentType.DOCUMENTATION,
+ ]
+
+ if weights is None:
+ weights = DEFAULT_UNIFIED_WEIGHTS
+ if min_score is None:
+ min_score = DEFAULT_MIN_SCORE
+
+ offset = (page - 1) * page_size
+
+ # Generate query embedding with graceful degradation
+ try:
+ query_embedding = await embed_query(query)
+ except Exception as e:
+ logger.warning(
+ f"Failed to generate query embedding - falling back to lexical-only search: {e}. "
+ "Check that openai_internal_api_key is configured and OpenAI API is accessible."
+ )
+ query_embedding = [0.0] * EMBEDDING_DIM
+ # Redistribute semantic weight to lexical
+ total_non_semantic = weights.lexical + weights.category + weights.recency
+ if total_non_semantic > 0:
+ factor = 1.0 / total_non_semantic
+ weights = UnifiedSearchWeights(
+ semantic=0.0,
+ lexical=weights.lexical * factor,
+ category=weights.category * factor,
+ recency=weights.recency * factor,
+ )
+ else:
+ weights = UnifiedSearchWeights(
+ semantic=0.0, lexical=1.0, category=0.0, recency=0.0
+ )
+
+ # Build parameters
+ params: list[Any] = []
+ param_idx = 1
+
+ # Query for lexical search (may differ from semantic ``query`` —
+ # see the ``lexical_query`` parameter doc). Carries the prefix
+ # ``to_tsquery`` form when ``prefix_match`` is set.
+ params.append(lexical_tsquery)
+ query_param = f"${param_idx}"
+ param_idx += 1
+
+ # Query lowercase for category matching — keep tied to the
+ # natural-language ``query`` so categories still match the user's
+ # intent words, not the stripped lexical form.
+ params.append(query.lower())
+ query_lower_param = f"${param_idx}"
+ param_idx += 1
+
+ # Embedding
+ embedding_str = embedding_to_vector_string(query_embedding)
+ params.append(embedding_str)
+ embedding_param = f"${param_idx}"
+ param_idx += 1
+
+ # Content types
+ content_type_values = [ct.value for ct in content_types]
+ params.append(content_type_values)
+ content_types_param = f"${param_idx}"
+ param_idx += 1
+
+ # User ID filter (for private content).
+ # Defense-in-depth: a per-user row with NULL userId would be matched by
+ # the public ``OR userId IS NULL`` branch and leak across users. All of
+ # LIBRARY_AGENT, WORKSPACE_FILE, and CHAT_SESSION are always per-user;
+ # explicitly exclude any of those combined with a NULL userId. The DB
+ # CHECK constraint in migration ``20260526120000`` makes that
+ # impossible at write time, but this query-side guard keeps the
+ # invariant local to the search code too.
+ _USER_SCOPED_TYPES_SQL = (
+ "('LIBRARY_AGENT'::{schema_prefix}\"ContentType\", "
+ "'WORKSPACE_FILE'::{schema_prefix}\"ContentType\", "
+ "'CHAT_SESSION'::{schema_prefix}\"ContentType\")"
+ )
+ user_filter = ""
+ if user_id is not None:
+ params.append(user_id)
+ user_filter = (
+ f'AND (uce."userId" = ${param_idx} OR uce."userId" IS NULL) '
+ f'AND NOT (uce."contentType" IN {_USER_SCOPED_TYPES_SQL} '
+ f'AND uce."userId" IS NULL)'
+ )
+ param_idx += 1
+ else:
+ user_filter = 'AND uce."userId" IS NULL'
+
+ # Weights
+ params.append(weights.semantic)
+ w_semantic = f"${param_idx}"
+ param_idx += 1
+
+ params.append(weights.lexical)
+ w_lexical = f"${param_idx}"
+ param_idx += 1
+
+ params.append(weights.category)
+ w_category = f"${param_idx}"
+ param_idx += 1
+
+ params.append(weights.recency)
+ w_recency = f"${param_idx}"
+ param_idx += 1
+
+ # Min score
+ params.append(min_score)
+ min_score_param = f"${param_idx}"
+ param_idx += 1
+
+ # Pagination
+ params.append(page_size)
+ limit_param = f"${param_idx}"
+ param_idx += 1
+
+ params.append(offset)
+ offset_param = f"${param_idx}"
+ param_idx += 1
+
+ # Unified search query on UnifiedContentEmbedding
+ sql_query = f"""
+ WITH candidates AS (
+ -- Lexical matches (uses GIN index on search column)
+ -- Bounded to 500 rows so a broad prefix like ``a:*`` can't
+ -- drag the whole table into ts_rank_cd scoring. Matches the
+ -- semantic branch's bounded design.
+ (
+ SELECT uce.id, uce."contentType", uce."contentId"
+ FROM {{schema_prefix}}"UnifiedContentEmbedding" uce
+ WHERE uce."contentType" = ANY({content_types_param}::{{schema_prefix}}"ContentType"[])
+ {user_filter}
+ AND uce.search @@ {lexical_fn}('english', {query_param})
+ LIMIT 500
+ )
+
+ UNION
+
+ -- Semantic matches (uses HNSW index on embedding)
+ (
+ SELECT uce.id, uce."contentType", uce."contentId"
+ FROM {{schema_prefix}}"UnifiedContentEmbedding" uce
+ WHERE uce."contentType" = ANY({content_types_param}::{{schema_prefix}}"ContentType"[])
+ {user_filter}
+ ORDER BY uce.embedding <=> {embedding_param}::vector
+ LIMIT 200
+ )
+ ),
+ search_scores AS (
+ SELECT
+ uce."contentType" as content_type,
+ uce."contentId" as content_id,
+ uce."searchableText" as searchable_text,
+ uce.metadata,
+ uce."updatedAt" as updated_at,
+ -- Semantic score: cosine similarity (1 - distance)
+ COALESCE(1 - (uce.embedding <=> {embedding_param}::vector), 0) as semantic_score,
+ -- Lexical score: ts_rank_cd
+ COALESCE(ts_rank_cd(uce.search, {lexical_fn}('english', {query_param})), 0) as lexical_raw,
+ -- Category match from metadata
+ CASE
+ WHEN uce.metadata ? 'categories' AND EXISTS (
+ SELECT 1 FROM jsonb_array_elements_text(uce.metadata->'categories') cat
+ WHERE LOWER(cat) LIKE '%' || {query_lower_param} || '%'
+ )
+ THEN 1.0
+ ELSE 0.0
+ END as category_score,
+ -- Recency score: linear decay over 90 days
+ GREATEST(0, 1 - EXTRACT(EPOCH FROM (NOW() - uce."updatedAt")) / (90 * 24 * 3600)) as recency_score
+ FROM candidates c
+ INNER JOIN {{schema_prefix}}"UnifiedContentEmbedding" uce ON c.id = uce.id
+ ),
+ max_lexical AS (
+ SELECT GREATEST(MAX(lexical_raw), 0.001) as max_val FROM search_scores
+ ),
+ normalized AS (
+ SELECT
+ ss.*,
+ ss.lexical_raw / ml.max_val as lexical_score
+ FROM search_scores ss
+ CROSS JOIN max_lexical ml
+ ),
+ scored AS (
+ SELECT
+ content_type,
+ content_id,
+ searchable_text,
+ metadata,
+ updated_at,
+ semantic_score,
+ lexical_score,
+ category_score,
+ recency_score,
+ (
+ {w_semantic} * semantic_score +
+ {w_lexical} * lexical_score +
+ {w_category} * category_score +
+ {w_recency} * recency_score
+ ) as combined_score
+ FROM normalized
+ ),
+ filtered AS (
+ SELECT *, COUNT(*) OVER () as total_count
+ FROM scored
+ WHERE combined_score >= {min_score_param}
+ )
+ SELECT * FROM filtered
+ -- ``content_id`` is the deterministic tiebreaker: when several
+ -- rows tie on ``combined_score`` (common with identical
+ -- embeddings or repeated text), Postgres is free to return any
+ -- order absent a secondary key, which lets the same row land on
+ -- multiple pages under ``LIMIT/OFFSET`` paging.
+ ORDER BY combined_score DESC, content_id ASC
+ LIMIT {limit_param} OFFSET {offset_param}
+ """
+
+ try:
+ raw_results = await query_raw_with_schema(sql_query, *params)
+ except Exception as e:
+ await _log_vector_error_diagnostics(e)
+ raise
+
+ # ``query_raw_with_schema`` returns ``list[dict[str, Any]]`` — the
+ # SQL columns align with the ``HybridSearchRow`` shape, so a single
+ # cast at the boundary lets the rest of the pipeline operate on a
+ # typed row without per-call ``# type: ignore``s.
+ results: list[HybridSearchRow] = cast(list[HybridSearchRow], raw_results)
+
+ total = results[0].get("total_count", 0) if results else 0
+ # Apply BM25 reranking
+ if results:
+ results = bm25_rerank(
+ query=query,
+ results=results,
+ text_field="searchable_text",
+ bm25_weight=0.3,
+ original_score_field="combined_score",
+ )
+
+ # Clean up results
+ for result in results:
+ result.pop("total_count", None)
+
+ logger.info(f"Unified hybrid search: {len(results)} results, {total} total")
+
+ return results, total
+
+
+# ============================================================================
+# Diagnostics
+# ============================================================================
+
+# Rate limit: only log vector error diagnostics once per this interval
+_VECTOR_DIAG_INTERVAL_SECONDS = 60
+_last_vector_diag_time: float = 0
+
+
+async def _log_vector_error_diagnostics(error: Exception) -> None:
+ """Log diagnostic info when 'type vector does not exist' error occurs.
+
+ Note: Diagnostic queries use query_raw_with_schema which may run on a different
+ pooled connection than the one that failed. Session-level search_path can differ,
+ so these diagnostics show cluster-wide state, not necessarily the failed session.
+
+ Includes rate limiting to avoid log spam - only logs once per minute.
+ Caller should re-raise the error after calling this function.
+ """
+ global _last_vector_diag_time
+
+ # Check if this is the vector type error
+ error_str = str(error).lower()
+ if not (
+ "type" in error_str and "vector" in error_str and "does not exist" in error_str
+ ):
+ return
+
+ # Rate limit: only log once per interval
+ now = time.time()
+ if now - _last_vector_diag_time < _VECTOR_DIAG_INTERVAL_SECONDS:
+ return
+ _last_vector_diag_time = now
+
+ try:
+ diagnostics: dict[str, object] = {}
+
+ try:
+ search_path_result = await query_raw_with_schema("SHOW search_path")
+ diagnostics["search_path"] = search_path_result
+ except Exception as e:
+ diagnostics["search_path"] = f"Error: {e}"
+
+ try:
+ schema_result = await query_raw_with_schema("SELECT current_schema()")
+ diagnostics["current_schema"] = schema_result
+ except Exception as e:
+ diagnostics["current_schema"] = f"Error: {e}"
+
+ try:
+ user_result = await query_raw_with_schema(
+ "SELECT current_user, session_user, current_database()"
+ )
+ diagnostics["user_info"] = user_result
+ except Exception as e:
+ diagnostics["user_info"] = f"Error: {e}"
+
+ try:
+ # Check pgvector extension installation (cluster-wide, stable info)
+ ext_result = await query_raw_with_schema(
+ "SELECT extname, extversion, nspname as schema "
+ "FROM pg_extension e "
+ "JOIN pg_namespace n ON e.extnamespace = n.oid "
+ "WHERE extname = 'vector'"
+ )
+ diagnostics["pgvector_extension"] = ext_result
+ except Exception as e:
+ diagnostics["pgvector_extension"] = f"Error: {e}"
+
+ logger.error(
+ f"Vector type error diagnostics:\n"
+ f" Error: {error}\n"
+ f" search_path: {diagnostics.get('search_path')}\n"
+ f" current_schema: {diagnostics.get('current_schema')}\n"
+ f" user_info: {diagnostics.get('user_info')}\n"
+ f" pgvector_extension: {diagnostics.get('pgvector_extension')}"
+ )
+ except Exception as diag_error:
+ logger.error(f"Failed to collect vector error diagnostics: {diag_error}")
diff --git a/autogpt_platform/backend/backend/api/features/search/hybrid_search_test.py b/autogpt_platform/backend/backend/api/features/search/hybrid_search_test.py
new file mode 100644
index 0000000000..2ecf819bc9
--- /dev/null
+++ b/autogpt_platform/backend/backend/api/features/search/hybrid_search_test.py
@@ -0,0 +1,372 @@
+"""
+Tests for the generic unified hybrid search engine.
+"""
+
+from unittest.mock import patch
+
+import pytest
+from prisma.enums import ContentType
+
+from backend.api.features.search import embeddings
+from backend.api.features.search.hybrid_search import (
+ UnifiedSearchWeights,
+ tokenize,
+ unified_hybrid_search,
+)
+
+# ---------------------------------------------------------------------------
+# tokenize (BM25)
+# ---------------------------------------------------------------------------
+
+
+ "input_text, expected",
+ [
+ ("AITextGeneratorBlock", ["aitextgeneratorblock"]),
+ ("hello world", ["hello", "world"]),
+ ("", []),
+ ("HTTPRequest", ["httprequest"]),
+ ],
+)
+def test_tokenize(input_text: str, expected: list[str]):
+ assert tokenize(input_text) == expected
+
+
+# ---------------------------------------------------------------------------
+# unified_hybrid_search
+# ---------------------------------------------------------------------------
+
+
[email protected](loop_scope="session")
+async def test_unified_hybrid_search_basic():
+ """Test basic unified hybrid search across all content types."""
+ mock_results = [
+ {
+ "content_type": "STORE_AGENT",
+ "content_id": "agent-1",
+ "searchable_text": "Test Agent Description",
+ "metadata": {"name": "Test Agent"},
+ "updated_at": "2025-01-01T00:00:00Z",
+ "semantic_score": 0.7,
+ "lexical_score": 0.8,
+ "category_score": 0.5,
+ "recency_score": 0.3,
+ "combined_score": 0.6,
+ "total_count": 2,
+ },
+ {
+ "content_type": "BLOCK",
+ "content_id": "block-1",
+ "searchable_text": "Test Block Description",
+ "metadata": {"name": "Test Block"},
+ "updated_at": "2025-01-01T00:00:00Z",
+ "semantic_score": 0.6,
+ "lexical_score": 0.7,
+ "category_score": 0.4,
+ "recency_score": 0.2,
+ "combined_score": 0.5,
+ "total_count": 2,
+ },
+ ]
+
+ with patch(
+ "backend.api.features.search.hybrid_search.query_raw_with_schema"
+ ) as mock_query:
+ with patch(
+ "backend.api.features.search.hybrid_search.embed_query"
+ ) as mock_embed:
+ mock_query.return_value = mock_results
+ mock_embed.return_value = [0.1] * embeddings.EMBEDDING_DIM
+
+ results, total = await unified_hybrid_search(
+ query="test",
+ page=1,
+ page_size=20,
+ )
+
+ assert len(results) == 2
+ assert total == 2
+ assert results[0]["content_type"] == "STORE_AGENT"
+ assert results[1]["content_type"] == "BLOCK"
+
+
[email protected](loop_scope="session")
+async def test_unified_hybrid_search_filter_by_content_type():
+ """Test unified search filtering by specific content types."""
+ mock_results = [
+ {
+ "content_type": "BLOCK",
+ "content_id": "block-1",
+ "searchable_text": "Test Block",
+ "metadata": {},
+ "updated_at": "2025-01-01T00:00:00Z",
+ "semantic_score": 0.7,
+ "lexical_score": 0.8,
+ "category_score": 0.0,
+ "recency_score": 0.3,
+ "combined_score": 0.5,
+ "total_count": 1,
+ },
+ ]
+
+ with patch(
+ "backend.api.features.search.hybrid_search.query_raw_with_schema"
+ ) as mock_query:
+ with patch(
+ "backend.api.features.search.hybrid_search.embed_query"
+ ) as mock_embed:
+ mock_query.return_value = mock_results
+ mock_embed.return_value = [0.1] * embeddings.EMBEDDING_DIM
+
+ results, total = await unified_hybrid_search(
+ query="test",
+ content_types=[ContentType.BLOCK],
+ page=1,
+ page_size=20,
+ )
+
+ call_args = mock_query.call_args
+ params = call_args[0][1:]
+ assert ["BLOCK"] in params
+
+ assert len(results) == 1
+ assert total == 1
+
+
[email protected](loop_scope="session")
+async def test_unified_hybrid_search_with_user_id():
+ """Test unified search with user_id for private content."""
+ mock_results = [
+ {
+ "content_type": "STORE_AGENT",
+ "content_id": "agent-1",
+ "searchable_text": "My Private Agent",
+ "metadata": {},
+ "updated_at": "2025-01-01T00:00:00Z",
+ "semantic_score": 0.7,
+ "lexical_score": 0.8,
+ "category_score": 0.0,
+ "recency_score": 0.3,
+ "combined_score": 0.6,
+ "total_count": 1,
+ },
+ ]
+
+ with patch(
+ "backend.api.features.search.hybrid_search.query_raw_with_schema"
+ ) as mock_query:
+ with patch(
+ "backend.api.features.search.hybrid_search.embed_query"
+ ) as mock_embed:
+ mock_query.return_value = mock_results
+ mock_embed.return_value = [0.1] * embeddings.EMBEDDING_DIM
+
+ results, total = await unified_hybrid_search(
+ query="test",
+ user_id="user-123",
+ page=1,
+ page_size=20,
+ )
+
+ call_args = mock_query.call_args
+ sql_template = call_args[0][0]
+ params = call_args[0][1:]
+
+ assert 'uce."userId"' in sql_template
+ assert "user-123" in params
+
+
[email protected](loop_scope="session")
+async def test_unified_hybrid_search_excludes_null_user_library_agents():
+ """Defense-in-depth: the SQL filter must NEVER let a LIBRARY_AGENT row
+ with NULL userId match an authenticated query — that would leak private
+ library data across users."""
+ with patch(
+ "backend.api.features.search.hybrid_search.query_raw_with_schema"
+ ) as mock_query:
+ with patch(
+ "backend.api.features.search.hybrid_search.embed_query"
+ ) as mock_embed:
+ mock_query.return_value = []
+ mock_embed.return_value = [0.1] * embeddings.EMBEDDING_DIM
+
+ await unified_hybrid_search(
+ query="email summarizer",
+ user_id="user-123",
+ page=1,
+ page_size=20,
+ )
+
+ sql_template = mock_query.call_args[0][0]
+ assert "LIBRARY_AGENT" in sql_template
+ assert "WORKSPACE_FILE" in sql_template
+ assert "CHAT_SESSION" in sql_template
+
+
[email protected](loop_scope="session")
+async def test_unified_hybrid_search_custom_weights():
+ """Test unified search with custom weights."""
+ custom_weights = UnifiedSearchWeights(
+ semantic=0.6,
+ lexical=0.2,
+ category=0.1,
+ recency=0.1,
+ )
+
+ with patch(
+ "backend.api.features.search.hybrid_search.query_raw_with_schema"
+ ) as mock_query:
+ with patch(
+ "backend.api.features.search.hybrid_search.embed_query"
+ ) as mock_embed:
+ mock_query.return_value = []
+ mock_embed.return_value = [0.1] * embeddings.EMBEDDING_DIM
+
+ results, total = await unified_hybrid_search(
+ query="test",
+ weights=custom_weights,
+ page=1,
+ page_size=20,
+ )
+
+ call_args = mock_query.call_args
+ params = call_args[0][1:]
+
+ assert 0.6 in params # semantic weight
+ assert 0.2 in params # lexical weight
+
+
[email protected](loop_scope="session")
+async def test_unified_hybrid_search_graceful_degradation():
+ """Test unified search gracefully degrades when embeddings unavailable."""
+ mock_results = [
+ {
+ "content_type": "DOCUMENTATION",
+ "content_id": "doc-1",
+ "searchable_text": "API Documentation",
+ "metadata": {},
+ "updated_at": "2025-01-01T00:00:00Z",
+ "semantic_score": 0.0,
+ "lexical_score": 0.8,
+ "category_score": 0.0,
+ "recency_score": 0.2,
+ "combined_score": 0.5,
+ "total_count": 1,
+ },
+ ]
+
+ with patch(
+ "backend.api.features.search.hybrid_search.query_raw_with_schema"
+ ) as mock_query:
+ with patch(
+ "backend.api.features.search.hybrid_search.embed_query"
+ ) as mock_embed:
+ mock_query.return_value = mock_results
+ mock_embed.side_effect = Exception("Embedding generation failed")
+
+ # Should NOT raise - graceful degradation
+ results, total = await unified_hybrid_search(
+ query="test",
+ page=1,
+ page_size=20,
+ )
+
+ assert len(results) == 1
+ assert total == 1
+
+
[email protected](loop_scope="session")
+async def test_unified_hybrid_search_empty_query():
+ """Test unified search with empty query returns empty results."""
+ results, total = await unified_hybrid_search(
+ query="",
+ page=1,
+ page_size=20,
+ )
+
+ assert results == []
+ assert total == 0
+
+
[email protected](loop_scope="session")
+async def test_unified_hybrid_search_pagination():
+ """Test unified search pagination with BM25 reranking."""
+ mock_results = [
+ {
+ "content_type": "STORE_AGENT",
+ "content_id": f"agent-{i}",
+ "searchable_text": f"Agent {i} description",
+ "metadata": {"name": f"Agent {i}"},
+ "updated_at": "2025-01-01T00:00:00Z",
+ "semantic_score": 0.7,
+ "lexical_score": 0.8 - (i * 0.01),
+ "category_score": 0.5,
+ "recency_score": 0.3,
+ "combined_score": 0.6 - (i * 0.01),
+ "total_count": 50,
+ }
+ for i in range(15)
+ ]
+
+ with patch(
+ "backend.api.features.search.hybrid_search.query_raw_with_schema"
+ ) as mock_query:
+ with patch(
+ "backend.api.features.search.hybrid_search.embed_query"
+ ) as mock_embed:
+ mock_query.return_value = mock_results
+ mock_embed.return_value = [0.1] * embeddings.EMBEDDING_DIM
+
+ results, total = await unified_hybrid_search(
+ query="test",
+ page=3,
+ page_size=15,
+ )
+
+ assert len(results) == 15
+ assert total == 50
+
+ call_args = mock_query.call_args
+ params = call_args[0]
+ page_size_param = params[-2]
+ offset_param = params[-1]
+ assert page_size_param == 15
+ assert offset_param == 30 # (page 3 - 1) * 15
+
+
[email protected](loop_scope="session")
+async def test_unified_hybrid_search_schema_prefix():
+ """Test unified search uses schema_prefix placeholder."""
+ with patch(
+ "backend.api.features.search.hybrid_search.query_raw_with_schema"
+ ) as mock_query:
+ with patch(
+ "backend.api.features.search.hybrid_search.embed_query"
+ ) as mock_embed:
+ mock_query.return_value = []
+ mock_embed.return_value = [0.1] * embeddings.EMBEDDING_DIM
+
+ await unified_hybrid_search(
+ query="test",
+ page=1,
+ page_size=20,
+ )
+
+ call_args = mock_query.call_args
+ sql_template = call_args[0][0]
+
+ assert "{schema_prefix}" in sql_template
+ assert '"UnifiedContentEmbedding"' in sql_template
+
+
+if __name__ == "__main__":
+ pytest.main([__file__, "-v", "-s"])
diff --git a/autogpt_platform/backend/backend/api/features/search/model.py b/autogpt_platform/backend/backend/api/features/search/model.py
new file mode 100644
index 0000000000..0c7a31a2bd
--- /dev/null
+++ b/autogpt_platform/backend/backend/api/features/search/model.py
@@ -0,0 +1,47 @@
+"""Pydantic response models for the unified ``/api/search`` endpoint."""
+
+from __future__ import annotations
+
+from datetime import datetime
+from typing import Any, Literal
+
+import pydantic
+
+SearchItemType = Literal[
+ "library_agent",
+ "store_agent",
+ "workspace_file",
+ "chat_session",
+]
+
+
+class SearchResultItem(pydantic.BaseModel):
+ """One row in a /search/global bucket.
+
+ Kept intentionally generic so the frontend can render all three
+ buckets through a single component. Type-specific details live in
+ ``metadata`` (e.g. mime_type for files, creator for store agents).
+ """
+
+ id: str
+ type: SearchItemType
+ title: str
+ subtitle: str | None = None
+ metadata: dict[str, Any] = pydantic.Field(default_factory=dict)
+ # ``None`` for empty-query (recent) responses — no relevance ranking,
+ # just newest-first. Populated for hybrid-search responses.
+ score: float | None = None
+ updated_at: datetime | None = None
+
+
+class GlobalSearchResponse(pydantic.BaseModel):
+ """Top-N hits bucketed by item type.
+
+ For a non-empty query this contains hybrid-search results.
+ For an empty query this contains the most-recently-updated items
+ per bucket (cached per-user with a short TTL).
+ """
+
+ agents: list[SearchResultItem] = pydantic.Field(default_factory=list)
+ files: list[SearchResultItem] = pydantic.Field(default_factory=list)
+ chats: list[SearchResultItem] = pydantic.Field(default_factory=list)
diff --git a/autogpt_platform/backend/backend/api/features/search/rate_limit.py b/autogpt_platform/backend/backend/api/features/search/rate_limit.py
new file mode 100644
index 0000000000..76f9fcecc4
--- /dev/null
+++ b/autogpt_platform/backend/backend/api/features/search/rate_limit.py
@@ -0,0 +1,77 @@
+"""Per-user QPS rate limit for ``/api/search/global``.
+
+Search-as-you-type calls fan out to a paid OpenAI embedding for every
+non-empty query (see ``unified_hybrid_search`` → ``embed_query``). The
+200 ms frontend debounce keeps normal typing well under any reasonable
+cap, so this limiter exists to put a hard ceiling on key-held clients
+or scripts that bypass the debounce and would otherwise burn embedding
+spend without backpressure.
+
+Fixed-window counter in Redis (INCR + EXPIRE on first hit). Approximate
+by design — under high concurrency two calls may both pass against the
+same snapshot, same trade-off as :mod:`backend.copilot.rate_limit`.
+"""
+
+from __future__ import annotations
+
+import logging
+from datetime import UTC, datetime
+
+import fastapi
+
+from backend.data.redis_client import get_redis_async
+
+logger = logging.getLogger(__name__)
+
+# 120 req/minute / user = 2 QPS sustained, generous for the 200 ms-debounced
+# search-as-you-type frontend (~5 keystrokes/sec collapse to ~1 call/sec)
+# while still catching runaway clients.
+GLOBAL_SEARCH_WINDOW_SECONDS = 60
+GLOBAL_SEARCH_MAX_REQUESTS = 120
+
+
+def _window_key(user_id: str, *, now: datetime) -> str:
+ """Per-user fixed-window key. Bucket aligned to ``GLOBAL_SEARCH_WINDOW_SECONDS``
+ so each user has at most one active counter per window."""
+ bucket = int(now.timestamp()) // GLOBAL_SEARCH_WINDOW_SECONDS
+ return f"search:global:rl:{user_id}:{bucket}"
+
+
+async def enforce_global_search_rate_limit(user_id: str) -> None:
+ """Raise HTTP 429 when ``user_id`` exceeds the per-window cap.
+
+ On Redis brown-out we fail *open* (log and let the call through):
+ the worst case is one user briefly burning a few extra embedding
+ calls, which is far less bad than a global Redis blip blocking
+ search for every user.
+ """
+ now = datetime.now(UTC)
+ key = _window_key(user_id, now=now)
+ try:
+ redis = await get_redis_async()
+ # Atomic create-with-TTL then INCR. The previous ``INCR`` +
+ # conditional ``EXPIRE`` was racy: if EXPIRE failed on the
+ # first-hit path (e.g. transient network blip — the exact
+ # "Redis brown-out" the except below catches) the key stuck
+ # around without a TTL until Redis evicted it. ``SET NX EX``
+ # makes the TTL part of the same write that creates the key;
+ # subsequent INCRs preserve it.
+ await redis.set(key, 0, ex=GLOBAL_SEARCH_WINDOW_SECONDS, nx=True)
+ count = await redis.incr(key)
+ except Exception as e:
+ logger.warning(
+ "Global-search rate-limit check failed open for user %s: %s",
+ user_id,
+ e,
+ )
+ return
+
+ if count > GLOBAL_SEARCH_MAX_REQUESTS:
+ raise fastapi.HTTPException(
+ status_code=429,
+ detail=(
+ f"Global search rate limit exceeded "
+ f"({GLOBAL_SEARCH_MAX_REQUESTS} requests per "
+ f"{GLOBAL_SEARCH_WINDOW_SECONDS}s). Try again shortly."
+ ),
+ )
diff --git a/autogpt_platform/backend/backend/api/features/search/rate_limit_test.py b/autogpt_platform/backend/backend/api/features/search/rate_limit_test.py
new file mode 100644
index 0000000000..58c98ecdd7
--- /dev/null
+++ b/autogpt_platform/backend/backend/api/features/search/rate_limit_test.py
@@ -0,0 +1,101 @@
+"""Unit tests for the /search/global per-user QPS rate limiter."""
+
+from __future__ import annotations
+
+from unittest.mock import AsyncMock, MagicMock
+
+import fastapi
+import pytest
+
+from backend.api.features.search import rate_limit
+
+
+def fake_redis(mocker):
+ """Patch ``get_redis_async`` to return a MagicMock with awaitable
+ ``set`` and ``incr`` so each test can drive the counter directly.
+ """
+ redis = MagicMock()
+ redis.set = AsyncMock()
+ redis.incr = AsyncMock()
+ mocker.patch(
+ "backend.api.features.search.rate_limit.get_redis_async",
+ new=AsyncMock(return_value=redis),
+ )
+ return redis
+
+
+async def test_first_hit_creates_key_with_ttl(fake_redis):
+ """``SET NX EX`` is the atomic create-with-TTL that runs on every
+ hit — subsequent hits are no-ops on the SET (key already exists),
+ so the TTL is set exactly once when the window opens."""
+ fake_redis.incr.return_value = 1
+ await rate_limit.enforce_global_search_rate_limit("u1")
+ fake_redis.set.assert_awaited_once()
+ args, kwargs = fake_redis.set.await_args
+ key = args[0]
+ assert kwargs["ex"] == rate_limit.GLOBAL_SEARCH_WINDOW_SECONDS
+ assert kwargs["nx"] is True
+ assert "u1" in key
+
+
+async def test_subsequent_hit_still_calls_set_nx(fake_redis):
+ """Every hit issues ``SET NX EX`` — Redis no-ops the actual write
+ when the key already exists, so the TTL stays put. The point of NX
+ is to make this safe to call unconditionally."""
+ fake_redis.incr.return_value = 2
+ await rate_limit.enforce_global_search_rate_limit("u1")
+ fake_redis.set.assert_awaited_once()
+
+
+async def test_under_limit_passes(fake_redis):
+ """At-the-limit (exactly MAX) is still allowed — the cap is
+ exclusive (count > MAX raises)."""
+ fake_redis.incr.return_value = rate_limit.GLOBAL_SEARCH_MAX_REQUESTS
+ await rate_limit.enforce_global_search_rate_limit("u1")
+
+
+async def test_over_limit_raises_429(fake_redis):
+ """One past the cap raises HTTP 429 with a descriptive detail."""
+ fake_redis.incr.return_value = rate_limit.GLOBAL_SEARCH_MAX_REQUESTS + 1
+ with pytest.raises(fastapi.HTTPException) as exc_info:
+ await rate_limit.enforce_global_search_rate_limit("u1")
+ assert exc_info.value.status_code == 429
+ detail = str(exc_info.value.detail)
+ assert str(rate_limit.GLOBAL_SEARCH_MAX_REQUESTS) in detail
+
+
+async def test_fails_open_on_redis_error(mocker):
+ """Redis brown-out must not block search — fail-open keeps everyone
+ moving while a small burst of one user's embedding spend is the only
+ downside. Caller never sees an exception in this path."""
+ mocker.patch(
+ "backend.api.features.search.rate_limit.get_redis_async",
+ new=AsyncMock(side_effect=RuntimeError("redis down")),
+ )
+ # Should NOT raise.
+ await rate_limit.enforce_global_search_rate_limit("u1")
+
+
+async def test_per_user_keys_are_distinct(fake_redis):
+ """The window key must be derived from ``user_id`` so two users'
+ counters never collide. We can't observe the key directly from
+ ``incr``'s mock without inspecting args."""
+ fake_redis.incr.return_value = 1
+ await rate_limit.enforce_global_search_rate_limit("alice")
+ key_a = fake_redis.incr.await_args.args[0]
+
+ fake_redis.incr.reset_mock()
+ fake_redis.incr.return_value = 1
+ await rate_limit.enforce_global_search_rate_limit("bob")
+ key_b = fake_redis.incr.await_args.args[0]
+
+ assert key_a != key_b
+ assert "alice" in key_a
+ assert "bob" in key_b
diff --git a/autogpt_platform/backend/backend/api/features/search/routes.py b/autogpt_platform/backend/backend/api/features/search/routes.py
new file mode 100644
index 0000000000..5cfffc1fdd
--- /dev/null
+++ b/autogpt_platform/backend/backend/api/features/search/routes.py
@@ -0,0 +1,70 @@
+"""HTTP routes for the unified ``/api/search`` feature.
+
+``GET /api/search/global?q=…&per_type_limit=4`` — bucketed search across
+agents (library + store), workspace files, and chat sessions.
+
+- When ``q`` is non-empty: hybrid search (semantic + lexical + BM25),
+ ranked independently inside each bucket and capped at
+ ``per_type_limit`` items.
+- When ``q`` is empty/omitted: most-recently-updated items per bucket
+ (cached per-user with a short TTL). Same response shape — the
+ frontend renders both with one component.
+"""
+
+from __future__ import annotations
+
+import autogpt_libs.auth as autogpt_auth_lib
+from fastapi import APIRouter, Query, Security
+
+from . import service
+from .model import GlobalSearchResponse
+from .rate_limit import enforce_global_search_rate_limit
+
+# Bound on ``q`` enforced at the API edge so a 100 KB query string
+# cannot reach ``embed_query`` (paid OpenAI call) or the BM25 reranker.
+# 512 chars comfortably covers the human-typeable range; UIs that need
+# more for some internal flow should hit the dedicated feature search
+# endpoints, not the global one.
+MAX_QUERY_LENGTH = 512
+
+router = APIRouter(
+ tags=["search"],
+ dependencies=[Security(autogpt_auth_lib.requires_user)],
+)
+
+
+ "/global",
+ summary="Global search",
+ response_model=GlobalSearchResponse,
+)
+async def global_search_endpoint(
+ q: str = Query(
+ "",
+ max_length=MAX_QUERY_LENGTH,
+ description=(
+ "Free-form search query (semantic + lexical). When empty or "
+ "omitted, the endpoint returns the most-recently-updated "
+ "items per bucket instead."
+ ),
+ ),
+ per_type_limit: int = Query(
+ 4,
+ ge=1,
+ le=10,
+ description=(
+ "Maximum items returned per bucket. The response contains "
+ "*at most* this many agents, files, and chats — fewer when "
+ "the bucket has no/few matches."
+ ),
+ ),
+ user_id: str = Security(autogpt_auth_lib.get_user_id),
+) -> GlobalSearchResponse:
+ # Skip rate-limit on the empty-query branch — it hits the per-user
+ # cache, not OpenAI. Only the non-empty path triggers paid embedding
+ # calls, so that's where the QPS cap matters.
+ if q.strip():
+ await enforce_global_search_rate_limit(user_id)
+ return await service.global_search(
+ query=q, user_id=user_id, per_type_limit=per_type_limit
+ )
diff --git a/autogpt_platform/backend/backend/api/features/store/semantic_search_test.py b/autogpt_platform/backend/backend/api/features/search/semantic_search_test.py
similarity index 85%
rename from autogpt_platform/backend/backend/api/features/store/semantic_search_test.py
rename to autogpt_platform/backend/backend/api/features/search/semantic_search_test.py
index b52f924ee8..578cd3ca31 100644
--- a/autogpt_platform/backend/backend/api/features/store/semantic_search_test.py
+++ b/autogpt_platform/backend/backend/api/features/search/semantic_search_test.py
@@ -3,7 +3,7 @@
import pytest
from prisma.enums import ContentType
-from backend.api.features.store.embeddings import EMBEDDING_DIM, semantic_search
+from backend.api.features.search.embeddings import EMBEDDING_DIM, semantic_search
@pytest.mark.asyncio
@@ -12,7 +12,7 @@ async def test_search_blocks_only(mocker):
# Mock embed_query to return a test embedding
mock_embedding = [0.1] * EMBEDDING_DIM
mocker.patch(
- "backend.api.features.store.embeddings.embed_query",
+ "backend.api.features.search.embeddings.embed_query",
return_value=mock_embedding,
)
@@ -27,7 +27,7 @@ async def test_search_blocks_only(mocker):
}
]
mocker.patch(
- "backend.api.features.store.embeddings.query_raw_with_schema",
+ "backend.api.features.search.embeddings.query_raw_with_schema",
return_value=mock_results,
)
@@ -47,7 +47,7 @@ async def test_search_multiple_content_types(mocker):
"""Test searching multiple content types simultaneously."""
mock_embedding = [0.1] * EMBEDDING_DIM
mocker.patch(
- "backend.api.features.store.embeddings.embed_query",
+ "backend.api.features.search.embeddings.embed_query",
return_value=mock_embedding,
)
@@ -68,7 +68,7 @@ async def test_search_multiple_content_types(mocker):
},
]
mocker.patch(
- "backend.api.features.store.embeddings.query_raw_with_schema",
+ "backend.api.features.search.embeddings.query_raw_with_schema",
return_value=mock_results,
)
@@ -87,7 +87,7 @@ async def test_search_with_min_similarity_threshold(mocker):
"""Test that results below min_similarity are filtered out."""
mock_embedding = [0.1] * EMBEDDING_DIM
mocker.patch(
- "backend.api.features.store.embeddings.embed_query",
+ "backend.api.features.search.embeddings.embed_query",
return_value=mock_embedding,
)
@@ -102,7 +102,7 @@ async def test_search_with_min_similarity_threshold(mocker):
}
]
mocker.patch(
- "backend.api.features.store.embeddings.query_raw_with_schema",
+ "backend.api.features.search.embeddings.query_raw_with_schema",
return_value=mock_results,
)
@@ -121,7 +121,7 @@ async def test_search_fallback_to_lexical(mocker):
"""Test fallback to lexical search when embeddings fail."""
# Mock embed_query to return None (embeddings unavailable)
mocker.patch(
- "backend.api.features.store.embeddings.embed_query",
+ "backend.api.features.search.embeddings.embed_query",
return_value=None,
)
@@ -135,7 +135,7 @@ async def test_search_fallback_to_lexical(mocker):
}
]
mocker.patch(
- "backend.api.features.store.embeddings.query_raw_with_schema",
+ "backend.api.features.search.embeddings.query_raw_with_schema",
return_value=mock_lexical_results,
)
@@ -163,7 +163,7 @@ async def test_search_with_user_id_filter(mocker):
"""Test searching with user_id filter for private content."""
mock_embedding = [0.1] * EMBEDDING_DIM
mocker.patch(
- "backend.api.features.store.embeddings.embed_query",
+ "backend.api.features.search.embeddings.embed_query",
return_value=mock_embedding,
)
@@ -177,7 +177,7 @@ async def test_search_with_user_id_filter(mocker):
}
]
mocker.patch(
- "backend.api.features.store.embeddings.query_raw_with_schema",
+ "backend.api.features.search.embeddings.query_raw_with_schema",
return_value=mock_results,
)
@@ -196,7 +196,7 @@ async def test_search_limit_parameter(mocker):
"""Test that limit parameter correctly limits results."""
mock_embedding = [0.1] * EMBEDDING_DIM
mocker.patch(
- "backend.api.features.store.embeddings.embed_query",
+ "backend.api.features.search.embeddings.embed_query",
return_value=mock_embedding,
)
@@ -212,7 +212,7 @@ async def test_search_limit_parameter(mocker):
for i in range(5)
]
mocker.patch(
- "backend.api.features.store.embeddings.query_raw_with_schema",
+ "backend.api.features.search.embeddings.query_raw_with_schema",
return_value=mock_results,
)
@@ -230,12 +230,12 @@ async def test_search_default_content_types(mocker):
"""Test that default content_types includes BLOCK, STORE_AGENT, and DOCUMENTATION."""
mock_embedding = [0.1] * EMBEDDING_DIM
mocker.patch(
- "backend.api.features.store.embeddings.embed_query",
+ "backend.api.features.search.embeddings.embed_query",
return_value=mock_embedding,
)
mock_query_raw = mocker.patch(
- "backend.api.features.store.embeddings.query_raw_with_schema",
+ "backend.api.features.search.embeddings.query_raw_with_schema",
return_value=[],
)
@@ -253,13 +253,13 @@ async def test_search_handles_database_error(mocker):
"""Test that database errors are handled gracefully."""
mock_embedding = [0.1] * EMBEDDING_DIM
mocker.patch(
- "backend.api.features.store.embeddings.embed_query",
+ "backend.api.features.search.embeddings.embed_query",
return_value=mock_embedding,
)
# Simulate database error
mocker.patch(
- "backend.api.features.store.embeddings.query_raw_with_schema",
+ "backend.api.features.search.embeddings.query_raw_with_schema",
side_effect=Exception("Database connection failed"),
)
diff --git a/autogpt_platform/backend/backend/api/features/search/service.py b/autogpt_platform/backend/backend/api/features/search/service.py
new file mode 100644
index 0000000000..83032173f5
--- /dev/null
+++ b/autogpt_platform/backend/backend/api/features/search/service.py
@@ -0,0 +1,466 @@
+"""Service layer for the unified ``/api/search/global`` endpoint.
+
+For a non-empty query, ``global_search`` fan-outs to
+``unified_hybrid_search`` three times in parallel (once per bucket) so a
+slow embedding/lexical call for one content type doesn't stall the
+others.
+
+For an empty/whitespace query, ``global_search`` returns the
+most-recently-updated items per bucket. That branch uses three plain DB
+queries — no embedding needed since it's purely
+``ORDER BY updatedAt DESC LIMIT N`` — and the result is cached per-user
+with a short TTL because the UI surfaces this listing on every
+page-load.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+from typing import Any
+
+import pydantic
+from prisma.enums import ContentType
+
+from backend.api.features.search import hybrid_search
+from backend.api.features.search.model import (
+ GlobalSearchResponse,
+ SearchItemType,
+ SearchResultItem,
+)
+from backend.copilot.model import get_user_sessions
+from backend.data.db import query_raw_with_schema
+from backend.data.db_accessors import library_db, search
+from backend.data.workspace import get_workspace
+from backend.util.cache import cached
+from backend.util.workspace import WorkspaceManager
+
+logger = logging.getLogger(__name__)
+
+
+# Minimum length of the trailing token before we enable
+# ``to_tsquery(... 'token:*')`` prefix matching for hybrid search.
+# A single-char prefix would match a large fraction of the embedding
+# table (the lexical candidate branch in ``unified_hybrid_search`` has
+# no LIMIT before scoring), so we wait for the user to type the second
+# character before turning prefix-match on.
+_PREFIX_MATCH_MIN_TAIL_LEN = 2
+
+
+def _should_prefix_match(query: str) -> bool:
+ """Whether to enable prefix-match for a search-as-you-type query.
+
+ Looks at the trailing token because that's the one the user is
+ actively typing — earlier tokens are assumed complete.
+ """
+ tail = query.rsplit(maxsplit=1)[-1] if query.strip() else ""
+ return len(tail) >= _PREFIX_MATCH_MIN_TAIL_LEN
+
+
+# ----- helpers ---------------------------------------------------------------
+
+
+# Pydantic coerces both ``ContentType`` enum members (in-process accessor
+# path) and bare strings (raw-SQL / RPC shim path) into the enum at the
+# boundary, so the dispatch below can rely on a typed value rather than
+# duck-typing the input with ``getattr``.
+_CONTENT_TYPE_ADAPTER = pydantic.TypeAdapter(ContentType)
+
+# Hybrid-search rows carry their metadata as a JSON column; Prisma
+# deserialises it to a dict on the way out. A typed adapter at the
+# boundary turns "you probably get a dict" into "you definitely get a
+# dict (or this row gets skipped)" so downstream ``.get(...)`` is safe.
+_METADATA_ADAPTER = pydantic.TypeAdapter(dict[str, Any])
+
+
+def _hybrid_row_to_item(row: hybrid_search.HybridSearchRow) -> SearchResultItem | None:
+ """Convert one ``unified_hybrid_search`` row into a ``SearchResultItem``.
+
+ Returns None when the row's content type is one we don't surface
+ through /search/global (e.g. BLOCK / DOCUMENTATION rows that might
+ sneak in if a caller widens the type filter), or when the row's
+ ``content_type`` / ``metadata`` columns fail to coerce to their
+ expected shapes (defensive — should not happen in practice).
+ """
+ try:
+ content_type = _CONTENT_TYPE_ADAPTER.validate_python(row.get("content_type"))
+ metadata = _METADATA_ADAPTER.validate_python(row.get("metadata") or {})
+ except pydantic.ValidationError:
+ return None
+
+ item_type: SearchItemType
+ if content_type is ContentType.LIBRARY_AGENT:
+ title = metadata.get("name") or row.get("searchable_text") or ""
+ item_type = "library_agent"
+ subtitle = None
+ elif content_type is ContentType.STORE_AGENT:
+ title = metadata.get("name") or row.get("searchable_text") or ""
+ item_type = "store_agent"
+ cats = metadata.get("categories") or []
+ subtitle = ", ".join(cats[:2]) if cats else None
+ elif content_type is ContentType.WORKSPACE_FILE:
+ title = metadata.get("name") or row.get("searchable_text") or ""
+ item_type = "workspace_file"
+ subtitle = metadata.get("mime_type") or metadata.get("path")
+ elif content_type is ContentType.CHAT_SESSION:
+ title = metadata.get("title") or row.get("searchable_text") or ""
+ item_type = "chat_session"
+ subtitle = None
+ else:
+ return None
+
+ score = row.get("relevance")
+ if score is None:
+ score = row.get("combined_score")
+
+ return SearchResultItem(
+ id=row.get("content_id") or "",
+ type=item_type,
+ title=title,
+ subtitle=subtitle,
+ metadata=metadata,
+ score=float(score) if score is not None else None,
+ updated_at=row.get("updated_at"),
+ )
+
+
+async def _search_bucket(
+ query: str,
+ user_id: str,
+ content_types: list[ContentType],
+ limit: int,
+) -> list[SearchResultItem]:
+ """Run unified_hybrid_search for one bucket; swallow failures so a
+ broken bucket doesn't 500 the whole response."""
+ try:
+ rows, _total = await search().unified_hybrid_search(
+ query=query,
+ content_types=content_types,
+ user_id=user_id,
+ page=1,
+ page_size=limit,
+ # The command palette must return *nothing* for unrelated
+ # queries — cosine similarity between two embeddings is
+ # almost never 0, so a low floor lets the semantic UNION
+ # surface arbitrary recent rows for queries like
+ # "hello walhalla". Match the store-agent threshold (0.20).
+ min_score=hybrid_search.DEFAULT_STORE_AGENT_MIN_SCORE,
+ # Search-as-you-type: the trailing token is usually a partial
+ # word, so prefix-match the lexical signal ("se" -> "se:*").
+ # Gated on the trailing token having >= 2 chars: a single
+ # letter ("s:*") would match a huge fraction of the embedding
+ # table and let the lexical candidate set explode before
+ # ts_rank_cd scoring (the lexical branch isn't capped like
+ # the semantic branch's LIMIT 200). For 1-char queries we
+ # fall back to whole-word plainto_tsquery, which usually
+ # yields no matches and lets the semantic signal take over.
+ prefix_match=_should_prefix_match(query),
+ )
+ except Exception as e:
+ logger.warning(
+ "Hybrid search failed for bucket %s: %s",
+ [ct.value for ct in content_types],
+ e,
+ )
+ return []
+
+ items: list[SearchResultItem] = []
+ for row in rows:
+ item = _hybrid_row_to_item(row)
+ if item is not None:
+ items.append(item)
+ if len(items) >= limit:
+ break
+
+ # Read-time enrichment so store-agent rows embedded BEFORE the
+ # ``creator``/``slug`` metadata fields were added still link to the
+ # marketplace correctly. Newer embeds carry these in the JSON
+ # column; older ones need a join back to ``StoreListing`` + Profile
+ # to be clickable.
+ await _enrich_store_agent_metadata(items)
+ return items
+
+
+async def _enrich_store_agent_metadata(items: list[SearchResultItem]) -> None:
+ """Backfill ``creator`` and ``slug`` on any store-agent items that
+ are missing them.
+
+ Mutates ``items`` in place. Single batched SQL roundtrip, runs only
+ when at least one store-agent item is missing the fields, so the
+ common-case (all rows embedded post-fix) pays nothing.
+ """
+ stale_ids = [
+ item.id
+ for item in items
+ if item.type == "store_agent"
+ and not (item.metadata.get("creator") and item.metadata.get("slug"))
+ ]
+ if not stale_ids:
+ return
+
+ try:
+ rows = await query_raw_with_schema(
+ """
+ SELECT slv.id, sl.slug, p.username AS creator
+ FROM {schema_prefix}"StoreListingVersion" slv
+ JOIN {schema_prefix}"StoreListing" sl
+ ON slv."storeListingId" = sl.id
+ JOIN {schema_prefix}"Profile" p
+ ON p."userId" = sl."owningUserId"
+ WHERE slv.id = ANY($1::text[])
+ AND sl."isDeleted" = false
+ """,
+ stale_ids,
+ )
+ except Exception as e:
+ # Enrichment is purely a click-routing concern — never let it
+ # 500 the whole bucket. A search result with a missing
+ # marketplace URL renders as inert (handled by the FE click
+ # guard), which matches current behaviour.
+ logger.warning("Failed to enrich store-agent metadata: %s", e)
+ return
+
+ by_id = {row["id"]: row for row in rows}
+ for item in items:
+ if item.type != "store_agent":
+ continue
+ enrichment = by_id.get(item.id)
+ if enrichment is None:
+ continue
+ item.metadata["creator"] = enrichment["creator"]
+ item.metadata["slug"] = enrichment["slug"]
+
+
+# ----- recent (empty-query) buckets ------------------------------------------
+
+
+async def _recent_agents(user_id: str, limit: int) -> list[SearchResultItem]:
+ """Most-recently-updated library agents for the user."""
+ # Local import — library.model pulls in a heavy graph chain that we
+ # don't want to load at search-module import time.
+ from backend.api.features.library import model as library_model
+
+ try:
+ resp = await library_db().list_library_agents(
+ user_id=user_id,
+ page=1,
+ page_size=limit,
+ sort_by=library_model.LibraryAgentSort.UPDATED_AT,
+ )
+ except Exception as e:
+ logger.warning("Failed to list recent library agents for %s: %s", user_id, e)
+ return []
+
+ items: list[SearchResultItem] = []
+ for agent in resp.agents[:limit]:
+ items.append(
+ SearchResultItem(
+ id=agent.id,
+ type="library_agent",
+ title=agent.name,
+ subtitle=agent.description or None,
+ metadata={
+ "graph_id": agent.graph_id,
+ "image_url": agent.image_url,
+ "is_favorite": agent.is_favorite,
+ },
+ updated_at=agent.updated_at,
+ )
+ )
+ return items
+
+
+# Over-fetch so the in-Python relevance rerank has enough candidates
+# to actually re-order. Bounded so a user with thousands of name-
+# matching files doesn't blow the bucket cost.
+_RELEVANCE_OVERFETCH_CAP = 32
+
+
+def _title_relevance_score(title: str, query: str) -> int:
+ """Three-tier literal-match score, highest = best.
+
+ Exact case-insensitive match (3) > startswith (2) > contains (1).
+ Falls back to 0 when nothing matches, which shouldn't happen in
+ practice because the DB filter already applied ``contains``.
+ """
+ title_lower = title.lower()
+ query_lower = query.lower()
+ if title_lower == query_lower:
+ return 3
+ if title_lower.startswith(query_lower):
+ return 2
+ if query_lower in title_lower:
+ return 1
+ return 0
+
+
+async def _files_bucket(
+ user_id: str, limit: int, query: str | None = None
+) -> list[SearchResultItem]:
+ """Workspace files, filtered by ``query`` substring when provided.
+
+ Uses a direct ``ILIKE`` filter on ``name`` instead of the embedding
+ index so newly-written files are findable immediately. Workspace
+ file embeddings only encode the name anyway, so we lose nothing in
+ quality and gain freshness.
+
+ When ``query`` is non-empty we over-fetch up to
+ ``_RELEVANCE_OVERFETCH_CAP`` candidates and re-rank by literal
+ name-match relevance before truncating to ``limit``, so a stale
+ exact match still beats four fresh substring matches. The recents
+ branch (empty query) keeps the DB-side ``createdAt DESC`` ordering
+ untouched.
+ """
+ try:
+ workspace = await get_workspace(user_id)
+ if workspace is None:
+ # User has no workspace yet — nothing to list, no need to
+ # create one just for the recents view.
+ return []
+ manager = WorkspaceManager(user_id, workspace.id, session_id=None)
+ fetch_limit = max(limit, _RELEVANCE_OVERFETCH_CAP) if query else limit
+ files = await manager.list_files(
+ limit=fetch_limit,
+ include_all_sessions=True,
+ name_contains=query or None,
+ )
+ except Exception as e:
+ logger.warning("Failed to list workspace files for %s: %s", user_id, e)
+ return []
+
+ if query:
+ # Sort by relevance desc, then by ``createdAt`` desc as
+ # tiebreaker so the freshness ordering is preserved within a
+ # relevance bucket. ``list_files`` already returns
+ # createdAt-desc, so a stable sort keeps that tiebreaker.
+ files = sorted(
+ files, key=lambda f: _title_relevance_score(f.name, query), reverse=True
+ )
+
+ items: list[SearchResultItem] = []
+ for file in files[:limit]:
+ items.append(
+ SearchResultItem(
+ id=file.id,
+ type="workspace_file",
+ title=file.name,
+ subtitle=file.mime_type or file.path,
+ metadata={
+ "path": file.path,
+ "mime_type": file.mime_type,
+ "size_bytes": file.size_bytes,
+ },
+ updated_at=file.updated_at,
+ )
+ )
+ return items
+
+
+async def _chats_bucket(
+ user_id: str, limit: int, query: str | None = None
+) -> list[SearchResultItem]:
+ """Chat sessions, filtered by ``query`` substring when provided.
+
+ Uses a direct ``ILIKE`` filter on ``title`` instead of the embedding
+ index so newly-renamed sessions are findable immediately. Chat
+ session embeddings only encode the title anyway.
+
+ Non-empty queries over-fetch + relevance-rerank exactly like
+ ``_files_bucket`` (see its docstring). Empty queries keep the
+ ``updatedAt DESC`` recents ordering.
+ """
+ try:
+ fetch_limit = max(limit, _RELEVANCE_OVERFETCH_CAP) if query else limit
+ sessions, _total = await get_user_sessions(
+ user_id=user_id,
+ limit=fetch_limit,
+ offset=0,
+ title_contains=query or None,
+ )
+ except Exception as e:
+ logger.warning("Failed to list chat sessions for %s: %s", user_id, e)
+ return []
+
+ if query:
+ # Untitled sessions still ride the recents path elsewhere; this
+ # branch only fires with a query, where ``title_contains``
+ # already filters them out. The placeholder fallback below
+ # ("Untitled chat") is therefore unreachable here, but stays as
+ # a defensive read.
+ sessions = sorted(
+ sessions,
+ key=lambda s: _title_relevance_score(s.title or "", query),
+ reverse=True,
+ )
+
+ items: list[SearchResultItem] = []
+ for session in sessions[:limit]:
+ items.append(
+ SearchResultItem(
+ id=session.session_id,
+ type="chat_session",
+ title=session.title or "Untitled chat",
+ metadata={
+ "chat_status": session.chat_status,
+ # Exposed so the search result UI can swap the generic
+ # chat icon for a platform-specific one (e.g. Discord)
+ # when the session originated from an external chat.
+ "source_platform": session.metadata.source_platform,
+ },
+ updated_at=session.updated_at,
+ )
+ )
+ return items
+
+
+# 60-second TTL: short enough that freshly-created items show up
+# promptly, long enough to absorb rapid page-load chatter from the
+# sidebar. Redis-backed so the cache survives across worker processes.
+@cached(ttl_seconds=60, shared_cache=True, cache_none=False)
+async def _cached_recent_buckets(user_id: str, limit: int) -> GlobalSearchResponse:
+ agents, files, chats = await asyncio.gather(
+ _recent_agents(user_id, limit),
+ _files_bucket(user_id, limit),
+ _chats_bucket(user_id, limit),
+ )
+ return GlobalSearchResponse(agents=agents, files=files, chats=chats)
+
+
+# ----- /search/global --------------------------------------------------------
+
+
+async def global_search(
+ query: str, user_id: str, per_type_limit: int = 4
+) -> GlobalSearchResponse:
+ """Bucketed search across agents (library + store), files, chat sessions.
+
+ - Non-empty ``query``:
+ - Agents: hybrid search (semantic + lexical) via the embedding
+ index — store agents have rich descriptions worth embedding.
+ - Files & chats: direct ``ILIKE`` on ``name`` / ``title`` so
+ freshly-created rows are findable without waiting on async
+ embedding generation. Their embeddings only encode the
+ name/title anyway, so we lose nothing in quality.
+ - Empty/whitespace ``query``: most-recently-updated items per bucket,
+ cached per-user for 60s.
+ """
+ query = (query or "").strip()
+ limit = max(1, min(per_type_limit, 10))
+
+ if not query:
+ return await _cached_recent_buckets(user_id, limit)
+
+ agents, files, chats = await asyncio.gather(
+ _search_bucket(
+ query,
+ user_id,
+ [ContentType.LIBRARY_AGENT, ContentType.STORE_AGENT],
+ limit,
+ ),
+ # Files & chats bypass the embedding index — see _files_bucket /
+ # _chats_bucket docstrings. Direct ILIKE keeps freshly-created
+ # rows findable without waiting on async embedding generation.
+ _files_bucket(user_id, limit, query=query),
+ _chats_bucket(user_id, limit, query=query),
+ )
+ return GlobalSearchResponse(agents=agents, files=files, chats=chats)
diff --git a/autogpt_platform/backend/backend/api/features/search/service_test.py b/autogpt_platform/backend/backend/api/features/search/service_test.py
new file mode 100644
index 0000000000..766e541f77
--- /dev/null
+++ b/autogpt_platform/backend/backend/api/features/search/service_test.py
@@ -0,0 +1,477 @@
+"""Unit tests for the unified search service."""
+
+from __future__ import annotations
+
+import datetime
+from unittest.mock import AsyncMock, MagicMock, patch
+
+import pytest
+from prisma.enums import ContentType
+
+from backend.api.features.search import service
+from backend.api.features.search.model import GlobalSearchResponse
+
+
+def _patch_hybrid(return_value):
+ """Patch the db_accessors.search() shim used by service.global_search."""
+ mock_shim = MagicMock()
+ mock_shim.unified_hybrid_search = AsyncMock(return_value=return_value)
+ return (
+ patch(
+ "backend.api.features.search.service.search",
+ return_value=mock_shim,
+ ),
+ mock_shim,
+ )
+
+
+# ============================================================================
+# Fixtures + fakes
+# ============================================================================
+
+
+def _fake_library_agent(agent_id: str, name: str = "A"):
+ from backend.api.features.library import model as library_model
+ from backend.util.models import Pagination
+
+ agent = library_model.LibraryAgent.model_construct(
+ id=agent_id,
+ graph_id=agent_id,
+ graph_version=1,
+ name=name,
+ description="d",
+ image_url=None,
+ creator_name="c",
+ creator_image_url="",
+ input_schema={},
+ output_schema={},
+ credentials_input_schema=None,
+ has_external_trigger=False,
+ has_human_in_the_loop=False,
+ has_sensitive_action=False,
+ status=library_model.LibraryAgentStatus.COMPLETED,
+ new_output=False,
+ can_access_graph=True,
+ is_latest_version=True,
+ is_favorite=False,
+ created_at=datetime.datetime(2024, 1, 1),
+ updated_at=datetime.datetime(2024, 1, 1),
+ )
+ return library_model.LibraryAgentResponse(
+ agents=[agent],
+ pagination=Pagination(
+ total_items=1, total_pages=1, current_page=1, page_size=3
+ ),
+ )
+
+
+def _fake_workspace_file(file_id: str = "f1"):
+ from backend.data.workspace import WorkspaceFile
+
+ return WorkspaceFile(
+ id=file_id,
+ workspace_id="ws-1",
+ created_at=datetime.datetime(2024, 1, 1),
+ updated_at=datetime.datetime(2024, 1, 2),
+ name="report.pdf",
+ path="/documents/report.pdf",
+ storage_path="ws-1/documents/report.pdf",
+ mime_type="application/pdf",
+ size_bytes=123,
+ )
+
+
+def _fake_chat_session(session_id: str = "s1", title: str | None = "Title"):
+ from backend.copilot.model import ChatSessionInfo
+
+ return ChatSessionInfo(
+ session_id=session_id,
+ user_id="u1",
+ title=title,
+ usage=[],
+ started_at=datetime.datetime(2024, 1, 1),
+ updated_at=datetime.datetime(2024, 1, 5),
+ )
+
+
[email protected](autouse=True)
+def _clear_recent_cache():
+ """The empty-query branch is decorated with ``@cached`` — clear the
+ cache between tests so stale entries don't leak."""
+ service._cached_recent_buckets.cache_clear()
+ yield
+ service._cached_recent_buckets.cache_clear()
+
+
+def _mock_recent_sources(
+ mocker,
+ *,
+ library_response=None,
+ workspace=None,
+ files=None,
+ sessions=(),
+):
+ """Patch the three recent-bucket sources with sensible defaults.
+
+ Returns the library_db mock so tests can assert on its call count.
+ """
+ mock_lib = MagicMock()
+ mock_lib.list_library_agents = AsyncMock(
+ return_value=library_response or _fake_library_agent("a-1")
+ )
+ mocker.patch(
+ "backend.api.features.search.service.library_db", return_value=mock_lib
+ )
+
+ # Mock at the *use* site (where ``service`` rebinds the imported
+ # symbol) rather than the *source* site. Service now imports these
+ # at module top level (per ``AGENTS.md``), so patching the source
+ # module no longer affects the already-bound name inside service.
+ mocker.patch(
+ "backend.api.features.search.service.get_workspace",
+ new=AsyncMock(return_value=workspace),
+ )
+ if files is not None:
+ mock_manager = MagicMock()
+ mock_manager.list_files = AsyncMock(return_value=files)
+ mocker.patch(
+ "backend.api.features.search.service.WorkspaceManager",
+ return_value=mock_manager,
+ )
+
+ mocker.patch(
+ "backend.api.features.search.service.get_user_sessions",
+ new=AsyncMock(return_value=(list(sessions), len(sessions))),
+ )
+ return mock_lib
+
+
+# ============================================================================
+# global_search — hybrid branch (non-empty query)
+# ============================================================================
+
+
+async def test_global_search_buckets_results_by_content_type(mocker):
+ """One hybrid call for agents; files & chats use direct DB queries —
+ rows are sorted into the right bucket regardless of the source path."""
+
+ def hybrid_side_effect(*, content_types, **_):
+ type_set = {ct.value for ct in content_types}
+ if "LIBRARY_AGENT" in type_set or "STORE_AGENT" in type_set:
+ return (
+ [
+ {
+ "content_id": "agent-1",
+ "content_type": ContentType.LIBRARY_AGENT,
+ "metadata": {"name": "My Agent"},
+ "combined_score": 0.8,
+ "updated_at": datetime.datetime(2024, 1, 1),
+ },
+ {
+ "content_id": "store-1",
+ "content_type": ContentType.STORE_AGENT,
+ "metadata": {"name": "Store Agent", "categories": ["ai"]},
+ "combined_score": 0.7,
+ "updated_at": datetime.datetime(2024, 1, 2),
+ },
+ ],
+ 2,
+ )
+ return ([], 0)
+
+ _mock_recent_sources(
+ mocker,
+ workspace=MagicMock(id="ws-1"),
+ files=[_fake_workspace_file("file-1")],
+ sessions=[_fake_chat_session("chat-1", title="My Chat")],
+ )
+
+ mock_shim = MagicMock()
+ mock_shim.unified_hybrid_search = AsyncMock(side_effect=hybrid_side_effect)
+ with patch("backend.api.features.search.service.search", return_value=mock_shim):
+ result = await service.global_search(
+ query="anything", user_id="u1", per_type_limit=4
+ )
+
+ # Only the agents bucket still goes through unified_hybrid_search;
+ # files and chats now route through direct DB queries (see
+ # _files_bucket / _chats_bucket).
+ assert mock_shim.unified_hybrid_search.await_count == 1
+
+ assert [a.id for a in result.agents] == ["agent-1", "store-1"]
+ assert [a.type for a in result.agents] == ["library_agent", "store_agent"]
+ assert [f.id for f in result.files] == ["file-1"]
+ assert result.files[0].type == "workspace_file"
+ assert [c.id for c in result.chats] == ["chat-1"]
+ assert result.chats[0].type == "chat_session"
+
+
+async def test_global_search_respects_per_type_limit(mocker):
+ """Bucket cap (``per_type_limit``) trims the rendered response after
+ in-Python relevance reranking. The DB fetch limit is the larger of
+ the bucket cap and ``_RELEVANCE_OVERFETCH_CAP`` (32) so the rerank
+ has enough candidates to do meaningful work — see the docstring on
+ ``_files_bucket``."""
+ files = [_fake_workspace_file(f"file-{i}") for i in range(10)]
+
+ mock_manager = MagicMock()
+ mock_manager.list_files = AsyncMock(return_value=files)
+ mocker.patch(
+ "backend.api.features.search.service.WorkspaceManager",
+ return_value=mock_manager,
+ )
+ mocker.patch(
+ "backend.api.features.search.service.get_workspace",
+ new=AsyncMock(return_value=MagicMock(id="ws-1")),
+ )
+
+ # Agents/chats are out of scope here — stub them to empty so the
+ # gather() in global_search resolves cleanly.
+ patcher, _ = _patch_hybrid(([], 0))
+ mocker.patch(
+ "backend.api.features.search.service.get_user_sessions",
+ new=AsyncMock(return_value=([], 0)),
+ )
+
+ with patcher:
+ result = await service.global_search(query="x", user_id="u1", per_type_limit=3)
+
+ # Rendered response is trimmed to per_type_limit even though the
+ # bucket fetched more for reranking.
+ assert len(result.files) == 3
+ # The DB fetch uses the relevance overfetch cap, NOT the bucket
+ # cap — anything else would mean we're ranking too small a sample.
+ assert (
+ mock_manager.list_files.await_args.kwargs["limit"]
+ == service._RELEVANCE_OVERFETCH_CAP
+ )
+
+
+async def test_global_search_bucket_failure_does_not_kill_other_buckets(mocker):
+ """If the files DB query raises, the chats & agents buckets still
+ return their results — fan-out failures must not 500 the response."""
+
+ mocker.patch(
+ "backend.api.features.search.service.get_workspace",
+ new=AsyncMock(side_effect=RuntimeError("simulated DB failure")),
+ )
+ mocker.patch(
+ "backend.api.features.search.service.get_user_sessions",
+ new=AsyncMock(return_value=([_fake_chat_session("chat-1", title="Hi")], 1)),
+ )
+
+ patcher, _ = _patch_hybrid(([], 0))
+ with patcher:
+ result = await service.global_search(query="x", user_id="u1")
+
+ assert result.files == [] # failed bucket -> empty, not 500
+ assert [c.id for c in result.chats] == ["chat-1"]
+
+
+async def test_global_search_forwards_user_id_to_every_bucket():
+ patcher, mock_shim = _patch_hybrid(([], 0))
+ with patcher:
+ await service.global_search(query="x", user_id="user-42")
+
+ for call in mock_shim.unified_hybrid_search.await_args_list:
+ assert call.kwargs["user_id"] == "user-42"
+
+
+# ============================================================================
+# global_search — recent branch (empty query)
+# ============================================================================
+
+
+async def test_global_search_empty_query_returns_recent_buckets(mocker):
+ """Empty/whitespace ``q`` short-circuits the hybrid path and returns
+ the most-recently-updated items per bucket."""
+ _mock_recent_sources(
+ mocker,
+ workspace=MagicMock(id="ws-1"),
+ files=[_fake_workspace_file()],
+ sessions=[_fake_chat_session()],
+ )
+
+ # Patch the hybrid shim too so we can assert it's NOT invoked.
+ patcher, mock_shim = _patch_hybrid(([], 0))
+ with patcher:
+ result = await service.global_search(query=" ", user_id="u1")
+
+ assert isinstance(result, GlobalSearchResponse)
+ assert [a.id for a in result.agents] == ["a-1"]
+ assert result.agents[0].type == "library_agent"
+ assert [f.id for f in result.files] == ["f1"]
+ assert result.files[0].type == "workspace_file"
+ assert [c.id for c in result.chats] == ["s1"]
+ assert result.chats[0].type == "chat_session"
+
+ mock_shim.unified_hybrid_search.assert_not_called()
+
+
+async def test_global_search_empty_query_caches_per_user(mocker):
+ """Two empty-query calls with the same args reuse the cached response."""
+ mock_lib = _mock_recent_sources(mocker)
+
+ await service.global_search(query="", user_id="u1", per_type_limit=3)
+ await service.global_search(query="", user_id="u1", per_type_limit=3)
+
+ # Underlying list_library_agents was hit once — second call was cached.
+ assert mock_lib.list_library_agents.await_count == 1
+
+
+async def test_global_search_empty_query_handles_user_without_workspace(mocker):
+ """A brand-new user with no workspace row returns empty files without
+ creating a workspace as a side-effect."""
+ _mock_recent_sources(mocker) # workspace=None by default
+
+ result = await service.global_search(query="", user_id="u1")
+ assert result.files == []
+
+
+async def test_global_search_empty_query_untitled_chat_falls_back(mocker):
+ _mock_recent_sources(mocker, sessions=[_fake_chat_session(title=None)])
+
+ result = await service.global_search(query="", user_id="u1")
+ assert result.chats[0].title == "Untitled chat"
+
+
+def test_db_accessor_search_resolves_unified_hybrid_search():
+ """Regression: ``db_accessors.search()`` must expose
+ ``unified_hybrid_search`` on the in-process branch.
+
+ The unit tests above patch ``service.search`` so a wrong import in
+ the accessor wouldn't surface. This guards against a repeat of the
+ bug where the accessor was still pointing at
+ ``backend.api.features.store.hybrid_search`` after the engine moved
+ to ``backend.api.features.search.hybrid_search``.
+ """
+ from backend.data import db, db_accessors
+
+ with patch.object(db, "is_connected", return_value=True):
+ shim = db_accessors.search()
+ assert callable(getattr(shim, "unified_hybrid_search", None)), (
+ "db_accessors.search() must expose unified_hybrid_search; "
+ "service.global_search hits this attribute on every non-empty "
+ "query in production."
+ )
+
+
+# ============================================================================
+# Relevance ordering — non-empty query reranks by literal name match
+# ============================================================================
+
+
+def _fake_workspace_file_named(file_id: str, name: str, created_at: datetime.datetime):
+ from backend.data.workspace import WorkspaceFile
+
+ return WorkspaceFile(
+ id=file_id,
+ workspace_id="ws-1",
+ created_at=created_at,
+ updated_at=created_at,
+ name=name,
+ path=f"/documents/{name}",
+ storage_path=f"ws-1/documents/{name}",
+ mime_type="application/pdf",
+ size_bytes=1,
+ )
+
+
+def _fake_chat_session_named(
+ session_id: str, title: str, updated_at: datetime.datetime
+):
+ from backend.copilot.model import ChatSessionInfo
+
+ return ChatSessionInfo(
+ session_id=session_id,
+ user_id="u1",
+ title=title,
+ usage=[],
+ started_at=updated_at,
+ updated_at=updated_at,
+ )
+
+
+def test_title_relevance_score_ranks_exact_above_prefix_above_contains():
+ """The three-tier literal-match score is the single knob behind the
+ files/chats rerank — assert each rung individually so a future
+ refactor can't silently flatten the scale."""
+ assert service._title_relevance_score("Quarterly Report", "quarterly") == 2
+ assert service._title_relevance_score("My Quarterly", "quarterly") == 1
+ assert service._title_relevance_score("Quarterly", "quarterly") == 3
+ assert service._title_relevance_score("Annual", "quarterly") == 0
+
+
+async def test_files_bucket_reranks_by_relevance_not_freshness(mocker):
+ """Repro of the blocker on service.py:326 — a stale exact / prefix
+ match must beat fresher contains-only matches within ``limit``."""
+ # Freshness DESC from list_files: newest first
+ files = [
+ _fake_workspace_file_named(
+ "f-new-1", "final-report-v3.pdf", datetime.datetime(2026, 5, 28)
+ ),
+ _fake_workspace_file_named(
+ "f-new-2", "report-2026-Q1.pdf", datetime.datetime(2026, 5, 27)
+ ),
+ _fake_workspace_file_named(
+ "f-new-3", "monthly-report.xlsx", datetime.datetime(2026, 5, 26)
+ ),
+ _fake_workspace_file_named(
+ "f-new-4", "weekly-report.md", datetime.datetime(2026, 5, 25)
+ ),
+ # Older, but the literal best match (startswith)
+ _fake_workspace_file_named(
+ "f-old", "quarterly-report-overview.md", datetime.datetime(2026, 5, 20)
+ ),
+ ]
+ mock_manager = MagicMock()
+ mock_manager.list_files = AsyncMock(return_value=files)
+ mocker.patch(
+ "backend.api.features.search.service.WorkspaceManager",
+ return_value=mock_manager,
+ )
+ mocker.patch(
+ "backend.api.features.search.service.get_workspace",
+ new=AsyncMock(return_value=MagicMock(id="ws-1")),
+ )
+
+ result = await service._files_bucket("u1", limit=1, query="quarterly")
+
+ # The stale startswith match must outrank the fresh contains matches.
+ assert [f.id for f in result] == ["f-old"]
+ # Over-fetch must request at least _RELEVANCE_OVERFETCH_CAP candidates
+ # so the rerank has enough headroom.
+ assert (
+ mock_manager.list_files.await_args.kwargs["limit"]
+ >= service._RELEVANCE_OVERFETCH_CAP
+ )
+
+
+async def test_chats_bucket_reranks_by_relevance_not_recency(mocker):
+ """Same rerank guarantee for ``_chats_bucket`` — an older exact-title
+ chat must beat fresher contains-only matches."""
+ sessions = [
+ _fake_chat_session_named(
+ "s-new", "Reporting on Q2", datetime.datetime(2026, 5, 28)
+ ),
+ _fake_chat_session_named("s-old", "report", datetime.datetime(2026, 5, 20)),
+ ]
+ mocker.patch(
+ "backend.api.features.search.service.get_user_sessions",
+ new=AsyncMock(return_value=(sessions, len(sessions))),
+ )
+
+ result = await service._chats_bucket("u1", limit=1, query="report")
+
+ # Exact case-insensitive match wins despite being older.
+ assert [c.id for c in result] == ["s-old"]
diff --git a/autogpt_platform/backend/backend/api/features/store/db.py b/autogpt_platform/backend/backend/api/features/store/db.py
index add126eae9..8c9db11b67 100644
--- a/autogpt_platform/backend/backend/api/features/store/db.py
+++ b/autogpt_platform/backend/backend/api/features/store/db.py
@@ -1198,8 +1198,7 @@ async def get_my_agents(
) -> store_model.MyUnpublishedAgentsResponse:
"""Get the agents for the authenticated user"""
logger.debug(
- f"Getting my agents for user {user_id}, page={page}, "
- f"sort_by={sort_by.value}"
+ f"Getting my agents for user {user_id}, page={page}, sort_by={sort_by.value}"
)
try:
diff --git a/autogpt_platform/backend/backend/api/features/store/db_test.py b/autogpt_platform/backend/backend/api/features/store/db_test.py
index fc619590c1..4996cc8ec5 100644
--- a/autogpt_platform/backend/backend/api/features/store/db_test.py
+++ b/autogpt_platform/backend/backend/api/features/store/db_test.py
@@ -1,5 +1,5 @@
from datetime import datetime
-from unittest.mock import AsyncMock
+from unittest.mock import AsyncMock, patch
import prisma.enums
import prisma.errors
@@ -350,49 +350,76 @@ async def test_get_user_profile(mocker):
@pytest.mark.asyncio(loop_scope="session")
-async def test_get_store_agents_with_search_parameterized(mocker):
- """Test that search query uses parameterized SQL - validates the fix works"""
-
- # Call function with search query containing potential SQL injection
- malicious_search = "test'; DROP TABLE StoreAgent; --"
- result = await db.get_store_agents(search_query=malicious_search)
+async def test_get_store_agents_with_search_parameterized():
+ """Malicious search input is passed as a parameter, not concatenated into SQL."""
+ with patch(
+ "backend.api.features.store.hybrid_search.query_raw_with_schema",
+ AsyncMock(return_value=[]),
+ ) as hybrid_query_raw, patch(
+ "backend.api.features.store.db.query_raw_with_schema",
+ AsyncMock(return_value=[]),
+ ) as fallback_query_raw:
+ malicious_search = "test'; DROP TABLE StoreAgent; --"
+ result = await db.get_store_agents(search_query=malicious_search)
- # Verify query executed safely
- assert isinstance(result.agents, list)
+ assert isinstance(result.agents, list)
+ hybrid_sql, *hybrid_params = hybrid_query_raw.call_args.args
+ fallback_sql, *fallback_params = fallback_query_raw.call_args.args
+ assert malicious_search in hybrid_params
+ assert malicious_search not in hybrid_sql
+ assert malicious_search in fallback_params
+ assert malicious_search not in fallback_sql
@pytest.mark.asyncio(loop_scope="session")
async def test_get_store_agents_with_search_and_filters_parameterized():
- """Test parameterized SQL with multiple filters"""
-
- # Call with multiple filters including potential injection attempts
- result = await db.get_store_agents(
- search_query="test",
- creators=["creator1'; DROP TABLE Users; --", "creator2"],
- category="AI'; DELETE FROM StoreAgent; --",
- featured=True,
- sorted_by=db.StoreAgentsSortOptions.RATING,
- page=1,
- page_size=20,
- )
+ """Malicious creator/category values are bound as parameters across all filters."""
+ with patch(
+ "backend.api.features.store.hybrid_search.query_raw_with_schema",
+ AsyncMock(return_value=[]),
+ ), patch(
+ "backend.api.features.store.db.query_raw_with_schema",
+ AsyncMock(return_value=[]),
+ ) as fallback_query_raw:
+ malicious_creator = "creator1'; DROP TABLE Users; --"
+ malicious_category = "AI'; DELETE FROM StoreAgent; --"
+ result = await db.get_store_agents(
+ search_query="test",
+ creators=[malicious_creator, "creator2"],
+ category=malicious_category,
+ featured=True,
+ sorted_by=db.StoreAgentsSortOptions.RATING,
+ page=1,
+ page_size=20,
+ )
- # Verify the query executed without error
- assert isinstance(result.agents, list)
+ assert isinstance(result.agents, list)
+ fallback_sql, *fallback_params = fallback_query_raw.call_args.args
+ assert malicious_category in fallback_params
+ assert malicious_category not in fallback_sql
+ assert [malicious_creator, "creator2"] in fallback_params
@pytest.mark.asyncio(loop_scope="session")
async def test_get_store_agents_search_category_array_injection():
- """Test that category parameter is safely passed as a parameter"""
- # Try SQL injection via category
- malicious_category = "AI'; DROP TABLE StoreAgent; --"
- result = await db.get_store_agents(
- search_query="test",
- category=malicious_category,
- )
+ """Category injection attempt is bound as a parameter, not interpolated."""
+ with patch(
+ "backend.api.features.store.hybrid_search.query_raw_with_schema",
+ AsyncMock(return_value=[]),
+ ), patch(
+ "backend.api.features.store.db.query_raw_with_schema",
+ AsyncMock(return_value=[]),
+ ) as fallback_query_raw:
+ malicious_category = "AI'; DROP TABLE StoreAgent; --"
+ result = await db.get_store_agents(
+ search_query="test",
+ category=malicious_category,
+ )
- # Verify the query executed without error
- # Category should be parameterized, preventing SQL injection
- assert isinstance(result.agents, list)
+ assert isinstance(result.agents, list)
+ fallback_sql, *fallback_params = fallback_query_raw.call_args.args
+ assert malicious_category in fallback_params
+ assert malicious_category not in fallback_sql
@pytest.mark.asyncio(loop_scope="session")
diff --git a/autogpt_platform/backend/backend/api/features/store/embeddings.py b/autogpt_platform/backend/backend/api/features/store/embeddings.py
index b9920a7c49..33e66712b4 100644
--- a/autogpt_platform/backend/backend/api/features/store/embeddings.py
+++ b/autogpt_platform/backend/backend/api/features/store/embeddings.py
@@ -1,105 +1,29 @@
"""
-Unified Content Embeddings Service
+Store listing embedding helpers.
-Handles generation and storage of OpenAI embeddings for all content types
-(store listings, blocks, documentation, library agents) to enable semantic/hybrid search.
+Thin store-specific wrappers around the generic embedding service in
+``backend.api.features.search.embeddings``. These exist for backward
+compatibility with code that addresses store listings by
+``StoreListingVersion.id`` directly.
"""
-import asyncio
import logging
-import time
from typing import Any
import prisma
from prisma.enums import ContentType
-from tiktoken import encoding_for_model
-from backend.api.features.store.content_handlers import CONTENT_HANDLERS
-from backend.blocks import get_blocks
-from backend.data.db import execute_raw_with_schema, query_raw_with_schema
-from backend.util.clients import get_openai_client
-from backend.util.json import dumps
+from backend.api.features.search.embeddings import (
+ backfill_all_content_types,
+ build_searchable_text,
+ delete_content_embedding,
+ generate_embedding,
+ get_content_embedding,
+ store_content_embedding,
+)
logger = logging.getLogger(__name__)
-# OpenAI embedding model configuration
-EMBEDDING_MODEL = "text-embedding-3-small"
-# Embedding dimension for the model above
-# text-embedding-3-small: 1536, text-embedding-3-large: 3072
-EMBEDDING_DIM = 1536
-# OpenAI embedding token limit (8,191 with 1 token buffer for safety)
-EMBEDDING_MAX_TOKENS = 8191
-
-
-def build_searchable_text(
- name: str,
- description: str,
- sub_heading: str,
- categories: list[str],
-) -> str:
- """
- Build searchable text from listing version fields.
-
- Combines relevant fields into a single string for embedding.
- """
- parts = []
-
- # Name is important - include it
- if name:
- parts.append(name)
-
- # Sub-heading provides context
- if sub_heading:
- parts.append(sub_heading)
-
- # Description is the main content
- if description:
- parts.append(description)
-
- # Categories help with semantic matching
- if categories:
- parts.append(" ".join(categories))
-
- return " ".join(parts)
-
-
-async def generate_embedding(text: str) -> list[float]:
- """
- Generate embedding for text using OpenAI API.
-
- Raises exceptions on failure - caller should handle.
- """
- client = get_openai_client()
- if not client:
- raise RuntimeError("openai_internal_api_key not set, cannot generate embedding")
-
- # Truncate text to token limit using tiktoken
- # Character-based truncation is insufficient because token ratios vary by content type
- enc = encoding_for_model(EMBEDDING_MODEL)
- tokens = enc.encode(text)
- if len(tokens) > EMBEDDING_MAX_TOKENS:
- tokens = tokens[:EMBEDDING_MAX_TOKENS]
- truncated_text = enc.decode(tokens)
- logger.info(
- f"Truncated text from {len(enc.encode(text))} to {len(tokens)} tokens"
- )
- else:
- truncated_text = text
-
- start_time = time.time()
- response = await client.embeddings.create(
- model=EMBEDDING_MODEL,
- input=truncated_text,
- )
- latency_ms = (time.time() - start_time) * 1000
-
- embedding = response.data[0].embedding
- logger.info(
- f"Generated embedding: {len(embedding)} dims, "
- f"{len(tokens)} tokens, {latency_ms:.0f}ms"
- )
- return embedding
-
async def store_embedding(
version_id: str,
@@ -123,61 +47,6 @@ async def store_embedding(
)
-async def store_content_embedding(
- content_type: ContentType,
- content_id: str,
- embedding: list[float],
- searchable_text: str,
- metadata: dict | None = None,
- user_id: str | None = None,
- tx: prisma.Prisma | None = None,
-) -> bool:
- """
- Store embedding in the unified content embeddings table.
-
- New function for unified content embedding storage.
- Uses raw SQL since Prisma doesn't natively support pgvector.
-
- Raises exceptions on failure - caller should handle.
- """
- client = tx if tx else prisma.get_client()
-
- # Convert embedding to PostgreSQL vector format
- embedding_str = embedding_to_vector_string(embedding)
- metadata_json = dumps(metadata or {})
-
- # Upsert the embedding
- # WHERE clause in DO UPDATE prevents PostgreSQL 15 bug with NULLS NOT DISTINCT
- # Use unqualified ::vector - pgvector is in search_path on all environments
- await execute_raw_with_schema(
- """
- INSERT INTO {schema_prefix}"UnifiedContentEmbedding" (
- "id", "contentType", "contentId", "userId", "embedding", "searchableText", "metadata", "createdAt", "updatedAt"
- )
- VALUES (gen_random_uuid()::text, $1::{schema_prefix}"ContentType", $2, $3, $4::vector, $5, $6::jsonb, NOW(), NOW())
- ON CONFLICT ("contentType", "contentId", "userId")
- DO UPDATE SET
- "embedding" = $4::vector,
- "searchableText" = $5,
- "metadata" = $6::jsonb,
- "updatedAt" = NOW()
- WHERE {schema_prefix}"UnifiedContentEmbedding"."contentType" = $1::{schema_prefix}"ContentType"
- AND {schema_prefix}"UnifiedContentEmbedding"."contentId" = $2
- AND ({schema_prefix}"UnifiedContentEmbedding"."userId" = $3 OR ($3 IS NULL AND {schema_prefix}"UnifiedContentEmbedding"."userId" IS NULL))
- """,
- content_type,
- content_id,
- user_id,
- embedding_str,
- searchable_text,
- metadata_json,
- client=client,
- )
-
- logger.info(f"Stored embedding for {content_type}:{content_id}")
- return True
-
-
async def get_embedding(version_id: str) -> dict[str, Any] | None:
"""
Retrieve embedding record for a listing version.
@@ -199,41 +68,6 @@ async def get_embedding(version_id: str) -> dict[str, Any] | None:
return None
-async def get_content_embedding(
- content_type: ContentType, content_id: str, user_id: str | None = None
-) -> dict[str, Any] | None:
- """
- Retrieve embedding record for any content type.
-
- New function for unified content embedding retrieval.
- Returns dict with contentType, contentId, embedding, timestamps or None if not found.
-
- Raises exceptions on failure - caller should handle.
- """
- result = await query_raw_with_schema(
- """
- SELECT
- "contentType",
- "contentId",
- "userId",
- "embedding"::text as "embedding",
- "searchableText",
- "metadata",
- "createdAt",
- "updatedAt"
- FROM {schema_prefix}"UnifiedContentEmbedding"
- WHERE "contentType" = $1::{schema_prefix}"ContentType" AND "contentId" = $2 AND ("userId" = $3 OR ($3 IS NULL AND "userId" IS NULL))
- """,
- content_type,
- content_id,
- user_id,
- )
-
- if result and len(result) > 0:
- return result[0]
- return None
-
-
async def ensure_embedding(
version_id: str,
name: str,
@@ -304,120 +138,6 @@ async def delete_embedding(version_id: str) -> bool:
return await delete_content_embedding(ContentType.STORE_AGENT, version_id)
-async def delete_content_embedding(
- content_type: ContentType, content_id: str, user_id: str | None = None
-) -> bool:
- """
- Delete embedding for any content type.
-
- New function for unified content embedding deletion.
- Note: This is usually handled automatically by CASCADE delete,
- but provided for manual cleanup if needed.
-
- Args:
- content_type: The type of content (STORE_AGENT, LIBRARY_AGENT, etc.)
- content_id: The unique identifier for the content
- user_id: Optional user ID. For public content (STORE_AGENT, BLOCK), pass None.
- For user-scoped content (LIBRARY_AGENT), pass the user's ID to avoid
- deleting embeddings belonging to other users.
-
- Returns:
- True if deletion succeeded, False otherwise
- """
- try:
- client = prisma.get_client()
-
- await execute_raw_with_schema(
- """
- DELETE FROM {schema_prefix}"UnifiedContentEmbedding"
- WHERE "contentType" = $1::{schema_prefix}"ContentType"
- AND "contentId" = $2
- AND ("userId" = $3 OR ($3 IS NULL AND "userId" IS NULL))
- """,
- content_type,
- content_id,
- user_id,
- client=client,
- )
-
- user_str = f" (user: {user_id})" if user_id else ""
- logger.info(f"Deleted embedding for {content_type}:{content_id}{user_str}")
- return True
-
- except Exception as e:
- logger.error(f"Failed to delete embedding for {content_type}:{content_id}: {e}")
- return False
-
-
-async def get_embedding_stats() -> dict[str, Any]:
- """
- Get statistics about embedding coverage for all content types.
-
- Returns stats per content type and overall totals.
- """
- try:
- stats_by_type = {}
- total_items = 0
- total_with_embeddings = 0
- total_without_embeddings = 0
-
- # Aggregate stats from all handlers
- for content_type, handler in CONTENT_HANDLERS.items():
- try:
- stats = await handler.get_stats()
- stats_by_type[content_type.value] = {
- "total": stats["total"],
- "with_embeddings": stats["with_embeddings"],
- "without_embeddings": stats["without_embeddings"],
- "coverage_percent": (
- round(stats["with_embeddings"] / stats["total"] * 100, 1)
- if stats["total"] > 0
- else 0
- ),
- }
-
- total_items += stats["total"]
- total_with_embeddings += stats["with_embeddings"]
- total_without_embeddings += stats["without_embeddings"]
-
- except Exception as e:
- logger.error(f"Failed to get stats for {content_type.value}: {e}")
- stats_by_type[content_type.value] = {
- "total": 0,
- "with_embeddings": 0,
- "without_embeddings": 0,
- "coverage_percent": 0,
- "error": str(e),
- }
-
- return {
- "by_type": stats_by_type,
- "totals": {
- "total": total_items,
- "with_embeddings": total_with_embeddings,
- "without_embeddings": total_without_embeddings,
- "coverage_percent": (
- round(total_with_embeddings / total_items * 100, 1)
- if total_items > 0
- else 0
- ),
- },
- }
-
- except Exception as e:
- logger.error(f"Failed to get embedding stats: {e}")
- return {
- "by_type": {},
- "totals": {
- "total": 0,
- "with_embeddings": 0,
- "without_embeddings": 0,
- "coverage_percent": 0,
- },
- "error": str(e),
- }
-
-
async def backfill_missing_embeddings(batch_size: int = 10) -> dict[str, Any]:
"""
Generate embeddings for approved listings that don't have them.
@@ -436,544 +156,3 @@ async def backfill_missing_embeddings(batch_size: int = 10) -> dict[str, Any]:
# Return in the old format for backward compatibility
return result["totals"]
-
-
-async def backfill_all_content_types(batch_size: int = 10) -> dict[str, Any]:
- """
- Generate embeddings for all content types using registered handlers.
-
- Processes content types in order: BLOCK → STORE_AGENT → DOCUMENTATION.
- This ensures foundational content (blocks) are searchable first.
-
- Args:
- batch_size: Number of embeddings to generate per content type
-
- Returns:
- Dict with stats per content type and overall totals
- """
- results_by_type = {}
- total_processed = 0
- total_success = 0
- total_failed = 0
- all_errors: dict[str, int] = {} # Aggregate errors across all content types
-
- # Process content types in explicit order
- processing_order = [
- ContentType.BLOCK,
- ContentType.STORE_AGENT,
- ContentType.DOCUMENTATION,
- ContentType.LIBRARY_AGENT,
- ]
-
- for content_type in processing_order:
- handler = CONTENT_HANDLERS.get(content_type)
- if not handler:
- logger.warning(f"No handler registered for {content_type.value}")
- continue
- try:
- logger.info(f"Processing {content_type.value} content type...")
-
- # Get missing items from handler
- missing_items = await handler.get_missing_items(batch_size)
-
- if not missing_items:
- results_by_type[content_type.value] = {
- "processed": 0,
- "success": 0,
- "failed": 0,
- "message": "No missing embeddings",
- }
- continue
-
- # Process embeddings concurrently for better performance
- embedding_tasks = [
- ensure_content_embedding(
- content_type=item.content_type,
- content_id=item.content_id,
- searchable_text=item.searchable_text,
- metadata=item.metadata,
- user_id=item.user_id,
- )
- for item in missing_items
- ]
-
- results = await asyncio.gather(*embedding_tasks, return_exceptions=True)
-
- success = sum(1 for result in results if result is True)
- failed = len(results) - success
-
- # Aggregate errors across all content types
- if failed > 0:
- for result in results:
- if isinstance(result, Exception):
- error_key = f"{type(result).__name__}: {str(result)}"
- all_errors[error_key] = all_errors.get(error_key, 0) + 1
-
- results_by_type[content_type.value] = {
- "processed": len(missing_items),
- "success": success,
- "failed": failed,
- "message": f"Backfilled {success} embeddings, {failed} failed",
- }
-
- total_processed += len(missing_items)
- total_success += success
- total_failed += failed
-
- logger.info(
- f"{content_type.value}: processed {len(missing_items)}, "
- f"success {success}, failed {failed}"
- )
-
- except Exception as e:
- logger.error(f"Failed to process {content_type.value}: {e}")
- results_by_type[content_type.value] = {
- "processed": 0,
- "success": 0,
- "failed": 0,
- "error": str(e),
- }
-
- # Log aggregated errors once at the end
- if all_errors:
- error_details = ", ".join(
- f"{error} ({count}x)" for error, count in all_errors.items()
- )
- logger.error(f"Embedding backfill errors: {error_details}")
-
- return {
- "by_type": results_by_type,
- "totals": {
- "processed": total_processed,
- "success": total_success,
- "failed": total_failed,
- "message": f"Overall: {total_success} succeeded, {total_failed} failed",
- },
- }
-
-
-async def embed_query(query: str) -> list[float]:
- """
- Generate embedding for a search query.
-
- Same as generate_embedding but with clearer intent.
- Raises exceptions on failure - caller should handle.
- """
- return await generate_embedding(query)
-
-
-def embedding_to_vector_string(embedding: list[float]) -> str:
- """Convert embedding list to PostgreSQL vector string format."""
- return "[" + ",".join(str(x) for x in embedding) + "]"
-
-
-async def ensure_content_embedding(
- content_type: ContentType,
- content_id: str,
- searchable_text: str,
- metadata: dict | None = None,
- user_id: str | None = None,
- force: bool = False,
- tx: prisma.Prisma | None = None,
-) -> bool:
- """
- Ensure an embedding exists for any content type.
-
- Generic function for creating embeddings for store agents, blocks, docs, etc.
-
- Args:
- content_type: ContentType enum value (STORE_AGENT, BLOCK, etc.)
- content_id: Unique identifier for the content
- searchable_text: Combined text for embedding generation
- metadata: Optional metadata to store with embedding
- force: Force regeneration even if embedding exists
- tx: Optional transaction client
-
- Returns:
- True if embedding exists/was created
-
- Raises exceptions on failure - caller should handle.
- """
- # Check if embedding already exists
- if not force:
- existing = await get_content_embedding(content_type, content_id, user_id)
- if existing and existing.get("embedding"):
- logger.debug(f"Embedding for {content_type}:{content_id} already exists")
- return True
-
- # Generate new embedding
- embedding = await generate_embedding(searchable_text)
-
- # Store the embedding
- return await store_content_embedding(
- content_type=content_type,
- content_id=content_id,
- embedding=embedding,
- searchable_text=searchable_text,
- metadata=metadata or {},
- user_id=user_id,
- tx=tx,
- )
-
-
-async def cleanup_orphaned_embeddings() -> dict[str, Any]:
- """
- Clean up embeddings for content that no longer exists or is no longer valid.
-
- Compares current content with embeddings in database and removes orphaned records:
- - STORE_AGENT: Removes embeddings for rejected/deleted store listings
- - BLOCK: Removes embeddings for blocks no longer registered
- - DOCUMENTATION: Removes embeddings for deleted doc files
- - LIBRARY_AGENT: Removes embeddings for soft-deleted/hidden library agents
- (matches the LibraryAgentHandler.get_missing_items filter)
-
- Returns:
- Dict with cleanup statistics per content type
- """
- results_by_type = {}
- total_deleted = 0
-
- # Cleanup orphaned embeddings for all content types
- cleanup_types = [
- ContentType.STORE_AGENT,
- ContentType.BLOCK,
- ContentType.DOCUMENTATION,
- ContentType.LIBRARY_AGENT,
- ]
-
- for content_type in cleanup_types:
- try:
- handler = CONTENT_HANDLERS.get(content_type)
- if not handler:
- logger.warning(f"No handler registered for {content_type}")
- results_by_type[content_type.value] = {
- "deleted": 0,
- "error": "No handler registered",
- }
- continue
-
- # Get all current content IDs from handler
- if content_type == ContentType.STORE_AGENT:
- # Get IDs of approved store listing versions from non-deleted listings
- valid_agents = await query_raw_with_schema(
- """
- SELECT slv.id
- FROM {schema_prefix}"StoreListingVersion" slv
- JOIN {schema_prefix}"StoreListing" sl ON slv."storeListingId" = sl.id
- WHERE slv."submissionStatus" = 'APPROVED'
- AND slv."isDeleted" = false
- AND sl."isDeleted" = false
- """,
- )
- current_ids = {row["id"] for row in valid_agents}
- elif content_type == ContentType.BLOCK:
- current_ids = set(get_blocks().keys())
- elif content_type == ContentType.LIBRARY_AGENT:
- # Match LibraryAgentHandler.get_missing_items: only non-deleted,
- # non-hidden LibraryAgent rows are considered valid.
- valid_agents = await query_raw_with_schema(
- """
- SELECT id
- FROM {schema_prefix}"LibraryAgent"
- WHERE "isDeleted" = false AND "isHidden" = false
- """,
- )
- current_ids = {row["id"] for row in valid_agents}
- elif content_type == ContentType.DOCUMENTATION:
- # Use DocumentationHandler to get section-based content IDs
- from backend.api.features.store.content_handlers import (
- DocumentationHandler,
- )
-
- doc_handler = CONTENT_HANDLERS.get(ContentType.DOCUMENTATION)
- if isinstance(doc_handler, DocumentationHandler):
- docs_root = doc_handler._get_docs_root()
- if docs_root.exists():
- current_ids = doc_handler._get_all_section_content_ids(
- docs_root
- )
- else:
- current_ids = set()
- else:
- current_ids = set()
- else:
- # Skip unknown content types to avoid accidental deletion
- logger.warning(
- f"Skipping cleanup for unknown content type: {content_type}"
- )
- results_by_type[content_type.value] = {
- "deleted": 0,
- "error": "Unknown content type - skipped for safety",
- }
- continue
-
- # Get all embedding IDs from database
- db_embeddings = await query_raw_with_schema(
- """
- SELECT "contentId"
- FROM {schema_prefix}"UnifiedContentEmbedding"
- WHERE "contentType" = $1::{schema_prefix}"ContentType"
- """,
- content_type,
- )
-
- db_ids = {row["contentId"] for row in db_embeddings}
-
- # Find orphaned embeddings (in DB but not in current content)
- orphaned_ids = db_ids - current_ids
-
- if not orphaned_ids:
- logger.info(f"{content_type.value}: No orphaned embeddings found")
- results_by_type[content_type.value] = {
- "deleted": 0,
- "message": "No orphaned embeddings",
- }
- continue
-
- # Delete orphaned embeddings in batch for better performance
- orphaned_list = list(orphaned_ids)
- try:
- await execute_raw_with_schema(
- """
- DELETE FROM {schema_prefix}"UnifiedContentEmbedding"
- WHERE "contentType" = $1::{schema_prefix}"ContentType"
- AND "contentId" = ANY($2::text[])
- """,
- content_type,
- orphaned_list,
- )
- deleted = len(orphaned_list)
- except Exception as e:
- logger.error(f"Failed to batch delete orphaned embeddings: {e}")
- deleted = 0
-
- logger.info(
- f"{content_type.value}: Deleted {deleted}/{len(orphaned_ids)} orphaned embeddings"
- )
- results_by_type[content_type.value] = {
- "deleted": deleted,
- "orphaned": len(orphaned_ids),
- "message": f"Deleted {deleted} orphaned embeddings",
- }
-
- total_deleted += deleted
-
- except Exception as e:
- logger.error(f"Failed to cleanup {content_type.value}: {e}")
- results_by_type[content_type.value] = {
- "deleted": 0,
- "error": str(e),
- }
-
- return {
- "by_type": results_by_type,
- "totals": {
- "deleted": total_deleted,
- "message": f"Deleted {total_deleted} orphaned embeddings",
- },
- }
-
-
-async def semantic_search(
- query: str,
- content_types: list[ContentType] | None = None,
- user_id: str | None = None,
- limit: int = 20,
- min_similarity: float = 0.5,
-) -> list[dict[str, Any]]:
- """
- Semantic search across content types using embeddings.
-
- Performs vector similarity search on UnifiedContentEmbedding table.
- Used directly for blocks/docs/library agents, or as the semantic component
- within hybrid_search for store agents.
-
- If embedding generation fails, falls back to lexical search on searchableText.
-
- Args:
- query: Search query string
- content_types: List of ContentType to search. Defaults to [BLOCK, STORE_AGENT, DOCUMENTATION]
- user_id: Optional user ID for searching private content (library agents)
- limit: Maximum number of results to return (default: 20)
- min_similarity: Minimum cosine similarity threshold (0-1, default: 0.5)
-
- Returns:
- List of search results with the following structure:
- [
- {
- "content_id": str,
- "content_type": str, # "BLOCK", "STORE_AGENT", "DOCUMENTATION", or "LIBRARY_AGENT"
- "searchable_text": str,
- "metadata": dict,
- "similarity": float, # Cosine similarity score (0-1)
- },
- ...
- ]
-
- Examples:
- # Search blocks only
- results = await semantic_search("calculate", content_types=[ContentType.BLOCK])
-
- # Search blocks and documentation
- results = await semantic_search(
- "how to use API",
- content_types=[ContentType.BLOCK, ContentType.DOCUMENTATION]
- )
-
- # Search all public content (default)
- results = await semantic_search("AI agent")
-
- # Search user's library agents
- results = await semantic_search(
- "my custom agent",
- content_types=[ContentType.LIBRARY_AGENT],
- user_id="user123"
- )
- """
- # Default to searching all public content types
- if content_types is None:
- content_types = [
- ContentType.BLOCK,
- ContentType.STORE_AGENT,
- ContentType.DOCUMENTATION,
- ]
-
- # Validate inputs
- if not content_types:
- return [] # Empty content_types would cause invalid SQL (IN ())
-
- query = query.strip()
- if not query:
- return []
-
- if limit < 1:
- limit = 1
- if limit > 100:
- limit = 100
-
- # Generate query embedding
- try:
- query_embedding = await embed_query(query)
- # Semantic search with embeddings
- embedding_str = embedding_to_vector_string(query_embedding)
-
- # Build params in order: limit, then user_id (if provided), then content types
- params: list[Any] = [limit]
- user_filter = ""
- if user_id is not None:
- user_filter = 'AND "userId" = ${}'.format(len(params) + 1)
- params.append(user_id)
-
- # Add content type parameters and build placeholders dynamically
- content_type_start_idx = len(params) + 1
- content_type_placeholders = ", ".join(
- "$" + str(content_type_start_idx + i) + '::{schema_prefix}"ContentType"'
- for i in range(len(content_types))
- )
- params.extend([ct.value for ct in content_types])
-
- # Build min_similarity param index before appending
- min_similarity_idx = len(params) + 1
- params.append(min_similarity)
-
- # Use unqualified ::vector and <=> operator - pgvector is in search_path on all environments
- sql = (
- """
- SELECT
- "contentId" as content_id,
- "contentType" as content_type,
- "searchableText" as searchable_text,
- metadata,
- 1 - (embedding <=> '"""
- + embedding_str
- + """'::vector) as similarity
- FROM {schema_prefix}"UnifiedContentEmbedding"
- WHERE "contentType" IN ("""
- + content_type_placeholders
- + """)
- """
- + user_filter
- + """
- AND 1 - (embedding <=> '"""
- + embedding_str
- + """'::vector) >= $"""
- + str(min_similarity_idx)
- + """
- ORDER BY similarity DESC
- LIMIT $1
- """
- )
-
- results = await query_raw_with_schema(sql, *params)
- return [
- {
- "content_id": row["content_id"],
- "content_type": row["content_type"],
- "searchable_text": row["searchable_text"],
- "metadata": row["metadata"],
- "similarity": float(row["similarity"]),
- }
- for row in results
- ]
- except Exception as e:
- logger.warning(f"Semantic search failed, falling back to lexical search: {e}")
-
- # Fallback to lexical search if embeddings unavailable
-
- params_lexical: list[Any] = [limit]
- user_filter = ""
- if user_id is not None:
- user_filter = 'AND "userId" = ${}'.format(len(params_lexical) + 1)
- params_lexical.append(user_id)
-
- # Add content type parameters and build placeholders dynamically
- content_type_start_idx = len(params_lexical) + 1
- content_type_placeholders_lexical = ", ".join(
- "$" + str(content_type_start_idx + i) + '::{schema_prefix}"ContentType"'
- for i in range(len(content_types))
- )
- params_lexical.extend([ct.value for ct in content_types])
-
- # Build query param index before appending
- query_param_idx = len(params_lexical) + 1
- params_lexical.append(f"%{query}%")
-
- # Use regular string (not f-string) for template to preserve {schema_prefix} placeholders
- sql_lexical = (
- """
- SELECT
- "contentId" as content_id,
- "contentType" as content_type,
- "searchableText" as searchable_text,
- metadata,
- 0.0 as similarity
- FROM {schema_prefix}"UnifiedContentEmbedding"
- WHERE "contentType" IN ("""
- + content_type_placeholders_lexical
- + """)
- """
- + user_filter
- + """
- AND "searchableText" ILIKE $"""
- + str(query_param_idx)
- + """
- ORDER BY "updatedAt" DESC
- LIMIT $1
- """
- )
-
- try:
- results = await query_raw_with_schema(sql_lexical, *params_lexical)
- return [
- {
- "content_id": row["content_id"],
- "content_type": row["content_type"],
- "searchable_text": row["searchable_text"],
- "metadata": row["metadata"],
- "similarity": 0.0, # Lexical search doesn't provide similarity
- }
- for row in results
- ]
- except Exception as e:
- logger.error(f"Lexical search failed: {e}")
- return []
diff --git a/autogpt_platform/backend/backend/api/features/store/embeddings_test.py b/autogpt_platform/backend/backend/api/features/store/embeddings_test.py
index 0d5e5ce4a2..e477de318c 100644
--- a/autogpt_platform/backend/backend/api/features/store/embeddings_test.py
+++ b/autogpt_platform/backend/backend/api/features/store/embeddings_test.py
@@ -1,3 +1,5 @@
+"""Tests for store-listing embedding wrappers."""
+
from unittest.mock import AsyncMock, MagicMock, patch
import prisma
@@ -5,6 +7,7 @@ import pytest
from prisma import Prisma
from prisma.enums import ContentType
+from backend.api.features.search import embeddings as search_embeddings
from backend.api.features.store import embeddings
@@ -18,128 +21,6 @@ async def setup_prisma():
yield
[email protected](loop_scope="session")
-async def test_build_searchable_text():
- """Test searchable text building from listing fields."""
- result = embeddings.build_searchable_text(
- name="AI Assistant",
- description="A helpful AI assistant for productivity",
- sub_heading="Boost your productivity",
- categories=["AI", "Productivity"],
- )
-
- expected = "AI Assistant Boost your productivity A helpful AI assistant for productivity AI Productivity"
- assert result == expected
-
-
[email protected](loop_scope="session")
-async def test_build_searchable_text_empty_fields():
- """Test searchable text building with empty fields."""
- result = embeddings.build_searchable_text(
- name="", description="Test description", sub_heading="", categories=[]
- )
-
- assert result == "Test description"
-
-
[email protected](loop_scope="session")
-async def test_generate_embedding_success():
- """Test successful embedding generation."""
- # Mock OpenAI response
- mock_client = MagicMock()
- mock_response = MagicMock()
- mock_response.data = [MagicMock()]
- mock_response.data[0].embedding = [0.1, 0.2, 0.3] * 512 # 1536 dimensions
-
- # Use AsyncMock for async embeddings.create method
- mock_client.embeddings.create = AsyncMock(return_value=mock_response)
-
- # Patch at the point of use in embeddings.py
- with patch(
- "backend.api.features.store.embeddings.get_openai_client"
- ) as mock_get_client:
- mock_get_client.return_value = mock_client
-
- result = await embeddings.generate_embedding("test text")
-
- assert result is not None
- assert len(result) == embeddings.EMBEDDING_DIM
- assert result[0] == 0.1
-
- mock_client.embeddings.create.assert_called_once_with(
- model="text-embedding-3-small", input="test text"
- )
-
-
[email protected](loop_scope="session")
-async def test_generate_embedding_no_api_key():
- """Test embedding generation without API key."""
- # Patch at the point of use in embeddings.py
- with patch(
- "backend.api.features.store.embeddings.get_openai_client"
- ) as mock_get_client:
- mock_get_client.return_value = None
-
- with pytest.raises(RuntimeError, match="openai_internal_api_key not set"):
- await embeddings.generate_embedding("test text")
-
-
[email protected](loop_scope="session")
-async def test_generate_embedding_api_error():
- """Test embedding generation with API error."""
- mock_client = MagicMock()
- mock_client.embeddings.create = AsyncMock(side_effect=Exception("API Error"))
-
- # Patch at the point of use in embeddings.py
- with patch(
- "backend.api.features.store.embeddings.get_openai_client"
- ) as mock_get_client:
- mock_get_client.return_value = mock_client
-
- with pytest.raises(Exception, match="API Error"):
- await embeddings.generate_embedding("test text")
-
-
[email protected](loop_scope="session")
-async def test_generate_embedding_text_truncation():
- """Test that long text is properly truncated using tiktoken."""
- from tiktoken import encoding_for_model
-
- mock_client = MagicMock()
- mock_response = MagicMock()
- mock_response.data = [MagicMock()]
- mock_response.data[0].embedding = [0.1] * embeddings.EMBEDDING_DIM
-
- # Use AsyncMock for async embeddings.create method
- mock_client.embeddings.create = AsyncMock(return_value=mock_response)
-
- # Patch at the point of use in embeddings.py
- with patch(
- "backend.api.features.store.embeddings.get_openai_client"
- ) as mock_get_client:
- mock_get_client.return_value = mock_client
-
- # Create text that will exceed 8191 tokens
- # Use varied characters to ensure token-heavy text: each word is ~1 token
- words = [f"word{i}" for i in range(10000)]
- long_text = " ".join(words) # ~10000 tokens
-
- await embeddings.generate_embedding(long_text)
-
- # Verify text was truncated to 8191 tokens
- call_args = mock_client.embeddings.create.call_args
- truncated_text = call_args.kwargs["input"]
-
- # Count actual tokens in truncated text
- enc = encoding_for_model("text-embedding-3-small")
- actual_tokens = len(enc.encode(truncated_text))
-
- # Should be at or just under 8191 tokens
- assert actual_tokens <= 8191
- # Should be close to the limit (not over-truncated)
- assert actual_tokens >= 8100
-
-
@pytest.mark.asyncio(loop_scope="session")
async def test_store_embedding_success(mocker):
"""Test successful embedding storage."""
@@ -194,7 +75,7 @@ async def test_get_embedding_success():
]
with patch(
- "backend.api.features.store.embeddings.query_raw_with_schema",
+ "backend.api.features.search.embeddings.query_raw_with_schema",
return_value=mock_result,
):
result = await embeddings.get_embedding("test-version-id")
@@ -208,7 +89,7 @@ async def test_get_embedding_success():
async def test_get_embedding_not_found():
"""Test embedding retrieval when not found."""
with patch(
- "backend.api.features.store.embeddings.query_raw_with_schema",
+ "backend.api.features.search.embeddings.query_raw_with_schema",
return_value=[],
):
result = await embeddings.get_embedding("test-version-id")
@@ -218,7 +99,7 @@ async def test_get_embedding_not_found():
@pytest.mark.asyncio(loop_scope="session")
@patch("backend.api.features.store.embeddings.generate_embedding")
-@patch("backend.api.features.store.embeddings.store_embedding")
+@patch("backend.api.features.store.embeddings.store_content_embedding")
@patch("backend.api.features.store.embeddings.get_embedding")
async def test_ensure_embedding_already_exists(mock_get, mock_store, mock_generate):
"""Test ensure_embedding when embedding already exists."""
@@ -287,39 +168,10 @@ async def test_ensure_embedding_generation_fails(mock_get, mock_generate):
@pytest.mark.asyncio(loop_scope="session")
-async def test_get_embedding_stats():
- """Test embedding statistics retrieval."""
- # Mock handler stats for each content type
- mock_handler = MagicMock()
- mock_handler.get_stats = AsyncMock(
- return_value={
- "total": 100,
- "with_embeddings": 75,
- "without_embeddings": 25,
- }
- )
-
- # Patch the CONTENT_HANDLERS where it's used (in embeddings module)
- with patch(
- "backend.api.features.store.embeddings.CONTENT_HANDLERS",
- {ContentType.STORE_AGENT: mock_handler},
- ):
- result = await embeddings.get_embedding_stats()
-
- assert "by_type" in result
- assert "totals" in result
- assert result["totals"]["total"] == 100
- assert result["totals"]["with_embeddings"] == 75
- assert result["totals"]["without_embeddings"] == 25
- assert result["totals"]["coverage_percent"] == 75.0
-
-
[email protected](loop_scope="session")
-@patch("backend.api.features.store.embeddings.store_content_embedding")
+@patch("backend.api.features.search.embeddings.store_content_embedding")
async def test_backfill_missing_embeddings_success(mock_store):
"""Test backfill with successful embedding generation."""
- # Mock ContentItem from handlers
- from backend.api.features.store.content_handlers import ContentItem
+ from backend.api.features.search.content_handlers import ContentItem
mock_items = [
ContentItem(
@@ -336,38 +188,39 @@ async def test_backfill_missing_embeddings_success(mock_store):
),
]
- # Mock handler to return missing items
mock_handler = MagicMock()
mock_handler.get_missing_items = AsyncMock(return_value=mock_items)
- # Mock store_content_embedding to succeed for first, fail for second
mock_store.side_effect = [True, False]
with patch(
- "backend.api.features.store.embeddings.CONTENT_HANDLERS",
+ "backend.api.features.search.embeddings.CONTENT_HANDLERS",
{ContentType.STORE_AGENT: mock_handler},
):
with patch(
- "backend.api.features.store.embeddings.generate_embedding",
- return_value=[0.1] * embeddings.EMBEDDING_DIM,
+ "backend.api.features.search.embeddings.get_content_embedding",
+ AsyncMock(return_value=None),
):
- result = await embeddings.backfill_missing_embeddings(batch_size=5)
+ with patch(
+ "backend.api.features.search.embeddings.generate_embedding",
+ return_value=[0.1] * search_embeddings.EMBEDDING_DIM,
+ ):
+ result = await embeddings.backfill_missing_embeddings(batch_size=5)
- assert result["processed"] == 2
- assert result["success"] == 1
- assert result["failed"] == 1
- assert mock_store.call_count == 2
+ assert result["processed"] == 2
+ assert result["success"] == 1
+ assert result["failed"] == 1
+ assert mock_store.call_count == 2
@pytest.mark.asyncio(loop_scope="session")
async def test_backfill_missing_embeddings_no_missing():
"""Test backfill when no embeddings are missing."""
- # Mock handler to return no missing items
mock_handler = MagicMock()
mock_handler.get_missing_items = AsyncMock(return_value=[])
with patch(
- "backend.api.features.store.embeddings.CONTENT_HANDLERS",
+ "backend.api.features.search.embeddings.CONTENT_HANDLERS",
{ContentType.STORE_AGENT: mock_handler},
):
result = await embeddings.backfill_missing_embeddings(batch_size=5)
@@ -375,25 +228,3 @@ async def test_backfill_missing_embeddings_no_missing():
assert result["processed"] == 0
assert result["success"] == 0
assert result["failed"] == 0
-
-
[email protected](loop_scope="session")
-async def test_embedding_to_vector_string():
- """Test embedding to PostgreSQL vector string conversion."""
- embedding = [0.1, 0.2, 0.3, -0.4]
- result = embeddings.embedding_to_vector_string(embedding)
- assert result == "[0.1,0.2,0.3,-0.4]"
-
-
[email protected](loop_scope="session")
-async def test_embed_query():
- """Test embed_query function (alias for generate_embedding)."""
- with patch(
- "backend.api.features.store.embeddings.generate_embedding"
- ) as mock_generate:
- mock_generate.return_value = [0.1, 0.2, 0.3]
-
- result = await embeddings.embed_query("test query")
-
- assert result == [0.1, 0.2, 0.3]
- mock_generate.assert_called_once_with("test query")
diff --git a/autogpt_platform/backend/backend/api/features/store/hybrid_search.py b/autogpt_platform/backend/backend/api/features/store/hybrid_search.py
index 593bd5201e..a8a1c26259 100644
--- a/autogpt_platform/backend/backend/api/features/store/hybrid_search.py
+++ b/autogpt_platform/backend/backend/api/features/store/hybrid_search.py
@@ -1,421 +1,34 @@
"""
-Unified Hybrid Search
+Hybrid search for store agents.
-Combines semantic (embedding) search with lexical (tsvector) search
-for improved relevance across all content types (agents, blocks, docs).
-Includes BM25 reranking for improved lexical relevance.
+Wraps the generic search engine in ``backend.api.features.search.hybrid_search``
+with store-specific concerns: a ``StoreAgent``-joined SQL query that pulls
+display metadata (runs, rating, featured, creator) and a weight schema
+that adds a ``popularity`` signal.
+
+For generic cross-content search use ``search.hybrid_search.unified_hybrid_search``.
"""
import logging
-import re
-import time
from dataclasses import dataclass
-from typing import Any, Literal
-
-from prisma.enums import ContentType
-from rank_bm25 import BM25Okapi # type: ignore[import-untyped]
+from typing import Any, Literal, cast
-from backend.api.features.store.embeddings import (
+from backend.api.features.search.embeddings import (
EMBEDDING_DIM,
embed_query,
embedding_to_vector_string,
)
+from backend.api.features.search.hybrid_search import (
+ DEFAULT_STORE_AGENT_MIN_SCORE,
+ HybridSearchRow,
+ _log_vector_error_diagnostics,
+ bm25_rerank,
+)
from backend.data.db import query_raw_with_schema
logger = logging.getLogger(__name__)
-# ============================================================================
-# BM25 Reranking
-# ============================================================================
-
-
-def tokenize(text: str) -> list[str]:
- """Tokenize text for BM25."""
- if not text:
- return []
- return re.findall(r"\b\w+\b", text.lower())
-
-
-def bm25_rerank(
- query: str,
- results: list[dict[str, Any]],
- text_field: str = "searchable_text",
- bm25_weight: float = 0.3,
- original_score_field: str = "combined_score",
-) -> list[dict[str, Any]]:
- """
- Rerank search results using BM25.
-
- Combines the original combined_score with BM25 score for improved
- lexical relevance, especially for exact term matches.
-
- Args:
- query: The search query
- results: List of result dicts with text_field and original_score_field
- text_field: Field name containing the text to score
- bm25_weight: Weight for BM25 score (0-1). Original score gets (1 - bm25_weight)
- original_score_field: Field name containing the original score
-
- Returns:
- Results list sorted by combined score (BM25 + original)
- """
- if not results or not query:
- return results
-
- # Extract texts and tokenize
- corpus = [tokenize(r.get(text_field, "") or "") for r in results]
-
- # Handle edge case where all documents are empty
- if all(len(doc) == 0 for doc in corpus):
- return results
-
- # Build BM25 index
- bm25 = BM25Okapi(corpus)
-
- # Score query against corpus
- query_tokens = tokenize(query)
- if not query_tokens:
- return results
-
- bm25_scores = bm25.get_scores(query_tokens)
-
- # Normalize BM25 scores to 0-1 range
- max_bm25 = max(bm25_scores) if max(bm25_scores) > 0 else 1.0
- normalized_bm25 = [s / max_bm25 for s in bm25_scores]
-
- # Combine scores
- original_weight = 1.0 - bm25_weight
- for i, result in enumerate(results):
- original_score = result.get(original_score_field, 0) or 0
- result["bm25_score"] = normalized_bm25[i]
- final_score = (
- original_weight * original_score + bm25_weight * normalized_bm25[i]
- )
- result["final_score"] = final_score
- result["relevance"] = final_score
-
- # Sort by relevance descending
- results.sort(key=lambda x: x.get("relevance", 0), reverse=True)
-
- return results
-
-
-@dataclass
-class UnifiedSearchWeights:
- """Weights for unified search (no popularity signal)."""
-
- semantic: float = 0.40 # Embedding cosine similarity
- lexical: float = 0.40 # tsvector ts_rank_cd score
- category: float = 0.10 # Category match boost (for types that have categories)
- recency: float = 0.10 # Newer content ranked higher
-
- def __post_init__(self):
- """Validate weights are non-negative and sum to approximately 1.0."""
- total = self.semantic + self.lexical + self.category + self.recency
-
- if any(
- w < 0 for w in [self.semantic, self.lexical, self.category, self.recency]
- ):
- raise ValueError("All weights must be non-negative")
-
- if not (0.99 <= total <= 1.01):
- raise ValueError(f"Weights must sum to ~1.0, got {total:.3f}")
-
-
-# Default weights for unified search
-DEFAULT_UNIFIED_WEIGHTS = UnifiedSearchWeights()
-
-# Minimum relevance score thresholds
-DEFAULT_MIN_SCORE = 0.15 # For unified search (more permissive)
-DEFAULT_STORE_AGENT_MIN_SCORE = 0.20 # For store agent search (original threshold)
-
-
-async def unified_hybrid_search(
- query: str,
- content_types: list[ContentType] | None = None,
- category: str | None = None,
- page: int = 1,
- page_size: int = 20,
- weights: UnifiedSearchWeights | None = None,
- min_score: float | None = None,
- user_id: str | None = None,
- lexical_query: str | None = None,
-) -> tuple[list[dict[str, Any]], int]:
- """
- Unified hybrid search across all content types.
-
- Searches UnifiedContentEmbedding using both semantic (vector) and lexical (tsvector) signals.
-
- Args:
- query: Search query string (used for embedding + lexical unless
- ``lexical_query`` is provided).
- content_types: List of content types to search. Defaults to all public types.
- category: Filter by category (for content types that support it)
- page: Page number (1-indexed)
- page_size: Results per page
- weights: Custom weights for search signals
- min_score: Minimum relevance score threshold (0-1)
- user_id: User ID for searching private content (library agents)
- lexical_query: Optional override for the tsvector ``@@`` candidate
- selection and ``ts_rank_cd`` scoring. Use when the natural-
- language query is too long for ``plainto_tsquery``'s
- AND-of-terms semantics — typically a keyword-extracted form
- of ``query`` (e.g. ``"youtube summarize video"`` for a goal
- like ``"summarize a YouTube video..."``). Defaults to
- ``query`` when None.
-
- Returns:
- Tuple of (results list, total count)
- """
- # Validate inputs
- query = query.strip()
- if not query:
- return [], 0
- # Default the lexical query to the semantic query for backwards-
- # compatibility; callers (e.g. library similarity search) override
- # when ``plainto_tsquery``'s AND semantics over a long natural goal
- # would zero out every match.
- lexical_query = (lexical_query or query).strip() or query
-
- if page < 1:
- page = 1
- if page_size < 1:
- page_size = 1
- if page_size > 100:
- page_size = 100
-
- if content_types is None:
- content_types = [
- ContentType.STORE_AGENT,
- ContentType.BLOCK,
- ContentType.DOCUMENTATION,
- ]
-
- if weights is None:
- weights = DEFAULT_UNIFIED_WEIGHTS
- if min_score is None:
- min_score = DEFAULT_MIN_SCORE
-
- offset = (page - 1) * page_size
-
- # Generate query embedding with graceful degradation
- try:
- query_embedding = await embed_query(query)
- except Exception as e:
- logger.warning(
- f"Failed to generate query embedding - falling back to lexical-only search: {e}. "
- "Check that openai_internal_api_key is configured and OpenAI API is accessible."
- )
- query_embedding = [0.0] * EMBEDDING_DIM
- # Redistribute semantic weight to lexical
- total_non_semantic = weights.lexical + weights.category + weights.recency
- if total_non_semantic > 0:
- factor = 1.0 / total_non_semantic
- weights = UnifiedSearchWeights(
- semantic=0.0,
- lexical=weights.lexical * factor,
- category=weights.category * factor,
- recency=weights.recency * factor,
- )
- else:
- weights = UnifiedSearchWeights(
- semantic=0.0, lexical=1.0, category=0.0, recency=0.0
- )
-
- # Build parameters
- params: list[Any] = []
- param_idx = 1
-
- # Query for lexical search (may differ from semantic ``query`` —
- # see the ``lexical_query`` parameter doc).
- params.append(lexical_query)
- query_param = f"${param_idx}"
- param_idx += 1
-
- # Query lowercase for category matching — keep tied to the
- # natural-language ``query`` so categories still match the user's
- # intent words, not the stripped lexical form.
- params.append(query.lower())
- query_lower_param = f"${param_idx}"
- param_idx += 1
-
- # Embedding
- embedding_str = embedding_to_vector_string(query_embedding)
- params.append(embedding_str)
- embedding_param = f"${param_idx}"
- param_idx += 1
-
- # Content types
- content_type_values = [ct.value for ct in content_types]
- params.append(content_type_values)
- content_types_param = f"${param_idx}"
- param_idx += 1
-
- # User ID filter (for private content).
- # Defense-in-depth: a LIBRARY_AGENT row with NULL userId would be matched
- # by the public ``OR userId IS NULL`` branch and leak across users.
- # LIBRARY_AGENT rows are always per-user; explicitly exclude that combo.
- user_filter = ""
- if user_id is not None:
- params.append(user_id)
- user_filter = (
- f'AND (uce."userId" = ${param_idx} OR uce."userId" IS NULL) '
- f'AND NOT (uce."contentType" = \'LIBRARY_AGENT\'::{{schema_prefix}}"ContentType" '
- f'AND uce."userId" IS NULL)'
- )
- param_idx += 1
- else:
- user_filter = 'AND uce."userId" IS NULL'
-
- # Weights
- params.append(weights.semantic)
- w_semantic = f"${param_idx}"
- param_idx += 1
-
- params.append(weights.lexical)
- w_lexical = f"${param_idx}"
- param_idx += 1
-
- params.append(weights.category)
- w_category = f"${param_idx}"
- param_idx += 1
-
- params.append(weights.recency)
- w_recency = f"${param_idx}"
- param_idx += 1
-
- # Min score
- params.append(min_score)
- min_score_param = f"${param_idx}"
- param_idx += 1
-
- # Pagination
- params.append(page_size)
- limit_param = f"${param_idx}"
- param_idx += 1
-
- params.append(offset)
- offset_param = f"${param_idx}"
- param_idx += 1
-
- # Unified search query on UnifiedContentEmbedding
- sql_query = f"""
- WITH candidates AS (
- -- Lexical matches (uses GIN index on search column)
- SELECT uce.id, uce."contentType", uce."contentId"
- FROM {{schema_prefix}}"UnifiedContentEmbedding" uce
- WHERE uce."contentType" = ANY({content_types_param}::{{schema_prefix}}"ContentType"[])
- {user_filter}
- AND uce.search @@ plainto_tsquery('english', {query_param})
-
- UNION
-
- -- Semantic matches (uses HNSW index on embedding)
- (
- SELECT uce.id, uce."contentType", uce."contentId"
- FROM {{schema_prefix}}"UnifiedContentEmbedding" uce
- WHERE uce."contentType" = ANY({content_types_param}::{{schema_prefix}}"ContentType"[])
- {user_filter}
- ORDER BY uce.embedding <=> {embedding_param}::vector
- LIMIT 200
- )
- ),
- search_scores AS (
- SELECT
- uce."contentType" as content_type,
- uce."contentId" as content_id,
- uce."searchableText" as searchable_text,
- uce.metadata,
- uce."updatedAt" as updated_at,
- -- Semantic score: cosine similarity (1 - distance)
- COALESCE(1 - (uce.embedding <=> {embedding_param}::vector), 0) as semantic_score,
- -- Lexical score: ts_rank_cd
- COALESCE(ts_rank_cd(uce.search, plainto_tsquery('english', {query_param})), 0) as lexical_raw,
- -- Category match from metadata
- CASE
- WHEN uce.metadata ? 'categories' AND EXISTS (
- SELECT 1 FROM jsonb_array_elements_text(uce.metadata->'categories') cat
- WHERE LOWER(cat) LIKE '%' || {query_lower_param} || '%'
- )
- THEN 1.0
- ELSE 0.0
- END as category_score,
- -- Recency score: linear decay over 90 days
- GREATEST(0, 1 - EXTRACT(EPOCH FROM (NOW() - uce."updatedAt")) / (90 * 24 * 3600)) as recency_score
- FROM candidates c
- INNER JOIN {{schema_prefix}}"UnifiedContentEmbedding" uce ON c.id = uce.id
- ),
- max_lexical AS (
- SELECT GREATEST(MAX(lexical_raw), 0.001) as max_val FROM search_scores
- ),
- normalized AS (
- SELECT
- ss.*,
- ss.lexical_raw / ml.max_val as lexical_score
- FROM search_scores ss
- CROSS JOIN max_lexical ml
- ),
- scored AS (
- SELECT
- content_type,
- content_id,
- searchable_text,
- metadata,
- updated_at,
- semantic_score,
- lexical_score,
- category_score,
- recency_score,
- (
- {w_semantic} * semantic_score +
- {w_lexical} * lexical_score +
- {w_category} * category_score +
- {w_recency} * recency_score
- ) as combined_score
- FROM normalized
- ),
- filtered AS (
- SELECT *, COUNT(*) OVER () as total_count
- FROM scored
- WHERE combined_score >= {min_score_param}
- )
- SELECT * FROM filtered
- ORDER BY combined_score DESC
- LIMIT {limit_param} OFFSET {offset_param}
- """
-
- try:
- results = await query_raw_with_schema(sql_query, *params)
- except Exception as e:
- await _log_vector_error_diagnostics(e)
- raise
-
- total = results[0]["total_count"] if results else 0
- # Apply BM25 reranking
- if results:
- results = bm25_rerank(
- query=query,
- results=results,
- text_field="searchable_text",
- bm25_weight=0.3,
- original_score_field="combined_score",
- )
-
- # Clean up results
- for result in results:
- result.pop("total_count", None)
-
- logger.info(f"Unified hybrid search: {len(results)} results, {total} total")
-
- return results, total
-
-
-# ============================================================================
-# Store Agent specific search (with full metadata)
-# ============================================================================
-
-
@dataclass
class StoreAgentSearchWeights:
"""Weights for store agent search including popularity."""
@@ -714,12 +327,15 @@ async def hybrid_search(
"""
try:
- results = await query_raw_with_schema(sql_query, *params)
+ raw_results = await query_raw_with_schema(sql_query, *params)
except Exception as e:
await _log_vector_error_diagnostics(e)
raise
- total = results[0]["total_count"] if results else 0
+ # Cast at the SQL boundary so the rest of the pipeline operates on
+ # the typed row shape (matches search/hybrid_search.py's pattern).
+ results: list[HybridSearchRow] = cast(list[HybridSearchRow], raw_results)
+ total = results[0].get("total_count", 0) if results else 0
# Apply BM25 reranking
if results:
@@ -737,7 +353,7 @@ async def hybrid_search(
logger.info(f"Hybrid search (store agents): {len(results)} results, {total} total")
- return results, total
+ return cast(list[dict[str, Any]], results), total
async def hybrid_search_simple(
@@ -749,87 +365,6 @@ async def hybrid_search_simple(
return await hybrid_search(query=query, page=page, page_size=page_size)
-# ============================================================================
-# Diagnostics
-# ============================================================================
-
-# Rate limit: only log vector error diagnostics once per this interval
-_VECTOR_DIAG_INTERVAL_SECONDS = 60
-_last_vector_diag_time: float = 0
-
-
-async def _log_vector_error_diagnostics(error: Exception) -> None:
- """Log diagnostic info when 'type vector does not exist' error occurs.
-
- Note: Diagnostic queries use query_raw_with_schema which may run on a different
- pooled connection than the one that failed. Session-level search_path can differ,
- so these diagnostics show cluster-wide state, not necessarily the failed session.
-
- Includes rate limiting to avoid log spam - only logs once per minute.
- Caller should re-raise the error after calling this function.
- """
- global _last_vector_diag_time
-
- # Check if this is the vector type error
- error_str = str(error).lower()
- if not (
- "type" in error_str and "vector" in error_str and "does not exist" in error_str
- ):
- return
-
- # Rate limit: only log once per interval
- now = time.time()
- if now - _last_vector_diag_time < _VECTOR_DIAG_INTERVAL_SECONDS:
- return
- _last_vector_diag_time = now
-
- try:
- diagnostics: dict[str, object] = {}
-
- try:
- search_path_result = await query_raw_with_schema("SHOW search_path")
- diagnostics["search_path"] = search_path_result
- except Exception as e:
- diagnostics["search_path"] = f"Error: {e}"
-
- try:
- schema_result = await query_raw_with_schema("SELECT current_schema()")
- diagnostics["current_schema"] = schema_result
- except Exception as e:
- diagnostics["current_schema"] = f"Error: {e}"
-
- try:
- user_result = await query_raw_with_schema(
- "SELECT current_user, session_user, current_database()"
- )
- diagnostics["user_info"] = user_result
- except Exception as e:
- diagnostics["user_info"] = f"Error: {e}"
-
- try:
- # Check pgvector extension installation (cluster-wide, stable info)
- ext_result = await query_raw_with_schema(
- "SELECT extname, extversion, nspname as schema "
- "FROM pg_extension e "
- "JOIN pg_namespace n ON e.extnamespace = n.oid "
- "WHERE extname = 'vector'"
- )
- diagnostics["pgvector_extension"] = ext_result
- except Exception as e:
- diagnostics["pgvector_extension"] = f"Error: {e}"
-
- logger.error(
- f"Vector type error diagnostics:\n"
- f" Error: {error}\n"
- f" search_path: {diagnostics.get('search_path')}\n"
- f" current_schema: {diagnostics.get('current_schema')}\n"
- f" user_info: {diagnostics.get('user_info')}\n"
- f" pgvector_extension: {diagnostics.get('pgvector_extension')}"
- )
- except Exception as diag_error:
- logger.error(f"Failed to collect vector error diagnostics: {diag_error}")
-
-
# Backward compatibility alias - HybridSearchWeights maps to StoreAgentSearchWeights
# for existing code that expects the popularity parameter
HybridSearchWeights = StoreAgentSearchWeights
diff --git a/autogpt_platform/backend/backend/api/features/store/hybrid_search_test.py b/autogpt_platform/backend/backend/api/features/store/hybrid_search_test.py
index b21a602c5d..4d0ce3c1e5 100644
--- a/autogpt_platform/backend/backend/api/features/store/hybrid_search_test.py
+++ b/autogpt_platform/backend/backend/api/features/store/hybrid_search_test.py
@@ -1,52 +1,24 @@
"""
-Integration tests for hybrid search with schema handling.
-
-These tests verify that hybrid search works correctly across different database schemas.
+Integration tests for the store-agent hybrid search.
"""
from unittest.mock import patch
import pytest
-from prisma.enums import ContentType
-
-from backend.api.features.store import embeddings
-from backend.api.features.store.hybrid_search import (
- HybridSearchWeights,
- UnifiedSearchWeights,
- hybrid_search,
- tokenize,
- unified_hybrid_search,
-)
-
-# ---------------------------------------------------------------------------
-# tokenize (BM25)
-# ---------------------------------------------------------------------------
-
-
- "input_text, expected",
- [
- ("AITextGeneratorBlock", ["aitextgeneratorblock"]),
- ("hello world", ["hello", "world"]),
- ("", []),
- ("HTTPRequest", ["httprequest"]),
- ],
-)
-def test_tokenize(input_text: str, expected: list[str]):
- assert tokenize(input_text) == expected
+
+from backend.api.features.search import embeddings
+from backend.api.features.store.hybrid_search import HybridSearchWeights, hybrid_search
@pytest.mark.asyncio(loop_scope="session")
@pytest.mark.integration
async def test_hybrid_search_with_schema_handling():
"""Test that hybrid search correctly handles database schema prefixes."""
- # Test with a mock query to ensure schema handling works
query = "test agent"
with patch(
"backend.api.features.store.hybrid_search.query_raw_with_schema"
) as mock_query:
- # Mock the query result
mock_query.return_value = [
{
"slug": "test/agent",
@@ -74,7 +46,7 @@ async def test_hybrid_search_with_schema_handling():
with patch(
"backend.api.features.store.hybrid_search.embed_query"
) as mock_embed:
- mock_embed.return_value = [0.1] * embeddings.EMBEDDING_DIM # Mock embedding
+ mock_embed.return_value = [0.1] * embeddings.EMBEDDING_DIM
results, total = await hybrid_search(
query=query,
@@ -82,14 +54,11 @@ async def test_hybrid_search_with_schema_handling():
page_size=20,
)
- # Verify the query was called
assert mock_query.called
- # Verify the SQL template uses schema_prefix placeholder
call_args = mock_query.call_args
sql_template = call_args[0][0]
assert "{schema_prefix}" in sql_template
- # Verify results
assert len(results) == 1
assert total == 1
assert results[0]["slug"] == "test/agent"
@@ -118,10 +87,7 @@ async def test_hybrid_search_with_public_schema():
page_size=20,
)
- # Verify the mock was set up correctly
assert mock_schema.return_value == "public"
-
- # Results should work even with empty results
assert results == []
assert total == 0
@@ -149,9 +115,7 @@ async def test_hybrid_search_with_custom_schema():
page_size=20,
)
- # Verify the mock was set up correctly
assert mock_schema.return_value == "platform"
-
assert results == []
assert total == 0
@@ -160,7 +124,6 @@ async def test_hybrid_search_with_custom_schema():
@pytest.mark.integration
async def test_hybrid_search_without_embeddings():
"""Test hybrid search gracefully degrades when embeddings are unavailable."""
- # Mock database to return some results
mock_results = [
{
"slug": "test-agent",
@@ -176,7 +139,7 @@ async def test_hybrid_search_without_embeddings():
"featured": False,
"is_available": True,
"updated_at": "2025-01-01T00:00:00Z",
- "semantic_score": 0.0, # Zero because no embedding
+ "semantic_score": 0.0,
"lexical_score": 0.5,
"category_score": 0.0,
"recency_score": 0.1,
@@ -190,18 +153,15 @@ async def test_hybrid_search_without_embeddings():
with patch(
"backend.api.features.store.hybrid_search.query_raw_with_schema"
) as mock_query:
- # Simulate embedding failure by raising exception
mock_embed.side_effect = Exception("Embedding generation failed")
mock_query.return_value = mock_results
- # Should NOT raise - graceful degradation
results, total = await hybrid_search(
query="test",
page=1,
page_size=20,
)
- # Verify it returns results even without embeddings
assert len(results) == 1
assert results[0]["slug"] == "test-agent"
assert total == 1
@@ -221,7 +181,6 @@ async def test_hybrid_search_with_filters():
) as mock_embed:
mock_embed.return_value = [0.1] * embeddings.EMBEDDING_DIM
- # Test with featured filter
results, total = await hybrid_search(
query="test",
featured=True,
@@ -231,11 +190,8 @@ async def test_hybrid_search_with_filters():
page_size=10,
)
- # Verify filters were applied in the query
call_args = mock_query.call_args
- params = call_args[0][1:] # Skip SQL template
-
- # Should have query, query_lower, creators array, category
+ params = call_args[0][1:]
assert len(params) >= 4
@@ -268,18 +224,14 @@ async def test_hybrid_search_weights():
page_size=20,
)
- # Verify custom weights were used in the query
call_args = mock_query.call_args
sql_template = call_args[0][0]
- params = call_args[0][1:] # Get all parameters passed
-
- # Check that SQL uses parameterized weights (not f-string interpolation)
- assert "$" in sql_template # Verify parameterization is used
+ params = call_args[0][1:]
- # Check that custom weights are in the params
- assert 0.5 in params # semantic weight
- assert 0.3 in params # lexical weight
- assert 0.1 in params # category and recency weights
+ assert "$" in sql_template
+ assert 0.5 in params
+ assert 0.3 in params
+ assert 0.1 in params
@pytest.mark.asyncio(loop_scope="session")
@@ -289,14 +241,12 @@ async def test_hybrid_search_min_score_filtering():
with patch(
"backend.api.features.store.hybrid_search.query_raw_with_schema"
) as mock_query:
- # Return results with varying scores
mock_query.return_value = [
{
"slug": "high-score/agent",
"agent_name": "High Score Agent",
"combined_score": 0.8,
"total_count": 1,
- # ... other fields
}
]
@@ -305,36 +255,25 @@ async def test_hybrid_search_min_score_filtering():
) as mock_embed:
mock_embed.return_value = [0.1] * embeddings.EMBEDDING_DIM
- # Test with custom min_score
results, total = await hybrid_search(
query="test",
- min_score=0.5, # High threshold
+ min_score=0.5,
page=1,
page_size=20,
)
- # Verify min_score was applied in query
call_args = mock_query.call_args
sql_template = call_args[0][0]
- params = call_args[0][1:] # Get all parameters
+ params = call_args[0][1:]
- # Check that SQL uses parameterized min_score
assert "combined_score >=" in sql_template
- assert "$" in sql_template # Verify parameterization
-
- # Check that custom min_score is in the params
assert 0.5 in params
@pytest.mark.asyncio(loop_scope="session")
@pytest.mark.integration
async def test_hybrid_search_pagination():
- """Test hybrid search pagination.
-
- Pagination happens in SQL (LIMIT/OFFSET), then BM25 reranking is applied
- to the paginated results.
- """
- # Create mock results that SQL would return for a page
+ """Test hybrid search pagination."""
mock_results = [
{
"slug": f"agent-{i}",
@@ -359,7 +298,7 @@ async def test_hybrid_search_pagination():
"popularity_score": 0.3,
"total_count": 25,
}
- for i in range(10) # SQL returns page_size results
+ for i in range(10)
]
with patch(
@@ -372,21 +311,17 @@ async def test_hybrid_search_pagination():
) as mock_embed:
mock_embed.return_value = [0.1] * embeddings.EMBEDDING_DIM
- # Test page 2 with page_size 10
results, total = await hybrid_search(
query="test",
page=2,
page_size=10,
)
- # Verify results returned
assert len(results) == 10
- assert total == 25 # Total from SQL COUNT(*) OVER()
+ assert total == 25
- # Verify the SQL query uses page_size and offset
call_args = mock_query.call_args
params = call_args[0]
- # Last two params are page_size and offset
page_size_param = params[-2]
offset_param = params[-1]
assert page_size_param == 10
@@ -400,7 +335,6 @@ async def test_hybrid_search_error_handling():
with patch(
"backend.api.features.store.hybrid_search.query_raw_with_schema"
) as mock_query:
- # Simulate database error
mock_query.side_effect = Exception("Database connection error")
with patch(
@@ -408,7 +342,6 @@ async def test_hybrid_search_error_handling():
) as mock_embed:
mock_embed.return_value = [0.1] * embeddings.EMBEDDING_DIM
- # Should raise exception
with pytest.raises(Exception) as exc_info:
await hybrid_search(
query="test",
@@ -417,360 +350,3 @@ async def test_hybrid_search_error_handling():
)
assert "Database connection error" in str(exc_info.value)
-
-
-# =============================================================================
-# Unified Hybrid Search Tests
-# =============================================================================
-
-
[email protected](loop_scope="session")
-async def test_unified_hybrid_search_basic():
- """Test basic unified hybrid search across all content types."""
- mock_results = [
- {
- "content_type": "STORE_AGENT",
- "content_id": "agent-1",
- "searchable_text": "Test Agent Description",
- "metadata": {"name": "Test Agent"},
- "updated_at": "2025-01-01T00:00:00Z",
- "semantic_score": 0.7,
- "lexical_score": 0.8,
- "category_score": 0.5,
- "recency_score": 0.3,
- "combined_score": 0.6,
- "total_count": 2,
- },
- {
- "content_type": "BLOCK",
- "content_id": "block-1",
- "searchable_text": "Test Block Description",
- "metadata": {"name": "Test Block"},
- "updated_at": "2025-01-01T00:00:00Z",
- "semantic_score": 0.6,
- "lexical_score": 0.7,
- "category_score": 0.4,
- "recency_score": 0.2,
- "combined_score": 0.5,
- "total_count": 2,
- },
- ]
-
- with patch(
- "backend.api.features.store.hybrid_search.query_raw_with_schema"
- ) as mock_query:
- with patch(
- "backend.api.features.store.hybrid_search.embed_query"
- ) as mock_embed:
- mock_query.return_value = mock_results
- mock_embed.return_value = [0.1] * embeddings.EMBEDDING_DIM
-
- results, total = await unified_hybrid_search(
- query="test",
- page=1,
- page_size=20,
- )
-
- assert len(results) == 2
- assert total == 2
- assert results[0]["content_type"] == "STORE_AGENT"
- assert results[1]["content_type"] == "BLOCK"
-
-
[email protected](loop_scope="session")
-async def test_unified_hybrid_search_filter_by_content_type():
- """Test unified search filtering by specific content types."""
- mock_results = [
- {
- "content_type": "BLOCK",
- "content_id": "block-1",
- "searchable_text": "Test Block",
- "metadata": {},
- "updated_at": "2025-01-01T00:00:00Z",
- "semantic_score": 0.7,
- "lexical_score": 0.8,
- "category_score": 0.0,
- "recency_score": 0.3,
- "combined_score": 0.5,
- "total_count": 1,
- },
- ]
-
- with patch(
- "backend.api.features.store.hybrid_search.query_raw_with_schema"
- ) as mock_query:
- with patch(
- "backend.api.features.store.hybrid_search.embed_query"
- ) as mock_embed:
- mock_query.return_value = mock_results
- mock_embed.return_value = [0.1] * embeddings.EMBEDDING_DIM
-
- results, total = await unified_hybrid_search(
- query="test",
- content_types=[ContentType.BLOCK],
- page=1,
- page_size=20,
- )
-
- # Verify content_types parameter was passed correctly
- call_args = mock_query.call_args
- params = call_args[0][1:]
- # The content types should be in the params as a list
- assert ["BLOCK"] in params
-
- assert len(results) == 1
- assert total == 1
-
-
[email protected](loop_scope="session")
-async def test_unified_hybrid_search_with_user_id():
- """Test unified search with user_id for private content."""
- mock_results = [
- {
- "content_type": "STORE_AGENT",
- "content_id": "agent-1",
- "searchable_text": "My Private Agent",
- "metadata": {},
- "updated_at": "2025-01-01T00:00:00Z",
- "semantic_score": 0.7,
- "lexical_score": 0.8,
- "category_score": 0.0,
- "recency_score": 0.3,
- "combined_score": 0.6,
- "total_count": 1,
- },
- ]
-
- with patch(
- "backend.api.features.store.hybrid_search.query_raw_with_schema"
- ) as mock_query:
- with patch(
- "backend.api.features.store.hybrid_search.embed_query"
- ) as mock_embed:
- mock_query.return_value = mock_results
- mock_embed.return_value = [0.1] * embeddings.EMBEDDING_DIM
-
- results, total = await unified_hybrid_search(
- query="test",
- user_id="user-123",
- page=1,
- page_size=20,
- )
-
- # Verify SQL contains user_id filter
- call_args = mock_query.call_args
- sql_template = call_args[0][0]
- params = call_args[0][1:]
-
- assert 'uce."userId"' in sql_template
- assert "user-123" in params
-
-
[email protected](loop_scope="session")
-async def test_unified_hybrid_search_excludes_null_user_library_agents():
- """Defense-in-depth: the SQL filter must NEVER let a LIBRARY_AGENT row
- with NULL userId match an authenticated query — that would leak private
- library data across users. Sentry's HIGH-severity bug-prediction on
- hybrid_search.py:259."""
- with patch(
- "backend.api.features.store.hybrid_search.query_raw_with_schema"
- ) as mock_query:
- with patch(
- "backend.api.features.store.hybrid_search.embed_query"
- ) as mock_embed:
- mock_query.return_value = []
- mock_embed.return_value = [0.1] * embeddings.EMBEDDING_DIM
-
- await unified_hybrid_search(
- query="email summarizer",
- user_id="user-123",
- page=1,
- page_size=20,
- )
-
- sql_template = mock_query.call_args[0][0]
- assert (
- "NOT (uce.\"contentType\" = 'LIBRARY_AGENT'::{schema_prefix}"
- '"ContentType" AND uce."userId" IS NULL)' in sql_template
- )
-
-
[email protected](loop_scope="session")
-async def test_unified_hybrid_search_custom_weights():
- """Test unified search with custom weights."""
- custom_weights = UnifiedSearchWeights(
- semantic=0.6,
- lexical=0.2,
- category=0.1,
- recency=0.1,
- )
-
- with patch(
- "backend.api.features.store.hybrid_search.query_raw_with_schema"
- ) as mock_query:
- with patch(
- "backend.api.features.store.hybrid_search.embed_query"
- ) as mock_embed:
- mock_query.return_value = []
- mock_embed.return_value = [0.1] * embeddings.EMBEDDING_DIM
-
- results, total = await unified_hybrid_search(
- query="test",
- weights=custom_weights,
- page=1,
- page_size=20,
- )
-
- # Verify custom weights are in parameters
- call_args = mock_query.call_args
- params = call_args[0][1:]
-
- assert 0.6 in params # semantic weight
- assert 0.2 in params # lexical weight
-
-
[email protected](loop_scope="session")
-async def test_unified_hybrid_search_graceful_degradation():
- """Test unified search gracefully degrades when embeddings unavailable."""
- mock_results = [
- {
- "content_type": "DOCUMENTATION",
- "content_id": "doc-1",
- "searchable_text": "API Documentation",
- "metadata": {},
- "updated_at": "2025-01-01T00:00:00Z",
- "semantic_score": 0.0, # Zero because no embedding
- "lexical_score": 0.8,
- "category_score": 0.0,
- "recency_score": 0.2,
- "combined_score": 0.5,
- "total_count": 1,
- },
- ]
-
- with patch(
- "backend.api.features.store.hybrid_search.query_raw_with_schema"
- ) as mock_query:
- with patch(
- "backend.api.features.store.hybrid_search.embed_query"
- ) as mock_embed:
- mock_query.return_value = mock_results
- mock_embed.side_effect = Exception(
- "Embedding generation failed"
- ) # Embedding failure
-
- # Should NOT raise - graceful degradation
- results, total = await unified_hybrid_search(
- query="test",
- page=1,
- page_size=20,
- )
-
- assert len(results) == 1
- assert total == 1
-
-
[email protected](loop_scope="session")
-async def test_unified_hybrid_search_empty_query():
- """Test unified search with empty query returns empty results."""
- results, total = await unified_hybrid_search(
- query="",
- page=1,
- page_size=20,
- )
-
- assert results == []
- assert total == 0
-
-
[email protected](loop_scope="session")
-async def test_unified_hybrid_search_pagination():
- """Test unified search pagination with BM25 reranking.
-
- Pagination happens in SQL (LIMIT/OFFSET), then BM25 reranking is applied
- to the paginated results.
- """
- # Create mock results that SQL would return for a page
- mock_results = [
- {
- "content_type": "STORE_AGENT",
- "content_id": f"agent-{i}",
- "searchable_text": f"Agent {i} description",
- "metadata": {"name": f"Agent {i}"},
- "updated_at": "2025-01-01T00:00:00Z",
- "semantic_score": 0.7,
- "lexical_score": 0.8 - (i * 0.01),
- "category_score": 0.5,
- "recency_score": 0.3,
- "combined_score": 0.6 - (i * 0.01),
- "total_count": 50,
- }
- for i in range(15) # SQL returns page_size results
- ]
-
- with patch(
- "backend.api.features.store.hybrid_search.query_raw_with_schema"
- ) as mock_query:
- with patch(
- "backend.api.features.store.hybrid_search.embed_query"
- ) as mock_embed:
- mock_query.return_value = mock_results
- mock_embed.return_value = [0.1] * embeddings.EMBEDDING_DIM
-
- results, total = await unified_hybrid_search(
- query="test",
- page=3,
- page_size=15,
- )
-
- # Verify results returned
- assert len(results) == 15
- assert total == 50 # Total from SQL COUNT(*) OVER()
-
- # Verify the SQL query uses page_size and offset
- call_args = mock_query.call_args
- params = call_args[0]
- # Last two params are page_size and offset
- page_size_param = params[-2]
- offset_param = params[-1]
- assert page_size_param == 15
- assert offset_param == 30 # (page 3 - 1) * 15
-
-
[email protected](loop_scope="session")
-async def test_unified_hybrid_search_schema_prefix():
- """Test unified search uses schema_prefix placeholder."""
- with patch(
- "backend.api.features.store.hybrid_search.query_raw_with_schema"
- ) as mock_query:
- with patch(
- "backend.api.features.store.hybrid_search.embed_query"
- ) as mock_embed:
- mock_query.return_value = []
- mock_embed.return_value = [0.1] * embeddings.EMBEDDING_DIM
-
- await unified_hybrid_search(
- query="test",
- page=1,
- page_size=20,
- )
-
- call_args = mock_query.call_args
- sql_template = call_args[0][0]
-
- # Verify schema_prefix placeholder is used for table references
- assert "{schema_prefix}" in sql_template
- assert '"UnifiedContentEmbedding"' in sql_template
-
-
-if __name__ == "__main__":
- pytest.main([__file__, "-v", "-s"])
diff --git a/autogpt_platform/backend/backend/api/features/store/media_test.py b/autogpt_platform/backend/backend/api/features/store/media_test.py
index 7f3899c8a5..264fc30774 100644
--- a/autogpt_platform/backend/backend/api/features/store/media_test.py
+++ b/autogpt_platform/backend/backend/api/features/store/media_test.py
@@ -47,7 +47,7 @@ def mock_storage_client(mocker):
async def test_upload_media_success(mock_settings, mock_storage_client):
# Create test JPEG data with valid signature
- test_data = b"\xFF\xD8\xFF" + b"test data"
+ test_data = b"\xff\xd8\xff" + b"test data"
test_file = fastapi.UploadFile(
filename="laptop.jpeg",
@@ -85,7 +85,7 @@ async def test_upload_media_missing_credentials(monkeypatch):
test_file = fastapi.UploadFile(
filename="laptop.jpeg",
- file=io.BytesIO(b"\xFF\xD8\xFF" + b"test data"), # Valid JPEG signature
+ file=io.BytesIO(b"\xff\xd8\xff" + b"test data"), # Valid JPEG signature
headers=starlette.datastructures.Headers({"content-type": "image/jpeg"}),
)
@@ -110,7 +110,7 @@ async def test_upload_media_video_type(mock_settings, mock_storage_client):
async def test_upload_media_file_too_large(mock_settings, mock_storage_client):
- large_data = b"\xFF\xD8\xFF" + b"x" * (
+ large_data = b"\xff\xd8\xff" + b"x" * (
50 * 1024 * 1024 + 1
) # 50MB + 1 byte with valid JPEG signature
test_file = fastapi.UploadFile(
diff --git a/autogpt_platform/backend/backend/api/features/store/routes.py b/autogpt_platform/backend/backend/api/features/store/routes.py
index 5085804296..cbc75b9e14 100644
--- a/autogpt_platform/backend/backend/api/features/store/routes.py
+++ b/autogpt_platform/backend/backend/api/features/store/routes.py
@@ -10,12 +10,12 @@ from pydantic import BaseModel
import backend.data.graph
import backend.util.json
+from backend.api.features.search import hybrid_search as search_engine
from backend.util.exceptions import NotFoundError
from backend.util.models import Pagination
from . import cache as store_cache
from . import db as store_db
-from . import hybrid_search as store_hybrid_search
from . import image_gen as store_image_gen
from . import media as store_media
from . import model as store_model
@@ -91,7 +91,7 @@ async def unified_search(
"""
# Perform unified hybrid search
- results, total = await store_hybrid_search.unified_hybrid_search(
+ results, total = await search_engine.unified_hybrid_search(
query=query,
content_types=content_types,
user_id=user_id,
diff --git a/autogpt_platform/backend/backend/api/features/workspace/embeddings.py b/autogpt_platform/backend/backend/api/features/workspace/embeddings.py
new file mode 100644
index 0000000000..da43cef46a
--- /dev/null
+++ b/autogpt_platform/backend/backend/api/features/workspace/embeddings.py
@@ -0,0 +1,79 @@
+"""Background embedding generation for ``UserWorkspaceFile`` rows.
+
+Mirrors ``backend.api.features.library.embeddings``: fire-and-forget
+scheduling so user-facing file writes don't pay the OpenAI embedding
+latency. Failures are logged, not raised — a missing embedding only
+degrades search quality, it never breaks correctness.
+
+Only the user-visible ``name`` (and the path stem, when it diverges) is
+embedded — file *contents* are intentionally out of scope.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+
+from prisma.enums import ContentType
+
+from backend.api.features.search.content_handlers import build_workspace_file_text
+from backend.api.features.search.embeddings import (
+ delete_content_embedding,
+ ensure_content_embedding,
+ get_content_embedding,
+)
+
+logger = logging.getLogger(__name__)
+
+# See ``library/embeddings.py`` for why we hold a strong ref to the task.
+_background_tasks: set[asyncio.Task[None]] = set()
+
+
+async def _run_embedding(file_id: str, user_id: str, name: str, path: str) -> None:
+ try:
+ searchable_text = build_workspace_file_text(name, path)
+ if not searchable_text:
+ logger.debug(
+ "Skipping workspace file embedding for %s: empty searchable text",
+ file_id,
+ )
+ return
+ existing = await get_content_embedding(
+ ContentType.WORKSPACE_FILE, file_id, user_id
+ )
+ if existing and existing.get("searchableText") == searchable_text:
+ return
+ await ensure_content_embedding(
+ content_type=ContentType.WORKSPACE_FILE,
+ content_id=file_id,
+ searchable_text=searchable_text,
+ metadata={"name": name, "path": path},
+ user_id=user_id,
+ force=True,
+ )
+ except Exception as e:
+ logger.warning(
+ "Failed to ensure workspace file embedding for %s: %s", file_id, e
+ )
+
+
+def schedule_workspace_file_embedding(
+ file_id: str, user_id: str, name: str, path: str
+) -> asyncio.Task[None]:
+ """Schedule a fire-and-forget (re-)embed of a workspace file."""
+ task = asyncio.create_task(_run_embedding(file_id, user_id, name, path))
+ _background_tasks.add(task)
+ task.add_done_callback(_background_tasks.discard)
+ return task
+
+
+async def delete_workspace_file_embedding(file_id: str, user_id: str) -> None:
+ """Best-effort embedding cleanup when a workspace file is deleted."""
+ try:
+ await delete_content_embedding(
+ ContentType.WORKSPACE_FILE, file_id, user_id=user_id
+ )
+ except Exception as e:
+ logger.warning(
+ "Failed to delete workspace file embedding for %s: %s", file_id, e
+ )
diff --git a/autogpt_platform/backend/backend/api/rest_api.py b/autogpt_platform/backend/backend/api/rest_api.py
index 006bb68bf0..fe49047271 100644
--- a/autogpt_platform/backend/backend/api/rest_api.py
+++ b/autogpt_platform/backend/backend/api/rest_api.py
@@ -37,6 +37,7 @@ import backend.api.features.otto.routes
import backend.api.features.platform_linking.routes
import backend.api.features.postmark.postmark
import backend.api.features.push.routes as push_routes
+import backend.api.features.search.routes as search_routes
import backend.api.features.store.model
import backend.api.features.store.routes
import backend.api.features.v1
@@ -423,6 +424,11 @@ app.include_router(
tags=["push"],
prefix="/api/push",
)
+app.include_router(
+ search_routes.router,
+ tags=["search"],
+ prefix="/api/search",
+)
app.include_router(
backend.api.features.platform_linking.routes.router,
tags=["platform-linking"],
diff --git a/autogpt_platform/backend/backend/conftest.py b/autogpt_platform/backend/backend/conftest.py
index 4fc6693f5e..9b3a19bfad 100644
--- a/autogpt_platform/backend/backend/conftest.py
+++ b/autogpt_platform/backend/backend/conftest.py
@@ -46,33 +46,46 @@ def target_user_id() -> str:
return "5e53486c-cf57-477e-ba2a-cb02dc828e1c"
+async def _create_user_with_loop_retry(user_data: dict) -> None:
+ """Create a user, retrying once on a transient ``Event loop is closed``.
+
+ Fire-and-forget background tasks elsewhere can leave the Prisma pool
+ bound to a now-closed test function loop. The first session-loop DB
+ call after that surfaces as ``RuntimeError: Event loop is closed``;
+ the pool re-establishes itself on the retry.
+ """
+ from backend.data.user import get_or_create_user
+ from backend.util.exceptions import DatabaseError
+
+ try:
+ await get_or_create_user(user_data)
+ except DatabaseError as e:
+ if "Event loop is closed" not in str(e):
+ raise
+ await get_or_create_user(user_data)
+
+
@pytest.fixture
async def setup_test_user(test_user_id):
"""Create test user in database before tests."""
- from backend.data.user import get_or_create_user
-
- # Create the test user in the database using JWT token format
user_data = {
"sub": test_user_id,
"email": "[email protected]",
"user_metadata": {"name": "Test User"},
}
- await get_or_create_user(user_data)
+ await _create_user_with_loop_retry(user_data)
return test_user_id
@pytest.fixture
async def setup_admin_user(admin_user_id):
"""Create admin user in database before tests."""
- from backend.data.user import get_or_create_user
-
- # Create the admin user in the database using JWT token format
user_data = {
"sub": admin_user_id,
"email": "[email protected]",
"user_metadata": {"name": "Test Admin"},
}
- await get_or_create_user(user_data)
+ await _create_user_with_loop_retry(user_data)
return admin_user_id
diff --git a/autogpt_platform/backend/backend/copilot/chat_session_embeddings.py b/autogpt_platform/backend/backend/copilot/chat_session_embeddings.py
new file mode 100644
index 0000000000..c25f47219e
--- /dev/null
+++ b/autogpt_platform/backend/backend/copilot/chat_session_embeddings.py
@@ -0,0 +1,75 @@
+"""Background embedding generation for ``ChatSession`` titles.
+
+Mirrors ``backend.api.features.library.embeddings``: fire-and-forget so
+title updates stay fast. Only the (user-set or auto-generated) title is
+embedded — message bodies are intentionally out of scope here.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+
+from prisma.enums import ContentType
+
+from backend.api.features.search.embeddings import (
+ delete_content_embedding,
+ ensure_content_embedding,
+ get_content_embedding,
+)
+
+logger = logging.getLogger(__name__)
+
+# See ``library/embeddings.py`` for why we hold a strong ref to the task.
+_background_tasks: set[asyncio.Task[None]] = set()
+
+
+async def _run_embedding(session_id: str, user_id: str, title: str) -> None:
+ try:
+ searchable_text = (title or "").strip()
+ if not searchable_text:
+ # Nothing to search on — drop any stale row so renaming back
+ # to "untitled" doesn't keep the old title indexed.
+ await delete_content_embedding(
+ ContentType.CHAT_SESSION, session_id, user_id=user_id
+ )
+ return
+ existing = await get_content_embedding(
+ ContentType.CHAT_SESSION, session_id, user_id
+ )
+ if existing and existing.get("searchableText") == searchable_text:
+ return
+ await ensure_content_embedding(
+ content_type=ContentType.CHAT_SESSION,
+ content_id=session_id,
+ searchable_text=searchable_text,
+ metadata={"title": searchable_text},
+ user_id=user_id,
+ force=True,
+ )
+ except Exception as e:
+ logger.warning(
+ "Failed to ensure chat session embedding for %s: %s", session_id, e
+ )
+
+
+def schedule_chat_session_embedding(
+ session_id: str, user_id: str, title: str | None
+) -> asyncio.Task[None]:
+ """Schedule a fire-and-forget (re-)embed of a chat session title."""
+ task = asyncio.create_task(_run_embedding(session_id, user_id, title or ""))
+ _background_tasks.add(task)
+ task.add_done_callback(_background_tasks.discard)
+ return task
+
+
+async def delete_chat_session_embedding(session_id: str, user_id: str) -> None:
+ """Best-effort embedding cleanup when a chat session is deleted."""
+ try:
+ await delete_content_embedding(
+ ContentType.CHAT_SESSION, session_id, user_id=user_id
+ )
+ except Exception as e:
+ logger.warning(
+ "Failed to delete chat session embedding for %s: %s", session_id, e
+ )
diff --git a/autogpt_platform/backend/backend/copilot/chat_session_embeddings_test.py b/autogpt_platform/backend/backend/copilot/chat_session_embeddings_test.py
new file mode 100644
index 0000000000..210a8f7974
--- /dev/null
+++ b/autogpt_platform/backend/backend/copilot/chat_session_embeddings_test.py
@@ -0,0 +1,160 @@
+"""Tests for the chat-session title embedding scheduler."""
+
+import asyncio
+from unittest.mock import AsyncMock, patch
+
+import pytest
+from prisma.enums import ContentType
+
+from backend.copilot.chat_session_embeddings import (
+ _run_embedding,
+ delete_chat_session_embedding,
+ schedule_chat_session_embedding,
+)
+
+
+async def test_run_embedding_deletes_when_title_is_empty():
+ """An empty/whitespace title drops any stale row instead of embedding."""
+ with (
+ patch(
+ "backend.copilot.chat_session_embeddings.delete_content_embedding",
+ new=AsyncMock(return_value=None),
+ ) as mock_delete,
+ patch(
+ "backend.copilot.chat_session_embeddings.ensure_content_embedding",
+ new=AsyncMock(return_value=True),
+ ) as mock_ensure,
+ ):
+ await _run_embedding("sess-1", "user-1", " ")
+ mock_ensure.assert_not_called()
+ mock_delete.assert_awaited_once_with(
+ ContentType.CHAT_SESSION, "sess-1", user_id="user-1"
+ )
+
+
+async def test_run_embedding_forwards_user_scope_and_force_true():
+ with (
+ patch(
+ "backend.copilot.chat_session_embeddings.get_content_embedding",
+ new=AsyncMock(return_value=None),
+ ),
+ patch(
+ "backend.copilot.chat_session_embeddings.ensure_content_embedding",
+ new=AsyncMock(return_value=True),
+ ) as mock_ensure,
+ ):
+ await _run_embedding("sess-1", "user-1", "My Chat")
+ kwargs = mock_ensure.call_args.kwargs
+ assert kwargs["content_type"] == ContentType.CHAT_SESSION
+ assert kwargs["content_id"] == "sess-1"
+ assert kwargs["user_id"] == "user-1"
+ assert kwargs["searchable_text"] == "My Chat"
+ assert kwargs["metadata"] == {"title": "My Chat"}
+ assert kwargs["force"] is True
+
+
+async def test_run_embedding_skips_when_title_unchanged():
+ """If the existing row already carries the same title, skip re-embedding."""
+ with (
+ patch(
+ "backend.copilot.chat_session_embeddings.get_content_embedding",
+ new=AsyncMock(return_value={"searchableText": "My Chat"}),
+ ),
+ patch(
+ "backend.copilot.chat_session_embeddings.ensure_content_embedding",
+ new=AsyncMock(return_value=True),
+ ) as mock_ensure,
+ ):
+ await _run_embedding("sess-1", "user-1", "My Chat")
+ mock_ensure.assert_not_called()
+
+
+async def test_run_embedding_refreshes_when_title_changed():
+ with (
+ patch(
+ "backend.copilot.chat_session_embeddings.get_content_embedding",
+ new=AsyncMock(return_value={"searchableText": "Old Title"}),
+ ),
+ patch(
+ "backend.copilot.chat_session_embeddings.ensure_content_embedding",
+ new=AsyncMock(return_value=True),
+ ) as mock_ensure,
+ ):
+ await _run_embedding("sess-1", "user-1", "New Title")
+ mock_ensure.assert_awaited_once()
+
+
+async def test_run_embedding_swallows_failures():
+ """Failures must not propagate — it's a best-effort background task."""
+ with (
+ patch(
+ "backend.copilot.chat_session_embeddings.get_content_embedding",
+ new=AsyncMock(return_value=None),
+ ),
+ patch(
+ "backend.copilot.chat_session_embeddings.ensure_content_embedding",
+ new=AsyncMock(side_effect=RuntimeError("openai down")),
+ ),
+ ):
+ # Must not raise.
+ await _run_embedding("sess-1", "user-1", "My Chat")
+
+
+async def test_schedule_returns_task_and_runs_in_background():
+ with (
+ patch(
+ "backend.copilot.chat_session_embeddings.get_content_embedding",
+ new=AsyncMock(return_value=None),
+ ),
+ patch(
+ "backend.copilot.chat_session_embeddings.ensure_content_embedding",
+ new=AsyncMock(return_value=True),
+ ) as mock_ensure,
+ ):
+ task = schedule_chat_session_embedding("sess-1", "user-1", "My Chat")
+ assert isinstance(task, asyncio.Task)
+ await task
+ mock_ensure.assert_awaited_once()
+
+
+async def test_schedule_handles_none_title():
+ """A ``None`` title is coerced to empty and triggers the delete path."""
+ with patch(
+ "backend.copilot.chat_session_embeddings.delete_content_embedding",
+ new=AsyncMock(return_value=None),
+ ) as mock_delete:
+ task = schedule_chat_session_embedding("sess-1", "user-1", None)
+ await task
+ mock_delete.assert_awaited_once_with(
+ ContentType.CHAT_SESSION, "sess-1", user_id="user-1"
+ )
+
+
+async def test_delete_chat_session_embedding_forwards_user_scope():
+ with patch(
+ "backend.copilot.chat_session_embeddings.delete_content_embedding",
+ new=AsyncMock(return_value=None),
+ ) as mock_delete:
+ await delete_chat_session_embedding("sess-1", "user-1")
+ mock_delete.assert_awaited_once_with(
+ ContentType.CHAT_SESSION, "sess-1", user_id="user-1"
+ )
+
+
+async def test_delete_chat_session_embedding_swallows_failures():
+ """Cleanup failures must not propagate."""
+ with patch(
+ "backend.copilot.chat_session_embeddings.delete_content_embedding",
+ new=AsyncMock(side_effect=RuntimeError("db down")),
+ ):
+ # Must not raise.
+ await delete_chat_session_embedding("sess-1", "user-1")
diff --git a/autogpt_platform/backend/backend/copilot/db.py b/autogpt_platform/backend/backend/copilot/db.py
index 04bd586680..1fe0e11510 100644
--- a/autogpt_platform/backend/backend/copilot/db.py
+++ b/autogpt_platform/backend/backend/copilot/db.py
@@ -526,10 +526,19 @@ async def get_user_chat_sessions(
user_id: str,
limit: int = 50,
offset: int = 0,
+ title_contains: str | None = None,
) -> list[ChatSessionInfo]:
- """Get chat sessions for a user, ordered by most recent."""
+ """Get chat sessions for a user, ordered by most recent.
+
+ ``title_contains`` is a case-insensitive substring filter used by
+ /search/global so sessions are findable by literal title match
+ without waiting on async embedding.
+ """
+ where: ChatSessionWhereInput = {"userId": user_id}
+ if title_contains:
+ where["title"] = {"contains": title_contains, "mode": "insensitive"}
prisma_sessions = await PrismaChatSession.prisma().find_many(
- where={"userId": user_id},
+ where=where,
order={"updatedAt": "desc"},
take=limit,
skip=offset,
diff --git a/autogpt_platform/backend/backend/copilot/model.py b/autogpt_platform/backend/backend/copilot/model.py
index 67fb9c3564..d2e8fd996d 100644
--- a/autogpt_platform/backend/backend/copilot/model.py
+++ b/autogpt_platform/backend/backend/copilot/model.py
@@ -1062,15 +1062,25 @@ async def get_user_sessions(
user_id: str,
limit: int = 50,
offset: int = 0,
+ title_contains: str | None = None,
) -> tuple[list[ChatSessionInfo], int]:
"""Get chat sessions for a user from the database with total count.
+ ``title_contains`` is a case-insensitive substring filter used by
+ /search/global so sessions are findable by literal title match
+ without waiting on async embedding.
+
Returns:
A tuple of (sessions, total_count) where total_count is the overall
number of sessions for the user (not just the current page).
"""
db = chat_db()
- sessions = await db.get_user_chat_sessions(user_id, limit, offset)
+ sessions = await db.get_user_chat_sessions(
+ user_id, limit, offset, title_contains=title_contains
+ )
+ # Total count ignores the filter — it's the user's overall session
+ # count, used by paginated listings. The /search/global caller
+ # discards it.
total_count = await db.get_user_session_count(user_id)
return sessions, total_count
@@ -1102,6 +1112,20 @@ async def delete_chat_session(session_id: str, user_id: str | None = None) -> bo
except Exception as e:
logger.warning(f"Failed to delete session {session_id} from cache: {e}")
+ # Best-effort embedding cleanup so deleted sessions stop appearing in
+ # /search/global hits. Only attempt when we have a user_id since the
+ # embedding row is user-scoped; without it the user-filter would miss
+ # and the orphan row (if any) gets cleaned up by the periodic embedder.
+ if user_id:
+ try:
+ from backend.copilot.chat_session_embeddings import (
+ delete_chat_session_embedding,
+ )
+
+ await delete_chat_session_embedding(session_id, user_id)
+ except Exception as e:
+ logger.warning(f"Failed to delete session embedding for {session_id}: {e}")
+
# Shut down any local browser daemon for this session (best-effort).
# Inline import required: all tool modules import ChatSession from this
# module, so any top-level import from tools.* would create a cycle.
@@ -1158,6 +1182,20 @@ async def update_session_title(
f"Cache title update failed for session {session_id} (non-critical): {e}"
)
+ # Fire-and-forget: index the new title so /search/global can find
+ # this session. Local import keeps the heavy embedding deps
+ # (OpenAI client, prisma raw SQL) off the model import path.
+ try:
+ from backend.copilot.chat_session_embeddings import (
+ schedule_chat_session_embedding,
+ )
+
+ schedule_chat_session_embedding(session_id, user_id, title)
+ except Exception as e:
+ logger.warning(
+ f"Failed to schedule session title embedding for {session_id}: {e}"
+ )
+
return True
except Exception as e:
logger.error(f"Failed to update title for session {session_id}: {e}")
diff --git a/autogpt_platform/backend/backend/copilot/tools/search_docs.py b/autogpt_platform/backend/backend/copilot/tools/search_docs.py
index b0d787f681..693361a2fd 100644
--- a/autogpt_platform/backend/backend/copilot/tools/search_docs.py
+++ b/autogpt_platform/backend/backend/copilot/tools/search_docs.py
@@ -5,6 +5,7 @@ from typing import Any
from prisma.enums import ContentType
+from backend.api.features.search.hybrid_search import HybridSearchRow
from backend.copilot.model import ChatSession
from backend.data.db_accessors import search
@@ -132,7 +133,7 @@ class SearchDocsTool(BaseTool):
)
# Deduplicate by document path (keep highest scoring section per doc)
- seen_docs: dict[str, dict[str, Any]] = {}
+ seen_docs: dict[str, HybridSearchRow] = {}
for result in results:
metadata = result.get("metadata", {})
doc_path = metadata.get("path", "")
diff --git a/autogpt_platform/backend/backend/data/db_accessors.py b/autogpt_platform/backend/backend/data/db_accessors.py
index 8598fe9d6f..2221866684 100644
--- a/autogpt_platform/backend/backend/data/db_accessors.py
+++ b/autogpt_platform/backend/backend/data/db_accessors.py
@@ -55,7 +55,7 @@ def store_db():
def search():
if db.is_connected():
- from backend.api.features.store import hybrid_search as _search
+ from backend.api.features.search import hybrid_search as _search
search = _search
else:
diff --git a/autogpt_platform/backend/backend/data/db_manager.py b/autogpt_platform/backend/backend/data/db_manager.py
index c71f0589b8..3b01a04146 100644
--- a/autogpt_platform/backend/backend/data/db_manager.py
+++ b/autogpt_platform/backend/backend/data/db_manager.py
@@ -21,18 +21,18 @@ from backend.api.features.library.db import (
update_graph_in_library,
update_library_agent,
)
+from backend.api.features.search.embeddings import (
+ cleanup_orphaned_embeddings,
+ get_embedding_stats,
+)
+from backend.api.features.search.hybrid_search import unified_hybrid_search
from backend.api.features.store.db import (
get_agent,
get_available_graph,
get_store_agent_details,
get_store_agents,
)
-from backend.api.features.store.embeddings import (
- backfill_missing_embeddings,
- cleanup_orphaned_embeddings,
- get_embedding_stats,
-)
-from backend.api.features.store.hybrid_search import unified_hybrid_search
+from backend.api.features.store.embeddings import backfill_missing_embeddings
from backend.copilot import db as chat_db
from backend.copilot.sharing.db import link_new_execution_to_chat_share
from backend.data import db
diff --git a/autogpt_platform/backend/backend/data/workspace.py b/autogpt_platform/backend/backend/data/workspace.py
index 62220b45fe..19384fde83 100644
--- a/autogpt_platform/backend/backend/data/workspace.py
+++ b/autogpt_platform/backend/backend/data/workspace.py
@@ -254,6 +254,7 @@ async def list_workspace_files(
include_deleted: bool = False,
limit: Optional[int] = None,
offset: int = 0,
+ name_contains: Optional[str] = None,
) -> list[WorkspaceFile]:
"""
List files in a workspace.
@@ -264,6 +265,9 @@ async def list_workspace_files(
include_deleted: Whether to include soft-deleted files
limit: Maximum number of files to return
offset: Number of files to skip
+ name_contains: Case-insensitive substring filter applied to
+ ``name``. Used by /search/global so files are findable by
+ literal name match without waiting on async embedding.
Returns:
List of WorkspaceFile instances
@@ -279,6 +283,9 @@ async def list_workspace_files(
path_prefix = f"/{path_prefix}"
where_clause["path"] = {"startswith": path_prefix}
+ if name_contains:
+ where_clause["name"] = {"contains": name_contains, "mode": "insensitive"}
+
files = await UserWorkspaceFile.prisma().find_many(
where=where_clause,
order={"createdAt": "desc"},
diff --git a/autogpt_platform/backend/backend/util/workspace.py b/autogpt_platform/backend/backend/util/workspace.py
index 8013014565..61d6e47e22 100644
--- a/autogpt_platform/backend/backend/util/workspace.py
+++ b/autogpt_platform/backend/backend/util/workspace.py
@@ -326,6 +326,21 @@ class WorkspaceManager:
f"at path {path}, size={len(content)} bytes"
)
+ # Fire-and-forget: index this file in the hybrid-search store so
+ # the user can find it from /search/global by name. No-ops cheaply
+ # when the existing embedding's text is unchanged.
+ try:
+ from backend.api.features.workspace.embeddings import (
+ schedule_workspace_file_embedding,
+ )
+
+ schedule_workspace_file_embedding(
+ file_id=file.id, user_id=self.user_id, name=file.name, path=file.path
+ )
+ except Exception as e:
+ # Embedding is purely a search-quality concern — never block writes.
+ logger.warning(f"Failed to schedule file embedding for {file.id}: {e}")
+
return file
async def list_files(
@@ -334,6 +349,7 @@ class WorkspaceManager:
limit: Optional[int] = None,
offset: int = 0,
include_all_sessions: bool = False,
+ name_contains: Optional[str] = None,
) -> list[WorkspaceFile]:
"""
List files in workspace.
@@ -359,6 +375,7 @@ class WorkspaceManager:
path_prefix=effective_path,
limit=limit,
offset=offset,
+ name_contains=name_contains,
)
async def delete_file(self, file_id: str) -> bool:
@@ -386,6 +403,21 @@ class WorkspaceManager:
# Soft-delete database record
result = await db.soft_delete_workspace_file(file_id, self.workspace_id)
+
+ # Best-effort cleanup of the search index so deleted files don't
+ # keep showing up in /search/global hits.
+ if result is not None:
+ try:
+ from backend.api.features.workspace.embeddings import (
+ delete_workspace_file_embedding,
+ )
+
+ await delete_workspace_file_embedding(
+ file_id=file_id, user_id=self.user_id
+ )
+ except Exception as e:
+ logger.warning(f"Failed to delete file embedding for {file_id}: {e}")
+
return result is not None
async def get_download_url(self, file_id: str, expires_in: int = 3600) -> str:
diff --git a/autogpt_platform/backend/migrations/20260526120000_add_file_and_chat_content_type_values/migration.sql b/autogpt_platform/backend/migrations/20260526120000_add_file_and_chat_content_type_values/migration.sql
new file mode 100644
index 0000000000..da9a1d1cf6
--- /dev/null
+++ b/autogpt_platform/backend/migrations/20260526120000_add_file_and_chat_content_type_values/migration.sql
@@ -0,0 +1,10 @@
+-- Step 1/2: add WORKSPACE_FILE and CHAT_SESSION to the ContentType enum.
+--
+-- Postgres requires `ALTER TYPE ... ADD VALUE` to be committed before the
+-- new value can be referenced in queries or constraints. Splitting this
+-- into its own migration keeps the follow-up migration
+-- (`20260526120001_user_scoped_content_type_check`) free to use the new
+-- values in `DELETE` / `CHECK` expressions.
+
+ALTER TYPE "ContentType" ADD VALUE IF NOT EXISTS 'WORKSPACE_FILE';
+ALTER TYPE "ContentType" ADD VALUE IF NOT EXISTS 'CHAT_SESSION';
diff --git a/autogpt_platform/backend/migrations/20260526120001_user_scoped_content_type_check/migration.sql b/autogpt_platform/backend/migrations/20260526120001_user_scoped_content_type_check/migration.sql
new file mode 100644
index 0000000000..a9431b0f87
--- /dev/null
+++ b/autogpt_platform/backend/migrations/20260526120001_user_scoped_content_type_check/migration.sql
@@ -0,0 +1,27 @@
+-- Step 2/2: tighten the per-user CHECK constraint on UnifiedContentEmbedding
+-- to cover WORKSPACE_FILE and CHAT_SESSION (added in the preceding
+-- migration). The split is required because Postgres errors with
+-- "unsafe use of new value" if a CHECK / DELETE in the same transaction
+-- as `ALTER TYPE ... ADD VALUE` references the new enum values.
+
+-- Sweep any stray rows with NULL userId for the per-user types so the
+-- new CHECK doesn't fail in environments where someone hand-inserted
+-- (no rows are expected in practice).
+DELETE FROM "UnifiedContentEmbedding"
+WHERE "contentType" IN ('WORKSPACE_FILE', 'CHAT_SESSION')
+ AND "userId" IS NULL;
+
+-- Replace the LIBRARY_AGENT-only CHECK with one that covers all three
+-- per-user content types. Drop-then-add keeps idempotency on re-runs.
+ALTER TABLE "UnifiedContentEmbedding"
+DROP CONSTRAINT IF EXISTS "UnifiedContentEmbedding_library_agent_requires_user";
+
+ALTER TABLE "UnifiedContentEmbedding"
+DROP CONSTRAINT IF EXISTS "UnifiedContentEmbedding_user_scoped_requires_user";
+
+ALTER TABLE "UnifiedContentEmbedding"
+ADD CONSTRAINT "UnifiedContentEmbedding_user_scoped_requires_user"
+CHECK (
+ "contentType" NOT IN ('LIBRARY_AGENT', 'WORKSPACE_FILE', 'CHAT_SESSION')
+ OR "userId" IS NOT NULL
+);
diff --git a/autogpt_platform/backend/schema.prisma b/autogpt_platform/backend/schema.prisma
index 81a4128298..8ce424a00f 100644
--- a/autogpt_platform/backend/schema.prisma
+++ b/autogpt_platform/backend/schema.prisma
@@ -1283,6 +1283,8 @@ enum ContentType {
INTEGRATION // File-based: Python classes (blocks with credentials)
DOCUMENTATION // File-based: .md/.mdx files
LIBRARY_AGENT // Database: User's personal agents
+ WORKSPACE_FILE // Database: UserWorkspaceFile (per-user)
+ CHAT_SESSION // Database: ChatSession (per-user)
}
// Unified embeddings table for all searchable content types
diff --git a/autogpt_platform/frontend/.storybook/main.ts b/autogpt_platform/frontend/.storybook/main.ts
index 235dbf4749..40b0203026 100644
--- a/autogpt_platform/frontend/.storybook/main.ts
+++ b/autogpt_platform/frontend/.storybook/main.ts
@@ -6,9 +6,11 @@ const config: StorybookConfig = {
"../src/components/tokens/**/*.stories.@(js|jsx|mjs|ts|tsx)",
"../src/components/atoms/**/*.stories.@(js|jsx|mjs|ts|tsx)",
"../src/components/molecules/**/*.stories.@(js|jsx|mjs|ts|tsx)",
+ "../src/components/organisms/**/*.stories.@(js|jsx|mjs|ts|tsx)",
"../src/components/ai-elements/**/*.stories.@(js|jsx|mjs|ts|tsx)",
"../src/components/renderers/**/*.stories.@(js|jsx|mjs|ts|tsx)",
"../src/app/[(]platform[)]/copilot/**/*.stories.@(js|jsx|mjs|ts|tsx)",
+ "../src/app/[(]platform[)]/components/**/*.stories.@(js|jsx|mjs|ts|tsx)",
],
addons: [
"@storybook/addon-a11y",
diff --git a/autogpt_platform/frontend/.storybook/preview.tsx b/autogpt_platform/frontend/.storybook/preview.tsx
index d8c67a0a3f..29fb7b08e0 100644
--- a/autogpt_platform/frontend/.storybook/preview.tsx
+++ b/autogpt_platform/frontend/.storybook/preview.tsx
@@ -7,6 +7,7 @@ import {
Title,
} from "@storybook/addon-docs/blocks";
import { Preview } from "@storybook/nextjs";
+import { QueryClient, QueryClientProvider } from "@tanstack/react-query";
import { initialize, mswLoader } from "msw-storybook-addon";
import React from "react";
import "../src/app/globals.css";
@@ -16,6 +17,16 @@ import { theme } from "./theme";
// Initialize MSW
initialize();
+// One QueryClient per browser session is fine for Storybook — retries
+// are off so failing MSW handlers surface immediately instead of being
+// hidden behind exponential backoff.
+const storyQueryClient = new QueryClient({
+ defaultOptions: {
+ queries: { retry: false, refetchOnWindowFocus: false },
+ mutations: { retry: false },
+ },
+});
+
const preview: Preview = {
parameters: {
nextjs: {
@@ -39,9 +50,11 @@ const preview: Preview = {
loaders: [mswLoader],
decorators: [
(Story) => (
- <div className="bg-background p-8">
- <Story />
- </div>
+ <QueryClientProvider client={storyQueryClient}>
+ <div className="bg-background p-8">
+ <Story />
+ </div>
+ </QueryClientProvider>
),
],
};
diff --git a/autogpt_platform/frontend/src/app/(platform)/components/GlobalSearchModal/GlobalSearchModal.tsx b/autogpt_platform/frontend/src/app/(platform)/components/GlobalSearchModal/GlobalSearchModal.tsx
new file mode 100644
index 0000000000..3502f7ebed
--- /dev/null
+++ b/autogpt_platform/frontend/src/app/(platform)/components/GlobalSearchModal/GlobalSearchModal.tsx
@@ -0,0 +1,38 @@
+import type { SearchResultItem } from "@/app/api/__generated__/models/searchResultItem";
+import { SearchCommandModal } from "@/components/organisms/SearchCommandModal/SearchCommandModal";
+import { useGlobalSearch } from "./useGlobalSearch";
+
+interface Props {
+ isOpen: boolean;
+ onClose: () => void;
+ onSelectItem: (item: SearchResultItem) => void;
+}
+
+export function GlobalSearchModal({ isOpen, onClose, onSelectItem }: Props) {
+ const { query, setQuery, buckets, itemsById, isFetching, isError } =
+ useGlobalSearch(isOpen);
+
+ function handleSelect(id: string) {
+ const apiItem = itemsById.get(id);
+ if (!apiItem) return;
+ onSelectItem(apiItem);
+ onClose();
+ }
+
+ return (
+ <SearchCommandModal
+ isOpen={isOpen}
+ onClose={onClose}
+ query={query}
+ onQueryChange={setQuery}
+ buckets={buckets}
+ isLoading={isFetching}
+ isError={isError}
+ placeholder="Search agents, files, chats..."
+ inputAriaLabel="Global search"
+ idleEmptyLabel="No recent items"
+ searchingEmptyLabel="No results found"
+ onSelectItem={(item) => handleSelect(item.id)}
+ />
+ );
+}
diff --git a/autogpt_platform/frontend/src/app/(platform)/components/GlobalSearchModal/__tests__/GlobalSearchModal.test.tsx b/autogpt_platform/frontend/src/app/(platform)/components/GlobalSearchModal/__tests__/GlobalSearchModal.test.tsx
new file mode 100644
index 0000000000..c4aff14e24
--- /dev/null
+++ b/autogpt_platform/frontend/src/app/(platform)/components/GlobalSearchModal/__tests__/GlobalSearchModal.test.tsx
@@ -0,0 +1,249 @@
+import {
+ getGetSearchGlobalSearchMockHandler,
+ getGetSearchGlobalSearchMockHandler200,
+} from "@/app/api/__generated__/endpoints/search/search.msw";
+import type { GlobalSearchResponse } from "@/app/api/__generated__/models/globalSearchResponse";
+import { server } from "@/mocks/mock-server";
+import { fireEvent, render, screen } from "@/tests/integrations/test-utils";
+import userEvent from "@testing-library/user-event";
+import { describe, expect, it, vi } from "vitest";
+import { GlobalSearchModal } from "../GlobalSearchModal";
+
+function fixedResponse(
+ overrides: Partial<GlobalSearchResponse> = {},
+): GlobalSearchResponse {
+ return {
+ agents: [
+ {
+ id: "a1",
+ type: "library_agent",
+ title: "Alpha agent",
+ subtitle: "library row",
+ score: 0.9,
+ updated_at: new Date("2025-01-01T00:00:00Z"),
+ },
+ {
+ id: "a2",
+ type: "store_agent",
+ title: "Bravo store",
+ subtitle: "by acme",
+ score: 0.8,
+ updated_at: new Date("2025-01-01T00:00:00Z"),
+ },
+ ],
+ files: [
+ {
+ id: "f1",
+ type: "workspace_file",
+ title: "notes.md",
+ subtitle: "/docs",
+ score: 0.7,
+ updated_at: new Date("2025-01-01T00:00:00Z"),
+ },
+ ],
+ chats: [
+ {
+ id: "c1",
+ type: "chat_session",
+ title: "Charlie chat",
+ score: 0.6,
+ updated_at: new Date("2025-01-01T00:00:00Z"),
+ },
+ ],
+ ...overrides,
+ };
+}
+
+function renderModal(
+ props: {
+ isOpen?: boolean;
+ onClose?: () => void;
+ onSelectItem?: (item: { id: string }) => void;
+ } = {},
+) {
+ const onClose = props.onClose ?? vi.fn();
+ const onSelectItem = props.onSelectItem ?? vi.fn();
+ const utils = render(
+ <GlobalSearchModal
+ isOpen={props.isOpen ?? true}
+ onClose={onClose}
+ onSelectItem={onSelectItem}
+ />,
+ );
+ return { ...utils, onClose, onSelectItem };
+}
+
+describe("GlobalSearchModal", () => {
+ it("returns null when closed", () => {
+ renderModal({ isOpen: false });
+ expect(screen.queryByRole("dialog")).toBeNull();
+ });
+
+ it("auto-focuses the search input when opened", async () => {
+ server.use(getGetSearchGlobalSearchMockHandler(fixedResponse()));
+ renderModal();
+ const input = await screen.findByRole("textbox", {
+ name: /global search/i,
+ });
+ await vi.waitFor(() => expect(document.activeElement).toBe(input));
+ });
+
+ it("renders bucketed results from the API", async () => {
+ server.use(getGetSearchGlobalSearchMockHandler200(fixedResponse()));
+ renderModal();
+
+ expect(await screen.findByText("Alpha agent")).toBeDefined();
+ expect(screen.getByText("Bravo store")).toBeDefined();
+ expect(screen.getByText("notes.md")).toBeDefined();
+ expect(screen.getByText("Charlie chat")).toBeDefined();
+
+ // Section headers are rendered for each non-empty bucket.
+ expect(screen.getByText("Agents")).toBeDefined();
+ expect(screen.getByText("Files")).toBeDefined();
+ expect(screen.getByText("Chats")).toBeDefined();
+ });
+
+ it("hides empty buckets", async () => {
+ server.use(
+ getGetSearchGlobalSearchMockHandler200(
+ fixedResponse({ files: [], chats: [] }),
+ ),
+ );
+ renderModal();
+
+ expect(await screen.findByText("Alpha agent")).toBeDefined();
+ expect(screen.queryByText("Files")).toBeNull();
+ expect(screen.queryByText("Chats")).toBeNull();
+ });
+
+ it("shows the empty state when the API returns no items", async () => {
+ server.use(
+ getGetSearchGlobalSearchMockHandler200({
+ agents: [],
+ files: [],
+ chats: [],
+ }),
+ );
+ renderModal();
+
+ expect(await screen.findByText("No recent items")).toBeDefined();
+ });
+
+ it("shows a query-specific empty state when searching with no results", async () => {
+ const user = userEvent.setup();
+ server.use(
+ getGetSearchGlobalSearchMockHandler200({
+ agents: [],
+ files: [],
+ chats: [],
+ }),
+ );
+ renderModal();
+
+ const input = await screen.findByRole("textbox", {
+ name: /global search/i,
+ });
+ await user.type(input, "zzzz");
+ expect(await screen.findByText("No results found")).toBeDefined();
+ });
+
+ it("invokes onSelectItem and onClose when a result is clicked", async () => {
+ const user = userEvent.setup();
+ server.use(getGetSearchGlobalSearchMockHandler200(fixedResponse()));
+ const onClose = vi.fn();
+ const onSelectItem = vi.fn();
+ renderModal({ onClose, onSelectItem });
+
+ const result = await screen.findByRole("option", { name: /alpha agent/i });
+ await user.click(result);
+
+ expect(onSelectItem).toHaveBeenCalledTimes(1);
+ expect(onSelectItem.mock.calls[0][0]).toMatchObject({
+ id: "a1",
+ type: "library_agent",
+ });
+ expect(onClose).toHaveBeenCalledTimes(1);
+ });
+
+ it("navigates results with arrow keys and selects with Enter across buckets", async () => {
+ server.use(getGetSearchGlobalSearchMockHandler200(fixedResponse()));
+ const onSelectItem = vi.fn();
+ renderModal({ onSelectItem });
+
+ // Wait for results to be present.
+ await screen.findByText("Charlie chat");
+ const dialog = screen.getByRole("dialog");
+
+ // Bucket order is Chats → Agents → Files; first chat is highlighted
+ // by default.
+ const charlie = screen.getByRole("option", { name: /charlie chat/i });
+ expect(charlie.getAttribute("aria-selected")).toBe("true");
+
+ // Three arrow-downs walks chat → agent-1 → agent-2 → files (notes.md).
+ fireEvent.keyDown(dialog, { key: "ArrowDown" });
+ fireEvent.keyDown(dialog, { key: "ArrowDown" });
+ fireEvent.keyDown(dialog, { key: "ArrowDown" });
+ await vi.waitFor(() => {
+ const notes = screen.getByRole("option", { name: /notes\.md/i });
+ expect(notes.getAttribute("aria-selected")).toBe("true");
+ });
+
+ // Enter selects the currently highlighted item.
+ fireEvent.keyDown(dialog, { key: "Enter" });
+ expect(onSelectItem).toHaveBeenCalledTimes(1);
+ expect(onSelectItem.mock.calls[0][0]).toMatchObject({
+ id: "f1",
+ type: "workspace_file",
+ });
+ });
+
+ it("closes when Escape is pressed", async () => {
+ server.use(getGetSearchGlobalSearchMockHandler200(fixedResponse()));
+ const onClose = vi.fn();
+ renderModal({ onClose });
+
+ await screen.findByText("Alpha agent");
+ const dialog = screen.getByRole("dialog");
+ fireEvent.keyDown(dialog, { key: "Escape" });
+ expect(onClose).toHaveBeenCalledTimes(1);
+ });
+
+ it("clears the query when the clear button is pressed", async () => {
+ const user = userEvent.setup();
+ server.use(getGetSearchGlobalSearchMockHandler200(fixedResponse()));
+ renderModal();
+
+ const input = (await screen.findByRole("textbox", {
+ name: /global search/i,
+ })) as HTMLInputElement;
+
+ await user.type(input, "alpha");
+ expect(input.value).toBe("alpha");
+
+ const clearButton = await screen.findByRole("button", {
+ name: /clear search/i,
+ });
+ await user.click(clearButton);
+ expect(input.value).toBe("");
+ });
+
+ it("highlights the matching query substring inside the title", async () => {
+ const user = userEvent.setup();
+ server.use(getGetSearchGlobalSearchMockHandler200(fixedResponse()));
+ renderModal();
+
+ await screen.findByText("Alpha agent");
+ const input = screen.getByRole("textbox", { name: /global search/i });
+ await user.type(input, "alpha");
+
+ // After debounce, the matching substring is rendered as a bold span.
+ const bolded = await screen.findByText(
+ (_content, element) =>
+ !!element &&
+ element.tagName === "SPAN" &&
+ element.className.includes("font-semibold") &&
+ element.textContent?.toLocaleLowerCase() === "alpha",
+ );
+ expect(bolded).toBeDefined();
+ });
+});
diff --git a/autogpt_platform/frontend/src/app/(platform)/components/GlobalSearchModal/helpers.tsx b/autogpt_platform/frontend/src/app/(platform)/components/GlobalSearchModal/helpers.tsx
new file mode 100644
index 0000000000..61b7a890b9
--- /dev/null
+++ b/autogpt_platform/frontend/src/app/(platform)/components/GlobalSearchModal/helpers.tsx
@@ -0,0 +1,124 @@
+import type { GlobalSearchResponse } from "@/app/api/__generated__/models/globalSearchResponse";
+import type { SearchResultItem } from "@/app/api/__generated__/models/searchResultItem";
+import type { SearchResultItemType } from "@/app/api/__generated__/models/searchResultItemType";
+import type {
+ SearchCommandBucket,
+ SearchCommandItem,
+} from "@/components/organisms/SearchCommandModal/helpers";
+import { cn } from "@/lib/utils";
+import {
+ ChatCircleIcon,
+ FileIcon,
+ StorefrontIcon,
+ TreeStructureIcon,
+} from "@phosphor-icons/react";
+import Image from "next/image";
+import type { ComponentType } from "react";
+import {
+ type PlatformLogo,
+ resolvePlatformLogo,
+} from "@/app/(platform)/copilot/components/ChatOriginIcon/platformLogos";
+
+export type BucketKey = "agents" | "files" | "chats";
+
+const ICON_BY_TYPE: Record<
+ SearchResultItemType,
+ ComponentType<{ className?: string }>
+> = {
+ // Workflow / node-graph cue maps better to "an agent" than a robot
+ // mascot — agents in this product are visual graphs of nodes, not
+ // anthropomorphic assistants.
+ library_agent: TreeStructureIcon,
+ store_agent: StorefrontIcon,
+ workspace_file: FileIcon,
+ chat_session: ChatCircleIcon,
+};
+
+// Chat sessions opened from external platforms (Discord, Slack, …)
+// carry their origin in ``metadata.source_platform``. We reuse the
+// ChatOriginIcon platform-logo table so the search row shows the same
+// branded image as the sidebar / chat header — consistent visual
+// language across surfaces.
+function makePlatformIconComponent(
+ logo: PlatformLogo,
+): ComponentType<{ className?: string }> {
+ function PlatformIcon({ className }: { className?: string }) {
+ return (
+ <Image
+ src={logo.src}
+ alt={logo.name}
+ width={16}
+ height={16}
+ loading="lazy"
+ className={cn("size-4 object-contain", className)}
+ />
+ );
+ }
+ PlatformIcon.displayName = `PlatformIcon(${logo.name})`;
+ return PlatformIcon;
+}
+
+function resolveIcon(
+ item: SearchResultItem,
+): ComponentType<{ className?: string }> {
+ if (item.type === "chat_session") {
+ const logo = resolvePlatformLogo(
+ item.metadata?.source_platform as string | undefined,
+ );
+ if (logo) return makePlatformIconComponent(logo);
+ }
+ return ICON_BY_TYPE[item.type];
+}
+
+const BUCKET_LABEL: Record<BucketKey, string> = {
+ agents: "Agents",
+ files: "Files",
+ chats: "Chats",
+};
+
+const BUCKET_ORDER: BucketKey[] = ["chats", "agents", "files"];
+
+// Namespace the key by type: raw IDs can collide across buckets (e.g. a
+// library agent and a chat session sharing an ID), which would clobber
+// ``itemsById`` and route a click to the wrong target.
+function itemKey(item: SearchResultItem): string {
+ return `${item.type}:${item.id}`;
+}
+
+function toCommandItem(item: SearchResultItem): SearchCommandItem {
+ return {
+ id: itemKey(item),
+ title: item.title,
+ subtitle: item.subtitle ?? null,
+ icon: resolveIcon(item),
+ };
+}
+
+/**
+ * Adapt the bucketed API response into the generic ``SearchCommandModal``
+ * shape. Bucket order matches ``BUCKET_ORDER``; the original
+ * ``SearchResultItem`` for each row is returned alongside via
+ * ``itemsById`` so the container can route on the original ``type``
+ * after a click.
+ */
+export function buildBucketsFromResponse(
+ response: GlobalSearchResponse | undefined,
+): {
+ buckets: SearchCommandBucket[];
+ itemsById: Map<string, SearchResultItem>;
+} {
+ const itemsById = new Map<string, SearchResultItem>();
+ if (!response) return { buckets: [], itemsById };
+
+ const buckets: SearchCommandBucket[] = BUCKET_ORDER.map((key) => {
+ const apiItems = response[key] ?? [];
+ for (const apiItem of apiItems) itemsById.set(itemKey(apiItem), apiItem);
+ return {
+ key,
+ label: BUCKET_LABEL[key],
+ items: apiItems.map(toCommandItem),
+ };
+ });
+
+ return { buckets, itemsById };
+}
diff --git a/autogpt_platform/frontend/src/app/(platform)/components/GlobalSearchModal/useGlobalSearch.ts b/autogpt_platform/frontend/src/app/(platform)/components/GlobalSearchModal/useGlobalSearch.ts
new file mode 100644
index 0000000000..6c4e5468f4
--- /dev/null
+++ b/autogpt_platform/frontend/src/app/(platform)/components/GlobalSearchModal/useGlobalSearch.ts
@@ -0,0 +1,56 @@
+import { useEffect, useState } from "react";
+import { useGetSearchGlobalSearch } from "@/app/api/__generated__/endpoints/search/search";
+import type { GlobalSearchResponse } from "@/app/api/__generated__/models/globalSearchResponse";
+import { buildBucketsFromResponse } from "./helpers";
+
+const DEBOUNCE_MS = 200;
+const PER_TYPE_LIMIT = 4;
+
+export function useGlobalSearch(isOpen: boolean) {
+ const [query, setQuery] = useState("");
+ const [debouncedQuery, setDebouncedQuery] = useState("");
+
+ useEffect(() => {
+ if (!isOpen) {
+ setQuery("");
+ setDebouncedQuery("");
+ }
+ }, [isOpen]);
+
+ useEffect(() => {
+ const timeout = window.setTimeout(() => {
+ setDebouncedQuery(query);
+ }, DEBOUNCE_MS);
+ return () => window.clearTimeout(timeout);
+ }, [query]);
+
+ const { data, isFetching, isError } = useGetSearchGlobalSearch(
+ { q: debouncedQuery.trim(), per_type_limit: PER_TYPE_LIMIT },
+ {
+ query: {
+ enabled: isOpen,
+ // Keep the previous bucket of results visible while a new
+ // query is in-flight so the modal doesn't flash empty on every
+ // keystroke.
+ placeholderData: (prev) => prev,
+ // Recent-items (empty query) responses are cached server-side
+ // but a short client cache cuts the request on quick re-opens.
+ staleTime: 10_000,
+ },
+ },
+ );
+
+ const response =
+ data?.status === 200 ? (data.data as GlobalSearchResponse) : undefined;
+
+ const { buckets, itemsById } = buildBucketsFromResponse(response);
+
+ return {
+ query,
+ setQuery,
+ buckets,
+ itemsById,
+ isFetching,
+ isError,
+ };
+}
diff --git a/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatOriginIcon/ChatOriginIcon.tsx b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatOriginIcon/ChatOriginIcon.tsx
index 84c9330f33..b43c670a41 100644
--- a/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatOriginIcon/ChatOriginIcon.tsx
+++ b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatOriginIcon/ChatOriginIcon.tsx
@@ -2,24 +2,14 @@
import Image from "next/image";
import { useState } from "react";
-
-const PLATFORM_LOGOS: Record<string, { name: string; src: string }> = {
- discord: { name: "Discord", src: "/integrations/discord.png" },
- github: { name: "GitHub", src: "/integrations/github.png" },
- linear: { name: "Linear", src: "/integrations/linear.png" },
- slack: { name: "Slack", src: "/integrations/slack.png" },
- teams: { name: "Teams", src: "/integrations/teams.png" },
- telegram: { name: "Telegram", src: "/integrations/telegram.png" },
- whatsapp: { name: "WhatsApp", src: "/integrations/whatsapp.png" },
-};
+import { resolvePlatformLogo } from "./platformLogos";
interface Props {
sourcePlatform?: string | null;
}
export function ChatOriginIcon({ sourcePlatform }: Props) {
- const platform = sourcePlatform?.trim().toLowerCase();
- const logo = platform ? PLATFORM_LOGOS[platform] : undefined;
+ const logo = resolvePlatformLogo(sourcePlatform);
const [brokenSrc, setBrokenSrc] = useState<string | null>(null);
if (!logo || brokenSrc === logo.src) return null;
diff --git a/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatOriginIcon/platformLogos.ts b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatOriginIcon/platformLogos.ts
new file mode 100644
index 0000000000..d205549a0a
--- /dev/null
+++ b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatOriginIcon/platformLogos.ts
@@ -0,0 +1,21 @@
+export interface PlatformLogo {
+ name: string;
+ src: string;
+}
+
+export const PLATFORM_LOGOS: Record<string, PlatformLogo> = {
+ discord: { name: "Discord", src: "/integrations/discord.png" },
+ github: { name: "GitHub", src: "/integrations/github.png" },
+ linear: { name: "Linear", src: "/integrations/linear.png" },
+ slack: { name: "Slack", src: "/integrations/slack.png" },
+ teams: { name: "Teams", src: "/integrations/teams.png" },
+ telegram: { name: "Telegram", src: "/integrations/telegram.png" },
+ whatsapp: { name: "WhatsApp", src: "/integrations/whatsapp.png" },
+};
+
+export function resolvePlatformLogo(
+ sourcePlatform?: string | null,
+): PlatformLogo | undefined {
+ const key = sourcePlatform?.trim().toLowerCase();
+ return key ? PLATFORM_LOGOS[key] : undefined;
+}
diff --git a/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatSearchModal/ChatSearchModal.tsx b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatSearchModal/ChatSearchModal.tsx
deleted file mode 100644
index 0344dd00fa..0000000000
--- a/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatSearchModal/ChatSearchModal.tsx
+++ /dev/null
@@ -1,202 +0,0 @@
-import { Button } from "@/components/ui/button";
-import { Input } from "@/components/ui/input";
-import { Separator } from "@/components/ui/separator";
-import { cn } from "@/lib/utils";
-import { MagnifyingGlassIcon, XIcon } from "@phosphor-icons/react";
-import { useEffect, useRef } from "react";
-import type { KeyboardEvent } from "react";
-import { useCopilotChatRuntimeStore } from "../../copilotChatRegistry";
-import { useCopilotUIStore } from "../../store";
-import { ChatSearchResults } from "./ChatSearchResults";
-import type { SearchSession } from "./helpers";
-import { useChatSearch } from "./useChatSearch";
-
-interface Props {
- sessions: SearchSession[];
- currentSessionId: string | null;
- onSelectSession: (id: string) => void;
-}
-
-export function ChatSearchModal({
- sessions,
- currentSessionId,
- onSelectSession,
-}: Props) {
- const inputRef = useRef<HTMLInputElement>(null);
- const isSearchOpen = useCopilotUIStore((state) => state.isSearchOpen);
- const setSearchOpen = useCopilotUIStore((state) => state.setSearchOpen);
- const completedSessionIDs = useCopilotUIStore(
- (state) => state.completedSessionIDs,
- );
- const sessionNeedsReload = useCopilotChatRuntimeStore(
- (state) => state.sessionNeedsReload,
- );
- const {
- query,
- debouncedQuery,
- setQuery,
- clearQuery,
- results,
- highlightedIndex,
- setHighlightedIndex,
- highlightedResultRef,
- moveHighlight,
- isSearching,
- } = useChatSearch(sessions, isSearchOpen);
-
- useEffect(() => {
- if (!isSearchOpen) return;
- window.setTimeout(() => inputRef.current?.focus(), 0);
- }, [isSearchOpen]);
-
- if (!isSearchOpen) return null;
-
- function closeSearch() {
- setSearchOpen(false);
- }
-
- function selectHighlightedSession() {
- const session = results[highlightedIndex];
- if (!session) return;
- onSelectSession(session.id);
- closeSearch();
- }
-
- function handleKeyDown(event: KeyboardEvent<HTMLDivElement>) {
- if (event.key === "Escape") {
- event.preventDefault();
- closeSearch();
- } else if (event.key === "ArrowDown") {
- event.preventDefault();
- moveHighlight(1);
- } else if (event.key === "ArrowUp") {
- event.preventDefault();
- moveHighlight(-1);
- } else if (event.key === "Enter") {
- event.preventDefault();
- selectHighlightedSession();
- }
- }
-
- return (
- <div
- className="fixed inset-0 z-[80] flex items-start justify-center bg-black/20 px-4 pt-[18vh] backdrop-blur-sm"
- onMouseDown={closeSearch}
- >
- <div
- role="dialog"
- aria-modal="true"
- aria-labelledby="chat-search-title"
- className="w-full max-w-xl overflow-hidden rounded-3xl bg-white shadow-2xl ring-1 ring-zinc-200"
- onMouseDown={(event) => event.stopPropagation()}
- onKeyDown={handleKeyDown}
- >
- <div className="flex items-center gap-3 bg-zinc-50 p-3">
- <MagnifyingGlassIcon className="h-5 w-5 shrink-0 text-zinc-800" />
- <Input
- ref={inputRef}
- value={query}
- onChange={(event) => setQuery(event.target.value)}
- placeholder="Search chats..."
- aria-label="Search chats"
- aria-controls="chat-search-results"
- aria-activedescendant={
- results[highlightedIndex]
- ? `chat-search-result-${results[highlightedIndex].id}`
- : undefined
- }
- autoComplete="off"
- className="h-9 border-0 bg-transparent px-0 text-base text-zinc-950 shadow-none placeholder:text-zinc-700 focus-visible:ring-0"
- />
- {query ? (
- <Button
- type="button"
- variant="ghost"
- size="icon-sm"
- aria-label="Clear search"
- onClick={clearQuery}
- className="shrink-0"
- >
- <XIcon className="h-4 w-4" />
- </Button>
- ) : null}
- <Button
- type="button"
- variant="outline"
- size="icon-sm"
- aria-label="Close search"
- onClick={closeSearch}
- className="shrink-0 rounded-full border-zinc-400"
- >
- <XIcon className="h-4 w-4" />
- </Button>
- </div>
- <Separator />
- <div className="py-2">
- <div
- id="chat-search-title"
- className="px-4 pb-2 text-[11px] font-semibold uppercase tracking-wide text-zinc-700"
- >
- {isSearching ? "Results" : "Recent chats"}
- </div>
- <div
- id="chat-search-results"
- className={cn(
- "max-h-[22rem] overflow-y-auto scrollbar-thin scrollbar-track-transparent scrollbar-thumb-zinc-200",
- results.length === 0 && "max-h-none",
- )}
- >
- {results.length > 0 ? (
- <div className="px-2">
- <ChatSearchResults
- results={results}
- query={debouncedQuery}
- highlightedIndex={highlightedIndex}
- highlightedResultRef={highlightedResultRef}
- currentSessionId={currentSessionId}
- completedSessionIDs={completedSessionIDs}
- sessionNeedsReload={sessionNeedsReload}
- onHighlight={setHighlightedIndex}
- onSelect={(id) => {
- onSelectSession(id);
- closeSearch();
- }}
- />
- </div>
- ) : (
- <div className="px-3 py-8 text-center text-sm text-zinc-500">
- No chats found
- </div>
- )}
- </div>
- </div>
- <Separator />
- <div className="flex items-center justify-between gap-4 bg-zinc-50 px-4 py-4 text-xs text-zinc-700">
- <div className="flex items-center gap-4">
- <div className="flex items-center gap-1.5">
- <kbd className="inline-flex h-5 w-5 items-center justify-center rounded-md border border-zinc-200 bg-white font-sans text-[11px] text-zinc-800 shadow-[inset_0_-1px_0_rgba(15,15,20,0.04),0_1px_1px_rgba(15,15,20,0.04)]">
- ↑
- </kbd>
- <kbd className="inline-flex h-5 w-5 items-center justify-center rounded-md border border-zinc-200 bg-white font-sans text-[11px] text-zinc-800 shadow-[inset_0_-1px_0_rgba(15,15,20,0.04),0_1px_1px_rgba(15,15,20,0.04)]">
- ↓
- </kbd>
- <span>Navigate</span>
- </div>
- <div className="flex items-center gap-1.5">
- <kbd className="inline-flex h-5 min-w-5 items-center justify-center rounded-md border border-zinc-200 bg-white px-1 font-sans text-[11px] text-zinc-800 shadow-[inset_0_-1px_0_rgba(15,15,20,0.04),0_1px_1px_rgba(15,15,20,0.04)]">
- ↵
- </kbd>
- <span>Select</span>
- </div>
- </div>
- <div className="flex items-center gap-1.5">
- <kbd className="inline-flex h-5 items-center justify-center rounded-md border border-zinc-200 bg-white px-1.5 font-sans text-[11px] text-zinc-800 shadow-[inset_0_-1px_0_rgba(15,15,20,0.04),0_1px_1px_rgba(15,15,20,0.04)]">
- esc
- </kbd>
- <span>Close</span>
- </div>
- </div>
- </div>
- </div>
- );
-}
diff --git a/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatSearchModal/__tests__/ChatSearchModal.test.tsx b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatSearchModal/__tests__/ChatSearchModal.test.tsx
deleted file mode 100644
index 6e72b33732..0000000000
--- a/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatSearchModal/__tests__/ChatSearchModal.test.tsx
+++ /dev/null
@@ -1,295 +0,0 @@
-import {
- fireEvent,
- render,
- screen,
- within,
-} from "@/tests/integrations/test-utils";
-import userEvent from "@testing-library/user-event";
-import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
-import { useCopilotChatRuntimeStore } from "../../../copilotChatRegistry";
-import { useCopilotUIStore } from "../../../store";
-import { ChatSearchModal } from "../ChatSearchModal";
-import type { SearchSession } from "../helpers";
-
-function makeSession(
- overrides: Partial<SearchSession> & { id: string },
-): SearchSession {
- return {
- title: `Session ${overrides.id}`,
- updated_at: "2025-01-01T00:00:00Z",
- ...overrides,
- };
-}
-
-function renderModal(props: {
- sessions?: SearchSession[];
- currentSessionId?: string | null;
- onSelectSession?: (id: string) => void;
-}) {
- const onSelectSession = props.onSelectSession ?? vi.fn();
- return {
- onSelectSession,
- ...render(
- <ChatSearchModal
- sessions={props.sessions ?? []}
- currentSessionId={props.currentSessionId ?? null}
- onSelectSession={onSelectSession}
- />,
- ),
- };
-}
-
-describe("ChatSearchModal", () => {
- beforeEach(() => {
- useCopilotUIStore.setState({
- isSearchOpen: true,
- completedSessionIDs: new Set<string>(),
- });
- useCopilotChatRuntimeStore.setState({ sessionNeedsReload: {} });
- });
-
- afterEach(() => {
- useCopilotUIStore.setState({ isSearchOpen: false });
- });
-
- it("returns null when the search modal is closed", () => {
- useCopilotUIStore.setState({ isSearchOpen: false });
- renderModal({ sessions: [makeSession({ id: "a" })] });
- expect(screen.queryByRole("dialog")).toBeNull();
- });
-
- it("auto-focuses the input when opened", async () => {
- renderModal({ sessions: [makeSession({ id: "a", title: "Alpha" })] });
- const input = await screen.findByRole("textbox", { name: /search chats/i });
- await vi.waitFor(() => expect(document.activeElement).toBe(input));
- });
-
- it("shows 'No chats found' when query has no matches and hides the clear button when empty", async () => {
- const user = userEvent.setup();
- renderModal({ sessions: [makeSession({ id: "a", title: "Alpha" })] });
-
- expect(screen.queryByRole("button", { name: /clear search/i })).toBeNull();
-
- const input = screen.getByRole("textbox", { name: /search chats/i });
- await user.type(input, "zzz");
- expect(await screen.findByText("No chats found")).toBeDefined();
- expect(screen.getByRole("button", { name: /clear search/i })).toBeDefined();
- });
-
- it("closes when the backdrop is clicked but not when the dialog body is clicked", async () => {
- renderModal({ sessions: [makeSession({ id: "a" })] });
- const dialog = await screen.findByRole("dialog");
-
- fireEvent.mouseDown(dialog);
- expect(useCopilotUIStore.getState().isSearchOpen).toBe(true);
-
- const backdrop = dialog.parentElement!;
- fireEvent.mouseDown(backdrop);
- expect(useCopilotUIStore.getState().isSearchOpen).toBe(false);
- });
-
- it("closes when the close button is clicked", async () => {
- renderModal({ sessions: [makeSession({ id: "a" })] });
- fireEvent.click(
- await screen.findByRole("button", { name: /close search/i }),
- );
- expect(useCopilotUIStore.getState().isSearchOpen).toBe(false);
- });
-
- it("closes on Escape", async () => {
- renderModal({ sessions: [makeSession({ id: "a" })] });
- const dialog = await screen.findByRole("dialog");
- fireEvent.keyDown(dialog, { key: "Escape" });
- expect(useCopilotUIStore.getState().isSearchOpen).toBe(false);
- });
-
- it("navigates with ArrowDown/ArrowUp, clamps at the bounds, and selects via Enter", async () => {
- const onSelect = vi.fn();
- const sessions = [
- makeSession({
- id: "a",
- title: "Alpha",
- updated_at: "2025-01-03T00:00:00Z",
- }),
- makeSession({
- id: "b",
- title: "Beta",
- updated_at: "2025-01-02T00:00:00Z",
- }),
- makeSession({
- id: "c",
- title: "Gamma",
- updated_at: "2025-01-01T00:00:00Z",
- }),
- ];
- renderModal({ sessions, onSelectSession: onSelect });
-
- const dialog = await screen.findByRole("dialog");
- const options = within(dialog).getAllByRole("option");
- expect(options[0].getAttribute("aria-selected")).toBe("true");
-
- fireEvent.keyDown(dialog, { key: "ArrowUp" });
- expect(
- within(dialog).getAllByRole("option")[0].getAttribute("aria-selected"),
- ).toBe("true");
-
- fireEvent.keyDown(dialog, { key: "ArrowDown" });
- expect(
- within(dialog).getAllByRole("option")[1].getAttribute("aria-selected"),
- ).toBe("true");
-
- fireEvent.keyDown(dialog, { key: "ArrowDown" });
- fireEvent.keyDown(dialog, { key: "ArrowDown" });
- fireEvent.keyDown(dialog, { key: "ArrowDown" });
- expect(
- within(dialog).getAllByRole("option")[2].getAttribute("aria-selected"),
- ).toBe("true");
-
- fireEvent.keyDown(dialog, { key: "Enter" });
- expect(onSelect).toHaveBeenCalledWith("c");
- expect(useCopilotUIStore.getState().isSearchOpen).toBe(false);
- });
-
- it("Enter is a no-op when there are no results", async () => {
- const onSelect = vi.fn();
- renderModal({
- sessions: [makeSession({ id: "a", title: "Alpha" })],
- onSelectSession: onSelect,
- });
- const user = userEvent.setup();
- const dialog = await screen.findByRole("dialog");
- await user.type(
- screen.getByRole("textbox", { name: /search chats/i }),
- "nope",
- );
- await screen.findByText("No chats found");
- fireEvent.keyDown(dialog, { key: "Enter" });
- expect(onSelect).not.toHaveBeenCalled();
- expect(useCopilotUIStore.getState().isSearchOpen).toBe(true);
- });
-
- it("highlights a session on hover and selects it on click", async () => {
- const onSelect = vi.fn();
- renderModal({
- sessions: [
- makeSession({
- id: "a",
- title: "Alpha",
- updated_at: "2025-01-02T00:00:00Z",
- }),
- makeSession({
- id: "b",
- title: "Beta",
- updated_at: "2025-01-01T00:00:00Z",
- }),
- ],
- onSelectSession: onSelect,
- });
- const dialog = await screen.findByRole("dialog");
- const betaOption = within(dialog).getByRole("option", { name: /beta/i });
- fireEvent.mouseEnter(betaOption);
- expect(betaOption.getAttribute("aria-selected")).toBe("true");
-
- fireEvent.click(betaOption);
- expect(onSelect).toHaveBeenCalledWith("b");
- expect(useCopilotUIStore.getState().isSearchOpen).toBe(false);
- });
-
- it("clears the query when the clear button is clicked", async () => {
- const user = userEvent.setup();
- renderModal({ sessions: [makeSession({ id: "a", title: "Alpha" })] });
- const input = screen.getByRole("textbox", { name: /search chats/i });
- await user.type(input, "alp");
- expect(await screen.findByText("Results")).toBeDefined();
-
- fireEvent.click(screen.getByRole("button", { name: /clear search/i }));
- expect((input as HTMLInputElement).value).toBe("");
- expect(await screen.findByText("Recent chats")).toBeDefined();
- });
-
- it("renders running, queued, processing, and completed indicators", async () => {
- useCopilotUIStore.setState({
- isSearchOpen: true,
- completedSessionIDs: new Set<string>(["done"]),
- });
- useCopilotChatRuntimeStore.setState({
- sessionNeedsReload: { processing: false },
- });
-
- renderModal({
- currentSessionId: "current",
- sessions: [
- makeSession({
- id: "running",
- title: "Running chat",
- chat_status: "running",
- updated_at: "2025-01-05T00:00:00Z",
- }),
- makeSession({
- id: "queued",
- title: "Queued chat",
- chat_status: "queued",
- updated_at: "2025-01-04T00:00:00Z",
- }),
- makeSession({
- id: "processing",
- title: "Processing chat",
- is_processing: true,
- updated_at: "2025-01-03T00:00:00Z",
- }),
- makeSession({
- id: "done",
- title: "Done chat",
- updated_at: "2025-01-02T00:00:00Z",
- }),
- makeSession({
- id: "current",
- title: "Current chat",
- updated_at: "2025-01-01T00:00:00Z",
- }),
- ],
- });
-
- expect(await screen.findByLabelText("Session running")).toBeDefined();
- expect(screen.getByLabelText("Session queued")).toBeDefined();
- expect(screen.getByLabelText("Session processing")).toBeDefined();
- expect(screen.getByLabelText("Session completed")).toBeDefined();
- });
-
- it("hides the processing indicator when needsReload is set for that session", async () => {
- useCopilotChatRuntimeStore.setState({
- sessionNeedsReload: { stale: true },
- });
- renderModal({
- sessions: [
- makeSession({
- id: "stale",
- title: "Stale chat",
- is_processing: true,
- }),
- ],
- });
- await screen.findByText("Stale chat");
- expect(screen.queryByLabelText("Session processing")).toBeNull();
- });
-
- it("falls back to 'Untitled chat' when the session has no title", async () => {
- renderModal({ sessions: [makeSession({ id: "a", title: null })] });
- expect(await screen.findByText("Untitled chat")).toBeDefined();
- });
-
- it("debounces query changes and bolds the matched substring", async () => {
- const user = userEvent.setup();
- renderModal({
- sessions: [makeSession({ id: "a", title: "Revenue forecast" })],
- });
- const input = screen.getByRole("textbox", { name: /search chats/i });
- await user.type(input, "fore");
-
- const option = await screen.findByRole("option", {
- name: /revenue forecast/i,
- });
- const bold = await within(option).findByText("fore");
- expect(bold.className).toContain("font-semibold");
- });
-});
diff --git a/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatSidebar/ChatSidebar.tsx b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatSidebar/ChatSidebar.tsx
index f53bbe2327..8e2670b662 100644
--- a/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatSidebar/ChatSidebar.tsx
+++ b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatSidebar/ChatSidebar.tsx
@@ -46,7 +46,9 @@ import { shouldShowSessionProcessingIndicator } from "../../sessionActivity";
import { useCopilotUIStore } from "../../store";
import { useSessionDeletion } from "../../useSessionDeletion";
import { SESSION_LIST_QUERY_KEY, useSessionList } from "../../useSessionList";
-import { ChatSearchModal } from "../ChatSearchModal/ChatSearchModal";
+import type { SearchResultItem } from "@/app/api/__generated__/models/searchResultItem";
+import { GlobalSearchModal } from "@/app/(platform)/components/GlobalSearchModal/GlobalSearchModal";
+import { useRouter } from "next/navigation";
import { ChatSessionBlock } from "../ChatSessionBlock/ChatSessionBlock";
import { DeleteChatDialog } from "../DeleteChatDialog/DeleteChatDialog";
import { UsagePopover } from "../UsageLimits/UsagePopover/UsagePopover";
@@ -56,9 +58,14 @@ export function ChatSidebar() {
const { state } = useSidebar();
const isCollapsed = state === "collapsed";
const [sessionId, setSessionId] = useQueryState("sessionId", parseAsString);
- const { completedSessionIDs, clearCompletedSession, setSearchOpen } =
- useCopilotUIStore();
+ const {
+ completedSessionIDs,
+ clearCompletedSession,
+ isSearchOpen,
+ setSearchOpen,
+ } = useCopilotUIStore();
const isChatSearchEnabled = useGetFlag(Flag.CHAT_SEARCH);
+ const router = useRouter();
const sessionNeedsReload = useCopilotChatRuntimeStore(
(state) => state.sessionNeedsReload,
);
@@ -157,6 +164,41 @@ export function ChatSidebar() {
setSearchOpen(false);
}
+ function handleSelectSearchItem(item: SearchResultItem) {
+ if (item.type === "chat_session") {
+ setSessionId(item.id);
+ return;
+ }
+ if (item.type === "library_agent") {
+ router.push(`/library/agents/${item.id}`);
+ return;
+ }
+ if (item.type === "store_agent") {
+ // Store-agent rows carry creator + slug in ``metadata`` so we can
+ // build the marketplace URL without an extra fetch.
+ const metadata = (item.metadata ?? {}) as {
+ creator?: string;
+ slug?: string;
+ };
+ if (metadata.creator && metadata.slug) {
+ router.push(
+ `/marketplace/agent/${encodeURIComponent(metadata.creator)}/${encodeURIComponent(metadata.slug)}`,
+ );
+ }
+ return;
+ }
+ if (item.type === "workspace_file") {
+ // No dedicated viewer route — open the file's download URL in a
+ // new tab so the user gets the content immediately.
+ window.open(
+ `/api/proxy/api/workspace/files/${item.id}/download`,
+ "_blank",
+ "noopener,noreferrer",
+ );
+ return;
+ }
+ }
+
function handleRenameClick(
e: React.MouseEvent,
id: string,
@@ -530,10 +572,10 @@ export function ChatSidebar() {
)}
{isChatSearchEnabled ? (
- <ChatSearchModal
- sessions={sessions}
- currentSessionId={sessionId}
- onSelectSession={handleSelectSession}
+ <GlobalSearchModal
+ isOpen={isSearchOpen}
+ onClose={() => setSearchOpen(false)}
+ onSelectItem={handleSelectSearchItem}
/>
) : null}
</>
diff --git a/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatSidebar/__tests__/ChatSidebar.test.tsx b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatSidebar/__tests__/ChatSidebar.test.tsx
index 0b24b0e452..083d9abeca 100644
--- a/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatSidebar/__tests__/ChatSidebar.test.tsx
+++ b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatSidebar/__tests__/ChatSidebar.test.tsx
@@ -3,6 +3,8 @@ import {
getDeleteV2DeleteSessionMockHandler422,
getGetV2ListSessionsMockHandler200,
} from "@/app/api/__generated__/endpoints/chat/chat.msw";
+import { getGetSearchGlobalSearchMockHandler200 } from "@/app/api/__generated__/endpoints/search/search.msw";
+import type { GlobalSearchResponse } from "@/app/api/__generated__/models/globalSearchResponse";
import { server } from "@/mocks/mock-server";
import {
fireEvent,
@@ -180,35 +182,61 @@ describe("ChatSidebar — delete flow", () => {
});
});
-describe("ChatSidebar — search modal", () => {
+describe("ChatSidebar — global search modal", () => {
+ function makeSearchResponse(
+ overrides: Partial<GlobalSearchResponse> = {},
+ ): GlobalSearchResponse {
+ return {
+ agents: [],
+ files: [],
+ chats: [
+ {
+ id: "newer",
+ type: "chat_session",
+ title: "Revenue forecast",
+ score: 0.9,
+ updated_at: new Date("2025-01-03T00:00:00Z"),
+ },
+ {
+ id: "middle",
+ type: "chat_session",
+ title: "Forecast follow-up",
+ score: 0.8,
+ updated_at: new Date("2025-01-02T00:00:00Z"),
+ },
+ {
+ id: "older",
+ type: "chat_session",
+ title: "Budget notes",
+ score: 0.6,
+ updated_at: new Date("2025-01-01T00:00:00Z"),
+ },
+ ],
+ ...overrides,
+ };
+ }
+
beforeEach(() => {
useCopilotUIStore.setState({ isSearchOpen: false });
server.use(
getGetV2ListSessionsMockHandler200({
- sessions: [
- {
- id: "older",
- title: "Budget notes",
- is_processing: false,
- created_at: "2025-01-01T00:00:00Z",
- updated_at: "2025-01-01T00:00:00Z",
- },
- {
- id: "newer",
- title: "Revenue forecast",
- is_processing: false,
- created_at: "2025-01-03T00:00:00Z",
- updated_at: "2025-01-03T00:00:00Z",
- },
- {
- id: "middle",
- title: "Forecast follow-up",
- is_processing: false,
- created_at: "2025-01-02T00:00:00Z",
- updated_at: "2025-01-02T00:00:00Z",
- },
- ],
- total: 3,
+ sessions: [],
+ total: 0,
+ }),
+ // Search endpoint is filtered server-side by the ``q`` param. To
+ // keep the test deterministic we narrow the chat bucket here
+ // instead of relying on backend semantics.
+ getGetSearchGlobalSearchMockHandler200(({ request }) => {
+ const url = new URL(request.url);
+ const q = (url.searchParams.get("q") ?? "").trim().toLowerCase();
+ const response = makeSearchResponse();
+ if (!q) {
+ return response;
+ }
+ response.chats = (response.chats ?? []).filter((chat) =>
+ chat.title.toLowerCase().includes(q),
+ );
+ return response;
}),
);
});
@@ -222,10 +250,9 @@ describe("ChatSidebar — search modal", () => {
);
const dialog = await screen.findByRole("dialog");
- const input = screen.getByRole("textbox", { name: /search chats/i });
+ const input = screen.getByRole("textbox", { name: /global search/i });
await vi.waitFor(() => expect(document.activeElement).toBe(input));
- expect(within(dialog).getByText("Recent chats")).toBeDefined();
- expect(within(dialog).getByText("Revenue forecast")).toBeDefined();
+ expect(await within(dialog).findByText("Revenue forecast")).toBeDefined();
});
it("filters results, shows empty copy, and clears the query", async () => {
@@ -236,34 +263,38 @@ describe("ChatSidebar — search modal", () => {
await screen.findByRole("button", { name: /search chats/i }),
);
await user.type(
- screen.getByRole("textbox", { name: /search chats/i }),
+ screen.getByRole("textbox", { name: /global search/i }),
"forecast",
);
- await screen.findByText("Results");
const dialog = screen.getByRole("dialog");
+ // The hook keeps previous results visible via ``placeholderData`` to
+ // avoid a flash-of-empty on every keystroke — wait for the filtered
+ // response to land before asserting that ``Budget notes`` is gone.
+ await vi.waitFor(() => {
+ expect(
+ within(dialog).queryByRole("option", { name: /budget notes/i }),
+ ).toBeNull();
+ });
expect(
within(dialog).getByRole("option", { name: /revenue forecast/i }),
).toBeDefined();
expect(
within(dialog).getByRole("option", { name: /forecast follow-up/i }),
).toBeDefined();
- expect(
- within(dialog).queryByRole("option", { name: /budget notes/i }),
- ).toBeNull();
- await user.clear(screen.getByRole("textbox", { name: /search chats/i }));
+ await user.clear(screen.getByRole("textbox", { name: /global search/i }));
await user.type(
- screen.getByRole("textbox", { name: /search chats/i }),
+ screen.getByRole("textbox", { name: /global search/i }),
"missing",
);
- expect(await screen.findByText("No chats found")).toBeDefined();
+ expect(await screen.findByText("No results found")).toBeDefined();
await user.click(screen.getByRole("button", { name: /clear search/i }));
expect(
(
screen.getByRole("textbox", {
- name: /search chats/i,
+ name: /global search/i,
}) as HTMLInputElement
).value,
).toBe("");
@@ -274,14 +305,17 @@ describe("ChatSidebar — search modal", () => {
renderSidebar();
fireEvent.keyDown(document, { key: "k", metaKey: true });
- expect(await screen.findByRole("dialog")).toBeDefined();
+ const dialog = await screen.findByRole("dialog");
await user.type(
- screen.getByRole("textbox", { name: /search chats/i }),
+ screen.getByRole("textbox", { name: /global search/i }),
"forecast",
);
- await screen.findByText("Results");
- await user.keyboard("{ArrowDown}{Enter}");
+ // Wait for the highlighted result to settle on the top match.
+ await within(dialog).findByRole("option", { name: /revenue forecast/i });
+
+ fireEvent.keyDown(dialog, { key: "ArrowDown" });
+ fireEvent.keyDown(dialog, { key: "Enter" });
await vi.waitFor(() => {
expect(screen.queryByRole("dialog")).toBeNull();
diff --git a/autogpt_platform/frontend/src/app/api/openapi.json b/autogpt_platform/frontend/src/app/api/openapi.json
index 115e5e3bc1..21a6a8fa3e 100644
--- a/autogpt_platform/frontend/src/app/api/openapi.json
+++ b/autogpt_platform/frontend/src/app/api/openapi.json
@@ -8442,6 +8442,66 @@
}
}
},
+ "/api/search/global": {
+ "get": {
+ "tags": ["search", "search"],
+ "summary": "Global search",
+ "operationId": "getSearchGlobal search",
+ "security": [{ "HTTPBearerJWT": [] }],
+ "parameters": [
+ {
+ "name": "q",
+ "in": "query",
+ "required": false,
+ "schema": {
+ "type": "string",
+ "maxLength": 512,
+ "description": "Free-form search query (semantic + lexical). When empty or omitted, the endpoint returns the most-recently-updated items per bucket instead.",
+ "default": "",
+ "title": "Q"
+ },
+ "description": "Free-form search query (semantic + lexical). When empty or omitted, the endpoint returns the most-recently-updated items per bucket instead."
+ },
+ {
+ "name": "per_type_limit",
+ "in": "query",
+ "required": false,
+ "schema": {
+ "type": "integer",
+ "maximum": 10,
+ "minimum": 1,
+ "description": "Maximum items returned per bucket. The response contains *at most* this many agents, files, and chats — fewer when the bucket has no/few matches.",
+ "default": 4,
+ "title": "Per Type Limit"
+ },
+ "description": "Maximum items returned per bucket. The response contains *at most* this many agents, files, and chats — fewer when the bucket has no/few matches."
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/GlobalSearchResponse"
+ }
+ }
+ }
+ },
+ "401": {
+ "$ref": "#/components/responses/HTTP401NotAuthenticatedError"
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": { "$ref": "#/components/schemas/HTTPValidationError" }
+ }
+ }
+ }
+ }
+ }
+ },
"/api/skills": {
"get": {
"tags": ["v1", "skills"],
@@ -11578,7 +11638,9 @@
"BLOCK",
"INTEGRATION",
"DOCUMENTATION",
- "LIBRARY_AGENT"
+ "LIBRARY_AGENT",
+ "WORKSPACE_FILE",
+ "CHAT_SESSION"
],
"title": "ContentType"
},
@@ -12867,6 +12929,28 @@
"title": "FolderUpdateRequest",
"description": "Request model for updating a folder."
},
+ "GlobalSearchResponse": {
+ "properties": {
+ "agents": {
+ "items": { "$ref": "#/components/schemas/SearchResultItem" },
+ "type": "array",
+ "title": "Agents"
+ },
+ "files": {
+ "items": { "$ref": "#/components/schemas/SearchResultItem" },
+ "type": "array",
+ "title": "Files"
+ },
+ "chats": {
+ "items": { "$ref": "#/components/schemas/SearchResultItem" },
+ "type": "array",
+ "title": "Chats"
+ }
+ },
+ "type": "object",
+ "title": "GlobalSearchResponse",
+ "description": "Top-N hits bucketed by item type.\n\nFor a non-empty query this contains hybrid-search results.\nFor an empty query this contains the most-recently-updated items\nper bucket (cached per-user with a short TTL)."
+ },
"Graph": {
"properties": {
"id": { "type": "string", "title": "Id" },
@@ -16759,6 +16843,46 @@
"required": ["items", "search_id", "total_items", "pagination"],
"title": "SearchResponse"
},
+ "SearchResultItem": {
+ "properties": {
+ "id": { "type": "string", "title": "Id" },
+ "type": {
+ "type": "string",
+ "enum": [
+ "library_agent",
+ "store_agent",
+ "workspace_file",
+ "chat_session"
+ ],
+ "title": "Type"
+ },
+ "title": { "type": "string", "title": "Title" },
+ "subtitle": {
+ "anyOf": [{ "type": "string" }, { "type": "null" }],
+ "title": "Subtitle"
+ },
+ "metadata": {
+ "additionalProperties": true,
+ "type": "object",
+ "title": "Metadata"
+ },
+ "score": {
+ "anyOf": [{ "type": "number" }, { "type": "null" }],
+ "title": "Score"
+ },
+ "updated_at": {
+ "anyOf": [
+ { "type": "string", "format": "date-time" },
+ { "type": "null" }
+ ],
+ "title": "Updated At"
+ }
+ },
+ "type": "object",
+ "required": ["id", "type", "title"],
+ "title": "SearchResultItem",
+ "description": "One row in a /search/global bucket.\n\nKept intentionally generic so the frontend can render all three\nbuckets through a single component. Type-specific details live in\n``metadata`` (e.g. mime_type for files, creator for store agents)."
+ },
"SessionDetailResponse": {
"properties": {
"id": { "type": "string", "title": "Id" },
diff --git a/autogpt_platform/frontend/src/components/organisms/SearchCommandModal/SearchCommandModal.stories.tsx b/autogpt_platform/frontend/src/components/organisms/SearchCommandModal/SearchCommandModal.stories.tsx
new file mode 100644
index 0000000000..f38391bfa7
--- /dev/null
+++ b/autogpt_platform/frontend/src/components/organisms/SearchCommandModal/SearchCommandModal.stories.tsx
@@ -0,0 +1,214 @@
+import type { Meta, StoryObj } from "@storybook/nextjs";
+import {
+ BookOpenIcon,
+ ChatCircleIcon,
+ FileIcon,
+ StorefrontIcon,
+} from "@phosphor-icons/react";
+import { useState } from "react";
+import { SearchCommandModal } from "./SearchCommandModal";
+import type { SearchCommandBucket } from "./helpers";
+
+const FIXTURE_BUCKETS: SearchCommandBucket[] = [
+ {
+ key: "chats",
+ label: "Chats",
+ items: [
+ {
+ id: "chat-1",
+ title: "Debugging the YouTube agent failures",
+ icon: ChatCircleIcon,
+ },
+ {
+ id: "chat-2",
+ title: "Planning the launch announcement",
+ icon: ChatCircleIcon,
+ },
+ ],
+ },
+ {
+ key: "agents",
+ label: "Agents",
+ items: [
+ {
+ id: "agent-1",
+ title: "YouTube Video Summarizer",
+ subtitle: "Summarize any YouTube video into bullet points",
+ icon: BookOpenIcon,
+ },
+ {
+ id: "agent-2",
+ title: "Email Triage Bot",
+ subtitle: "by hackergrrl",
+ icon: StorefrontIcon,
+ },
+ {
+ id: "agent-3",
+ title: "PDF Question Answerer",
+ subtitle: "Ask questions about uploaded PDFs",
+ icon: BookOpenIcon,
+ },
+ ],
+ },
+ {
+ key: "files",
+ label: "Files",
+ items: [
+ {
+ id: "file-1",
+ title: "Q4-roadmap.pdf",
+ subtitle: "/projects/planning",
+ icon: FileIcon,
+ },
+ {
+ id: "file-2",
+ title: "competitor-analysis.md",
+ subtitle: "/research",
+ icon: FileIcon,
+ },
+ ],
+ },
+];
+
+interface DemoArgs {
+ buckets: SearchCommandBucket[];
+ isLoading?: boolean;
+ isError?: boolean;
+ initialQuery?: string;
+ placeholder?: string;
+ inputAriaLabel?: string;
+ idleEmptyLabel?: string;
+ searchingEmptyLabel?: string;
+}
+
+function SearchCommandModalDemo({
+ buckets,
+ isLoading,
+ isError,
+ initialQuery = "",
+ placeholder,
+ inputAriaLabel,
+ idleEmptyLabel,
+ searchingEmptyLabel,
+}: DemoArgs) {
+ const [isOpen, setIsOpen] = useState(true);
+ const [query, setQuery] = useState(initialQuery);
+
+ const trimmed = query.trim().toLowerCase();
+ // Lightweight in-memory filter so the story behaves like a real
+ // command palette as you type. Empty query shows the full fixture
+ // (which acts as the "recent items" state in the real app).
+ const filteredBuckets = trimmed
+ ? buckets.map((bucket) => ({
+ ...bucket,
+ items: bucket.items.filter((item) =>
+ item.title.toLowerCase().includes(trimmed),
+ ),
+ }))
+ : buckets;
+
+ return (
+ // Storybook preview shim: a sized box that becomes the containing
+ // block for the modal's ``position: fixed`` descendants. Any
+ // ancestor with a ``transform`` (or ``filter`` / ``contain: paint``)
+ // anchors fixed children to itself instead of the viewport, which
+ // lets the full dialog render inside the docs iframe without
+ // overflow or scrollbars. ``[&_.fixed]:!absolute`` also retargets
+ // the backdrop's positioning so it sits inside this box.
+ <div
+ className="relative h-[720px] w-full overflow-hidden rounded-md bg-zinc-100 [&_.fixed]:!absolute [&_.pt-\[18vh\]]:!pt-12"
+ style={{ transform: "translateZ(0)" }}
+ >
+ {!isOpen && (
+ <button
+ type="button"
+ className="m-4 rounded-md border border-zinc-200 bg-white px-3 py-2 text-sm shadow-sm"
+ onClick={() => setIsOpen(true)}
+ >
+ Open search
+ </button>
+ )}
+ <SearchCommandModal
+ isOpen={isOpen}
+ onClose={() => setIsOpen(false)}
+ query={query}
+ onQueryChange={setQuery}
+ buckets={filteredBuckets}
+ isLoading={isLoading}
+ isError={isError}
+ placeholder={placeholder}
+ inputAriaLabel={inputAriaLabel}
+ idleEmptyLabel={idleEmptyLabel}
+ searchingEmptyLabel={searchingEmptyLabel}
+ onSelectItem={() => {
+ setIsOpen(false);
+ }}
+ />
+ </div>
+ );
+}
+
+const meta: Meta<typeof SearchCommandModalDemo> = {
+ title: "Organisms/SearchCommandModal",
+ component: SearchCommandModalDemo,
+ tags: ["autodocs"],
+ parameters: {
+ layout: "fullscreen",
+ docs: {
+ description: {
+ component:
+ "Generic, controlled command-palette modal with bucketed results, keyboard navigation, and slot-based empty/error/loading states. App-agnostic — see ``GlobalSearchModal`` for the wired-up container that talks to the backend.",
+ },
+ // Safety net so the autodocs iframe is always tall enough for
+ // the sized wrapper that anchors the modal inside the preview.
+ // See the comment on the wrapper ``<div>`` in
+ // ``SearchCommandModalDemo`` for the positioning trick.
+ story: { iframeHeight: 800 },
+ },
+ },
+ args: {
+ buckets: FIXTURE_BUCKETS,
+ placeholder: "Search agents, files, chats…",
+ inputAriaLabel: "Global search",
+ },
+};
+
+export default meta;
+type Story = StoryObj<typeof meta>;
+
+export const WithResults: Story = {};
+
+export const Loading: Story = {
+ args: {
+ buckets: [],
+ isLoading: true,
+ },
+};
+
+export const Empty: Story = {
+ args: {
+ buckets: [],
+ idleEmptyLabel: "No recent items",
+ },
+};
+
+export const NoMatches: Story = {
+ args: {
+ buckets: [],
+ initialQuery: "zzz",
+ searchingEmptyLabel: "No results found",
+ },
+};
+
+export const SingleBucket: Story = {
+ args: {
+ buckets: [FIXTURE_BUCKETS[0]],
+ },
+};
+
+export const ErrorState: Story = {
+ args: {
+ buckets: [],
+ isError: true,
+ },
+};
diff --git a/autogpt_platform/frontend/src/components/organisms/SearchCommandModal/SearchCommandModal.tsx b/autogpt_platform/frontend/src/components/organisms/SearchCommandModal/SearchCommandModal.tsx
new file mode 100644
index 0000000000..3eb8cef5bc
--- /dev/null
+++ b/autogpt_platform/frontend/src/components/organisms/SearchCommandModal/SearchCommandModal.tsx
@@ -0,0 +1,229 @@
+import { LoadingSpinner } from "@/components/atoms/LoadingSpinner/LoadingSpinner";
+import { Button } from "@/components/ui/button";
+import { Input } from "@/components/ui/input";
+import { Separator } from "@/components/ui/separator";
+import { cn } from "@/lib/utils";
+import { MagnifyingGlassIcon, XIcon } from "@phosphor-icons/react";
+import * as RXDialog from "@radix-ui/react-dialog";
+import { useId, useRef, type KeyboardEvent, type ReactNode } from "react";
+import {
+ flattenBuckets,
+ getTotalCount,
+ type SearchCommandBucket,
+ type SearchCommandItem,
+} from "./helpers";
+import { SearchCommandResults } from "./SearchCommandResults";
+import { SearchCommandSkeleton } from "./SearchCommandSkeleton";
+import { useKeyboardNav } from "./useKeyboardNav";
+
+interface Props {
+ isOpen: boolean;
+ onClose: () => void;
+ query: string;
+ onQueryChange: (next: string) => void;
+ /** Controlled bucketed results. Bucket order is preserved as-is. */
+ buckets: SearchCommandBucket[];
+ onSelectItem: (item: SearchCommandItem, bucketKey: string) => void;
+ /** Shown when results are empty and ``query`` is empty. */
+ idleEmptyLabel?: ReactNode;
+ /** Shown when results are empty and ``query`` is non-empty. */
+ searchingEmptyLabel?: ReactNode;
+ /** Shown when ``isError`` is true. Replaces the result list entirely. */
+ errorLabel?: ReactNode;
+ isLoading?: boolean;
+ isError?: boolean;
+ placeholder?: string;
+ inputAriaLabel?: string;
+}
+
+export function SearchCommandModal({
+ isOpen,
+ onClose,
+ query,
+ onQueryChange,
+ buckets,
+ onSelectItem,
+ idleEmptyLabel = "No items",
+ searchingEmptyLabel = "No results found",
+ errorLabel = "Something went wrong. Try again.",
+ isLoading = false,
+ isError = false,
+ placeholder = "Search…",
+ inputAriaLabel = "Search",
+}: Props) {
+ const inputRef = useRef<HTMLInputElement>(null);
+ const reactId = useId();
+ // Strip the colons React uses in ``useId`` so the result survives a
+ // selector / id-based lookup in tests and CSS.
+ const idPrefix = `search-cmd-${reactId.replace(/:/g, "")}`;
+
+ const flatResults = flattenBuckets(buckets);
+ const totalCount = getTotalCount(buckets);
+ const trimmedQuery = query.trim();
+ const isSearching = trimmedQuery.length > 0;
+
+ const {
+ highlightedIndex,
+ setHighlightedIndex,
+ highlightedRef,
+ moveHighlight,
+ } = useKeyboardNav(totalCount, trimmedQuery);
+
+ const highlightedFlat = flatResults[highlightedIndex];
+
+ function selectHighlightedItem() {
+ if (!highlightedFlat) return;
+ onSelectItem(highlightedFlat.item, highlightedFlat.bucketKey);
+ }
+
+ function handleKeyDown(event: KeyboardEvent<HTMLDivElement>) {
+ // Escape closure is handled by Radix's ``onOpenChange``; arrow
+ // keys and Enter stay here so the input keeps focus while the user
+ // navigates the result list.
+ if (event.key === "ArrowDown") {
+ event.preventDefault();
+ moveHighlight(1);
+ } else if (event.key === "ArrowUp") {
+ event.preventDefault();
+ moveHighlight(-1);
+ } else if (event.key === "Enter") {
+ event.preventDefault();
+ selectHighlightedItem();
+ }
+ }
+
+ function handleOpenAutoFocus(event: Event) {
+ // Radix's default is to focus the first focusable element. For
+ // search-as-you-type we want the input regardless of where it
+ // sits in the tab order.
+ event.preventDefault();
+ inputRef.current?.focus();
+ }
+
+ const showResults = !isError && totalCount > 0;
+ const showLoading = !isError && totalCount === 0 && isLoading;
+ const showEmptyState = !isError && totalCount === 0 && !isLoading;
+
+ return (
+ <RXDialog.Root
+ open={isOpen}
+ onOpenChange={(open) => {
+ if (!open) onClose();
+ }}
+ >
+ <RXDialog.Portal>
+ <RXDialog.Overlay className="fixed inset-0 z-[80] bg-black/20 backdrop-blur-sm" />
+ <RXDialog.Content
+ aria-labelledby={`${idPrefix}-title`}
+ onOpenAutoFocus={handleOpenAutoFocus}
+ className="fixed left-1/2 top-[18vh] z-[80] w-[calc(100%-2rem)] max-w-xl -translate-x-1/2 overflow-hidden rounded-3xl bg-white shadow-2xl ring-1 ring-zinc-200 focus:outline-none"
+ onKeyDown={handleKeyDown}
+ >
+ <RXDialog.Title id={`${idPrefix}-title`} className="sr-only">
+ Search
+ </RXDialog.Title>
+ <div className="flex items-center gap-3 bg-zinc-50 p-3">
+ <MagnifyingGlassIcon className="h-5 w-5 shrink-0 text-zinc-800" />
+ <Input
+ ref={inputRef}
+ value={query}
+ onChange={(event) => onQueryChange(event.target.value)}
+ placeholder={placeholder}
+ aria-label={inputAriaLabel}
+ aria-controls={`${idPrefix}-results`}
+ aria-activedescendant={
+ highlightedFlat
+ ? `${idPrefix}-${highlightedFlat.item.id}`
+ : undefined
+ }
+ autoComplete="off"
+ className="h-9 border-0 bg-transparent px-0 text-base text-zinc-950 shadow-none placeholder:text-zinc-700 focus-visible:ring-0"
+ />
+ {isLoading && isSearching ? (
+ <LoadingSpinner
+ size="small"
+ aria-label="Searching"
+ className="shrink-0 text-zinc-500"
+ />
+ ) : null}
+ {query ? (
+ <Button
+ type="button"
+ variant="ghost"
+ size="icon-sm"
+ aria-label="Clear search"
+ onClick={() => onQueryChange("")}
+ className="shrink-0"
+ >
+ <XIcon className="h-4 w-4" />
+ </Button>
+ ) : null}
+ </div>
+ <Separator />
+ <div className="py-2">
+ <div
+ id={`${idPrefix}-results`}
+ className={cn(
+ "max-h-[26rem] overflow-y-auto scrollbar-thin scrollbar-track-transparent scrollbar-thumb-zinc-200",
+ totalCount === 0 && "max-h-none",
+ )}
+ >
+ {isError ? (
+ <div className="px-3 py-8 text-center text-sm text-red-500">
+ {errorLabel}
+ </div>
+ ) : showResults ? (
+ <SearchCommandResults
+ buckets={buckets}
+ idPrefix={idPrefix}
+ query={trimmedQuery}
+ highlightedIndex={highlightedIndex}
+ highlightedRef={highlightedRef}
+ onHighlight={setHighlightedIndex}
+ onSelect={onSelectItem}
+ />
+ ) : showLoading ? (
+ // Skeleton over text: the input-side spinner already
+ // signals "searching", so the body should mirror the
+ // shape of what's about to appear (staggered rows) rather
+ // than block the eye with a centred string. Emil
+ // Kowalski's strategy-feedback-immediate rule.
+ <SearchCommandSkeleton />
+ ) : showEmptyState ? (
+ <div className="px-3 py-8 text-center text-sm text-zinc-500">
+ {isSearching ? searchingEmptyLabel : idleEmptyLabel}
+ </div>
+ ) : null}
+ </div>
+ </div>
+ <Separator />
+ <div className="flex items-center justify-between gap-4 bg-zinc-50 px-4 py-4 text-xs text-zinc-700">
+ <div className="flex items-center gap-4">
+ <div className="flex items-center gap-1.5">
+ <kbd className="inline-flex h-5 w-5 items-center justify-center rounded-md border border-zinc-200 bg-white font-sans text-[11px] text-zinc-800 shadow-[inset_0_-1px_0_rgba(15,15,20,0.04),0_1px_1px_rgba(15,15,20,0.04)]">
+ ↑
+ </kbd>
+ <kbd className="inline-flex h-5 w-5 items-center justify-center rounded-md border border-zinc-200 bg-white font-sans text-[11px] text-zinc-800 shadow-[inset_0_-1px_0_rgba(15,15,20,0.04),0_1px_1px_rgba(15,15,20,0.04)]">
+ ↓
+ </kbd>
+ <span>Navigate</span>
+ </div>
+ <div className="flex items-center gap-1.5">
+ <kbd className="inline-flex h-5 min-w-5 items-center justify-center rounded-md border border-zinc-200 bg-white px-1 font-sans text-[11px] text-zinc-800 shadow-[inset_0_-1px_0_rgba(15,15,20,0.04),0_1px_1px_rgba(15,15,20,0.04)]">
+ ↵
+ </kbd>
+ <span>Select</span>
+ </div>
+ </div>
+ <div className="flex items-center gap-1.5">
+ <kbd className="inline-flex h-5 items-center justify-center rounded-md border border-zinc-200 bg-white px-1.5 font-sans text-[11px] text-zinc-800 shadow-[inset_0_-1px_0_rgba(15,15,20,0.04),0_1px_1px_rgba(15,15,20,0.04)]">
+ esc
+ </kbd>
+ <span>Close</span>
+ </div>
+ </div>
+ </RXDialog.Content>
+ </RXDialog.Portal>
+ </RXDialog.Root>
+ );
+}
diff --git a/autogpt_platform/frontend/src/components/organisms/SearchCommandModal/SearchCommandResultItem.tsx b/autogpt_platform/frontend/src/components/organisms/SearchCommandModal/SearchCommandResultItem.tsx
new file mode 100644
index 0000000000..b73a6bce07
--- /dev/null
+++ b/autogpt_platform/frontend/src/components/organisms/SearchCommandModal/SearchCommandResultItem.tsx
@@ -0,0 +1,107 @@
+import { Button } from "@/components/ui/button";
+import { cn } from "@/lib/utils";
+import type { MutableRefObject } from "react";
+import { highlightMatch, type SearchCommandItem } from "./helpers";
+
+interface Props {
+ item: SearchCommandItem;
+ /** Unique DOM id prefix so multiple modals on a page never collide. */
+ idPrefix: string;
+ query: string;
+ isHighlighted: boolean;
+ highlightedRef?: MutableRefObject<HTMLButtonElement | null>;
+ onHighlight: () => void;
+ onSelect: () => void;
+ /** Zero-based position used to stagger the enter animation. */
+ enterIndex?: number;
+}
+
+// Per Emil Kowalski's "faster is better" rule: keep the stagger tight
+// so the list never feels like it's *waiting* to render. 35ms × 12
+// items ≈ 420ms total before the last item finishes its own 180ms
+// animation, which stays under the 300ms perceived-snappy budget for
+// the typical first-visible row.
+const ENTER_STAGGER_MS = 35;
+
+export function SearchCommandResultItem({
+ item,
+ idPrefix,
+ query,
+ isHighlighted,
+ highlightedRef,
+ onHighlight,
+ onSelect,
+ enterIndex = 0,
+}: Props) {
+ const Icon = item.icon;
+
+ return (
+ <Button
+ ref={isHighlighted ? highlightedRef : undefined}
+ id={`${idPrefix}-${item.id}`}
+ type="button"
+ variant="ghost"
+ role="option"
+ aria-selected={isHighlighted}
+ onMouseEnter={onHighlight}
+ onClick={onSelect}
+ style={{ animationDelay: `${enterIndex * ENTER_STAGGER_MS}ms` }}
+ className={cn(
+ "relative h-auto w-full justify-start rounded-md px-3 py-2 text-left transition-colors duration-150",
+ "motion-safe:animate-search-item-in",
+ isHighlighted ? "bg-zinc-100 hover:bg-zinc-100" : "hover:bg-zinc-50",
+ )}
+ >
+ <span
+ aria-hidden="true"
+ className={cn(
+ "absolute inset-y-0 left-0 my-auto h-5 w-[3px] rounded-full bg-zinc-900 transition-opacity duration-150",
+ isHighlighted ? "opacity-100" : "opacity-0",
+ )}
+ />
+ <div className="relative z-10 flex min-w-0 flex-1 items-center gap-2.5">
+ {Icon ? (
+ <span aria-hidden className="contents">
+ <Icon
+ className={cn(
+ "h-4 w-4 shrink-0 transition-colors duration-150",
+ isHighlighted ? "text-zinc-900" : "text-zinc-500",
+ )}
+ />
+ </span>
+ ) : null}
+ <div className="min-w-0 flex-1">
+ <div
+ className={cn(
+ "truncate text-sm font-normal transition-colors duration-150",
+ isHighlighted ? "text-zinc-950" : "text-zinc-800",
+ )}
+ >
+ {highlightMatch(item.title, query).map((part, partIndex) => (
+ <span
+ key={`${part.text}-${partIndex}`}
+ className={part.isMatch ? "font-semibold" : undefined}
+ >
+ {part.text}
+ </span>
+ ))}
+ </div>
+ {item.subtitle ? (
+ <div className="mt-0.5 truncate text-xs text-zinc-500">
+ {item.subtitle}
+ </div>
+ ) : null}
+ </div>
+ <span
+ aria-hidden="true"
+ className={cn(
+ "inline-flex h-5 min-w-5 shrink-0 items-center justify-center rounded-md border border-zinc-200 bg-white px-1 font-sans text-[11px] text-zinc-600 shadow-[inset_0_-1px_0_rgba(15,15,20,0.04),0_1px_1px_rgba(15,15,20,0.04)] transition-opacity duration-150",
+ isHighlighted ? "opacity-100" : "opacity-0",
+ )}
+ >
+ ↵
+ </span>
+ </div>
+ </Button>
+ );
+}
diff --git a/autogpt_platform/frontend/src/components/organisms/SearchCommandModal/SearchCommandResults.tsx b/autogpt_platform/frontend/src/components/organisms/SearchCommandModal/SearchCommandResults.tsx
new file mode 100644
index 0000000000..fcabdaa6da
--- /dev/null
+++ b/autogpt_platform/frontend/src/components/organisms/SearchCommandModal/SearchCommandResults.tsx
@@ -0,0 +1,78 @@
+import type { MutableRefObject } from "react";
+import {
+ flattenBuckets,
+ type FlatBucketItem,
+ type SearchCommandBucket,
+ type SearchCommandItem,
+} from "./helpers";
+import { SearchCommandResultItem } from "./SearchCommandResultItem";
+
+interface Props {
+ buckets: SearchCommandBucket[];
+ idPrefix: string;
+ query: string;
+ highlightedIndex: number;
+ highlightedRef: MutableRefObject<HTMLButtonElement | null>;
+ onHighlight: (index: number) => void;
+ onSelect: (item: SearchCommandItem, bucketKey: string) => void;
+}
+
+export function SearchCommandResults({
+ buckets,
+ idPrefix,
+ query,
+ highlightedIndex,
+ highlightedRef,
+ onHighlight,
+ onSelect,
+}: Props) {
+ const flat = flattenBuckets(buckets);
+ // Group the flat list back by bucket so the absolute ``index`` survives
+ // for keyboard nav while we still render each section under its
+ // header.
+ const grouped = new Map<string, FlatBucketItem[]>();
+ for (const bucket of buckets) grouped.set(bucket.key, []);
+ for (const entry of flat) grouped.get(entry.bucketKey)?.push(entry);
+
+ return (
+ <div
+ role="listbox"
+ aria-label="Search results"
+ className="flex flex-col gap-1"
+ >
+ {buckets.map((bucket) => {
+ const entries = grouped.get(bucket.key) ?? [];
+ if (entries.length === 0) return null;
+ return (
+ <div key={bucket.key} className="px-2">
+ <div className="px-3 pb-1 pt-2 text-[11px] font-semibold uppercase tracking-wide text-zinc-500">
+ {bucket.label}
+ </div>
+ <div className="flex flex-col">
+ {entries.map((entry) => (
+ <SearchCommandResultItem
+ key={entry.item.id}
+ item={entry.item}
+ idPrefix={idPrefix}
+ query={query}
+ isHighlighted={entry.index === highlightedIndex}
+ highlightedRef={
+ entry.index === highlightedIndex
+ ? highlightedRef
+ : undefined
+ }
+ onHighlight={() => onHighlight(entry.index)}
+ onSelect={() => onSelect(entry.item, entry.bucketKey)}
+ // Absolute index across all buckets so the stagger
+ // cascades naturally section-by-section instead of
+ // restarting at each header.
+ enterIndex={entry.index}
+ />
+ ))}
+ </div>
+ </div>
+ );
+ })}
+ </div>
+ );
+}
diff --git a/autogpt_platform/frontend/src/components/organisms/SearchCommandModal/SearchCommandSkeleton.tsx b/autogpt_platform/frontend/src/components/organisms/SearchCommandModal/SearchCommandSkeleton.tsx
new file mode 100644
index 0000000000..6281cbe1b9
--- /dev/null
+++ b/autogpt_platform/frontend/src/components/organisms/SearchCommandModal/SearchCommandSkeleton.tsx
@@ -0,0 +1,39 @@
+import { Skeleton } from "@/components/atoms/Skeleton/Skeleton";
+
+interface Props {
+ /** Number of skeleton rows to render. Defaults to 4 — roughly the
+ * average bucket size in the global search modal. */
+ rows?: number;
+}
+
+const STAGGER_MS = 35;
+
+export function SearchCommandSkeleton({ rows = 4 }: Props) {
+ return (
+ <div
+ role="presentation"
+ aria-hidden="true"
+ data-testid="search-command-skeleton"
+ className="flex flex-col gap-1 px-2"
+ >
+ <div className="px-3 pb-1 pt-2">
+ <Skeleton className="h-3 w-16" />
+ </div>
+ <div className="flex flex-col">
+ {Array.from({ length: rows }, (_, idx) => (
+ <div
+ key={idx}
+ className="flex items-center gap-2.5 rounded-md px-3 py-2 motion-safe:animate-search-item-in"
+ style={{ animationDelay: `${idx * STAGGER_MS}ms` }}
+ >
+ <Skeleton className="h-4 w-4 rounded" />
+ <div className="flex min-w-0 flex-1 flex-col gap-1">
+ <Skeleton className="h-3.5 w-[55%]" />
+ <Skeleton className="h-3 w-[35%]" />
+ </div>
+ </div>
+ ))}
+ </div>
+ </div>
+ );
+}
diff --git a/autogpt_platform/frontend/src/components/organisms/SearchCommandModal/__tests__/SearchCommandModal.test.tsx b/autogpt_platform/frontend/src/components/organisms/SearchCommandModal/__tests__/SearchCommandModal.test.tsx
new file mode 100644
index 0000000000..0326ec1a62
--- /dev/null
+++ b/autogpt_platform/frontend/src/components/organisms/SearchCommandModal/__tests__/SearchCommandModal.test.tsx
@@ -0,0 +1,249 @@
+import { BookOpenIcon, ChatCircleIcon } from "@phosphor-icons/react";
+import { fireEvent, render, screen } from "@/tests/integrations/test-utils";
+import userEvent from "@testing-library/user-event";
+import { useState } from "react";
+import { describe, expect, it, vi } from "vitest";
+import type { SearchCommandBucket } from "../helpers";
+import { SearchCommandModal } from "../SearchCommandModal";
+
+const BASE_BUCKETS: SearchCommandBucket[] = [
+ {
+ key: "agents",
+ label: "Agents",
+ items: [
+ {
+ id: "a1",
+ title: "Alpha agent",
+ subtitle: "library row",
+ icon: BookOpenIcon,
+ },
+ {
+ id: "a2",
+ title: "Bravo store",
+ subtitle: "by acme",
+ icon: BookOpenIcon,
+ },
+ ],
+ },
+ {
+ key: "files",
+ label: "Files",
+ items: [
+ {
+ id: "f1",
+ title: "notes.md",
+ subtitle: "/docs",
+ icon: BookOpenIcon,
+ },
+ ],
+ },
+ {
+ key: "chats",
+ label: "Chats",
+ items: [
+ {
+ id: "c1",
+ title: "Charlie chat",
+ icon: ChatCircleIcon,
+ },
+ ],
+ },
+];
+
+interface HarnessProps {
+ isOpen?: boolean;
+ onClose?: () => void;
+ buckets?: SearchCommandBucket[];
+ onSelectItem?: Parameters<typeof SearchCommandModal>[0]["onSelectItem"];
+ isLoading?: boolean;
+ isError?: boolean;
+ initialQuery?: string;
+}
+
+function Harness({
+ isOpen = true,
+ onClose = () => {},
+ buckets = BASE_BUCKETS,
+ onSelectItem = () => {},
+ isLoading,
+ isError,
+ initialQuery = "",
+}: HarnessProps) {
+ const [query, setQuery] = useState(initialQuery);
+ return (
+ <SearchCommandModal
+ isOpen={isOpen}
+ onClose={onClose}
+ query={query}
+ onQueryChange={setQuery}
+ buckets={buckets}
+ isLoading={isLoading}
+ isError={isError}
+ onSelectItem={onSelectItem}
+ inputAriaLabel="Test search"
+ placeholder="Search…"
+ idleEmptyLabel="No items"
+ searchingEmptyLabel="No results"
+ />
+ );
+}
+
+describe("SearchCommandModal", () => {
+ it("returns null when closed", () => {
+ render(<Harness isOpen={false} />);
+ expect(screen.queryByRole("dialog")).toBeNull();
+ });
+
+ it("auto-focuses the search input when opened", async () => {
+ render(<Harness />);
+ const input = await screen.findByRole("textbox", { name: /test search/i });
+ await vi.waitFor(() => expect(document.activeElement).toBe(input));
+ });
+
+ it("renders bucketed results with section headers", () => {
+ render(<Harness />);
+ expect(screen.getByText("Alpha agent")).toBeDefined();
+ expect(screen.getByText("Bravo store")).toBeDefined();
+ expect(screen.getByText("notes.md")).toBeDefined();
+ expect(screen.getByText("Charlie chat")).toBeDefined();
+ expect(screen.getByText("Agents")).toBeDefined();
+ expect(screen.getByText("Files")).toBeDefined();
+ expect(screen.getByText("Chats")).toBeDefined();
+ });
+
+ it("hides empty buckets", () => {
+ render(
+ <Harness
+ buckets={[
+ BASE_BUCKETS[0],
+ { ...BASE_BUCKETS[1], items: [] },
+ { ...BASE_BUCKETS[2], items: [] },
+ ]}
+ />,
+ );
+ expect(screen.queryByText("Files")).toBeNull();
+ expect(screen.queryByText("Chats")).toBeNull();
+ });
+
+ it("shows the idle empty label with no items and no query", () => {
+ render(<Harness buckets={[]} />);
+ expect(screen.getByText("No items")).toBeDefined();
+ });
+
+ it("shows the searching empty label when a query is active and no items match", () => {
+ render(<Harness buckets={[]} initialQuery="zzz" />);
+ expect(screen.getByText("No results")).toBeDefined();
+ });
+
+ it("renders the loading skeleton", () => {
+ render(<Harness buckets={[]} isLoading />);
+ expect(screen.getByTestId("search-command-skeleton")).toBeDefined();
+ });
+
+ it("renders the error label", () => {
+ render(<Harness buckets={[]} isError />);
+ expect(screen.getByText("Something went wrong. Try again.")).toBeDefined();
+ });
+
+ it("invokes onSelectItem when a result is clicked", async () => {
+ const user = userEvent.setup();
+ const onSelectItem = vi.fn();
+ render(<Harness onSelectItem={onSelectItem} />);
+
+ await user.click(screen.getByRole("option", { name: /alpha agent/i }));
+ expect(onSelectItem).toHaveBeenCalledTimes(1);
+ expect(onSelectItem.mock.calls[0][0]).toMatchObject({ id: "a1" });
+ expect(onSelectItem.mock.calls[0][1]).toBe("agents");
+ });
+
+ it("navigates with arrow keys and selects with Enter across buckets", async () => {
+ const onSelectItem = vi.fn();
+ render(<Harness onSelectItem={onSelectItem} />);
+ const dialog = screen.getByRole("dialog");
+
+ expect(
+ screen
+ .getByRole("option", { name: /alpha agent/i })
+ .getAttribute("aria-selected"),
+ ).toBe("true");
+
+ // Two arrow-downs walk past both Agents items into Files.
+ fireEvent.keyDown(dialog, { key: "ArrowDown" });
+ fireEvent.keyDown(dialog, { key: "ArrowDown" });
+ expect(
+ screen
+ .getByRole("option", { name: /notes\.md/i })
+ .getAttribute("aria-selected"),
+ ).toBe("true");
+
+ fireEvent.keyDown(dialog, { key: "Enter" });
+ expect(onSelectItem).toHaveBeenCalledTimes(1);
+ expect(onSelectItem.mock.calls[0][0]).toMatchObject({ id: "f1" });
+ expect(onSelectItem.mock.calls[0][1]).toBe("files");
+ });
+
+ it("clamps arrow navigation at the bounds", () => {
+ render(<Harness />);
+ const dialog = screen.getByRole("dialog");
+
+ // ArrowUp at the top stays on the first item.
+ fireEvent.keyDown(dialog, { key: "ArrowUp" });
+ expect(
+ screen
+ .getByRole("option", { name: /alpha agent/i })
+ .getAttribute("aria-selected"),
+ ).toBe("true");
+
+ // Press ArrowDown more times than there are items.
+ for (let i = 0; i < 10; i += 1) {
+ fireEvent.keyDown(dialog, { key: "ArrowDown" });
+ }
+ expect(
+ screen
+ .getByRole("option", { name: /charlie chat/i })
+ .getAttribute("aria-selected"),
+ ).toBe("true");
+ });
+
+ it("closes when Escape is pressed", () => {
+ const onClose = vi.fn();
+ render(<Harness onClose={onClose} />);
+ // Radix listens at document level for Escape inside its Dialog;
+ // a keyDown on the document mirrors what a user keystroke would.
+ fireEvent.keyDown(document, { key: "Escape" });
+ expect(onClose).toHaveBeenCalledTimes(1);
+ });
+
+ it("does not close when the dialog body itself is clicked", () => {
+ const onClose = vi.fn();
+ render(<Harness onClose={onClose} />);
+ fireEvent.mouseDown(screen.getByRole("dialog"));
+ expect(onClose).not.toHaveBeenCalled();
+ });
+
+ it("clears the query when the clear button is pressed", async () => {
+ const user = userEvent.setup();
+ render(<Harness initialQuery="alpha" />);
+
+ const input = screen.getByRole("textbox", {
+ name: /test search/i,
+ }) as HTMLInputElement;
+ expect(input.value).toBe("alpha");
+
+ await user.click(screen.getByRole("button", { name: /clear search/i }));
+ expect(input.value).toBe("");
+ });
+
+ it("highlights the matching query substring inside the title", () => {
+ render(<Harness initialQuery="alpha" />);
+
+ const bolded = screen.getByText(
+ (_content, element) =>
+ !!element &&
+ element.tagName === "SPAN" &&
+ element.className.includes("font-semibold") &&
+ element.textContent?.toLocaleLowerCase() === "alpha",
+ );
+ expect(bolded).toBeDefined();
+ });
+});
diff --git a/autogpt_platform/frontend/src/components/organisms/SearchCommandModal/helpers.ts b/autogpt_platform/frontend/src/components/organisms/SearchCommandModal/helpers.ts
new file mode 100644
index 0000000000..fa0cbcb485
--- /dev/null
+++ b/autogpt_platform/frontend/src/components/organisms/SearchCommandModal/helpers.ts
@@ -0,0 +1,70 @@
+import type { ComponentType } from "react";
+
+export interface SearchCommandItem {
+ id: string;
+ title: string;
+ subtitle?: string | null;
+ // Icon rendered at the start of the row. Accepts any component
+ // accepting ``className`` — Phosphor / lucide / heroicons all match.
+ // Library-specific ``aria-hidden`` / ``weight`` props vary in their
+ // exact signatures, so this stays loose with ``ComponentType<{
+ // className?: string }>``. Callers pass the icon component directly.
+ icon?: ComponentType<{ className?: string }>;
+}
+
+export interface SearchCommandBucket {
+ // Stable id for keys + ``onSelectItem`` callbacks.
+ key: string;
+ label: string;
+ items: SearchCommandItem[];
+}
+
+export interface FlatBucketItem {
+ item: SearchCommandItem;
+ bucketKey: string;
+ /** Absolute index across all buckets (0-based). */
+ index: number;
+}
+
+export interface HighlightPart {
+ text: string;
+ isMatch: boolean;
+}
+
+export function flattenBuckets(
+ buckets: SearchCommandBucket[],
+): FlatBucketItem[] {
+ const flat: FlatBucketItem[] = [];
+ for (const bucket of buckets) {
+ for (const item of bucket.items) {
+ flat.push({ item, bucketKey: bucket.key, index: flat.length });
+ }
+ }
+ return flat;
+}
+
+export function getTotalCount(buckets: SearchCommandBucket[]): number {
+ let total = 0;
+ for (const bucket of buckets) total += bucket.items.length;
+ return total;
+}
+
+/**
+ * Splits ``title`` so the substring matching ``query`` (case-insensitive)
+ * is marked ``isMatch: true``. Returns the title unchanged when the query
+ * is empty or no match is found.
+ */
+export function highlightMatch(title: string, query: string): HighlightPart[] {
+ const normalizedQuery = query.trim().toLocaleLowerCase();
+ if (!normalizedQuery) return [{ text: title, isMatch: false }];
+
+ const matchIndex = title.toLocaleLowerCase().indexOf(normalizedQuery);
+ if (matchIndex === -1) return [{ text: title, isMatch: false }];
+
+ const matchEnd = matchIndex + normalizedQuery.length;
+ return [
+ { text: title.slice(0, matchIndex), isMatch: false },
+ { text: title.slice(matchIndex, matchEnd), isMatch: true },
+ { text: title.slice(matchEnd), isMatch: false },
+ ].filter((part) => part.text.length > 0);
+}
diff --git a/autogpt_platform/frontend/src/components/organisms/SearchCommandModal/useKeyboardNav.ts b/autogpt_platform/frontend/src/components/organisms/SearchCommandModal/useKeyboardNav.ts
new file mode 100644
index 0000000000..5f6fa789fd
--- /dev/null
+++ b/autogpt_platform/frontend/src/components/organisms/SearchCommandModal/useKeyboardNav.ts
@@ -0,0 +1,47 @@
+import { useEffect, useRef, useState } from "react";
+
+/**
+ * Pure state machine for the highlighted-row cursor in a vertical
+ * keyboard-driven list. Decoupled from any specific data shape so it
+ * can be reused by any list-style organism.
+ *
+ * - Resets to ``0`` whenever ``resetKey`` changes (e.g. when the search
+ * query changes and the result set is replaced).
+ * - Clamps the cursor inside ``[0, totalCount)`` whenever the list
+ * shrinks.
+ * - Scrolls the highlighted row into view via ``highlightedRef``.
+ */
+export function useKeyboardNav(totalCount: number, resetKey: unknown) {
+ const [highlightedIndex, setHighlightedIndex] = useState(0);
+ const highlightedRef = useRef<HTMLButtonElement | null>(null);
+
+ useEffect(() => {
+ setHighlightedIndex(0);
+ }, [resetKey, totalCount]);
+
+ useEffect(() => {
+ if (highlightedIndex >= totalCount) {
+ setHighlightedIndex(Math.max(0, totalCount - 1));
+ }
+ }, [highlightedIndex, totalCount]);
+
+ useEffect(() => {
+ highlightedRef.current?.scrollIntoView({ block: "nearest" });
+ }, [highlightedIndex]);
+
+ function moveHighlight(direction: 1 | -1) {
+ if (totalCount === 0) return;
+ setHighlightedIndex((current) => {
+ const next = current + direction;
+ if (next < 0 || next >= totalCount) return current;
+ return next;
+ });
+ }
+
+ return {
+ highlightedIndex,
+ setHighlightedIndex,
+ highlightedRef,
+ moveHighlight,
+ };
+}
diff --git a/autogpt_platform/frontend/src/mocks/mock-handlers.ts b/autogpt_platform/frontend/src/mocks/mock-handlers.ts
index 4b173c5ff6..49c2392bdb 100644
--- a/autogpt_platform/frontend/src/mocks/mock-handlers.ts
+++ b/autogpt_platform/frontend/src/mocks/mock-handlers.ts
@@ -19,6 +19,7 @@ import { getOnboardingMock } from "@/app/api/__generated__/endpoints/onboarding/
import { getOttoMock } from "@/app/api/__generated__/endpoints/otto/otto.msw";
import { getPresetsMock } from "@/app/api/__generated__/endpoints/presets/presets.msw";
import { getSchedulesMock } from "@/app/api/__generated__/endpoints/schedules/schedules.msw";
+import { getSearchMock } from "@/app/api/__generated__/endpoints/search/search.msw";
import { getStoreMock } from "@/app/api/__generated__/endpoints/store/store.msw";
// Pass hard-coded data to individual handler functions to override faker-generated data.
@@ -44,5 +45,6 @@ export const mockHandlers = [
...getOttoMock(),
...getPresetsMock(),
...getSchedulesMock(),
+ ...getSearchMock(),
...getStoreMock(),
];
diff --git a/autogpt_platform/frontend/tailwind.config.ts b/autogpt_platform/frontend/tailwind.config.ts
index e12b1819c3..76cd15ff0a 100644
--- a/autogpt_platform/frontend/tailwind.config.ts
+++ b/autogpt_platform/frontend/tailwind.config.ts
@@ -159,6 +159,19 @@ const config = {
opacity: "1",
},
},
+ // Emil Kowalski enter pattern: translate from below + fade,
+ // ease-out, under 300ms. Driven by inline ``animation-delay``
+ // for per-item staggering.
+ "search-item-in": {
+ "0%": {
+ opacity: "0",
+ transform: "translateY(4px)",
+ },
+ "100%": {
+ opacity: "1",
+ transform: "translateY(0)",
+ },
+ },
shimmer: {
"0%": {
backgroundPosition: "200% 0",
@@ -191,6 +204,12 @@ const config = {
"accordion-down": "accordion-down 0.2s ease-out",
"accordion-up": "accordion-up 0.2s ease-out",
"fade-in": "fade-in 0.2s ease-out",
+ // 180ms / ease-out keeps the entry inside Emil Kowalski's
+ // "faster is better" UI budget. Backwards mode so the item
+ // stays invisible until its staggered delay elapses, instead
+ // of flashing at frame zero.
+ "search-item-in":
+ "search-item-in 180ms cubic-bezier(0.16, 1, 0.3, 1) both",
shimmer: "shimmer 4s ease-in-out infinite",
loader: "loader 1s infinite",
shake: "shake 0.5s ease-in-out",