BrainAPI is a knowledge graph–powered AI memory layer that transforms unstructured data into structured knowledge, enabling intelligent search, recommendations, and contextual memory for AI agents and applications.
Python ai-agentsai-memoryknowledge-graphknowledge-managementknowledge-representationreccomendation-system
Add hybrid search and catalog graph retrieval
Christian73dd2d28cb5b
diff --git a/.env.example b/.env.exampleindex 83262bf..f79f4b9 100644--- a/.env.example+++ b/.env.example@@ -167,6 +167,26 @@ INGEST_ARCHITECT_DENSE_ENTITY_THRESHOLD="12"INGEST_ARCHITECT_DENSE_MAX_CHARS="1200"INGEST_ARCHITECT_PRIOR_CONTEXT="auto"INGEST_ARCHITECT_SCRATCHPAD_TOKEN_CAP="500"+# Search+# SEARCH_ENABLED => true registers GET|POST /retrieve/search, writes tsvector + halfvec HNSW+# (when dim > 2000). Default false keeps memory /retrieve/context identical (dense ∪ ILIKE).+# Requires DATA_DB=postgresql. Search p50 SLO is < 200 ms excluding embed.query RTT.+# SEARCH_USE_DENSE / SEARCH_USE_BM25 => at least one must be true when search is on.+# Both true (default) fuses with SEARCH_FUSION=rrf (or cc with SEARCH_FUSION_ALPHA).+# SEARCH_COMMUNITY_LABELS => hub labels for the communities search channel.+# SEARCH_NEIGHBOR_FANOUT => max 1-hop members per seed when expand=neighbors.+# CONTEXT_PASSAGE_MODE => hybrid|bm25|dense|ilike; only applies when SEARCH_ENABLED=true.+# hybrid uses BM25 ∪ dense RRF on /retrieve/context; ilike freezes the old lexical leg.+SEARCH_ENABLED="false"+SEARCH_USE_DENSE="true"+SEARCH_USE_BM25="true"+SEARCH_FUSION="rrf"+SEARCH_FUSION_ALPHA="0.5"+SEARCH_BM25_K1="1.2"+SEARCH_BM25_B="0.75"+SEARCH_COMMUNITY_LABELS="TYPE,CLASS,TOPIC"+SEARCH_NEIGHBOR_FANOUT="50"+CONTEXT_PASSAGE_MODE="hybrid"GRAPH_DB="networkx"DATA_DB="postgresql"VECTOR_DB="postgresql"diff --git a/README.md b/README.mdindex f4bedb8..b139a29 100644--- a/README.md+++ b/README.md@@ -210,6 +210,8 @@ BrainAPI follows an **append-only philosophy**: nothing is deleted from the grap- **`accurate`** (default) — the full swarm runs: Scout extracts every entity, the Observations agent writes notes, and the Janitor validates and deduplicates the Architect's relationships.- **`lightweight`** — a faster, cheaper path that extracts only the most important entities and skips the heavier validation steps. Switch with `--pipeline lightweight` or `PIPELINE_MODE` in `.env`.+Search (`GET|POST /retrieve/search`) is **off by default**. Set `SEARCH_ENABLED=true` (requires `DATA_DB=postgresql`) to register the hit-list API, write BM25 `tsvector` indexes, and use halfvec HNSW when embeddings are >2000-d. Default retrieval is both channels fused (`SEARCH_FUSION=rrf`). The search p50 SLO is **< 200 ms excluding `embed.query`** — ask for `profile_stages` to split embed RTT from retrieve. `/retrieve/context` stays dense ∪ ILIKE unless search is on and `CONTEXT_PASSAGE_MODE` is `hybrid` / `bm25` / `dense`.+---## 🔎 Retrieving Knowledge@@ -219,6 +221,7 @@ Once your data is in, BrainAPI exposes purpose-built retrieval endpoints (REST,| Endpoint | Method | What you get || ---------------------------- | ------ | ---------------------------------------------------------------------------------------------------------------- || `/retrieve/context` | `POST` | **Relevant information for a piece of text** — the matching graph triples plus historical context |+| `/retrieve/search` | `GET`/`POST` | **Ranked search hits** (opt-in: `SEARCH_ENABLED=true`) — passages (BM25 ∪ dense) by default; optional `entities` / `events` / `communities` channels and `expand=neighbors`; p50 < 200 ms excluding embed || `/retrieve/entity/status` | `GET` | **Existence check** for a specific entity — returns whether it exists, its node, relationships, and observations || `/retrieve/entity/synergies` | `GET` | **Similar things** related to a given one — effectively a recommendation algorithm over the graph |diff --git a/deploy/env.heavy.example b/deploy/env.heavy.exampleindex 47a9801..6897f12 100644--- a/deploy/env.heavy.example+++ b/deploy/env.heavy.example@@ -170,6 +170,16 @@ INGEST_ARCHITECT_DENSE_ENTITY_THRESHOLD="12"INGEST_ARCHITECT_DENSE_MAX_CHARS="1200"INGEST_ARCHITECT_PRIOR_CONTEXT="auto"INGEST_ARCHITECT_SCRATCHPAD_TOKEN_CAP="500"+# Search (off by default). SEARCH_ENABLED=true requires DATA_DB=postgresql.+# p50 SLO < 200 ms excluding embed.query RTT.+SEARCH_ENABLED="false"+SEARCH_USE_DENSE="true"+SEARCH_USE_BM25="true"+SEARCH_FUSION="rrf"+SEARCH_FUSION_ALPHA="0.5"+SEARCH_BM25_K1="1.2"+SEARCH_BM25_B="0.75"+CONTEXT_PASSAGE_MODE="hybrid"GRAPH_DB="neo4j"DATA_DB="mongo"VECTOR_DB="milvus"diff --git a/deploy/env.light.example b/deploy/env.light.exampleindex 7ab76ef..8eb88e9 100644--- a/deploy/env.light.example+++ b/deploy/env.light.example@@ -168,6 +168,15 @@ INGEST_ARCHITECT_DENSE_ENTITY_THRESHOLD="12"INGEST_ARCHITECT_DENSE_MAX_CHARS="1200"INGEST_ARCHITECT_PRIOR_CONTEXT="auto"INGEST_ARCHITECT_SCRATCHPAD_TOKEN_CAP="500"+# Search (off by default). p50 SLO < 200 ms excluding embed.query. Requires DATA_DB=postgresql.+SEARCH_ENABLED="false"+SEARCH_USE_DENSE="true"+SEARCH_USE_BM25="true"+SEARCH_FUSION="rrf"+SEARCH_FUSION_ALPHA="0.5"+SEARCH_BM25_K1="1.2"+SEARCH_BM25_B="0.75"+CONTEXT_PASSAGE_MODE="hybrid"GRAPH_DB="networkx"DATA_DB="postgresql"VECTOR_DB="postgresql"diff --git a/src/adapters/data.py b/src/adapters/data.pyindex dc12088..d0ac6bc 100644--- a/src/adapters/data.py+++ b/src/adapters/data.py@@ -49,6 +49,11 @@ class DataAdapter:"""return self.data.search(text=text, brain_id=brain_id)+ def search_bm25(+ self, text: str, brain_id: str = "default", limit: int = 10+ ) -> List[Tuple[TextChunk, float]]:+ return self.data.search_bm25(text=text, brain_id=brain_id, limit=limit)+def get_text_chunks_by_ids(self, ids: List[str], with_observations: bool, brain_id: str = "default") -> Tuple[List[TextChunk], List[Observation]]:diff --git a/src/adapters/graph.py b/src/adapters/graph.pyindex cc2cc5c..b720706 100644--- a/src/adapters/graph.py+++ b/src/adapters/graph.py@@ -355,6 +355,26 @@ class GraphAdapter:brain_id, limit, skip, node_labels, node_uuids, query_text)+ def search_nodes_bm25(+ self,+ query_text: str,+ brain_id: str = "default",+ *,+ limit: int = 10,+ node_labels: Optional[list[str]] = None,+ node_uuids: Optional[list[str]] = None,+ ):+ method = getattr(self.graph, "search_nodes_bm25", None)+ if method is None:+ return []+ return method(+ query_text,+ brain_id,+ limit=limit,+ node_labels=node_labels,+ node_uuids=node_uuids,+ )+def deprecate_relationship(self,subject: Node,diff --git a/src/adapters/interfaces/data.py b/src/adapters/interfaces/data.pyindex 80cb8da..cf99abd 100644--- a/src/adapters/interfaces/data.py+++ b/src/adapters/interfaces/data.py@@ -53,6 +53,13 @@ class DataClient(ABC):"""raise NotImplementedError("search method not implemented")+ def search_bm25(+ self, text: str, brain_id: str, limit: int = 10+ ) -> List[Tuple[TextChunk, float]]:+ raise NotImplementedError(+ "search_bm25 requires SEARCH_ENABLED=true and DATA_DB=postgresql"+ )+@abstractmethoddef get_text_chunks_by_ids(self, ids: List[str], with_observations: bool, brain_id: strdiff --git a/src/config.py b/src/config.pyindex 74bcc03..4a60e11 100644--- a/src/config.py+++ b/src/config.py@@ -71,6 +71,56 @@ def validate_ingest_architect_prior_context()+def validate_search_fusion(value: str | None) -> Literal["rrf", "cc"]:+ normalized = (value or "rrf").strip().lower()+ if normalized in ("rrf", "cc"):+ return normalized # type: ignore[return-value]+ raise ValueError(+ f"Invalid SEARCH_FUSION: {value!r}. Expected 'rrf' or 'cc'."+ )+++def validate_context_passage_mode(+ value: str | None,+) -> Literal["hybrid", "bm25", "dense", "ilike"]:+ normalized = (value or "hybrid").strip().lower()+ if normalized in ("hybrid", "bm25", "dense", "ilike"):+ return normalized # type: ignore[return-value]+ raise ValueError(+ f"Invalid CONTEXT_PASSAGE_MODE: {value!r}. "+ "Expected 'hybrid', 'bm25', 'dense', or 'ilike'."+ )+++def validate_search_config(+ *,+ enabled: bool,+ use_dense: bool,+ use_bm25: bool,+ data_db: str,+ bm25_k1: float,+ bm25_b: float,+) -> None:+ if bm25_k1 <= 0:+ raise ValueError(f"Invalid SEARCH_BM25_K1: {bm25_k1!r}. Must be > 0.")+ if not 0.0 <= bm25_b <= 1.0:+ raise ValueError(+ f"Invalid SEARCH_BM25_B: {bm25_b!r}. Must be between 0 and 1."+ )+ if not enabled:+ return+ if (data_db or "").strip().lower() != "postgresql":+ raise ValueError(+ "SEARCH_ENABLED=true requires DATA_DB=postgresql "+ "(v1 BM25 indexes are Postgres-only)."+ )+ if not use_dense and not use_bm25:+ raise ValueError(+ "SEARCH_ENABLED=true requires SEARCH_USE_DENSE=true "+ "and/or SEARCH_USE_BM25=true."+ )++class AzureConfig:"""Configuration class for the Azure configuration.@@ -452,6 +502,7 @@ class SpacyConfig:)+SEARCH_FTS_REGCONFIGS = frozenset({"italian", "spanish", "simple"})_MODES = ("local", "remote")_PROVIDERS = ("ollama",@@ -645,6 +696,55 @@ class Config:self.agentic_architecture: Literal["custom", "langchain"] = os.getenv("AGENTIC_ARCHITECTURE", "custom")+ self.search_enabled = os.getenv("SEARCH_ENABLED", "false") == "true"+ self.search_use_dense = os.getenv("SEARCH_USE_DENSE", "true") == "true"+ self.search_use_bm25 = os.getenv("SEARCH_USE_BM25", "true") == "true"+ community_raw = os.getenv("SEARCH_COMMUNITY_LABELS", "TYPE,CLASS,TOPIC")+ community_labels = [+ part.strip() for part in community_raw.split(",") if part.strip()+ ]+ self.search_community_labels = community_labels or [+ "TYPE",+ "CLASS",+ "TOPIC",+ ]+ self.search_neighbor_fanout = int(os.getenv("SEARCH_NEIGHBOR_FANOUT", "50"))+ self.search_fusion: Literal["rrf", "cc"] = validate_search_fusion(+ os.getenv("SEARCH_FUSION", "rrf")+ )+ self.search_fusion_alpha = float(os.getenv("SEARCH_FUSION_ALPHA", "0.5"))+ self.search_literal_fill = os.getenv("SEARCH_LITERAL_FILL", "false") == "true"+ self.search_bm25_k1 = float(os.getenv("SEARCH_BM25_K1", "1.2"))+ self.search_bm25_b = float(os.getenv("SEARCH_BM25_B", "0.75"))+ self.search_fts_regconfig = os.getenv("SEARCH_FTS_REGCONFIG", "").strip().lower()+ self.search_fts_brains = frozenset(+ part.strip()+ for part in os.getenv("SEARCH_FTS_BRAINS", "").split(",")+ if part.strip()+ )+ self.context_passage_mode: Literal[+ "hybrid", "bm25", "dense", "ilike"+ ] = validate_context_passage_mode(+ os.getenv("CONTEXT_PASSAGE_MODE", "hybrid")+ )+ validate_search_config(+ enabled=self.search_enabled,+ use_dense=self.search_use_dense,+ use_bm25=self.search_use_bm25,+ data_db=self.data_db,+ bm25_k1=self.search_bm25_k1,+ bm25_b=self.search_bm25_b,+ )++ def search_fts_regconfig_for_brain(self, brain_id: str) -> str | None:+ bid = (brain_id or "").strip()+ if not bid.lower().startswith("searchbench"):+ return None+ if bid not in self.search_fts_brains:+ return None+ if self.search_fts_regconfig not in SEARCH_FTS_REGCONFIGS:+ return None+ return self.search_fts_regconfigconfig = Config()diff --git a/src/constants/tasks/ingestion.py b/src/constants/tasks/ingestion.pyindex fee5335..3941d11 100644--- a/src/constants/tasks/ingestion.py+++ b/src/constants/tasks/ingestion.py@@ -85,3 +85,10 @@ class IngestionTaskArgs(BaseModel):"Used to resolve relative dates like 'yesterday' into absolute dates."),)+ skip_enrichment: bool = Field(+ default=False,+ description=(+ "Persist the text chunk and embedding only. Skip observations and "+ "knowledge-graph enrichment (Scout/Architect)."+ ),+ )diff --git a/src/core/agents/architect_agent.py b/src/core/agents/architect_agent.pyindex e07cf34..be744b3 100644--- a/src/core/agents/architect_agent.py+++ b/src/core/agents/architect_agent.py@@ -73,6 +73,7 @@ from src.core.saving.identity import (stable_relationship_id,)from src.core.saving.ingestion_manager import IngestionManager+from src.core.search.catalog_graph import is_static_has_triplefrom src.services.api.constants.requests import IngestionTripleSetfrom src.utils.cleanup import strip_propertiesfrom src.utils.dates import normalize_date_string@@ -184,8 +185,28 @@ def ingestion_triples_to_relationships(if not cr.subject or not cr.subj_event:continuesubject = _triple_entity(cr.subject)- event = _triple_entity(cr.event)obj = _triple_entity(cr.object)+ if cr.event is None or is_static_has_triple(cr):+ pred_name = cr.subj_event.name or "HAS"+ rel_uuid = cr.subj_event.uuid or stable_relationship_id(+ subject.uuid, pred_name, obj.uuid+ )+ triple_relationships.append(+ ArchitectAgentRelationship(+ tail=subject,+ name=pred_name,+ tip=obj,+ description=cr.subj_event.description,+ amount=cr.subj_event.amount,+ properties=cr.subj_event.properties or {},+ uuid=rel_uuid,+ flow_key=rel_uuid,+ )+ )+ continue+ if not cr.event or not cr.event_obj:+ continue+ event = _triple_entity(cr.event)flow_key = stable_flow_key(event_uuid=event.uuid,event_name=event.name,@@ -224,6 +245,8 @@ def ingestion_triples_to_relationships(])for pt in partial_triples:+ if not pt.event or not pt.event_obj:+ continueevent = _triple_entity(pt.event)obj = _triple_entity(pt.object)if pt.subject:diff --git a/src/core/plugins/context.py b/src/core/plugins/context.pyindex 0bf5e7e..04a8f70 100644--- a/src/core/plugins/context.py+++ b/src/core/plugins/context.py@@ -53,6 +53,8 @@ class PluginContext:self.config = configself._routers: list[tuple["APIRouter", dict[str, Any]]] = []self._event_handlers: dict[str, list[Callable]] = {}+ self._search_retrievers: dict[str, Callable] = {}+ self._search_rerankers: dict[str, Callable] = {}@classmethoddef _build_adapters(cls) -> PluginAdapters:@@ -133,3 +135,15 @@ class PluginContext:def add_event_handler(self, event: str, handler: Callable) -> None:self._event_handlers.setdefault(event, []).append(handler)++ def register_search_retriever(self, name: str, fn: Callable) -> None:+ from src.core.search.hooks import register_search_retriever++ register_search_retriever(name, fn)+ self._search_retrievers[name] = fn++ def register_search_reranker(self, name: str, fn: Callable) -> None:+ from src.core.search.hooks import register_search_reranker++ register_search_reranker(name, fn)+ self._search_rerankers[name] = fndiff --git a/src/core/saving/ingestion_manager.py b/src/core/saving/ingestion_manager.pyindex b3a9887..d8be498 100644--- a/src/core/saving/ingestion_manager.py+++ b/src/core/saving/ingestion_manager.py@@ -12,14 +12,16 @@ Modified By: Christian Nonis <[email protected]>from src.adapters.embeddings import EmbeddingsAdapterfrom src.adapters.graph import GraphAdapterfrom src.adapters.embeddings import VectorStoreAdapter+from src.constants.agents import ArchitectAgentRelationship+from src.constants.embeddings import Vectorfrom src.core.agents.scout_agent import ScoutEntity-from src.core.agents.architect_agent import ArchitectAgentRelationshipfrom src.core.saving.identity import (stable_flow_key,stable_node_id,stable_relationship_id,stable_uuid,)+from src.core.search.catalog_graph import node_embed_text__all__ = ["IngestionManager",@@ -53,6 +55,16 @@ class IngestionManager:self.kg = graph_adapterself.resolved_cache = {}self.metadata = {}+ self._text_embed_cache: dict[str, list[float]] = {}++ def _embed_text_cached(self, text: str) -> Vector:+ cached = self._text_embed_cache.get(text)+ if cached is not None:+ return Vector(id="", embeddings=list(cached), metadata={})+ vector = self.embeddings.embed_text(text)+ if vector.embeddings:+ self._text_embed_cache[text] = list(vector.embeddings)+ return vectordef process_node_vectors(self, node_data: ScoutEntity, brain_id):"""@@ -70,12 +82,19 @@ class IngestionManager:Returns:The node's UUID."""- if node_data.name in self.resolved_cache:+ text, cache_key = node_embed_text(node_data)+ if cache_key in self.resolved_cache:return node_data.uuid- v_sub = self.embeddings.embed_text(node_data.name)+ v_sub = self._embed_text_cached(text)+ labels = [node_data.type]+ extra = (node_data.properties or {}).get("catalog_labels") or []+ for item in extra:+ label = str(item).strip()+ if label and label not in labels:+ labels.append(label)v_sub.metadata = {- "labels": [node_data.type],+ "labels": labels,"name": node_data.name,"uuid": node_data.uuid,}@@ -87,7 +106,7 @@ class IngestionManager:**(node_data.properties or {}),"v_id": v_ids[0],}- self.resolved_cache[node_data.name] = v_ids[0]+ self.resolved_cache[cache_key] = v_ids[0]return node_data.uuidelse:print("[ ! ]Node not embedded:", node_data)@@ -118,7 +137,7 @@ class IngestionManager:)text_to_embed = (rel_data.description or "").strip() or rel_data.nameif text_to_embed:- v_rel = self.embeddings.embed_text(text_to_embed)+ v_rel = self._embed_text_cached(text_to_embed)v_rel.metadata = {**(self.metadata or {}),"uuid": rel_data.uuid,diff --git a/src/core/search/bm25.py b/src/core/search/bm25.pynew file mode 100644index 0000000..1ae9dd2--- /dev/null+++ b/src/core/search/bm25.py@@ -0,0 +1,43 @@+import math+++def okapi_idf(n_docs: float, df: float) -> float:+ n_docs = max(float(n_docs), 0.0)+ df = max(float(df), 0.0)+ return math.log(1.0 + (n_docs - df + 0.5) / (df + 0.5))+++def okapi_bm25_term(+ tf: float,+ dl: float,+ avgdl: float,+ idf: float,+ *,+ k1: float = 1.2,+ b: float = 0.75,+) -> float:+ if tf <= 0 or idf == 0:+ return 0.0+ avgdl = avgdl if avgdl > 0 else 1.0+ dl = dl if dl > 0 else 1.0+ denom = tf + k1 * (1.0 - b + b * dl / avgdl)+ if denom == 0:+ return 0.0+ return idf * (tf * (k1 + 1.0)) / denom+++def okapi_bm25_document(+ term_tfs: dict[str, float],+ *,+ dl: float,+ avgdl: float,+ n_docs: float,+ dfs: dict[str, float],+ k1: float = 1.2,+ b: float = 0.75,+) -> float:+ score = 0.0+ for term, tf in term_tfs.items():+ idf = okapi_idf(n_docs, dfs.get(term, 0.0))+ score += okapi_bm25_term(tf, dl, avgdl, idf, k1=k1, b=b)+ return scorediff --git a/src/core/search/catalog_graph.py b/src/core/search/catalog_graph.pynew file mode 100644index 0000000..6e315ba--- /dev/null+++ b/src/core/search/catalog_graph.py@@ -0,0 +1,768 @@+from __future__ import annotations++import json+import re+from pathlib import Path+from typing import Any, Sequence++from src.core.search.recommend import behavior_weight++HAS_EVENT_NAME = "HAS"+ENTITY_TYPE = "ENTITY"+CLASS_LABEL = "CLASS"+TYPE_LABEL = "TYPE"+ATTR_LABEL = "ATTR"+EVENT_LABEL = "EVENT"+USER_TYPE = "USER"+OPTION_ALLOWLIST = frozenset(+ {"color", "material", "category", "brand", "size", "style"}+)+_BEHAVIOR_EVENT = {+ "view": "View",+ "viewed": "View",+ "click": "View",+ "clicked": "View",+ "cart": "AddToCart",+ "add_to_cart": "AddToCart",+ "added_to_cart": "AddToCart",+ "addtocart": "AddToCart",+ "purchase": "Purchase",+ "purchased": "Purchase",+ "buy": "Purchase",+ "bought": "Purchase",+ "favorite": "Favorite",+ "favourite": "Favorite",+ "favorites": "Favorite",+ "favourites": "Favorite",+ "wishlist": "Wishlist",+ "add_to_favorite": "Favorite",+ "addtofavorite": "Favorite",+ "add_to_favourite": "Favorite",+ "addtofavourite": "Favorite",+ "add_to_wishlist": "Wishlist",+ "addtowishlist": "Wishlist",+ "follow": "Follow",+ "flw": "Follow",+}+FEATURE_KEY_CAP = 16+FEATURE_KEY_PRIORITY = (+ "style",+ "mood",+ "color",+ "colour",+ "material",+ "shape",+ "finish",+ "fabric",+ "design",+ "woodtone",+)+FEATURE_SKIP_TOKENS = (+ "width",+ "height",+ "weight",+ "length",+ "depth",+ "capacity",+ "temperature",+ "rating",+ "price",+ "count",+ "warranty",+ "origin",+ "assembly",+ "thickness",+ "diameter",+ "clearance",+ "sku",+)+CONTINUOUS_PROPERTY_KEYS = (+ "price",+ "rating",+ "average_rating",+ "rating_count",+ "review_count",+ "width",+ "height",+ "depth",+ "weight",+ "length",+)+BOOLEAN_NOISE = {+ "",+ "yes",+ "no",+ "true",+ "false",+ "y",+ "n",+ "0",+ "1",+ "none",+ "n/a",+ "na",+ "unknown",+ "null",+}+_LABEL_RE = re.compile(r"[^A-Za-z0-9]+")+_TEXT_FIELD_RE = re.compile(+ r"^(Title|Brand|Color|Class|Category|Features|Locale|Hierarchy|Description|Price|Rating)\s*:\s*(.+)$",+ re.IGNORECASE,+)+_HIER_SPLIT = re.compile(r"\s*[>/|]+\s*")+++def entity_uuid(doc_id: str) -> str:+ return str(doc_id)+++def hub_uuid(kind: str, value: str) -> str:+ return f"hub:{kind}:{value.strip().lower()}"+++def event_uuid(owner: str, kind: str, seq: int) -> str:+ return f"evt:{owner}:{kind}:{seq}"+++def rel_uuid(owner: str, kind: str, seq: int) -> str:+ return f"rel:{owner}:{kind}:{seq}"+++def sanitize_label(raw: str) -> str:+ cleaned = _LABEL_RE.sub("_", (raw or "").strip().upper()).strip("_")+ if not cleaned or cleaned[0].isdigit():+ return ATTR_LABEL+ return cleaned[:40]+++_DOC_ID_RE = re.compile(r"DOCID\s+(\S+)")+++def compose_search_text(*parts: str | None) -> str:+ chunks = [str(part).strip() for part in parts if str(part or "").strip()]+ return " ".join(chunks)+++def doc_id_from_text(text: str | None) -> str | None:+ match = _DOC_ID_RE.search(str(text or ""))+ if not match:+ return None+ doc_id = match.group(1).strip().rstrip(".")+ return doc_id or None+++def node_id_from_passage_text(text: str | None) -> str | None:+ doc_id = doc_id_from_text(text)+ if not doc_id:+ return None+ return entity_uuid(doc_id)+++def entity_search_text(doc: dict[str, Any], fields: dict[str, Any] | None = None) -> str:+ raw = str(doc.get("text") or "").strip()+ if raw:+ return raw+ resolved = fields if fields is not None else doc_fields(doc)+ return compose_search_text(+ resolved.get("title"),+ resolved.get("class"),+ resolved.get("hierarchy"),+ resolved.get("brand"),+ resolved.get("color"),+ resolved.get("features"),+ resolved.get("description"),+ )+++def node_embed_text(node_data: Any) -> tuple[str, str]:+ props = getattr(node_data, "properties", None) or {}+ if not isinstance(props, dict):+ props = {}+ search_text = str(props.get("search_text") or "").strip()+ name = str(getattr(node_data, "name", "") or "").strip()+ if search_text:+ uuid = str(getattr(node_data, "uuid", "") or "").strip()+ return search_text, f"uuid:{uuid}" if uuid else search_text+ return name, name+++def split_hierarchy(raw: str | None) -> list[str]:+ text = str(raw or "").strip()+ if not text:+ return []+ parts = [item.strip() for item in _HIER_SPLIT.split(text) if item.strip()]+ seen: set[str] = set()+ out: list[str] = []+ for part in parts:+ key = part.lower()+ if key in seen:+ continue+ seen.add(key)+ out.append(part)+ return out+++def _feature_key_rank(key: str) -> int:+ lowered = key.lower().replace(" ", "")+ for index, token in enumerate(FEATURE_KEY_PRIORITY):+ if token in lowered:+ return index+ return len(FEATURE_KEY_PRIORITY)+++def _skip_feature_key(key: str) -> bool:+ lowered = key.lower().replace(" ", "")+ if not lowered:+ return True+ return any(token in lowered for token in FEATURE_SKIP_TOKENS)+++def parse_feature_string(+ raw: str | None, *, cap: int = FEATURE_KEY_CAP+) -> list[tuple[str, str]]:+ text = str(raw or "").strip()+ if not text:+ return []+ pairs: list[tuple[str, str]] = []+ seen: set[tuple[str, str]] = set()+ parts = [part.strip() for part in text.split("|") if part.strip()]+ if ":" in text and not parts:+ parts = [text]+ for part in parts:+ if ":" not in part:+ continue+ key, value = part.split(":", 1)+ key = key.strip()+ value = value.strip()+ if not key or not value:+ continue+ if _skip_feature_key(key):+ continue+ if value.lower() in BOOLEAN_NOISE:+ continue+ item = (key, value)+ if item in seen:+ continue+ seen.add(item)+ pairs.append(item)+ pairs.sort(key=lambda item: (_feature_key_rank(item[0]), item[0].lower()))+ if cap <= 0:+ return pairs+ kept: list[tuple[str, str]] = []+ keys: set[str] = set()+ for key, value in pairs:+ key_id = key.lower()+ if key_id not in keys and len(keys) >= cap:+ continue+ keys.add(key_id)+ kept.append((key, value))+ return kept+++def _fields_from_text(text: str) -> dict[str, str]:+ out: dict[str, str] = {}+ for line in (text or "").splitlines():+ match = _TEXT_FIELD_RE.match(line.strip())+ if not match:+ continue+ key = match.group(1).strip().lower()+ value = match.group(2).strip()+ if value:+ out[key] = value+ return out+++def doc_fields(doc: dict[str, Any]) -> dict[str, Any]:+ text_fields = _fields_from_text(str(doc.get("text") or ""))+ title = str(doc.get("title") or text_fields.get("title") or "").strip()+ class_name = str(+ doc.get("class")+ or doc.get("product_class")+ or text_fields.get("class")+ or ""+ ).strip()+ hierarchy = str(+ doc.get("hierarchy")+ or doc.get("category_hierarchy")+ or text_fields.get("hierarchy")+ or text_fields.get("category")+ or ""+ ).strip()+ if not class_name:+ class_name = str(text_fields.get("category") or "").strip()+ brand = str(doc.get("brand") or text_fields.get("brand") or "").strip()+ color = str(doc.get("color") or text_fields.get("color") or "").strip()+ locale = str(doc.get("locale") or text_fields.get("locale") or "").strip()+ features = str(doc.get("features") or text_fields.get("features") or "").strip()+ description = str(+ doc.get("description") or text_fields.get("description") or ""+ ).strip()+ dataset = str(doc.get("dataset") or "").strip()+ properties: dict[str, str] = {}+ for key in CONTINUOUS_PROPERTY_KEYS:+ value = str(doc.get(key) or text_fields.get(key) or "").strip()+ if value:+ properties[key] = value+ return {+ "title": title,+ "class": class_name,+ "hierarchy": hierarchy,+ "brand": brand,+ "color": color,+ "locale": locale,+ "features": features,+ "description": description,+ "dataset": dataset,+ "properties": properties,+ }+++def _triple_pred_name(value: Any) -> str:+ if value is None:+ return ""+ if isinstance(value, dict):+ return str(value.get("name") or "").strip().upper()+ return str(getattr(value, "name", "") or "").strip().upper()+++def is_static_has_triple(triple: Any) -> bool:+ if isinstance(triple, dict):+ event = triple.get("event")+ subj_event = triple.get("subj_event")+ event_obj = triple.get("event_obj")+ else:+ event = getattr(triple, "event", None)+ subj_event = getattr(triple, "subj_event", None)+ event_obj = getattr(triple, "event_obj", None)+ happened = None+ event_name = ""+ if isinstance(event, dict):+ happened = event.get("happened_at")+ event_name = str(event.get("name") or "").strip().upper()+ elif event is not None:+ happened = getattr(event, "happened_at", None)+ event_name = str(getattr(event, "name", "") or "").strip().upper()+ if happened:+ return False+ pred = _triple_pred_name(subj_event) or HAS_EVENT_NAME+ if event is None:+ return pred == HAS_EVENT_NAME+ other = _triple_pred_name(event_obj) or HAS_EVENT_NAME+ return event_name == HAS_EVENT_NAME and pred == HAS_EVENT_NAME and other == HAS_EVENT_NAME+++def has_triple(+ *,+ subject: dict[str, Any],+ object_node: dict[str, Any],+ seq: int,+ kind: str,+) -> dict[str, Any]:+ owner = str(subject.get("uuid") or subject.get("name") or "node")+ return {+ "subject": subject,+ "subj_event": {+ "name": HAS_EVENT_NAME,+ "uuid": rel_uuid(owner, f"{kind}_has", seq),+ },+ "object": object_node,+ }+++def _entity_node(+ doc_id: str,+ name: str,+ *,+ description: str | None = None,+ properties: dict[str, Any] | None = None,+ search_text: str | None = None,+) -> dict[str, Any]:+ props = dict(properties or {})+ blob = str(search_text or "").strip() or compose_search_text(name, description)+ if blob:+ props["search_text"] = blob+ node: dict[str, Any] = {+ "name": name or doc_id,+ "type": ENTITY_TYPE,+ "uuid": entity_uuid(doc_id),+ "labels": [ENTITY_TYPE],+ "properties": props,+ }+ if description:+ node["description"] = description+ return node+++def _hub_node(kind: str, value: str, extra_label: str | None = None) -> dict[str, Any]:+ labels = [kind]+ if extra_label:+ sanitized = sanitize_label(extra_label)+ if sanitized and sanitized not in labels:+ labels.append(sanitized)+ search_text = compose_search_text(extra_label or kind, value)+ return {+ "name": value,+ "type": kind,+ "uuid": hub_uuid(kind.lower(), value),+ "labels": labels,+ "description": search_text,+ "properties": {+ "search_text": search_text,+ "catalog_labels": labels,+ },+ }+++def doc_to_triples(doc: dict[str, Any]) -> list[dict[str, Any]]:+ doc_id = str(doc.get("doc_id") or "").strip()+ if not doc_id:+ return []+ fields = doc_fields(doc)+ name = fields["title"] or doc_id+ subject = _entity_node(+ doc_id,+ name,+ description=fields["description"] or None,+ properties=dict(fields["properties"] or {}),+ search_text=entity_search_text(doc, fields),+ )+ triples: list[dict[str, Any]] = []+ seq = 0+ class_name = fields["class"]+ if class_name:+ triples.append(+ has_triple(+ subject=subject,+ object_node=_hub_node(CLASS_LABEL, class_name),+ seq=seq,+ kind="class",+ )+ )+ seq += 1+ class_key = class_name.strip().lower()+ for part in split_hierarchy(fields["hierarchy"]):+ if part.strip().lower() == class_key:+ continue+ triples.append(+ has_triple(+ subject=subject,+ object_node=_hub_node(TYPE_LABEL, part, extra_label="CATEGORY"),+ seq=seq,+ kind="hierarchy",+ )+ )+ seq += 1+ if fields["brand"]:+ triples.append(+ has_triple(+ subject=subject,+ object_node=_hub_node(ATTR_LABEL, fields["brand"], extra_label="BRAND"),+ seq=seq,+ kind="brand",+ )+ )+ seq += 1+ if fields["color"]:+ triples.append(+ has_triple(+ subject=subject,+ object_node=_hub_node(ATTR_LABEL, fields["color"], extra_label="COLOR"),+ seq=seq,+ kind="color",+ )+ )+ seq += 1+ if fields["locale"]:+ triples.append(+ has_triple(+ subject=subject,+ object_node=_hub_node(TYPE_LABEL, fields["locale"], extra_label="LOCALE"),+ seq=seq,+ kind="locale",+ )+ )+ seq += 1+ for key, value in parse_feature_string(fields["features"]):+ triples.append(+ has_triple(+ subject=subject,+ object_node=_hub_node(ATTR_LABEL, value, extra_label=key),+ seq=seq,+ kind=f"feat:{sanitize_label(key).lower()}",+ )+ )+ seq += 1++ if not triples:+ dataset = fields["dataset"] or "ITEM"+ triples.append(+ has_triple(+ subject=subject,+ object_node=_hub_node(TYPE_LABEL, dataset),+ seq=seq,+ kind="type",+ )+ )+ return triples+++def docs_to_triples(docs: Sequence[dict[str, Any]]) -> list[dict[str, Any]]:+ triples: list[dict[str, Any]] = []+ for doc in docs:+ triples.extend(doc_to_triples(doc))+ return triples+++def catalog_entity_backfill_rows(docs: Sequence[dict[str, Any]]) -> list[dict[str, str]]:+ rows: list[dict[str, str]] = []+ for doc in docs:+ doc_id = str(doc.get("doc_id") or "").strip()+ if not doc_id:+ continue+ fields = doc_fields(doc)+ rows.append(+ {+ "uuid": entity_uuid(doc_id),+ "name": str(fields.get("title") or doc_id),+ "search_text": entity_search_text(doc, fields),+ }+ )+ return rows+++def format_happened_at(timestamp: str | None) -> str | None:+ if not timestamp:+ return None+ raw = str(timestamp).strip()+ if not raw:+ return None+ from datetime import datetime++ if len(raw) == 10 and raw[2] == "/" and raw[5] == "/":+ return raw+ for fmt in (+ "%Y-%m-%dT%H:%M:%S%z",+ "%Y-%m-%dT%H:%M:%S.%f%z",+ "%Y-%m-%dT%H:%M:%SZ",+ "%Y-%m-%dT%H:%M:%S",+ "%Y-%m-%d %H:%M:%S",+ "%Y-%m-%d",+ "%m/%d/%Y",+ "%d/%m/%Y",+ ):+ try:+ parsed = datetime.strptime(+ raw.replace("Z", "+0000") if fmt.endswith("%z") and raw.endswith("Z") else raw,+ fmt,+ )+ return parsed.strftime("%m/%d/%Y")+ except ValueError:+ continue+ try:+ parsed = datetime.fromisoformat(raw.replace("Z", "+00:00"))+ return parsed.strftime("%m/%d/%Y")+ except ValueError:+ return raw+++def _interaction_event_name(behavior: str) -> str:+ key = (behavior or "").strip().lower().replace("-", "_").replace(" ", "_")+ if key in _BEHAVIOR_EVENT:+ return _BEHAVIOR_EVENT[key]+ compact = key.replace("_", "")+ for alias, name in _BEHAVIOR_EVENT.items():+ if alias.replace("_", "") == compact:+ return name+ label = (behavior or "Interaction").strip() or "Interaction"+ return label[:1].upper() + label[1:]+++def _option_pairs(row: dict[str, Any]) -> list[tuple[str, str]]:+ pairs: list[tuple[str, str]] = []+ seen: set[tuple[str, str]] = set()+ for src in (row.get("options"), row.get("attributes")):+ if not isinstance(src, dict):+ continue+ for key, raw in src.items():+ facet = str(key).strip().lower()+ if facet not in OPTION_ALLOWLIST or raw is None:+ continue+ value = str(raw).strip()+ if not value:+ continue+ pair = (facet, value)+ if pair in seen:+ continue+ seen.add(pair)+ pairs.append(pair)+ return pairs+++def prefers_triple(+ *,+ user_id: str,+ facet: str,+ value: str,+ seq: int,+ amount: float,+) -> dict[str, Any]:+ extra = "CATEGORY" if facet == "category" else facet.upper()+ kind = CLASS_LABEL if facet == "category" else ATTR_LABEL+ return {+ "subject": {+ "name": user_id,+ "type": USER_TYPE,+ "uuid": f"user:{user_id}",+ "labels": [USER_TYPE],+ },+ "subj_event": {+ "name": "PREFERS",+ "uuid": rel_uuid(user_id, f"prefers:{facet}:{value}", seq),+ "amount": amount,+ "properties": {"facet": facet, "value": value, "weight": amount},+ },+ "object": _hub_node(kind, value, extra_label=extra),+ }+++def interaction_to_triples(row: dict[str, Any], *, seq: int) -> list[dict[str, Any]]:+ user_id = str(row.get("user_id") or "").strip()+ item_id = str(row.get("item_id") or "").strip()+ if not user_id or not item_id:+ return []+ behavior = str(row.get("behavior") or "interaction").strip() or "interaction"+ event_name = _interaction_event_name(behavior)+ happened_at = format_happened_at(row.get("timestamp") or row.get("ts"))+ item_name = str(row.get("title") or row.get("item_name") or item_id)+ event_node: dict[str, Any] = {+ "name": event_name,+ "type": EVENT_LABEL,+ "uuid": event_uuid(user_id, f"{item_id}:{behavior}", seq),+ "labels": [EVENT_LABEL],+ }+ if happened_at:+ event_node["happened_at"] = happened_at+ user_node = {+ "name": user_id,+ "type": USER_TYPE,+ "uuid": f"user:{user_id}",+ "labels": [USER_TYPE],+ }+ triples = [+ {+ "subject": user_node,+ "subj_event": {+ "name": "MADE",+ "uuid": rel_uuid(user_id, f"made:{item_id}", seq),+ },+ "event": event_node,+ "event_obj": {+ "name": "TARGETED",+ "uuid": rel_uuid(user_id, f"tgt:{item_id}", seq),+ },+ "object": _entity_node(item_id, item_name),+ }+ ]+ weight = behavior_weight(event_name)+ for facet, value in _option_pairs(row):+ triples.append(+ prefers_triple(+ user_id=user_id,+ facet=facet,+ value=value,+ seq=seq,+ amount=weight,+ )+ )+ catalog_doc = {+ "doc_id": item_id,+ "title": item_name,+ "class": row.get("category") or row.get("class"),+ "brand": row.get("brand"),+ "color": row.get("color"),+ "features": row.get("features"),+ "dataset": "interaction",+ }+ existing = {+ str((t.get("event") or {}).get("uuid") or "")+ for t in triples+ if (t.get("event") or {}).get("uuid")+ }+ for triple in doc_to_triples(catalog_doc):+ event_id = str((triple.get("event") or {}).get("uuid") or "")+ if event_id and event_id in existing:+ continue+ triples.append(triple)+ return triples+++def interactions_to_triples(+ rows: Sequence[dict[str, Any]],+ *,+ seq_start: int = 1,+) -> list[dict[str, Any]]:+ triples: list[dict[str, Any]] = []+ for i, row in enumerate(rows):+ triples.extend(interaction_to_triples(row, seq=seq_start + i))+ return triples+++def load_interaction_rows(path: Path) -> list[dict[str, Any]]:+ rows: list[dict[str, Any]] = []+ for line in path.read_text(encoding="utf-8").splitlines():+ text = line.strip()+ if not text:+ continue+ obj = json.loads(text)+ if not isinstance(obj, dict):+ continue+ if obj.get("user_id") and obj.get("item_id"):+ rows.append(obj)+ return rows+++def structured_ingest_body(+ triples: Sequence[dict[str, Any]],+ *,+ brain_id: str,+ mode: str = "deterministic",+) -> dict[str, Any]:+ return {+ "mode": mode,+ "brain_id": brain_id,+ "data": list(triples),+ }+++__all__ = [+ "HAS_EVENT_NAME",+ "ENTITY_TYPE",+ "CLASS_LABEL",+ "TYPE_LABEL",+ "ATTR_LABEL",+ "EVENT_LABEL",+ "USER_TYPE",+ "FEATURE_KEY_CAP",+ "compose_search_text",+ "doc_fields",+ "doc_id_from_text",+ "doc_to_triples",+ "entity_search_text",+ "node_id_from_passage_text",+ "docs_to_triples",+ "catalog_entity_backfill_rows",+ "entity_uuid",+ "format_happened_at",+ "has_triple",+ "is_static_has_triple",+ "hub_uuid",+ "interaction_to_triples",+ "prefers_triple",+ "interactions_to_triples",+ "load_interaction_rows",+ "node_embed_text",+ "parse_feature_string",+ "sanitize_label",+ "split_hierarchy",+ "structured_ingest_body",+]diff --git a/src/core/search/graph_channels.py b/src/core/search/graph_channels.pynew file mode 100644index 0000000..ea84004--- /dev/null+++ b/src/core/search/graph_channels.py@@ -0,0 +1,903 @@+from __future__ import annotations++import math+import re+from dataclasses import dataclass, field+from datetime import datetime+from typing import Any, Iterable++from src.core.search.hybrid import dense_similarity, fuse_passage_lists, passage_snippet+from src.utils.dates import parse_date_string, to_naive_utc++PASSAGES_CHANNEL = "passages"+ENTITIES_CHANNEL = "entities"+EVENTS_CHANNEL = "events"+COMMUNITIES_CHANNEL = "communities"+NEIGHBORS_CHANNEL = "neighbors"++CORE_CHANNELS = frozenset(+ {+ PASSAGES_CHANNEL,+ ENTITIES_CHANNEL,+ EVENTS_CHANNEL,+ COMMUNITIES_CHANNEL,+ }+)+GRAPH_CHANNELS = frozenset(+ {ENTITIES_CHANNEL, EVENTS_CHANNEL, COMMUNITIES_CHANNEL}+)++DEFAULT_COMMUNITY_LABELS = ("TYPE", "CLASS", "TOPIC")+DEFAULT_NEIGHBOR_FANOUT = 50+EVENT_LABEL = "EVENT"+ITEM_LABEL = "ENTITY"+HUB_ID_PREFIX = "hub:"+HUB_LABELS = frozenset({"ATTR", "TYPE", "CLASS", "TOPIC"})+EVENT_RECENCY_HALF_LIFE_DAYS = 365.0+_TOKEN_RE = re.compile(r"[A-Za-z0-9]{3,}")+_MAX_NAME_TOKENS = 6+_TOKEN_STOPWORDS = frozenset(+ {+ "the",+ "and",+ "for",+ "with",+ "from",+ "that",+ "this",+ "are",+ "was",+ "not",+ "without",+ "your",+ "you",+ "any",+ "all",+ }+)+++@dataclass+class GraphHit:+ id: str+ channel: str+ score: float+ snippet: str+ labels: list[str] = field(default_factory=list)+ extras: dict[str, Any] | None = None+++def parse_community_labels(raw: str | None) -> list[str]:+ items = [part.strip() for part in str(raw or "").split(",") if part.strip()]+ return items or list(DEFAULT_COMMUNITY_LABELS)+++def selected_graph_channels(channels: list[str] | None) -> list[str]:+ selected: list[str] = []+ seen: set[str] = set()+ for item in channels or []:+ lowered = str(item or "").strip().lower()+ if lowered in GRAPH_CHANNELS and lowered not in seen:+ selected.append(lowered)+ seen.add(lowered)+ return selected+++def _upper_set(values: Iterable[str] | None) -> set[str]:+ return {str(item).strip().upper() for item in (values or []) if str(item).strip()}+++def _node_labels(node: Any) -> list[str]:+ raw = getattr(node, "labels", None) or []+ return [str(item) for item in raw if str(item).strip()]+++def _node_uuid(node: Any) -> str:+ return str(getattr(node, "uuid", "") or "").strip()+++def _node_name(node: Any) -> str:+ name = getattr(node, "name", None)+ if name:+ return str(name)+ return _node_uuid(node)+++def _node_search_text(node: Any) -> str:+ props = getattr(node, "properties", None) or {}+ search_text = ""+ if isinstance(props, dict):+ search_text = str(props.get("search_text") or "").strip()+ description = str(getattr(node, "description", None) or "").strip()+ return search_text or compose_parts(_node_name(node), description)+++def compose_parts(*parts: str) -> str:+ return " ".join(item for item in parts if item)+++def _hub_kind(labels: Iterable[str] | None) -> str | None:+ have = _upper_set(labels)+ for kind in ("CLASS", "ATTR", "TYPE", "TOPIC"):+ if kind in have:+ return kind+ return None+++def _happened_at(node: Any) -> str | None:+ raw = getattr(node, "happened_at", None)+ if raw:+ text = str(raw).strip()+ if text:+ return text+ props = getattr(node, "properties", None) or {}+ if isinstance(props, dict):+ value = props.get("happened_at")+ if value:+ text = str(value).strip()+ if text:+ return text+ return None+++def _labels_match(node_labels: list[str], wanted: list[str] | None) -> bool:+ if not wanted:+ return True+ have = _upper_set(node_labels)+ need = _upper_set(wanted)+ return bool(have & need)+++def is_item_entity(uuid: str, labels: Iterable[str] | None = None) -> bool:+ node_id = str(uuid or "").strip()+ if not node_id or node_id.startswith(HUB_ID_PREFIX):+ return False+ have = _upper_set(labels)+ if EVENT_LABEL in have and ITEM_LABEL not in have:+ return False+ if have & HUB_LABELS and ITEM_LABEL not in have:+ return False+ return True+++def _entity_search_labels(node_labels: list[str] | None) -> list[str]:+ wanted = [str(item).strip() for item in (node_labels or []) if str(item).strip()]+ return wanted or [ITEM_LABEL]+++def _name_match_score(name: str, query: str) -> float:+ text = (name or "").strip().lower()+ needle = (query or "").strip().lower()+ if not text:+ return 0.0+ score = 0.0+ if needle:+ if text == needle:+ score += 5.0+ elif needle in text:+ score += 3.0+ tokens = _query_tokens(query)+ if tokens:+ matched = [token for token in tokens if token in text]+ score += float(len(matched))+ if matched:+ score += len(matched) / len(tokens)+ for left, right in zip(tokens, tokens[1:]):+ if f"{left} {right}" in text:+ score += 0.75+ return score+++def _search_tokens(query: str) -> list[str]:+ query_lower = (query or "").strip().lower()+ tokens: list[str] = []+ for token in _query_tokens(query):+ if token in _TOKEN_STOPWORDS or token == query_lower:+ continue+ tokens.append(token)+ return tokens+++def _degree_idf(degree: int) -> float:+ return 1.0 / math.log2(2.0 + max(0, int(degree)))+++def _parse_when(raw: str | None) -> datetime | None:+ if not raw:+ return None+ text = str(raw).strip()+ if not text:+ return None+ candidates = [text]+ if text.endswith("Z") and "+0000" not in text:+ candidates.append(text[:-1] + "+0000")+ for item in candidates:+ for fmt in (+ "%Y-%m-%dT%H:%M:%S%z",+ "%Y-%m-%dT%H:%M:%SZ",+ "%Y-%m-%dT%H:%M:%S",+ "%m/%d/%Y",+ "%Y-%m-%d",+ ):+ try:+ parsed = datetime.strptime(item, fmt)+ return parsed+ except ValueError:+ continue+ return parse_date_string(text)+++def recency_weight(+ happened_at: str | None,+ *,+ half_life_days: float = EVENT_RECENCY_HALF_LIFE_DAYS,+ now: datetime | None = None,+) -> float:+ if not happened_at or half_life_days <= 0:+ return 1.0+ parsed = _parse_when(happened_at)+ if parsed is None:+ return 1.0+ ref = now or datetime.now()+ age_days = max(+ 0.0,+ (to_naive_utc(ref) - to_naive_utc(parsed)).total_seconds() / 86400.0,+ )+ return float(0.5 ** (age_days / float(half_life_days)))+++def _query_tokens(query: str) -> list[str]:+ seen: set[str] = set()+ tokens: list[str] = []+ for match in _TOKEN_RE.finditer(query or ""):+ token = match.group(0).lower()+ if token in seen:+ continue+ seen.add(token)+ tokens.append(token)+ if len(tokens) >= _MAX_NAME_TOKENS:+ break+ return tokens+++def _hit_from_node(+ node: Any,+ *,+ channel: str,+ score: float,+ extras: dict[str, Any] | None = None,+) -> GraphHit | None:+ node_id = _node_uuid(node)+ if not node_id:+ return None+ name = _node_name(node)+ payload = dict(extras or {})+ happened = _happened_at(node)+ if happened and "happened_at" not in payload:+ payload["happened_at"] = happened+ return GraphHit(+ id=node_id,+ channel=channel,+ score=float(score),+ snippet=passage_snippet(name),+ labels=_node_labels(node),+ extras=payload or None,+ )+++def _merge_hit(bucket: dict[str, GraphHit], hit: GraphHit | None) -> None:+ if hit is None or not hit.id:+ return+ current = bucket.get(hit.id)+ if current is None or hit.score > current.score:+ bucket[hit.id] = hit+++def _ranked_hits(bucket: dict[str, GraphHit], k: int) -> list[GraphHit]:+ ordered = sorted(bucket.values(), key=lambda item: (-item.score, item.id))+ if k <= 0:+ return ordered+ return ordered[:k]+++def _search_entities(+ graph: Any,+ *,+ brain_id: str,+ query_text: str | None,+ node_labels: list[str] | None,+ limit: int,+) -> list[Any]:+ if graph is None or limit <= 0:+ return []+ try:+ result = graph.search_entities(+ brain_id=brain_id,+ limit=limit,+ skip=0,+ node_labels=node_labels,+ query_text=query_text,+ )+ except Exception:+ return []+ raw = getattr(result, "results", None) or []+ if not isinstance(raw, (list, tuple)):+ return []+ return list(raw)+++def _hydrate_nodes(graph: Any, uuids: list[str], brain_id: str) -> dict[str, Any]:+ if graph is None or not uuids:+ return {}+ unique = []+ seen: set[str] = set()+ for item in uuids:+ key = str(item or "").strip()+ if not key or key in seen:+ continue+ seen.add(key)+ unique.append(key)+ if not unique:+ return {}+ try:+ nodes = graph.get_by_uuids(unique, brain_id=brain_id)+ except Exception:+ nodes = []+ for key in unique:+ try:+ node = graph.get_by_uuid(key, brain_id=brain_id)+ except Exception:+ continue+ if node is not None:+ nodes.append(node)+ found: dict[str, Any] = {}+ for node in nodes or []:+ node_id = _node_uuid(node)+ if node_id:+ found[node_id] = node+ return found+++def collect_entity_hits(+ *,+ query: str,+ brain_id: str,+ k: int,+ graph: Any,+ vector_search: Any = None,+ query_vector: list[float] | None = None,+ node_labels: list[str] | None = None,+ channel: str = ENTITIES_CHANNEL,+) -> list[GraphHit]:+ if k <= 0 or graph is None:+ return []+ bucket: dict[str, GraphHit] = {}+ wanted = _entity_search_labels(node_labels)+ restrict_items = not node_labels+ fetch_k = max(k * 8, 40)++ def _keep(node_id: str, labels: list[str]) -> bool:+ if not _labels_match(labels, wanted):+ return False+ if restrict_items and not is_item_entity(node_id, labels):+ return False+ return True++ def _add_node(node: Any, extra: float = 0.0) -> None:+ node_id = _node_uuid(node)+ labels = _node_labels(node)+ if not node_id or not _keep(node_id, labels):+ return+ score = _name_match_score(_node_search_text(node), query) + extra+ _merge_hit(+ bucket,+ _hit_from_node(node, channel=channel, score=score),+ )++ nodes = _search_entities(+ graph,+ brain_id=brain_id,+ query_text=query,+ node_labels=wanted,+ limit=fetch_k,+ )+ for node in nodes:+ _add_node(node)++ for token in _search_tokens(query):+ token_nodes = _search_entities(+ graph,+ brain_id=brain_id,+ query_text=token,+ node_labels=wanted,+ limit=fetch_k,+ )+ for node in token_nodes:+ _add_node(node)++ bm25_method = getattr(graph, "search_nodes_bm25", None)+ if callable(bm25_method):+ try:+ bm25_hits = bm25_method(+ query,+ brain_id,+ limit=fetch_k,+ node_labels=wanted,+ )+ except TypeError:+ try:+ bm25_hits = bm25_method(query, brain_id)+ except Exception:+ bm25_hits = []+ except Exception:+ bm25_hits = []+ for item in bm25_hits or []:+ node = item[0] if isinstance(item, (tuple, list)) and item else item+ score = 0.0+ if isinstance(item, (tuple, list)) and len(item) >= 2:+ try:+ score = float(item[1])+ except (TypeError, ValueError):+ score = 0.0+ extra = min(3.0, max(0.0, score))+ _add_node(node, extra=extra)++ if vector_search is not None and query_vector:+ try:+ vectors = vector_search.search_nodes(+ query_vector, brain_id=brain_id, k=fetch_k+ )+ except Exception:+ vectors = []+ ann_ids: list[str] = []+ ann_scores: dict[str, float] = {}+ ann_meta: dict[str, dict[str, Any]] = {}+ for vector in vectors or []:+ meta = getattr(vector, "metadata", None) or {}+ node_id = str(meta.get("uuid") or getattr(vector, "id", "") or "").strip()+ if not node_id:+ continue+ ann_ids.append(node_id)+ distance = getattr(vector, "distance", None)+ ann_scores[node_id] = dense_similarity(+ float(distance) if distance is not None else float("inf")+ )+ if isinstance(meta, dict):+ ann_meta[node_id] = meta+ hydrated = _hydrate_nodes(graph, ann_ids, brain_id)+ for node_id in ann_ids:+ node = hydrated.get(node_id)+ labels = _node_labels(node) if node is not None else []+ extra = 0.5 * float(ann_scores.get(node_id, 0.0))+ if node is None:+ meta = ann_meta.get(node_id) or {}+ raw_labels = meta.get("labels") or []+ if isinstance(raw_labels, str):+ raw_labels = [raw_labels]+ labels = [str(item) for item in raw_labels if str(item).strip()]+ if not labels and meta.get("type"):+ labels = [str(meta.get("type"))]+ if not _keep(node_id, labels):+ continue+ name = str(meta.get("name") or node_id)+ _merge_hit(+ bucket,+ GraphHit(+ id=node_id,+ channel=channel,+ score=_name_match_score(name, query) + extra,+ snippet=passage_snippet(name),+ labels=labels,+ extras=None,+ ),+ )+ continue+ _add_node(node, extra=extra)++ return _ranked_hits(bucket, k)+++def collect_event_hits(+ *,+ query: str,+ brain_id: str,+ k: int,+ graph: Any,+ vector_search: Any = None,+ query_vector: list[float] | None = None,+) -> list[GraphHit]:+ hits = collect_entity_hits(+ query=query,+ brain_id=brain_id,+ k=k,+ graph=graph,+ vector_search=vector_search,+ query_vector=query_vector,+ node_labels=[EVENT_LABEL],+ channel=EVENTS_CHANNEL,+ )+ scored: list[GraphHit] = []+ for hit in hits:+ happened = None+ if hit.extras:+ happened = hit.extras.get("happened_at")+ weight = recency_weight(str(happened) if happened else None)+ extras = dict(hit.extras or {})+ if happened:+ extras["recency"] = weight+ scored.append(+ GraphHit(+ id=hit.id,+ channel=EVENTS_CHANNEL,+ score=float(hit.score) * weight,+ snippet=hit.snippet,+ labels=list(hit.labels or []),+ extras=extras or None,+ )+ )+ return _ranked_hits({item.id: item for item in scored}, k)+++def _event_centric_others(+ graph: Any,+ uuid: str,+ brain_id: str,+ *,+ skip_labels: set[str],+ fanout: int,+) -> list[Any]:+ found: list[Any] = []+ seen = {uuid}+ try:+ triples = graph.get_event_centric_neighbors([uuid], brain_id=brain_id)+ except Exception:+ triples = []+ for triple in triples or []:+ if not isinstance(triple, (tuple, list)) or len(triple) < 5:+ continue+ n, _p1, m, _p2, b = triple[:5]+ for node in (n, m, b):+ node_id = _node_uuid(node)+ if not node_id or node_id in seen:+ continue+ labels = _upper_set(_node_labels(node))+ if skip_labels and labels & skip_labels:+ continue+ seen.add(node_id)+ found.append(node)+ if len(found) >= fanout:+ return found+ return found+++def _direct_others(+ graph: Any,+ uuid: str,+ brain_id: str,+ *,+ skip_labels: set[str],+ fanout: int,+ skip_event_wrappers: bool,+) -> list[Any]:+ found: list[Any] = []+ seen = {uuid}+ try:+ neighbors = graph.get_neighbors([uuid], brain_id=brain_id)+ except Exception:+ return found+ rows = []+ if isinstance(neighbors, dict):+ rows = neighbors.get(uuid) or []+ for entry in rows:+ node = entry[1] if isinstance(entry, (tuple, list)) and len(entry) >= 2 else entry+ node_id = _node_uuid(node)+ if not node_id or node_id in seen:+ continue+ labels = _upper_set(_node_labels(node))+ if skip_event_wrappers and EVENT_LABEL in labels:+ try:+ inner = graph.get_neighbors([node_id], brain_id=brain_id)+ except Exception:+ inner = {}+ inner_rows = []+ if isinstance(inner, dict):+ inner_rows = inner.get(node_id) or []+ for inner_entry in inner_rows:+ tip = (+ inner_entry[1]+ if isinstance(inner_entry, (tuple, list)) and len(inner_entry) >= 2+ else inner_entry+ )+ tip_id = _node_uuid(tip)+ if not tip_id or tip_id in seen:+ continue+ tip_labels = _upper_set(_node_labels(tip))+ if skip_labels and tip_labels & skip_labels:+ continue+ if EVENT_LABEL in tip_labels:+ continue+ seen.add(tip_id)+ found.append(tip)+ if len(found) >= fanout:+ return found+ continue+ if skip_labels and labels & skip_labels:+ continue+ seen.add(node_id)+ found.append(node)+ if len(found) >= fanout:+ return found+ return found+++def adjacent_nodes(+ graph: Any,+ uuid: str,+ brain_id: str,+ *,+ skip_labels: Iterable[str] | None = None,+ fanout: int = DEFAULT_NEIGHBOR_FANOUT,+ skip_event_wrappers: bool = True,+) -> list[Any]:+ if graph is None or not uuid or fanout <= 0:+ return []+ skip = _upper_set(skip_labels)+ found = _event_centric_others(+ graph, uuid, brain_id, skip_labels=skip, fanout=fanout+ )+ if len(found) >= fanout:+ return found[:fanout]+ seen = {uuid, *(_node_uuid(node) for node in found)}+ for node in _direct_others(+ graph,+ uuid,+ brain_id,+ skip_labels=skip,+ fanout=fanout,+ skip_event_wrappers=skip_event_wrappers,+ ):+ node_id = _node_uuid(node)+ if not node_id or node_id in seen:+ continue+ seen.add(node_id)+ found.append(node)+ if len(found) >= fanout:+ break+ return found[:fanout]+++def _member_hits_for_hub(+ *,+ hub: GraphHit,+ graph: Any,+ brain_id: str,+ query: str,+ skip: set[str],+ fanout: int,+) -> tuple[list[GraphHit], dict[str, Any]]:+ members = adjacent_nodes(+ graph,+ hub.id,+ brain_id,+ skip_labels=skip,+ fanout=fanout,+ skip_event_wrappers=True,+ )+ degree = max(len(members), 1)+ idf = _degree_idf(degree)+ scored: list[GraphHit] = []+ nodes: dict[str, Any] = {}+ for member in members:+ labels_of = _node_labels(member)+ member_id = _node_uuid(member)+ if not is_item_entity(member_id, labels_of):+ continue+ lexical = _name_match_score(_node_search_text(member), query)+ hit = _hit_from_node(+ member,+ channel=COMMUNITIES_CHANNEL,+ score=float(hub.score) * idf * (1.0 + lexical),+ extras={"hub_id": hub.id, "hub_name": hub.snippet},+ )+ if hit is None:+ continue+ scored.append(hit)+ nodes[member_id] = member+ scored.sort(key=lambda item: (-item.score, item.id))+ return scored, nodes+++def _hybrid_rank_community_hits(+ hits: list[GraphHit],+ *,+ query: str,+ brain_id: str,+ k: int,+ graph: Any,+ vector_search: Any = None,+ query_vector: list[float] | None = None,+) -> list[GraphHit]:+ if not hits:+ return []+ by_id = {hit.id: hit for hit in hits}+ candidate_ids = list(by_id.keys())+ lexical_ids: list[str] = []+ bm25_method = getattr(graph, "search_nodes_bm25", None)+ if callable(bm25_method):+ try:+ bm25_hits = bm25_method(+ query,+ brain_id,+ limit=max(k * 8, len(candidate_ids)),+ node_uuids=candidate_ids,+ )+ except Exception:+ bm25_hits = []+ for item in bm25_hits or []:+ node = item[0] if isinstance(item, (tuple, list)) and item else item+ node_id = _node_uuid(node)+ if node_id in by_id and node_id not in lexical_ids:+ lexical_ids.append(node_id)+ dense_ids: list[str] = []+ if vector_search is not None and query_vector:+ try:+ vectors = vector_search.search_nodes(+ query_vector, brain_id=brain_id, k=max(k * 8, len(candidate_ids))+ )+ except Exception:+ vectors = []+ for vector in vectors or []:+ meta = getattr(vector, "metadata", None) or {}+ node_id = str(meta.get("uuid") or getattr(vector, "id", "") or "").strip()+ if node_id in by_id and node_id not in dense_ids:+ dense_ids.append(node_id)+ fused = fuse_passage_lists(dense_ids, lexical_ids)+ if not fused:+ return _ranked_hits(by_id, k)+ bucket: dict[str, GraphHit] = {}+ for node_id, score in fused:+ hit = by_id.get(node_id)+ if hit is None:+ continue+ bucket[node_id] = GraphHit(+ id=hit.id,+ channel=hit.channel,+ score=float(score),+ snippet=hit.snippet,+ labels=list(hit.labels or []),+ extras=dict(hit.extras or {}) or None,+ )+ for hit in hits:+ if hit.id not in bucket:+ bucket[hit.id] = GraphHit(+ id=hit.id,+ channel=hit.channel,+ score=0.0,+ snippet=hit.snippet,+ labels=list(hit.labels or []),+ extras=dict(hit.extras or {}) or None,+ )+ return _ranked_hits(bucket, k)+++def collect_community_hits(+ *,+ query: str,+ brain_id: str,+ k: int,+ graph: Any,+ vector_search: Any = None,+ query_vector: list[float] | None = None,+ community_labels: list[str] | None = None,+ fanout: int = DEFAULT_NEIGHBOR_FANOUT,+) -> list[GraphHit]:+ labels = [item for item in (community_labels or []) if str(item).strip()]+ if not labels:+ labels = list(DEFAULT_COMMUNITY_LABELS)+ hub_labels = list(labels)+ if "ATTR" not in _upper_set(hub_labels):+ hub_labels.append("ATTR")+ hubs = collect_entity_hits(+ query=query,+ brain_id=brain_id,+ k=k,+ graph=graph,+ vector_search=vector_search,+ query_vector=query_vector,+ node_labels=hub_labels,+ channel=COMMUNITIES_CHANNEL,+ )+ skip = _upper_set(hub_labels) | HUB_LABELS | {EVENT_LABEL}+ cap = max(1, int(fanout))+ hub_cap = max(1, min(len(hubs), 8))+ kinds = {+ kind+ for kind in (_hub_kind(hub.labels) for hub in hubs[:hub_cap])+ if kind+ }+ expand_cap = max(cap, 200) if len(kinds) >= 2 else cap+ members_by_kind: dict[str, set[str]] = {}+ union_bucket: dict[str, GraphHit] = {}+ for hub in hubs[:hub_cap]:+ kind = _hub_kind(hub.labels) or "ATTR"+ scored, _nodes = _member_hits_for_hub(+ hub=hub,+ graph=graph,+ brain_id=brain_id,+ query=query,+ skip=skip,+ fanout=expand_cap,+ )+ kind_ids = members_by_kind.setdefault(kind, set())+ for hit in scored[:expand_cap]:+ kind_ids.add(hit.id)+ _merge_hit(union_bucket, hit)+ kind_sets = [ids for ids in members_by_kind.values() if ids]+ selected_ids: set[str] | None = None+ if len(kind_sets) >= 2:+ selected_ids = set(kind_sets[0])+ for ids in kind_sets[1:]:+ selected_ids &= ids+ if not selected_ids:+ selected_ids = None+ if selected_ids is None:+ candidates = list(union_bucket.values())+ else:+ candidates = [+ hit for hit in union_bucket.values() if hit.id in selected_ids+ ]+ return _hybrid_rank_community_hits(+ candidates,+ query=query,+ brain_id=brain_id,+ k=k,+ graph=graph,+ vector_search=vector_search,+ query_vector=query_vector,+ )+++def expand_neighbor_hits(+ seeds: list[GraphHit],+ *,+ brain_id: str,+ k: int,+ graph: Any,+ community_labels: list[str] | None = None,+ fanout: int = DEFAULT_NEIGHBOR_FANOUT,+) -> list[GraphHit]:+ if not seeds or graph is None or k <= 0:+ return []+ skip = _upper_set(community_labels)+ seed_ids = {hit.id for hit in seeds}+ bucket: dict[str, GraphHit] = {}+ cap = max(1, int(fanout))+ for seed in seeds:+ skip_events = seed.channel != EVENTS_CHANNEL+ members = adjacent_nodes(+ graph,+ seed.id,+ brain_id,+ skip_labels=skip,+ fanout=cap,+ skip_event_wrappers=skip_events,+ )+ degree = max(len(members), 1)+ idf = _degree_idf(degree)+ scored: list[GraphHit] = []+ for member in members:+ node_id = _node_uuid(member)+ if not node_id or node_id in seed_ids:+ continue+ hit = _hit_from_node(+ member,+ channel=NEIGHBORS_CHANNEL,+ score=float(seed.score) * idf,+ extras={"seed_id": seed.id, "seed_channel": seed.channel},+ )+ if hit is not None:+ scored.append(hit)+ scored.sort(key=lambda item: (-item.score, item.id))+ for hit in scored[:cap]:+ _merge_hit(bucket, hit)+ return _ranked_hits(bucket, k)diff --git a/src/core/search/hooks.py b/src/core/search/hooks.pynew file mode 100644index 0000000..28dfde6--- /dev/null+++ b/src/core/search/hooks.py@@ -0,0 +1,142 @@+from __future__ import annotations++from typing import Any, Callable, Optional++from src.core.search.graph_channels import CORE_CHANNELS++RERANK_MAX_K = 10+CATALOG_RERANK_MAX_K = 50+CATALOG_RETRIEVE_MIN_K = 50+CATALOG_RETRIEVE_MAX_K = 200+++def retrieve_k_for_mode(mode: str | None, k: int) -> int:+ if (mode or "default") == "catalog":+ return min(CATALOG_RETRIEVE_MAX_K, max(int(k), CATALOG_RETRIEVE_MIN_K))+ return int(k)+++def rerank_max_k_for_mode(mode: str | None) -> int:+ if (mode or "default") == "catalog":+ return CATALOG_RERANK_MAX_K+ return RERANK_MAX_K+++RetrieveFn = Callable[+ [str, str, int],+ tuple[list[str], dict[str, float], dict[str, str]],+]+RerankFn = Callable[[str, list[dict[str, Any]], int], list[dict[str, Any]]]++_retrievers: dict[str, RetrieveFn] = {}+_rerankers: dict[str, RerankFn] = {}+++class SearchPluginError(ValueError):+ pass+++def reset_search_plugins() -> None:+ _retrievers.clear()+ _rerankers.clear()+++def register_search_retriever(name: str, fn: RetrieveFn) -> None:+ key = _clean_name(name)+ if not key:+ raise SearchPluginError("Retriever name is required.")+ _retrievers[key] = fn+++def register_search_reranker(name: str, fn: RerankFn) -> None:+ key = _clean_name(name)+ if not key:+ raise SearchPluginError("Reranker name is required.")+ _rerankers[key] = fn+++def get_search_retriever(name: str) -> Optional[RetrieveFn]:+ return _retrievers.get(_clean_name(name))+++def get_search_reranker(name: str) -> Optional[RerankFn]:+ return _rerankers.get(_clean_name(name))+++def listed_retrievers() -> list[str]:+ return sorted(_retrievers)+++def listed_rerankers() -> list[str]:+ return sorted(_rerankers)+++def _clean_name(name: str) -> str:+ return str(name or "").strip().lower()+++def parse_rerank(value: Optional[str]) -> Optional[str]:+ raw = (value or "").strip()+ if not raw or raw.lower() == "none":+ return None+ lowered = raw.lower()+ if lowered == "linear":+ raise SearchPluginError(+ "rerank=linear is not implemented. Use none or plugin:<name>."+ )+ if not lowered.startswith("plugin:"):+ raise SearchPluginError(+ f"Invalid rerank={raw!r}. Expected none or plugin:<name>."+ )+ name = _clean_name(raw.split(":", 1)[1])+ if not name:+ raise SearchPluginError("rerank=plugin: requires a plugin name.")+ return name+++def parse_plugin_channels(channels: Optional[list[str]]) -> list[str]:+ names: list[str] = []+ for item in channels or []:+ raw = str(item or "").strip()+ if not raw:+ continue+ lowered = raw.lower()+ if lowered in CORE_CHANNELS:+ continue+ if lowered.startswith("plugin:"):+ name = _clean_name(raw.split(":", 1)[1])+ if not name:+ raise SearchPluginError("channels plugin: requires a plugin name.")+ names.append(name)+ continue+ if get_search_retriever(lowered) is not None:+ names.append(lowered)+ continue+ raise SearchPluginError(f"Unknown search channel {raw!r}.")+ return names+++def resolve_reranker(value: Optional[str]) -> Optional[RerankFn]:+ name = parse_rerank(value)+ if name is None:+ return None+ fn = get_search_reranker(name)+ if fn is None:+ available = ", ".join(listed_rerankers()) or "(none loaded)"+ raise SearchPluginError(+ f"Unknown search rerank plugin {name!r}. Loaded: {available}."+ )+ return fn+++def resolve_retrievers(channels: Optional[list[str]]) -> list[tuple[str, RetrieveFn]]:+ resolved: list[tuple[str, RetrieveFn]] = []+ for name in parse_plugin_channels(channels):+ fn = get_search_retriever(name)+ if fn is None:+ available = ", ".join(listed_retrievers()) or "(none loaded)"+ raise SearchPluginError(+ f"Unknown search retriever plugin {name!r}. Loaded: {available}."+ )+ resolved.append((name, fn))+ return resolveddiff --git a/src/core/search/hybrid.py b/src/core/search/hybrid.pynew file mode 100644index 0000000..005a852--- /dev/null+++ b/src/core/search/hybrid.py@@ -0,0 +1,339 @@+import re+from typing import Any, Optional++from src.core.search.fact_filter import reciprocal_rank_fusion++_TOKEN_RE = re.compile(r"[^\W_]+", re.UNICODE)+_LITERAL_STOP = frozenset(+ {+ "the",+ "and",+ "for",+ "with",+ "from",+ "this",+ "that",+ "are",+ "was",+ }+)++_EXTRAS_SKIP = frozenset(+ {"resource_id", "uuid", "id", "inserted_at", "brain_version"}+)+++def passage_snippet(text: str, max_len: int = 240) -> str:+ body = (text or "").strip()+ if len(body) <= max_len:+ return body+ return body[: max_len - 1].rstrip() + "…"+++def dense_similarity(distance: float) -> float:+ return 1.0 - float(distance)+++def extras_from_metadata(meta: Any) -> dict[str, str] | None:+ if not isinstance(meta, dict) or not meta:+ return None+ out: dict[str, str] = {}+ for key, value in meta.items():+ name = str(key or "").strip()+ if not name or name.lower() in _EXTRAS_SKIP:+ continue+ if value is None or isinstance(value, (dict, list, tuple, set)):+ continue+ text = str(value).strip()+ if not text:+ continue+ out[name] = text+ return out or None+++def merge_hit_extras(*parts: dict[str, Any] | None) -> dict[str, Any] | None:+ merged: dict[str, Any] = {}+ for part in parts:+ if not part:+ continue+ for key, value in part.items():+ if value is None:+ continue+ merged[str(key)] = value+ return merged or None+++def hit_matches_extras(+ have: dict[str, Any] | None,+ wanted: dict[str, str] | None,+) -> bool:+ if not wanted:+ return True+ lookup = {+ str(key).strip().lower(): str(value).strip().lower()+ for key, value in (have or {}).items()+ if value is not None+ }+ for key, value in wanted.items():+ if lookup.get(str(key).strip().lower()) != str(value).strip().lower():+ return False+ return True+++def facet_counts_from_extras(+ extras_list: list[dict[str, Any] | None],+) -> dict[str, dict[str, int]] | None:+ buckets: dict[str, dict[str, int]] = {}+ for extras in extras_list:+ if not extras:+ continue+ for key, value in extras.items():+ name = str(key or "").strip()+ if not name or name.lower() in _EXTRAS_SKIP:+ continue+ if value is None or isinstance(value, (dict, list, tuple, set)):+ continue+ text = str(value).strip()+ if not text:+ continue+ inner = buckets.setdefault(name, {})+ inner[text] = inner.get(text, 0) + 1+ return buckets or None+++def collect_dense_passages(+ vector_search: Any,+ query_vector: list[float],+ brain_id: str,+ k: int,+ extras_out: Optional[dict[str, dict[str, str]]] = None,+) -> tuple[list[str], dict[str, float]]:+ if not query_vector or k <= 0:+ return [], {}+ hits = vector_search.search_data(+ query_vector, brain_id=brain_id, k=k+ )+ ids: list[str] = []+ distances: dict[str, float] = {}+ for vector in hits:+ meta = vector.metadata or {}+ resource_id = meta.get("resource_id") or getattr(vector, "id", None)+ if not resource_id:+ continue+ resource_id = str(resource_id)+ ids.append(resource_id)+ distances[resource_id] = (+ float(vector.distance) if vector.distance is not None else float("inf")+ )+ if extras_out is not None:+ parsed = extras_from_metadata(meta)+ if parsed:+ extras_out[resource_id] = parsed+ return ids, distances+++def collect_bm25_passages(+ data_adapter: Any,+ query: str,+ brain_id: str,+ k: int,+ extras_out: Optional[dict[str, dict[str, str]]] = None,+) -> tuple[list[str], dict[str, float], dict[str, str]]:+ if not query or k <= 0:+ return [], {}, {}+ ranked = data_adapter.search_bm25(query, brain_id, limit=k)+ ids: list[str] = []+ scores: dict[str, float] = {}+ texts: dict[str, str] = {}+ for chunk, score in ranked or []:+ chunk_id = str(getattr(chunk, "id", "") or "")+ if not chunk_id:+ continue+ ids.append(chunk_id)+ scores[chunk_id] = float(score)+ texts[chunk_id] = getattr(chunk, "text", "") or ""+ if extras_out is not None:+ parsed = extras_from_metadata(getattr(chunk, "metadata", None))+ if parsed:+ extras_out[chunk_id] = parsed+ return ids, scores, texts+++def collect_ilike_passages(+ data_adapter: Any,+ query: str,+ brain_id: str,+) -> tuple[list[str], dict[str, str]]:+ ids: list[str] = []+ texts: dict[str, str] = {}+ search_result = data_adapter.search(query, brain_id)+ for chunk in getattr(search_result, "text_chunks", None) or []:+ chunk_id = str(getattr(chunk, "id", "") or "")+ if not chunk_id:+ continue+ ids.append(chunk_id)+ texts[chunk_id] = getattr(chunk, "text", "") or ""+ return ids, texts+++def query_tokens(text: str) -> list[str]:+ found: list[str] = []+ seen: set[str] = set()+ for raw in _TOKEN_RE.findall(text or ""):+ token = raw.lower()+ if len(token) < 3 or token in _LITERAL_STOP or token in seen:+ continue+ seen.add(token)+ found.append(token)+ return found+++def literal_overlap_ids(+ query: str,+ texts_by_id: dict[str, str],+ *,+ k: int = 50,+) -> list[str]:+ qtoks = set(query_tokens(query))+ if not qtoks or k <= 0:+ return []+ scored: list[tuple[int, str]] = []+ for doc_id, body in texts_by_id.items():+ key = str(doc_id or "")+ if not key:+ continue+ overlap = len(qtoks & set(query_tokens(body or "")))+ if overlap <= 0:+ continue+ scored.append((overlap, key))+ scored.sort(key=lambda item: (-item[0], item[1]))+ return [doc_id for _, doc_id in scored[:k]]+++def collect_literal_residual(+ data_adapter: Any,+ query: str,+ brain_id: str,+ k: int,+) -> tuple[list[str], dict[str, str]]:+ getter = getattr(data_adapter, "get_text_chunks", None)+ if getter is None or not query or k <= 0:+ return [], {}+ texts: dict[str, str] = {}+ for token in query_tokens(query):+ try:+ chunks, _total = getter(brain_id, limit=k, query_text=token)+ except TypeError:+ try:+ chunks, _total = getter(brain_id, k, 0, token)+ except Exception:+ continue+ except Exception:+ continue+ for chunk in chunks or []:+ chunk_id = str(getattr(chunk, "id", "") or "")+ if not chunk_id:+ continue+ texts[chunk_id] = getattr(chunk, "text", "") or ""+ return literal_overlap_ids(query, texts, k=k), texts+++def frozen_head_merge(+ ranked_ids: list[str],+ extra_id_lists: Optional[list[list[str]]] = None,+ *,+ head_k: int = 10,+ k: int = 50,+ prefer_ids: Optional[set[str]] = None,+) -> list[str]:+ ranked = [str(item) for item in ranked_ids if item]+ head = ranked[: max(int(head_k), 0)]+ ranked_k = ranked[: max(int(k), 0)]+ passages_k = set(ranked_k)+ extras: list[str] = []+ seen_extra: set[str] = set()+ for extra in extra_id_lists or []:+ for hid in extra[: max(int(k), 0)]:+ key = str(hid)+ if not key or key in seen_extra or key in passages_k:+ continue+ if prefer_ids is not None and key not in prefer_ids:+ continue+ seen_extra.add(key)+ extras.append(key)+ rest = [item for item in ranked_k[len(head) :] if item not in set(head)]+ if prefer_ids is not None:+ fill = (+ extras+ + [item for item in rest if item in prefer_ids]+ + [item for item in rest if item not in prefer_ids]+ )+ else:+ fill = extras + rest+ tail: list[str] = []+ seen = set(head)+ limit = max(int(k), 0)+ for hid in fill:+ if hid in seen:+ continue+ seen.add(hid)+ tail.append(hid)+ if len(head) + len(tail) >= limit:+ break+ return (head + tail)[:limit]+++def _minmax(values: dict[str, float]) -> dict[str, float]:+ if not values:+ return {}+ lo = min(values.values())+ hi = max(values.values())+ if hi <= lo:+ return {key: 1.0 for key in values}+ scale = hi - lo+ return {key: (value - lo) / scale for key, value in values.items()}+++def convex_combination(+ dense_scores: dict[str, float],+ bm25_scores: dict[str, float],+ alpha: float,+) -> list[tuple[str, float]]:+ alpha = min(1.0, max(0.0, float(alpha)))+ dense_n = _minmax(dense_scores)+ bm25_n = _minmax(bm25_scores)+ ids = set(dense_n) | set(bm25_n)+ fused = {+ item: alpha * dense_n.get(item, 0.0) + (1.0 - alpha) * bm25_n.get(item, 0.0)+ for item in ids+ }+ return sorted(fused.items(), key=lambda pair: pair[1], reverse=True)+++def fuse_passage_lists(+ dense_ids: list[str],+ lexical_ids: list[str],+ *,+ fusion: str = "rrf",+ alpha: float = 0.5,+ dense_similarities: Optional[dict[str, float]] = None,+ bm25_scores: Optional[dict[str, float]] = None,+ extra_id_lists: Optional[list[list[str]]] = None,+) -> list[tuple[str, float]]:+ extra = [ids for ids in (extra_id_lists or []) if ids]+ lists = [ids for ids in (dense_ids, lexical_ids) if ids]+ if fusion == "cc" and (dense_ids or lexical_ids):+ core = convex_combination(+ dense_similarities or {},+ bm25_scores or {},+ alpha,+ )+ if not extra:+ return core+ return reciprocal_rank_fusion([[item for item, _ in core], *extra])+ lists.extend(extra)+ if not lists:+ return []+ if len(lists) == 1:+ return [(item, 1.0 / (rank + 1)) for rank, item in enumerate(lists[0])]+ return reciprocal_rank_fusion(lists)diff --git a/src/core/search/personalize.py b/src/core/search/personalize.pynew file mode 100644index 0000000..210d82a--- /dev/null+++ b/src/core/search/personalize.py@@ -0,0 +1,273 @@+from __future__ import annotations++import re+from collections import defaultdict+from typing import Dict, Iterable, List, Optional++from src.constants.kg import Node+from src.core.search.entity_sibilings import _predicate_currently_valid+from src.core.search.recommend import (+ _is_attr_node,+ _prefers_weight,+ behavior_weight,+ recency_decay,+)+from src.services.kg_agent.main import graph_adapter++SHORT_TERM_HALF_LIFE_DAYS = 14.0+ITEM_LABELS = frozenset({"ENTITY", "PRODUCT"})+PREF_PREDICATES = frozenset({"PREFERS", "PREFER", "LIKES"})+_TOKEN_RE = re.compile(r"[A-Za-z0-9]+")+_STOP = frozenset(+ {+ "the",+ "and",+ "for",+ "with",+ "from",+ "this",+ "that",+ "are",+ "was",+ "a",+ "an",+ "of",+ "to",+ "in",+ "on",+ }+)+++def query_personalize_lambda(query: str) -> float:+ tokens = [+ token.lower()+ for token in _TOKEN_RE.findall(query or "")+ if token.lower() not in _STOP+ ]+ if not tokens:+ return 0.0+ if any(any(char.isdigit() for char in token) for token in tokens):+ return 0.0+ count = len(tokens)+ if count == 1:+ return 0.85+ if count == 2:+ return 0.5+ if count == 3:+ return 0.25+ return 0.1+++def _minmax(values: Dict[str, float]) -> Dict[str, float]:+ if not values:+ return {}+ lo = min(values.values())+ hi = max(values.values())+ if hi <= lo:+ return {key: 0.0 for key in values}+ span = hi - lo+ return {key: (value - lo) / span for key, value in values.items()}+++def blend_ranked(+ ids: List[str],+ retrieve_scores: Dict[str, float],+ pref_scores: Dict[str, float],+ lam: float,+) -> tuple[list[str], dict[str, float]]:+ ordered_ids = list(ids)+ if not ordered_ids:+ return [], {}+ lam = max(0.0, min(1.0, float(lam)))+ retrieve_out = {+ item_id: float(retrieve_scores.get(item_id, 0.0)) for item_id in ordered_ids+ }+ if lam <= 0.0 or not any(+ float(pref_scores.get(item_id, 0.0)) for item_id in ordered_ids+ ):+ return ordered_ids, retrieve_out+ retrieve_norm = _minmax(retrieve_out)+ pref_norm = _minmax(+ {item_id: float(pref_scores.get(item_id, 0.0)) for item_id in ordered_ids}+ )+ blended = {+ item_id: (1.0 - lam) * retrieve_norm[item_id] + lam * pref_norm[item_id]+ for item_id in ordered_ids+ }+ ranked = sorted(+ ordered_ids,+ key=lambda item_id: (-blended[item_id], ordered_ids.index(item_id)),+ )+ return ranked, blended+++def _is_item_node(node: Node) -> bool:+ labels = {str(label).upper() for label in (node.labels or [])}+ return bool(labels & ITEM_LABELS)+++def _neighbors(uuid: str, brain_id: str, of_types: Optional[list[str]] = None):+ try:+ payload = graph_adapter.get_neighbors(+ [uuid], of_types=of_types, brain_id=brain_id+ )+ except TypeError:+ payload = graph_adapter.get_neighbors([uuid], brain_id=brain_id)+ except Exception:+ return []+ return payload.get(uuid, []) if isinstance(payload, dict) else []+++def _attr_nodes_from_item(item: Node, brain_id: str) -> List[Node]:+ attrs: List[Node] = []+ for pred, neighbor in _neighbors(item.uuid, brain_id):+ if not _predicate_currently_valid(pred):+ continue+ if _is_attr_node(neighbor):+ attrs.append(neighbor)+ continue+ if "EVENT" in {str(label).upper() for label in (neighbor.labels or [])}:+ for pred2, tip in _neighbors(neighbor.uuid, brain_id):+ if not _predicate_currently_valid(pred2):+ continue+ if _is_attr_node(tip):+ attrs.append(tip)+ return attrs+++def resolve_user_node(target: str, brain_id: str) -> Optional[Node]:+ text = (target or "").strip()+ if not text:+ return None+ candidates = [text]+ if not text.startswith("user:"):+ candidates.append(f"user:{text}")+ try:+ nodes = graph_adapter.get_by_uuids(candidates, brain_id=brain_id) or []+ except Exception:+ nodes = []+ for node in nodes:+ if node is not None and getattr(node, "uuid", None):+ return node+ for uuid in candidates:+ try:+ node = graph_adapter.get_by_uuid(uuid, brain_id=brain_id)+ except Exception:+ node = None+ if node is not None and getattr(node, "uuid", None):+ return node+ return None+++def user_pref_weights(+ seed: Node | None,+ brain_id: str,+ *,+ short_half_life_days: float = SHORT_TERM_HALF_LIFE_DAYS,+ behavior_weights: Optional[Dict[str, float]] = None,+) -> Dict[str, float]:+ scores: Dict[str, float] = defaultdict(float)+ if seed is None or not getattr(seed, "uuid", None):+ return {}+ for pred, neighbor in _neighbors(seed.uuid, brain_id):+ if not _predicate_currently_valid(pred):+ continue+ pred_name = (pred.name or "").upper()+ if pred_name in PREF_PREDICATES or "PREFER" in pred_name or "LIKE" in pred_name:+ if _is_attr_node(neighbor):+ scores[neighbor.uuid] += _prefers_weight(pred)+ for pred, event_node in _neighbors(seed.uuid, brain_id, of_types=["EVENT"]):+ if not _predicate_currently_valid(pred):+ continue+ if "EVENT" not in {str(label).upper() for label in (event_node.labels or [])}:+ continue+ weight = behavior_weight(event_node.name, behavior_weights)+ decay = recency_decay(event_node.happened_at, short_half_life_days)+ for pred2, tip in _neighbors(event_node.uuid, brain_id):+ if not _predicate_currently_valid(pred2):+ continue+ if tip.uuid == seed.uuid:+ continue+ if not _is_item_node(tip):+ continue+ contrib = float(weight) * float(decay)+ for attr in _attr_nodes_from_item(tip, brain_id):+ scores[attr.uuid] += contrib+ return {uuid: float(value) for uuid, value in scores.items() if value}+++def score_nodes_for_user(+ node_ids: Iterable[str],+ prefs: Dict[str, float],+ brain_id: str,+) -> Dict[str, float]:+ if not prefs:+ return {str(node_id): 0.0 for node_id in node_ids if node_id}+ out: Dict[str, float] = {}+ for node_id in node_ids:+ key = str(node_id or "")+ if not key:+ continue+ total = 0.0+ try:+ node = graph_adapter.get_by_uuid(key, brain_id=brain_id)+ except Exception:+ node = None+ if node is None:+ out[key] = 0.0+ continue+ seen: set[str] = set()+ for attr in _attr_nodes_from_item(node, brain_id):+ if attr.uuid in seen:+ continue+ seen.add(attr.uuid)+ total += float(prefs.get(attr.uuid, 0.0))+ out[key] = total+ return out+++def personalize_ranked_ids(+ *,+ query: str,+ ranked_ids: List[str],+ retrieve_scores: Dict[str, float],+ node_id_by_hit: Dict[str, Optional[str]],+ target: Optional[str],+ brain_id: str,+) -> tuple[list[str], dict[str, float]]:+ ids = list(ranked_ids)+ if not target or not ids:+ return ids, {}+ seed = resolve_user_node(target, brain_id)+ if seed is None:+ return ids, {}+ prefs = user_pref_weights(seed, brain_id)+ if not prefs:+ return ids, {}+ unique_nodes = []+ seen_nodes: set[str] = set()+ for hit_id in ids:+ node_id = node_id_by_hit.get(hit_id)+ if not node_id or node_id in seen_nodes:+ continue+ seen_nodes.add(node_id)+ unique_nodes.append(node_id)+ node_prefs = score_nodes_for_user(unique_nodes, prefs, brain_id)+ pref_by_hit = {+ hit_id: float(node_prefs.get(node_id_by_hit.get(hit_id) or "", 0.0))+ for hit_id in ids+ }+ lam = query_personalize_lambda(query)+ return blend_ranked(ids, retrieve_scores, pref_by_hit, lam)+++__all__ = [+ "SHORT_TERM_HALF_LIFE_DAYS",+ "blend_ranked",+ "personalize_ranked_ids",+ "query_personalize_lambda",+ "resolve_user_node",+ "score_nodes_for_user",+ "user_pref_weights",+]diff --git a/src/core/search/recommend.py b/src/core/search/recommend.pyindex 80a0e27..5197f7d 100644--- a/src/core/search/recommend.py+++ b/src/core/search/recommend.py@@ -14,6 +14,8 @@ from src.core.search.entity_sibilings import (from src.utils.dates import parse_date_string, to_naive_utcfrom src.services.kg_agent.main import graph_adapter+UNKNOWN_BEHAVIOR_WEIGHT = 0.2+DEFAULT_BEHAVIOR_WEIGHTS: Dict[str, float] = {"view": 0.2,"click": 0.2,@@ -24,6 +26,19 @@ DEFAULT_BEHAVIOR_WEIGHTS: Dict[str, float] = {"buy": 1.0,"purchased": 1.0,"bought": 1.0,+ "favorite": 0.7,+ "favourite": 0.7,+ "favorites": 0.7,+ "favourites": 0.7,+ "wishlist": 0.7,+ "add_to_favorite": 0.7,+ "addtofavorite": 0.7,+ "add_to_favourite": 0.7,+ "addtofavourite": 0.7,+ "add_to_wishlist": 0.7,+ "addtowishlist": 0.7,+ "follow": 0.5,+ "flw": 0.5,}ATTR_LABELS = frozenset(@@ -46,7 +61,7 @@ def behavior_weight(compact = alias.replace("-", "").replace("_", "").replace(" ", "")if compact == key:return float(value)- return 1.0+ return UNKNOWN_BEHAVIOR_WEIGHTdef recency_decay(diff --git a/src/lib/postgresql/data.py b/src/lib/postgresql/data.pyindex 93affcd..48e83ca 100644--- a/src/lib/postgresql/data.py+++ b/src/lib/postgresql/data.py@@ -16,7 +16,7 @@ import psycopg2import psycopg2.extrasfrom src.adapters.interfaces.data import DataClient, SearchResult-from src.config import config+from src.config import SEARCH_FTS_REGCONFIGS, configfrom src.constants.data import (Brain,KGChanges,@@ -100,6 +100,126 @@ CREATE INDEX IF NOT EXISTS idx_data_kg_changes_timestampON data_kg_changes (timestamp DESC);"""+_SEARCH_DDL = """+ALTER TABLE data_text_chunks+ ADD COLUMN IF NOT EXISTS search_tsv tsvector+ GENERATED ALWAYS AS (to_tsvector('english', coalesce(text, ''))) STORED;+ALTER TABLE data_text_chunks+ ADD COLUMN IF NOT EXISTS search_len integer+ GENERATED ALWAYS AS (+ GREATEST(length(to_tsvector('english', coalesce(text, ''))), 0)+ ) STORED;+CREATE INDEX IF NOT EXISTS idx_data_text_chunks_search_tsv+ ON data_text_chunks USING gin (search_tsv);+"""+++def _search_alt_ddl(regconfig: str) -> str:+ if regconfig not in SEARCH_FTS_REGCONFIGS:+ raise ValueError(f"Unsupported FTS regconfig {regconfig!r}")+ return f"""+ALTER TABLE data_text_chunks+ ADD COLUMN IF NOT EXISTS search_tsv_alt tsvector+ GENERATED ALWAYS AS (to_tsvector('{regconfig}', coalesce(text, ''))) STORED;+ALTER TABLE data_text_chunks+ ADD COLUMN IF NOT EXISTS search_len_alt integer+ GENERATED ALWAYS AS (+ GREATEST(length(to_tsvector('{regconfig}', coalesce(text, ''))), 0)+ ) STORED;+CREATE INDEX IF NOT EXISTS idx_data_text_chunks_search_tsv_alt+ ON data_text_chunks USING gin (search_tsv_alt);+"""+++def _bm25_sql(+ regconfig: str,+ tsv_col: str,+ len_col: str,+ *,+ query_match: str | None = None,+) -> str:+ if tsv_col not in {"search_tsv", "search_tsv_alt"}:+ raise ValueError(f"Unsupported tsv column {tsv_col!r}")+ if len_col not in {"search_len", "search_len_alt"}:+ raise ValueError(f"Unsupported len column {len_col!r}")+ if regconfig not in SEARCH_FTS_REGCONFIGS and regconfig != "english":+ raise ValueError(f"Unsupported FTS regconfig {regconfig!r}")+ if query_match is None:+ query_match = "or" if tsv_col == "search_tsv_alt" else "and"+ if query_match not in {"and", "or"}:+ raise ValueError(f"Unsupported BM25 query_match {query_match!r}")+ if query_match == "and":+ match_sql = f"c.{tsv_col} @@ plainto_tsquery('{regconfig}', %s)"+ else:+ match_sql = "TRUE"+ return f"""+ WITH query_lexemes AS (+ SELECT unnest(tsvector_to_array(to_tsvector('{regconfig}', %s)))+ AS lexeme+ ),+ coll AS (+ SELECT+ COUNT(*)::double precision AS n_docs,+ COALESCE(+ AVG(GREATEST({len_col}, 1)),+ 1+ )::double precision AS avgdl+ FROM data_text_chunks+ ),+ idf AS (+ SELECT+ s.word AS lexeme,+ ln(+ 1.0 + (coll.n_docs - s.ndoc + 0.5)+ / (s.ndoc + 0.5)+ ) AS idf+ FROM ts_stat(+ 'SELECT {tsv_col} FROM data_text_chunks'+ ) AS s+ CROSS JOIN coll+ WHERE s.word IN (SELECT lexeme FROM query_lexemes)+ ),+ docs AS (+ SELECT+ c.id,+ c.document,+ c.text,+ GREATEST(c.{len_col}, 1)::double precision AS dl,+ lex.lexeme,+ COALESCE(+ array_length(lex.positions, 1),+ 1+ )::double precision AS tf+ FROM data_text_chunks c+ CROSS JOIN LATERAL unnest(c.{tsv_col})+ AS lex(lexeme, positions)+ WHERE {match_sql}+ AND lex.lexeme IN (SELECT lexeme FROM query_lexemes)+ )+ SELECT+ d.id,+ d.document,+ d.text,+ SUM(+ COALESCE(i.idf, 0)+ * (d.tf * (%s + 1.0))+ / (+ d.tf+ + %s * (+ 1.0 - %s+ + %s * d.dl+ / NULLIF(c.avgdl, 0)+ )+ )+ ) AS bm25+ FROM docs d+ CROSS JOIN coll c+ LEFT JOIN idf i ON i.lexeme = d.lexeme+ GROUP BY d.id, d.document, d.text+ ORDER BY bm25 DESC, d.id ASC+ LIMIT %s+ """+def _ilike_pattern(query_text: str) -> str:escaped = query_text.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_")@@ -130,6 +250,8 @@ class PostgreSQLDataClient(DataClient):self._lock = threading.Lock()self._system_initialized = Falseself._initialized_brains: set[str] = set()+ self._search_ready_brains: set[str] = set()+ self._search_alt_brains: set[str] = set()@contextmanagerdef _system_connection(self) -> Iterator[psycopg2.extensions.connection]:@@ -159,17 +281,47 @@ class PostgreSQLDataClient(DataClient):self._system_initialized = Truedef _ensure_brain_schema(self, brain_id: str) -> None:- if brain_id in self._initialized_brains:+ alt_regconfig = config.search_fts_regconfig_for_brain(brain_id)+ needs_base = brain_id not in self._initialized_brains+ needs_search = (+ config.search_enabled and brain_id not in self._search_ready_brains+ )+ needs_alt = (+ config.search_enabled+ and alt_regconfig is not None+ and brain_id not in self._search_alt_brains+ )+ if not needs_base and not needs_search and not needs_alt:returnwith self._lock:- if brain_id in self._initialized_brains:+ alt_regconfig = config.search_fts_regconfig_for_brain(brain_id)+ needs_base = brain_id not in self._initialized_brains+ needs_search = (+ config.search_enabled and brain_id not in self._search_ready_brains+ )+ needs_alt = (+ config.search_enabled+ and alt_regconfig is not None+ and brain_id not in self._search_alt_brains+ )+ if not needs_base and not needs_search and not needs_alt:returnensure_brain_database(brain_id)with borrow(get_brain_pool(brain_id)) as conn:with conn.cursor() as cur:- cur.execute(_BRAIN_DDL)+ if needs_base:+ cur.execute(_BRAIN_DDL)+ if needs_search:+ cur.execute(_SEARCH_DDL)+ if needs_alt and alt_regconfig is not None:+ cur.execute(_search_alt_ddl(alt_regconfig))conn.commit()- self._initialized_brains.add(brain_id)+ if needs_base:+ self._initialized_brains.add(brain_id)+ if needs_search:+ self._search_ready_brains.add(brain_id)+ if needs_alt:+ self._search_alt_brains.add(brain_id)def save_text_chunk(self, text_chunk: TextChunk, brain_id: str) -> TextChunk:document = text_chunk.model_dump(mode="json")@@ -279,6 +431,39 @@ class PostgreSQLDataClient(DataClient):]return SearchResult(text_chunks=text_chunks, observations=observations)+ def search_bm25(+ self,+ text: str,+ brain_id: str,+ collection: str = "text_chunks",+ limit: int = 10,+ ) -> List[Tuple[TextChunk, float]]:+ if not text or not text.strip() or limit <= 0:+ return []+ if collection not in ("*", "text_chunks"):+ return []+ k1 = float(config.search_bm25_k1)+ b = float(config.search_bm25_b)+ alt_regconfig = config.search_fts_regconfig_for_brain(brain_id)+ if alt_regconfig:+ sql = _bm25_sql(alt_regconfig, "search_tsv_alt", "search_len_alt")+ params = (text, k1, k1, b, b, limit)+ else:+ sql = _bm25_sql("english", "search_tsv", "search_len")+ params = (text, text, k1, k1, b, b, limit)+ with self._brain_connection(brain_id) as conn:+ with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cur:+ cur.execute(+ sql,+ params,+ )+ rows = cur.fetchall()+ results: List[Tuple[TextChunk, float]] = []+ for row in rows:+ chunk = TextChunk.model_validate(row["document"])+ results.append((chunk, float(row["bm25"] or 0.0)))+ return results+def get_text_chunks_by_ids(self, ids: List[str], with_observations: bool = False, brain_id: str = "default") -> Tuple[List[TextChunk], List[Observation]]:diff --git a/src/lib/postgresql/graph_store.py b/src/lib/postgresql/graph_store.pyindex 0e7b88d..b0fdabf 100644--- a/src/lib/postgresql/graph_store.py+++ b/src/lib/postgresql/graph_store.py@@ -194,25 +194,64 @@ class PostgreSQLGraphStore:ON kg_topic_sessions(session_id);"""+ _SEARCH_DOCUMENT_SQL = (+ "coalesce(data->>'name', '') || ' ' || "+ "coalesce(data->>'description', '') || ' ' || "+ "coalesce(data->>'search_text', '')"+ )+ _SEARCH_DDL = f"""+ ALTER TABLE kg_nodes+ ADD COLUMN IF NOT EXISTS search_tsv tsvector+ GENERATED ALWAYS AS (+ to_tsvector('english'::regconfig, {_SEARCH_DOCUMENT_SQL})+ ) STORED;+ ALTER TABLE kg_nodes+ ADD COLUMN IF NOT EXISTS search_len integer+ GENERATED ALWAYS AS (+ GREATEST(+ length(to_tsvector('english'::regconfig, {_SEARCH_DOCUMENT_SQL})),+ 0+ )+ ) STORED;+ CREATE INDEX IF NOT EXISTS idx_kg_nodes_search_tsv+ ON kg_nodes USING gin (search_tsv);+ """+def __init__(self) -> None:config.postgresql.validate_credentials()self._brains: dict[str, _BrainGraph] = {}self._schema_ready: set[str] = set()+ self._search_ready_brains: set[str] = set()self._schema_lock = threading.Lock()self._brains_lock = threading.RLock()def _ensure_brain_schema(self, brain_id: str) -> None:- if brain_id in self._schema_ready:+ needs_base = brain_id not in self._schema_ready+ needs_search = (+ bool(config.search_enabled) and brain_id not in self._search_ready_brains+ )+ if not needs_base and not needs_search:returnwith self._schema_lock:- if brain_id in self._schema_ready:+ needs_base = brain_id not in self._schema_ready+ needs_search = (+ bool(config.search_enabled)+ and brain_id not in self._search_ready_brains+ )+ if not needs_base and not needs_search:returnensure_brain_database(brain_id)with borrow(get_brain_pool(brain_id)) as conn:- with conn.cursor() as cur:- cur.execute(self._DDL)- conn.commit()- self._schema_ready.add(brain_id)+ if needs_base:+ with conn.cursor() as cur:+ cur.execute(self._DDL)+ conn.commit()+ self._schema_ready.add(brain_id)+ if needs_search:+ with conn.cursor() as cur:+ cur.execute(self._SEARCH_DDL)+ conn.commit()+ self._search_ready_brains.add(brain_id)@contextmanagerdef _connection(self, brain_id: str):@@ -223,6 +262,122 @@ class PostgreSQLGraphStore:def _ensure_brain_row(self, brain_id: str) -> None:self._ensure_brain_schema(brain_id)+ def search_nodes_bm25(+ self,+ text: str,+ brain_id: str,+ *,+ limit: int = 10,+ node_labels: list[str] | None = None,+ node_uuids: list[str] | None = None,+ ) -> list[tuple[str, float, dict]]:+ if not config.search_enabled or not text or not str(text).strip() or limit <= 0:+ return []+ self._ensure_brain_schema(brain_id)+ k1 = float(config.search_bm25_k1)+ b = float(config.search_bm25_b)+ labels = [+ str(item).strip().upper()+ for item in (node_labels or [])+ if str(item).strip()+ ]+ uuids = [str(item).strip() for item in (node_uuids or []) if str(item).strip()]+ sql = """+ WITH query_lexemes AS (+ SELECT unnest(tsvector_to_array(to_tsvector('english', %s)))+ AS lexeme+ ),+ coll AS (+ SELECT+ COUNT(*)::double precision AS n_docs,+ COALESCE(AVG(GREATEST(search_len, 1)), 1)::double precision AS avgdl+ FROM kg_nodes+ ),+ idf AS (+ SELECT+ s.word AS lexeme,+ ln(+ 1.0 + (coll.n_docs - s.ndoc + 0.5)+ / (s.ndoc + 0.5)+ ) AS idf+ FROM ts_stat('SELECT search_tsv FROM kg_nodes') AS s+ CROSS JOIN coll+ WHERE s.word IN (SELECT lexeme FROM query_lexemes)+ ),+ docs AS (+ SELECT+ n.uuid,+ n.data,+ GREATEST(n.search_len, 1)::double precision AS dl,+ lex.lexeme,+ COALESCE(array_length(lex.positions, 1), 1)::double precision AS tf+ FROM kg_nodes n+ CROSS JOIN LATERAL unnest(n.search_tsv) AS lex(lexeme, positions)+ WHERE n.search_tsv @@ plainto_tsquery('english', %s)+ AND lex.lexeme IN (SELECT lexeme FROM query_lexemes)+ AND (%s::text[] IS NULL OR n.uuid = ANY(%s))+ AND (+ %s::text[] IS NULL+ OR EXISTS (+ SELECT 1+ FROM jsonb_array_elements_text(+ coalesce(n.data->'labels', '[]'::jsonb)+ ) AS lbl+ WHERE upper(lbl) = ANY(%s)+ )+ )+ )+ SELECT+ d.uuid,+ d.data,+ SUM(+ COALESCE(i.idf, 0)+ * (d.tf * (%s + 1.0))+ / (+ d.tf+ + %s * (+ 1.0 - %s+ + %s * d.dl / NULLIF(c.avgdl, 0)+ )+ )+ ) AS bm25+ FROM docs d+ CROSS JOIN coll c+ LEFT JOIN idf i ON i.lexeme = d.lexeme+ GROUP BY d.uuid, d.data+ ORDER BY bm25 DESC, d.uuid ASC+ LIMIT %s+ """+ label_param = labels or None+ uuid_param = uuids or None+ params = (+ text,+ text,+ uuid_param,+ uuid_param,+ label_param,+ label_param,+ k1,+ k1,+ b,+ b,+ limit,+ )+ with self._connection(brain_id) as conn:+ with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cur:+ cur.execute(sql, params)+ rows = cur.fetchall()+ out: list[tuple[str, float, dict]] = []+ for row in rows:+ node_id = str(row.get("uuid") or "").strip()+ if not node_id:+ continue+ data = row.get("data") or {}+ if not isinstance(data, dict):+ data = {}+ out.append((node_id, float(row.get("bm25") or 0.0), data))+ return out+def get_brain(self, brain_id: str) -> _BrainGraph:return self._load_brain(brain_id)diff --git a/src/lib/postgresql/networkx_client.py b/src/lib/postgresql/networkx_client.pyindex 2dd14e6..8a20c65 100644--- a/src/lib/postgresql/networkx_client.py+++ b/src/lib/postgresql/networkx_client.py@@ -711,6 +711,40 @@ class NetworkXGraphClient(GraphClient):)return SearchEntitiesResult(results=nodes, total=total)+ def search_nodes_bm25(+ self,+ query_text: str,+ brain_id: str,+ *,+ limit: int = 10,+ node_labels: Optional[list[str]] = None,+ node_uuids: Optional[list[str]] = None,+ ) -> list[tuple[Node, float]]:+ rows = self._store.search_nodes_bm25(+ query_text,+ brain_id,+ limit=limit,+ node_labels=node_labels,+ node_uuids=node_uuids,+ )+ hits: list[tuple[Node, float]] = []+ for node_uuid, score, data in rows:+ labels = [str(item) for item in (data.get("labels") or []) if str(item).strip()]+ properties = {k: v for k, v in data.items() if k != "labels"}+ hits.append(+ (+ Node(+ uuid=str(node_uuid),+ name=str(data.get("name") or node_uuid),+ labels=labels,+ description=data.get("description"),+ properties=properties,+ ),+ float(score),+ )+ )+ return hits+def deprecate_relationship(self,subject: Node,diff --git a/src/lib/postgresql/vectors.py b/src/lib/postgresql/vectors.pyindex e8b5402..5e77a32 100644--- a/src/lib/postgresql/vectors.py+++ b/src/lib/postgresql/vectors.py@@ -20,6 +20,7 @@ import psycopg2.extrasfrom pgvector.psycopg2 import register_vectorfrom src.adapters.interfaces.embeddings import VectorStoreClient+from src.config import configfrom src.constants.embeddings import EMBEDDING_STORES_SIZES, Vectorfrom ._provisioning import borrow, ensure_brain_database, get_brain_pool@@ -57,14 +58,34 @@ def _table_name(store: str) -> str:return f"vectors_{_safe_store(store)}"-def _vector_index_ddl(table: str, dimension: int) -> str:+def _vector_index_ddl(+ table: str, dimension: int, *, search_enabled: bool | None = None+) -> str:dim = int(dimension)- if dim > 2000:- return ""- return f"""+ if search_enabled is None:+ search_enabled = bool(config.search_enabled)+ if dim <= 2000:+ return f"""CREATE INDEX IF NOT EXISTS idx_{table}_embeddingsON {table} USING hnsw (embeddings vector_cosine_ops);"""+ if not search_enabled:+ return ""+ return f"""+ CREATE INDEX IF NOT EXISTS idx_{table}_embeddings_halfvec+ ON {table} USING hnsw+ ((embeddings::halfvec({dim})) halfvec_cosine_ops);+ """+++def _uses_halfvec_ann(dimension: int, *, search_enabled: bool | None = None) -> bool:+ if search_enabled is None:+ search_enabled = bool(config.search_enabled)+ try:+ dim = int(dimension)+ except (TypeError, ValueError):+ return False+ return search_enabled and dim > 2000def _table_vector_dimension(cur: psycopg2.extensions.cursor, table: str) -> Optional[int]:@@ -132,6 +153,7 @@ class PostgreSQLVectorStoreClient(VectorStoreClient):returndimension = EMBEDDING_STORES_SIZES[store]table = _table_name(store)+ index_ddl = _vector_index_ddl(table, dimension)ddl = f"""CREATE TABLE IF NOT EXISTS {table} (id BIGINT PRIMARY KEY,@@ -141,7 +163,7 @@ class PostgreSQLVectorStoreClient(VectorStoreClient):);CREATE INDEX IF NOT EXISTS idx_{table}_uuidON {table} (uuid);- {_vector_index_ddl(table, dimension)}+ {index_ddl}"""with self._connection(brain_id) as conn:with conn.cursor() as cur:@@ -155,7 +177,17 @@ class PostgreSQLVectorStoreClient(VectorStoreClient):dimension,)cur.execute(f"DROP TABLE IF EXISTS {table} CASCADE")- cur.execute(ddl)+ try:+ cur.execute(ddl)+ except Exception as exc:+ if _uses_halfvec_ann(dimension):+ raise RuntimeError(+ "SEARCH_ENABLED=true requires a halfvec HNSW index "+ f"on {table} (dimension {dimension} > 2000). "+ "The vector backend could not create it: "+ f"{exc}"+ ) from exc+ raiseconn.commit()self._initialized_stores.add(key)@@ -212,6 +244,15 @@ class PostgreSQLVectorStoreClient(VectorStoreClient):return []table = _table_name(store)fetch_k = ann_overfetch_k(k)+ dimension = EMBEDDING_STORES_SIZES[store]+ use_halfvec = _uses_halfvec_ann(dimension)+ if use_halfvec:+ order_sql = (+ f"embeddings::halfvec({dimension}) "+ f"<=> (%s::vector)::halfvec({dimension})"+ )+ else:+ order_sql = "embeddings <=> %s::vector"with self._connection(brain_id) as conn:with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cur:cur.execute(@@ -223,7 +264,7 @@ class PostgreSQLVectorStoreClient(VectorStoreClient):SELECT id, uuid, metadata,(embeddings <=> %s::vector) AS distanceFROM {table}- ORDER BY embeddings <=> %s::vector, uuid ASC, id ASC+ ORDER BY {order_sql}, uuid ASC, id ASCLIMIT %s""",(data_vector, data_vector, fetch_k),diff --git a/src/services/api/constants/requests.py b/src/services/api/constants/requests.pyindex f71ea48..855c736 100644--- a/src/services/api/constants/requests.py+++ b/src/services/api/constants/requests.py@@ -107,10 +107,30 @@ class IngestionTripleSet(BaseModel):None,description="The optional event predicate of the triple.",)- event: RequestPartialNode- event_obj: RequestPartialPredicate+ event: Optional[RequestPartialNode] = Field(+ None,+ description="The optional event node. Omit for a direct subject-predicate-object edge.",+ )+ event_obj: Optional[RequestPartialPredicate] = Field(+ None,+ description="The optional event-to-object predicate. Paired with event when present.",+ )object: RequestPartialNode+ @model_validator(mode="after")+ def require_event_pair_or_direct_edge(self):+ has_event = self.event is not None+ has_event_obj = self.event_obj is not None+ if has_event != has_event_obj:+ raise ValueError("event and event_obj must both be set or both omitted")+ if has_event:+ return self+ if not self.subject or not self.subj_event:+ raise ValueError(+ "direct triples require subject, subj_event, and object"+ )+ return self+class PartialNodeFilter(BaseModel):"""@@ -469,3 +489,74 @@ class GetContextResponse(BaseModel):paths: Optional[List[dict[str, Any]]] = Nonetopics: Optional[List[dict[str, Any]]] = Nonestage_timings: Optional[dict[str, Any]] = None+++class SearchRequestBody(BaseModel):+ query: str+ brain_id: str = "default"+ k: int = Field(10, ge=1, le=200)+ channels: List[str] = Field(default_factory=lambda: ["passages"])+ node_labels: Optional[List[str]] = None+ community_labels: Optional[List[str]] = None+ expand: Literal["none", "neighbors"] = "none"+ fusion: Optional[Literal["rrf", "cc"]] = None+ fusion_alpha: Optional[float] = Field(default=None, ge=0.0, le=1.0)+ rerank: Optional[str] = Field(+ default=None,+ description="none or plugin:<name>. Unknown plugin names return 400.",+ )+ mode: Literal["default", "catalog"] = "default"+ profile_stages: bool = False+ extras: Optional[Dict[str, str]] = None+ target: Optional[str] = None++ @field_validator("target", mode="before")+ @classmethod+ def empty_target(cls, value):+ if value is None:+ return None+ text = str(value).strip()+ return text or None++ @field_validator("extras", mode="before")+ @classmethod+ def coerce_extras_filter(cls, value):+ if value is None or value == {}:+ return None+ if not isinstance(value, dict):+ raise ValueError("extras must be an object of string keys and values")+ return {+ str(key): str(item)+ for key, item in value.items()+ if item is not None+ } or None+++class SearchHitScores(BaseModel):+ bm25: Optional[float] = None+ dense: Optional[float] = None+ rrf: Optional[float] = None+ cc: Optional[float] = None+ rerank: Optional[float] = None+ plugin: Optional[dict[str, float]] = None+ graph: Optional[float] = None+ personalize: Optional[float] = None+++class SearchHit(BaseModel):+ id: str+ channel: str+ score: float+ scores: SearchHitScores+ snippet: str+ labels: List[str] = Field(default_factory=list)+ extras: Optional[dict[str, Any]] = None+ node_id: Optional[str] = None+++class SearchResponse(BaseModel):+ hits: List[SearchHit]+ stage_timings: Optional[dict[str, Any]] = None+ channel_lists: Optional[dict[str, list[str]]] = None+ facets: Optional[dict[str, dict[str, int]]] = None+ node_ids: List[str] = Field(default_factory=list)diff --git a/src/services/api/controllers/retrieve.py b/src/services/api/controllers/retrieve.pyindex a62434f..086c70b 100644--- a/src/services/api/controllers/retrieve.py+++ b/src/services/api/controllers/retrieve.py@@ -26,6 +26,12 @@ from src.core.search.fact_filter import (personalized_pagerank,reciprocal_rank_fusion,)+from src.core.search.hybrid import (+ collect_bm25_passages,+ collect_dense_passages,+ collect_ilike_passages,+ fuse_passage_lists,+)from src.core.search.relationships import search_relationshipsfrom src.lib.tracing.profiler import profile_request, profile_stagefrom src.utils.vector_search import VectorSearchFacade@@ -43,6 +49,7 @@ from src.services.kg_agent.main import embeddings_adapter, vector_store_adapterfrom src.utils.dates import parse_date_string, to_naive_utcfrom src.utils.similarity.vectors import cosine_similarityfrom src.utils.nlp.ner import _entity_extractor+from src.config import configvector_search = VectorSearchFacade(vector_store_adapter)@@ -1681,45 +1688,52 @@ def _retrieve_passages(text: str, brain_id: str, *, limit: int) -> list[tuple[str, float, str]]:"""Return (chunk_id, score, text) ranked passages via vector + keyword fusion."""+ mode = (+ config.context_passage_mode if config.search_enabled else "ilike"+ )+ use_dense = mode in ("hybrid", "dense", "ilike")+ use_bm25 = mode in ("hybrid", "bm25")+ use_ilike = mode == "ilike"with profile_stage("passages.retrieve", blocking=True, queries=1):- with profile_stage("embed.query"):- text_embeddings = embeddings_adapter.embed_text(text)- with profile_stage("vector.search_data", k=max(limit, _PASSAGE_K)):- vector_hits = vector_search.search_data(- text_embeddings.embeddings,- brain_id=brain_id,- k=max(limit, _PASSAGE_K),- )vector_ids: list[str] = []- id_to_text: dict[str, str] = {}id_to_distance: dict[str, float] = {}- for vector in vector_hits:- meta = vector.metadata or {}- resource_id = meta.get("resource_id") or getattr(vector, "id", None)- if not resource_id:- continue- resource_id = str(resource_id)- vector_ids.append(resource_id)- id_to_distance[resource_id] = (- float(vector.distance)- if vector.distance is not None- else float("inf")- )+ if use_dense:+ with profile_stage("embed.query"):+ text_embeddings = embeddings_adapter.embed_text(text)+ with profile_stage("vector.search_data", k=max(limit, _PASSAGE_K)):+ vector_ids, id_to_distance = collect_dense_passages(+ vector_search,+ text_embeddings.embeddings,+ brain_id,+ max(limit, _PASSAGE_K),+ )keyword_ids: list[str] = []- with profile_stage("data.keyword_search"):- try:- search_result = data_adapter.search(text, brain_id)- for chunk in getattr(search_result, "text_chunks", None) or []:- chunk_id = str(getattr(chunk, "id", "") or "")- if not chunk_id:- continue- keyword_ids.append(chunk_id)- id_to_text[chunk_id] = getattr(chunk, "text", "") or ""- except Exception:- pass+ id_to_text: dict[str, str] = {}+ if use_bm25:+ with profile_stage("data.bm25_search"):+ try:+ keyword_ids, _, id_to_text = collect_bm25_passages(+ data_adapter,+ text,+ brain_id,+ max(limit, _PASSAGE_K),+ )+ except Exception:+ pass+ elif use_ilike:+ with profile_stage("data.keyword_search"):+ try:+ keyword_ids, id_to_text = collect_ilike_passages(+ data_adapter, text, brain_id+ )+ except Exception:+ pass- fused = reciprocal_rank_fusion([vector_ids, keyword_ids])+ fused = fuse_passage_lists(+ vector_ids if use_dense else [],+ keyword_ids,+ )ranked_ids = [item for item, _ in fused[:limit]]missing = [cid for cid in ranked_ids if cid not in id_to_text]if missing:diff --git a/src/services/api/controllers/search.py b/src/services/api/controllers/search.pynew file mode 100644index 0000000..65e4459--- /dev/null+++ b/src/services/api/controllers/search.py@@ -0,0 +1,514 @@+import asyncio+from typing import Any, Optional++from fastapi import HTTPException++from src.config import config+from src.core.search.catalog_graph import node_id_from_passage_text+from src.core.search.personalize import personalize_ranked_ids+from src.core.search.graph_channels import (+ COMMUNITIES_CHANNEL,+ ENTITIES_CHANNEL,+ EVENTS_CHANNEL,+ GraphHit,+ NEIGHBORS_CHANNEL,+ PASSAGES_CHANNEL,+ collect_community_hits,+ collect_entity_hits,+ collect_event_hits,+ expand_neighbor_hits,+ is_item_entity,+ selected_graph_channels,+)+from src.core.search.hooks import (+ SearchPluginError,+ rerank_max_k_for_mode,+ resolve_reranker,+ resolve_retrievers,+ retrieve_k_for_mode,+)+from src.core.search.hybrid import (+ collect_bm25_passages,+ collect_dense_passages,+ collect_literal_residual,+ dense_similarity,+ extras_from_metadata,+ facet_counts_from_extras,+ frozen_head_merge,+ fuse_passage_lists,+ hit_matches_extras,+ merge_hit_extras,+ passage_snippet,+)+from src.lib.tracing.profiler import profile_request, profile_stage+from src.services.api.constants.requests import (+ SearchHit,+ SearchHitScores,+ SearchRequestBody,+ SearchResponse,+)+from src.services.data.main import data_adapter+from src.services.kg_agent.main import embeddings_adapter, graph_adapter, vector_store_adapter+from src.utils.vector_search import VectorSearchFacade++vector_search = VectorSearchFacade(vector_store_adapter)+++def _require_search_enabled() -> None:+ if not config.search_enabled:+ raise HTTPException(+ status_code=404,+ detail="Search is disabled. Set SEARCH_ENABLED=true to enable /retrieve/search.",+ )+++def _plugin_http(exc: SearchPluginError) -> HTTPException:+ return HTTPException(status_code=400, detail=str(exc))+++def _fetch_snippets(ids: list[str], known: dict[str, str], brain_id: str) -> dict[str, str]:+ texts, _ = _fetch_chunk_fields(ids, known, {}, brain_id)+ return texts+++def _fetch_chunk_fields(+ ids: list[str],+ known_texts: dict[str, str],+ known_extras: dict[str, dict[str, Any]],+ brain_id: str,+) -> tuple[dict[str, str], dict[str, dict[str, Any]]]:+ texts = dict(known_texts)+ extras: dict[str, dict[str, Any]] = dict(known_extras)+ missing = [+ chunk_id+ for chunk_id in ids+ if chunk_id not in texts or chunk_id not in extras+ ]+ if not missing:+ return texts, extras+ try:+ chunks, _ = data_adapter.get_text_chunks_by_ids(missing, False, brain_id)+ except Exception:+ return texts, extras+ if not isinstance(chunks, (list, tuple)):+ return texts, extras+ for chunk in chunks:+ chunk_id = str(getattr(chunk, "id", "") or "")+ if not chunk_id:+ continue+ if chunk_id not in texts:+ texts[chunk_id] = getattr(chunk, "text", "") or ""+ parsed = extras_from_metadata(getattr(chunk, "metadata", None))+ if parsed:+ extras[chunk_id] = merge_hit_extras(extras.get(chunk_id), parsed) or parsed+ elif chunk_id not in extras:+ extras[chunk_id] = {}+ return texts, extras+++def _run_dense(+ query_vector: list[float],+ brain_id: str,+ k: int,+ extras_out: dict[str, dict[str, str]] | None = None,+):+ with profile_stage("search.dense"):+ return collect_dense_passages(+ vector_search, query_vector, brain_id, k, extras_out=extras_out+ )+++def _run_bm25(+ query: str,+ brain_id: str,+ k: int,+ extras_out: dict[str, dict[str, str]] | None = None,+):+ with profile_stage("search.bm25"):+ return collect_bm25_passages(+ data_adapter, query, brain_id, k, extras_out=extras_out+ )+++def _want_passages(channels: list[str] | None) -> bool:+ if not channels:+ return True+ return any(str(item).strip().lower() == PASSAGES_CHANNEL for item in channels)+++def _collect_graph_hits(+ request: SearchRequestBody,+ *,+ query_vector: list[float] | None,+ graph_channels: list[str],+) -> list[GraphHit]:+ k = request.k+ brain_id = request.brain_id+ fanout = int(getattr(config, "search_neighbor_fanout", 50) or 50)+ community_labels = request.community_labels+ if community_labels is None:+ community_labels = list(config.search_community_labels)+ hits: list[GraphHit] = []+ if ENTITIES_CHANNEL in graph_channels:+ hits.extend(+ collect_entity_hits(+ query=request.query,+ brain_id=brain_id,+ k=k,+ graph=graph_adapter,+ vector_search=vector_search,+ query_vector=query_vector,+ node_labels=request.node_labels,+ )+ )+ if EVENTS_CHANNEL in graph_channels:+ hits.extend(+ collect_event_hits(+ query=request.query,+ brain_id=brain_id,+ k=k,+ graph=graph_adapter,+ vector_search=vector_search,+ query_vector=query_vector,+ )+ )+ if COMMUNITIES_CHANNEL in graph_channels:+ hits.extend(+ collect_community_hits(+ query=request.query,+ brain_id=brain_id,+ k=k,+ graph=graph_adapter,+ vector_search=vector_search,+ query_vector=query_vector,+ community_labels=community_labels,+ fanout=fanout,+ )+ )+ if request.expand == "neighbors" and hits:+ hits.extend(+ expand_neighbor_hits(+ hits,+ brain_id=brain_id,+ k=k,+ graph=graph_adapter,+ community_labels=community_labels,+ fanout=fanout,+ )+ )+ return hits+++async def search(request: SearchRequestBody) -> SearchResponse:+ _require_search_enabled()+ with profile_request(+ "retrieve.search", enabled=request.profile_stages+ ) as profiler:+ response = await _search(request)+ if profiler is not None:+ response.stage_timings = profiler.last_report+ return response+++async def _search(request: SearchRequestBody) -> SearchResponse:+ try:+ reranker = resolve_reranker(request.rerank)+ plugin_retrievers = resolve_retrievers(request.channels)+ except SearchPluginError as exc:+ raise _plugin_http(exc) from exc++ k = request.k+ k_ret = retrieve_k_for_mode(request.mode, k)+ rerank_max = rerank_max_k_for_mode(request.mode)+ brain_id = request.brain_id+ fusion = request.fusion or config.search_fusion+ fusion_alpha = (+ request.fusion_alpha+ if request.fusion_alpha is not None+ else config.search_fusion_alpha+ )+ want_passages = _want_passages(request.channels)+ graph_channels = selected_graph_channels(request.channels)+ want_graph = bool(graph_channels)+ use_dense = want_passages and config.search_use_dense+ use_bm25 = want_passages and config.search_use_bm25+ query_vector: Optional[list[float]] = None+ if use_dense or want_graph:+ with profile_stage("embed.query", blocking=False):+ embedded = await asyncio.to_thread(+ embeddings_adapter.embed_text, request.query+ )+ query_vector = list(embedded.embeddings or [])++ dense_extras: dict[str, dict[str, str]] = {}+ bm25_extras: dict[str, dict[str, str]] = {}+ retrieve_jobs = []+ if use_dense:+ retrieve_jobs.append(+ asyncio.to_thread(+ _run_dense, query_vector or [], brain_id, k_ret, dense_extras+ )+ )+ if use_bm25:+ retrieve_jobs.append(+ asyncio.to_thread(+ _run_bm25, request.query, brain_id, k_ret, bm25_extras+ )+ )++ dense_ids: list[str] = []+ dense_distances: dict[str, float] = {}+ bm25_ids: list[str] = []+ bm25_scores: dict[str, float] = {}+ texts: dict[str, str] = {}+ plugin_id_lists: list[list[str]] = []+ plugin_scores: dict[str, dict[str, float]] = {}+ plugin_channels: dict[str, str] = {}+ graph_hits: list[GraphHit] = []+ literal_ids: list[str] = []+ with profile_stage("search.retrieve", blocking=False):+ gathered = await asyncio.gather(*retrieve_jobs) if retrieve_jobs else []+ offset = 0+ if use_dense:+ dense_ids, dense_distances = gathered[offset]+ offset += 1+ if use_bm25:+ bm25_ids, bm25_scores, texts = gathered[offset]+ for name, retriever in plugin_retrievers:+ with profile_stage(f"search.plugin.{name}"):+ ids, scores, plugin_texts = await asyncio.to_thread(+ retriever, request.query, brain_id, k_ret+ )+ plugin_id_lists.append(ids)+ plugin_scores[name] = scores+ texts.update(plugin_texts)+ for item_id in ids:+ plugin_channels.setdefault(str(item_id), f"plugin:{name}")+ if want_graph:+ try:+ graph_hits = await asyncio.to_thread(+ _collect_graph_hits,+ request,+ query_vector=query_vector,+ graph_channels=graph_channels,+ )+ except Exception:+ graph_hits = []+ if getattr(config, "search_literal_fill", False):+ lit_ids, literal_texts = collect_literal_residual(+ data_adapter, request.query, brain_id, k_ret+ )+ literal_ids = list(lit_ids)+ texts.update(literal_texts)+ for item_id in literal_ids:+ plugin_channels.setdefault(str(item_id), PASSAGES_CHANNEL)++ passage_extras = {**dense_extras, **bm25_extras}+ dense_sims = {+ chunk_id: dense_similarity(distance)+ for chunk_id, distance in dense_distances.items()+ }+ extra_id_lists: list[list[str]] = []+ graph_by_channel: dict[str, list[str]] = {}+ graph_by_id: dict[str, GraphHit] = {}+ for hit in graph_hits:+ graph_by_channel.setdefault(hit.channel, []).append(hit.id)+ current = graph_by_id.get(hit.id)+ if current is None or hit.score > current.score:+ graph_by_id[hit.id] = hit++ def _item_ids(channel_name: str) -> list[str]:+ ids = graph_by_channel.get(channel_name) or []+ if channel_name != EVENTS_CHANNEL:+ ids = [+ item_id+ for item_id in ids+ if is_item_entity(+ item_id,+ graph_by_id[item_id].labels if item_id in graph_by_id else [],+ )+ ]+ return ids++ graph_channel_ids = {+ ENTITIES_CHANNEL: _item_ids(ENTITIES_CHANNEL),+ EVENTS_CHANNEL: _item_ids(EVENTS_CHANNEL),+ COMMUNITIES_CHANNEL: _item_ids(COMMUNITIES_CHANNEL),+ NEIGHBORS_CHANNEL: _item_ids(NEIGHBORS_CHANNEL),+ }+ for ids in graph_channel_ids.values():+ if ids:+ extra_id_lists.append(ids)+ for item_id in ids:+ texts.setdefault(item_id, graph_by_id[item_id].snippet)+ fused = fuse_passage_lists(+ dense_ids if use_dense else [],+ bm25_ids if use_bm25 else [],+ fusion=fusion,+ alpha=fusion_alpha,+ dense_similarities=dense_sims,+ bm25_scores=bm25_scores,+ extra_id_lists=extra_id_lists,+ )+ ranked_ids = [item for item, _ in fused]+ sidecar_lists = [ids for ids in plugin_id_lists if ids]+ if literal_ids:+ sidecar_lists.append(literal_ids)+ if sidecar_lists:+ ranked_ids = frozen_head_merge(+ ranked_ids,+ sidecar_lists,+ head_k=10,+ k=k_ret,+ )+ else:+ ranked_ids = ranked_ids[:k_ret]+ rerank_scores: dict[str, float] = {}+ if reranker is not None and ranked_ids:+ head_n = min(rerank_max, len(ranked_ids))+ head_ids = ranked_ids[:head_n]+ tail_ids = ranked_ids[head_n:]+ with profile_stage("search.rerank", blocking=False, chunks=head_n):+ texts = await asyncio.to_thread(+ _fetch_snippets, head_ids, texts, brain_id+ )+ candidates = [+ {+ "id": chunk_id,+ "text": texts.get(chunk_id) or "",+ "score": float(dict(fused).get(chunk_id, 0.0)),+ }+ for chunk_id in head_ids+ ]+ reranked = await asyncio.to_thread(+ reranker, request.query, candidates, head_n+ )+ ordered: list[str] = []+ seen: set[str] = set()+ for item in reranked or []:+ chunk_id = str((item or {}).get("id") or "")+ if not chunk_id or chunk_id not in head_ids or chunk_id in seen:+ continue+ ordered.append(chunk_id)+ seen.add(chunk_id)+ if item.get("score") is not None:+ rerank_scores[chunk_id] = float(item["score"])+ for chunk_id in head_ids:+ if chunk_id not in seen:+ ordered.append(chunk_id)+ ranked_ids = ordered + tail_ids+ ranked_ids = ranked_ids[:k_ret]+ fused_map = dict(fused)++ extras_by_id: dict[str, dict[str, Any]] = dict(passage_extras)+ for item_id, graph_hit in graph_by_id.items():+ extras_by_id[item_id] = (+ merge_hit_extras(extras_by_id.get(item_id), graph_hit.extras) or {}+ )++ with profile_stage("search.snippets", blocking=False, chunks=len(ranked_ids)):+ texts, extras_by_id = await asyncio.to_thread(+ _fetch_chunk_fields, ranked_ids, texts, extras_by_id, brain_id+ )++ if request.extras:+ ranked_ids = [+ item_id+ for item_id in ranked_ids+ if hit_matches_extras(extras_by_id.get(item_id), request.extras)+ ]+ node_id_by_hit: dict[str, str | None] = {}+ for item_id in ranked_ids:+ graph_hit = graph_by_id.get(item_id)+ if graph_hit is not None and is_item_entity(graph_hit.id, graph_hit.labels):+ node_id_by_hit[item_id] = graph_hit.id+ else:+ node_id_by_hit[item_id] = node_id_from_passage_text(+ texts.get(item_id) or ""+ )+ personalize_scores: dict[str, float] = {}+ if request.target:+ retrieve_for_blend = {+ item_id: float(+ rerank_scores.get(item_id, fused_map.get(item_id, 0.0))+ )+ for item_id in ranked_ids+ }+ ranked_ids, personalize_scores = personalize_ranked_ids(+ query=request.query,+ ranked_ids=ranked_ids,+ retrieve_scores=retrieve_for_blend,+ node_id_by_hit=node_id_by_hit,+ target=request.target,+ brain_id=brain_id,+ )+ ranked_ids = ranked_ids[:k]++ channel_of: dict[str, str] = {}+ for chunk_id in list(dense_ids) + list(bm25_ids):+ channel_of.setdefault(chunk_id, PASSAGES_CHANNEL)+ for hit in graph_hits:+ channel_of.setdefault(hit.id, hit.channel)+ for chunk_id, plugin_channel in plugin_channels.items():+ channel_of.setdefault(chunk_id, plugin_channel)++ hits: list[SearchHit] = []+ for chunk_id in ranked_ids:+ body = texts.get(chunk_id) or ""+ rrf = fused_map.get(chunk_id)+ plugin = {+ name: scores[chunk_id]+ for name, scores in plugin_scores.items()+ if chunk_id in scores+ }+ graph_hit = graph_by_id.get(chunk_id)+ extras = merge_hit_extras(extras_by_id.get(chunk_id), graph_hit.extras if graph_hit else None)+ scores = SearchHitScores(+ bm25=bm25_scores.get(chunk_id),+ dense=dense_sims.get(chunk_id),+ rrf=rrf if fusion == "rrf" else None,+ cc=rrf if fusion == "cc" else None,+ rerank=rerank_scores.get(chunk_id),+ plugin=plugin or None,+ graph=graph_hit.score if graph_hit is not None else None,+ personalize=personalize_scores.get(chunk_id),+ )+ node_id = node_id_by_hit.get(chunk_id)+ hits.append(+ SearchHit(+ id=chunk_id,+ channel=channel_of.get(chunk_id, PASSAGES_CHANNEL),+ score=float(+ personalize_scores.get(+ chunk_id,+ rerank_scores.get(chunk_id, fused_map.get(chunk_id, 0.0)),+ )+ ),+ scores=scores,+ snippet=passage_snippet(body),+ labels=list(graph_hit.labels) if graph_hit is not None else [],+ extras=extras,+ node_id=node_id,+ )+ )+ node_ids: list[str] = []+ seen_nodes: set[str] = set()+ for hit in hits:+ if not hit.node_id or hit.node_id in seen_nodes:+ continue+ seen_nodes.add(hit.node_id)+ node_ids.append(hit.node_id)+ return SearchResponse(+ hits=hits,+ facets=facet_counts_from_extras([hit.extras for hit in hits]),+ channel_lists={+ "dense": list(dense_ids),+ "bm25": list(bm25_ids),+ "entities": list(graph_channel_ids[ENTITIES_CHANNEL]),+ "events": list(graph_channel_ids[EVENTS_CHANNEL]),+ "communities": list(graph_channel_ids[COMMUNITIES_CHANNEL]),+ "neighbors": list(graph_channel_ids[NEIGHBORS_CHANNEL]),+ "literal": list(literal_ids),+ },+ node_ids=node_ids,+ )diff --git a/src/services/api/routes/retrieve.py b/src/services/api/routes/retrieve.pyindex d3e0c7d..a57eff7 100644--- a/src/services/api/routes/retrieve.py+++ b/src/services/api/routes/retrieve.py@@ -21,6 +21,8 @@ from src.services.api.constants.requests import (RetrieveNeighborsWithIdentificationParamsRequestBody,RetrieveRequestResponse,RetrieveNeighborsRequestResponse,+ SearchRequestBody,+ SearchResponse,)from src.services.api.controllers.retrieve import (retrieve_neighbors as retrieve_neighbors_controller,@@ -33,6 +35,7 @@ from src.services.api.controllers.kg import get_hops as get_hops_controllerfrom src.services.api.controllers.retrieve import (retrieve_data as retrieve_data_controller,)+from src.services.api.controllers.search import search as search_controllerfrom src.services.api.controllers.structured_data import (get_structured_data_by_id as get_structured_data_by_id_controller,get_structured_data_list as get_structured_data_list_controller,@@ -594,6 +597,83 @@ async def post_recommend()+@retrieve_router.get(path="/search", response_model=SearchResponse)+async def get_search(+ query: str = Query(..., description="The search query."),+ k: int = Query(10, ge=1, le=200, description="Number of hits to return."),+ channels: Optional[str] = Query(+ "passages",+ description=(+ "Comma-separated channels: passages, entities, events, "+ "communities, and/or plugin:<name>."+ ),+ ),+ node_labels: Optional[str] = Query(+ None,+ description="Comma-separated node labels to filter the entities channel.",+ ),+ community_labels: Optional[str] = Query(+ None,+ description="Comma-separated hub labels for the communities channel.",+ ),+ expand: Literal["none", "neighbors"] = Query(+ "none",+ description="Optional 1-hop expansion from graph channel seeds.",+ ),+ fusion: Optional[Literal["rrf", "cc"]] = Query(+ None, description="Fusion override. Default is SEARCH_FUSION."+ ),+ rerank: Optional[str] = Query(+ None,+ description="none or plugin:<name>. Unknown plugin names return 400.",+ ),+ target: Optional[str] = Query(+ None,+ description="Optional USER uuid or id for query-gated rerank of retrieved hits.",+ ),+ profile_stages: bool = Query(False),+ brain_id: str = Depends(get_brain_id),+):+ channel_list = [+ item.strip()+ for item in (channels or "passages").split(",")+ if item.strip()+ ] or ["passages"]+ node_label_list = [+ item.strip()+ for item in (node_labels or "").split(",")+ if item.strip()+ ] or None+ community_label_list = [+ item.strip()+ for item in (community_labels or "").split(",")+ if item.strip()+ ] or None+ request = SearchRequestBody(+ query=query,+ k=k,+ channels=channel_list,+ node_labels=node_label_list,+ community_labels=community_label_list,+ expand=expand,+ fusion=fusion,+ rerank=rerank,+ target=target,+ profile_stages=profile_stages,+ brain_id=brain_id,+ )+ return await search_controller(request)+++@retrieve_router.post(path="/search", response_model=SearchResponse)+async def post_search(+ body: SearchRequestBody,+ brain_id: str = Depends(get_brain_id),+):+ body.brain_id = brain_id or body.brain_id+ return await search_controller(body)++@retrieve_router.get(path="/entity/status")async def get_entity_status(target: str,diff --git a/src/workers/tasks/ingestion.py b/src/workers/tasks/ingestion.pyindex 069d2ec..c14c864 100644--- a/src/workers/tasks/ingestion.py+++ b/src/workers/tasks/ingestion.py@@ -656,6 +656,16 @@ def ingest_data(self, args: dict):with track_stage(cost_ledger, "embed"):cost_ledger.embed.add_usage(calls=1)+ if payload.skip_enrichment:+ set_ingestion_task_status(+ self.request.id,+ payload.brain_id,+ "completed",+ stage="completed",+ cost=cost_ledger.to_dict(),+ )+ return self.request.id+if config.pipeline_mode == "accurate" and config.run_observations:with track_stage(cost_ledger, "observations"):observations = observations_agent.observe(@@ -967,19 +977,26 @@ def process_architect_relationships(self, args: dict):print(f"> Processing node {node_data.name}")try:future.result(timeout=180)+ props = {+ k: v+ for k, v in (node_data.properties or {}).items()+ if v is not None+ }+ labels = [node_data.type]+ extra = props.get("catalog_labels") or []+ if isinstance(extra, str):+ extra = [extra]+ for item in extra:+ label = str(item).strip()+ if label and label not in labels:+ labels.append(label)graph_nodes.append(Node(uuid=node_data.uuid,- labels=[node_data.type],+ labels=labels,name=node_data.name,description=node_data.description,- properties={- k: v- for k, v in (- node_data.properties or {}- ).items()- if v is not None- },+ properties=props,polarity=(node_data.polarityif node_data.polaritydiff --git a/tests/test_catalog_graph_search.py b/tests/test_catalog_graph_search.pynew file mode 100644index 0000000..ebedcf4--- /dev/null+++ b/tests/test_catalog_graph_search.py@@ -0,0 +1,102 @@+import os+import unittest+from unittest.mock import MagicMock++ENV_DEFAULTS = {+ "BRAINPAT_TOKEN": "test-token",+ "MODELS_MODE": "local",+ "EMBEDDINGS_LOCAL_MODEL": "local-model",+ "EMBEDDINGS_SMALL_MODEL": "small-model",+ "EMBEDDING_NODES_DIMENSION": "3",+ "EMBEDDING_TRIPLETS_DIMENSION": "3",+ "EMBEDDING_OBSERVATIONS_DIMENSION": "3",+ "EMBEDDING_DATA_DIMENSION": "3",+ "EMBEDDING_RELATIONSHIPS_DIMENSION": "3",+ "REDIS_HOST": "localhost",+ "REDIS_PORT": "6379",+ "NEO4J_HOST": "localhost",+ "NEO4J_PORT": "7687",+ "NEO4J_USERNAME": "neo4j",+ "NEO4J_PASSWORD": "password",+ "MILVUS_HOST": "localhost",+ "MILVUS_PORT": "19530",+ "MONGO_CONNECTION_STRING": "mongodb://localhost:27017",+ "CELERY_WORKER_CONCURRENCY": "1",+ "OLLAMA_HOST": "localhost",+ "OLLAMA_PORT": "11434",+ "OLLAMA_LLM_SMALL_MODEL": "small",+ "OLLAMA_LLM_LARGE_MODEL": "large",+}+for key, value in ENV_DEFAULTS.items():+ os.environ.setdefault(key, value)++from src.constants.embeddings import Vector+from src.core.agents.scout_agent import ScoutEntity+from src.core.search.catalog_graph import node_embed_text+from src.core.saving.ingestion_manager import IngestionManager+from src.lib.postgresql.graph_store import PostgreSQLGraphStore+from src.services.api.constants.requests import SearchRequestBody+++class CatalogGraphSearchUnitTests(unittest.TestCase):+ def test_search_request_body_stays_passages_without_catalog_fields(self):+ body = SearchRequestBody(query="modern sofas")+ self.assertEqual(body.channels, ["passages"])+ fields = set(SearchRequestBody.model_fields)+ self.assertIn("target", fields)+ for banned in ("product_id", "sku", "brand", "user_id"):+ self.assertNotIn(banned, fields)++ def test_node_embed_text_uses_search_text_only_when_set(self):+ catalog = ScoutEntity(+ type="ENTITY",+ name="sofa",+ uuid="sofa-1",+ properties={"search_text": "velvet sofa navy modern"},+ )+ text, cache_key = node_embed_text(catalog)+ self.assertEqual(text, "velvet sofa navy modern")+ self.assertEqual(cache_key, "uuid:sofa-1")+ memory = ScoutEntity(type="PERSON", name="Ada", uuid="p1")+ mem_text, mem_key = node_embed_text(memory)+ self.assertEqual(mem_text, "Ada")+ self.assertEqual(mem_key, "Ada")++ def test_process_node_vectors_embeds_search_text(self):+ embeddings = MagicMock()+ embeddings.embed_text.return_value = Vector(+ id="v1", embeddings=[0.1, 0.2], metadata={}+ )+ store = MagicMock()+ store.add_vectors.return_value = ["vid-1"]+ manager = IngestionManager(embeddings, store, MagicMock())+ node = ScoutEntity(+ type="ENTITY",+ name="sofa",+ uuid="sofa-1",+ properties={"search_text": "velvet sofa for small rooms"},+ )+ manager.process_node_vectors(node, "searchbenchsmoke")+ embeddings.embed_text.assert_called_once_with("velvet sofa for small rooms")+ meta = embeddings.embed_text.return_value.metadata+ self.assertEqual(meta["uuid"], "sofa-1")+ self.assertEqual(meta["name"], "sofa")+ memory = ScoutEntity(type="PERSON", name="Ada", uuid="p1")+ manager.process_node_vectors(memory, "searchbenchsmoke")+ embeddings.embed_text.assert_called_with("Ada")++ def test_node_search_ddl_is_english_and_gated(self):+ ddl = PostgreSQLGraphStore._SEARCH_DDL+ self.assertIn("kg_nodes", ddl)+ self.assertIn("search_tsv", ddl)+ self.assertIn("to_tsvector", ddl)+ self.assertIn("'english'", ddl)+ self.assertNotIn("'italian'", ddl)+ self.assertIn("data->>'search_text'", ddl)+ self.assertNotIn("concat_ws", ddl)+ self.assertIn("::regconfig", ddl)+ self.assertIn("||", ddl)+++if __name__ == "__main__":+ unittest.main()diff --git a/tests/test_context_retrieval.py b/tests/test_context_retrieval.pyindex b57f106..d7c4709 100644--- a/tests/test_context_retrieval.py+++ b/tests/test_context_retrieval.py@@ -3,6 +3,7 @@ import osimport sysimport typesimport unittest+from types import SimpleNamespacefrom unittest.mock import MagicMock, patch@@ -41,6 +42,7 @@ for key, value in ENV_DEFAULTS.items():sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))from src.constants.embeddings import Vector+from src.constants.data import TextChunkfrom src.constants.kg import Node, Predicatefrom src.core.search.entity_info import EventSynergyRetrieverfrom src.services.api.constants.requests import GetContextRequestBody@@ -483,5 +485,99 @@ class EventSynergyHardeningTests(unittest.TestCase):self.assertLessEqual(work[0], 3)+class ContextPassageModeTests(unittest.TestCase):+ def test_search_off_keeps_ilike_lexical_leg(self):+ from src.services.api.controllers import retrieve as retrieve_mod++ mock_embeddings = MagicMock()+ mock_embeddings.embed_text.return_value = Vector(+ id="q", embeddings=[0.1], metadata={}+ )+ mock_vs = MagicMock()+ mock_vs.search_data.return_value = [+ Vector(id="d", metadata={"resource_id": "dense-1"}, distance=0.2)+ ]+ ilike_chunk = TextChunk(id="ilike-1", text="ilike hit about license")+ mock_data = MagicMock()+ mock_data.search.return_value = SimpleNamespace(text_chunks=[ilike_chunk])+ mock_data.get_text_chunks_by_ids.return_value = (+ [TextChunk(id="dense-1", text="dense body")],+ [],+ )++ with (+ patch.object(retrieve_mod, "embeddings_adapter", mock_embeddings),+ patch.object(retrieve_mod, "vector_search", mock_vs),+ patch.object(retrieve_mod, "data_adapter", mock_data),+ patch.object(retrieve_mod.config, "search_enabled", False),+ patch.object(retrieve_mod.config, "context_passage_mode", "hybrid"),+ ):+ hits = retrieve_mod._retrieve_passages("license", "brain-a", limit=8)++ mock_data.search.assert_called()+ mock_data.search_bm25.assert_not_called()+ self.assertTrue(any(item[0] == "ilike-1" for item in hits))++ def test_search_on_hybrid_uses_bm25(self):+ from src.services.api.controllers import retrieve as retrieve_mod++ mock_embeddings = MagicMock()+ mock_embeddings.embed_text.return_value = Vector(+ id="q", embeddings=[0.1], metadata={}+ )+ mock_vs = MagicMock()+ mock_vs.search_data.return_value = [+ Vector(id="d", metadata={"resource_id": "dense-1"}, distance=0.2)+ ]+ bm25_chunk = TextChunk(id="bm25-1", text="bm25 hit about license")+ mock_data = MagicMock()+ mock_data.search_bm25.return_value = [(bm25_chunk, 5.0)]+ mock_data.get_text_chunks_by_ids.return_value = (+ [TextChunk(id="dense-1", text="dense body")],+ [],+ )++ with (+ patch.object(retrieve_mod, "embeddings_adapter", mock_embeddings),+ patch.object(retrieve_mod, "vector_search", mock_vs),+ patch.object(retrieve_mod, "data_adapter", mock_data),+ patch.object(retrieve_mod.config, "search_enabled", True),+ patch.object(retrieve_mod.config, "context_passage_mode", "hybrid"),+ ):+ hits = retrieve_mod._retrieve_passages("license", "brain-a", limit=8)++ mock_data.search_bm25.assert_called()+ mock_data.search.assert_not_called()+ ids = [item[0] for item in hits]+ self.assertIn("bm25-1", ids)+ self.assertIn("dense-1", ids)++ def test_search_on_ilike_freezes_old_lexical_leg(self):+ from src.services.api.controllers import retrieve as retrieve_mod++ mock_embeddings = MagicMock()+ mock_embeddings.embed_text.return_value = Vector(+ id="q", embeddings=[0.1], metadata={}+ )+ mock_vs = MagicMock()+ mock_vs.search_data.return_value = []+ ilike_chunk = TextChunk(id="ilike-2", text="legacy ilike")+ mock_data = MagicMock()+ mock_data.search.return_value = SimpleNamespace(text_chunks=[ilike_chunk])++ with (+ patch.object(retrieve_mod, "embeddings_adapter", mock_embeddings),+ patch.object(retrieve_mod, "vector_search", mock_vs),+ patch.object(retrieve_mod, "data_adapter", mock_data),+ patch.object(retrieve_mod.config, "search_enabled", True),+ patch.object(retrieve_mod.config, "context_passage_mode", "ilike"),+ ):+ hits = retrieve_mod._retrieve_passages("license", "brain-a", limit=8)++ mock_data.search.assert_called()+ mock_data.search_bm25.assert_not_called()+ self.assertEqual(hits[0][0], "ilike-2")++if __name__ == "__main__":unittest.main()diff --git a/tests/test_halfvec_ann.py b/tests/test_halfvec_ann.pynew file mode 100644index 0000000..5ee34c1--- /dev/null+++ b/tests/test_halfvec_ann.py@@ -0,0 +1,182 @@+import os+import unittest+from contextlib import contextmanager+from unittest.mock import MagicMock, patch++ENV_DEFAULTS = {+ "BRAINPAT_TOKEN": "test-token",+ "MODELS_MODE": "local",+ "EMBEDDINGS_LOCAL_MODEL": "local-model",+ "EMBEDDINGS_SMALL_MODEL": "small-model",+ "EMBEDDING_NODES_DIMENSION": "3",+ "EMBEDDING_TRIPLETS_DIMENSION": "3",+ "EMBEDDING_OBSERVATIONS_DIMENSION": "3",+ "EMBEDDING_DATA_DIMENSION": "3",+ "EMBEDDING_RELATIONSHIPS_DIMENSION": "3",+ "REDIS_HOST": "localhost",+ "REDIS_PORT": "6379",+ "NEO4J_HOST": "localhost",+ "NEO4J_PORT": "7687",+ "NEO4J_USERNAME": "neo4j",+ "NEO4J_PASSWORD": "password",+ "MILVUS_HOST": "localhost",+ "MILVUS_PORT": "19530",+ "MONGO_CONNECTION_STRING": "mongodb://localhost:27017",+ "CELERY_WORKER_CONCURRENCY": "1",+ "OLLAMA_HOST": "localhost",+ "OLLAMA_PORT": "11434",+ "OLLAMA_LLM_SMALL_MODEL": "small",+ "OLLAMA_LLM_LARGE_MODEL": "large",+}+for key, value in ENV_DEFAULTS.items():+ os.environ.setdefault(key, value)++from src.constants.embeddings import Vector+from src.lib.postgresql import vectors as vectors_mod+from src.utils.vector_search import stable_top_k_vectors+++def _postgres_available() -> bool:+ try:+ import psycopg2++ from src.config import config++ pg = config.postgresql+ pg.validate_credentials()+ conn = psycopg2.connect(+ host=pg.host,+ port=pg.port,+ user=pg.username,+ password=pg.password,+ dbname=pg.maintenance_database or "postgres",+ connect_timeout=2,+ )+ conn.close()+ return True+ except Exception:+ return False+++class HalfvecAnnTests(unittest.TestCase):+ def test_3072_creates_halfvec_hnsw_when_search_on(self):+ ddl = vectors_mod._vector_index_ddl(+ "vectors_data", 3072, search_enabled=True+ )+ self.assertIn("halfvec(3072)", ddl)+ self.assertIn("hnsw", ddl)+ self.assertIn("halfvec_cosine_ops", ddl)++ def test_3072_skips_index_when_search_off(self):+ ddl = vectors_mod._vector_index_ddl(+ "vectors_data", 3072, search_enabled=False+ )+ self.assertEqual(ddl.strip(), "")++ def test_under_2000_keeps_float32_hnsw(self):+ ddl = vectors_mod._vector_index_ddl(+ "vectors_data", 1536, search_enabled=True+ )+ self.assertIn("vector_cosine_ops", ddl)+ self.assertNotIn("halfvec", ddl)++ def test_overfetch_float32_rerank_matches_exact_topk(self):+ ann_order = [+ Vector(id="b", metadata={"uuid": "b", "resource_id": "b"}, distance=0.05),+ Vector(id="a", metadata={"uuid": "a", "resource_id": "a"}, distance=0.01),+ Vector(id="c", metadata={"uuid": "c", "resource_id": "c"}, distance=0.20),+ ]+ reranked = stable_top_k_vectors(ann_order, 2)+ self.assertEqual([v.id for v in reranked], ["a", "b"])++ def test_search_vectors_orders_by_halfvec_and_reranks_float32(self):+ client = vectors_mod.PostgreSQLVectorStoreClient.__new__(+ vectors_mod.PostgreSQLVectorStoreClient+ )+ executed: list[str] = []++ class Cursor:+ def execute(self, sql, params=None):+ executed.append(str(sql))++ def fetchall(self):+ return [+ {+ "id": 2,+ "uuid": "b",+ "metadata": {"resource_id": "b"},+ "distance": 0.2,+ },+ {+ "id": 1,+ "uuid": "a",+ "metadata": {"resource_id": "a"},+ "distance": 0.05,+ },+ ]++ @contextmanager+ def fake_connection(_brain_id):+ conn = MagicMock()+ cur = Cursor()++ @contextmanager+ def fake_cursor(**_kwargs):+ yield cur++ conn.cursor.side_effect = lambda **kwargs: fake_cursor()+ yield conn++ with (+ patch.object(client, "_ensure_store"),+ patch.object(client, "_connection", fake_connection),+ patch.object(vectors_mod, "EMBEDDING_STORES_SIZES", {"data": 3072}),+ patch.object(vectors_mod.config, "search_enabled", True),+ ):+ hits = client.search_vectors([0.0] * 3072, "brain", "data", k=1)++ joined = "\n".join(executed)+ self.assertIn("halfvec(3072)", joined)+ self.assertIn("embeddings <=> %s::vector", joined)+ self.assertEqual([h.id for h in hits], ["a"])++ @unittest.skipUnless(_postgres_available(), "postgres not available")+ def test_postgres_creates_halfvec_hnsw_for_3072(self):+ import psycopg2.extras++ from src.lib.postgresql._naming import brain_db_name+ from src.lib.postgresql._provisioning import borrow, get_brain_pool++ brain_id = "searchbench_halfvec_unit"+ sizes = {+ "nodes": 3072,+ "triplets": 3072,+ "observations": 3072,+ "data": 3072,+ "relationships": 3072,+ }+ client = vectors_mod.PostgreSQLVectorStoreClient()+ with (+ patch.object(vectors_mod.config, "search_enabled", True),+ patch.object(vectors_mod, "EMBEDDING_STORES_SIZES", sizes),+ ):+ client._initialized_stores.clear()+ client._ensure_store("data", brain_id)+ with borrow(get_brain_pool(brain_id)) as conn:+ with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cur:+ cur.execute(+ """+ SELECT indexdef+ FROM pg_indexes+ WHERE tablename = 'vectors_data'+ AND indexname = 'idx_vectors_data_embeddings_halfvec'+ """+ )+ row = cur.fetchone()+ self.assertIsNotNone(row)+ self.assertIn("halfvec", row["indexdef"])+ self.assertIn("hnsw", row["indexdef"].lower())+++if __name__ == "__main__":+ unittest.main()diff --git a/tests/test_ingestion_standard.py b/tests/test_ingestion_standard.pyindex 47a483c..be2c2c5 100644--- a/tests/test_ingestion_standard.py+++ b/tests/test_ingestion_standard.py@@ -48,6 +48,13 @@ class SourceTextHelperTests(unittest.TestCase):data={"data_type": "text", "text_data": "hello world"},brain_id="brain1",)+ self.assertFalse(payload.skip_enrichment)+ skipped = IngestionTaskArgs(+ data={"data_type": "text", "text_data": "catalog row"},+ brain_id="brain1",+ skip_enrichment=True,+ )+ self.assertTrue(skipped.skip_enrichment)self.assertEqual(source_text_from_payload(payload), "hello world")def test_source_text_from_json_payload(self):@@ -115,7 +122,7 @@ class IngestDataObservationAndEnrichmentTests(unittest.TestCase):)return ["obs-1"]- def fake_enrich(text, targeting=None, brain_id="default"):+ def fake_enrich(text, targeting=None, brain_id="default", **kwargs):from src.core.saving.auto_kg import EnrichmentOrchestrationResultenrich_calls.append({"text": text, "brain_id": brain_id})@@ -143,6 +150,7 @@ class IngestDataObservationAndEnrichmentTests(unittest.TestCase):patch.object(ingestion_mod, "observations_agent") as observations_agent,patch.object(ingestion_mod, "enrich_kg_from_input", side_effect=fake_enrich),patch.object(ingestion_mod.config, "pipeline_mode", "accurate"),+ patch.object(ingestion_mod.config, "run_observations", True),patch.object(ingestion_mod, "set_ingestion_task_status"),patch("celery.chain") as chain_mock,):@@ -171,6 +179,65 @@ class IngestDataObservationAndEnrichmentTests(unittest.TestCase):self.assertEqual(enrich_calls[0]["text"], expected_text)self.assertEqual(enrich_calls[0]["brain_id"], "tenant-a")+ def test_skip_enrichment_writes_chunk_without_kg(self):+ from src.constants.embeddings import Vector+ from src.workers.tasks import ingestion as ingestion_mod++ enrich_calls = []+ observe_calls = []++ def fake_save_text_chunk(chunk, brain_id="default"):+ return chunk++ def fake_enrich(*args, **kwargs):+ enrich_calls.append(kwargs)+ raise AssertionError("enrichment should be skipped")++ def fake_observe(**kwargs):+ observe_calls.append(kwargs)+ raise AssertionError("observations should be skipped")++ task = ingestion_mod.ingest_data+ request = self._make_request()+ args = {+ "data": {"data_type": "text", "text_data": "DOCID p1. Title: kettle"},+ "brain_id": "searchbenchesci",+ "skip_enrichment": True,+ }++ with (+ patch.object(ingestion_mod, "data_adapter") as data_adapter,+ patch.object(ingestion_mod, "embeddings_adapter") as embeddings_adapter,+ patch.object(ingestion_mod, "vector_store_adapter") as vector_store_adapter,+ patch.object(ingestion_mod, "cache_adapter"),+ patch.object(ingestion_mod, "observations_agent") as observations_agent,+ patch.object(ingestion_mod, "enrich_kg_from_input", side_effect=fake_enrich),+ patch.object(ingestion_mod.config, "pipeline_mode", "accurate"),+ patch.object(ingestion_mod, "set_ingestion_task_status") as set_status,+ ):+ data_adapter.save_text_chunk.side_effect = fake_save_text_chunk+ embeddings_adapter.embed_text.return_value = Vector(+ id="v1", embeddings=[0.1, 0.2], metadata={}+ )+ observations_agent.observe.side_effect = fake_observe+ vector_store_adapter.add_vectors.return_value = ["v1"]+ result = task.run.__func__(+ type("Bound", (), {"request": request})(),+ args,+ )++ self.assertEqual(result, "task-123")+ self.assertEqual(observe_calls, [])+ self.assertEqual(enrich_calls, [])+ self.assertTrue(data_adapter.save_text_chunk.called)+ self.assertTrue(embeddings_adapter.embed_text.called)+ completed = [+ call+ for call in set_status.call_args_list+ if call.args and call.args[2] == "completed"+ ]+ self.assertTrue(completed)+def test_invalid_pipeline_mode_fails_ingest(self):from src.workers.tasks import ingestion as ingestion_moddiff --git a/tests/test_ingestion_structured_contract.py b/tests/test_ingestion_structured_contract.pyindex 0c8124a..ff55362 100644--- a/tests/test_ingestion_structured_contract.py+++ b/tests/test_ingestion_structured_contract.py@@ -51,6 +51,31 @@ class StructuredRequestValidationTests(unittest.TestCase):self.assertIsNone(body.anchor)self.assertIsNone(body.text)+ def test_direct_has_triple_validates(self):+ from src.services.api.constants.requests import IngestionTripleSet++ triple = IngestionTripleSet(+ subject={"name": "bed", "type": "ENTITY", "uuid": "0"},+ subj_event={"name": "HAS", "uuid": "rel-1"},+ object={"name": "navy", "type": "ATTR", "uuid": "hub-navy"},+ )+ self.assertIsNone(triple.event)+ self.assertIsNone(triple.event_obj)+ prefers = IngestionTripleSet(+ subject={"name": "u01", "type": "USER", "uuid": "user:u01"},+ subj_event={"name": "PREFERS", "uuid": "rel-pref"},+ object={"name": "70s", "type": "ATTR", "uuid": "hub:attr:70s"},+ )+ self.assertIsNone(prefers.event)+ self.assertEqual(prefers.subj_event.name, "PREFERS")+ with self.assertRaises(ValidationError):+ IngestionTripleSet(+ subject={"name": "bed", "type": "ENTITY", "uuid": "0"},+ subj_event={"name": "HAS"},+ event={"name": "HAS", "type": "EVENT"},+ object={"name": "navy", "type": "ATTR"},+ )+def test_string_anchor_is_rejected(self):from src.services.api.constants.requests import IngestionStructuredRequestBodydiff --git a/tests/test_recommend_recsys.py b/tests/test_recommend_recsys.pyindex 45a6b16..1485226 100644--- a/tests/test_recommend_recsys.py+++ b/tests/test_recommend_recsys.py@@ -76,8 +76,22 @@ class BehaviorWeightTests(unittest.TestCase):self.assertEqual(behavior_weight("View"), 0.2)self.assertEqual(behavior_weight("AddToCart"), 0.5)- def test_unknown_defaults_to_one(self):- self.assertEqual(behavior_weight("WeirdEvent"), 1.0)+ def test_unknown_defaults_to_view_weight(self):+ self.assertEqual(behavior_weight("WeirdEvent"), 0.2)++ def test_favorite_between_cart_and_purchase(self):+ self.assertEqual(behavior_weight("Favorite"), 0.7)+ self.assertEqual(behavior_weight("wishlist"), 0.7)+ self.assertEqual(behavior_weight("add_to_favorite"), 0.7)+ self.assertGreater(behavior_weight("Favorite"), behavior_weight("AddToCart"))+ self.assertLess(behavior_weight("Favorite"), behavior_weight("Purchase"))++ def test_follow_is_half(self):+ self.assertEqual(behavior_weight("Follow"), 0.5)+ self.assertEqual(behavior_weight("FLW"), 0.5)+ self.assertEqual(behavior_weight("follow"), 0.5)+ self.assertGreater(behavior_weight("Follow"), behavior_weight("WeirdEvent"))+ self.assertLess(behavior_weight("Follow"), behavior_weight("Favorite"))def test_custom_table(self):self.assertEqual(behavior_weight("View", {"view": 0.9}), 0.9)diff --git a/tests/test_search_api.py b/tests/test_search_api.pynew file mode 100644index 0000000..07feb93--- /dev/null+++ b/tests/test_search_api.py@@ -0,0 +1,393 @@+import os+import unittest+from unittest.mock import MagicMock, patch++ENV_DEFAULTS = {+ "BRAINPAT_TOKEN": "test-token",+ "MODELS_MODE": "local",+ "EMBEDDINGS_LOCAL_MODEL": "local-model",+ "EMBEDDINGS_SMALL_MODEL": "small-model",+ "EMBEDDING_NODES_DIMENSION": "3",+ "EMBEDDING_TRIPLETS_DIMENSION": "3",+ "EMBEDDING_OBSERVATIONS_DIMENSION": "3",+ "EMBEDDING_DATA_DIMENSION": "3",+ "EMBEDDING_RELATIONSHIPS_DIMENSION": "3",+ "REDIS_HOST": "localhost",+ "REDIS_PORT": "6379",+ "NEO4J_HOST": "localhost",+ "NEO4J_PORT": "7687",+ "NEO4J_USERNAME": "neo4j",+ "NEO4J_PASSWORD": "password",+ "MILVUS_HOST": "localhost",+ "MILVUS_PORT": "19530",+ "MONGO_CONNECTION_STRING": "mongodb://localhost:27017",+ "CELERY_WORKER_CONCURRENCY": "1",+ "OLLAMA_HOST": "localhost",+ "OLLAMA_PORT": "11434",+ "OLLAMA_LLM_SMALL_MODEL": "small",+ "OLLAMA_LLM_LARGE_MODEL": "large",+}+for key, value in ENV_DEFAULTS.items():+ os.environ.setdefault(key, value)++from fastapi import HTTPException++from src.constants.embeddings import Vector+from src.constants.data import TextChunk+from src.services.api.constants.requests import SearchRequestBody+++class SearchApiTests(unittest.IsolatedAsyncioTestCase):+ async def test_404_when_search_disabled(self):+ from src.services.api.controllers import search as search_mod++ with patch.object(search_mod.config, "search_enabled", False):+ with self.assertRaises(HTTPException) as ctx:+ await search_mod.search(SearchRequestBody(query="license"))+ self.assertEqual(ctx.exception.status_code, 404)++ async def test_fused_hits_when_both_channels_on(self):+ from src.services.api.controllers import search as search_mod++ dense_vec = Vector(+ id="dense-1",+ metadata={"resource_id": "chunk-dense"},+ distance=0.1,+ )+ bm25_chunk = TextChunk(id="chunk-bm25", text="counseling license renewal")+ mock_embeddings = MagicMock()+ mock_embeddings.embed_text.return_value = Vector(+ id="q", embeddings=[0.1, 0.2], metadata={}+ )+ mock_vs = MagicMock()+ mock_vs.search_data.return_value = [dense_vec]+ mock_data = MagicMock()+ mock_data.search_bm25.return_value = [(bm25_chunk, 4.2)]+ mock_data.get_text_chunks_by_ids.return_value = (+ [TextChunk(id="chunk-dense", text="dense passage about alice")],+ [],+ )++ with (+ patch.object(search_mod.config, "search_enabled", True),+ patch.object(search_mod.config, "search_use_dense", True),+ patch.object(search_mod.config, "search_use_bm25", True),+ patch.object(search_mod.config, "search_fusion", "rrf"),+ patch.object(search_mod, "embeddings_adapter", mock_embeddings),+ patch.object(search_mod, "vector_search", mock_vs),+ patch.object(search_mod, "data_adapter", mock_data),+ ):+ response = await search_mod.search(+ SearchRequestBody(query="license", k=10, profile_stages=True)+ )++ ids = [hit.id for hit in response.hits]+ self.assertIn("chunk-dense", ids)+ self.assertIn("chunk-bm25", ids)+ self.assertTrue(any(hit.scores.rrf for hit in response.hits))+ mock_embeddings.embed_text.assert_called_once_with("license")+ self.assertIsNotNone(response.stage_timings)+ stage_names = [s["stage"] for s in response.stage_timings.get("stages", [])]+ self.assertIn("embed.query", stage_names)+ self.assertTrue(+ any(name.startswith("search.") for name in stage_names),+ stage_names,+ )++ async def test_dense_only_skips_bm25(self):+ from src.services.api.controllers import search as search_mod++ mock_embeddings = MagicMock()+ mock_embeddings.embed_text.return_value = Vector(+ id="q", embeddings=[0.1], metadata={}+ )+ mock_vs = MagicMock()+ mock_vs.search_data.return_value = [+ Vector(id="d1", metadata={"resource_id": "only-dense"}, distance=0.2)+ ]+ mock_data = MagicMock()+ mock_data.get_text_chunks_by_ids.return_value = (+ [TextChunk(id="only-dense", text="vector hit")],+ [],+ )++ with (+ patch.object(search_mod.config, "search_enabled", True),+ patch.object(search_mod.config, "search_use_dense", True),+ patch.object(search_mod.config, "search_use_bm25", False),+ patch.object(search_mod, "embeddings_adapter", mock_embeddings),+ patch.object(search_mod, "vector_search", mock_vs),+ patch.object(search_mod, "data_adapter", mock_data),+ ):+ response = await search_mod.search(SearchRequestBody(query="alice", k=5))++ self.assertEqual([hit.id for hit in response.hits], ["only-dense"])+ mock_data.search_bm25.assert_not_called()+ self.assertIsNotNone(response.hits[0].scores.dense)++ async def test_bm25_only_skips_embed(self):+ from src.services.api.controllers import search as search_mod++ mock_embeddings = MagicMock()+ mock_data = MagicMock()+ mock_data.search_bm25.return_value = [+ (TextChunk(id="lex", text="license board"), 3.1)+ ]+ mock_vs = MagicMock()++ with (+ patch.object(search_mod.config, "search_enabled", True),+ patch.object(search_mod.config, "search_use_dense", False),+ patch.object(search_mod.config, "search_use_bm25", True),+ patch.object(search_mod, "embeddings_adapter", mock_embeddings),+ patch.object(search_mod, "vector_search", mock_vs),+ patch.object(search_mod, "data_adapter", mock_data),+ ):+ response = await search_mod.search(SearchRequestBody(query="license", k=5))++ self.assertEqual([hit.id for hit in response.hits], ["lex"])+ mock_embeddings.embed_text.assert_not_called()+ mock_vs.search_data.assert_not_called()+ self.assertEqual(response.hits[0].scores.bm25, 3.1)+ self.assertIsNone(response.hits[0].node_id)+ self.assertEqual(response.node_ids, [])++ async def test_passage_docid_sets_node_id(self):+ from src.services.api.controllers import search as search_mod++ mock_embeddings = MagicMock()+ mock_data = MagicMock()+ mock_data.search_bm25.return_value = [+ (+ TextChunk(+ id="chunk-bed",+ text="DOCID 0.\nTitle: solid wood platform bed\nClass: Beds",+ ),+ 5.0,+ )+ ]+ mock_vs = MagicMock()++ with (+ patch.object(search_mod.config, "search_enabled", True),+ patch.object(search_mod.config, "search_use_dense", False),+ patch.object(search_mod.config, "search_use_bm25", True),+ patch.object(search_mod, "embeddings_adapter", mock_embeddings),+ patch.object(search_mod, "vector_search", mock_vs),+ patch.object(search_mod, "data_adapter", mock_data),+ ):+ response = await search_mod.search(+ SearchRequestBody(query="platform bed", k=5)+ )++ self.assertEqual([hit.id for hit in response.hits], ["chunk-bed"])+ self.assertEqual(response.hits[0].channel, "passages")+ self.assertEqual(response.hits[0].node_id, "0")+ self.assertEqual(response.node_ids, ["0"])++ def test_default_search_mode_is_default(self):+ body = SearchRequestBody(query="license")+ self.assertEqual(body.mode, "default")+ self.assertEqual(body.k, 10)+ self.assertIsNone(body.extras)++ async def test_extras_equality_filter_and_facets(self):+ from src.services.api.controllers import search as search_mod+ from src.core.search.hybrid import (+ facet_counts_from_extras,+ hit_matches_extras,+ )++ self.assertTrue(hit_matches_extras({"locale": "it"}, {"locale": "IT"}))+ self.assertFalse(hit_matches_extras({"color": "argento"}, {"color": "nope"}))+ self.assertEqual(+ facet_counts_from_extras(+ [+ {"locale": "it", "color": "argento", "brand": "CasaLuce"},+ {"locale": "it", "color": "blu navy", "brand": "AtelierNord"},+ None,+ ]+ ),+ {+ "locale": {"it": 2},+ "color": {"argento": 1, "blu navy": 1},+ "brand": {"CasaLuce": 1, "AtelierNord": 1},+ },+ )+ self.assertIsNone(facet_counts_from_extras([None, {}]))+ self.assertEqual(+ facet_counts_from_extras(+ [{"locale": "it", "uuid": "skip-me", "resource_id": "chunk"}]+ ),+ {"locale": {"it": 1}},+ )++ mock_data = MagicMock()+ mock_data.search_bm25.return_value = [+ (+ TextChunk(+ id="it-1",+ text="bollitore acciaio",+ metadata={"locale": "it", "color": "argento", "brand": "CasaLuce"},+ ),+ 3.0,+ ),+ (+ TextChunk(+ id="it-2",+ text="divano velluto",+ metadata={"locale": "it", "color": "blu navy", "brand": "AtelierNord"},+ ),+ 2.0,+ ),+ ]+ mock_data.get_text_chunks_by_ids.return_value = ([], [])+ mock_embeddings = MagicMock()+ mock_vs = MagicMock()++ with (+ patch.object(search_mod.config, "search_enabled", True),+ patch.object(search_mod.config, "search_use_dense", False),+ patch.object(search_mod.config, "search_use_bm25", True),+ patch.object(search_mod, "embeddings_adapter", mock_embeddings),+ patch.object(search_mod, "vector_search", mock_vs),+ patch.object(search_mod, "data_adapter", mock_data),+ ):+ kept = await search_mod.search(+ SearchRequestBody(query="divano", k=10, extras={"locale": "it"})+ )+ missed = await search_mod.search(+ SearchRequestBody(query="divano", k=10, extras={"color": "nope"})+ )+ unfiltered = await search_mod.search(+ SearchRequestBody(query="divano", k=10)+ )++ self.assertEqual({hit.id for hit in kept.hits}, {"it-1", "it-2"})+ self.assertEqual(kept.hits[0].extras["locale"], "it")+ self.assertEqual(kept.facets["locale"]["it"], 2)+ self.assertEqual(kept.facets["brand"]["CasaLuce"], 1)+ self.assertEqual(missed.hits, [])+ self.assertIsNone(missed.facets)+ self.assertEqual({hit.id for hit in unfiltered.hits}, {"it-1", "it-2"})+ self.assertEqual(unfiltered.hits[0].channel, "passages")+ mock_embeddings.embed_text.assert_not_called()++ def test_literal_overlap_and_frozen_head_helpers(self):+ from src.core.search.hybrid import (+ frozen_head_merge,+ fuse_passage_lists,+ literal_overlap_ids,+ )++ ranked = frozen_head_merge(+ [f"h{index}" for index in range(12)],+ [["extra", "h0"]],+ head_k=10,+ k=15,+ )+ self.assertEqual(ranked[:10], [f"h{index}" for index in range(10)])+ self.assertEqual(ranked[10], "extra")+ ids = literal_overlap_ids(+ "navy velvet sofa",+ {+ "hit": "Title: Navy velvet sofa two seats",+ "miss": "plastic kettle lid",+ "partial": "velvet throw pillow",+ },+ k=10,+ )+ self.assertEqual(ids[0], "hit")+ self.assertIn("partial", ids)+ self.assertNotIn("miss", ids)+ fused = fuse_passage_lists(+ ["d1", "d2"],+ ["b1", "d1"],+ fusion="cc",+ alpha=0.7,+ dense_similarities={"d1": 1.0, "d2": 0.2},+ bm25_scores={"b1": 4.0, "d1": 1.0},+ )+ self.assertTrue(fused)+ self.assertEqual(fused[0][0], "d1")++ async def test_fusion_alpha_request_overrides_config(self):+ from src.services.api.controllers import search as search_mod++ mock_data = MagicMock()+ mock_data.search_bm25.return_value = [+ (TextChunk(id="lex", text="license board"), 3.1)+ ]+ mock_embeddings = MagicMock()+ mock_embeddings.embed_text.return_value = Vector(+ id="q", embeddings=[0.1], metadata={}+ )+ mock_vs = MagicMock()+ mock_vs.search_data.return_value = [+ Vector(id="d1", metadata={"resource_id": "dense-1"}, distance=0.1)+ ]+ mock_data.get_text_chunks_by_ids.return_value = (+ [TextChunk(id="dense-1", text="vector hit")],+ [],+ )++ with (+ patch.object(search_mod.config, "search_enabled", True),+ patch.object(search_mod.config, "search_use_dense", True),+ patch.object(search_mod.config, "search_use_bm25", True),+ patch.object(search_mod.config, "search_fusion", "rrf"),+ patch.object(search_mod.config, "search_fusion_alpha", 0.1),+ patch.object(search_mod.config, "search_literal_fill", False),+ patch.object(search_mod, "embeddings_adapter", mock_embeddings),+ patch.object(search_mod, "vector_search", mock_vs),+ patch.object(search_mod, "data_adapter", mock_data),+ ):+ response = await search_mod.search(+ SearchRequestBody(+ query="license",+ k=5,+ fusion="cc",+ fusion_alpha=0.9,+ )+ )++ ids = [hit.id for hit in response.hits]+ self.assertIn("dense-1", ids)+ self.assertTrue(any(hit.scores.cc is not None for hit in response.hits))++ async def test_literal_fill_uses_frozen_head(self):+ from src.services.api.controllers import search as search_mod++ mock_data = MagicMock()+ mock_data.search_bm25.return_value = [+ (TextChunk(id=f"h{index}", text=f"t{index}"), float(20 - index))+ for index in range(12)+ ]+ mock_data.get_text_chunks.return_value = (+ [TextChunk(id="title-hit", text="navy velvet sofa")],+ 1,+ )+ mock_embeddings = MagicMock()+ mock_vs = MagicMock()++ with (+ patch.object(search_mod.config, "search_enabled", True),+ patch.object(search_mod.config, "search_use_dense", False),+ patch.object(search_mod.config, "search_use_bm25", True),+ patch.object(search_mod.config, "search_literal_fill", True),+ patch.object(search_mod, "embeddings_adapter", mock_embeddings),+ patch.object(search_mod, "vector_search", mock_vs),+ patch.object(search_mod, "data_adapter", mock_data),+ ):+ response = await search_mod.search(+ SearchRequestBody(query="navy velvet sofa", k=15)+ )++ ids = [hit.id for hit in response.hits]+ self.assertEqual(ids[:10], [f"h{index}" for index in range(10)])+ self.assertIn("title-hit", ids[10:])+ self.assertEqual(response.channel_lists["literal"], ["title-hit"])+++if __name__ == "__main__":+ unittest.main()diff --git a/tests/test_search_bm25.py b/tests/test_search_bm25.pynew file mode 100644index 0000000..65040a0--- /dev/null+++ b/tests/test_search_bm25.py@@ -0,0 +1,256 @@+import os+import unittest+from unittest.mock import MagicMock, patch++ENV_DEFAULTS = {+ "BRAINPAT_TOKEN": "test-token",+ "MODELS_MODE": "local",+ "EMBEDDINGS_LOCAL_MODEL": "local-model",+ "EMBEDDINGS_SMALL_MODEL": "small-model",+ "EMBEDDING_NODES_DIMENSION": "3",+ "EMBEDDING_TRIPLETS_DIMENSION": "3",+ "EMBEDDING_OBSERVATIONS_DIMENSION": "3",+ "EMBEDDING_DATA_DIMENSION": "3",+ "EMBEDDING_RELATIONSHIPS_DIMENSION": "3",+ "REDIS_HOST": "localhost",+ "REDIS_PORT": "6379",+ "NEO4J_HOST": "localhost",+ "NEO4J_PORT": "7687",+ "NEO4J_USERNAME": "neo4j",+ "NEO4J_PASSWORD": "password",+ "MILVUS_HOST": "localhost",+ "MILVUS_PORT": "19530",+ "MONGO_CONNECTION_STRING": "mongodb://localhost:27017",+ "CELERY_WORKER_CONCURRENCY": "1",+ "OLLAMA_HOST": "localhost",+ "OLLAMA_PORT": "11434",+ "OLLAMA_LLM_SMALL_MODEL": "small",+ "OLLAMA_LLM_LARGE_MODEL": "large",+}+for key, value in ENV_DEFAULTS.items():+ os.environ.setdefault(key, value)++from src.core.search.bm25 import okapi_bm25_document+from src.constants.data import TextChunk+from src.lib.postgresql import data as pgdata+++def _postgres_available() -> bool:+ try:+ import psycopg2++ pg = pgdata.config.postgresql+ pg.validate_credentials()+ conn = psycopg2.connect(+ host=pg.host,+ port=pg.port,+ user=pg.username,+ password=pg.password,+ dbname=pg.maintenance_database or "postgres",+ connect_timeout=2,+ )+ conn.close()+ return True+ except Exception:+ return False+++class OkapiBm25Tests(unittest.TestCase):+ def test_term_overlap_ranks_above_non_overlap(self):+ overlap = okapi_bm25_document(+ {"license": 2.0},+ dl=8,+ avgdl=8,+ n_docs=2,+ dfs={"license": 1},+ )+ non_overlap = okapi_bm25_document(+ {},+ dl=8,+ avgdl=8,+ n_docs=2,+ dfs={"license": 1},+ )+ self.assertGreater(overlap, non_overlap)+ self.assertEqual(non_overlap, 0.0)++ def test_higher_tf_ranks_higher(self):+ high = okapi_bm25_document(+ {"license": 3.0},+ dl=10,+ avgdl=10,+ n_docs=4,+ dfs={"license": 2},+ )+ low = okapi_bm25_document(+ {"license": 1.0},+ dl=10,+ avgdl=10,+ n_docs=4,+ dfs={"license": 2},+ )+ self.assertGreater(high, low)++ def test_search_ddl_is_gated_off_base_schema(self):+ self.assertNotIn("search_tsv", pgdata._BRAIN_DDL)+ self.assertIn("search_tsv", pgdata._SEARCH_DDL)+ self.assertIn("gin", pgdata._SEARCH_DDL.lower())+ self.assertIn("to_tsvector", pgdata._SEARCH_DDL)+ self.assertIn("to_tsvector('english'", pgdata._SEARCH_DDL)+ self.assertNotIn("to_tsvector('italian'", pgdata._SEARCH_DDL)+ self.assertNotIn("to_tsvector('spanish'", pgdata._SEARCH_DDL)+ self.assertNotIn("search_tsv_alt", pgdata._SEARCH_DDL)+ alt = pgdata._search_alt_ddl("italian")+ self.assertIn("to_tsvector('italian'", alt)+ self.assertIn("search_tsv_alt", alt)+ with self.assertRaises(ValueError):+ pgdata._search_alt_ddl("english")++ def test_english_bm25_sql_keeps_plainto_and(self):+ sql = pgdata._bm25_sql("english", "search_tsv", "search_len")+ self.assertIn("plainto_tsquery('english'", sql)+ self.assertIn("search_tsv", sql)+ self.assertNotIn("search_tsv_alt", sql)+ self.assertNotIn("WHERE TRUE", sql)++ def test_alt_bm25_sql_matches_any_lexeme(self):+ sql = pgdata._bm25_sql("italian", "search_tsv_alt", "search_len_alt")+ self.assertNotIn("plainto_tsquery", sql)+ self.assertIn("WHERE TRUE", sql)+ self.assertIn("search_tsv_alt", sql)+ self.assertIn("search_len_alt", sql)+ and_sql = pgdata._bm25_sql(+ "italian", "search_tsv_alt", "search_len_alt", query_match="and"+ )+ self.assertIn("plainto_tsquery('italian'", and_sql)++ def test_searchbenchesci74_stays_english_and_path(self):+ with (+ patch.object(+ pgdata.config, "search_fts_brains", frozenset({"searchbenchitsmoke"})+ ),+ patch.object(pgdata.config, "search_fts_regconfig", "italian"),+ ):+ self.assertIsNone(+ pgdata.config.search_fts_regconfig_for_brain("searchbenchesci74")+ )+ self.assertIsNone(+ pgdata.config.search_fts_regconfig_for_brain("locomoconv26")+ )++ def test_node_search_schema_skipped_when_search_disabled(self):+ from src.lib.postgresql.graph_store import PostgreSQLGraphStore++ store = PostgreSQLGraphStore.__new__(PostgreSQLGraphStore)+ store._schema_ready = {"brain-a"}+ store._search_ready_brains = set()+ store._schema_lock = MagicMock()+ with patch.object(pgdata.config, "search_enabled", False):+ store._ensure_brain_schema("brain-a")+ self.assertEqual(store._search_ready_brains, set())+ store._schema_lock.assert_not_called()++ def test_search_schema_skipped_when_search_disabled(self):+ client = pgdata.PostgreSQLDataClient.__new__(pgdata.PostgreSQLDataClient)+ client._initialized_brains = {"brain-a"}+ client._search_ready_brains = set()+ client._search_alt_brains = set()+ client._lock = MagicMock()+ with patch.object(pgdata.config, "search_enabled", False):+ client._ensure_brain_schema("brain-a")+ self.assertEqual(client._search_ready_brains, set())++ @unittest.skipUnless(_postgres_available(), "postgres not available")+ def test_postgres_bm25_ranks_term_overlap_first(self):+ brain_id = "searchbench_bm25_unit"+ with patch.object(pgdata.config, "search_enabled", True):+ client = pgdata.PostgreSQLDataClient()+ overlap = TextChunk(+ id="overlap",+ text="alice counseling license renewal board",+ )+ other = TextChunk(+ id="other",+ text="weather forecast sunny tomorrow picnic",+ )+ client.save_text_chunk(overlap, brain_id)+ client.save_text_chunk(other, brain_id)+ ranked = client.search_bm25("counseling license", brain_id, limit=10)+ ids = [chunk.id for chunk, _score in ranked]+ self.assertTrue(ids, "expected at least one BM25 hit")+ self.assertEqual(ids[0], "overlap")+ self.assertNotIn("other", ids)++ def test_fts_regconfig_only_allowlisted_searchbench(self):+ with (+ patch.object(+ pgdata.config,+ "search_fts_brains",+ frozenset({"locomoconv26", "searchbenchitsmoke", "demorecsys"}),+ ),+ patch.object(pgdata.config, "search_fts_regconfig", "italian"),+ ):+ self.assertEqual(+ pgdata.config.search_fts_regconfig_for_brain("searchbenchitsmoke"),+ "italian",+ )+ self.assertIsNone(+ pgdata.config.search_fts_regconfig_for_brain("searchbenchescies")+ )+ self.assertIsNone(pgdata.config.search_fts_regconfig_for_brain("locomoconv26"))+ self.assertIsNone(pgdata.config.search_fts_regconfig_for_brain("demorecsys"))+ self.assertIsNone(pgdata.config.search_fts_regconfig_for_brain("beam1m1clean"))++ def test_alt_ddl_not_executed_for_memory_brain(self):+ client = pgdata.PostgreSQLDataClient.__new__(pgdata.PostgreSQLDataClient)+ client._initialized_brains = {"locomoconv26"}+ client._search_ready_brains = {"locomoconv26"}+ client._search_alt_brains = set()+ client._lock = MagicMock()+ with (+ patch.object(pgdata.config, "search_enabled", True),+ patch.object(+ pgdata.config, "search_fts_regconfig_for_brain", return_value=None+ ),+ patch.object(pgdata, "ensure_brain_database") as ensure_db,+ ):+ client._ensure_brain_schema("locomoconv26")+ ensure_db.assert_not_called()+ self.assertEqual(client._search_alt_brains, set())++ def test_alt_ddl_executed_for_allowlisted_searchbench(self):+ from contextlib import contextmanager++ client = pgdata.PostgreSQLDataClient.__new__(pgdata.PostgreSQLDataClient)+ client._initialized_brains = {"searchbenchitsmoke"}+ client._search_ready_brains = {"searchbenchitsmoke"}+ client._search_alt_brains = set()+ client._lock = MagicMock()+ cur = MagicMock()+ conn = MagicMock()+ conn.cursor.return_value.__enter__.return_value = cur+ conn.cursor.return_value.__exit__.return_value = False++ @contextmanager+ def fake_borrow(_pool):+ yield conn++ with (+ patch.object(pgdata.config, "search_enabled", True),+ patch.object(+ pgdata.config, "search_fts_regconfig_for_brain", return_value="italian"+ ),+ patch.object(pgdata, "ensure_brain_database"),+ patch.object(pgdata, "get_brain_pool"),+ patch.object(pgdata, "borrow", fake_borrow),+ ):+ client._ensure_brain_schema("searchbenchitsmoke")+ executed = " ".join(str(call.args[0]) for call in cur.execute.call_args_list)+ self.assertIn("search_tsv_alt", executed)+ self.assertIn("to_tsvector('italian'", executed)+ self.assertNotIn("to_tsvector('english'", executed)+ self.assertIn("searchbenchitsmoke", client._search_alt_brains)+++if __name__ == "__main__":+ unittest.main()diff --git a/tests/test_search_config.py b/tests/test_search_config.pynew file mode 100644index 0000000..0b9db26--- /dev/null+++ b/tests/test_search_config.py@@ -0,0 +1,135 @@+import os+import unittest+from unittest.mock import patch++ENV_DEFAULTS = {+ "BRAINPAT_TOKEN": "test-token",+ "MODELS_MODE": "local",+ "EMBEDDINGS_LOCAL_MODEL": "local-model",+ "EMBEDDINGS_SMALL_MODEL": "small-model",+ "EMBEDDING_NODES_DIMENSION": "3",+ "EMBEDDING_TRIPLETS_DIMENSION": "3",+ "EMBEDDING_OBSERVATIONS_DIMENSION": "3",+ "EMBEDDING_DATA_DIMENSION": "3",+ "EMBEDDING_RELATIONSHIPS_DIMENSION": "3",+ "REDIS_HOST": "localhost",+ "REDIS_PORT": "6379",+ "NEO4J_HOST": "localhost",+ "NEO4J_PORT": "7687",+ "NEO4J_USERNAME": "neo4j",+ "NEO4J_PASSWORD": "password",+ "MILVUS_HOST": "localhost",+ "MILVUS_PORT": "19530",+ "MONGO_CONNECTION_STRING": "mongodb://localhost:27017",+ "CELERY_WORKER_CONCURRENCY": "1",+ "OLLAMA_HOST": "localhost",+ "OLLAMA_PORT": "11434",+ "OLLAMA_LLM_SMALL_MODEL": "small",+ "OLLAMA_LLM_LARGE_MODEL": "large",+ "PIPELINE_MODE": "accurate",+}+for key, value in ENV_DEFAULTS.items():+ os.environ.setdefault(key, value)++from src.config import (+ validate_context_passage_mode,+ validate_search_config,+ validate_search_fusion,+)+++class SearchConfigValidationTests(unittest.TestCase):+ def test_defaults_are_search_off_and_fused(self):+ self.assertEqual(validate_search_fusion(None), "rrf")+ self.assertEqual(validate_context_passage_mode(None), "hybrid")+ validate_search_config(+ enabled=False,+ use_dense=False,+ use_bm25=False,+ data_db="mongo",+ bm25_k1=1.2,+ bm25_b=0.75,+ )++ def test_enabled_requires_postgres_data_db(self):+ with self.assertRaises(ValueError) as ctx:+ validate_search_config(+ enabled=True,+ use_dense=True,+ use_bm25=True,+ data_db="mongo",+ bm25_k1=1.2,+ bm25_b=0.75,+ )+ self.assertIn("DATA_DB=postgresql", str(ctx.exception))++ def test_enabled_requires_at_least_one_channel(self):+ with self.assertRaises(ValueError) as ctx:+ validate_search_config(+ enabled=True,+ use_dense=False,+ use_bm25=False,+ data_db="postgresql",+ bm25_k1=1.2,+ bm25_b=0.75,+ )+ self.assertIn("SEARCH_USE_DENSE", str(ctx.exception))++ def test_invalid_fusion_and_passage_mode(self):+ with self.assertRaises(ValueError):+ validate_search_fusion("weighted")+ with self.assertRaises(ValueError):+ validate_context_passage_mode("splade")++ def test_config_reads_search_defaults(self):+ env = {+ "BRAINPAT_TOKEN": "test-token",+ "MODELS_MODE": "local",+ "EMBEDDINGS_LOCAL_MODEL": "local-model",+ "EMBEDDINGS_SMALL_MODEL": "small-model",+ "EMBEDDING_NODES_DIMENSION": "3",+ "EMBEDDING_TRIPLETS_DIMENSION": "3",+ "EMBEDDING_OBSERVATIONS_DIMENSION": "3",+ "EMBEDDING_DATA_DIMENSION": "3",+ "EMBEDDING_RELATIONSHIPS_DIMENSION": "3",+ "REDIS_HOST": "localhost",+ "REDIS_PORT": "6379",+ "NEO4J_HOST": "localhost",+ "NEO4J_PORT": "7687",+ "NEO4J_USERNAME": "neo4j",+ "NEO4J_PASSWORD": "password",+ "MILVUS_HOST": "localhost",+ "MILVUS_PORT": "19530",+ "MONGO_CONNECTION_STRING": "mongodb://localhost:27017",+ "CELERY_WORKER_CONCURRENCY": "1",+ "OLLAMA_HOST": "localhost",+ "OLLAMA_PORT": "11434",+ "OLLAMA_LLM_SMALL_MODEL": "small",+ "OLLAMA_LLM_LARGE_MODEL": "large",+ "DATA_DB": "postgresql",+ }+ with patch.dict(os.environ, env, clear=False):+ for key in (+ "SEARCH_ENABLED",+ "SEARCH_USE_DENSE",+ "SEARCH_USE_BM25",+ "SEARCH_FUSION",+ "SEARCH_LITERAL_FILL",+ "CONTEXT_PASSAGE_MODE",+ ):+ os.environ.pop(key, None)+ from src.config import Config++ cfg = Config()+ self.assertFalse(cfg.search_enabled)+ self.assertTrue(cfg.search_use_dense)+ self.assertTrue(cfg.search_use_bm25)+ self.assertEqual(cfg.search_fusion, "rrf")+ self.assertEqual(cfg.search_bm25_k1, 1.2)+ self.assertEqual(cfg.search_bm25_b, 0.75)+ self.assertEqual(cfg.context_passage_mode, "hybrid")+ self.assertFalse(cfg.search_literal_fill)+++if __name__ == "__main__":+ unittest.main()diff --git a/tests/test_search_graph.py b/tests/test_search_graph.pynew file mode 100644index 0000000..e1bf35d--- /dev/null+++ b/tests/test_search_graph.py@@ -0,0 +1,713 @@+import os+import unittest+from unittest.mock import MagicMock, patch++ENV_DEFAULTS = {+ "BRAINPAT_TOKEN": "test-token",+ "MODELS_MODE": "local",+ "EMBEDDINGS_LOCAL_MODEL": "local-model",+ "EMBEDDINGS_SMALL_MODEL": "small-model",+ "EMBEDDING_NODES_DIMENSION": "3",+ "EMBEDDING_TRIPLETS_DIMENSION": "3",+ "EMBEDDING_OBSERVATIONS_DIMENSION": "3",+ "EMBEDDING_DATA_DIMENSION": "3",+ "EMBEDDING_RELATIONSHIPS_DIMENSION": "3",+ "REDIS_HOST": "localhost",+ "REDIS_PORT": "6379",+ "NEO4J_HOST": "localhost",+ "NEO4J_PORT": "7687",+ "NEO4J_USERNAME": "neo4j",+ "NEO4J_PASSWORD": "password",+ "MILVUS_HOST": "localhost",+ "MILVUS_PORT": "19530",+ "MONGO_CONNECTION_STRING": "mongodb://localhost:27017",+ "CELERY_WORKER_CONCURRENCY": "1",+ "OLLAMA_HOST": "localhost",+ "OLLAMA_PORT": "11434",+ "OLLAMA_LLM_SMALL_MODEL": "small",+ "OLLAMA_LLM_LARGE_MODEL": "large",+}+for key, value in ENV_DEFAULTS.items():+ os.environ.setdefault(key, value)++from fastapi import HTTPException++from src.constants.embeddings import Vector+from src.constants.kg import Node, SearchEntitiesResult+from src.core.search.graph_channels import (+ DEFAULT_COMMUNITY_LABELS,+ collect_community_hits,+ collect_entity_hits,+ collect_event_hits,+ expand_neighbor_hits,+)+from src.core.search.hybrid import fuse_passage_lists+from src.services.api.constants.requests import SearchHit, SearchRequestBody, SearchResponse+++def _node(+ uuid: str,+ name: str,+ labels: list[str],+ happened_at: str | None = None,+ description: str | None = None,+ properties: dict | None = None,+) -> Node:+ props = dict(properties or {})+ if happened_at:+ props["happened_at"] = happened_at+ return Node(+ uuid=uuid,+ name=name,+ labels=labels,+ happened_at=happened_at,+ description=description,+ properties=props,+ )+++class FakeGraph:+ def __init__(self, nodes: list[Node], neighbors: dict[str, list[Node]] | None = None):+ self.nodes = {node.uuid: node for node in nodes}+ self.neighbors = neighbors or {}+ self.neighbor_calls = 0++ def search_entities(+ self,+ brain_id: str = "default",+ limit: int = 10,+ skip: int = 0,+ node_labels: list[str] | None = None,+ query_text: str | None = None,+ ) -> SearchEntitiesResult:+ matched: list[Node] = []+ want = {str(item).upper() for item in (node_labels or [])}+ needle = (query_text or "").lower()+ for node in self.nodes.values():+ have = {str(item).upper() for item in node.labels}+ if want and not (have & want):+ continue+ if needle and needle not in node.name.lower():+ continue+ matched.append(node)+ page = matched[skip : skip + limit]+ return SearchEntitiesResult(results=page, total=len(matched))++ def search_nodes_bm25(+ self,+ query_text: str,+ brain_id: str = "default",+ limit: int = 10,+ node_labels: list[str] | None = None,+ node_uuids: list[str] | None = None,+ ):+ needle = (query_text or "").lower()+ want = {str(item).upper() for item in (node_labels or [])}+ allowed = {str(item) for item in (node_uuids or [])} or None+ scored: list[tuple[Node, float]] = []+ for node in self.nodes.values():+ if allowed is not None and node.uuid not in allowed:+ continue+ have = {str(item).upper() for item in node.labels}+ if want and not (have & want):+ continue+ props = node.properties or {}+ text = " ".join(+ [+ node.name or "",+ node.description or "",+ str(props.get("search_text") or ""),+ ]+ ).lower()+ score = 0.0+ if needle and needle in text:+ score += 5.0+ for token in needle.split():+ if token and token in text:+ score += 1.0+ if score:+ scored.append((node, score))+ scored.sort(key=lambda item: (-item[1], item[0].uuid))+ return scored[:limit]++ def get_by_uuids(self, uuids: list[str], brain_id: str = "default") -> list[Node]:+ return [self.nodes[item] for item in uuids if item in self.nodes]++ def get_by_uuid(self, uuid: str, brain_id: str = "default") -> Node:+ return self.nodes[uuid]++ def get_neighbors(self, nodes, brain_id: str = "default", **kwargs):+ self.neighbor_calls += 1+ keys = []+ for item in nodes:+ keys.append(item if isinstance(item, str) else item.uuid)+ out = {}+ for key in keys:+ out[key] = [(None, node) for node in self.neighbors.get(key, [])]+ return out++ def get_event_centric_neighbors(self, nodes, brain_id: str = "default"):+ return []+++class SearchGraphChannelTests(unittest.IsolatedAsyncioTestCase):+ def test_contract_has_graph_fields_not_catalog_enums(self):+ body = SearchRequestBody(query="sofa")+ self.assertEqual(body.channels, ["passages"])+ self.assertEqual(body.expand, "none")+ self.assertEqual(body.mode, "default")+ self.assertIsNone(body.node_labels)+ fields = set(SearchRequestBody.model_fields)+ hit_fields = set(SearchHit.model_fields)+ for banned in (+ "product_id",+ "product_features",+ "sku",+ "category",+ "brand",+ "user_id",+ ):+ self.assertNotIn(banned, fields)+ self.assertNotIn(banned, hit_fields)+ self.assertIn("labels", hit_fields)+ self.assertIn("extras", hit_fields)+ self.assertIn("node_id", hit_fields)+ self.assertIn("node_ids", set(SearchResponse.model_fields))+ self.assertIn("extras", fields)+ self.assertEqual(list(DEFAULT_COMMUNITY_LABELS), ["TYPE", "CLASS", "TOPIC"])++ async def test_unknown_channel_is_400(self):+ from src.services.api.controllers import search as search_mod++ with patch.object(search_mod.config, "search_enabled", True):+ with self.assertRaises(HTTPException) as ctx:+ await search_mod.search(+ SearchRequestBody(query="sofa", channels=["foo"])+ )+ self.assertEqual(ctx.exception.status_code, 400)+ self.assertIn("Unknown search channel", str(ctx.exception.detail))++ async def test_entities_empty_graph_is_200(self):+ from src.services.api.controllers import search as search_mod++ mock_embeddings = MagicMock()+ mock_embeddings.embed_text.return_value = Vector(+ id="q", embeddings=[0.1], metadata={}+ )+ mock_vs = MagicMock()+ mock_vs.search_nodes.return_value = []+ graph = FakeGraph([])++ with (+ patch.object(search_mod.config, "search_enabled", True),+ patch.object(search_mod.config, "search_use_dense", False),+ patch.object(search_mod.config, "search_use_bm25", True),+ patch.object(search_mod, "embeddings_adapter", mock_embeddings),+ patch.object(search_mod, "vector_search", mock_vs),+ patch.object(search_mod, "graph_adapter", graph),+ patch.object(search_mod, "data_adapter", MagicMock()),+ ):+ response = await search_mod.search(+ SearchRequestBody(query="sofa", channels=["entities"], k=5)+ )++ self.assertEqual(response.hits, [])++ async def test_seeded_entity_name_in_top_k(self):+ from src.services.api.controllers import search as search_mod++ graph = FakeGraph([_node("sku-1", "navy velvet sofa", ["ENTITY"])])+ mock_embeddings = MagicMock()+ mock_embeddings.embed_text.return_value = Vector(+ id="q", embeddings=[0.2], metadata={}+ )+ mock_vs = MagicMock()+ mock_vs.search_nodes.return_value = []++ with (+ patch.object(search_mod.config, "search_enabled", True),+ patch.object(search_mod.config, "search_use_dense", False),+ patch.object(search_mod.config, "search_use_bm25", True),+ patch.object(search_mod, "embeddings_adapter", mock_embeddings),+ patch.object(search_mod, "vector_search", mock_vs),+ patch.object(search_mod, "graph_adapter", graph),+ patch.object(search_mod, "data_adapter", MagicMock()),+ ):+ response = await search_mod.search(+ SearchRequestBody(query="velvet sofa", channels=["entities"], k=5)+ )++ self.assertEqual([hit.id for hit in response.hits], ["sku-1"])+ self.assertEqual(response.hits[0].channel, "entities")+ self.assertIn("ENTITY", response.hits[0].labels)+ self.assertEqual(response.hits[0].node_id, "sku-1")+ self.assertEqual(response.node_ids, ["sku-1"])++ async def test_node_labels_mismatch_is_empty_not_400(self):+ from src.services.api.controllers import search as search_mod++ graph = FakeGraph([_node("sku-1", "navy velvet sofa", ["ENTITY"])])+ mock_embeddings = MagicMock()+ mock_embeddings.embed_text.return_value = Vector(+ id="q", embeddings=[0.2], metadata={}+ )+ mock_vs = MagicMock()+ mock_vs.search_nodes.return_value = []++ with (+ patch.object(search_mod.config, "search_enabled", True),+ patch.object(search_mod, "embeddings_adapter", mock_embeddings),+ patch.object(search_mod, "vector_search", mock_vs),+ patch.object(search_mod, "graph_adapter", graph),+ patch.object(search_mod, "data_adapter", MagicMock()),+ ):+ response = await search_mod.search(+ SearchRequestBody(+ query="velvet sofa",+ channels=["entities"],+ node_labels=["EVENT"],+ k=5,+ )+ )++ self.assertEqual(response.hits, [])++ async def test_default_passages_does_not_touch_graph(self):+ from src.services.api.controllers import search as search_mod+ from src.constants.data import TextChunk++ mock_data = MagicMock()+ mock_data.search_bm25.return_value = [+ (TextChunk(id="lex", text="license board"), 3.1)+ ]+ mock_graph = MagicMock()+ mock_embeddings = MagicMock()+ mock_vs = MagicMock()++ with (+ patch.object(search_mod.config, "search_enabled", True),+ patch.object(search_mod.config, "search_use_dense", False),+ patch.object(search_mod.config, "search_use_bm25", True),+ patch.object(search_mod, "embeddings_adapter", mock_embeddings),+ patch.object(search_mod, "vector_search", mock_vs),+ patch.object(search_mod, "graph_adapter", mock_graph),+ patch.object(search_mod, "data_adapter", mock_data),+ ):+ response = await search_mod.search(SearchRequestBody(query="license", k=5))++ self.assertEqual([hit.id for hit in response.hits], ["lex"])+ self.assertEqual(response.hits[0].channel, "passages")+ mock_graph.search_entities.assert_not_called()+ mock_embeddings.embed_text.assert_not_called()++ async def test_events_empty_ok(self):+ from src.services.api.controllers import search as search_mod++ graph = FakeGraph([_node("sku-1", "navy velvet sofa", ["ENTITY"])])+ mock_embeddings = MagicMock()+ mock_embeddings.embed_text.return_value = Vector(+ id="q", embeddings=[0.2], metadata={}+ )+ mock_vs = MagicMock()+ mock_vs.search_nodes.return_value = []++ with (+ patch.object(search_mod.config, "search_enabled", True),+ patch.object(search_mod, "embeddings_adapter", mock_embeddings),+ patch.object(search_mod, "vector_search", mock_vs),+ patch.object(search_mod, "graph_adapter", graph),+ patch.object(search_mod, "data_adapter", MagicMock()),+ ):+ response = await search_mod.search(+ SearchRequestBody(query="view", channels=["events"], k=5)+ )++ self.assertEqual(response.hits, [])++ async def test_events_with_happened_at(self):+ from src.services.api.controllers import search as search_mod++ graph = FakeGraph(+ [+ _node(+ "evt-1",+ "View",+ ["EVENT"],+ happened_at="2024-01-05T12:00:00Z",+ )+ ]+ )+ mock_embeddings = MagicMock()+ mock_embeddings.embed_text.return_value = Vector(+ id="q", embeddings=[0.2], metadata={}+ )+ mock_vs = MagicMock()+ mock_vs.search_nodes.return_value = []++ with (+ patch.object(search_mod.config, "search_enabled", True),+ patch.object(search_mod, "embeddings_adapter", mock_embeddings),+ patch.object(search_mod, "vector_search", mock_vs),+ patch.object(search_mod, "graph_adapter", graph),+ patch.object(search_mod, "data_adapter", MagicMock()),+ ):+ response = await search_mod.search(+ SearchRequestBody(query="View", channels=["events"], k=5)+ )++ self.assertEqual([hit.id for hit in response.hits], ["evt-1"])+ self.assertEqual(response.hits[0].channel, "events")+ self.assertEqual(response.hits[0].extras["happened_at"], "2024-01-05T12:00:00Z")++ def test_community_hub_returns_members_with_fanout_cap(self):+ hub = _node("class-sofas", "sofas", ["CLASS"])+ members = [+ _node("e1", "velvet sofa", ["ENTITY"]),+ _node("e2", "leather sofa", ["ENTITY"]),+ _node("e3", "loveseat", ["ENTITY"]),+ ]+ unrelated = _node("e4", "lamp", ["ENTITY"])+ graph = FakeGraph(+ [hub, *members, unrelated],+ neighbors={"class-sofas": members},+ )+ hits = collect_community_hits(+ query="sofas",+ brain_id="searchbenchsmoke",+ k=10,+ graph=graph,+ fanout=2,+ )+ ids = [hit.id for hit in hits]+ self.assertTrue(set(ids).issubset({"e1", "e2", "e3"}))+ self.assertNotIn("e4", ids)+ self.assertNotIn("class-sofas", ids)+ self.assertEqual(len(ids), 2)+ self.assertTrue(all(hit.channel == "communities" for hit in hits))++ def test_expand_neighbors_default_off(self):+ seed = _node("sku-1", "velvet sofa", ["ENTITY"])+ neighbor = _node("class-sofas", "sofas", ["CLASS"])+ graph = FakeGraph([seed, neighbor], neighbors={"sku-1": [neighbor]})+ seeds = collect_entity_hits(+ query="velvet sofa",+ brain_id="searchbenchsmoke",+ k=5,+ graph=graph,+ )+ self.assertEqual(graph.neighbor_calls, 0)+ expanded = expand_neighbor_hits(+ seeds,+ brain_id="searchbenchsmoke",+ k=5,+ graph=graph,+ fanout=10,+ )+ self.assertEqual([hit.id for hit in expanded], ["class-sofas"])+ self.assertEqual(expanded[0].channel, "neighbors")++ async def test_expand_omitted_skips_hops(self):+ from src.services.api.controllers import search as search_mod++ seed = _node("sku-1", "velvet sofa", ["ENTITY"])+ neighbor = _node("n1", "pillow", ["ENTITY"])+ graph = FakeGraph([seed, neighbor], neighbors={"sku-1": [neighbor]})+ mock_embeddings = MagicMock()+ mock_embeddings.embed_text.return_value = Vector(+ id="q", embeddings=[0.2], metadata={}+ )+ mock_vs = MagicMock()+ mock_vs.search_nodes.return_value = []++ with (+ patch.object(search_mod.config, "search_enabled", True),+ patch.object(search_mod, "embeddings_adapter", mock_embeddings),+ patch.object(search_mod, "vector_search", mock_vs),+ patch.object(search_mod, "graph_adapter", graph),+ patch.object(search_mod, "data_adapter", MagicMock()),+ ):+ response = await search_mod.search(+ SearchRequestBody(query="velvet sofa", channels=["entities"], k=5)+ )++ self.assertEqual([hit.id for hit in response.hits], ["sku-1"])+ self.assertEqual(graph.neighbor_calls, 0)++ async def test_expand_neighbors_adds_hop(self):+ from src.services.api.controllers import search as search_mod++ seed = _node("sku-1", "velvet sofa", ["ENTITY"])+ neighbor = _node("n1", "pillow", ["ENTITY"])+ graph = FakeGraph([seed, neighbor], neighbors={"sku-1": [neighbor]})+ mock_embeddings = MagicMock()+ mock_embeddings.embed_text.return_value = Vector(+ id="q", embeddings=[0.2], metadata={}+ )+ mock_vs = MagicMock()+ mock_vs.search_nodes.return_value = []++ with (+ patch.object(search_mod.config, "search_enabled", True),+ patch.object(search_mod, "embeddings_adapter", mock_embeddings),+ patch.object(search_mod, "vector_search", mock_vs),+ patch.object(search_mod, "graph_adapter", graph),+ patch.object(search_mod, "data_adapter", MagicMock()),+ ):+ response = await search_mod.search(+ SearchRequestBody(+ query="velvet sofa",+ channels=["entities"],+ expand="neighbors",+ k=5,+ )+ )++ ids = [hit.id for hit in response.hits]+ self.assertIn("sku-1", ids)+ self.assertIn("n1", ids)+ self.assertTrue(any(hit.channel == "neighbors" for hit in response.hits))++ def test_fuse_mixed_chunk_and_node_ids(self):+ fused = fuse_passage_lists(+ ["chunk-1"],+ [],+ extra_id_lists=[["sku-1"]],+ )+ ids = [item for item, _ in fused]+ self.assertEqual(set(ids), {"chunk-1", "sku-1"})++ async def test_fused_graph_lists_drop_hub_ids(self):+ from src.constants.data import TextChunk+ from src.services.api.controllers import search as search_mod++ hub = _node("hub:attr:red", "red", ["ATTR"])+ product = _node("sku-1", "red velvet sofa", ["ENTITY"])+ graph = FakeGraph([hub, product], neighbors={"hub:attr:red": [product]})+ mock_data = MagicMock()+ mock_data.search_bm25.return_value = [+ (TextChunk(id="chunk-1", text="red velvet sofa"), 2.0)+ ]+ mock_embeddings = MagicMock()+ mock_embeddings.embed_text.return_value = Vector(+ id="q", embeddings=[0.2], metadata={}+ )+ mock_vs = MagicMock()+ mock_vs.search_nodes.return_value = []++ with (+ patch.object(search_mod.config, "search_enabled", True),+ patch.object(search_mod.config, "search_use_dense", False),+ patch.object(search_mod.config, "search_use_bm25", True),+ patch.object(search_mod, "embeddings_adapter", mock_embeddings),+ patch.object(search_mod, "vector_search", mock_vs),+ patch.object(search_mod, "graph_adapter", graph),+ patch.object(search_mod, "data_adapter", mock_data),+ ):+ response = await search_mod.search(+ SearchRequestBody(+ query="red sofa",+ channels=["passages", "entities", "communities"],+ k=10,+ )+ )++ ids = [hit.id for hit in response.hits]+ self.assertNotIn("hub:attr:red", ids)+ self.assertIn("sku-1", ids)+ self.assertIn("chunk-1", ids)+ self.assertEqual(response.channel_lists["bm25"], ["chunk-1"])+ self.assertIn("sku-1", response.channel_lists["entities"])+ self.assertNotIn("hub:attr:red", response.channel_lists["entities"])+ self.assertNotIn("hub:attr:red", response.channel_lists["communities"])++ def test_entities_drop_attr_hubs(self):+ hub = _node("hub:attr:red", "red", ["ATTR"])+ product = _node("sku-1", "red velvet sofa", ["ENTITY"])+ graph = FakeGraph([hub, product])+ hits = collect_entity_hits(+ query="red sofa",+ brain_id="searchbenchsmoke",+ k=5,+ graph=graph,+ )+ ids = [hit.id for hit in hits]+ self.assertEqual(ids, ["sku-1"])+ self.assertTrue(all(hit.channel == "entities" for hit in hits))++ def test_community_skips_attr_member_hubs(self):+ hub = _node("class-sofas", "sofas", ["CLASS"])+ product = _node("e1", "velvet sofa", ["ENTITY"])+ attr = _node("hub:attr:red", "red", ["ATTR"])+ graph = FakeGraph(+ [hub, product, attr],+ neighbors={"class-sofas": [product, attr]},+ )+ hits = collect_community_hits(+ query="sofas",+ brain_id="searchbenchsmoke",+ k=10,+ graph=graph,+ fanout=5,+ )+ ids = [hit.id for hit in hits]+ self.assertEqual(ids, ["e1"])+ self.assertNotIn("hub:attr:red", ids)++ def test_community_attr_hub_returns_products(self):+ hub = _node("hub:attr:red", "red", ["ATTR"])+ product = _node("e1", "red sofa", ["ENTITY"])+ graph = FakeGraph([hub, product], neighbors={"hub:attr:red": [product]})+ hits = collect_community_hits(+ query="red sofa",+ brain_id="searchbenchsmoke",+ k=10,+ graph=graph,+ fanout=5,+ )+ self.assertEqual([hit.id for hit in hits], ["e1"])+ self.assertEqual(hits[0].channel, "communities")++ def test_search_controller_does_not_import_retrieve_assembler(self):+ from pathlib import Path++ root = Path(__file__).resolve().parents[1]+ search_text = (+ root / "src" / "services" / "api" / "controllers" / "search.py"+ ).read_text(encoding="utf-8")+ retrieve_text = (+ root / "src" / "services" / "api" / "controllers" / "retrieve.py"+ ).read_text(encoding="utf-8")+ self.assertNotIn("controllers.retrieve", search_text)+ self.assertNotIn("kg_topic_sessions", search_text)+ self.assertNotIn("graph_channels", retrieve_text)+ schema = (+ root / "src" / "services" / "api" / "constants" / "requests.py"+ ).read_text(encoding="utf-8")+ self.assertNotIn("product_id", schema.split("class SearchRequestBody")[1].split("class SearchHitScores")[0])+++class SearchGraphHelperTests(unittest.TestCase):+ def test_entity_ann_and_name_match(self):+ named = _node("sku-1", "navy velvet sofa", ["ENTITY"])+ vector_only = _node("sku-2", "other item", ["ENTITY"])+ graph = FakeGraph([named, vector_only])+ vs = MagicMock()+ vs.search_nodes.return_value = [+ Vector(id="v2", metadata={"uuid": "sku-2", "name": "other item"}, distance=0.1)+ ]+ hits = collect_entity_hits(+ query="velvet sofa",+ brain_id="searchbenchsmoke",+ k=5,+ graph=graph,+ vector_search=vs,+ query_vector=[0.1, 0.2],+ )+ ids = [hit.id for hit in hits]+ self.assertIn("sku-1", ids)+ self.assertIn("sku-2", ids)++ def test_events_rank_recent_higher(self):+ older = _node("evt-old", "View", ["EVENT"], happened_at="2020-01-01T00:00:00Z")+ newer = _node("evt-new", "View", ["EVENT"], happened_at="2026-08-01T00:00:00Z")+ graph = FakeGraph([older, newer])+ hits = collect_event_hits(+ query="View",+ brain_id="searchbenchsmoke",+ k=5,+ graph=graph,+ )+ self.assertEqual(hits[0].id, "evt-new")+ self.assertGreater(hits[0].score, hits[1].score)++ def test_entity_bm25_matches_description_only(self):+ named = _node("sku-1", "catalog item", ["ENTITY"])+ described = _node(+ "sku-2",+ "item two",+ ["ENTITY"],+ description="navy velvet ottoman for small rooms",+ properties={"search_text": "item two navy velvet ottoman for small rooms"},+ )+ graph = FakeGraph([named, described])+ hits = collect_entity_hits(+ query="ottoman",+ brain_id="searchbenchsmoke",+ k=5,+ graph=graph,+ )+ ids = [hit.id for hit in hits]+ self.assertIn("sku-2", ids)+ self.assertNotIn("sku-1", ids)++ def test_community_intersects_class_and_attr_hubs(self):+ class_hub = _node("hub:class:sofas", "sofas", ["CLASS"])+ attr_hub = _node("hub:attr:modern", "modern", ["ATTR"])+ both = _node("e1", "velvet sofa", ["ENTITY"])+ class_only = _node("e2", "leather sofa", ["ENTITY"])+ graph = FakeGraph(+ [class_hub, attr_hub, both, class_only],+ neighbors={+ "hub:class:sofas": [both, class_only],+ "hub:attr:modern": [both],+ },+ )+ hits = collect_community_hits(+ query="modern sofas",+ brain_id="searchbenchsmoke",+ k=10,+ graph=graph,+ fanout=5,+ )+ ids = [hit.id for hit in hits]+ self.assertEqual(ids, ["e1"])+ self.assertNotIn("e2", ids)+ self.assertNotIn("hub:class:sofas", ids)++ def test_community_union_when_single_hub_kind(self):+ class_hub = _node("hub:class:sofas", "sofas", ["CLASS"])+ a = _node("e1", "velvet sofa", ["ENTITY"])+ b = _node("e2", "leather sofa", ["ENTITY"])+ graph = FakeGraph(+ [class_hub, a, b],+ neighbors={"hub:class:sofas": [a, b]},+ )+ hits = collect_community_hits(+ query="sofas",+ brain_id="searchbenchsmoke",+ k=10,+ graph=graph,+ fanout=5,+ )+ self.assertEqual(set(hit.id for hit in hits), {"e1", "e2"})++ def test_community_hybrid_ranks_by_node_bm25_not_degree(self):+ class_hub = _node("hub:class:sofas", "sofas", ["CLASS"])+ attr_hub = _node("hub:attr:modern", "modern", ["ATTR"])+ weak = _node("e1", "sofa", ["ENTITY"], description="basic")+ strong = _node(+ "e2",+ "plush seating",+ ["ENTITY"],+ description="modern sofas extra plush",+ properties={"search_text": "plush seating modern sofas extra plush"},+ )+ graph = FakeGraph(+ [class_hub, attr_hub, weak, strong],+ neighbors={+ "hub:class:sofas": [weak, strong],+ "hub:attr:modern": [weak, strong],+ },+ )+ hits = collect_community_hits(+ query="modern sofas",+ brain_id="searchbenchsmoke",+ k=10,+ graph=graph,+ fanout=5,+ )+ self.assertEqual([hit.id for hit in hits], ["e2", "e1"])+++if __name__ == "__main__":+ unittest.main()diff --git a/tests/test_search_personalize.py b/tests/test_search_personalize.pynew file mode 100644index 0000000..e0ebc7b--- /dev/null+++ b/tests/test_search_personalize.py@@ -0,0 +1,324 @@+import os+import unittest+from unittest.mock import MagicMock, patch++ENV_DEFAULTS = {+ "BRAINPAT_TOKEN": "test-token",+ "MODELS_MODE": "local",+ "EMBEDDINGS_LOCAL_MODEL": "local-model",+ "EMBEDDINGS_SMALL_MODEL": "small-model",+ "EMBEDDING_NODES_DIMENSION": "3",+ "EMBEDDING_TRIPLETS_DIMENSION": "3",+ "EMBEDDING_OBSERVATIONS_DIMENSION": "3",+ "EMBEDDING_DATA_DIMENSION": "3",+ "EMBEDDING_RELATIONSHIPS_DIMENSION": "3",+ "REDIS_HOST": "localhost",+ "REDIS_PORT": "6379",+ "NEO4J_HOST": "localhost",+ "NEO4J_PORT": "7687",+ "NEO4J_USERNAME": "neo4j",+ "NEO4J_PASSWORD": "password",+ "MILVUS_HOST": "localhost",+ "MILVUS_PORT": "19530",+ "MONGO_CONNECTION_STRING": "mongodb://localhost:27017",+ "CELERY_WORKER_CONCURRENCY": "1",+ "OLLAMA_HOST": "localhost",+ "OLLAMA_PORT": "11434",+ "OLLAMA_LLM_SMALL_MODEL": "small",+ "OLLAMA_LLM_LARGE_MODEL": "large",+}+for key, value in ENV_DEFAULTS.items():+ os.environ.setdefault(key, value)++from src.constants.data import TextChunk+from src.constants.kg import Node, Predicate+from src.core.search.personalize import (+ blend_ranked,+ personalize_ranked_ids,+ query_personalize_lambda,+ score_nodes_for_user,+ user_pref_weights,+)+from src.services.api.constants.requests import SearchRequestBody+++def _node(uuid: str, name: str, labels: list[str], happened_at: str | None = None) -> Node:+ return Node(+ uuid=uuid,+ name=name,+ labels=labels,+ happened_at=happened_at,+ properties={},+ )+++def _pred(name: str, amount: float | None = None, weight: float | None = None) -> Predicate:+ props = {}+ if weight is not None:+ props["weight"] = weight+ return Predicate(+ uuid=f"rel:{name}:{id(name)}",+ name=name,+ description="",+ amount=amount,+ properties=props,+ )+++class FakeGraph:+ def __init__(self):+ self.nodes: dict[str, Node] = {}+ self.edges: dict[str, list[tuple[Predicate, Node]]] = {}++ def add(self, node: Node) -> Node:+ self.nodes[node.uuid] = node+ self.edges.setdefault(node.uuid, [])+ return node++ def link(self, tail: str, pred: Predicate, tip: str) -> None:+ self.edges.setdefault(tail, []).append((pred, self.nodes[tip]))++ def get_by_uuids(self, uuids, brain_id="default"):+ return [self.nodes[uid] for uid in uuids if uid in self.nodes]++ def get_by_uuid(self, uuid, brain_id="default"):+ return self.nodes.get(uuid)++ def get_neighbors(self, nodes, brain_id="default", of_types=None, **kwargs):+ out = {}+ for item in nodes:+ uid = item if isinstance(item, str) else item.uuid+ neigh = list(self.edges.get(uid, []))+ if of_types:+ wanted = {label.upper() for label in of_types}+ neigh = [+ (pred, node)+ for pred, node in neigh+ if wanted & {label.upper() for label in (node.labels or [])}+ ]+ out[uid] = neigh+ return out+++def _style_graph() -> tuple[FakeGraph, Node]:+ graph = FakeGraph()+ user = graph.add(_node("user:u01", "u01", ["USER"]))+ recent = graph.add(+ _node("evt-recent", "View", ["EVENT"], happened_at="08/20/2026")+ )+ old = graph.add(_node("evt-old", "View", ["EVENT"], happened_at="08/21/2025"))+ seventies = graph.add(_node("70s-item", "lamp 70s", ["ENTITY"]))+ modern = graph.add(_node("modern-item", "lamp modern", ["ENTITY"]))+ attr_70s = graph.add(_node("hub:attr:70s", "70s", ["ATTR", "STYLE"]))+ attr_modern = graph.add(_node("hub:attr:modern", "modern", ["ATTR", "STYLE"]))+ graph.link("user:u01", _pred("MADE"), "evt-recent")+ graph.link("evt-recent", _pred("TARGETED"), "70s-item")+ graph.link("70s-item", _pred("HAS"), "hub:attr:70s")+ graph.link("user:u01", _pred("MADE"), "evt-old")+ graph.link("evt-old", _pred("TARGETED"), "modern-item")+ graph.link("modern-item", _pred("HAS"), "hub:attr:modern")+ graph.link(+ "user:u01",+ _pred("PREFERS", amount=0.4, weight=0.4),+ "hub:attr:70s",+ )+ return graph, user+++class QueryLambdaTests(unittest.TestCase):+ def test_generic_one_token(self):+ self.assertAlmostEqual(query_personalize_lambda("lamp"), 0.85)++ def test_two_and_three_tokens(self):+ self.assertAlmostEqual(query_personalize_lambda("oak table"), 0.5)+ self.assertAlmostEqual(query_personalize_lambda("oak dining table"), 0.25)++ def test_four_plus_tokens(self):+ self.assertAlmostEqual(query_personalize_lambda("oak dining table set"), 0.1)++ def test_digits_are_specific(self):+ self.assertEqual(query_personalize_lambda("oak dining table 180cm"), 0.0)+ self.assertEqual(query_personalize_lambda("sku 7468"), 0.0)+++class BlendTests(unittest.TestCase):+ def test_zero_lambda_preserves_order(self):+ ids = ["a", "b", "c"]+ ordered, _ = blend_ranked(ids, {"a": 3, "b": 2, "c": 1}, {"c": 9}, 0.0)+ self.assertEqual(ordered, ids)++ def test_all_zero_prefs_preserve_order(self):+ ids = ["a", "b", "c"]+ ordered, _ = blend_ranked(ids, {"a": 3, "b": 2, "c": 1}, {}, 0.9)+ self.assertEqual(ordered, ids)++ def test_high_lambda_promotes_overlap_without_dropping(self):+ ids = ["a", "b", "c"]+ ordered, _ = blend_ranked(+ ids, {"a": 3, "b": 2, "c": 1}, {"a": 0, "b": 0, "c": 10}, 1.0+ )+ self.assertEqual(ordered[0], "c")+ self.assertEqual(set(ordered), set(ids))+++class PrefVectorTests(unittest.TestCase):+ def test_recent_70s_outranks_old_modern_and_combines_prefers(self):+ graph, user = _style_graph()+ with patch("src.core.search.personalize.graph_adapter", graph):+ weights = user_pref_weights(user, "brain")+ self.assertGreater(weights["hub:attr:70s"], weights["hub:attr:modern"])+ self.assertGreater(weights["hub:attr:70s"], 0.4)++ def test_missing_user_is_empty(self):+ graph = FakeGraph()+ with patch("src.core.search.personalize.graph_adapter", graph):+ self.assertEqual(user_pref_weights(None, "brain"), {})+ self.assertEqual(+ user_pref_weights(_node("missing", "x", ["USER"]), "brain"),+ {},+ )++ def test_score_nodes_overlap_only(self):+ graph, user = _style_graph()+ with patch("src.core.search.personalize.graph_adapter", graph):+ prefs = user_pref_weights(user, "brain")+ scores = score_nodes_for_user(+ ["70s-item", "modern-item", "ghost"], prefs, "brain"+ )+ self.assertGreater(scores["70s-item"], scores["modern-item"])+ self.assertEqual(scores["ghost"], 0.0)+++class SearchPersonalizeTests(unittest.IsolatedAsyncioTestCase):+ async def _search(self, body: SearchRequestBody, chunks: list):+ from src.services.api.controllers import search as search_mod++ mock_data = MagicMock()+ mock_data.search_bm25.return_value = chunks+ mock_data.get_text_chunks_by_ids.return_value = (+ [item[0] for item in chunks],+ None,+ )+ with (+ patch.object(search_mod.config, "search_enabled", True),+ patch.object(search_mod.config, "search_use_dense", False),+ patch.object(search_mod.config, "search_use_bm25", True),+ patch.object(search_mod.config, "search_literal_fill", False),+ patch.object(search_mod, "embeddings_adapter", MagicMock()),+ patch.object(search_mod, "vector_search", MagicMock()),+ patch.object(search_mod, "data_adapter", mock_data),+ ):+ return await search_mod.search(body)++ def test_schema_allows_target_not_user_id(self):+ body = SearchRequestBody(query="lamp", target="u01")+ self.assertEqual(body.target, "u01")+ self.assertEqual(body.channels, ["passages"])+ fields = set(SearchRequestBody.model_fields)+ self.assertIn("target", fields)+ for banned in ("product_id", "sku", "brand", "user_id"):+ self.assertNotIn(banned, fields)++ async def test_omitted_target_keeps_retrieve_order(self):+ chunks = [+ (TextChunk(id="first", text="DOCID 70s-item. lamp"), 9.0),+ (TextChunk(id="second", text="DOCID modern-item. lamp"), 8.0),+ (TextChunk(id="third", text="no join"), 7.0),+ ]+ response = await self._search(SearchRequestBody(query="lamp", k=3), chunks)+ self.assertEqual([hit.id for hit in response.hits], ["first", "second", "third"])+ self.assertTrue(all(hit.scores.personalize is None for hit in response.hits))++ async def test_target_promotes_70s_on_generic_query(self):+ graph, _ = _style_graph()+ chunks = [+ (TextChunk(id="first", text="DOCID modern-item. lamp"), 9.0),+ (TextChunk(id="second", text="DOCID 70s-item. lamp"), 8.0),+ (TextChunk(id="third", text="no join lamp"), 7.0),+ ]+ with patch("src.core.search.personalize.graph_adapter", graph):+ response = await self._search(+ SearchRequestBody(query="lamp", k=3, target="u01"),+ chunks,+ )+ ids = [hit.id for hit in response.hits]+ self.assertEqual(ids[0], "second")+ self.assertIn("third", ids)+ self.assertEqual(response.hits[0].node_id, "70s-item")+ self.assertIsNotNone(response.hits[0].scores.personalize)++ async def test_specific_query_does_not_reorder(self):+ graph, _ = _style_graph()+ chunks = [+ (TextChunk(id="first", text="DOCID modern-item. oak"), 9.0),+ (TextChunk(id="second", text="DOCID 70s-item. oak"), 8.0),+ ]+ with patch("src.core.search.personalize.graph_adapter", graph):+ response = await self._search(+ SearchRequestBody(+ query="oak dining table 180cm", k=2, target="u01"+ ),+ chunks,+ )+ self.assertEqual([hit.id for hit in response.hits], ["first", "second"])++ async def test_extras_and_survives_personalize(self):+ graph, _ = _style_graph()+ chunks = [+ (+ TextChunk(+ id="first",+ text="DOCID modern-item. lamp",+ metadata={"color": "navy"},+ ),+ 9.0,+ ),+ (+ TextChunk(+ id="second",+ text="DOCID 70s-item. lamp",+ metadata={"color": "olive"},+ ),+ 8.0,+ ),+ (+ TextChunk(+ id="third",+ text="DOCID other-item. lamp",+ metadata={"color": "navy"},+ ),+ 7.0,+ ),+ ]+ with patch("src.core.search.personalize.graph_adapter", graph):+ response = await self._search(+ SearchRequestBody(+ query="lamp",+ k=3,+ target="u01",+ extras={"color": "navy"},+ ),+ chunks,+ )+ ids = [hit.id for hit in response.hits]+ self.assertNotIn("second", ids)+ self.assertEqual(set(ids), {"first", "third"})+ self.assertTrue(+ all((hit.extras or {}).get("color") == "navy" for hit in response.hits)+ )+ self.assertTrue(+ all((hit.extras or {}).get("style") != "70s" for hit in response.hits)+ )++ def test_personalize_never_filters_ids(self):+ graph, _ = _style_graph()+ with patch("src.core.search.personalize.graph_adapter", graph):+ ordered, _ = personalize_ranked_ids(+ query="lamp",+ ranked_ids=["a", "b"],+ retrieve_scores={"a": 2, "b": 1},+ node_id_by_hit={"a": "modern-item", "b": "70s-item"},+ target="u01",+ brain_id="brain",+ )+ self.assertEqual(set(ordered), {"a", "b"})diff --git a/tests/test_search_plugins.py b/tests/test_search_plugins.pynew file mode 100644index 0000000..70c762b--- /dev/null+++ b/tests/test_search_plugins.py@@ -0,0 +1,452 @@+import os+import sys+import unittest+from pathlib import Path+from unittest.mock import MagicMock, patch++ENV_DEFAULTS = {+ "BRAINPAT_TOKEN": "test-token",+ "MODELS_MODE": "local",+ "EMBEDDINGS_LOCAL_MODEL": "local-model",+ "EMBEDDINGS_SMALL_MODEL": "small-model",+ "EMBEDDING_NODES_DIMENSION": "3",+ "EMBEDDING_TRIPLETS_DIMENSION": "3",+ "EMBEDDING_OBSERVATIONS_DIMENSION": "3",+ "EMBEDDING_DATA_DIMENSION": "3",+ "EMBEDDING_RELATIONSHIPS_DIMENSION": "3",+ "REDIS_HOST": "localhost",+ "REDIS_PORT": "6379",+ "NEO4J_HOST": "localhost",+ "NEO4J_PORT": "7687",+ "NEO4J_USERNAME": "neo4j",+ "NEO4J_PASSWORD": "password",+ "MILVUS_HOST": "localhost",+ "MILVUS_PORT": "19530",+ "MONGO_CONNECTION_STRING": "mongodb://localhost:27017",+ "CELERY_WORKER_CONCURRENCY": "1",+ "OLLAMA_HOST": "localhost",+ "OLLAMA_PORT": "11434",+ "OLLAMA_LLM_SMALL_MODEL": "small",+ "OLLAMA_LLM_LARGE_MODEL": "large",+}+for key, value in ENV_DEFAULTS.items():+ os.environ.setdefault(key, value)++from fastapi import HTTPException++from src.constants.data import TextChunk+from src.core.search.hooks import (+ register_search_reranker,+ register_search_retriever,+ reset_search_plugins,+)+from src.services.api.constants.requests import SearchRequestBody++ROOT = Path(__file__).resolve().parents[1]+++class SearchPluginHookTests(unittest.IsolatedAsyncioTestCase):+ def setUp(self):+ reset_search_plugins()++ def tearDown(self):+ reset_search_plugins()++ async def test_unknown_rerank_plugin_is_400(self):+ from src.services.api.controllers import search as search_mod++ with patch.object(search_mod.config, "search_enabled", True):+ with self.assertRaises(HTTPException) as ctx:+ await search_mod.search(+ SearchRequestBody(+ query="license",+ rerank="plugin:missing-ce",+ )+ )+ self.assertEqual(ctx.exception.status_code, 400)+ self.assertIn("Unknown search rerank plugin", str(ctx.exception.detail))++ async def test_unknown_retriever_plugin_is_400(self):+ from src.services.api.controllers import search as search_mod++ with patch.object(search_mod.config, "search_enabled", True):+ with self.assertRaises(HTTPException) as ctx:+ await search_mod.search(+ SearchRequestBody(+ query="license",+ channels=["plugin:splade"],+ )+ )+ self.assertEqual(ctx.exception.status_code, 400)+ self.assertIn("Unknown search retriever plugin", str(ctx.exception.detail))++ async def test_linear_rerank_is_400_not_silent(self):+ from src.services.api.controllers import search as search_mod++ with patch.object(search_mod.config, "search_enabled", True):+ with self.assertRaises(HTTPException) as ctx:+ await search_mod.search(+ SearchRequestBody(query="license", rerank="linear")+ )+ self.assertEqual(ctx.exception.status_code, 400)++ async def test_cross_encoder_reranks_top_hits(self):+ from src.services.api.controllers import search as search_mod++ def reverse_rerank(query, candidates, k):+ ordered = list(reversed(candidates))+ for index, item in enumerate(ordered):+ item["score"] = 10 - index+ return ordered[:k]++ register_search_reranker("cross-encoder", reverse_rerank)+ mock_data = MagicMock()+ mock_data.search_bm25.return_value = [+ (TextChunk(id="first", text="alpha"), 9.0),+ (TextChunk(id="second", text="beta"), 8.0),+ (TextChunk(id="third", text="gamma"), 7.0),+ ]+ mock_embeddings = MagicMock()+ mock_vs = MagicMock()++ with (+ patch.object(search_mod.config, "search_enabled", True),+ patch.object(search_mod.config, "search_use_dense", False),+ patch.object(search_mod.config, "search_use_bm25", True),+ patch.object(search_mod, "embeddings_adapter", mock_embeddings),+ patch.object(search_mod, "vector_search", mock_vs),+ patch.object(search_mod, "data_adapter", mock_data),+ ):+ response = await search_mod.search(+ SearchRequestBody(+ query="license",+ k=3,+ rerank="plugin:cross-encoder",+ )+ )++ self.assertEqual([hit.id for hit in response.hits], ["third", "second", "first"])+ self.assertEqual(response.hits[0].scores.rerank, 10)+ mock_embeddings.embed_text.assert_not_called()++ def test_catalog_retrieve_and_rerank_caps(self):+ from src.core.search.hooks import (+ CATALOG_RERANK_MAX_K,+ RERANK_MAX_K,+ rerank_max_k_for_mode,+ retrieve_k_for_mode,+ )++ self.assertEqual(RERANK_MAX_K, 10)+ self.assertEqual(CATALOG_RERANK_MAX_K, 50)+ self.assertEqual(retrieve_k_for_mode("default", 10), 10)+ self.assertEqual(retrieve_k_for_mode("catalog", 10), 50)+ self.assertEqual(retrieve_k_for_mode("catalog", 50), 50)+ self.assertEqual(retrieve_k_for_mode("catalog", 200), 200)+ self.assertEqual(rerank_max_k_for_mode("default"), 10)+ self.assertEqual(rerank_max_k_for_mode("catalog"), 50)++ async def test_default_reranks_at_most_ten(self):+ from src.core.search.hooks import RERANK_MAX_K+ from src.services.api.controllers import search as search_mod++ seen: list[tuple[int, int]] = []++ def capture_rerank(query, candidates, k):+ seen.append((len(candidates), k))+ return list(candidates)[:k]++ register_search_reranker("cross-encoder", capture_rerank)+ mock_data = MagicMock()+ mock_data.search_bm25.return_value = [+ (TextChunk(id=f"h{index}", text=f"t{index}"), float(60 - index))+ for index in range(50)+ ]+ mock_embeddings = MagicMock()+ mock_vs = MagicMock()++ with (+ patch.object(search_mod.config, "search_enabled", True),+ patch.object(search_mod.config, "search_use_dense", False),+ patch.object(search_mod.config, "search_use_bm25", True),+ patch.object(search_mod, "embeddings_adapter", mock_embeddings),+ patch.object(search_mod, "vector_search", mock_vs),+ patch.object(search_mod, "data_adapter", mock_data),+ ):+ response = await search_mod.search(+ SearchRequestBody(+ query="license",+ k=50,+ rerank="plugin:cross-encoder",+ )+ )++ self.assertEqual(RERANK_MAX_K, 10)+ self.assertEqual(seen, [(10, 10)])+ self.assertEqual(len(response.hits), 50)+ mock_data.search_bm25.assert_called_with("license", "default", limit=50)++ async def test_catalog_mode_reranks_fifty_when_k_fifty(self):+ from src.core.search.hooks import CATALOG_RERANK_MAX_K+ from src.services.api.controllers import search as search_mod++ seen: list[tuple[int, int]] = []++ def capture_rerank(query, candidates, k):+ seen.append((len(candidates), k))+ ordered = list(reversed(candidates))+ return ordered[:k]++ register_search_reranker("cross-encoder", capture_rerank)+ mock_data = MagicMock()+ mock_data.search_bm25.return_value = [+ (TextChunk(id=f"h{index}", text=f"t{index}"), float(60 - index))+ for index in range(50)+ ]+ mock_embeddings = MagicMock()+ mock_vs = MagicMock()++ with (+ patch.object(search_mod.config, "search_enabled", True),+ patch.object(search_mod.config, "search_use_dense", False),+ patch.object(search_mod.config, "search_use_bm25", True),+ patch.object(search_mod, "embeddings_adapter", mock_embeddings),+ patch.object(search_mod, "vector_search", mock_vs),+ patch.object(search_mod, "data_adapter", mock_data),+ ):+ response = await search_mod.search(+ SearchRequestBody(+ query="license",+ k=50,+ mode="catalog",+ rerank="plugin:cross-encoder",+ )+ )++ self.assertEqual(CATALOG_RERANK_MAX_K, 50)+ self.assertEqual(seen, [(50, 50)])+ self.assertEqual(len(response.hits), 50)+ self.assertEqual(response.hits[0].id, "h49")+ mock_data.search_bm25.assert_called_with("license", "default", limit=50)++ async def test_catalog_mode_retrieves_fifty_when_k_ten(self):+ from src.services.api.controllers import search as search_mod++ seen: list[tuple[int, int]] = []++ def capture_rerank(query, candidates, k):+ seen.append((len(candidates), k))+ return list(candidates)[:k]++ register_search_reranker("cross-encoder", capture_rerank)+ mock_data = MagicMock()+ mock_data.search_bm25.return_value = [+ (TextChunk(id=f"h{index}", text=f"t{index}"), float(60 - index))+ for index in range(50)+ ]+ mock_embeddings = MagicMock()+ mock_vs = MagicMock()++ with (+ patch.object(search_mod.config, "search_enabled", True),+ patch.object(search_mod.config, "search_use_dense", False),+ patch.object(search_mod.config, "search_use_bm25", True),+ patch.object(search_mod, "embeddings_adapter", mock_embeddings),+ patch.object(search_mod, "vector_search", mock_vs),+ patch.object(search_mod, "data_adapter", mock_data),+ ):+ response = await search_mod.search(+ SearchRequestBody(+ query="license",+ k=10,+ mode="catalog",+ rerank="plugin:cross-encoder",+ )+ )++ self.assertEqual(seen, [(50, 50)])+ self.assertEqual(len(response.hits), 10)+ mock_data.search_bm25.assert_called_with("license", "default", limit=50)++ async def test_splade_channel_fuses_without_silent_bm25_fallback(self):+ from src.services.api.controllers import search as search_mod++ def fake_splade(query, brain_id, k):+ return ["splade-hit"], {"splade-hit": 4.2}, {"splade-hit": "sparse hit"}++ register_search_retriever("splade", fake_splade)+ mock_data = MagicMock()+ mock_embeddings = MagicMock()+ mock_vs = MagicMock()++ with (+ patch.object(search_mod.config, "search_enabled", True),+ patch.object(search_mod.config, "search_use_dense", False),+ patch.object(search_mod.config, "search_use_bm25", True),+ patch.object(search_mod, "embeddings_adapter", mock_embeddings),+ patch.object(search_mod, "vector_search", mock_vs),+ patch.object(search_mod, "data_adapter", mock_data),+ ):+ response = await search_mod.search(+ SearchRequestBody(+ query="license",+ k=5,+ channels=["plugin:splade"],+ )+ )++ self.assertEqual([hit.id for hit in response.hits], ["splade-hit"])+ self.assertEqual(response.hits[0].scores.plugin["splade"], 4.2)+ mock_data.search_bm25.assert_not_called()++ async def test_plugin_lists_fill_frozen_tail_not_head(self):+ from src.services.api.controllers import search as search_mod++ def fake_splade(query, brain_id, k):+ return (+ ["plugin-extra"],+ {"plugin-extra": 9.9},+ {"plugin-extra": "sparse extra"},+ )++ register_search_retriever("splade", fake_splade)+ mock_data = MagicMock()+ mock_data.search_bm25.return_value = [+ (TextChunk(id=f"h{index}", text=f"t{index}"), float(20 - index))+ for index in range(12)+ ]+ mock_embeddings = MagicMock()+ mock_vs = MagicMock()++ with (+ patch.object(search_mod.config, "search_enabled", True),+ patch.object(search_mod.config, "search_use_dense", False),+ patch.object(search_mod.config, "search_use_bm25", True),+ patch.object(search_mod.config, "search_literal_fill", False),+ patch.object(search_mod, "embeddings_adapter", mock_embeddings),+ patch.object(search_mod, "vector_search", mock_vs),+ patch.object(search_mod, "data_adapter", mock_data),+ ):+ response = await search_mod.search(+ SearchRequestBody(+ query="license",+ k=15,+ channels=["passages", "plugin:splade"],+ )+ )++ ids = [hit.id for hit in response.hits]+ self.assertEqual(ids[:10], [f"h{index}" for index in range(10)])+ self.assertIn("plugin-extra", ids[10:])+ self.assertNotIn("plugin-extra", ids[:10])++ def test_context_controller_does_not_import_search_hooks(self):+ retrieve = ROOT / "src" / "services" / "api" / "controllers" / "retrieve.py"+ text = retrieve.read_text(encoding="utf-8")+ self.assertNotIn("src.core.search.hooks", text)+ self.assertNotIn("src.core.search.graph_channels", text)+ self.assertNotIn("register_search_reranker", text)+ self.assertNotIn("plugin:cross-encoder", text)+++class SearchPluginPackageTests(unittest.TestCase):+ def setUp(self):+ reset_search_plugins()+ self._extra_paths: list[str] = []+ self._extra_modules: list[str] = []++ def tearDown(self):+ reset_search_plugins()+ for name in self._extra_modules:+ sys.modules.pop(name, None)+ for path in self._extra_paths:+ if path in sys.path:+ sys.path.remove(path)++ def _load_plugin_dir(self, name: str):+ plugin_dir = str(ROOT / "plugins" / name)+ sys.path.insert(0, plugin_dir)+ self._extra_paths.append(plugin_dir)+ return plugin_dir++ def test_cross_encoder_plugin_reranks_with_injected_predict(self):+ self._load_plugin_dir("search-rerank")+ import rerank as ce++ self._extra_modules.extend(["rerank", "main", "routes"])+ ce.set_predict(lambda pairs: [float(len(text)) for _, text in pairs])+ ranked = ce.rerank(+ "q",+ [+ {"id": "short", "text": "ab", "score": 1},+ {"id": "long", "text": "abcdefghij", "score": 1},+ ],+ 10,+ )+ self.assertEqual([row["id"] for row in ranked], ["long", "short"])+ from main import register as register_ce++ class Ctx:+ _app = None++ def register_search_reranker(self, name, fn):+ register_search_reranker(name, fn)++ def register_search_retriever(self, name, fn):+ register_search_retriever(name, fn)++ register_ce(Ctx())+ from src.core.search.hooks import get_search_reranker++ self.assertIsNotNone(get_search_reranker("cross-encoder"))++ def test_splade_index_and_retrieve_with_fake_encoder(self):+ self._load_plugin_dir("search-splade")+ import encode as splade_encode+ import index as splade_index++ self._extra_modules.extend(["encode", "index", "routes", "main"])+ splade_encode.set_encoder(+ lambda text: {token: 1.0 for token in text.split() if token}+ )+ splade_index.index_chunks(+ "searchbenchsmoke",+ [+ {"id": "a", "text": "alice license portland"},+ {"id": "b", "text": "piano thursday"},+ ],+ )+ ids, scores, texts = splade_index.retrieve(+ "alice license", "searchbenchsmoke", 2+ )+ self.assertEqual(ids[0], "a")+ self.assertGreater(scores["a"], scores.get("b", 0))+ self.assertIn("alice", texts["a"])++ def test_colbert_maxsim_retrieve_with_fake_encoder(self):+ self._load_plugin_dir("search-colbert")+ import encode as colbert_encode+ import index as colbert_index++ self._extra_modules.extend(["encode", "index", "routes", "main"])++ def fake_tokens(text: str):+ table = {+ "q": [[1.0, 0.0]],+ "match": [[1.0, 0.0]],+ "other": [[0.0, 1.0]],+ }+ return table.get(text, [[0.0, 0.0]])++ colbert_encode.set_encoder(fake_tokens)+ colbert_index.index_chunks(+ "searchbenchsmoke",+ [{"id": "hit", "text": "match"}, {"id": "miss", "text": "other"}],+ )+ ids, scores, _ = colbert_index.retrieve("q", "searchbenchsmoke", 2)+ self.assertEqual(ids[0], "hit")+ self.assertGreater(scores["hit"], scores["miss"])+++if __name__ == "__main__":+ unittest.main()diff --git a/tests/test_structured_triple_persistence.py b/tests/test_structured_triple_persistence.pyindex 16ba756..d4ca4c3 100644--- a/tests/test_structured_triple_persistence.py+++ b/tests/test_structured_triple_persistence.py@@ -85,6 +85,91 @@ class TripleConversionTests(unittest.TestCase):self.assertIn(("alice", "person"), entities)self.assertIn(("purchase", "event", "01/01/2024"), entities)+ def test_direct_has_is_one_edge_not_event(self):+ from src.core.agents.architect_agent import ingestion_triples_to_relationships+ from src.services.api.constants.requests import IngestionTripleSet++ triple = IngestionTripleSet(+ subject={"name": "bed", "type": "ENTITY", "uuid": "0"},+ subj_event={"name": "HAS", "uuid": "rel-has-1"},+ object={"name": "rubberwood", "type": "ATTR", "uuid": "hub-attr-rubberwood"},+ )+ relationships, entities = ingestion_triples_to_relationships([triple], [])+ self.assertEqual(len(relationships), 1)+ self.assertEqual(relationships[0].tail.uuid, "0")+ self.assertEqual(relationships[0].name, "HAS")+ self.assertEqual(relationships[0].tip.uuid, "hub-attr-rubberwood")+ self.assertEqual(relationships[0].tip.type, "ATTR")+ self.assertNotIn(("has", "event", ""), entities)+ self.assertNotIn(("has", "event"), entities)++ def test_has_wrapper_without_happened_at_collapses(self):+ from src.core.agents.architect_agent import ingestion_triples_to_relationships+ from src.services.api.constants.requests import IngestionTripleSet++ triple = IngestionTripleSet(+ subject={"name": "bed", "type": "ENTITY", "uuid": "0"},+ subj_event={"name": "HAS", "uuid": "rel-1"},+ event={"name": "HAS", "type": "EVENT", "uuid": "evt-has"},+ event_obj={"name": "HAS", "uuid": "rel-2"},+ object={"name": "modern", "type": "ATTR", "uuid": "hub-modern"},+ )+ relationships, entities = ingestion_triples_to_relationships([triple], [])+ self.assertEqual(len(relationships), 1)+ self.assertEqual(relationships[0].tail.uuid, "0")+ self.assertEqual(relationships[0].tip.uuid, "hub-modern")+ self.assertEqual(relationships[0].name, "HAS")+ self.assertTrue(+ all(getattr(rel.tail, "type", "").upper() != "EVENT" for rel in relationships)+ )+ self.assertTrue(+ all(getattr(rel.tip, "type", "").upper() != "EVENT" for rel in relationships)+ )++ def test_has_event_with_happened_at_stays_two_edges(self):+ from src.core.agents.architect_agent import ingestion_triples_to_relationships+ from src.services.api.constants.requests import IngestionTripleSet++ triple = IngestionTripleSet(+ subject={"name": "u1", "type": "USER", "uuid": "user-1"},+ subj_event={"name": "HAS", "uuid": "rel-1"},+ event={+ "name": "HAS",+ "type": "EVENT",+ "uuid": "evt-dated",+ "happened_at": "01/02/2024",+ },+ event_obj={"name": "HAS", "uuid": "rel-2"},+ object={"name": "sku-1", "type": "ENTITY", "uuid": "sku-1"},+ )+ relationships, _ = ingestion_triples_to_relationships([triple], [])+ self.assertEqual(len(relationships), 2)+ self.assertEqual(relationships[0].tip.uuid, "evt-dated")+ self.assertEqual(relationships[1].tail.uuid, "evt-dated")++ def test_direct_prefers_is_one_edge(self):+ from src.core.agents.architect_agent import ingestion_triples_to_relationships+ from src.services.api.constants.requests import IngestionTripleSet++ triple = IngestionTripleSet(+ subject={"name": "u01", "type": "USER", "uuid": "user:u01"},+ subj_event={+ "name": "PREFERS",+ "uuid": "rel-pref-1",+ "amount": 0.2,+ "properties": {"facet": "style", "value": "70s"},+ },+ object={"name": "70s", "type": "ATTR", "uuid": "hub:attr:70s"},+ )+ relationships, entities = ingestion_triples_to_relationships([triple], [])+ self.assertEqual(len(relationships), 1)+ self.assertEqual(relationships[0].tail.uuid, "user:u01")+ self.assertEqual(relationships[0].name, "PREFERS")+ self.assertEqual(relationships[0].tip.uuid, "hub:attr:70s")+ self.assertEqual(relationships[0].amount, 0.2)+ self.assertNotIn(("prefers", "event", ""), entities)+ self.assertNotIn(("prefers", "event"), entities)+class StructuredTriplePersistenceTests(unittest.TestCase):def test_triple_only_persists_without_llm(self):