Graph-Native Infrastructure for Context and Accountable AI Systems
Python agent-memoryaiai-governanceai-infrastructureartificial-intelligencecontext-engineeringcontext-graphsdata-engineeringdecision-intelligencedeveloper-toolsexplainable-aigenerative-aigraph-ragknowledge-graphllmontologyprovenancepythonreasoningsemantic-search
perf(explorer): add indexed search for large graphs
Zohaib Hassnain6f93f429c4ce
diff --git a/bench_search.py b/bench_search.pynew file mode 100644index 00000000..77e616e1--- /dev/null+++ b/bench_search.py@@ -0,0 +1,113 @@+from __future__ import annotations++import argparse+import statistics+import time+from pathlib import Path++from semantica.explorer.session import GraphSession++DEFAULT_GRAPH = Path(r".\demo_out_100k\primekg_demo_graph.json")+DEFAULT_QUERIES = [+ "metformin",+ "metf",+ "AICA ribonucleotide",+ "drug:DB06736",+ "protein",+ "glucophagezz",+]+++def parse_args() -> argparse.Namespace:+ parser = argparse.ArgumentParser(+ description="Benchmark GraphSession.search() against a local explorer graph JSON.",+ )+ parser.add_argument(+ "--graph",+ default=str(DEFAULT_GRAPH),+ help=f"Path to graph JSON (default: {DEFAULT_GRAPH})",+ )+ parser.add_argument(+ "--query",+ dest="queries",+ action="append",+ help="Query to benchmark. Repeat the flag to add multiple queries.",+ )+ parser.add_argument(+ "--rounds",+ type=int,+ default=80,+ help="Total search calls per query, including warmup (default: 80)",+ )+ parser.add_argument(+ "--warmup",+ type=int,+ default=10,+ help="How many initial calls to exclude from warm metrics (default: 10)",+ )+ parser.add_argument(+ "--limit",+ type=int,+ default=8,+ help="Search result limit to pass to GraphSession.search() (default: 8)",+ )+ return parser.parse_args()+++def format_ms(value: float) -> str:+ return f"{value:8.3f}"+++def main() -> None:+ args = parse_args()+ graph_path = Path(args.graph).expanduser().resolve()+ if not graph_path.exists():+ raise FileNotFoundError(f"Graph JSON not found: {graph_path}")++ queries = args.queries or list(DEFAULT_QUERIES)+ warmup = max(0, min(args.warmup, args.rounds - 1))++ print(f"Loading graph: {graph_path}")+ load_started = time.perf_counter()+ session = GraphSession.from_file(str(graph_path))+ load_ms = (time.perf_counter() - load_started) * 1000++ stats = session.get_stats()+ print(+ f"Loaded {stats.get('node_count', 0):,} nodes / {stats.get('edge_count', 0):,} edges "+ f"in {load_ms:.1f} ms"+ )+ print(+ f"Rounds per query: {args.rounds} (warmup excluded from warm stats: {warmup}) | limit={args.limit}"+ )+ print("")++ header = (+ f"{'query':24} {'cold':>8} {'warm_med':>10} {'warm_min':>10} "+ f"{'warm_max':>10} {'warm_avg':>10}"+ )+ print(header)+ print("-" * len(header))++ for query in queries:+ timings: list[float] = []+ for _ in range(args.rounds):+ started = time.perf_counter()+ session.search(query, args.limit, {})+ timings.append((time.perf_counter() - started) * 1000)++ cold = timings[0]+ warm = timings[warmup:] if warmup < len(timings) else timings++ print(+ f"{query[:24]:24} "+ f"{format_ms(cold)} "+ f"{format_ms(statistics.median(warm))} "+ f"{format_ms(min(warm))} "+ f"{format_ms(max(warm))} "+ f"{format_ms(statistics.fmean(warm))}"+ )+++if __name__ == "__main__":+ main()diff --git a/preprocess_primekg.py b/preprocess_primekg.pynew file mode 100644index 00000000..02178f87--- /dev/null+++ b/preprocess_primekg.py@@ -0,0 +1,469 @@+from __future__ import annotations++import argparse+import math+from collections import defaultdict+from dataclasses import dataclass+from pathlib import Path+from typing import Any++import pandas as pd++from semantica.context.context_graph import ContextGraph+++REQUIRED_COLUMNS = [+ "relation",+ "display_relation",+ "x_id",+ "x_type",+ "x_name",+ "x_source",+ "y_id",+ "y_type",+ "y_name",+ "y_source",+]+++@dataclass+class NodeRecord:+ node_id: str+ node_type: str+ content: str+ metadata: dict[str, Any]+++@dataclass+class EdgeRecord:+ source: str+ target: str+ edge_type: str+ metadata: dict[str, Any]+++def parse_args() -> argparse.Namespace:+ parser = argparse.ArgumentParser(+ description="Build a Semantica Explorer demo graph from PrimeKG."+ )+ parser.add_argument("--input", required=True, help="Path to PrimeKG kg.csv")+ parser.add_argument(+ "--output-dir",+ required=True,+ help="Directory where demo CSVs and graph JSON will be written",+ )+ parser.add_argument(+ "--anchors",+ nargs="+",+ required=True,+ help="Anchor terms used to seed the neighborhood search",+ )+ parser.add_argument(+ "--hops",+ type=int,+ default=2,+ help="Number of neighborhood expansion hops (default: 2)",+ )+ parser.add_argument(+ "--chunksize",+ type=int,+ default=200_000,+ help="CSV chunk size for streaming reads (default: 200000)",+ )+ parser.add_argument(+ "--max-edges",+ type=int,+ default=50_000,+ help="Maximum number of unique edges to keep (default: 50000)",+ )+ parser.add_argument(+ "--max-nodes",+ type=int,+ default=150_000,+ help="Maximum number of unique nodes to keep before expansion stops (default: 150000)",+ )+ parser.add_argument(+ "--novelty-fanout-per-node",+ type=int,+ default=96,+ help=(+ "Maximum unseen-node edges to keep per frontier node on each hop. "+ "Higher values trade breadth for local density (default: 96)"+ ),+ )+ parser.add_argument(+ "--context-fanout-per-node",+ type=int,+ default=12,+ help=(+ "Maximum already-known context edges to keep per frontier node on each hop "+ "after novelty edges are collected (default: 12)"+ ),+ )+ return parser.parse_args()+++def normalize_text(value: Any) -> str:+ if value is None:+ return ""+ text = str(value).strip().lower()+ return " ".join(text.split())+++def coerce_scalar(value: Any) -> str:+ if value is None:+ return ""+ if pd.isna(value):+ return ""+ if isinstance(value, float):+ if math.isnan(value):+ return ""+ if value.is_integer():+ return str(int(value))+ return str(value).strip()+++def canonical_node_id(node_type: Any, raw_id: Any) -> str:+ return f"{coerce_scalar(node_type)}:{coerce_scalar(raw_id)}"+++def matches_anchor(name: Any, anchor_terms: set[str]) -> bool:+ normalized = normalize_text(name)+ if not normalized:+ return False+ if normalized in anchor_terms:+ return True+ return any(term in normalized for term in anchor_terms)+++def chunk_reader(csv_path: Path, chunksize: int):+ return pd.read_csv(+ csv_path,+ usecols=REQUIRED_COLUMNS,+ chunksize=chunksize,+ dtype={column: "string" for column in REQUIRED_COLUMNS},+ keep_default_na=False,+ low_memory=False,+ )+++def build_node_record(row: pd.Series, prefix: str) -> NodeRecord:+ node_type = coerce_scalar(row[f"{prefix}_type"])+ raw_id = coerce_scalar(row[f"{prefix}_id"])+ name = coerce_scalar(row[f"{prefix}_name"]) or raw_id+ source = coerce_scalar(row[f"{prefix}_source"])+ node_id = canonical_node_id(node_type, raw_id)+ metadata = {+ "name": name,+ "source_system": source,+ "primekg_id": raw_id,+ "category": node_type,+ }+ return NodeRecord(+ node_id=node_id,+ node_type=node_type or "entity",+ content=name,+ metadata=metadata,+ )+++def build_edge_record(row: pd.Series) -> EdgeRecord:+ relation = coerce_scalar(row["relation"]) or "related_to"+ display_relation = coerce_scalar(row["display_relation"]) or relation+ source = canonical_node_id(row["x_type"], row["x_id"])+ target = canonical_node_id(row["y_type"], row["y_id"])+ metadata = {+ "display_relation": display_relation,+ "source_type": coerce_scalar(row["x_type"]),+ "target_type": coerce_scalar(row["y_type"]),+ "source_name": coerce_scalar(row["x_name"]),+ "target_name": coerce_scalar(row["y_name"]),+ "source_system_x": coerce_scalar(row["x_source"]),+ "source_system_y": coerce_scalar(row["y_source"]),+ "weight": 1.0,+ }+ return EdgeRecord(source=source, target=target, edge_type=relation, metadata=metadata)+++def find_anchor_nodes(csv_path: Path, anchor_terms: set[str], chunksize: int) -> dict[str, NodeRecord]:+ anchor_nodes: dict[str, NodeRecord] = {}++ for chunk in chunk_reader(csv_path, chunksize):+ for _, row in chunk.iterrows():+ if matches_anchor(row["x_name"], anchor_terms):+ record = build_node_record(row, "x")+ anchor_nodes.setdefault(record.node_id, record)+ if matches_anchor(row["y_name"], anchor_terms):+ record = build_node_record(row, "y")+ anchor_nodes.setdefault(record.node_id, record)++ return anchor_nodes+++def expand_neighborhood(+ csv_path: Path,+ seed_nodes: dict[str, NodeRecord],+ hops: int,+ chunksize: int,+ max_edges: int,+ max_nodes: int,+ novelty_fanout_per_node: int,+ context_fanout_per_node: int,+) -> tuple[dict[str, NodeRecord], list[EdgeRecord]]:+ selected_nodes: dict[str, NodeRecord] = dict(seed_nodes)+ selected_edges: dict[tuple[str, str, str], EdgeRecord] = {}++ frontier = set(seed_nodes.keys())+ visited = set(seed_nodes.keys())++ def _frontier_endpoints(source_id: str, target_id: str) -> list[str]:+ endpoints = []+ if source_id in frontier:+ endpoints.append(source_id)+ if target_id in frontier and target_id != source_id:+ endpoints.append(target_id)+ return endpoints++ def _can_accept_edge(+ active_frontier_nodes: list[str],+ counts: dict[str, int],+ limit_per_node: int,+ ) -> bool:+ if not active_frontier_nodes or limit_per_node <= 0:+ return limit_per_node <= 0 and not active_frontier_nodes+ return any(counts[node_id] < limit_per_node for node_id in active_frontier_nodes)++ def _mark_edge_consumed(active_frontier_nodes: list[str], counts: dict[str, int]) -> None:+ for node_id in active_frontier_nodes:+ counts[node_id] += 1++ for hop_index in range(hops):+ if (+ not frontier+ or len(selected_edges) >= max_edges+ or len(selected_nodes) >= max_nodes+ ):+ break++ next_frontier: set[str] = set()+ novelty_counts: dict[str, int] = defaultdict(int)+ context_counts: dict[str, int] = defaultdict(int)+ print(f"Scanning hop {hop_index + 1}/{hops} from {len(frontier)} frontier nodes...")++ start_node_count = len(selected_nodes)+ start_edge_count = len(selected_edges)+ stop = False++ # Pass 1: prioritize edges that introduce at least one unseen node.+ for chunk in chunk_reader(csv_path, chunksize):+ for _, row in chunk.iterrows():+ source_id = canonical_node_id(row["x_type"], row["x_id"])+ target_id = canonical_node_id(row["y_type"], row["y_id"])++ if source_id not in frontier and target_id not in frontier:+ continue+ if source_id in selected_nodes and target_id in selected_nodes:+ continue++ active_frontier_nodes = _frontier_endpoints(source_id, target_id)+ if not _can_accept_edge(+ active_frontier_nodes,+ novelty_counts,+ novelty_fanout_per_node,+ ):+ continue++ new_node_ids = [node_id for node_id in (source_id, target_id) if node_id not in selected_nodes]+ if new_node_ids and len(selected_nodes) + len(new_node_ids) > max_nodes:+ stop = True+ break++ edge_record = build_edge_record(row)+ edge_key = (edge_record.source, edge_record.target, edge_record.edge_type)+ if edge_key in selected_edges:+ continue++ source_record = build_node_record(row, "x")+ target_record = build_node_record(row, "y")+ selected_nodes.setdefault(source_record.node_id, source_record)+ selected_nodes.setdefault(target_record.node_id, target_record)++ selected_edges[edge_key] = edge_record+ _mark_edge_consumed(active_frontier_nodes, novelty_counts)+ if len(selected_edges) >= max_edges or len(selected_nodes) >= max_nodes:+ stop = True++ if source_id not in visited:+ next_frontier.add(source_id)+ if target_id not in visited:+ next_frontier.add(target_id)++ if stop:+ break+ if stop:+ break++ # Pass 2: add a small amount of context among already-selected nodes so the+ # extracted graph doesn't look amputated, but keep this secondary to novelty.+ if (+ not stop+ and context_fanout_per_node > 0+ and len(selected_edges) < max_edges+ ):+ for chunk in chunk_reader(csv_path, chunksize):+ for _, row in chunk.iterrows():+ source_id = canonical_node_id(row["x_type"], row["x_id"])+ target_id = canonical_node_id(row["y_type"], row["y_id"])++ if source_id not in frontier and target_id not in frontier:+ continue+ if source_id not in selected_nodes or target_id not in selected_nodes:+ continue++ edge_record = build_edge_record(row)+ edge_key = (edge_record.source, edge_record.target, edge_record.edge_type)+ if edge_key in selected_edges:+ continue++ active_frontier_nodes = _frontier_endpoints(source_id, target_id)+ if not _can_accept_edge(+ active_frontier_nodes,+ context_counts,+ context_fanout_per_node,+ ):+ continue++ selected_edges[edge_key] = edge_record+ _mark_edge_consumed(active_frontier_nodes, context_counts)+ if len(selected_edges) >= max_edges:+ stop = True+ break+ if stop:+ break++ next_frontier -= visited+ visited |= next_frontier+ frontier = next_frontier+ print(+ f" Added {len(selected_nodes) - start_node_count:,} node(s) and "+ f"{len(selected_edges) - start_edge_count:,} edge(s) on this hop."+ )++ return selected_nodes, list(selected_edges.values())+++def write_csvs(output_dir: Path, nodes: dict[str, NodeRecord], edges: list[EdgeRecord]) -> None:+ node_rows = []+ for node in nodes.values():+ node_rows.append(+ {+ "id": node.node_id,+ "name": node.content,+ "type": node.node_type,+ "source_system": node.metadata.get("source_system", ""),+ "primekg_id": node.metadata.get("primekg_id", ""),+ }+ )++ edge_rows = []+ for edge in edges:+ edge_rows.append(+ {+ "source": edge.source,+ "target": edge.target,+ "type": edge.edge_type,+ "display_relation": edge.metadata.get("display_relation", edge.edge_type),+ "source_name": edge.metadata.get("source_name", ""),+ "target_name": edge.metadata.get("target_name", ""),+ }+ )++ pd.DataFrame(node_rows).sort_values(["type", "name", "id"]).to_csv(+ output_dir / "primekg_demo_nodes.csv", index=False+ )+ pd.DataFrame(edge_rows).to_csv(output_dir / "primekg_demo_edges.csv", index=False)+++def write_graph_json(output_dir: Path, nodes: dict[str, NodeRecord], edges: list[EdgeRecord]) -> Path:+ graph = ContextGraph()++ graph.add_nodes(+ [+ {+ "id": node.node_id,+ "type": node.node_type,+ "content": node.content,+ "properties": node.metadata,+ }+ for node in nodes.values()+ ]+ )++ graph.add_edges(+ [+ {+ "source": edge.source,+ "target": edge.target,+ "type": edge.edge_type,+ "weight": 1.0,+ "properties": edge.metadata,+ }+ for edge in edges+ ]+ )++ graph_path = output_dir / "primekg_demo_graph.json"+ graph.save_to_file(str(graph_path))+ return graph_path+++def main() -> None:+ args = parse_args()+ csv_path = Path(args.input).expanduser().resolve()+ output_dir = Path(args.output_dir).expanduser().resolve()++ if not csv_path.exists():+ raise FileNotFoundError(f"Input CSV not found: {csv_path}")++ output_dir.mkdir(parents=True, exist_ok=True)++ anchor_terms = {normalize_text(term) for term in args.anchors if normalize_text(term)}+ if not anchor_terms:+ raise ValueError("At least one non-empty anchor term is required.")++ print(f"Loading anchors from {csv_path} ...")+ seed_nodes = find_anchor_nodes(csv_path, anchor_terms, args.chunksize)+ if not seed_nodes:+ raise RuntimeError(+ "No anchor nodes matched the provided terms. Check the anchor spellings against kg.csv."+ )++ print(f"Matched {len(seed_nodes)} anchor node(s).")+ nodes, edges = expand_neighborhood(+ csv_path=csv_path,+ seed_nodes=seed_nodes,+ hops=args.hops,+ chunksize=args.chunksize,+ max_edges=args.max_edges,+ max_nodes=args.max_nodes,+ novelty_fanout_per_node=args.novelty_fanout_per_node,+ context_fanout_per_node=args.context_fanout_per_node,+ )++ write_csvs(output_dir, nodes, edges)+ graph_path = write_graph_json(output_dir, nodes, edges)++ print("")+ print("PrimeKG demo graph ready.")+ print(f"Nodes: {len(nodes):,}")+ print(f"Edges: {len(edges):,}")+ print(f"Output directory: {output_dir}")+ print(f"Graph JSON: {graph_path}")+ print("")+ print(+ f"Next: python -m semantica.explorer --graph {graph_path} --host 127.0.0.1 --port 8000 --no-browser"+ )+++if __name__ == "__main__":+ main()diff --git a/semantica/explorer/app.py b/semantica/explorer/app.pyindex a9fa25ec..4ab5a020 100644--- a/semantica/explorer/app.py+++ b/semantica/explorer/app.py@@ -19,7 +19,12 @@ from .ws import ConnectionManagerdef _install_mutation_bridge(app: FastAPI, session: GraphSession) -> None:+ previous_callback = getattr(session.graph, "mutation_callback", None)+def on_mutation(event_type: str, entity_id: str, payload: dict) -> None:+ session.handle_graph_mutation(event_type, entity_id, payload)+ if callable(previous_callback):+ previous_callback(event_type, entity_id, payload)loop = getattr(app.state, "event_loop", None)manager = getattr(app.state, "ws_manager", None)if loop is None or manager is None or loop.is_closed():diff --git a/semantica/explorer/routes/enrich.py b/semantica/explorer/routes/enrich.pyindex be87c783..c7df5aff 100644--- a/semantica/explorer/routes/enrich.py+++ b/semantica/explorer/routes/enrich.py@@ -136,11 +136,11 @@ def _apply_inferred_edges(continuesource, target = argsif session.get_node(source) is None:- session.graph.add_node(source, "entity", content=source)+ session.add_node(source, "entity", content=source)if session.get_node(target) is None:- session.graph.add_node(target, "entity", content=target)+ session.add_node(target, "entity", content=target)edge_type = body.inferred_edge_type or predicate- session.graph.add_edge(+ session.add_edge(source,target,edge_type=edge_type,@@ -353,4 +353,6 @@ async def merge_nodes(return removed, edges_updatedremoved_ids, edges_updated = await asyncio.to_thread(_do_merge)+ if removed_ids:+ await asyncio.to_thread(session.rebuild_search_index)return MergeResponse(merged_into=primary_id, removed_ids=removed_ids, edges_updated=edges_updated)diff --git a/semantica/explorer/search_index.py b/semantica/explorer/search_index.pynew file mode 100644index 00000000..64c51a3b--- /dev/null+++ b/semantica/explorer/search_index.py@@ -0,0 +1,400 @@+"""+Explorer-local in-memory node search index.+"""++from __future__ import annotations++import heapq+import re+from collections import OrderedDict, defaultdict+from dataclasses import dataclass+from typing import Any, DefaultDict, Dict, Iterable, List, Optional, Tuple++_TOKEN_RE = re.compile(r"[a-z0-9]+")+_WHITESPACE_RE = re.compile(r"\s+")+_CURATED_ALIAS_KEYS = (+ "label",+ "name",+ "title",+ "pref_label",+ "preferred_label",+ "prefLabel",+ "aliases",+ "alias",+ "synonyms",+ "synonym",+ "symbol",+ "display_name",+ "displayName",+ "text",+ "content",+)+++def _normalize_text(value: Any) -> str:+ if value is None:+ return ""+ text = str(value).strip().lower()+ if not text:+ return ""+ return _WHITESPACE_RE.sub(" ", text)+++def _tokenize(text: str) -> Tuple[str, ...]:+ if not text:+ return ()+ return tuple(dict.fromkeys(_TOKEN_RE.findall(text)))+++def _collect_text_fragments(value: Any, fragments: List[str], *, limit: int = 64) -> None:+ if value is None or len(fragments) >= limit:+ return+ if isinstance(value, dict):+ for nested in value.values():+ _collect_text_fragments(nested, fragments, limit=limit)+ if len(fragments) >= limit:+ return+ return+ if isinstance(value, (list, tuple, set)):+ for nested in value:+ _collect_text_fragments(nested, fragments, limit=limit)+ if len(fragments) >= limit:+ return+ return++ normalized = _normalize_text(value)+ if normalized:+ fragments.append(normalized)+++def _coerce_float(value: Any) -> Optional[float]:+ if value is None or value == "":+ return None+ try:+ return float(value)+ except (TypeError, ValueError):+ return None+++@dataclass(frozen=True)+class IndexedNodeDocument:+ node_id: str+ normalized_id: str+ node_type: str+ exact_terms: frozenset[str]+ tokens: frozenset[str]+ primary_text: str+ secondary_text: str+ confidence: Optional[float]+ tags: Tuple[str, ...]+++class GraphSearchIndex:+ def __init__(+ self,+ *,+ cache_size: int = 128,+ prefix_min_length: int = 2,+ prefix_max_length: int = 12,+ secondary_scan_limit: int = 12000,+ ) -> None:+ self.cache_size = cache_size+ self.prefix_min_length = prefix_min_length+ self.prefix_max_length = prefix_max_length+ self.secondary_scan_limit = secondary_scan_limit+ self._documents: Dict[str, IndexedNodeDocument] = {}+ self._exact_index: DefaultDict[str, set[str]] = defaultdict(set)+ self._token_index: DefaultDict[str, set[str]] = defaultdict(set)+ self._prefix_index: DefaultDict[str, set[str]] = defaultdict(set)+ self._ordered_node_ids: List[str] = []+ self._cache: OrderedDict[Tuple[Any, ...], List[Tuple[str, float]]] = OrderedDict()++ def rebuild(self, nodes: Iterable[Dict[str, Any]]) -> None:+ self._documents.clear()+ self._exact_index.clear()+ self._token_index.clear()+ self._prefix_index.clear()+ self._ordered_node_ids = []+ self.clear_cache()++ for node in nodes:+ self.upsert(node, clear_cache=False)++ self._ordered_node_ids.sort()++ def clear_cache(self) -> None:+ self._cache.clear()++ def remove(self, node_id: str, *, clear_cache: bool = True) -> None:+ existing = self._documents.pop(node_id, None)+ if existing is None:+ return++ for term in existing.exact_terms:+ bucket = self._exact_index.get(term)+ if bucket is None:+ continue+ bucket.discard(node_id)+ if not bucket:+ self._exact_index.pop(term, None)++ for token in existing.tokens:+ bucket = self._token_index.get(token)+ if bucket is None:+ continue+ bucket.discard(node_id)+ if not bucket:+ self._token_index.pop(token, None)++ for length in range(self.prefix_min_length, min(len(token), self.prefix_max_length) + 1):+ prefix = token[:length]+ prefix_bucket = self._prefix_index.get(prefix)+ if prefix_bucket is None:+ continue+ prefix_bucket.discard(node_id)+ if not prefix_bucket:+ self._prefix_index.pop(prefix, None)++ try:+ self._ordered_node_ids.remove(node_id)+ except ValueError:+ pass++ if clear_cache:+ self.clear_cache()++ def upsert(self, node: Dict[str, Any], *, clear_cache: bool = True) -> None:+ node_id = str(node.get("id", "")).strip()+ if not node_id:+ return++ self.remove(node_id, clear_cache=False)+ document = self._build_document(node)+ self._documents[node_id] = document++ for term in document.exact_terms:+ self._exact_index[term].add(node_id)++ for token in document.tokens:+ self._token_index[token].add(node_id)+ for length in range(self.prefix_min_length, min(len(token), self.prefix_max_length) + 1):+ self._prefix_index[token[:length]].add(node_id)++ if node_id not in self._ordered_node_ids:+ self._ordered_node_ids.append(node_id)+ self._ordered_node_ids.sort()++ if clear_cache:+ self.clear_cache()++ def search(+ self,+ query: str,+ *,+ limit: int = 20,+ filters: Optional[Dict[str, Any]] = None,+ ) -> tuple[List[Tuple[str, float]], Dict[str, Any]]:+ normalized_query = _normalize_text(query)+ filters = filters or {}+ diagnostics: Dict[str, Any] = {+ "cache_hit": False,+ "path": "empty",+ "candidates": 0,+ }+ if not normalized_query:+ return [], diagnostics++ cache_key = self._cache_key(normalized_query, limit, filters)+ cached = self._cache.get(cache_key)+ if cached is not None:+ self._cache.move_to_end(cache_key)+ diagnostics.update({"cache_hit": True, "path": "cache", "candidates": len(cached)})+ return list(cached), diagnostics++ query_tokens = _tokenize(normalized_query)+ exact_ids = set(self._exact_index.get(normalized_query, set()))+ token_sets: List[set[str]] = []+ prefix_sets: List[set[str]] = []+ for token in query_tokens:+ exact_token_ids = set(self._token_index.get(token, set()))+ prefix_ids = set(self._prefix_index.get(token, set())) if len(token) >= self.prefix_min_length else set()+ if exact_token_ids:+ token_sets.append(exact_token_ids)+ if prefix_ids:+ prefix_sets.append(prefix_ids)++ candidate_ids: set[str] = set(exact_ids)+ if token_sets:+ intersected = set.intersection(*token_sets)+ candidate_ids.update(intersected if intersected else set().union(*token_sets))+ if prefix_sets:+ candidate_ids.update(set().union(*prefix_sets))++ diagnostics["path"] = "index"++ if not candidate_ids:+ diagnostics["path"] = "secondary_scan"+ candidate_ids = self._secondary_scan(normalized_query, limit)++ diagnostics["candidates"] = len(candidate_ids)++ scored: List[Tuple[float, int, int, str]] = []+ for node_id in candidate_ids:+ document = self._documents.get(node_id)+ if document is None or not self._passes_filters(document, filters):+ continue+ score = self._score_document(document, normalized_query, query_tokens)+ if score <= 0:+ continue+ token_hits = sum(1 for token in query_tokens if token in document.tokens)+ exactness = 1 if normalized_query == document.normalized_id or normalized_query in document.exact_terms else 0+ scored.append((score, exactness, token_hits, node_id))++ top_matches = heapq.nlargest(limit, scored, key=lambda item: (item[0], item[1], item[2], item[3]))+ results = [(node_id, round(score, 4)) for score, _, _, node_id in top_matches]+ self._store_cache(cache_key, results)+ return results, diagnostics++ def _secondary_scan(self, normalized_query: str, limit: int) -> set[str]:+ matches: set[str] = set()+ max_hits = max(limit * 20, 200)+ scanned = 0+ for node_id in self._ordered_node_ids:+ if scanned >= self.secondary_scan_limit or len(matches) >= max_hits:+ break+ scanned += 1+ document = self._documents.get(node_id)+ if document is None:+ continue+ if normalized_query in document.primary_text or normalized_query in document.secondary_text:+ matches.add(node_id)+ return matches++ def _score_document(+ self,+ document: IndexedNodeDocument,+ normalized_query: str,+ query_tokens: Tuple[str, ...],+ ) -> float:+ score = 0.0+ if normalized_query == document.normalized_id:+ score = max(score, 140.0)+ elif normalized_query in document.exact_terms:+ score = max(score, 120.0)++ if normalized_query and normalized_query in document.primary_text:+ score = max(score, 78.0 + min(len(normalized_query), 24) / 10.0)+ elif normalized_query and normalized_query in document.secondary_text:+ score = max(score, 26.0 + min(len(normalized_query), 24) / 20.0)++ token_hits = 0+ prefix_hits = 0+ for token in query_tokens:+ if token in document.tokens:+ token_hits += 1+ elif len(token) >= self.prefix_min_length and any(candidate.startswith(token) for candidate in document.tokens):+ prefix_hits += 1++ score += token_hits * 18.0+ score += prefix_hits * 10.0++ if len(query_tokens) > 1 and token_hits:+ score += token_hits * 4.0++ return score++ def _passes_filters(self, document: IndexedNodeDocument, filters: Dict[str, Any]) -> bool:+ filter_type = filters.get("type") or filters.get("node_type")+ if filter_type and document.node_type != str(filter_type):+ return False++ min_confidence = _coerce_float(filters.get("min_confidence"))+ if min_confidence is not None:+ if document.confidence is None or document.confidence < min_confidence:+ return False++ tags_filter = filters.get("tags")+ if tags_filter:+ if isinstance(tags_filter, str):+ required_tags = {_normalize_text(tags_filter)}+ else:+ required_tags = {+ normalized+ for normalized in (_normalize_text(tag) for tag in tags_filter)+ if normalized+ }+ if required_tags and not required_tags.issubset(set(document.tags)):+ return False++ return True++ def _cache_key(+ self,+ normalized_query: str,+ limit: int,+ filters: Dict[str, Any],+ ) -> Tuple[Any, ...]:+ serialized_filters: List[Tuple[str, Any]] = []+ for key in sorted(filters.keys()):+ value = filters[key]+ if isinstance(value, (list, tuple, set)):+ serialized_filters.append((key, tuple(str(item) for item in value)))+ else:+ serialized_filters.append((key, str(value)))+ return normalized_query, limit, tuple(serialized_filters)++ def _store_cache(self, cache_key: Tuple[Any, ...], results: List[Tuple[str, float]]) -> None:+ self._cache[cache_key] = list(results)+ self._cache.move_to_end(cache_key)+ while len(self._cache) > self.cache_size:+ self._cache.popitem(last=False)++ def _build_document(self, node: Dict[str, Any]) -> IndexedNodeDocument:+ node_id = str(node.get("id", "")).strip()+ node_type = str(node.get("type", "entity"))+ properties = dict(node.get("properties", {}) or {})++ primary_terms: List[str] = []+ for candidate in (node_id, node.get("content", "")):+ normalized = _normalize_text(candidate)+ if normalized:+ primary_terms.append(normalized)++ for alias_key in _CURATED_ALIAS_KEYS:+ _collect_text_fragments(properties.get(alias_key), primary_terms, limit=32)++ deduped_primary_terms = tuple(dict.fromkeys(term for term in primary_terms if term))+ primary_text = " ".join(deduped_primary_terms)+ tokens = frozenset(_tokenize(primary_text))++ secondary_fragments: List[str] = []+ for key, value in properties.items():+ if key in _CURATED_ALIAS_KEYS or key in {"content", "valid_from", "valid_until"}:+ continue+ _collect_text_fragments(value, secondary_fragments, limit=48)+ if len(secondary_fragments) >= 48:+ break++ secondary_text = " ".join(dict.fromkeys(fragment for fragment in secondary_fragments if fragment))+ confidence = _coerce_float(properties.get("confidence"))++ raw_tags = properties.get("tags") or []+ if isinstance(raw_tags, str):+ raw_tags = [raw_tags]+ tags = tuple(+ dict.fromkeys(+ normalized for normalized in (_normalize_text(tag) for tag in raw_tags) if normalized+ )+ )++ return IndexedNodeDocument(+ node_id=node_id,+ normalized_id=_normalize_text(node_id),+ node_type=node_type,+ exact_terms=frozenset(deduped_primary_terms),+ tokens=tokens,+ primary_text=primary_text,+ secondary_text=secondary_text,+ confidence=confidence,+ tags=tags,+ )diff --git a/semantica/explorer/session.py b/semantica/explorer/session.pyindex 9600f90f..857a4a3c 100644--- a/semantica/explorer/session.py+++ b/semantica/explorer/session.py@@ -4,12 +4,15 @@ Semantica Explorer session helpers.import base64import json+import loggingimport threading+import timeimport uuid-from datetime import datetime, UTC+from datetime import UTC, datetimefrom typing import Any, Dict, Iterable, List, Optionalfrom ..context.context_graph import ContextGraph, _resolve_edge_identity+from .search_index import GraphSearchIndex_KG_AVAILABLE = Falsetry:@@ -28,6 +31,8 @@ try:except ImportError:pass+logger = logging.getLogger(__name__)+class GraphSession:"""Thread-safe session wrapper around a loaded ``ContextGraph``."""@@ -35,6 +40,7 @@ class GraphSession:def __init__(self, graph: ContextGraph) -> None:self.graph = graphself._lock = threading.RLock()+ self._search_index = GraphSearchIndex()self.annotations: Dict[str, Dict[str, Any]] = {}@@ -46,6 +52,7 @@ class GraphSession:self._similarity: Any = Noneself._link_predictor: Any = Noneself._validator: Any = None+ self.rebuild_search_index()@classmethoddef from_file(cls, path: str) -> "GraphSession":@@ -390,6 +397,26 @@ class GraphSession:with self._lock:return self.graph.get_neighbors(node_id, hops=depth)+ def rebuild_search_index(self) -> None:+ with self._lock:+ normalized_nodes = [+ self.normalize_node(node.to_dict())+ for node in self.graph.nodes.values()+ if node is not None+ ]+ self._search_index.rebuild(normalized_nodes)++ def handle_graph_mutation(self, event_type: str, entity_id: str, payload: Dict[str, Any]) -> None:+ normalized_event = str(event_type or "").upper()+ if normalized_event in {"ADD_NODE", "UPDATE_NODE"}:+ normalized_node = self.normalize_node(payload or {})+ if normalized_node.get("id"):+ self._search_index.upsert(normalized_node)+ elif normalized_event in {"REMOVE_NODE", "DELETE_NODE"}:+ self._search_index.remove(str(entity_id))+ elif normalized_event in {"RELOAD_GRAPH", "RESET_GRAPH"}:+ self.rebuild_search_index()+def search(self,query: str,@@ -397,64 +424,34 @@ class GraphSession:filters: Optional[Dict[str, Any]] = None,) -> List[Dict[str, Any]]:filters = filters or {}- try:- with self._lock:- raw = self.graph.query(query)[:limit]- except Exception:- raw = []-- if not raw:- nodes, _ = self.get_nodes(search=query, skip=0, limit=max(limit * 5, limit))- scored = []- lowered_query = query.lower().strip()- for node in nodes:- haystacks = [- str(node.get("id", "")),- str(node.get("content", "")),- json.dumps(node.get("properties", {}), default=str),- ]- best_score = 0.0- for haystack in haystacks:- lowered = haystack.lower()- if lowered == lowered_query:- best_score = max(best_score, 1.0)- elif lowered_query in lowered:- best_score = max(best_score, min(0.9, len(lowered_query) / max(len(lowered), 1)))- if best_score > 0:- scored.append({"node": node, "score": round(best_score, 4)})- raw = sorted(scored, key=lambda item: item["score"], reverse=True)[:limit]-- normalized = []- for result in raw:- result_node = result.get("node", {})- node = (- self.normalize_node(result_node)- if "properties" in result_node or "metadata" in result_node or "content" in result_node- else result_node- )-- filter_type = filters.get("type") or filters.get("node_type")- if filter_type and node["type"] != filter_type:- continue-- min_confidence = self._coerce_float(filters.get("min_confidence"))- node_confidence = self._coerce_float(node["properties"].get("confidence"))- if min_confidence is not None and (- node_confidence is None or node_confidence < min_confidence- ):- continue+ started_at = time.perf_counter()+ matches, diagnostics = self._search_index.search(query, limit=limit, filters=filters)- tags_filter = filters.get("tags")- if tags_filter:- node_tags = node["properties"].get("tags") or []- if isinstance(node_tags, str):- node_tags = [node_tags]- if not set(tags_filter).issubset(set(node_tags)):+ normalized_results: List[Dict[str, Any]] = []+ with self._lock:+ for node_id, score in matches:+ raw_node = self.graph.find_node(node_id)+ if raw_node is None:continue+ node_payload = raw_node.to_dict() if hasattr(raw_node, "to_dict") else raw_node+ normalized_results.append(+ {+ "node": self.normalize_node(node_payload),+ "score": score,+ }+ )- normalized.append({"node": node, "score": result.get("score", 0.0)})-- return normalized[:limit]+ duration_ms = round((time.perf_counter() - started_at) * 1000, 2)+ logger.debug(+ "Explorer search query=%r limit=%s cache_hit=%s path=%s candidates=%s duration_ms=%s",+ query,+ limit,+ diagnostics.get("cache_hit"),+ diagnostics.get("path"),+ diagnostics.get("candidates"),+ duration_ms,+ )+ return normalized_results[:limit]def get_stats(self) -> Dict[str, Any]:with self._lock:@@ -594,8 +591,58 @@ class GraphSession:def add_nodes(self, nodes: List[Dict[str, Any]]) -> int:with self._lock:- return self.graph.add_nodes(nodes)+ added = self.graph.add_nodes(nodes)+ has_mutation_callback = callable(getattr(self.graph, "mutation_callback", None))+ if added and not has_mutation_callback:+ self.rebuild_search_index()+ return addeddef add_edges(self, edges: List[Dict[str, Any]]) -> int:with self._lock:- return self.graph.add_edges(edges)+ added = self.graph.add_edges(edges)+ has_mutation_callback = callable(getattr(self.graph, "mutation_callback", None))+ if added and not has_mutation_callback:+ self.rebuild_search_index()+ return added++ def add_node(+ self,+ node_id: str,+ node_type: str,+ content: Optional[str] = None,+ **properties: Any,+ ) -> bool:+ with self._lock:+ added = self.graph.add_node(node_id, node_type, content=content, **properties)+ has_mutation_callback = callable(getattr(self.graph, "mutation_callback", None))+ if added and not has_mutation_callback:+ normalized = self.get_node(node_id)+ if normalized is not None:+ self._search_index.upsert(normalized)+ return added++ def add_edge(+ self,+ source_id: str,+ target_id: str,+ edge_type: str = "related_to",+ weight: float = 1.0,+ **properties: Any,+ ) -> bool:+ with self._lock:+ added = self.graph.add_edge(+ source_id,+ target_id,+ edge_type=edge_type,+ weight=weight,+ **properties,+ )+ has_mutation_callback = callable(getattr(self.graph, "mutation_callback", None))+ if added and not has_mutation_callback:+ source = self.get_node(source_id)+ if source is not None:+ self._search_index.upsert(source)+ target = self.get_node(target_id)+ if target is not None:+ self._search_index.upsert(target)+ return addeddiff --git a/tests/explorer/test_explorer_api.py b/tests/explorer/test_explorer_api.pyindex f0d56859..8f30d794 100644--- a/tests/explorer/test_explorer_api.py+++ b/tests/explorer/test_explorer_api.py@@ -35,6 +35,16 @@ def _build_sample_graph() -> ContextGraph:graph.add_node("javascript", node_type="language", content="JavaScript programming language", x=100, y=120)graph.add_node("web_dev", node_type="concept", content="Web Development", x=24, y=30)graph.add_node("ml", node_type="concept", content="Machine Learning", x=45, y=60)+ graph.add_node(+ "metformin",+ node_type="drug",+ content="Metformin",+ aliases=["Glucophage"],+ confidence="0.97",+ tags=["drug", "featured"],+ x=22,+ y=33,+ )graph.add_node("decision_1",node_type="decision",@@ -243,6 +253,57 @@ class TestSearchAndStats:assert payload["total"] >= 1assert all(item["node"]["type"] == "language" for item in payload["results"])+ def test_search_exact_and_prefix(self, client):+ exact_response = client.post(+ "/api/graph/search",+ json={"query": "Metformin", "limit": 5},+ )+ assert exact_response.status_code == 200+ exact_payload = exact_response.json()+ assert exact_payload["results"][0]["node"]["id"] == "metformin"++ prefix_response = client.post(+ "/api/graph/search",+ json={"query": "metf", "limit": 5},+ )+ assert prefix_response.status_code == 200+ prefix_payload = prefix_response.json()+ assert any(item["node"]["id"] == "metformin" for item in prefix_payload["results"])++ def test_search_filters_and_cache_stability(self, client):+ body = {+ "query": "framework",+ "filters": {"type": "decision", "min_confidence": 0.8},+ "limit": 5,+ }+ first_response = client.post("/api/graph/search", json=body)+ second_response = client.post("/api/graph/search", json=body)++ assert first_response.status_code == 200+ assert second_response.status_code == 200+ assert first_response.json() == second_response.json()+ results = first_response.json()["results"]+ assert [item["node"]["id"] for item in results] == ["decision_1"]++ def test_search_sees_new_nodes_after_mutation(self, client):+ session = client.app.state.session+ assert session.add_node(+ "metformin_hcl",+ "drug",+ content="Metformin Hydrochloride",+ aliases=["Glucophage XR"],+ confidence="0.93",+ )++ response = client.post(+ "/api/graph/search",+ json={"query": "glucophage", "limit": 10},+ )+ assert response.status_code == 200+ result_ids = [item["node"]["id"] for item in response.json()["results"]]+ assert "metformin" in result_ids+ assert "metformin_hcl" in result_ids+def test_stats(self, client):response = client.get("/api/graph/stats")assert response.status_code == 200