Skip to content

Graph-Native Infrastructure for Context and Accountable AI Systems

perf(explorer): add indexed search for large graphs

Zohaib Hassnain6f93f429c4ce
diff --git a/bench_search.py b/bench_search.py
new file mode 100644
index 00000000..77e616e1
--- /dev/null
+++ b/bench_search.py
@@ -0,0 +1,113 @@
+from __future__ import annotations
+
+import argparse
+import statistics
+import time
+from pathlib import Path
+
+from semantica.explorer.session import GraphSession
+
+DEFAULT_GRAPH = Path(r".\demo_out_100k\primekg_demo_graph.json")
+DEFAULT_QUERIES = [
+ "metformin",
+ "metf",
+ "AICA ribonucleotide",
+ "drug:DB06736",
+ "protein",
+ "glucophagezz",
+]
+
+
+def parse_args() -> argparse.Namespace:
+ parser = argparse.ArgumentParser(
+ description="Benchmark GraphSession.search() against a local explorer graph JSON.",
+ )
+ parser.add_argument(
+ "--graph",
+ default=str(DEFAULT_GRAPH),
+ help=f"Path to graph JSON (default: {DEFAULT_GRAPH})",
+ )
+ parser.add_argument(
+ "--query",
+ dest="queries",
+ action="append",
+ help="Query to benchmark. Repeat the flag to add multiple queries.",
+ )
+ parser.add_argument(
+ "--rounds",
+ type=int,
+ default=80,
+ help="Total search calls per query, including warmup (default: 80)",
+ )
+ parser.add_argument(
+ "--warmup",
+ type=int,
+ default=10,
+ help="How many initial calls to exclude from warm metrics (default: 10)",
+ )
+ parser.add_argument(
+ "--limit",
+ type=int,
+ default=8,
+ help="Search result limit to pass to GraphSession.search() (default: 8)",
+ )
+ return parser.parse_args()
+
+
+def format_ms(value: float) -> str:
+ return f"{value:8.3f}"
+
+
+def main() -> None:
+ args = parse_args()
+ graph_path = Path(args.graph).expanduser().resolve()
+ if not graph_path.exists():
+ raise FileNotFoundError(f"Graph JSON not found: {graph_path}")
+
+ queries = args.queries or list(DEFAULT_QUERIES)
+ warmup = max(0, min(args.warmup, args.rounds - 1))
+
+ print(f"Loading graph: {graph_path}")
+ load_started = time.perf_counter()
+ session = GraphSession.from_file(str(graph_path))
+ load_ms = (time.perf_counter() - load_started) * 1000
+
+ stats = session.get_stats()
+ print(
+ f"Loaded {stats.get('node_count', 0):,} nodes / {stats.get('edge_count', 0):,} edges "
+ f"in {load_ms:.1f} ms"
+ )
+ print(
+ f"Rounds per query: {args.rounds} (warmup excluded from warm stats: {warmup}) | limit={args.limit}"
+ )
+ print("")
+
+ header = (
+ f"{'query':24} {'cold':>8} {'warm_med':>10} {'warm_min':>10} "
+ f"{'warm_max':>10} {'warm_avg':>10}"
+ )
+ print(header)
+ print("-" * len(header))
+
+ for query in queries:
+ timings: list[float] = []
+ for _ in range(args.rounds):
+ started = time.perf_counter()
+ session.search(query, args.limit, {})
+ timings.append((time.perf_counter() - started) * 1000)
+
+ cold = timings[0]
+ warm = timings[warmup:] if warmup < len(timings) else timings
+
+ print(
+ f"{query[:24]:24} "
+ f"{format_ms(cold)} "
+ f"{format_ms(statistics.median(warm))} "
+ f"{format_ms(min(warm))} "
+ f"{format_ms(max(warm))} "
+ f"{format_ms(statistics.fmean(warm))}"
+ )
+
+
+if __name__ == "__main__":
+ main()
diff --git a/preprocess_primekg.py b/preprocess_primekg.py
new file mode 100644
index 00000000..02178f87
--- /dev/null
+++ b/preprocess_primekg.py
@@ -0,0 +1,469 @@
+from __future__ import annotations
+
+import argparse
+import math
+from collections import defaultdict
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+import pandas as pd
+
+from semantica.context.context_graph import ContextGraph
+
+
+REQUIRED_COLUMNS = [
+ "relation",
+ "display_relation",
+ "x_id",
+ "x_type",
+ "x_name",
+ "x_source",
+ "y_id",
+ "y_type",
+ "y_name",
+ "y_source",
+]
+
+
+@dataclass
+class NodeRecord:
+ node_id: str
+ node_type: str
+ content: str
+ metadata: dict[str, Any]
+
+
+@dataclass
+class EdgeRecord:
+ source: str
+ target: str
+ edge_type: str
+ metadata: dict[str, Any]
+
+
+def parse_args() -> argparse.Namespace:
+ parser = argparse.ArgumentParser(
+ description="Build a Semantica Explorer demo graph from PrimeKG."
+ )
+ parser.add_argument("--input", required=True, help="Path to PrimeKG kg.csv")
+ parser.add_argument(
+ "--output-dir",
+ required=True,
+ help="Directory where demo CSVs and graph JSON will be written",
+ )
+ parser.add_argument(
+ "--anchors",
+ nargs="+",
+ required=True,
+ help="Anchor terms used to seed the neighborhood search",
+ )
+ parser.add_argument(
+ "--hops",
+ type=int,
+ default=2,
+ help="Number of neighborhood expansion hops (default: 2)",
+ )
+ parser.add_argument(
+ "--chunksize",
+ type=int,
+ default=200_000,
+ help="CSV chunk size for streaming reads (default: 200000)",
+ )
+ parser.add_argument(
+ "--max-edges",
+ type=int,
+ default=50_000,
+ help="Maximum number of unique edges to keep (default: 50000)",
+ )
+ parser.add_argument(
+ "--max-nodes",
+ type=int,
+ default=150_000,
+ help="Maximum number of unique nodes to keep before expansion stops (default: 150000)",
+ )
+ parser.add_argument(
+ "--novelty-fanout-per-node",
+ type=int,
+ default=96,
+ help=(
+ "Maximum unseen-node edges to keep per frontier node on each hop. "
+ "Higher values trade breadth for local density (default: 96)"
+ ),
+ )
+ parser.add_argument(
+ "--context-fanout-per-node",
+ type=int,
+ default=12,
+ help=(
+ "Maximum already-known context edges to keep per frontier node on each hop "
+ "after novelty edges are collected (default: 12)"
+ ),
+ )
+ return parser.parse_args()
+
+
+def normalize_text(value: Any) -> str:
+ if value is None:
+ return ""
+ text = str(value).strip().lower()
+ return " ".join(text.split())
+
+
+def coerce_scalar(value: Any) -> str:
+ if value is None:
+ return ""
+ if pd.isna(value):
+ return ""
+ if isinstance(value, float):
+ if math.isnan(value):
+ return ""
+ if value.is_integer():
+ return str(int(value))
+ return str(value).strip()
+
+
+def canonical_node_id(node_type: Any, raw_id: Any) -> str:
+ return f"{coerce_scalar(node_type)}:{coerce_scalar(raw_id)}"
+
+
+def matches_anchor(name: Any, anchor_terms: set[str]) -> bool:
+ normalized = normalize_text(name)
+ if not normalized:
+ return False
+ if normalized in anchor_terms:
+ return True
+ return any(term in normalized for term in anchor_terms)
+
+
+def chunk_reader(csv_path: Path, chunksize: int):
+ return pd.read_csv(
+ csv_path,
+ usecols=REQUIRED_COLUMNS,
+ chunksize=chunksize,
+ dtype={column: "string" for column in REQUIRED_COLUMNS},
+ keep_default_na=False,
+ low_memory=False,
+ )
+
+
+def build_node_record(row: pd.Series, prefix: str) -> NodeRecord:
+ node_type = coerce_scalar(row[f"{prefix}_type"])
+ raw_id = coerce_scalar(row[f"{prefix}_id"])
+ name = coerce_scalar(row[f"{prefix}_name"]) or raw_id
+ source = coerce_scalar(row[f"{prefix}_source"])
+ node_id = canonical_node_id(node_type, raw_id)
+ metadata = {
+ "name": name,
+ "source_system": source,
+ "primekg_id": raw_id,
+ "category": node_type,
+ }
+ return NodeRecord(
+ node_id=node_id,
+ node_type=node_type or "entity",
+ content=name,
+ metadata=metadata,
+ )
+
+
+def build_edge_record(row: pd.Series) -> EdgeRecord:
+ relation = coerce_scalar(row["relation"]) or "related_to"
+ display_relation = coerce_scalar(row["display_relation"]) or relation
+ source = canonical_node_id(row["x_type"], row["x_id"])
+ target = canonical_node_id(row["y_type"], row["y_id"])
+ metadata = {
+ "display_relation": display_relation,
+ "source_type": coerce_scalar(row["x_type"]),
+ "target_type": coerce_scalar(row["y_type"]),
+ "source_name": coerce_scalar(row["x_name"]),
+ "target_name": coerce_scalar(row["y_name"]),
+ "source_system_x": coerce_scalar(row["x_source"]),
+ "source_system_y": coerce_scalar(row["y_source"]),
+ "weight": 1.0,
+ }
+ return EdgeRecord(source=source, target=target, edge_type=relation, metadata=metadata)
+
+
+def find_anchor_nodes(csv_path: Path, anchor_terms: set[str], chunksize: int) -> dict[str, NodeRecord]:
+ anchor_nodes: dict[str, NodeRecord] = {}
+
+ for chunk in chunk_reader(csv_path, chunksize):
+ for _, row in chunk.iterrows():
+ if matches_anchor(row["x_name"], anchor_terms):
+ record = build_node_record(row, "x")
+ anchor_nodes.setdefault(record.node_id, record)
+ if matches_anchor(row["y_name"], anchor_terms):
+ record = build_node_record(row, "y")
+ anchor_nodes.setdefault(record.node_id, record)
+
+ return anchor_nodes
+
+
+def expand_neighborhood(
+ csv_path: Path,
+ seed_nodes: dict[str, NodeRecord],
+ hops: int,
+ chunksize: int,
+ max_edges: int,
+ max_nodes: int,
+ novelty_fanout_per_node: int,
+ context_fanout_per_node: int,
+) -> tuple[dict[str, NodeRecord], list[EdgeRecord]]:
+ selected_nodes: dict[str, NodeRecord] = dict(seed_nodes)
+ selected_edges: dict[tuple[str, str, str], EdgeRecord] = {}
+
+ frontier = set(seed_nodes.keys())
+ visited = set(seed_nodes.keys())
+
+ def _frontier_endpoints(source_id: str, target_id: str) -> list[str]:
+ endpoints = []
+ if source_id in frontier:
+ endpoints.append(source_id)
+ if target_id in frontier and target_id != source_id:
+ endpoints.append(target_id)
+ return endpoints
+
+ def _can_accept_edge(
+ active_frontier_nodes: list[str],
+ counts: dict[str, int],
+ limit_per_node: int,
+ ) -> bool:
+ if not active_frontier_nodes or limit_per_node <= 0:
+ return limit_per_node <= 0 and not active_frontier_nodes
+ return any(counts[node_id] < limit_per_node for node_id in active_frontier_nodes)
+
+ def _mark_edge_consumed(active_frontier_nodes: list[str], counts: dict[str, int]) -> None:
+ for node_id in active_frontier_nodes:
+ counts[node_id] += 1
+
+ for hop_index in range(hops):
+ if (
+ not frontier
+ or len(selected_edges) >= max_edges
+ or len(selected_nodes) >= max_nodes
+ ):
+ break
+
+ next_frontier: set[str] = set()
+ novelty_counts: dict[str, int] = defaultdict(int)
+ context_counts: dict[str, int] = defaultdict(int)
+ print(f"Scanning hop {hop_index + 1}/{hops} from {len(frontier)} frontier nodes...")
+
+ start_node_count = len(selected_nodes)
+ start_edge_count = len(selected_edges)
+ stop = False
+
+ # Pass 1: prioritize edges that introduce at least one unseen node.
+ for chunk in chunk_reader(csv_path, chunksize):
+ for _, row in chunk.iterrows():
+ source_id = canonical_node_id(row["x_type"], row["x_id"])
+ target_id = canonical_node_id(row["y_type"], row["y_id"])
+
+ if source_id not in frontier and target_id not in frontier:
+ continue
+ if source_id in selected_nodes and target_id in selected_nodes:
+ continue
+
+ active_frontier_nodes = _frontier_endpoints(source_id, target_id)
+ if not _can_accept_edge(
+ active_frontier_nodes,
+ novelty_counts,
+ novelty_fanout_per_node,
+ ):
+ continue
+
+ new_node_ids = [node_id for node_id in (source_id, target_id) if node_id not in selected_nodes]
+ if new_node_ids and len(selected_nodes) + len(new_node_ids) > max_nodes:
+ stop = True
+ break
+
+ edge_record = build_edge_record(row)
+ edge_key = (edge_record.source, edge_record.target, edge_record.edge_type)
+ if edge_key in selected_edges:
+ continue
+
+ source_record = build_node_record(row, "x")
+ target_record = build_node_record(row, "y")
+ selected_nodes.setdefault(source_record.node_id, source_record)
+ selected_nodes.setdefault(target_record.node_id, target_record)
+
+ selected_edges[edge_key] = edge_record
+ _mark_edge_consumed(active_frontier_nodes, novelty_counts)
+ if len(selected_edges) >= max_edges or len(selected_nodes) >= max_nodes:
+ stop = True
+
+ if source_id not in visited:
+ next_frontier.add(source_id)
+ if target_id not in visited:
+ next_frontier.add(target_id)
+
+ if stop:
+ break
+ if stop:
+ break
+
+ # Pass 2: add a small amount of context among already-selected nodes so the
+ # extracted graph doesn't look amputated, but keep this secondary to novelty.
+ if (
+ not stop
+ and context_fanout_per_node > 0
+ and len(selected_edges) < max_edges
+ ):
+ for chunk in chunk_reader(csv_path, chunksize):
+ for _, row in chunk.iterrows():
+ source_id = canonical_node_id(row["x_type"], row["x_id"])
+ target_id = canonical_node_id(row["y_type"], row["y_id"])
+
+ if source_id not in frontier and target_id not in frontier:
+ continue
+ if source_id not in selected_nodes or target_id not in selected_nodes:
+ continue
+
+ edge_record = build_edge_record(row)
+ edge_key = (edge_record.source, edge_record.target, edge_record.edge_type)
+ if edge_key in selected_edges:
+ continue
+
+ active_frontier_nodes = _frontier_endpoints(source_id, target_id)
+ if not _can_accept_edge(
+ active_frontier_nodes,
+ context_counts,
+ context_fanout_per_node,
+ ):
+ continue
+
+ selected_edges[edge_key] = edge_record
+ _mark_edge_consumed(active_frontier_nodes, context_counts)
+ if len(selected_edges) >= max_edges:
+ stop = True
+ break
+ if stop:
+ break
+
+ next_frontier -= visited
+ visited |= next_frontier
+ frontier = next_frontier
+ print(
+ f" Added {len(selected_nodes) - start_node_count:,} node(s) and "
+ f"{len(selected_edges) - start_edge_count:,} edge(s) on this hop."
+ )
+
+ return selected_nodes, list(selected_edges.values())
+
+
+def write_csvs(output_dir: Path, nodes: dict[str, NodeRecord], edges: list[EdgeRecord]) -> None:
+ node_rows = []
+ for node in nodes.values():
+ node_rows.append(
+ {
+ "id": node.node_id,
+ "name": node.content,
+ "type": node.node_type,
+ "source_system": node.metadata.get("source_system", ""),
+ "primekg_id": node.metadata.get("primekg_id", ""),
+ }
+ )
+
+ edge_rows = []
+ for edge in edges:
+ edge_rows.append(
+ {
+ "source": edge.source,
+ "target": edge.target,
+ "type": edge.edge_type,
+ "display_relation": edge.metadata.get("display_relation", edge.edge_type),
+ "source_name": edge.metadata.get("source_name", ""),
+ "target_name": edge.metadata.get("target_name", ""),
+ }
+ )
+
+ pd.DataFrame(node_rows).sort_values(["type", "name", "id"]).to_csv(
+ output_dir / "primekg_demo_nodes.csv", index=False
+ )
+ pd.DataFrame(edge_rows).to_csv(output_dir / "primekg_demo_edges.csv", index=False)
+
+
+def write_graph_json(output_dir: Path, nodes: dict[str, NodeRecord], edges: list[EdgeRecord]) -> Path:
+ graph = ContextGraph()
+
+ graph.add_nodes(
+ [
+ {
+ "id": node.node_id,
+ "type": node.node_type,
+ "content": node.content,
+ "properties": node.metadata,
+ }
+ for node in nodes.values()
+ ]
+ )
+
+ graph.add_edges(
+ [
+ {
+ "source": edge.source,
+ "target": edge.target,
+ "type": edge.edge_type,
+ "weight": 1.0,
+ "properties": edge.metadata,
+ }
+ for edge in edges
+ ]
+ )
+
+ graph_path = output_dir / "primekg_demo_graph.json"
+ graph.save_to_file(str(graph_path))
+ return graph_path
+
+
+def main() -> None:
+ args = parse_args()
+ csv_path = Path(args.input).expanduser().resolve()
+ output_dir = Path(args.output_dir).expanduser().resolve()
+
+ if not csv_path.exists():
+ raise FileNotFoundError(f"Input CSV not found: {csv_path}")
+
+ output_dir.mkdir(parents=True, exist_ok=True)
+
+ anchor_terms = {normalize_text(term) for term in args.anchors if normalize_text(term)}
+ if not anchor_terms:
+ raise ValueError("At least one non-empty anchor term is required.")
+
+ print(f"Loading anchors from {csv_path} ...")
+ seed_nodes = find_anchor_nodes(csv_path, anchor_terms, args.chunksize)
+ if not seed_nodes:
+ raise RuntimeError(
+ "No anchor nodes matched the provided terms. Check the anchor spellings against kg.csv."
+ )
+
+ print(f"Matched {len(seed_nodes)} anchor node(s).")
+ nodes, edges = expand_neighborhood(
+ csv_path=csv_path,
+ seed_nodes=seed_nodes,
+ hops=args.hops,
+ chunksize=args.chunksize,
+ max_edges=args.max_edges,
+ max_nodes=args.max_nodes,
+ novelty_fanout_per_node=args.novelty_fanout_per_node,
+ context_fanout_per_node=args.context_fanout_per_node,
+ )
+
+ write_csvs(output_dir, nodes, edges)
+ graph_path = write_graph_json(output_dir, nodes, edges)
+
+ print("")
+ print("PrimeKG demo graph ready.")
+ print(f"Nodes: {len(nodes):,}")
+ print(f"Edges: {len(edges):,}")
+ print(f"Output directory: {output_dir}")
+ print(f"Graph JSON: {graph_path}")
+ print("")
+ print(
+ f"Next: python -m semantica.explorer --graph {graph_path} --host 127.0.0.1 --port 8000 --no-browser"
+ )
+
+
+if __name__ == "__main__":
+ main()
diff --git a/semantica/explorer/app.py b/semantica/explorer/app.py
index a9fa25ec..4ab5a020 100644
--- a/semantica/explorer/app.py
+++ b/semantica/explorer/app.py
@@ -19,7 +19,12 @@ from .ws import ConnectionManager
def _install_mutation_bridge(app: FastAPI, session: GraphSession) -> None:
+ previous_callback = getattr(session.graph, "mutation_callback", None)
+
def on_mutation(event_type: str, entity_id: str, payload: dict) -> None:
+ session.handle_graph_mutation(event_type, entity_id, payload)
+ if callable(previous_callback):
+ previous_callback(event_type, entity_id, payload)
loop = getattr(app.state, "event_loop", None)
manager = getattr(app.state, "ws_manager", None)
if loop is None or manager is None or loop.is_closed():
diff --git a/semantica/explorer/routes/enrich.py b/semantica/explorer/routes/enrich.py
index be87c783..c7df5aff 100644
--- a/semantica/explorer/routes/enrich.py
+++ b/semantica/explorer/routes/enrich.py
@@ -136,11 +136,11 @@ def _apply_inferred_edges(
continue
source, target = args
if session.get_node(source) is None:
- session.graph.add_node(source, "entity", content=source)
+ session.add_node(source, "entity", content=source)
if session.get_node(target) is None:
- session.graph.add_node(target, "entity", content=target)
+ session.add_node(target, "entity", content=target)
edge_type = body.inferred_edge_type or predicate
- session.graph.add_edge(
+ session.add_edge(
source,
target,
edge_type=edge_type,
@@ -353,4 +353,6 @@ async def merge_nodes(
return removed, edges_updated
removed_ids, edges_updated = await asyncio.to_thread(_do_merge)
+ if removed_ids:
+ await asyncio.to_thread(session.rebuild_search_index)
return MergeResponse(merged_into=primary_id, removed_ids=removed_ids, edges_updated=edges_updated)
diff --git a/semantica/explorer/search_index.py b/semantica/explorer/search_index.py
new file mode 100644
index 00000000..64c51a3b
--- /dev/null
+++ b/semantica/explorer/search_index.py
@@ -0,0 +1,400 @@
+"""
+Explorer-local in-memory node search index.
+"""
+
+from __future__ import annotations
+
+import heapq
+import re
+from collections import OrderedDict, defaultdict
+from dataclasses import dataclass
+from typing import Any, DefaultDict, Dict, Iterable, List, Optional, Tuple
+
+_TOKEN_RE = re.compile(r"[a-z0-9]+")
+_WHITESPACE_RE = re.compile(r"\s+")
+_CURATED_ALIAS_KEYS = (
+ "label",
+ "name",
+ "title",
+ "pref_label",
+ "preferred_label",
+ "prefLabel",
+ "aliases",
+ "alias",
+ "synonyms",
+ "synonym",
+ "symbol",
+ "display_name",
+ "displayName",
+ "text",
+ "content",
+)
+
+
+def _normalize_text(value: Any) -> str:
+ if value is None:
+ return ""
+ text = str(value).strip().lower()
+ if not text:
+ return ""
+ return _WHITESPACE_RE.sub(" ", text)
+
+
+def _tokenize(text: str) -> Tuple[str, ...]:
+ if not text:
+ return ()
+ return tuple(dict.fromkeys(_TOKEN_RE.findall(text)))
+
+
+def _collect_text_fragments(value: Any, fragments: List[str], *, limit: int = 64) -> None:
+ if value is None or len(fragments) >= limit:
+ return
+ if isinstance(value, dict):
+ for nested in value.values():
+ _collect_text_fragments(nested, fragments, limit=limit)
+ if len(fragments) >= limit:
+ return
+ return
+ if isinstance(value, (list, tuple, set)):
+ for nested in value:
+ _collect_text_fragments(nested, fragments, limit=limit)
+ if len(fragments) >= limit:
+ return
+ return
+
+ normalized = _normalize_text(value)
+ if normalized:
+ fragments.append(normalized)
+
+
+def _coerce_float(value: Any) -> Optional[float]:
+ if value is None or value == "":
+ return None
+ try:
+ return float(value)
+ except (TypeError, ValueError):
+ return None
+
+
+@dataclass(frozen=True)
+class IndexedNodeDocument:
+ node_id: str
+ normalized_id: str
+ node_type: str
+ exact_terms: frozenset[str]
+ tokens: frozenset[str]
+ primary_text: str
+ secondary_text: str
+ confidence: Optional[float]
+ tags: Tuple[str, ...]
+
+
+class GraphSearchIndex:
+ def __init__(
+ self,
+ *,
+ cache_size: int = 128,
+ prefix_min_length: int = 2,
+ prefix_max_length: int = 12,
+ secondary_scan_limit: int = 12000,
+ ) -> None:
+ self.cache_size = cache_size
+ self.prefix_min_length = prefix_min_length
+ self.prefix_max_length = prefix_max_length
+ self.secondary_scan_limit = secondary_scan_limit
+ self._documents: Dict[str, IndexedNodeDocument] = {}
+ self._exact_index: DefaultDict[str, set[str]] = defaultdict(set)
+ self._token_index: DefaultDict[str, set[str]] = defaultdict(set)
+ self._prefix_index: DefaultDict[str, set[str]] = defaultdict(set)
+ self._ordered_node_ids: List[str] = []
+ self._cache: OrderedDict[Tuple[Any, ...], List[Tuple[str, float]]] = OrderedDict()
+
+ def rebuild(self, nodes: Iterable[Dict[str, Any]]) -> None:
+ self._documents.clear()
+ self._exact_index.clear()
+ self._token_index.clear()
+ self._prefix_index.clear()
+ self._ordered_node_ids = []
+ self.clear_cache()
+
+ for node in nodes:
+ self.upsert(node, clear_cache=False)
+
+ self._ordered_node_ids.sort()
+
+ def clear_cache(self) -> None:
+ self._cache.clear()
+
+ def remove(self, node_id: str, *, clear_cache: bool = True) -> None:
+ existing = self._documents.pop(node_id, None)
+ if existing is None:
+ return
+
+ for term in existing.exact_terms:
+ bucket = self._exact_index.get(term)
+ if bucket is None:
+ continue
+ bucket.discard(node_id)
+ if not bucket:
+ self._exact_index.pop(term, None)
+
+ for token in existing.tokens:
+ bucket = self._token_index.get(token)
+ if bucket is None:
+ continue
+ bucket.discard(node_id)
+ if not bucket:
+ self._token_index.pop(token, None)
+
+ for length in range(self.prefix_min_length, min(len(token), self.prefix_max_length) + 1):
+ prefix = token[:length]
+ prefix_bucket = self._prefix_index.get(prefix)
+ if prefix_bucket is None:
+ continue
+ prefix_bucket.discard(node_id)
+ if not prefix_bucket:
+ self._prefix_index.pop(prefix, None)
+
+ try:
+ self._ordered_node_ids.remove(node_id)
+ except ValueError:
+ pass
+
+ if clear_cache:
+ self.clear_cache()
+
+ def upsert(self, node: Dict[str, Any], *, clear_cache: bool = True) -> None:
+ node_id = str(node.get("id", "")).strip()
+ if not node_id:
+ return
+
+ self.remove(node_id, clear_cache=False)
+ document = self._build_document(node)
+ self._documents[node_id] = document
+
+ for term in document.exact_terms:
+ self._exact_index[term].add(node_id)
+
+ for token in document.tokens:
+ self._token_index[token].add(node_id)
+ for length in range(self.prefix_min_length, min(len(token), self.prefix_max_length) + 1):
+ self._prefix_index[token[:length]].add(node_id)
+
+ if node_id not in self._ordered_node_ids:
+ self._ordered_node_ids.append(node_id)
+ self._ordered_node_ids.sort()
+
+ if clear_cache:
+ self.clear_cache()
+
+ def search(
+ self,
+ query: str,
+ *,
+ limit: int = 20,
+ filters: Optional[Dict[str, Any]] = None,
+ ) -> tuple[List[Tuple[str, float]], Dict[str, Any]]:
+ normalized_query = _normalize_text(query)
+ filters = filters or {}
+ diagnostics: Dict[str, Any] = {
+ "cache_hit": False,
+ "path": "empty",
+ "candidates": 0,
+ }
+ if not normalized_query:
+ return [], diagnostics
+
+ cache_key = self._cache_key(normalized_query, limit, filters)
+ cached = self._cache.get(cache_key)
+ if cached is not None:
+ self._cache.move_to_end(cache_key)
+ diagnostics.update({"cache_hit": True, "path": "cache", "candidates": len(cached)})
+ return list(cached), diagnostics
+
+ query_tokens = _tokenize(normalized_query)
+ exact_ids = set(self._exact_index.get(normalized_query, set()))
+ token_sets: List[set[str]] = []
+ prefix_sets: List[set[str]] = []
+ for token in query_tokens:
+ exact_token_ids = set(self._token_index.get(token, set()))
+ prefix_ids = set(self._prefix_index.get(token, set())) if len(token) >= self.prefix_min_length else set()
+ if exact_token_ids:
+ token_sets.append(exact_token_ids)
+ if prefix_ids:
+ prefix_sets.append(prefix_ids)
+
+ candidate_ids: set[str] = set(exact_ids)
+ if token_sets:
+ intersected = set.intersection(*token_sets)
+ candidate_ids.update(intersected if intersected else set().union(*token_sets))
+ if prefix_sets:
+ candidate_ids.update(set().union(*prefix_sets))
+
+ diagnostics["path"] = "index"
+
+ if not candidate_ids:
+ diagnostics["path"] = "secondary_scan"
+ candidate_ids = self._secondary_scan(normalized_query, limit)
+
+ diagnostics["candidates"] = len(candidate_ids)
+
+ scored: List[Tuple[float, int, int, str]] = []
+ for node_id in candidate_ids:
+ document = self._documents.get(node_id)
+ if document is None or not self._passes_filters(document, filters):
+ continue
+ score = self._score_document(document, normalized_query, query_tokens)
+ if score <= 0:
+ continue
+ token_hits = sum(1 for token in query_tokens if token in document.tokens)
+ exactness = 1 if normalized_query == document.normalized_id or normalized_query in document.exact_terms else 0
+ scored.append((score, exactness, token_hits, node_id))
+
+ top_matches = heapq.nlargest(limit, scored, key=lambda item: (item[0], item[1], item[2], item[3]))
+ results = [(node_id, round(score, 4)) for score, _, _, node_id in top_matches]
+ self._store_cache(cache_key, results)
+ return results, diagnostics
+
+ def _secondary_scan(self, normalized_query: str, limit: int) -> set[str]:
+ matches: set[str] = set()
+ max_hits = max(limit * 20, 200)
+ scanned = 0
+ for node_id in self._ordered_node_ids:
+ if scanned >= self.secondary_scan_limit or len(matches) >= max_hits:
+ break
+ scanned += 1
+ document = self._documents.get(node_id)
+ if document is None:
+ continue
+ if normalized_query in document.primary_text or normalized_query in document.secondary_text:
+ matches.add(node_id)
+ return matches
+
+ def _score_document(
+ self,
+ document: IndexedNodeDocument,
+ normalized_query: str,
+ query_tokens: Tuple[str, ...],
+ ) -> float:
+ score = 0.0
+ if normalized_query == document.normalized_id:
+ score = max(score, 140.0)
+ elif normalized_query in document.exact_terms:
+ score = max(score, 120.0)
+
+ if normalized_query and normalized_query in document.primary_text:
+ score = max(score, 78.0 + min(len(normalized_query), 24) / 10.0)
+ elif normalized_query and normalized_query in document.secondary_text:
+ score = max(score, 26.0 + min(len(normalized_query), 24) / 20.0)
+
+ token_hits = 0
+ prefix_hits = 0
+ for token in query_tokens:
+ if token in document.tokens:
+ token_hits += 1
+ elif len(token) >= self.prefix_min_length and any(candidate.startswith(token) for candidate in document.tokens):
+ prefix_hits += 1
+
+ score += token_hits * 18.0
+ score += prefix_hits * 10.0
+
+ if len(query_tokens) > 1 and token_hits:
+ score += token_hits * 4.0
+
+ return score
+
+ def _passes_filters(self, document: IndexedNodeDocument, filters: Dict[str, Any]) -> bool:
+ filter_type = filters.get("type") or filters.get("node_type")
+ if filter_type and document.node_type != str(filter_type):
+ return False
+
+ min_confidence = _coerce_float(filters.get("min_confidence"))
+ if min_confidence is not None:
+ if document.confidence is None or document.confidence < min_confidence:
+ return False
+
+ tags_filter = filters.get("tags")
+ if tags_filter:
+ if isinstance(tags_filter, str):
+ required_tags = {_normalize_text(tags_filter)}
+ else:
+ required_tags = {
+ normalized
+ for normalized in (_normalize_text(tag) for tag in tags_filter)
+ if normalized
+ }
+ if required_tags and not required_tags.issubset(set(document.tags)):
+ return False
+
+ return True
+
+ def _cache_key(
+ self,
+ normalized_query: str,
+ limit: int,
+ filters: Dict[str, Any],
+ ) -> Tuple[Any, ...]:
+ serialized_filters: List[Tuple[str, Any]] = []
+ for key in sorted(filters.keys()):
+ value = filters[key]
+ if isinstance(value, (list, tuple, set)):
+ serialized_filters.append((key, tuple(str(item) for item in value)))
+ else:
+ serialized_filters.append((key, str(value)))
+ return normalized_query, limit, tuple(serialized_filters)
+
+ def _store_cache(self, cache_key: Tuple[Any, ...], results: List[Tuple[str, float]]) -> None:
+ self._cache[cache_key] = list(results)
+ self._cache.move_to_end(cache_key)
+ while len(self._cache) > self.cache_size:
+ self._cache.popitem(last=False)
+
+ def _build_document(self, node: Dict[str, Any]) -> IndexedNodeDocument:
+ node_id = str(node.get("id", "")).strip()
+ node_type = str(node.get("type", "entity"))
+ properties = dict(node.get("properties", {}) or {})
+
+ primary_terms: List[str] = []
+ for candidate in (node_id, node.get("content", "")):
+ normalized = _normalize_text(candidate)
+ if normalized:
+ primary_terms.append(normalized)
+
+ for alias_key in _CURATED_ALIAS_KEYS:
+ _collect_text_fragments(properties.get(alias_key), primary_terms, limit=32)
+
+ deduped_primary_terms = tuple(dict.fromkeys(term for term in primary_terms if term))
+ primary_text = " ".join(deduped_primary_terms)
+ tokens = frozenset(_tokenize(primary_text))
+
+ secondary_fragments: List[str] = []
+ for key, value in properties.items():
+ if key in _CURATED_ALIAS_KEYS or key in {"content", "valid_from", "valid_until"}:
+ continue
+ _collect_text_fragments(value, secondary_fragments, limit=48)
+ if len(secondary_fragments) >= 48:
+ break
+
+ secondary_text = " ".join(dict.fromkeys(fragment for fragment in secondary_fragments if fragment))
+ confidence = _coerce_float(properties.get("confidence"))
+
+ raw_tags = properties.get("tags") or []
+ if isinstance(raw_tags, str):
+ raw_tags = [raw_tags]
+ tags = tuple(
+ dict.fromkeys(
+ normalized for normalized in (_normalize_text(tag) for tag in raw_tags) if normalized
+ )
+ )
+
+ return IndexedNodeDocument(
+ node_id=node_id,
+ normalized_id=_normalize_text(node_id),
+ node_type=node_type,
+ exact_terms=frozenset(deduped_primary_terms),
+ tokens=tokens,
+ primary_text=primary_text,
+ secondary_text=secondary_text,
+ confidence=confidence,
+ tags=tags,
+ )
diff --git a/semantica/explorer/session.py b/semantica/explorer/session.py
index 9600f90f..857a4a3c 100644
--- a/semantica/explorer/session.py
+++ b/semantica/explorer/session.py
@@ -4,12 +4,15 @@ Semantica Explorer session helpers.
import base64
import json
+import logging
import threading
+import time
import uuid
-from datetime import datetime, UTC
+from datetime import UTC, datetime
from typing import Any, Dict, Iterable, List, Optional
from ..context.context_graph import ContextGraph, _resolve_edge_identity
+from .search_index import GraphSearchIndex
_KG_AVAILABLE = False
try:
@@ -28,6 +31,8 @@ try:
except ImportError:
pass
+logger = logging.getLogger(__name__)
+
class GraphSession:
"""Thread-safe session wrapper around a loaded ``ContextGraph``."""
@@ -35,6 +40,7 @@ class GraphSession:
def __init__(self, graph: ContextGraph) -> None:
self.graph = graph
self._lock = threading.RLock()
+ self._search_index = GraphSearchIndex()
self.annotations: Dict[str, Dict[str, Any]] = {}
@@ -46,6 +52,7 @@ class GraphSession:
self._similarity: Any = None
self._link_predictor: Any = None
self._validator: Any = None
+ self.rebuild_search_index()
@classmethod
def from_file(cls, path: str) -> "GraphSession":
@@ -390,6 +397,26 @@ class GraphSession:
with self._lock:
return self.graph.get_neighbors(node_id, hops=depth)
+ def rebuild_search_index(self) -> None:
+ with self._lock:
+ normalized_nodes = [
+ self.normalize_node(node.to_dict())
+ for node in self.graph.nodes.values()
+ if node is not None
+ ]
+ self._search_index.rebuild(normalized_nodes)
+
+ def handle_graph_mutation(self, event_type: str, entity_id: str, payload: Dict[str, Any]) -> None:
+ normalized_event = str(event_type or "").upper()
+ if normalized_event in {"ADD_NODE", "UPDATE_NODE"}:
+ normalized_node = self.normalize_node(payload or {})
+ if normalized_node.get("id"):
+ self._search_index.upsert(normalized_node)
+ elif normalized_event in {"REMOVE_NODE", "DELETE_NODE"}:
+ self._search_index.remove(str(entity_id))
+ elif normalized_event in {"RELOAD_GRAPH", "RESET_GRAPH"}:
+ self.rebuild_search_index()
+
def search(
self,
query: str,
@@ -397,64 +424,34 @@ class GraphSession:
filters: Optional[Dict[str, Any]] = None,
) -> List[Dict[str, Any]]:
filters = filters or {}
- try:
- with self._lock:
- raw = self.graph.query(query)[:limit]
- except Exception:
- raw = []
-
- if not raw:
- nodes, _ = self.get_nodes(search=query, skip=0, limit=max(limit * 5, limit))
- scored = []
- lowered_query = query.lower().strip()
- for node in nodes:
- haystacks = [
- str(node.get("id", "")),
- str(node.get("content", "")),
- json.dumps(node.get("properties", {}), default=str),
- ]
- best_score = 0.0
- for haystack in haystacks:
- lowered = haystack.lower()
- if lowered == lowered_query:
- best_score = max(best_score, 1.0)
- elif lowered_query in lowered:
- best_score = max(best_score, min(0.9, len(lowered_query) / max(len(lowered), 1)))
- if best_score > 0:
- scored.append({"node": node, "score": round(best_score, 4)})
- raw = sorted(scored, key=lambda item: item["score"], reverse=True)[:limit]
-
- normalized = []
- for result in raw:
- result_node = result.get("node", {})
- node = (
- self.normalize_node(result_node)
- if "properties" in result_node or "metadata" in result_node or "content" in result_node
- else result_node
- )
-
- filter_type = filters.get("type") or filters.get("node_type")
- if filter_type and node["type"] != filter_type:
- continue
-
- min_confidence = self._coerce_float(filters.get("min_confidence"))
- node_confidence = self._coerce_float(node["properties"].get("confidence"))
- if min_confidence is not None and (
- node_confidence is None or node_confidence < min_confidence
- ):
- continue
+ started_at = time.perf_counter()
+ matches, diagnostics = self._search_index.search(query, limit=limit, filters=filters)
- tags_filter = filters.get("tags")
- if tags_filter:
- node_tags = node["properties"].get("tags") or []
- if isinstance(node_tags, str):
- node_tags = [node_tags]
- if not set(tags_filter).issubset(set(node_tags)):
+ normalized_results: List[Dict[str, Any]] = []
+ with self._lock:
+ for node_id, score in matches:
+ raw_node = self.graph.find_node(node_id)
+ if raw_node is None:
continue
+ node_payload = raw_node.to_dict() if hasattr(raw_node, "to_dict") else raw_node
+ normalized_results.append(
+ {
+ "node": self.normalize_node(node_payload),
+ "score": score,
+ }
+ )
- normalized.append({"node": node, "score": result.get("score", 0.0)})
-
- return normalized[:limit]
+ duration_ms = round((time.perf_counter() - started_at) * 1000, 2)
+ logger.debug(
+ "Explorer search query=%r limit=%s cache_hit=%s path=%s candidates=%s duration_ms=%s",
+ query,
+ limit,
+ diagnostics.get("cache_hit"),
+ diagnostics.get("path"),
+ diagnostics.get("candidates"),
+ duration_ms,
+ )
+ return normalized_results[:limit]
def get_stats(self) -> Dict[str, Any]:
with self._lock:
@@ -594,8 +591,58 @@ class GraphSession:
def add_nodes(self, nodes: List[Dict[str, Any]]) -> int:
with self._lock:
- return self.graph.add_nodes(nodes)
+ added = self.graph.add_nodes(nodes)
+ has_mutation_callback = callable(getattr(self.graph, "mutation_callback", None))
+ if added and not has_mutation_callback:
+ self.rebuild_search_index()
+ return added
def add_edges(self, edges: List[Dict[str, Any]]) -> int:
with self._lock:
- return self.graph.add_edges(edges)
+ added = self.graph.add_edges(edges)
+ has_mutation_callback = callable(getattr(self.graph, "mutation_callback", None))
+ if added and not has_mutation_callback:
+ self.rebuild_search_index()
+ return added
+
+ def add_node(
+ self,
+ node_id: str,
+ node_type: str,
+ content: Optional[str] = None,
+ **properties: Any,
+ ) -> bool:
+ with self._lock:
+ added = self.graph.add_node(node_id, node_type, content=content, **properties)
+ has_mutation_callback = callable(getattr(self.graph, "mutation_callback", None))
+ if added and not has_mutation_callback:
+ normalized = self.get_node(node_id)
+ if normalized is not None:
+ self._search_index.upsert(normalized)
+ return added
+
+ def add_edge(
+ self,
+ source_id: str,
+ target_id: str,
+ edge_type: str = "related_to",
+ weight: float = 1.0,
+ **properties: Any,
+ ) -> bool:
+ with self._lock:
+ added = self.graph.add_edge(
+ source_id,
+ target_id,
+ edge_type=edge_type,
+ weight=weight,
+ **properties,
+ )
+ has_mutation_callback = callable(getattr(self.graph, "mutation_callback", None))
+ if added and not has_mutation_callback:
+ source = self.get_node(source_id)
+ if source is not None:
+ self._search_index.upsert(source)
+ target = self.get_node(target_id)
+ if target is not None:
+ self._search_index.upsert(target)
+ return added
diff --git a/tests/explorer/test_explorer_api.py b/tests/explorer/test_explorer_api.py
index f0d56859..8f30d794 100644
--- a/tests/explorer/test_explorer_api.py
+++ b/tests/explorer/test_explorer_api.py
@@ -35,6 +35,16 @@ def _build_sample_graph() -> ContextGraph:
graph.add_node("javascript", node_type="language", content="JavaScript programming language", x=100, y=120)
graph.add_node("web_dev", node_type="concept", content="Web Development", x=24, y=30)
graph.add_node("ml", node_type="concept", content="Machine Learning", x=45, y=60)
+ graph.add_node(
+ "metformin",
+ node_type="drug",
+ content="Metformin",
+ aliases=["Glucophage"],
+ confidence="0.97",
+ tags=["drug", "featured"],
+ x=22,
+ y=33,
+ )
graph.add_node(
"decision_1",
node_type="decision",
@@ -243,6 +253,57 @@ class TestSearchAndStats:
assert payload["total"] >= 1
assert all(item["node"]["type"] == "language" for item in payload["results"])
+ def test_search_exact_and_prefix(self, client):
+ exact_response = client.post(
+ "/api/graph/search",
+ json={"query": "Metformin", "limit": 5},
+ )
+ assert exact_response.status_code == 200
+ exact_payload = exact_response.json()
+ assert exact_payload["results"][0]["node"]["id"] == "metformin"
+
+ prefix_response = client.post(
+ "/api/graph/search",
+ json={"query": "metf", "limit": 5},
+ )
+ assert prefix_response.status_code == 200
+ prefix_payload = prefix_response.json()
+ assert any(item["node"]["id"] == "metformin" for item in prefix_payload["results"])
+
+ def test_search_filters_and_cache_stability(self, client):
+ body = {
+ "query": "framework",
+ "filters": {"type": "decision", "min_confidence": 0.8},
+ "limit": 5,
+ }
+ first_response = client.post("/api/graph/search", json=body)
+ second_response = client.post("/api/graph/search", json=body)
+
+ assert first_response.status_code == 200
+ assert second_response.status_code == 200
+ assert first_response.json() == second_response.json()
+ results = first_response.json()["results"]
+ assert [item["node"]["id"] for item in results] == ["decision_1"]
+
+ def test_search_sees_new_nodes_after_mutation(self, client):
+ session = client.app.state.session
+ assert session.add_node(
+ "metformin_hcl",
+ "drug",
+ content="Metformin Hydrochloride",
+ aliases=["Glucophage XR"],
+ confidence="0.93",
+ )
+
+ response = client.post(
+ "/api/graph/search",
+ json={"query": "glucophage", "limit": 10},
+ )
+ assert response.status_code == 200
+ result_ids = [item["node"]["id"] for item in response.json()["results"]]
+ assert "metformin" in result_ids
+ assert "metformin_hcl" in result_ids
+
def test_stats(self, client):
response = client.get("/api/graph/stats")
assert response.status_code == 200