Skip to content

BrainAPI is a knowledge graph–powered AI memory layer that transforms unstructured data into structured knowledge, enabling intelligent search, recommendations, and contextual memory for AI agents and applications.

feat(retrieval): Event GraphRAG hubs, topics, and provenance

Christian664027b583f6
diff --git a/src/adapters/embeddings.py b/src/adapters/embeddings.py
index 474fc92..e4e888b 100644
--- a/src/adapters/embeddings.py
+++ b/src/adapters/embeddings.py
@@ -168,13 +168,10 @@ class VectorStoreAdapter:
"""
Search vectors in the vector store and return the top k vectors.
"""
+ from src.utils.vector_search import stable_top_k_vectors
+
vectors = self.vector_store.search_vectors(data_vector, brain_id, store, k)
- return sorted(
- vectors,
- key=lambda x: (
- float("inf") if x.distance is None else float(x.distance)
- ),
- )
+ return stable_top_k_vectors(list(vectors), k)
def get_by_ids(
self, ids: list[str], store: str, brain_id: str = "default"
diff --git a/src/adapters/graph.py b/src/adapters/graph.py
index e78110f..3ce6f10 100644
--- a/src/adapters/graph.py
+++ b/src/adapters/graph.py
@@ -829,5 +829,34 @@ class GraphAdapter:
"""
return self.graph.list_relationships(subject, object, brain_id)
+ def get_event_hub_facts(
+ self,
+ event_uuids: list[str],
+ brain_id: str = "default",
+ ) -> list:
+ return self.graph.get_event_hub_facts(event_uuids, brain_id)
+
+ def rebuild_hub_bridge_index(self, brain_id: str = "default") -> int:
+ return self.graph.rebuild_hub_bridge_index(brain_id)
+
+ def get_hub_bridges(
+ self,
+ event_uuids: list[str],
+ brain_id: str = "default",
+ ) -> list:
+ return self.graph.get_hub_bridges(event_uuids, brain_id)
+
+ def rebuild_topic_index(self, brain_id: str = "default") -> int:
+ rebuild = getattr(self.graph, "rebuild_topic_index", None)
+ if rebuild is None:
+ return 0
+ return int(rebuild(brain_id) or 0)
+
+ def list_topic_memberships(self, brain_id: str = "default") -> list:
+ listing = getattr(self.graph, "list_topic_memberships", None)
+ if listing is None:
+ return []
+ return list(listing(brain_id) or [])
+
_graph_adapter = GraphAdapter()
diff --git a/src/constants/prompts/architect_agent.py b/src/constants/prompts/architect_agent.py
index a5b5b81..bdfb285 100644
--- a/src/constants/prompts/architect_agent.py
+++ b/src/constants/prompts/architect_agent.py
@@ -28,7 +28,6 @@ Entities Found by Scout: [
{{"uuid": "uuid_2", "type": "EVENT", "name": "WENT_TO", "description": "John went to New York City"}},
{{"uuid": "uuid_3", "type": "CITY", "name": "New York City"}},
{{"uuid": "uuid_4", "type": "EVENT", "name": "KNEW", "description": "John knew 12 new friends in New York City"}},
- {{"uuid": "uuid_5", "type": "UNIT", "name": "Friends", "description": "The number of friends John knew in New York City"}},
{{"uuid": "uuid_6", "type": "PERSON", "name": "Mary"}},
{{"uuid": "uuid_7", "type": "EVENT", "name": "WAS_IN", "description": "Mary was in San Francisco"}},
{{"uuid": "uuid_8", "type": "CITY", "name": "San Francisco"}},
@@ -56,6 +55,7 @@ Example output 1 (in the real output "tail" and "tip" must be the full entity ob
"tail": "uuid_1",
"name": "ACCOMPLISHED_ACTION",
"description": "John knew 12 new friends in New York City",
+ "amount": 12,
"tip": "uuid_4"
}},
{{
@@ -64,12 +64,6 @@ Example output 1 (in the real output "tail" and "tip" must be the full entity ob
"description": "John knew 12 new friends when he went to New York City",
"tip": "uuid_2"
}},
- {{
- "tail": "uuid_4",
- "name": "TARGETED",
- "description": "John knew 12 new friends in New York City",
- "tip": "uuid_5"
- }},
{{
"tail": "uuid_6",
"name": "EXPERIENCED",
@@ -106,7 +100,6 @@ Entities Found by Scout: [
{{"uuid": "uuid_8", "type": "ROLE", "name": "CEO", "description": "Mark Johnson covered the role of CEO of Acme Inc."}},
{{"uuid": "uuid_9", "type": "EVENT", "name": "COVERED_ROLE", "description": "Mark Johnson was the founder of Acme Inc."}},
{{"uuid": "uuid_10", "type": "EVENT", "name": "RAISED", "description": "Acme Inc. raised $100 million in funding", "happened_at": "19/01/2026"}},
- {{"uuid": "uuid_11", "type": "MONEY", "name": "Money", "description": "The amount of money Acme Inc. raised in funding"}},
]
Example output 2:
@@ -126,10 +119,11 @@ Example output 2:
"tip": "uuid_8"
}},
{{
- "tail": "uuid_10",
- "name": "TARGETED",
+ "tail": "uuid_1",
+ "name": "MADE",
"description": "Acme Inc. raised $100 million in funding",
- "tip": "uuid_11"
+ "amount": 100000000,
+ "tip": "uuid_10"
}},
... more relationships ...
],
@@ -151,7 +145,7 @@ DIRECTIONAL SLOT-FILLING:
- "tip": The end of the arrow (The Destination/Target, the object being affected).
- FORBIDDEN: Never swap "tail" and "tip": for an initiation vector the Actor is ALWAYS the "tail" and the Event is ALWAYS the "tip"; for a target vector the Event is ALWAYS the "tail" and the Recipient is ALWAYS the "tip".
- FORBIDDEN: Never link Actor nodes directly to Target nodes for dynamic actions.
-- FORBIDDEN: Never create nodes for numeric quantities.
+- FORBIDDEN: Never create nodes for numeric quantities, for their units, or placeholder nodes named after their own type (eg: MONEY:"Money", UNIT:"Friends"): node identity is derived from name and type, so every quantity in the graph would collapse onto one shared node. Carry the quantity as the 'amount' property of the relationship instead.
NEW NODES POLICY:
- Only add an entity to "new_nodes" if it appears NEITHER in the provided entities list NOR in the previously created relationships.
@@ -161,7 +155,7 @@ LOGIC CHECKLIST:
- Identify the Actor (Origin).
- Identify the Event Hub (Action Instance).
- Identify the Target (Destination).
-- If any quantity is specified in the text and the scout identified a Unit, attach the quantity value as 'amount' to the relationship properties.
+- If any quantity is specified in the text, attach the quantity value as 'amount' to the relationship properties.
- Nodes/Entities MUST be atomic and not composite (phrases) (eg: "Went to San Francisco" is not atomic, "Went to" + "San Francisco" is atomic)
Remember that the uuids are STANDARD uuids 8-4-4-4-12 hexadecimal character strings.
@@ -245,7 +239,6 @@ Entities Found by Scout: [
{{"uuid": "uuid_2", "type": "EVENT", "name": "Went", "description": "John went to New York City"}},
{{"uuid": "uuid_3", "type": "CITY", "name": "New York City"}},
{{"uuid": "uuid_4", "type": "EVENT", "name": "Knew", "description": "John knew 12 new friends in New York City"}},
- {{"uuid": "uuid_5", "type": "UNIT", "name": "Friends", "description": "The number of friends John knew in New York City"}},
{{"uuid": "uuid_6", "type": "PERSON", "name": "Mary"}},
{{"uuid": "uuid_7", "type": "EVENT", "name": "Was", "description": "Mary was in San Francisco"}},
{{"uuid": "uuid_8", "type": "CITY", "name": "San Francisco"}},
@@ -280,13 +273,6 @@ Example architect_agent_create_relationship tool input 1:
"predicate": "HAPPENED_WITHIN",
"description": "John knew 12 new friends when he went to New York City",
"object": "uuid_2"
- }},
- {{
- "subject": "uuid_4",
- "predicate": "TARGETED",
- "description": "John knew 12 new friends in New York City",
- "object": "uuid_5",
- "amount": 12
}}
]}}
Example architect_agent_create_relationship tool output:
@@ -334,13 +320,13 @@ DIRECTIONAL SLOT-FILLING:
- "subject": The start of the arrow (The Source of Energy/Origin).
- "object": The end of the arrow (The Destination/Target).
- FORBIDDEN: Never link Actor nodes directly to Target nodes for dynamic actions.
-- FORBIDDEN: Never create nodes for numeric quantities.
+- FORBIDDEN: Never create nodes for numeric quantities, for their units, or placeholder nodes named after their own type (eg: MONEY:"Money", UNIT:"Friends"): node identity is derived from name and type, so every quantity in the graph would collapse onto one shared node. Carry the quantity as the 'amount' property of the relationship instead.
LOGIC CHECKLIST:
- Identify the Actor (Origin).
- Identify the Event Hub (Action Instance).
- Identify the Target (Destination).
-- If any quantity is specified in the text and the scout identified a Unit, attach the quantity value as 'amount' to the relationship properties of the relationship with the Unit node.
+- If any quantity is specified in the text, attach the quantity value as 'amount' to the relationship properties of the event's relationships.
- Nodes/Entities MUST be atomic and not composite (phrases) (eg: "Went to San Francisco" is not atomic, "Went to" + "San Francisco" is atomic)
Your workflow must be:
@@ -376,7 +362,7 @@ Every action must flow through a central **EVENT hub**:
- **Directional Slot-Filling:** - `subject`: Origin/Source.
- `object`: Destination/Target.
- **Forbidden:** - No direct Actor-to-Target links for actions (must use Event hub).
- - No dedicated nodes for numbers; store quantities as `amount` properties on relationships.
+ - No dedicated nodes for numbers or their units, and no placeholder nodes named after their own type (eg: `MONEY:"Money"`, `UNIT:"Friends"`) — node identity is name plus type, so those collapse onto one shared node. Store quantities as `amount` properties on relationships.
- **Entity Coverage:** Use 100% of Scout-provided entities. Reuse entities across contexts as needed.
- **UUIDS:** Use the standard uuids 8-4-4-4-12 hexadecimal character strings.
- **Relationship Names:** Use general relationship names (eg: "TARGET_PRODUCT_OBJECT_CROISSANTS"=wrong, "TARGETED"=correct)
@@ -418,7 +404,7 @@ Every action must flow through a central **EVENT hub**:
- **Directional Slot-Filling:** - `subject`: Origin/Source.
- `object`: Destination/Target.
- **Forbidden:** - No direct Actor-to-Target links for actions (must use Event hub).
- - No dedicated nodes for numbers; store quantities as `amount` properties on relationships.
+ - No dedicated nodes for numbers or their units, and no placeholder nodes named after their own type (eg: `MONEY:"Money"`, `UNIT:"Friends"`) — node identity is name plus type, so those collapse onto one shared node. Store quantities as `amount` properties on relationships.
- **Entity Coverage:** Use 100% of Scout-provided entities. Reuse entities across contexts as needed.
- **UUIDS:** Use the standard uuids 8-4-4-4-12 hexadecimal character strings.
- **Relationship Names:** Use general relationship names (eg: "TARGET_PRODUCT_OBJECT_CROISSANTS"=wrong, "TARGETED"=correct)
@@ -449,7 +435,6 @@ Entities Found by Scout: [
{{"uuid": "uuid_2", "type": "EVENT", "name": "Went", "description": "John went to New York City"}},
{{"uuid": "uuid_3", "type": "CITY", "name": "New York City"}},
{{"uuid": "uuid_4", "type": "EVENT", "name": "Knew", "description": "John knew 12 new friends in New York City"}},
- {{"uuid": "uuid_5", "type": "UNIT", "name": "Friends", "description": "The number of friends John knew in New York City"}},
{{"uuid": "uuid_6", "type": "PERSON", "name": "Mary"}},
{{"uuid": "uuid_7", "type": "EVENT", "name": "Was", "description": "Mary was in San Francisco"}},
{{"uuid": "uuid_8", "type": "CITY", "name": "San Francisco"}},
@@ -484,13 +469,6 @@ Example architect_agent_create_relationship tool input 1:
"predicate": "HAPPENED_WITHIN",
"description": "John knew 12 new friends when he went to New York City",
"object": "uuid_2"
- }},
- {{
- "subject": "uuid_4",
- "predicate": "TARGETED",
- "description": "John knew 12 new friends in New York City",
- "object": "uuid_5",
- "amount": 12
}}
]}}
Example architect_agent_create_relationship tool output:
diff --git a/src/constants/prompts/scout_agent.py b/src/constants/prompts/scout_agent.py
index dac1db1..d829444 100644
--- a/src/constants/prompts/scout_agent.py
+++ b/src/constants/prompts/scout_agent.py
@@ -14,7 +14,8 @@ You are a "High-Recall Semantic Scout." Your goal is to decompose raw text into
ENTITY VS. PROPERTY LOGIC:
- STATIC ATTRIBUTES: Things that are unique to a specific entity and do not change often (e.g., ID numbers, telephone numbers, emails, descriptive text) must be stored as PROPERTIES of that entity.
- SHARED DIMENSIONS: Things that can be connected to multiple different entities (e.g., Currencies, Languages, Skills, Cities, Requirements) must be standalone ENTITIES.
-- DYNAMIC QUANTITIES: Do not create nodes for numbers. Identify the "Unit" (e.g., USD, Members, Hours) as an entity; the numeric value is a property to be handled by the Architect.
+- DYNAMIC QUANTITIES: Do not create nodes for numbers, for their units (e.g., USD, Members, Hours) or for the thing being counted; the numeric value and its unit are properties to be handled by the Architect.
+- FORBIDDEN: Never emit a placeholder entity named after its own type (e.g. MONEY:"Money", UNIT:"Friends"): every such entity would be the same node for every fact that mentions a quantity.
DECISION CRITERIA:
1. Does it change frequently? YES -> Entity.
@@ -48,8 +49,7 @@ Example output 1:
{{"type": "PERSON", "name": "John", "polarity": "neutral"}},
{{"type": "EVENT", "name": "Went", "description": "John went to New York City", "polarity": "neutral"}},
{{"type": "CITY", "name": "New York City", "polarity": "neutral"}},
- {{"type": "EVENT", "name": "Knew", "description": "John knew 12 new friends in New York City", "polarity": "neutral"}},
- {{"type": "FRIENDS", "name": "Friends", "description": "The number of friends John knew in New York City", "polarity": "neutral"}},
+ {{"type": "EVENT", "name": "Knew", "description": "John knew 12 new friends in New York City", "properties": {{ "amount": 12 }}, "polarity": "neutral"}},
{{"type": "PERSON", "name": "Mary", "polarity": "neutral"}},
{{"type": "EVENT", "name": "Was in", "description": "Mary was in San Francisco", "polarity": "neutral"}},
{{"type": "CITY", "name": "San Francisco", "polarity": "neutral"}},
@@ -74,18 +74,17 @@ Example output 2:
{{"type": "EVENT", "name": "Covered role", "description": "Mark Johnson was the founder of Acme Inc.", "polarity": "positive"}},
{{"type": "ROLE", "name": "Founder", "description": "Mark Johnson covered the role of founder of Acme Inc.", "polarity": "neutral"}},
{{"type": "EVENT", "name": "Raised", "description": "Acme Inc. raised $100 million in funding", "properties": {{ "amount": 100000000, "happened_at": "19/01/2026" }}, "polarity": "positive"}},
- {{"type": "MONEY", "name": "Money", "polarity": "neutral"}},
]
"""
SCOUT_AGENT_COARSE_SYSTEM_PROMPT = """
-You are a "High-Level Semantic Scout." Your goal is to extract the most important entities that can be used to reconstruct a meaningful narrative,
-the unique action instances (events), and the quantitative units.
+You are a "High-Level Semantic Scout." Your goal is to extract the most important entities that can be used to reconstruct a meaningful narrative
+and the unique action instances (events), carrying any quantity as a property.
ENTITY VS. PROPERTY LOGIC:
- STATIC ATTRIBUTES: Things that are unique to a specific entity and do not change often (e.g., ID numbers, telephone numbers, emails, descriptive text) must be stored as PROPERTIES of that entity.
- SHARED DIMENSIONS: Things that can be connected to multiple different entities (e.g., Currencies, Languages, Skills, Cities, Requirements) must be standalone ENTITIES.
-- DYNAMIC QUANTITIES: Do not create nodes for numbers. Identify the "Unit" (e.g., USD, Members, Hours) as an entity; the numeric value is a property to be handled by the Architect.
+- DYNAMIC QUANTITIES: Do not create nodes for numbers, for their units (e.g., USD, Members, Hours) or for the thing being counted; the numeric value and its unit are properties to be handled by the Architect.
DECISION CRITERIA:
1. Does it change frequently? YES -> Entity.
@@ -113,6 +112,7 @@ Before outputting an entity, you must pass the text through this logic to set th
4. **NO QUANTITY ENTITIES**
- You must not create entities for quantities (eg: X met 12 friends -> no entity for "12 friends" or "Number of friends").
+ - You must not create placeholder entities named after their own type (eg: MONEY:"Money"): every fact mentioning a quantity would share that single node.
- You can add details about quantities in the description or properties of other entities.
Example input 1:
@@ -138,7 +138,7 @@ Example output 2:
"""
SCOUT_AGENT_EXTRACT_ENTITIES_PROMPT = """
-Carefully read the text and extract ALL the entities, unique action instances (events), and quantitative units.
+Carefully read the text and extract ALL the entities and unique action instances (events), carrying any quantity as a property.
{targeting}
{reference_time}
@@ -158,8 +158,8 @@ Begin!
"""
SCOUT_AGENT_COARSE_EXTRACT_ENTITIES_PROMPT = """
-Carefully read the text and extract the most important entities that can be used to reconstruct a meaningful narrative,
-the unique action instances (events), and the quantitative units.
+Carefully read the text and extract the most important entities that can be used to reconstruct a meaningful narrative
+and the unique action instances (events), carrying any quantity as a property.
{targeting}
{reference_time}
@@ -173,14 +173,14 @@ OUTPUT RULES:
- Nodes/Entities MUST be atomic and not composite (phrases) (eg: "Went to San Francisco" is not atomic, "Went to" + "San Francisco" is atomic)
- Dates must be in the format "DD/MM/YYYY" and be stored as "happened_at" in the properties of the event nodes.
- When the reference date is provided, resolve relative dates (yesterday, last week, last Tuesday, etc.) against it into absolute DD/MM/YYYY values.
-- You must extract the most important entities that can be used to reconstruct a meaningful narrative,
-the unique action instances (events), and the quantitative units without omitting any concepts.
+- You must extract the most important entities that can be used to reconstruct a meaningful narrative
+and the unique action instances (events), without omitting any concepts and without creating quantity nodes.
Begin!
"""
SCOUT_AGENT_EXTRACT_STRUCTURED_ENTITIES_PROMPT = """
-Carefully read the text and extract ALL the entities, unique action instances (events), and quantitative units.
+Carefully read the text and extract ALL the entities and unique action instances (events), carrying any quantity as a property.
The following are the entities already identified that you can skip:
{current_entities}
diff --git a/src/core/saving/hub_bridges.py b/src/core/saving/hub_bridges.py
new file mode 100644
index 0000000..882119a
--- /dev/null
+++ b/src/core/saving/hub_bridges.py
@@ -0,0 +1,201 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import AbstractSet, Iterable, Mapping, Optional, Sequence
+
+
+_SPINE_ACTOR = ("MADE", "INITIATED", "PERFORMED", "EXPERIENCED", "COVERED")
+_SPINE_TARGET = ("TARGETED", "AFFECTED", "RESULTED")
+_CONTEXT_REL = ("OCCURRED", "WITHIN")
+
+
+@dataclass(frozen=True)
+class HubBridge:
+ event_a: str
+ event_b: str
+ shared_entity: str
+ shared_entity_name: str = ""
+ weight: float = 1.0
+
+ @property
+ def pair(self) -> tuple[str, str]:
+ return canonical_event_pair(self.event_a, self.event_b)
+
+
+def canonical_event_pair(event_a: str, event_b: str) -> tuple[str, str]:
+ a = str(event_a or "").strip()
+ b = str(event_b or "").strip()
+ if not a or not b or a == b:
+ raise ValueError("hub bridge requires two distinct event uuids")
+ return (a, b) if a < b else (b, a)
+
+
+def is_event_labels(labels: Optional[Sequence[str]]) -> bool:
+ return any(str(label).upper() == "EVENT" for label in (labels or ()))
+
+
+def is_spine_rel(name: Optional[str]) -> bool:
+ upper = (name or "").upper()
+ return any(token in upper for token in _SPINE_ACTOR) or any(
+ token in upper for token in _SPINE_TARGET
+ )
+
+
+def is_context_rel(name: Optional[str]) -> bool:
+ upper = (name or "").upper()
+ return any(token in upper for token in _CONTEXT_REL)
+
+
+def entity_event_memberships(
+ rows: Iterable[tuple[str, str, str, str]],
+) -> dict[str, dict[str, str]]:
+ """
+ Build shared-entity → {event_uuid: entity_name} from spine rows.
+
+ Each row is (event_uuid, entity_uuid, entity_name, rel_name).
+ Context / OCCURRED_WITHIN legs are ignored so bridges stay actor/object mediated.
+ """
+ membership: dict[str, dict[str, str]] = {}
+ for event_uuid, entity_uuid, entity_name, rel_name in rows:
+ event_id = str(event_uuid or "").strip()
+ entity_id = str(entity_uuid or "").strip()
+ if not event_id or not entity_id or event_id == entity_id:
+ continue
+ if is_context_rel(rel_name):
+ continue
+ if not is_spine_rel(rel_name):
+ continue
+ bucket = membership.setdefault(entity_id, {})
+ if event_id not in bucket:
+ bucket[event_id] = str(entity_name or "")
+ return membership
+
+
+def bridges_from_memberships(
+ membership: Mapping[str, Mapping[str, str]],
+) -> list[HubBridge]:
+ bridges: dict[tuple[str, str, str], HubBridge] = {}
+ for entity_id, events in membership.items():
+ event_ids = sorted(str(e) for e in events.keys() if e)
+ if len(event_ids) < 2:
+ continue
+ entity_name = ""
+ for name in events.values():
+ if name:
+ entity_name = str(name)
+ break
+ for i, left in enumerate(event_ids):
+ for right in event_ids[i + 1 :]:
+ a, b = canonical_event_pair(left, right)
+ key = (a, b, entity_id)
+ bridges[key] = HubBridge(
+ event_a=a,
+ event_b=b,
+ shared_entity=entity_id,
+ shared_entity_name=entity_name,
+ weight=1.0,
+ )
+ return sorted(
+ bridges.values(),
+ key=lambda br: (br.event_a, br.event_b, br.shared_entity),
+ )
+
+
+def _hub_session_set(
+ hub: str,
+ hub_sessions: Optional[Mapping[str, AbstractSet[str] | Sequence[str]]],
+) -> set[str]:
+ if not hub_sessions:
+ return set()
+ raw = hub_sessions.get(hub) or ()
+ return {str(s) for s in raw if s}
+
+
+def collect_bridge_neighbor_pool(
+ seed_hubs: Sequence[str],
+ bridges: Sequence[HubBridge],
+ *,
+ pool_cap: int,
+) -> list[tuple[str, HubBridge]]:
+ if pool_cap <= 0 or not seed_hubs or not bridges:
+ return []
+ seeds = [str(h) for h in seed_hubs if str(h or "").strip()]
+ seed_set = set(seeds)
+ if not seed_set:
+ return []
+
+ best: dict[str, tuple[tuple[float, str, str], HubBridge]] = {}
+ for bridge in bridges:
+ a, b = bridge.event_a, bridge.event_b
+ neighbor = None
+ if a in seed_set and b not in seed_set:
+ neighbor = b
+ elif b in seed_set and a not in seed_set:
+ neighbor = a
+ if neighbor is None:
+ continue
+ rank = (-float(bridge.weight), bridge.shared_entity, neighbor)
+ prev = best.get(neighbor)
+ if prev is None or rank < prev[0]:
+ best[neighbor] = (rank, bridge)
+
+ ranked = sorted(best.values(), key=lambda item: item[0])
+ chosen = [(rank[2], bridge) for rank, bridge in ranked[:pool_cap]]
+ chosen.sort(key=lambda item: (item[0], item[1].shared_entity, item[1].event_a))
+ return chosen
+
+
+def select_bridge_neighbors(
+ seed_hubs: Sequence[str],
+ bridges: Sequence[HubBridge],
+ *,
+ max_per_hub: int,
+ seed_sessions: Optional[AbstractSet[str]] = None,
+ hub_sessions: Optional[Mapping[str, AbstractSet[str] | Sequence[str]]] = None,
+) -> list[tuple[str, HubBridge]]:
+ """
+ Expand seed hubs by at most one bridge hop.
+
+ Returns deterministic (neighbor_event_uuid, bridge) pairs, unique by neighbor,
+ preferring novel source sessions vs the seed/selected set, then higher weight,
+ then lexicographic shared_entity / event ids.
+ """
+ if max_per_hub <= 0 or not seed_hubs or not bridges:
+ return []
+ seeds = [str(h) for h in seed_hubs if str(h or "").strip()]
+ seed_set = set(seeds)
+ if not seed_set:
+ return []
+
+ covered_sessions = {str(s) for s in (seed_sessions or ()) if s}
+ by_hub: dict[str, list[tuple[str, HubBridge]]] = {h: [] for h in seeds}
+ for bridge in bridges:
+ a, b = bridge.event_a, bridge.event_b
+ if a in seed_set and b not in seed_set:
+ by_hub.setdefault(a, []).append((b, bridge))
+ elif b in seed_set and a not in seed_set:
+ by_hub.setdefault(b, []).append((a, bridge))
+
+ chosen: list[tuple[str, HubBridge]] = []
+ seen_neighbors: set[str] = set()
+ for hub in sorted(by_hub.keys()):
+ taken = 0
+ remaining = list(by_hub[hub])
+ while remaining and taken < max_per_hub:
+ remaining.sort(
+ key=lambda item: (
+ -len(_hub_session_set(item[0], hub_sessions) - covered_sessions),
+ -float(item[1].weight),
+ item[1].shared_entity,
+ item[0],
+ )
+ )
+ neighbor, bridge = remaining.pop(0)
+ if neighbor in seen_neighbors or neighbor in seed_set:
+ continue
+ chosen.append((neighbor, bridge))
+ seen_neighbors.add(neighbor)
+ covered_sessions.update(_hub_session_set(neighbor, hub_sessions))
+ taken += 1
+ chosen.sort(key=lambda item: (item[0], item[1].shared_entity, item[1].event_a))
+ return chosen
diff --git a/src/core/saving/topic_hyperedges.py b/src/core/saving/topic_hyperedges.py
new file mode 100644
index 0000000..ecc2bdf
--- /dev/null
+++ b/src/core/saving/topic_hyperedges.py
@@ -0,0 +1,184 @@
+from __future__ import annotations
+
+import hashlib
+import re
+from dataclasses import dataclass
+from typing import Iterable, Mapping, Sequence
+
+
+_TOKEN_RE = re.compile(r"[a-z0-9]{3,}")
+_JACCARD_MERGE = 0.28
+_MAX_TOPIC_ENTITIES = 8
+
+
+@dataclass(frozen=True)
+class TopicSession:
+ topic_id: str
+ topic_label: str
+ session_id: str
+ weight: float = 1.0
+
+
+def _norm_tokens(text: str) -> set[str]:
+ return set(_TOKEN_RE.findall((text or "").lower()))
+
+
+def topic_id_for_label(label: str) -> str:
+ digest = hashlib.sha1(label.encode("utf-8")).hexdigest()[:12]
+ return f"topic:{digest}"
+
+
+def jaccard(a: set[str], b: set[str]) -> float:
+ if not a or not b:
+ return 0.0
+ inter = len(a & b)
+ if not inter:
+ return 0.0
+ return inter / len(a | b)
+
+
+def session_entity_sets(
+ rows: Iterable[tuple[str, str, str]],
+) -> dict[str, set[str]]:
+ """
+ Build session_id → entity-name tokens from (session_id, entity_uuid, entity_name).
+ """
+ out: dict[str, set[str]] = {}
+ for session_id, _entity_uuid, entity_name in rows:
+ sid = str(session_id or "").strip()
+ if not sid:
+ continue
+ tokens = _norm_tokens(str(entity_name or ""))
+ if not tokens:
+ continue
+ out.setdefault(sid, set()).update(tokens)
+ return out
+
+
+def cluster_sessions_into_topics(
+ session_entities: Mapping[str, set[str]],
+ *,
+ merge_threshold: float = _JACCARD_MERGE,
+) -> list[TopicSession]:
+ """Greedy Jaccard clustering of sessions by shared entity tokens."""
+ sessions = sorted(session_entities.keys())
+ if not sessions:
+ return []
+
+ clusters: list[dict[str, object]] = []
+ for sid in sessions:
+ ents = set(session_entities.get(sid) or ())
+ best_i = -1
+ best_score = -1.0
+ for i, cluster in enumerate(clusters):
+ score = jaccard(ents, set(cluster["entities"])) # type: ignore[arg-type]
+ if score > best_score:
+ best_score = score
+ best_i = i
+ if best_i >= 0 and best_score >= merge_threshold:
+ cluster = clusters[best_i]
+ members: list[str] = list(cluster["sessions"]) # type: ignore[arg-type]
+ members.append(sid)
+ cluster["sessions"] = members
+ merged = set(cluster["entities"]) # type: ignore[arg-type]
+ merged.update(ents)
+ cluster["entities"] = merged
+ else:
+ clusters.append({"sessions": [sid], "entities": ents})
+
+ results: list[TopicSession] = []
+ for cluster in clusters:
+ ents = sorted(set(cluster["entities"])) # type: ignore[arg-type]
+ label_tokens = ents[:_MAX_TOPIC_ENTITIES]
+ label = ", ".join(label_tokens) if label_tokens else "general"
+ tid = topic_id_for_label(label + "|" + ",".join(sorted(cluster["sessions"]))) # type: ignore[arg-type]
+ for sid in sorted(cluster["sessions"]): # type: ignore[arg-type]
+ results.append(
+ TopicSession(
+ topic_id=tid,
+ topic_label=label,
+ session_id=str(sid),
+ weight=1.0,
+ )
+ )
+ return results
+
+
+def lexical_topic_score(query: str, topic_label: str) -> float:
+ q = _norm_tokens(query)
+ t = _norm_tokens(topic_label)
+ if not q or not t:
+ return 0.0
+ return len(q & t) / max(1, len(q))
+
+
+def rrf_fuse(rank_lists: Sequence[Sequence[str]], *, k: int = 60) -> list[str]:
+ scores: dict[str, float] = {}
+ for ranks in rank_lists:
+ for i, item in enumerate(ranks):
+ key = str(item)
+ if not key:
+ continue
+ scores[key] = scores.get(key, 0.0) + 1.0 / (k + i + 1)
+ return [item for item, _ in sorted(scores.items(), key=lambda kv: (-kv[1], kv[0]))]
+
+
+def select_topics_and_sessions(
+ query: str,
+ memberships: Sequence[TopicSession],
+ seed_sessions: Sequence[str],
+ *,
+ k_topics: int = 10,
+ k_sessions: int = 10,
+) -> tuple[list[dict[str, object]], list[str]]:
+ """
+ Coarse-to-fine: score topics lexically (+ seed-session boost), expand sessions.
+ """
+ if not memberships:
+ return [], []
+
+ by_topic: dict[str, dict[str, object]] = {}
+ for row in memberships:
+ bucket = by_topic.setdefault(
+ row.topic_id,
+ {
+ "topic_id": row.topic_id,
+ "label": row.topic_label,
+ "sessions": set(),
+ },
+ )
+ sessions = bucket["sessions"]
+ assert isinstance(sessions, set)
+ sessions.add(row.session_id)
+
+ seed = {str(s) for s in seed_sessions if s}
+ scored: list[tuple[float, str]] = []
+ for topic_id, bucket in by_topic.items():
+ label = str(bucket["label"])
+ sessions = set(bucket["sessions"]) # type: ignore[arg-type]
+ lex = lexical_topic_score(query, label)
+ seed_hit = 1.0 if (sessions & seed) else 0.0
+ score = 0.7 * lex + 0.3 * seed_hit
+ scored.append((score, topic_id))
+ scored.sort(key=lambda item: (-item[0], item[1]))
+ top_topics = [tid for score, tid in scored[:k_topics] if score > 0 or seed]
+
+ topic_payload: list[dict[str, object]] = []
+ lexical_session_ranks: list[str] = []
+ seed_ranks = [s for s in seed_sessions if s]
+ for tid in top_topics:
+ bucket = by_topic[tid]
+ sessions = sorted(bucket["sessions"]) # type: ignore[arg-type]
+ topic_payload.append(
+ {
+ "topic_id": tid,
+ "label": bucket["label"],
+ "sessions": sessions,
+ }
+ )
+ for sid in sessions:
+ if sid not in lexical_session_ranks:
+ lexical_session_ranks.append(sid)
+
+ fused = rrf_fuse([seed_ranks, lexical_session_ranks])
+ return topic_payload, fused[:k_sessions]
diff --git a/src/core/search/entity_info.py b/src/core/search/entity_info.py
index d8015a2..c459f62 100644
--- a/src/core/search/entity_info.py
+++ b/src/core/search/entity_info.py
@@ -16,11 +16,13 @@ import numpy as np
from pydantic import BaseModel, Field
from src.constants.kg import Node, Predicate
+from src.lib.tracing.profiler import profile_stage
from src.services.kg_agent.main import graph_adapter
from src.services.kg_agent.main import embeddings_adapter
from src.services.kg_agent.main import vector_store_adapter
from src.utils.vector_search import VectorSearchFacade
from src.utils.similarity.vectors import cosine_similarity
+from src.utils.dates import parse_date_string, to_naive_utc
vector_search = VectorSearchFacade(vector_store_adapter)
@@ -43,6 +45,23 @@ class MatchPath(BaseModel):
children: List["MatchPath"] = Field(default_factory=list)
+def _recency_score(node: Node) -> float:
+ raw = node.happened_at
+ if not raw:
+ return 1.0
+ happened_at = parse_date_string(raw) if isinstance(raw, str) else raw
+ if not isinstance(happened_at, datetime):
+ print(
+ f"[DEBUG (_recency_score)]: unparseable happened_at {raw!r} "
+ f"on node {node.uuid}, recency left neutral"
+ )
+ return 1.0
+ days_ago = max(0, (datetime.now() - to_naive_utc(happened_at)).days)
+ if days_ago <= 0:
+ return 1.0
+ return 1 / (1 + np.log1p(days_ago))
+
+
class EventSynergyRetriever:
def __init__(self, memory_id: str):
"""
@@ -59,9 +78,10 @@ class EventSynergyRetriever:
current_node_id: str,
query_embedding: List[float],
) -> List[Tuple[Tuple[Predicate, Node], float]]:
- neighbors = graph_adapter.get_neighbors(
- [current_node_id], brain_id=self.brain_id
- )
+ with profile_stage("dossiers.get_neighbors"):
+ neighbors = graph_adapter.get_neighbors(
+ [current_node_id], brain_id=self.brain_id
+ )
if not neighbors or current_node_id not in neighbors:
return []
@@ -72,11 +92,12 @@ class EventSynergyRetriever:
v_id = cr.properties.get("v_id") if cr.properties else None
if v_id is None:
continue
- cr_vs = vector_store_adapter.get_by_ids(
- [v_id],
- store="relationships",
- brain_id=self.brain_id,
- )
+ with profile_stage("dossiers.edge_vector_fetch"):
+ cr_vs = vector_store_adapter.get_by_ids(
+ [v_id],
+ store="relationships",
+ brain_id=self.brain_id,
+ )
if not cr_vs or not getattr(cr_vs[0], "embeddings", None):
continue
scored.append(
@@ -164,12 +185,14 @@ class EventSynergyRetriever:
MatchPath: A hierarchical MatchPath rooted at the resolved target node containing the best-match path(s) and an aggregated similarity score. If the target node or any required embeddings are missing, returns a MatchPath with an empty path and similarity 0.0.
"""
- query_embedding = embeddings_adapter.embed_text(query)
- target_embedding = embeddings_adapter.embed_text(target)
+ with profile_stage("dossiers.embed", embeds=2):
+ query_embedding = embeddings_adapter.embed_text(query)
+ target_embedding = embeddings_adapter.embed_text(target)
- target_node_vs = vector_search.search_nodes(
- target_embedding.embeddings, brain_id=self.brain_id
- )
+ with profile_stage("dossiers.resolve_target"):
+ target_node_vs = vector_search.search_nodes(
+ target_embedding.embeddings, brain_id=self.brain_id
+ )
if not target_node_vs:
return MatchPath(
@@ -190,7 +213,10 @@ class EventSynergyRetriever:
children=[],
)
- target_node = graph_adapter.get_by_uuid(target_node_id, brain_id=self.brain_id)
+ with profile_stage("dossiers.load_target"):
+ target_node = graph_adapter.get_by_uuid(
+ target_node_id, brain_id=self.brain_id
+ )
if not target_node:
return MatchPath(
@@ -200,13 +226,16 @@ class EventSynergyRetriever:
children=[],
)
- synergy_paths = self._recursive_explorer(
- target_node_id,
- query_embedding.embeddings,
- depth=max_depth,
- visited_ids=set(),
- work_counter=[0],
- )
+ work_counter = [0]
+ with profile_stage("dossiers.explore", max_depth=max_depth) as detail:
+ synergy_paths = self._recursive_explorer(
+ target_node_id,
+ query_embedding.embeddings,
+ depth=max_depth,
+ visited_ids=set(),
+ work_counter=work_counter,
+ )
+ detail["visited_nodes"] = work_counter[0]
if not synergy_paths:
return MatchPath(
@@ -222,17 +251,7 @@ class EventSynergyRetriever:
if not node or not predicate:
continue
- days_ago = 0
- if node.happened_at:
- try:
- happened_at = node.happened_at
- if isinstance(happened_at, str):
- happened_at = datetime.fromisoformat(happened_at)
- days_ago = max(0, (datetime.now() - happened_at).days)
- except Exception:
- days_ago = 0
- recency = 1 / (1 + np.log1p(days_ago)) if days_ago > 0 else 1.0
- base_score = similarity * 0.6 + recency * 0.2
+ base_score = similarity * 0.6 + _recency_score(node) * 0.2
match_path = MatchPath(
target_node=target_node,
diff --git a/src/core/search/fact_filter.py b/src/core/search/fact_filter.py
index c5880e4..e2d15fb 100644
--- a/src/core/search/fact_filter.py
+++ b/src/core/search/fact_filter.py
@@ -18,7 +18,11 @@ def filter_relevant_facts(
) -> list[int]:
if not candidates:
return []
- if len(candidates) <= max_keep and llm_adapter is None:
+
+ # Without an LLM, preserve recall: return every candidate index and let the
+ # caller apply score-based capping. Truncating here would drop later
+ # high-value evidence that ranking already ordered.
+ if llm_adapter is None:
return list(range(len(candidates)))
indexed = "\n".join(f"{i}: {text}" for i, text in enumerate(candidates))
@@ -33,9 +37,6 @@ def filter_relevant_facts(
'Respond with JSON: {"keep_indices": [0, 2, ...]}'
)
- if llm_adapter is None:
- return list(range(min(max_keep, len(candidates))))
-
try:
response = llm_adapter.generate_structured(
prompt,
@@ -60,7 +61,7 @@ def filter_relevant_facts(
return cleaned[:max_keep]
except Exception:
pass
- return list(range(min(max_keep, len(candidates))))
+ return list(range(len(candidates)))
def reciprocal_rank_fusion(
@@ -75,3 +76,74 @@ def reciprocal_rank_fusion(
continue
scores[item] = scores.get(item, 0.0) + 1.0 / (k + rank + 1)
return sorted(scores.items(), key=lambda pair: pair[1], reverse=True)
+
+
+def _normalize_weighted_neighbors(
+ neighbors: list | dict | None,
+) -> list[tuple[str, float]]:
+ if not neighbors:
+ return []
+ if isinstance(neighbors, dict):
+ items = [(str(dst), float(weight)) for dst, weight in neighbors.items()]
+ else:
+ items = []
+ for entry in neighbors:
+ if isinstance(entry, tuple) and len(entry) == 2:
+ items.append((str(entry[0]), float(entry[1])))
+ else:
+ items.append((str(entry), 1.0))
+ cleaned: list[tuple[str, float]] = []
+ for dst, weight in items:
+ if not dst:
+ continue
+ cleaned.append((dst, max(0.0, weight)))
+ return cleaned
+
+
+def personalized_pagerank(
+ adjacency: dict[str, list | dict],
+ seeds: dict[str, float],
+ *,
+ damping: float = 0.85,
+ iterations: int = 20,
+) -> dict[str, float]:
+ if not adjacency and not seeds:
+ return {}
+ weighted: dict[str, list[tuple[str, float]]] = {}
+ nodes = set(seeds.keys())
+ for src, neighbors in adjacency.items():
+ src_key = str(src)
+ nodes.add(src_key)
+ edges = _normalize_weighted_neighbors(neighbors)
+ weighted[src_key] = edges
+ for dst, _weight in edges:
+ nodes.add(dst)
+ if not nodes:
+ return {}
+ node_list = sorted(nodes)
+ seed_mass = sum(max(0.0, w) for w in seeds.values()) or 1.0
+ personalization = {
+ node: (max(0.0, seeds.get(node, 0.0)) / seed_mass) for node in node_list
+ }
+ if sum(personalization.values()) <= 0:
+ uniform = 1.0 / len(node_list)
+ personalization = {node: uniform for node in node_list}
+ scores = dict(personalization)
+ for _ in range(max(1, iterations)):
+ next_scores = {
+ node: (1.0 - damping) * personalization[node] for node in node_list
+ }
+ for node in node_list:
+ neighbors = weighted.get(node) or []
+ total_weight = sum(weight for _dst, weight in neighbors)
+ if total_weight <= 0:
+ for target in node_list:
+ next_scores[target] += damping * scores[node] / len(node_list)
+ continue
+ mass = damping * scores[node]
+ for neighbor, weight in neighbors:
+ next_scores[neighbor] = next_scores.get(neighbor, 0.0) + (
+ mass * (weight / total_weight)
+ )
+ scores = next_scores
+ return scores
diff --git a/src/lib/milvus/client.py b/src/lib/milvus/client.py
index cc17785..2fbf2df 100644
--- a/src/lib/milvus/client.py
+++ b/src/lib/milvus/client.py
@@ -229,8 +229,11 @@ class MilvusClient(VectorStoreClient):
except Exception:
pass
+ from src.utils.vector_search import ann_overfetch_k, stable_top_k_vectors
+
+ fetch_k = ann_overfetch_k(k)
_results = client.search(
- store, data=[data_vector], limit=k, output_fields=["$meta"]
+ store, data=[data_vector], limit=fetch_k, output_fields=["$meta"]
)
results = []
for query_results in _results:
@@ -247,12 +250,7 @@ class MilvusClient(VectorStoreClient):
distance=1.0 - raw_score,
)
)
- results.sort(
- key=lambda x: (
- float("inf") if x.distance is None else float(x.distance)
- )
- )
- return results
+ return stable_top_k_vectors(results, k)
def get_by_ids(self, ids: list[str], store: str, brain_id: str) -> list[Vector]:
"""
diff --git a/src/lib/neo4j/client.py b/src/lib/neo4j/client.py
index e932397..de8affe 100644
--- a/src/lib/neo4j/client.py
+++ b/src/lib/neo4j/client.py
@@ -1670,6 +1670,8 @@ class Neo4jClient(GraphClient):
MATCH (n)-[r]-(m)-[r2]-(b)
WHERE n['uuid'] IN ["{uuids_list}"]
AND r2['flow_key'] = r['flow_key']
+ AND type(r) <> 'HUB_BRIDGE'
+ AND type(r2) <> 'HUB_BRIDGE'
RETURN
n['uuid'] as n_uuid, n['name'] as n_name, labels(n) as n_labels, n['description'] as n_description, properties(n) as n_properties, n['polarity'] as n_polarity, n['metadata'] as n_metadata, n['happened_at'] as n_happened_at, n['last_updated'] as n_last_updated, n['observations'] as n_observations,
r['uuid'] as r_uuid, type(r) as r_type, r['description'] as r_description, properties(r) as r_properties, r['flow_key'] as r_flow_key, r['last_updated'] as r_last_updated, r['observations'] as r_observations, r['amount'] as r_amount,
@@ -2050,4 +2052,209 @@ class Neo4jClient(GraphClient):
]
+ def get_event_hub_facts(
+ self,
+ event_uuids: list[str],
+ brain_id: str,
+ ) -> List[Tuple[Node, Predicate, Node, Predicate, Node]]:
+ unique = sorted({str(u) for u in event_uuids if u})
+ if not unique:
+ return []
+ self.ensure_database(brain_id)
+ uuids_list = '", "'.join(unique)
+ cypher_query = f"""
+ MATCH (n)-[r]-(m)-[r2]-(b)
+ WHERE m['uuid'] IN ["{uuids_list}"]
+ AND r2['flow_key'] = r['flow_key']
+ AND type(r) <> 'HUB_BRIDGE'
+ AND type(r2) <> 'HUB_BRIDGE'
+ AND 'EVENT' IN labels(m)
+ RETURN
+ n['uuid'] as n_uuid, n['name'] as n_name, labels(n) as n_labels, n['description'] as n_description, properties(n) as n_properties, n['polarity'] as n_polarity, n['metadata'] as n_metadata, n['happened_at'] as n_happened_at, n['last_updated'] as n_last_updated, n['observations'] as n_observations,
+ r['uuid'] as r_uuid, type(r) as r_type, r['description'] as r_description, properties(r) as r_properties, r['flow_key'] as r_flow_key, r['last_updated'] as r_last_updated, r['observations'] as r_observations, r['amount'] as r_amount,
+ CASE WHEN startNode(r) = n THEN 'out' ELSE 'in' END AS r_direction,
+ m['uuid'] as m_uuid, m['name'] as m_name, labels(m) as m_labels, m['description'] as m_description, properties(m) as m_properties, m['polarity'] as m_polarity, m['metadata'] as m_metadata, m['happened_at'] as m_happened_at, m['last_updated'] as m_last_updated, m['observations'] as m_observations,
+ r2['uuid'] as r2_uuid, type(r2) as r2_type, r2['description'] as r2_description, properties(r2) as r2_properties, r2['flow_key'] as r2_flow_key, r2['last_updated'] as r2_last_updated, r2['observations'] as r2_observations, r2['amount'] as r2_amount,
+ CASE WHEN startNode(r2) = m THEN 'out' ELSE 'in' END AS r2_direction,
+ b['uuid'] as b_uuid, b['name'] as b_name, labels(b) as b_labels, b['description'] as b_description, properties(b) as b_properties, b['polarity'] as b_polarity, b['metadata'] as b_metadata, b['happened_at'] as b_happened_at, b['last_updated'] as b_last_updated, b['observations'] as b_observations
+ ORDER BY m_uuid, r_uuid, r2_uuid
+ """
+ result = self.driver.execute_query(
+ cypher_query,
+ parameters_={"event_uuids": unique},
+ database_=brain_id,
+ )
+ return [
+ (
+ Node(
+ uuid=record.get("n_uuid", "") or "",
+ name=record.get("n_name", "") or "",
+ labels=record.get("n_labels", []) or [],
+ description=record.get("n_description", "") or "",
+ properties=always_dict(record.get("n_properties", {})),
+ polarity=record.get("n_polarity", "neutral"),
+ metadata=always_dict(record.get("n_metadata", {})),
+ happened_at=record.get("n_happened_at", "") or "",
+ last_updated=record.get("n_last_updated", "") or "",
+ observations=record.get("n_observations", []) or [],
+ ),
+ Predicate(
+ uuid=record.get("r_uuid", "") or "",
+ name=record.get("r_type", "") or "",
+ description=record.get("r_description", "") or "",
+ direction=record.get("r_direction", "neutral"),
+ properties=always_dict(record.get("r_properties", {})),
+ flow_key=record.get("r_flow_key", "") or "",
+ last_updated=record.get("r_last_updated", "") or "",
+ observations=record.get("r_observations", []) or [],
+ amount=record.get("r_amount"),
+ ),
+ Node(
+ uuid=record.get("m_uuid", "") or "",
+ name=record.get("m_name", "") or "",
+ labels=record.get("m_labels", []) or [],
+ description=record.get("m_description", "") or "",
+ properties=always_dict(record.get("m_properties", {})),
+ polarity=record.get("m_polarity", "neutral"),
+ metadata=always_dict(record.get("m_metadata", {})),
+ happened_at=record.get("m_happened_at", "") or "",
+ last_updated=record.get("m_last_updated", "") or "",
+ observations=record.get("m_observations", []) or [],
+ ),
+ Predicate(
+ uuid=record.get("r2_uuid", "") or "",
+ name=record.get("r2_type", "") or "",
+ description=record.get("r2_description", "") or "",
+ direction=record.get("r2_direction", "neutral"),
+ properties=always_dict(record.get("r2_properties", {})),
+ flow_key=record.get("r2_flow_key", "") or "",
+ last_updated=record.get("r2_last_updated", "") or "",
+ observations=record.get("r2_observations", []) or [],
+ amount=record.get("r2_amount"),
+ ),
+ Node(
+ uuid=record.get("b_uuid", "") or "",
+ name=record.get("b_name", "") or "",
+ labels=record.get("b_labels", []) or [],
+ description=record.get("b_description", "") or "",
+ properties=always_dict(record.get("b_properties", {})),
+ polarity=record.get("b_polarity", "neutral"),
+ metadata=always_dict(record.get("b_metadata", {})),
+ happened_at=record.get("b_happened_at", "") or "",
+ last_updated=record.get("b_last_updated", "") or "",
+ observations=record.get("b_observations", []) or [],
+ ),
+ )
+ for record in result.records
+ ]
+
+ def rebuild_hub_bridge_index(self, brain_id: str) -> int:
+ from src.core.saving.hub_bridges import (
+ bridges_from_memberships,
+ entity_event_memberships,
+ )
+
+ self.ensure_database(brain_id)
+ cypher_rows = """
+ MATCH (event)-[r]-(entity)
+ WHERE 'EVENT' IN labels(event)
+ AND NOT 'EVENT' IN labels(entity)
+ AND type(r) <> 'HUB_BRIDGE'
+ RETURN event['uuid'] AS event_uuid,
+ entity['uuid'] AS entity_uuid,
+ entity['name'] AS entity_name,
+ type(r) AS rel_name
+ """
+ result = self.driver.execute_query(cypher_rows, database_=brain_id)
+ rows = [
+ (
+ str(record.get("event_uuid") or ""),
+ str(record.get("entity_uuid") or ""),
+ str(record.get("entity_name") or ""),
+ str(record.get("rel_name") or ""),
+ )
+ for record in result.records
+ ]
+ bridges = bridges_from_memberships(entity_event_memberships(rows))
+ self.driver.execute_query(
+ "MATCH ()-[r:HUB_BRIDGE]->() DELETE r",
+ database_=brain_id,
+ )
+ for bridge in bridges:
+ self.driver.execute_query(
+ """
+ MATCH (a {uuid: $event_a}), (b {uuid: $event_b})
+ WHERE 'EVENT' IN labels(a) AND 'EVENT' IN labels(b)
+ MERGE (a)-[r:HUB_BRIDGE {
+ shared_entity: $shared_entity
+ }]->(b)
+ SET r.shared_entity_name = $shared_entity_name,
+ r.weight = $weight,
+ r.uuid = $bridge_uuid
+ """,
+ parameters_={
+ "event_a": bridge.event_a,
+ "event_b": bridge.event_b,
+ "shared_entity": bridge.shared_entity,
+ "shared_entity_name": bridge.shared_entity_name,
+ "weight": float(bridge.weight),
+ "bridge_uuid": (
+ f"hub-bridge:{bridge.event_a}:{bridge.event_b}:"
+ f"{bridge.shared_entity}"
+ ),
+ },
+ database_=brain_id,
+ )
+ return len(bridges)
+
+ def get_hub_bridges(
+ self,
+ event_uuids: list[str],
+ brain_id: str,
+ ) -> list:
+ from src.core.saving.hub_bridges import HubBridge
+
+ unique = sorted({str(u) for u in event_uuids if u})
+ if not unique:
+ return []
+ self.ensure_database(brain_id)
+ result = self.driver.execute_query(
+ """
+ MATCH (a)-[r:HUB_BRIDGE]-(b)
+ WHERE a['uuid'] IN $uuids OR b['uuid'] IN $uuids
+ RETURN a['uuid'] AS event_a,
+ b['uuid'] AS event_b,
+ r.shared_entity AS shared_entity,
+ r.shared_entity_name AS shared_entity_name,
+ r.weight AS weight
+ ORDER BY a['uuid'], b['uuid'], r.shared_entity
+ """,
+ parameters_={"uuids": unique},
+ database_=brain_id,
+ )
+ bridges: list[HubBridge] = []
+ seen: set[tuple[str, str, str]] = set()
+ for record in result.records:
+ a = str(record.get("event_a") or "")
+ b = str(record.get("event_b") or "")
+ entity = str(record.get("shared_entity") or "")
+ if not a or not b or a == b:
+ continue
+ lo, hi = (a, b) if a < b else (b, a)
+ key = (lo, hi, entity)
+ if key in seen:
+ continue
+ seen.add(key)
+ bridges.append(
+ HubBridge(
+ event_a=lo,
+ event_b=hi,
+ shared_entity=entity,
+ shared_entity_name=str(record.get("shared_entity_name") or ""),
+ weight=float(record.get("weight") or 1.0),
+ )
+ )
+ return bridges
+
+
_neo4j_client = Neo4jClient()
diff --git a/src/lib/postgresql/graph_store.py b/src/lib/postgresql/graph_store.py
index 864eb18..54b9b49 100644
--- a/src/lib/postgresql/graph_store.py
+++ b/src/lib/postgresql/graph_store.py
@@ -172,6 +172,26 @@ class PostgreSQLGraphStore:
);
CREATE INDEX IF NOT EXISTS idx_kg_relationships_endpoints
ON kg_relationships(source_uuid, target_uuid);
+ CREATE TABLE IF NOT EXISTS kg_hub_bridges (
+ event_a TEXT NOT NULL,
+ event_b TEXT NOT NULL,
+ shared_entity TEXT NOT NULL,
+ shared_entity_name TEXT NOT NULL DEFAULT '',
+ weight DOUBLE PRECISION NOT NULL DEFAULT 1.0,
+ PRIMARY KEY (event_a, event_b, shared_entity),
+ CHECK (event_a < event_b)
+ );
+ CREATE INDEX IF NOT EXISTS idx_kg_hub_bridges_a ON kg_hub_bridges(event_a);
+ CREATE INDEX IF NOT EXISTS idx_kg_hub_bridges_b ON kg_hub_bridges(event_b);
+ CREATE TABLE IF NOT EXISTS kg_topic_sessions (
+ topic_id TEXT NOT NULL,
+ topic_label TEXT NOT NULL DEFAULT '',
+ session_id TEXT NOT NULL,
+ weight DOUBLE PRECISION NOT NULL DEFAULT 1.0,
+ PRIMARY KEY (topic_id, session_id)
+ );
+ CREATE INDEX IF NOT EXISTS idx_kg_topic_sessions_session
+ ON kg_topic_sessions(session_id);
"""
def __init__(self) -> None:
@@ -786,3 +806,135 @@ class PostgreSQLGraphStore:
if limit is not None and len(records) >= limit:
return records
return records
+
+ def replace_hub_bridges(
+ self,
+ brain_id: str,
+ bridges: list[tuple[str, str, str, str, float]],
+ ) -> int:
+ self._ensure_brain_schema(brain_id)
+ with self._connection(brain_id) as conn:
+ with conn.cursor() as cur:
+ cur.execute("DELETE FROM kg_hub_bridges")
+ for event_a, event_b, shared_entity, shared_name, weight in bridges:
+ if not event_a or not event_b or event_a == event_b:
+ continue
+ a, b = (event_a, event_b) if event_a < event_b else (event_b, event_a)
+ cur.execute(
+ """
+ INSERT INTO kg_hub_bridges
+ (event_a, event_b, shared_entity, shared_entity_name, weight)
+ VALUES (%s, %s, %s, %s, %s)
+ ON CONFLICT (event_a, event_b, shared_entity) DO UPDATE SET
+ shared_entity_name = EXCLUDED.shared_entity_name,
+ weight = EXCLUDED.weight
+ """,
+ (a, b, shared_entity, shared_name or "", float(weight)),
+ )
+ conn.commit()
+ return len(bridges)
+
+ def get_hub_bridges_for_events(
+ self,
+ brain_id: str,
+ event_uuids: list[str],
+ ) -> list[tuple[str, str, str, str, float]]:
+ if not event_uuids:
+ return []
+ self._ensure_brain_schema(brain_id)
+ unique = sorted({str(u) for u in event_uuids if u})
+ if not unique:
+ return []
+ with self._connection(brain_id) as conn:
+ with conn.cursor() as cur:
+ cur.execute(
+ """
+ SELECT event_a, event_b, shared_entity, shared_entity_name, weight
+ FROM kg_hub_bridges
+ WHERE event_a = ANY(%s) OR event_b = ANY(%s)
+ ORDER BY event_a, event_b, shared_entity
+ """,
+ (unique, unique),
+ )
+ rows = cur.fetchall()
+ return [
+ (
+ str(row[0]),
+ str(row[1]),
+ str(row[2]),
+ str(row[3] or ""),
+ float(row[4] or 1.0),
+ )
+ for row in rows
+ ]
+
+ def count_hub_bridges(self, brain_id: str) -> int:
+ self._ensure_brain_schema(brain_id)
+ with self._connection(brain_id) as conn:
+ with conn.cursor() as cur:
+ cur.execute("SELECT COUNT(*) FROM kg_hub_bridges")
+ row = cur.fetchone()
+ return int(row[0] if row else 0)
+
+ def replace_topic_sessions(
+ self,
+ brain_id: str,
+ rows: list[tuple[str, str, str, float]],
+ ) -> int:
+ self._ensure_brain_schema(brain_id)
+ with self._connection(brain_id) as conn:
+ with conn.cursor() as cur:
+ cur.execute("DELETE FROM kg_topic_sessions")
+ for topic_id, topic_label, session_id, weight in rows:
+ if not topic_id or not session_id:
+ continue
+ cur.execute(
+ """
+ INSERT INTO kg_topic_sessions
+ (topic_id, topic_label, session_id, weight)
+ VALUES (%s, %s, %s, %s)
+ ON CONFLICT (topic_id, session_id) DO UPDATE SET
+ topic_label = EXCLUDED.topic_label,
+ weight = EXCLUDED.weight
+ """,
+ (
+ topic_id,
+ topic_label or "",
+ session_id,
+ float(weight),
+ ),
+ )
+ conn.commit()
+ return len(rows)
+
+ def list_topic_sessions(
+ self, brain_id: str
+ ) -> list[tuple[str, str, str, float]]:
+ self._ensure_brain_schema(brain_id)
+ with self._connection(brain_id) as conn:
+ with conn.cursor() as cur:
+ cur.execute(
+ """
+ SELECT topic_id, topic_label, session_id, weight
+ FROM kg_topic_sessions
+ ORDER BY topic_id, session_id
+ """
+ )
+ rows = cur.fetchall()
+ return [
+ (
+ str(row[0]),
+ str(row[1] or ""),
+ str(row[2]),
+ float(row[3] or 1.0),
+ )
+ for row in rows
+ ]
+
+ def count_topic_sessions(self, brain_id: str) -> int:
+ self._ensure_brain_schema(brain_id)
+ with self._connection(brain_id) as conn:
+ with conn.cursor() as cur:
+ cur.execute("SELECT COUNT(*) FROM kg_topic_sessions")
+ row = cur.fetchone()
+ return int(row[0] if row else 0)
diff --git a/src/lib/postgresql/networkx_client.py b/src/lib/postgresql/networkx_client.py
index 988ffde..1272285 100644
--- a/src/lib/postgresql/networkx_client.py
+++ b/src/lib/postgresql/networkx_client.py
@@ -1271,6 +1271,244 @@ class NetworkXGraphClient(GraphClient):
return triples
+ def get_event_hub_facts(
+ self,
+ event_uuids: list[str],
+ brain_id: str,
+ ) -> List[Tuple[Node, Predicate, Node, Predicate, Node]]:
+ unique = sorted({str(u) for u in event_uuids if u})
+ if not unique:
+ return []
+ self._store.ensure_database(brain_id)
+ brain = self._store.get_brain(brain_id)
+ event_set = set(unique)
+ neighbor_seeds: list[str] = []
+ seen_seeds: set[str] = set()
+ for event_uuid in unique:
+ for neighbor, _key, _data, _direction in self._incident_edges(
+ brain, event_uuid
+ ):
+ if neighbor in seen_seeds or neighbor in event_set:
+ continue
+ seen_seeds.add(neighbor)
+ neighbor_seeds.append(neighbor)
+ neighbor_seeds.sort()
+ results = []
+ seen_keys: set[tuple[str, str]] = set()
+ for (
+ n_uuid,
+ m_uuid,
+ b_uuid,
+ r1_data,
+ r2_data,
+ r_direction,
+ r2_direction,
+ ) in self._event_path_records(brain, neighbor_seeds):
+ if m_uuid not in event_set:
+ continue
+ key = (
+ str(r1_data.get("uuid") or ""),
+ str(r2_data.get("uuid") or ""),
+ )
+ if key in seen_keys:
+ continue
+ seen_keys.add(key)
+ results.append(
+ (
+ self._node_from_brain(brain, n_uuid),
+ self._predicate_from_edge(r1_data, r_direction),
+ self._node_from_brain(brain, m_uuid),
+ self._predicate_from_edge(r2_data, r2_direction),
+ self._node_from_brain(brain, b_uuid),
+ )
+ )
+ results.sort(
+ key=lambda row: (
+ str(getattr(row[2], "uuid", "") or ""),
+ str(getattr(row[1], "uuid", "") or ""),
+ str(getattr(row[3], "uuid", "") or ""),
+ )
+ )
+ return results
+
+ def rebuild_hub_bridge_index(self, brain_id: str) -> int:
+ from src.core.saving.hub_bridges import (
+ bridges_from_memberships,
+ entity_event_memberships,
+ is_event_labels,
+ )
+
+ self._store.ensure_database(brain_id)
+ brain = self._store.get_brain(brain_id)
+ rows: list[tuple[str, str, str, str]] = []
+ for source, target, _key, edge_data in brain.graph.edges(keys=True, data=True):
+ rel_name = str(edge_data.get("rel_type") or edge_data.get("name") or "")
+ source_labels = brain.labels(source)
+ target_labels = brain.labels(target)
+ source_is_event = is_event_labels(source_labels)
+ target_is_event = is_event_labels(target_labels)
+ if source_is_event and not target_is_event:
+ rows.append(
+ (
+ source,
+ target,
+ str(brain.node_data(target).get("name") or ""),
+ rel_name,
+ )
+ )
+ elif target_is_event and not source_is_event:
+ rows.append(
+ (
+ target,
+ source,
+ str(brain.node_data(source).get("name") or ""),
+ rel_name,
+ )
+ )
+ bridges = bridges_from_memberships(entity_event_memberships(rows))
+ payload = [
+ (
+ br.event_a,
+ br.event_b,
+ br.shared_entity,
+ br.shared_entity_name,
+ br.weight,
+ )
+ for br in bridges
+ ]
+ return self._store.replace_hub_bridges(brain_id, payload)
+
+ def get_hub_bridges(
+ self,
+ event_uuids: list[str],
+ brain_id: str,
+ ) -> list[dict]:
+ from src.core.saving.hub_bridges import HubBridge
+
+ rows = self._store.get_hub_bridges_for_events(brain_id, event_uuids)
+ return [
+ HubBridge(
+ event_a=a,
+ event_b=b,
+ shared_entity=entity,
+ shared_entity_name=name,
+ weight=weight,
+ )
+ for a, b, entity, name, weight in rows
+ ]
+
+ def rebuild_topic_index(self, brain_id: str) -> int:
+ import re
+
+ from src.core.saving.hub_bridges import (
+ is_context_rel,
+ is_event_labels,
+ is_spine_rel,
+ )
+ from src.core.saving.topic_hyperedges import (
+ cluster_sessions_into_topics,
+ session_entity_sets,
+ )
+
+ self._store.ensure_database(brain_id)
+ brain = self._store.get_brain(brain_id)
+ session_re = re.compile(r"session_(\d+)", re.IGNORECASE)
+
+ def sessions_from_text(text: str) -> list[str]:
+ found: list[str] = []
+ seen: set[str] = set()
+ for match in session_re.finditer(text or ""):
+ sid = f"session_{match.group(1)}"
+ if sid not in seen:
+ seen.add(sid)
+ found.append(sid)
+ return found
+
+ chunk_ids: list[str] = []
+ event_chunks: dict[str, list[str]] = {}
+ for node_uuid in brain.graph.nodes:
+ if not is_event_labels(brain.labels(node_uuid)):
+ continue
+ props = brain.node_data(node_uuid)
+ ids = [
+ str(x)
+ for x in (props.get("source_chunk_ids") or [])
+ if x
+ ]
+ single = props.get("source_chunk_id")
+ if single:
+ ids.append(str(single))
+ ids = sorted(set(ids))
+ event_chunks[node_uuid] = ids
+ chunk_ids.extend(ids)
+
+ chunk_sessions: dict[str, list[str]] = {}
+ if chunk_ids:
+ try:
+ from src.services.data.main import data_adapter
+
+ chunks, _ = data_adapter.get_text_chunks_by_ids(
+ sorted(set(chunk_ids))[:500],
+ False,
+ brain_id,
+ )
+ for chunk in chunks:
+ cid = str(getattr(chunk, "id", "") or "")
+ body = getattr(chunk, "text", "") or ""
+ if cid:
+ chunk_sessions[cid] = sessions_from_text(body)
+ except Exception:
+ chunk_sessions = {}
+
+ rows: list[tuple[str, str, str]] = []
+ for source, target, _key, edge_data in brain.graph.edges(keys=True, data=True):
+ rel_name = str(edge_data.get("rel_type") or edge_data.get("name") or "")
+ if is_context_rel(rel_name) or not is_spine_rel(rel_name):
+ continue
+ source_is_event = is_event_labels(brain.labels(source))
+ target_is_event = is_event_labels(brain.labels(target))
+ if source_is_event and not target_is_event:
+ event_uuid, entity_uuid = source, target
+ elif target_is_event and not source_is_event:
+ event_uuid, entity_uuid = target, source
+ else:
+ continue
+ entity_name = str(brain.node_data(entity_uuid).get("name") or "")
+ sessions: list[str] = []
+ for cid in event_chunks.get(event_uuid, []):
+ sessions.extend(chunk_sessions.get(cid, []))
+ if not sessions:
+ props = brain.node_data(event_uuid)
+ sessions = sessions_from_text(
+ " ".join(
+ str(props.get(k) or "")
+ for k in ("name", "description", "summary")
+ )
+ )
+ for sid in sessions:
+ rows.append((sid, entity_uuid, entity_name))
+
+ memberships = cluster_sessions_into_topics(session_entity_sets(rows))
+ payload = [
+ (m.topic_id, m.topic_label, m.session_id, m.weight) for m in memberships
+ ]
+ return self._store.replace_topic_sessions(brain_id, payload)
+
+ def list_topic_memberships(self, brain_id: str) -> list:
+ from src.core.saving.topic_hyperedges import TopicSession
+
+ rows = self._store.list_topic_sessions(brain_id)
+ return [
+ TopicSession(
+ topic_id=tid,
+ topic_label=label,
+ session_id=sid,
+ weight=weight,
+ )
+ for tid, label, sid, weight in rows
+ ]
+
+
def _records(data: list | Any) -> Any:
if hasattr(data, "records"):
return data
diff --git a/src/lib/postgresql/vectors.py b/src/lib/postgresql/vectors.py
index 7dba716..e8b5402 100644
--- a/src/lib/postgresql/vectors.py
+++ b/src/lib/postgresql/vectors.py
@@ -205,31 +205,44 @@ class PostgreSQLVectorStoreClient(VectorStoreClient):
def search_vectors(
self, data_vector: list[float], brain_id: str, store: str, k: int = 10
) -> list[Vector]:
+ from src.utils.vector_search import ann_overfetch_k, stable_top_k_vectors
+
self._ensure_store(store, brain_id)
+ if k <= 0:
+ return []
table = _table_name(store)
+ fetch_k = ann_overfetch_k(k)
with self._connection(brain_id) as conn:
with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cur:
+ cur.execute(
+ "SELECT set_config('hnsw.ef_search', %s, true)",
+ (str(max(100, fetch_k)),),
+ )
cur.execute(
f"""
SELECT id, uuid, metadata,
(embeddings <=> %s::vector) AS distance
FROM {table}
- ORDER BY embeddings <=> %s::vector
+ ORDER BY embeddings <=> %s::vector, uuid ASC, id ASC
LIMIT %s
""",
- (data_vector, data_vector, k),
+ (data_vector, data_vector, fetch_k),
)
rows = cur.fetchall()
- results = [
- Vector(
- id=str(row["id"]),
- metadata=dict(row["metadata"] or {}),
- distance=float(row["distance"]),
+ results: list[Vector] = []
+ for row in rows:
+ metadata = dict(row["metadata"] or {})
+ row_uuid = row.get("uuid")
+ if row_uuid and not metadata.get("uuid"):
+ metadata["uuid"] = str(row_uuid)
+ results.append(
+ Vector(
+ id=str(row_uuid or row["id"]),
+ metadata=metadata,
+ distance=float(row["distance"]),
+ )
)
- for row in rows
- ]
- results.sort(key=lambda x: x.distance)
- return results
+ return stable_top_k_vectors(results, k)
def get_by_ids(self, ids: list[str], store: str, brain_id: str) -> list[Vector]:
if not ids:
diff --git a/src/services/api/constants/requests.py b/src/services/api/constants/requests.py
index 79837c8..0ca0955 100644
--- a/src/services/api/constants/requests.py
+++ b/src/services/api/constants/requests.py
@@ -393,9 +393,13 @@ class GetContextRequestBody(BaseModel):
text: str
brain_id: str = "default"
historical_limit: int = 10
- max_facts: int = 40
+ max_facts: int = Field(40, ge=0)
max_passages: int = 8
apply_fact_filter: bool = True
+ use_ppr: bool = True
+ sufficiency_retry: bool = False
+ profile_stages: bool = False
+ cross_event_bridges: int = Field(3, ge=0)
class GetContextTriple(BaseModel):
@@ -403,6 +407,8 @@ class GetContextTriple(BaseModel):
identified_entity: str
triple: Tuple[Node, Predicate, Node, Predicate, Node]
+ source_chunk_ids: Optional[List[str]] = None
+ source_session_ids: Optional[List[str]] = None
class GetContextResponse(BaseModel):
@@ -412,3 +418,8 @@ class GetContextResponse(BaseModel):
triples: List[GetContextTriple]
historical_context: List[str] = []
source_passages: List[str] = []
+ graph_session_ids: Optional[List[str]] = None
+ temporal_conflicts: Optional[List[dict[str, Any]]] = None
+ paths: Optional[List[dict[str, Any]]] = None
+ topics: Optional[List[dict[str, Any]]] = None
+ stage_timings: Optional[dict[str, Any]] = None
diff --git a/src/services/api/controllers/retrieve.py b/src/services/api/controllers/retrieve.py
index 98d2f1c..4b48d5f 100644
--- a/src/services/api/controllers/retrieve.py
+++ b/src/services/api/controllers/retrieve.py
@@ -10,7 +10,9 @@ Modified By: Christian Nonis <[email protected]>
import asyncio
import json
-import threading
+import re
+from collections import defaultdict
+from datetime import datetime
from typing import Any, Optional
from fastapi import HTTPException
@@ -19,9 +21,13 @@ from starlette.responses import JSONResponse
from src.constants.embeddings import Vector
from src.constants.kg import IdentificationParams, Node, Predicate
from src.core.search.entities import search_entities
-from src.core.search.entity_info import EventSynergyRetriever, MatchPath
-from src.core.search.fact_filter import filter_relevant_facts, reciprocal_rank_fusion
+from src.core.search.fact_filter import (
+ filter_relevant_facts,
+ personalized_pagerank,
+ reciprocal_rank_fusion,
+)
from src.core.search.relationships import search_relationships
+from src.lib.tracing.profiler import profile_request, profile_stage
from src.utils.vector_search import VectorSearchFacade
from src.services.api.constants.requests import (
GetContextRequestBody,
@@ -34,16 +40,340 @@ from src.services.api.constants.requests import (
from src.services.kg_agent.main import graph_adapter, kg_agent
from src.services.data.main import data_adapter
from src.services.kg_agent.main import embeddings_adapter, vector_store_adapter
+from src.utils.dates import parse_date_string, to_naive_utc
from src.utils.similarity.vectors import cosine_similarity
from src.utils.nlp.ner import _entity_extractor
vector_search = VectorSearchFacade(vector_store_adapter)
_MAX_NOUN_CHUNKS = 5
-_MAX_DOSSIER_ENTITIES = 5
-_DOSSIER_MAX_DEPTH = 3
-_SEED_K = 20
-_PASSAGE_K = 12
+_SEED_K = 25
+_PASSAGE_K = 24
+_PPR_DAMPING = 0.85
+_PPR_ITERS = 20
+_SESSION_RE = re.compile(r"session_(\d+)", re.IGNORECASE)
+_SPINE_ACTOR = ("MADE", "INITIATED", "PERFORMED", "EXPERIENCED", "COVERED")
+_SPINE_TARGET = ("TARGETED", "AFFECTED", "RESULTED")
+_CONTEXT_REL = ("OCCURRED", "WITHIN")
+_DIVERSIFY_LAMBDA = 0.65
+_BRIDGE_SCORE_PENALTY = 0.15
+_BRIDGE_SEED_HUB_CAP = 12
+_BRIDGE_RESERVE_HUB_CAP = 6
+_BRIDGE_RESERVE_MIN = 3
+_BRIDGE_RESERVE_FRAC_DENOM = 4
+_DISTANCE_QUANT_DECIMALS = 3
+
+
+def _quantize_distance(distance: float) -> float:
+ try:
+ value = float(distance)
+ except (TypeError, ValueError):
+ return float("inf")
+ if value != value or value == float("inf") or value == float("-inf"):
+ return value
+ return round(value, _DISTANCE_QUANT_DECIMALS)
+
+
+def _candidate_event_uuid(candidate: dict[str, Any]) -> str:
+ return _event_hub_id(candidate)
+
+
+def _best_hub_fact_texts(candidates: list[dict[str, Any]]) -> dict[str, str]:
+ best: dict[str, tuple[float, str]] = {}
+ for candidate in candidates:
+ hub = _candidate_event_uuid(candidate)
+ if not hub:
+ continue
+ text = str(candidate.get("text") or "").strip()
+ if not text:
+ continue
+ score = _candidate_distance(candidate)
+ prev = best.get(hub)
+ if prev is None or score < prev[0] or (score == prev[0] and text < prev[1]):
+ best[hub] = (score, text)
+ return {hub: text for hub, (_score, text) in best.items()}
+
+
+def _paths_for_curated(
+ paths: list[dict[str, Any]],
+ curated: list[dict[str, Any]],
+) -> list[dict[str, Any]]:
+ """Keep bridge paths whose hubs both survived curation; attach fact legs."""
+ hub_texts = _best_hub_fact_texts(curated)
+ if not hub_texts or not paths:
+ return []
+ curated_hubs = set(hub_texts)
+ out: list[dict[str, Any]] = []
+ for path in paths:
+ hubs = [str(h) for h in (path.get("hubs") or []) if h]
+ if len(hubs) < 2 or not all(hub in curated_hubs for hub in hubs):
+ continue
+ out.append(
+ {
+ "hubs": hubs,
+ "shared_entity": path.get("shared_entity"),
+ "shared_entity_name": path.get("shared_entity_name"),
+ "weight": path.get("weight"),
+ "legs": [hub_texts[hub] for hub in hubs],
+ }
+ )
+ out.sort(
+ key=lambda p: (
+ tuple(p["hubs"]),
+ str(p.get("shared_entity") or ""),
+ )
+ )
+ return out
+
+
+def _candidate_distance(candidate: dict[str, Any]) -> float:
+ raw = candidate.get("score")
+ if raw is None:
+ return float("inf")
+ try:
+ return _quantize_distance(float(raw))
+ except (TypeError, ValueError):
+ return float("inf")
+
+
+def _rank_bridge_seed_hubs(
+ candidates: list[dict[str, Any]],
+ *,
+ cap: int,
+) -> list[str]:
+ hub_best_score: dict[str, float] = {}
+ for candidate in candidates:
+ hub = _candidate_event_uuid(candidate)
+ if not hub:
+ continue
+ score = _candidate_distance(candidate)
+ prev = hub_best_score.get(hub)
+ if prev is None or score < prev:
+ hub_best_score[hub] = score
+ return sorted(
+ hub_best_score.keys(),
+ key=lambda hub: (hub_best_score[hub], hub),
+ )[: max(0, cap)]
+
+
+def _resolve_chunk_sessions(
+ chunk_ids: list[str], brain_id: str
+) -> dict[str, list[str]]:
+ unique: list[str] = []
+ seen: set[str] = set()
+ for chunk_id in chunk_ids:
+ cid = str(chunk_id or "").strip()
+ if cid and cid not in seen:
+ seen.add(cid)
+ unique.append(cid)
+ if not unique:
+ return {}
+ out: dict[str, list[str]] = {}
+ try:
+ chunks, _ = data_adapter.get_text_chunks_by_ids(
+ unique[:120], False, brain_id
+ )
+ except Exception:
+ return {}
+ for chunk in chunks:
+ chunk_id = str(getattr(chunk, "id", "") or "")
+ body = (getattr(chunk, "text", None) or "").strip()
+ if not chunk_id or not body:
+ continue
+ sessions = _session_ids_from_text(body)
+ if sessions:
+ out[chunk_id] = sessions
+ return out
+
+
+def _sessions_from_chunk_map(
+ chunk_ids: list[str], chunk_sessions: dict[str, list[str]]
+) -> list[str]:
+ found: list[str] = []
+ seen: set[str] = set()
+ for chunk_id in chunk_ids:
+ for sid in chunk_sessions.get(str(chunk_id), []) or []:
+ if sid not in seen:
+ seen.add(sid)
+ found.append(sid)
+ return found
+
+
+def _expand_cross_event_bridges(
+ candidates: list[dict[str, Any]],
+ brain_id: str,
+ *,
+ max_per_hub: int,
+) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
+ from src.core.saving.hub_bridges import (
+ HubBridge,
+ select_bridge_neighbors,
+ )
+
+ if max_per_hub <= 0 or not candidates:
+ return candidates, []
+
+ hub_best_score: dict[str, float] = {}
+ seed_sessions: set[str] = set()
+ seed_chunk_ids: list[str] = []
+ for candidate in candidates:
+ hub = _candidate_event_uuid(candidate)
+ if hub:
+ score = _candidate_distance(candidate)
+ prev = hub_best_score.get(hub)
+ if prev is None or score < prev:
+ hub_best_score[hub] = score
+ for sid in _candidate_session_ids(candidate):
+ seed_sessions.add(sid)
+ seed_chunk_ids.extend(str(c) for c in (candidate.get("chunk_ids") or []) if c)
+
+ ranked_seeds = _rank_bridge_seed_hubs(
+ candidates, cap=_BRIDGE_SEED_HUB_CAP
+ )
+ if not ranked_seeds:
+ return candidates, []
+ reserve_hubs = set(
+ ranked_seeds[: min(_BRIDGE_RESERVE_HUB_CAP, len(ranked_seeds))]
+ )
+
+ try:
+ bridges = graph_adapter.get_hub_bridges(ranked_seeds, brain_id=brain_id)
+ except Exception:
+ return candidates, []
+ if not bridges:
+ return candidates, []
+
+ typed_bridges: list[HubBridge] = []
+ for item in bridges:
+ if isinstance(item, HubBridge):
+ typed_bridges.append(item)
+ else:
+ typed_bridges.append(
+ HubBridge(
+ event_a=str(item.get("event_a") or ""),
+ event_b=str(item.get("event_b") or ""),
+ shared_entity=str(item.get("shared_entity") or ""),
+ shared_entity_name=str(item.get("shared_entity_name") or ""),
+ weight=float(item.get("weight") or 1.0),
+ )
+ )
+
+ expansions = select_bridge_neighbors(
+ ranked_seeds,
+ typed_bridges,
+ max_per_hub=max_per_hub,
+ )
+ if not expansions:
+ return candidates, []
+
+ selected_neighbors = {neighbor for neighbor, _ in expansions}
+ neighbor_uuids = sorted(selected_neighbors)
+ with profile_stage(
+ "graph.bridge_neighbors", neighbors=len(neighbor_uuids)
+ ) as detail:
+ neighbors = graph_adapter.get_event_hub_facts(
+ neighbor_uuids, brain_id=brain_id
+ )
+ detail["rows"] = len(neighbors)
+
+ neighbor_chunk_ids: list[str] = []
+ facts_by_hub: dict[str, list[tuple[Any, Any, Any, Any, Any]]] = {}
+ for n, r, m, r2, b in neighbors:
+ event_uuid = str(getattr(m, "uuid", "") or "")
+ if not event_uuid:
+ continue
+ facts_by_hub.setdefault(event_uuid, []).append((n, r, m, r2, b))
+ neighbor_chunk_ids.extend(_extract_source_chunk_ids(n, r, m, r2, b))
+
+ chunk_sessions = _resolve_chunk_sessions(
+ seed_chunk_ids + neighbor_chunk_ids, brain_id
+ )
+ if not seed_sessions:
+ seed_sessions.update(_sessions_from_chunk_map(seed_chunk_ids, chunk_sessions))
+
+ hub_sessions: dict[str, set[str]] = {}
+ for event_uuid, rows in facts_by_hub.items():
+ sessions: set[str] = set()
+ for n, r, m, r2, b in rows:
+ chunk_ids = _extract_source_chunk_ids(n, r, m, r2, b)
+ sessions.update(_sessions_from_chunk_map(chunk_ids, chunk_sessions))
+ hub_sessions[event_uuid] = sessions
+
+ bridge_by_neighbor = {neighbor: bridge for neighbor, bridge in expansions}
+ seed_by_neighbor: dict[str, str] = {}
+ for neighbor, bridge in expansions:
+ if bridge.event_a == neighbor:
+ seed_by_neighbor[neighbor] = bridge.event_b
+ else:
+ seed_by_neighbor[neighbor] = bridge.event_a
+
+ path_meta: list[dict[str, Any]] = []
+ for neighbor, bridge in expansions:
+ seed = seed_by_neighbor.get(neighbor, "")
+ path_meta.append(
+ {
+ "hubs": sorted([seed, neighbor]),
+ "shared_entity": bridge.shared_entity,
+ "shared_entity_name": bridge.shared_entity_name,
+ "weight": bridge.weight,
+ }
+ )
+ path_meta.sort(
+ key=lambda p: (
+ tuple(p["hubs"]),
+ p["shared_entity"],
+ )
+ )
+
+ existing_keys = {
+ candidate.get("key")
+ for candidate in candidates
+ if candidate.get("key") is not None
+ }
+ expanded = list(candidates)
+ for event_uuid in sorted(selected_neighbors):
+ bridge = bridge_by_neighbor.get(event_uuid)
+ if bridge is None:
+ continue
+ seed = seed_by_neighbor.get(event_uuid, "")
+ base = hub_best_score.get(seed, float("inf"))
+ novel = hub_sessions.get(event_uuid, set()) - seed_sessions
+ score = (
+ base + _BRIDGE_SCORE_PENALTY
+ if base != float("inf")
+ else float("inf")
+ )
+ if novel and score != float("inf"):
+ score = max(0.0, score - min(0.05, 0.02 * len(novel)))
+ reserve_ok = seed in reserve_hubs
+ for n, r, m, r2, b in facts_by_hub.get(event_uuid, []):
+ if not _is_currently_valid(r) or not _is_currently_valid(r2):
+ continue
+ key = _fact_key(r, r2)
+ if key in existing_keys:
+ continue
+ existing_keys.add(key)
+ chunk_ids = _extract_source_chunk_ids(n, r, m, r2, b)
+ session_ids = _sessions_from_chunk_map(chunk_ids, chunk_sessions)
+ expanded.append(
+ {
+ "identified_entity": f"bridge:{bridge.shared_entity}",
+ "triple": (n, r, m, r2, b),
+ "score": score,
+ "key": key,
+ "text": _format_event_fact(n, r, m, r2, b),
+ "chunk_ids": chunk_ids,
+ "session_ids": session_ids,
+ "bridge": {
+ "from_hub": seed,
+ "to_hub": event_uuid,
+ "shared_entity": bridge.shared_entity,
+ "shared_entity_name": bridge.shared_entity_name,
+ "reserve_ok": reserve_ok,
+ },
+ }
+ )
+ return expanded, path_meta
async def retrieve_data(
@@ -401,13 +731,510 @@ def _format_kg_item(item: Any) -> str:
def _format_event_fact(n: Node, r: Predicate, m: Node, r2: Predicate, b: Node) -> str:
- return " | ".join(_format_kg_item(item) for item in (n, r, m, r2, b))
+ r2_name = (getattr(r2, "name", None) or "").upper()
+ end_label = (
+ "Context"
+ if any(token in r2_name for token in _CONTEXT_REL)
+ else "Target"
+ )
+ return (
+ f"Actor: {_format_kg_item(n)} | {_format_kg_item(r)} | "
+ f"Event: {_format_kg_item(m)} | {_format_kg_item(r2)} | "
+ f"{end_label}: {_format_kg_item(b)}"
+ )
def _fact_key(r: Predicate, r2: Predicate) -> tuple[str, str]:
return (getattr(r, "uuid", None) or "", getattr(r2, "uuid", None) or "")
+def _rel_name(predicate: Predicate | None) -> str:
+ return (getattr(predicate, "name", None) or "").upper()
+
+
+def _is_spine_actor_rel(name: str) -> bool:
+ return any(token in name for token in _SPINE_ACTOR)
+
+
+def _is_spine_target_rel(name: str) -> bool:
+ return any(token in name for token in _SPINE_TARGET)
+
+
+def _is_context_rel(name: str) -> bool:
+ return any(token in name for token in _CONTEXT_REL)
+
+
+def _event_hub_id(candidate: dict[str, Any]) -> str:
+ triple = candidate.get("triple") or ()
+ r = triple[1] if len(triple) > 1 else None
+ m = triple[2] if len(triple) > 2 else None
+ r2 = triple[3] if len(triple) > 3 else None
+ event_uuid = getattr(m, "uuid", None) if m is not None else None
+ if event_uuid:
+ return str(event_uuid)
+ flow_key = (
+ getattr(r, "flow_key", None)
+ or getattr(r2, "flow_key", None)
+ or ""
+ )
+ if flow_key:
+ return str(flow_key)
+ key = candidate.get("key")
+ if isinstance(key, tuple):
+ return "|".join(str(part or "") for part in key)
+ return str(candidate.get("text") or "")
+
+
+def _candidate_session_ids(candidate: dict[str, Any]) -> list[str]:
+ sessions = candidate.get("session_ids")
+ if isinstance(sessions, list) and sessions:
+ return [str(s) for s in sessions if s]
+ return []
+
+
+def _hub_leg_kind(candidate: dict[str, Any]) -> str:
+ triple = candidate.get("triple") or ()
+ r2 = triple[3] if len(triple) > 3 else None
+ name = _rel_name(r2)
+ if _is_context_rel(name):
+ return "context"
+ if _is_spine_target_rel(name):
+ return "spine"
+ return "other"
+
+
+def _hub_completeness_score(candidate: dict[str, Any]) -> float:
+ triple = candidate.get("triple") or ()
+ if len(triple) < 5:
+ return 0.0
+ n, r, m, r2, b = triple
+ score = 0.0
+ m_labels = {str(label).upper() for label in (getattr(m, "labels", None) or [])}
+ if "EVENT" in m_labels:
+ score += 1.0
+ r_name = _rel_name(r)
+ r2_name = _rel_name(r2)
+ if _is_spine_actor_rel(r_name):
+ score += 1.0
+ elif r_name:
+ score += 0.2
+ if _is_spine_target_rel(r2_name):
+ score += 1.0
+ elif _is_context_rel(r2_name):
+ score += 0.35
+ elif r2_name:
+ score += 0.15
+ if getattr(n, "uuid", None) and getattr(b, "uuid", None):
+ score += 0.5
+ if getattr(m, "happened_at", None):
+ score += 0.15
+ return score
+
+
+def _fact_recency_score(candidate: dict[str, Any]) -> float:
+ triple = candidate.get("triple") or ()
+ best = 0.0
+ now = datetime.now(tz=None)
+ for node in triple[0::2] if triple else ():
+ raw = getattr(node, "happened_at", None)
+ if not raw:
+ continue
+ parsed = parse_date_string(raw) if isinstance(raw, str) else raw
+ if not isinstance(parsed, datetime):
+ continue
+ days_ago = max(0, (now - to_naive_utc(parsed)).days)
+ score = 1.0 if days_ago <= 0 else 1.0 / (1.0 + days_ago)
+ if score > best:
+ best = score
+ return best
+
+
+def _relevance_score(candidate: dict[str, Any]) -> float:
+ distance = _candidate_distance(candidate)
+ distance_term = 1.0 / (1.0 + max(0.0, distance))
+ ppr_term = float(candidate.get("ppr_mass") or 0.0)
+ completeness = _hub_completeness_score(candidate) / 3.8
+ recency = _fact_recency_score(candidate)
+ topic_boost = 0.08 if candidate.get("topic_preferred") else 0.0
+ return (
+ 0.45 * distance_term
+ + 0.25 * ppr_term
+ + 0.20 * completeness
+ + 0.10 * recency
+ + topic_boost
+ )
+
+
+def _completeness_rank_key(candidate: dict[str, Any]) -> tuple:
+ return (
+ -_relevance_score(candidate),
+ _candidate_distance(candidate),
+ ) + _candidate_tiebreak_key(candidate)
+
+
+def _rank_facts_with_completeness(
+ candidates: list[dict[str, Any]],
+) -> list[dict[str, Any]]:
+ return sorted(candidates, key=_completeness_rank_key)
+
+
+def _is_complementary_hub_leg(
+ selected_meta: list[dict[str, Any]], item: dict[str, Any]
+) -> bool:
+ if item["kind"] not in {"spine", "context"}:
+ return False
+ for prev in selected_meta:
+ if prev["hub"] != item["hub"]:
+ continue
+ if {item["kind"], prev["kind"]} == {"spine", "context"}:
+ return True
+ return False
+
+
+def _bridge_reserved_slots(max_facts: int) -> int:
+ if max_facts <= 0:
+ return 0
+ return min(
+ max_facts,
+ max(_BRIDGE_RESERVE_MIN, max_facts // _BRIDGE_RESERVE_FRAC_DENOM),
+ )
+
+
+def _is_bridge_candidate(candidate: dict[str, Any]) -> bool:
+ return bool(candidate.get("bridge"))
+
+
+def _bridge_reserve_ok(candidate: dict[str, Any]) -> bool:
+ bridge = candidate.get("bridge")
+ if not bridge:
+ return False
+ if isinstance(bridge, dict) and "reserve_ok" in bridge:
+ return bool(bridge.get("reserve_ok"))
+ return True
+
+
+def _prepare_diversify_item(candidate: dict[str, Any]) -> dict[str, Any]:
+ key = candidate.get("key") or ("", "")
+ is_bridge = _is_bridge_candidate(candidate)
+ return {
+ "candidate": candidate,
+ "relevance": _relevance_score(candidate),
+ "hub": _event_hub_id(candidate),
+ "sessions": frozenset(_candidate_session_ids(candidate)),
+ "kind": _hub_leg_kind(candidate),
+ "tie": (str(key[0]) if key else "", str(key[1]) if len(key) > 1 else ""),
+ "is_bridge": is_bridge,
+ "reserve_ok": _bridge_reserve_ok(candidate) if is_bridge else False,
+ }
+
+
+def _mmr_pick_index(
+ remaining: list[dict[str, Any]],
+ selected_meta: list[dict[str, Any]],
+ selected_hubs: set[str],
+ selected_sessions: set[str],
+) -> int:
+ best_idx = 0
+ best_mmr = None
+ best_tie = None
+ for idx, item in enumerate(remaining):
+ relevance = item["relevance"]
+ hub = item["hub"]
+ sessions = item["sessions"]
+ complementary = _is_complementary_hub_leg(selected_meta, item)
+ if complementary:
+ hub_pen = 0.0
+ sess_pen = 0.0
+ relevance = relevance + 0.5
+ else:
+ hub_pen = 1.0 if hub in selected_hubs else 0.0
+ if sessions and selected_sessions:
+ sess_pen = len(sessions & selected_sessions) / max(
+ 1, len(sessions)
+ )
+ else:
+ sess_pen = 0.0
+ redundancy = 0.55 * hub_pen + 0.45 * sess_pen
+ mmr = _DIVERSIFY_LAMBDA * relevance - (1.0 - _DIVERSIFY_LAMBDA) * redundancy
+ tie = item["tie"]
+ if (
+ best_mmr is None
+ or mmr > best_mmr
+ or (mmr == best_mmr and (best_tie is None or tie < best_tie))
+ ):
+ best_mmr = mmr
+ best_tie = tie
+ best_idx = idx
+ return best_idx
+
+
+def _mmr_take(
+ remaining: list[dict[str, Any]],
+ *,
+ budget: int,
+ selected: list[dict[str, Any]],
+ selected_meta: list[dict[str, Any]],
+ selected_hubs: set[str],
+ selected_sessions: set[str],
+) -> None:
+ while budget > 0 and remaining:
+ best_idx = _mmr_pick_index(
+ remaining, selected_meta, selected_hubs, selected_sessions
+ )
+ chosen = remaining.pop(best_idx)
+ selected.append(chosen["candidate"])
+ selected_meta.append(chosen)
+ selected_hubs.add(chosen["hub"])
+ selected_sessions.update(chosen["sessions"])
+ budget -= 1
+
+
+def _take_novel_bridge_slots(
+ remaining: list[dict[str, Any]],
+ *,
+ budget: int,
+ covered_sessions: set[str],
+ selected: list[dict[str, Any]],
+ selected_meta: list[dict[str, Any]],
+ selected_hubs: set[str],
+ selected_sessions: set[str],
+) -> None:
+ while budget > 0 and remaining:
+ novel_sessions: set[str] = set()
+ for item in remaining:
+ if not item["is_bridge"] or not item["reserve_ok"]:
+ continue
+ novel_sessions.update(item["sessions"] - covered_sessions)
+ if not novel_sessions:
+ break
+ target = min(novel_sessions)
+ best_idx = None
+ best_key = None
+ for idx, item in enumerate(remaining):
+ if not item["is_bridge"] or not item["reserve_ok"]:
+ continue
+ if target not in item["sessions"]:
+ continue
+ key = (item["hub"], item["tie"])
+ if best_key is None or key < best_key:
+ best_key = key
+ best_idx = idx
+ if best_idx is None:
+ break
+ chosen = remaining.pop(best_idx)
+ selected.append(chosen["candidate"])
+ selected_meta.append(chosen)
+ selected_hubs.add(chosen["hub"])
+ selected_sessions.update(chosen["sessions"])
+ covered_sessions.update(chosen["sessions"])
+ budget -= 1
+
+
+def _diversify_facts(
+ ranked: list[dict[str, Any]],
+ *,
+ max_facts: int,
+) -> list[dict[str, Any]]:
+ if max_facts <= 0 or not ranked:
+ return []
+ if len(ranked) <= max_facts:
+ return list(ranked)
+
+ prepared = [_prepare_diversify_item(candidate) for candidate in ranked]
+ prepared.sort(key=lambda item: (-item["relevance"], item["tie"]))
+
+ selected: list[dict[str, Any]] = []
+ selected_meta: list[dict[str, Any]] = []
+ selected_hubs: set[str] = set()
+ selected_sessions: set[str] = set()
+
+ reserve = _bridge_reserved_slots(max_facts)
+ has_bridge = any(item["is_bridge"] and item["reserve_ok"] for item in prepared)
+ if not has_bridge:
+ reserve = 0
+ main_budget = max_facts - reserve
+
+ non_bridge = [item for item in prepared if not item["is_bridge"]]
+ bridge_items = [item for item in prepared if item["is_bridge"]]
+ stable_bridges = [item for item in bridge_items if item["reserve_ok"]]
+
+ _mmr_take(
+ non_bridge,
+ budget=main_budget,
+ selected=selected,
+ selected_meta=selected_meta,
+ selected_hubs=selected_hubs,
+ selected_sessions=selected_sessions,
+ )
+
+ covered_by_non_bridge = set(selected_sessions)
+ _take_novel_bridge_slots(
+ stable_bridges,
+ budget=reserve,
+ covered_sessions=covered_by_non_bridge,
+ selected=selected,
+ selected_meta=selected_meta,
+ selected_hubs=selected_hubs,
+ selected_sessions=selected_sessions,
+ )
+
+ remaining = non_bridge + stable_bridges
+ remaining.sort(key=lambda item: (-item["relevance"], item["tie"]))
+ leftover = max_facts - len(selected)
+ _mmr_take(
+ remaining,
+ budget=leftover,
+ selected=selected,
+ selected_meta=selected_meta,
+ selected_hubs=selected_hubs,
+ selected_sessions=selected_sessions,
+ )
+ return selected
+
+
+def _temporal_conflict_meta(
+ candidates: list[dict[str, Any]],
+) -> list[dict[str, Any]]:
+ by_entity: dict[str, list[dict[str, Any]]] = defaultdict(list)
+ for candidate in candidates:
+ triple = candidate.get("triple") or ()
+ for node in (triple[0], triple[4]) if len(triple) >= 5 else ():
+ uuid = getattr(node, "uuid", None)
+ if not uuid:
+ continue
+ happened = None
+ event = triple[2] if len(triple) > 2 else None
+ raw = getattr(event, "happened_at", None) if event is not None else None
+ if isinstance(raw, str) and raw.strip():
+ happened = raw.strip()
+ by_entity[str(uuid)].append(
+ {
+ "fact_key": list(candidate.get("key") or ()),
+ "hub_id": _event_hub_id(candidate),
+ "happened_at": happened,
+ }
+ )
+ conflicts: list[dict[str, Any]] = []
+ for entity_uuid, entries in sorted(by_entity.items()):
+ times = {
+ entry["happened_at"]
+ for entry in entries
+ if entry.get("happened_at")
+ }
+ hubs = {entry["hub_id"] for entry in entries}
+ if len(times) >= 2 and len(hubs) >= 2:
+ conflicts.append(
+ {
+ "entity_uuid": entity_uuid,
+ "hubs": sorted(hubs),
+ "happened_ats": sorted(times),
+ "facts": [entry["fact_key"] for entry in entries],
+ }
+ )
+ return conflicts
+
+
+def _edge_type_weight(predicate: Predicate | None, query: str | None = None) -> float:
+ name = _rel_name(predicate)
+ if _is_spine_actor_rel(name) or _is_spine_target_rel(name):
+ weight = 1.0
+ elif _is_context_rel(name):
+ weight = 0.35
+ else:
+ weight = 0.55
+ if query:
+ q_tokens = set(re.findall(r"[a-z0-9]+", query.lower()))
+ p_tokens = set(re.findall(r"[a-z0-9]+", name.lower()))
+ if q_tokens and p_tokens:
+ overlap = len(q_tokens & p_tokens) / max(1, len(p_tokens))
+ weight *= 1.0 + 0.5 * overlap
+ return weight
+
+
+def _add_weighted_edge(
+ adjacency: dict[str, dict[str, float]],
+ src: str | None,
+ dst: str | None,
+ weight: float,
+) -> None:
+ if not src or not dst or src == dst or weight <= 0:
+ return
+ bucket = adjacency.setdefault(str(src), {})
+ prev = bucket.get(str(dst), 0.0)
+ if weight > prev:
+ bucket[str(dst)] = weight
+
+
+def _candidate_tiebreak_key(candidate: dict[str, Any]) -> tuple[str, ...]:
+ triple = candidate.get("triple") or ()
+ n = triple[0] if len(triple) > 0 else None
+ r = triple[1] if len(triple) > 1 else None
+ m = triple[2] if len(triple) > 2 else None
+ r2 = triple[3] if len(triple) > 3 else None
+ b = triple[4] if len(triple) > 4 else None
+ flow_key = (
+ getattr(r, "flow_key", None)
+ or getattr(r2, "flow_key", None)
+ or ""
+ )
+ key = candidate.get("key")
+ if not (isinstance(key, tuple) and len(key) == 2):
+ key = _fact_key(r, r2) if r is not None and r2 is not None else ("", "")
+ r_uuid, r2_uuid = key
+ return (
+ str(flow_key or ""),
+ str(r_uuid or ""),
+ str(r2_uuid or ""),
+ str(getattr(n, "uuid", None) or ""),
+ str(getattr(m, "uuid", None) or ""),
+ str(getattr(b, "uuid", None) or ""),
+ str(candidate.get("identified_entity") or ""),
+ str(candidate.get("text") or ""),
+ )
+
+
+def _distance_rank_key(candidate: dict[str, Any]) -> tuple:
+ return (_candidate_distance(candidate),) + _candidate_tiebreak_key(
+ candidate
+ )
+
+
+def _ppr_rank_key(
+ candidate: dict[str, Any], ppr_scores: dict[str, float]
+) -> tuple:
+ return (_score_candidate_with_ppr(candidate, ppr_scores),) + _candidate_tiebreak_key(
+ candidate
+ )
+
+
+def _dedupe_candidates(candidates: list[dict[str, Any]]) -> list[dict[str, Any]]:
+ deduped: dict[tuple[str, str], dict[str, Any]] = {}
+ for candidate in candidates:
+ key = candidate["key"]
+ existing = deduped.get(key)
+ if existing is None:
+ deduped[key] = candidate
+ continue
+ if candidate["score"] < existing["score"]:
+ deduped[key] = candidate
+ continue
+ if candidate["score"] == existing["score"] and _candidate_tiebreak_key(
+ candidate
+ ) < _candidate_tiebreak_key(existing):
+ deduped[key] = candidate
+ return sorted(deduped.values(), key=_distance_rank_key)
+
+
+def _merge_variant_candidate_lists(
+ variant_lists: list[list[dict[str, Any]]],
+) -> list[dict[str, Any]]:
+ merged: list[dict[str, Any]] = []
+ for local in variant_lists:
+ ordered = sorted(local, key=_distance_rank_key)
+ merged.extend(ordered)
+ return merged
+
+
def _is_currently_valid(predicate: Predicate) -> bool:
props = getattr(predicate, "properties", None) or {}
if props.get("invalid_at"):
@@ -446,47 +1273,242 @@ def _collect_query_variants(text: str, elements) -> list[str]:
_add(chunk if isinstance(chunk, str) else None)
if len(variants) > before:
added_chunks += 1
+ for extra in _decompose_enumeration_queries(text, elements):
+ _add(extra)
return variants
+_ENUMERATION_RE = re.compile(
+ r"\b(what|which|list|ways?|types?|kinds?|events?|books?|items?|"
+ r"changes?|traits?|hobbies|activities)\b",
+ re.IGNORECASE,
+)
+
+
+def _is_enumeration_question(text: str) -> bool:
+ return bool(_ENUMERATION_RE.search(text or ""))
+
+
+def _decompose_enumeration_queries(text: str, elements) -> list[str]:
+ if not _is_enumeration_question(text):
+ return []
+ queries: list[str] = []
+ entities: list[str] = []
+ for token in getattr(elements, "tokens", None) or []:
+ if not isinstance(token, dict):
+ continue
+ label = str(token.get("label") or token.get("ent_type") or "").upper()
+ value = str(token.get("text") or "").strip()
+ if not value:
+ continue
+ if label in {"PERSON", "PER", "ORG", "GPE"} or (
+ value[:1].isupper() and " " not in value
+ ):
+ entities.append(value)
+ relation_terms = [
+ m.group(0).lower()
+ for m in re.finditer(
+ r"\b(books?|read|ways?|types?|kinds?|events?|hobbies|"
+ r"activities|changes?|traits?|friends?|places?)\b",
+ text or "",
+ re.IGNORECASE,
+ )
+ ]
+ seen: set[str] = set()
+ for entity in entities[:4]:
+ for relation in relation_terms[:4]:
+ q = f"{entity} {relation}"
+ key = q.lower()
+ if key not in seen and key != (text or "").strip().lower():
+ seen.add(key)
+ queries.append(q)
+ key = entity.lower()
+ if key not in seen:
+ seen.add(key)
+ queries.append(entity)
+ for relation in relation_terms[:3]:
+ if relation not in seen:
+ seen.add(relation)
+ queries.append(relation)
+ return queries[:8]
+
+
+def _seed_personalization(seeds: list[tuple[str, float, str]]) -> dict[str, float]:
+ weights: dict[str, float] = {}
+ for uuid, distance, _ in seeds:
+ weight = 1.0 / (1.0 + max(0.0, _quantize_distance(distance)))
+ prev = weights.get(uuid)
+ if prev is None or weight > prev:
+ weights[uuid] = weight
+ return weights
+
+
+def _build_adjacency_from_seeds(
+ seed_uuids: list[str],
+ brain_id: str,
+ *,
+ query: str | None = None,
+ query_embedding: list[float] | None = None,
+) -> dict[str, list[tuple[str, float]]]:
+ weighted: dict[str, dict[str, float]] = {}
+ if not seed_uuids:
+ return {}
+ try:
+ neighbors = graph_adapter.get_event_centric_neighbors(
+ seed_uuids, brain_id=brain_id
+ )
+ except Exception:
+ return {}
+ for n, r, m, r2, b in neighbors:
+ if not _is_currently_valid(r) or not _is_currently_valid(r2):
+ continue
+ n_uuid = getattr(n, "uuid", None)
+ m_uuid = getattr(m, "uuid", None)
+ b_uuid = getattr(b, "uuid", None)
+ w1 = _edge_type_weight(r, query=query)
+ w2 = _edge_type_weight(r2, query=query)
+ if query_embedding is not None:
+ for pred, base in ((r, w1), (r2, w2)):
+ emb = getattr(pred, "embeddings", None) or getattr(
+ pred, "embedding", None
+ )
+ if emb is None:
+ continue
+ try:
+ sim = float(cosine_similarity(query_embedding, emb))
+ boost = 0.5 + 0.5 * max(0.0, sim)
+ if pred is r:
+ w1 *= boost
+ else:
+ w2 *= boost
+ except Exception:
+ pass
+ _add_weighted_edge(weighted, n_uuid, m_uuid, w1)
+ _add_weighted_edge(weighted, m_uuid, n_uuid, w1)
+ _add_weighted_edge(weighted, m_uuid, b_uuid, w2)
+ _add_weighted_edge(weighted, b_uuid, m_uuid, w2)
+ if n_uuid and b_uuid:
+ bridge = min(w1, w2) * 0.5
+ _add_weighted_edge(weighted, n_uuid, b_uuid, bridge)
+ _add_weighted_edge(weighted, b_uuid, n_uuid, bridge)
+ return {
+ src: sorted(((dst, weight) for dst, weight in edges.items()), key=lambda x: x[0])
+ for src, edges in sorted(weighted.items(), key=lambda item: item[0])
+ }
+
+
+def _score_candidate_with_ppr(
+ candidate: dict[str, Any], ppr_scores: dict[str, float]
+) -> float:
+ n, _r, m, _r2, b = candidate["triple"]
+ scores = [
+ ppr_scores.get(str(getattr(node, "uuid", "")), 0.0)
+ for node in (n, m, b)
+ if getattr(node, "uuid", None)
+ ]
+ if not scores:
+ return _candidate_distance(candidate)
+ return -max(scores)
+
+
+def _context_looks_insufficient(
+ question: str, text_lines: list[str], passages: list[str]
+) -> bool:
+ blob = "\n".join(text_lines + passages).lower()
+ if len(blob.strip()) < 80:
+ return True
+ tokens = [
+ t
+ for t in re.findall(r"[a-z0-9']+", (question or "").lower())
+ if len(t) > 3
+ and t
+ not in {
+ "what",
+ "which",
+ "when",
+ "where",
+ "who",
+ "does",
+ "did",
+ "have",
+ "has",
+ "been",
+ "with",
+ "from",
+ "about",
+ "would",
+ "could",
+ "their",
+ "there",
+ "this",
+ "that",
+ "mentioned",
+ }
+ ]
+ if not tokens:
+ return False
+ hits = sum(1 for token in tokens if token in blob)
+ return hits / len(tokens) < 0.35
+
+
+def _stabilize_seed_hits(
+ seeds: list[tuple[str, float, str]],
+) -> list[tuple[str, float, str]]:
+ best: dict[str, tuple[float, str]] = {}
+ for uuid, distance, name in seeds:
+ uid = str(uuid or "").strip()
+ if not uid:
+ continue
+ dist = _quantize_distance(distance)
+ prev = best.get(uid)
+ if prev is None or dist < prev[0]:
+ best[uid] = (dist, str(name))
+ return sorted(
+ ((uid, dist, name) for uid, (dist, name) in best.items()),
+ key=lambda item: (item[1], item[0]),
+ )
+
+
def _seed_nodes_for_text(text: str, brain_id: str) -> list[tuple[str, float, str]]:
"""Return (node_uuid, distance, entity_name) seeds from nodes + relationships."""
- text_embeddings = embeddings_adapter.embed_text(text)
+ with profile_stage("embed.query"):
+ text_embeddings = embeddings_adapter.embed_text(text)
embeddings = text_embeddings.embeddings
seeds: list[tuple[str, float, str]] = []
- node_vectors = vector_search.search_nodes(
- embeddings,
- brain_id=brain_id,
- k=_SEED_K,
- )
+ with profile_stage("vector.search_nodes", k=_SEED_K):
+ node_vectors = vector_search.search_nodes(
+ embeddings,
+ brain_id=brain_id,
+ k=_SEED_K,
+ )
for vector in node_vectors:
meta = vector.metadata or {}
- uuid = meta.get("uuid")
+ uuid = meta.get("uuid") or getattr(vector, "id", None)
if not uuid:
continue
distance = (
vector.distance if vector.distance is not None else float("inf")
)
name = meta.get("name") or text
- seeds.append((uuid, float(distance), str(name)))
+ seeds.append((str(uuid), float(distance), str(name)))
- rel_vectors = vector_search.search_relationships(
- embeddings,
- brain_id=brain_id,
- k=_SEED_K,
- )
+ with profile_stage("vector.search_relationships", k=_SEED_K):
+ rel_vectors = vector_search.search_relationships(
+ embeddings,
+ brain_id=brain_id,
+ k=_SEED_K,
+ )
for vector in rel_vectors:
meta = vector.metadata or {}
distance = (
vector.distance if vector.distance is not None else float("inf")
)
- node_ids = meta.get("node_ids") or []
+ node_ids = sorted(str(n) for n in (meta.get("node_ids") or []) if n)
for node_id in node_ids:
- if node_id:
- seeds.append((str(node_id), float(distance), text))
+ seeds.append((node_id, float(distance), text))
- return seeds
+ return _stabilize_seed_hits(seeds)
def _extract_source_chunk_ids(*items: Any) -> list[str]:
@@ -508,299 +1530,572 @@ def _extract_source_chunk_ids(*items: Any) -> list[str]:
return ids
+def _session_ids_from_text(text: str) -> list[str]:
+ found: list[str] = []
+ seen: set[str] = set()
+ for match in _SESSION_RE.finditer(text or ""):
+ sid = f"session_{match.group(1)}"
+ if sid not in seen:
+ seen.add(sid)
+ found.append(sid)
+ return found
+
+
+def _session_ids_for_chunks(
+ chunk_ids: list[str], chunk_sessions: dict[str, list[str]]
+) -> list[str]:
+ found: list[str] = []
+ seen: set[str] = set()
+ for chunk_id in chunk_ids:
+ for sid in chunk_sessions.get(chunk_id) or []:
+ if sid not in seen:
+ seen.add(sid)
+ found.append(sid)
+ return found
+
+
+def _build_fact_channel(
+ curated: list[dict[str, Any]],
+ chunk_sessions: dict[str, list[str]],
+) -> tuple[list[str], list[GetContextTriple], list[str]]:
+ text_lines: list[str] = []
+ triples: list[GetContextTriple] = []
+ graph_session_ids: list[str] = []
+ seen_sessions: set[str] = set()
+ for candidate in curated:
+ n, r, m, r2, b = candidate["triple"]
+ chunk_ids = list(candidate.get("chunk_ids") or [])
+ session_ids = _candidate_session_ids(candidate)
+ if not session_ids:
+ session_ids = _session_ids_for_chunks(chunk_ids, chunk_sessions)
+ line = candidate["text"]
+ if session_ids:
+ line = f"{line} ({', '.join(session_ids)})"
+ text_lines.append(line)
+ triples.append(
+ GetContextTriple(
+ identified_entity=candidate["identified_entity"],
+ triple=(n, r, m, r2, b),
+ source_chunk_ids=chunk_ids or None,
+ source_session_ids=session_ids or None,
+ )
+ )
+ for sid in session_ids:
+ if sid not in seen_sessions:
+ seen_sessions.add(sid)
+ graph_session_ids.append(sid)
+ graph_session_ids.sort()
+ return text_lines, triples, graph_session_ids
+
+
def _retrieve_passages(
text: str, brain_id: str, *, limit: int
) -> list[tuple[str, float, str]]:
"""Return (chunk_id, score, text) ranked passages via vector + keyword fusion."""
- text_embeddings = embeddings_adapter.embed_text(text)
- vector_hits = vector_search.search_data(
- text_embeddings.embeddings,
- brain_id=brain_id,
- k=max(limit, _PASSAGE_K),
- )
- vector_ids: list[str] = []
- id_to_text: dict[str, str] = {}
- id_to_distance: dict[str, float] = {}
- for vector in vector_hits:
- meta = vector.metadata or {}
- resource_id = meta.get("resource_id") or getattr(vector, "id", None)
- if not resource_id:
- continue
- resource_id = str(resource_id)
- vector_ids.append(resource_id)
- id_to_distance[resource_id] = (
- float(vector.distance)
- if vector.distance is not None
- else float("inf")
- )
-
- keyword_ids: list[str] = []
- try:
- search_result = data_adapter.search(text, brain_id)
- for chunk in getattr(search_result, "text_chunks", None) or []:
- chunk_id = str(getattr(chunk, "id", "") or "")
- if not chunk_id:
+ with profile_stage("passages.retrieve", blocking=True, queries=1):
+ with profile_stage("embed.query"):
+ text_embeddings = embeddings_adapter.embed_text(text)
+ with profile_stage("vector.search_data", k=max(limit, _PASSAGE_K)):
+ vector_hits = vector_search.search_data(
+ text_embeddings.embeddings,
+ brain_id=brain_id,
+ k=max(limit, _PASSAGE_K),
+ )
+ vector_ids: list[str] = []
+ id_to_text: dict[str, str] = {}
+ id_to_distance: dict[str, float] = {}
+ for vector in vector_hits:
+ meta = vector.metadata or {}
+ resource_id = meta.get("resource_id") or getattr(vector, "id", None)
+ if not resource_id:
continue
- keyword_ids.append(chunk_id)
- id_to_text[chunk_id] = getattr(chunk, "text", "") or ""
- except Exception:
- pass
-
- fused = reciprocal_rank_fusion([vector_ids, keyword_ids])
- ranked_ids = [item for item, _ in fused[:limit]]
- missing = [cid for cid in ranked_ids if cid not in id_to_text]
- if missing:
- try:
- chunks, _ = data_adapter.get_text_chunks_by_ids(
- missing, False, brain_id
+ resource_id = str(resource_id)
+ vector_ids.append(resource_id)
+ id_to_distance[resource_id] = (
+ float(vector.distance)
+ if vector.distance is not None
+ else float("inf")
)
- for chunk in chunks:
- id_to_text[str(chunk.id)] = chunk.text or ""
- except Exception:
- pass
-
- passages: list[tuple[str, float, str]] = []
- for chunk_id in ranked_ids:
- body = (id_to_text.get(chunk_id) or "").strip()
- if not body:
- continue
- score = id_to_distance.get(chunk_id, 1.0 / (1.0 + len(passages)))
- passages.append((chunk_id, float(score), body))
- return passages
+ keyword_ids: list[str] = []
+ with profile_stage("data.keyword_search"):
+ try:
+ search_result = data_adapter.search(text, brain_id)
+ for chunk in getattr(search_result, "text_chunks", None) or []:
+ chunk_id = str(getattr(chunk, "id", "") or "")
+ if not chunk_id:
+ continue
+ keyword_ids.append(chunk_id)
+ id_to_text[chunk_id] = getattr(chunk, "text", "") or ""
+ except Exception:
+ pass
+
+ fused = reciprocal_rank_fusion([vector_ids, keyword_ids])
+ ranked_ids = [item for item, _ in fused[:limit]]
+ missing = [cid for cid in ranked_ids if cid not in id_to_text]
+ if missing:
+ with profile_stage("data.fetch_chunks", chunks=len(missing)):
+ try:
+ chunks, _ = data_adapter.get_text_chunks_by_ids(
+ missing, False, brain_id
+ )
+ for chunk in chunks:
+ id_to_text[str(chunk.id)] = chunk.text or ""
+ except Exception:
+ pass
-def _flatten_match_path(path: MatchPath) -> list[str]:
- lines: list[str] = []
- current: MatchPath | None = path
- while current is not None:
- predicate, node = current.path
- if node is None and not getattr(predicate, "name", None):
- break
- parts = []
- if predicate and getattr(predicate, "name", None):
- parts.append(_format_kg_item(predicate))
- if node is not None:
- parts.append(_format_kg_item(node))
- if parts:
- lines.append(" -> ".join(parts))
- if current.children:
- current = current.children[0]
- else:
- current = None
- return lines
+ passages: list[tuple[str, float, str]] = []
+ for chunk_id in ranked_ids:
+ body = (id_to_text.get(chunk_id) or "").strip()
+ if not body:
+ continue
+ score = id_to_distance.get(chunk_id, 1.0 / (1.0 + len(passages)))
+ passages.append((chunk_id, float(score), body))
+ return passages
-async def get_context(request: GetContextRequestBody) -> GetContextResponse:
+async def get_context(
+ request: GetContextRequestBody,
+ *,
+ fact_filter_adapter: Any = None,
+) -> GetContextResponse:
"""
Retrieve contextual information for a text.
- """
- elements = _entity_extractor.extract_elements(request.text)
- variants = _collect_query_variants(request.text, elements)
+ `fact_filter_adapter` is only supplied by deep-navigation callers: the
+ one-shot context path leaves it unset because an LLM call does not fit the
+ sub-second budget, and without it no fact filtering happens.
+
+ `profile_stages` attaches a per-stage wall-clock breakdown to the response
+ and publishes it as a latency trace event; it is off unless the caller asks
+ for it or `TRACE_STAGE_PROFILER_ENABLED` is set.
+ """
+ with profile_request(
+ "retrieve.context", enabled=request.profile_stages
+ ) as profiler:
+ response = await _build_context(request, fact_filter_adapter)
+ if profiler is not None:
+ response.stage_timings = profiler.last_report
+ return response
+
+
+async def _build_context(
+ request: GetContextRequestBody,
+ fact_filter_adapter: Any,
+) -> GetContextResponse:
+ with profile_stage("nlp.extract_elements"):
+ elements = _entity_extractor.extract_elements(request.text)
+ with profile_stage("nlp.query_variants") as detail:
+ variants = _collect_query_variants(request.text, elements)
+ detail["variants"] = len(variants)
+ max_passages = max(1, request.max_passages)
historical_context: list[str] = []
source_passages: list[str] = []
- candidate_lock = threading.Lock()
candidates: list[dict[str, Any]] = []
seed_hits: list[tuple[str, float, str]] = []
passage_hits: list[tuple[str, float, str]] = []
- def _collect_facts_for_variant(text: str) -> None:
- seeds = _seed_nodes_for_text(text, request.brain_id)
- if not seeds:
- return
- with candidate_lock:
- seed_hits.extend(seeds)
- uuid_to_distance: dict[str, float] = {}
- for uuid, distance, _ in seeds:
- prev = uuid_to_distance.get(uuid)
- if prev is None or distance < prev:
- uuid_to_distance[uuid] = distance
- seed_uuids = list(uuid_to_distance.keys())
- neighbors = graph_adapter.get_event_centric_neighbors(
- seed_uuids, brain_id=request.brain_id
- )
- local: list[dict[str, Any]] = []
- for n, r, m, r2, b in neighbors:
- if not _is_currently_valid(r) or not _is_currently_valid(r2):
- continue
- distances = [
- uuid_to_distance[u]
- for u in (
- getattr(n, "uuid", None),
- getattr(m, "uuid", None),
- getattr(b, "uuid", None),
- )
- if u in uuid_to_distance
- ]
- score = min(distances) if distances else float("inf")
- local.append(
- {
- "identified_entity": text,
- "triple": (n, r, m, r2, b),
- "score": score,
- "key": _fact_key(r, r2),
- "text": _format_event_fact(n, r, m, r2, b),
- "chunk_ids": _extract_source_chunk_ids(n, r, m, r2, b),
- }
+ def _collect_facts_for_variant(
+ text: str,
+ ) -> tuple[list[tuple[str, float, str]], list[dict[str, Any]]]:
+ with profile_stage("facts.variant") as detail:
+ with profile_stage("facts.seed_search"):
+ seeds = _seed_nodes_for_text(text, request.brain_id)
+ detail["seeds"] = len(seeds)
+ if not seeds:
+ return [], []
+ uuid_to_distance: dict[str, float] = {}
+ for uuid, distance, _ in seeds:
+ dist = _quantize_distance(distance)
+ prev = uuid_to_distance.get(uuid)
+ if prev is None or dist < prev:
+ uuid_to_distance[uuid] = dist
+ seed_uuids = sorted(
+ uuid_to_distance.keys(),
+ key=lambda uid: (uuid_to_distance[uid], uid),
)
- if local:
- with candidate_lock:
- candidates.extend(local)
+ with profile_stage(
+ "graph.event_neighbors", seed_uuids=len(seed_uuids)
+ ) as neighbors_detail:
+ neighbors = graph_adapter.get_event_centric_neighbors(
+ seed_uuids, brain_id=request.brain_id
+ )
+ neighbors_detail["rows"] = len(neighbors)
+ local: list[dict[str, Any]] = []
+ with profile_stage("facts.assemble"):
+ for n, r, m, r2, b in neighbors:
+ if not _is_currently_valid(r) or not _is_currently_valid(r2):
+ continue
+ distances = [
+ uuid_to_distance[u]
+ for u in (
+ getattr(n, "uuid", None),
+ getattr(m, "uuid", None),
+ getattr(b, "uuid", None),
+ )
+ if u in uuid_to_distance
+ ]
+ score = min(distances) if distances else float("inf")
+ local.append(
+ {
+ "identified_entity": text,
+ "triple": (n, r, m, r2, b),
+ "score": score,
+ "key": _fact_key(r, r2),
+ "text": _format_event_fact(n, r, m, r2, b),
+ "chunk_ids": _extract_source_chunk_ids(n, r, m, r2, b),
+ }
+ )
+ return list(seeds), local
def _collect_passages() -> None:
nonlocal passage_hits
- passage_hits = _retrieve_passages(
- request.text,
- request.brain_id,
- limit=max(1, int(getattr(request, "max_passages", 8) or 8)),
- )
+ with profile_stage("passages.collect") as detail:
+ queries = [request.text] + [
+ v
+ for v in variants
+ if v.strip().lower() != request.text.strip().lower()
+ ]
+ if _is_enumeration_question(request.text):
+ queries = queries[:6]
+ else:
+ queries = queries[:3]
+ detail["queries"] = len(queries)
+ fused_lists: list[list[str]] = []
+ id_to_text: dict[str, str] = {}
+ id_to_score: dict[str, float] = {}
+ for query in queries:
+ hits = _retrieve_passages(
+ query,
+ request.brain_id,
+ limit=max(max_passages, _PASSAGE_K),
+ )
+ ordered_ids: list[str] = []
+ for chunk_id, score, body in hits:
+ ordered_ids.append(chunk_id)
+ id_to_text[chunk_id] = body
+ prev = id_to_score.get(chunk_id)
+ if prev is None or score < prev:
+ id_to_score[chunk_id] = score
+ if ordered_ids:
+ fused_lists.append(ordered_ids)
+ if not fused_lists:
+ passage_hits = []
+ return
+ fused = reciprocal_rank_fusion(fused_lists)
+ ranked_ids = [item for item, _ in fused[: max(max_passages, _PASSAGE_K)]]
+ passage_hits = [
+ (cid, id_to_score.get(cid, 1.0), id_to_text.get(cid, ""))
+ for cid in ranked_ids
+ if id_to_text.get(cid)
+ ]
async def _get_historical_context():
nonlocal historical_context
- passages = await asyncio.to_thread(
- _retrieve_passages,
- request.text,
- request.brain_id,
- limit=max(request.historical_limit, _PASSAGE_K),
- )
- if passages:
- historical_context = [text for _, _, text in passages[: request.historical_limit]]
+ with profile_stage("historical.context", blocking=False):
+ passages = await asyncio.to_thread(
+ _retrieve_passages,
+ request.text,
+ request.brain_id,
+ limit=max(request.historical_limit, _PASSAGE_K),
+ )
+ if passages:
+ historical_context = [
+ text for _, _, text in passages[: request.historical_limit]
+ ]
+ return historical_context
+ with profile_stage("historical.fallback", blocking=False):
+ text_chunks, structured_data = await asyncio.gather(
+ asyncio.to_thread(
+ data_adapter.get_last_text_chunks,
+ brain_id=request.brain_id,
+ limit=request.historical_limit,
+ ),
+ asyncio.to_thread(
+ data_adapter.get_last_structured_data,
+ brain_id=request.brain_id,
+ limit=request.historical_limit,
+ ),
+ )
+ historical_context = [text_chunk.text for text_chunk in text_chunks] + [
+ json.dumps(structured_data.data)
+ for structured_data in structured_data
+ if len(structured_data.data.items()) > 0
+ ]
return historical_context
- text_chunks, structured_data = await asyncio.gather(
- asyncio.to_thread(
- data_adapter.get_last_text_chunks,
- brain_id=request.brain_id,
- limit=request.historical_limit,
- ),
- asyncio.to_thread(
- data_adapter.get_last_structured_data,
- brain_id=request.brain_id,
- limit=request.historical_limit,
- ),
- )
- historical_context = [text_chunk.text for text_chunk in text_chunks] + [
- json.dumps(structured_data.data)
- for structured_data in structured_data
- if len(structured_data.data.items()) > 0
- ]
- return historical_context
- futures = [
- asyncio.to_thread(_collect_facts_for_variant, variant) for variant in variants
- ]
- futures.append(asyncio.to_thread(_collect_passages))
- futures.append(_get_historical_context())
- await asyncio.gather(*futures)
-
- deduped: dict[tuple[str, str], dict[str, Any]] = {}
- for candidate in candidates:
- key = candidate["key"]
- existing = deduped.get(key)
- if existing is None or candidate["score"] < existing["score"]:
- deduped[key] = candidate
-
- ranked = sorted(deduped.values(), key=lambda c: c["score"])
- max_facts = max(1, int(getattr(request, "max_facts", 40) or 40))
-
- if getattr(request, "apply_fact_filter", True) and ranked:
- keep = filter_relevant_facts(
- request.text,
- [c["text"] for c in ranked],
- max_keep=max_facts,
+ with profile_stage(
+ "retrieval.fanout", blocking=False, variants=len(variants)
+ ):
+ fact_futures = [
+ asyncio.to_thread(_collect_facts_for_variant, variant)
+ for variant in variants
+ ]
+ other_futures = [
+ asyncio.to_thread(_collect_passages),
+ _get_historical_context(),
+ ]
+ all_results = await asyncio.gather(*fact_futures, *other_futures)
+ variant_outputs = all_results[: len(variants)]
+
+ seed_hits = []
+ variant_lists: list[list[dict[str, Any]]] = []
+ for seeds, local in variant_outputs:
+ seed_hits.extend(seeds)
+ variant_lists.append(local)
+ candidates = _merge_variant_candidate_lists(variant_lists)
+
+ bridge_paths: list[dict[str, Any]] = []
+ with profile_stage(
+ "facts.bridge_expand",
+ bridges=request.cross_event_bridges,
+ ) as detail:
+ candidates, bridge_paths = _expand_cross_event_bridges(
+ candidates,
+ request.brain_id,
+ max_per_hub=request.cross_event_bridges,
)
- curated = [ranked[i] for i in keep if 0 <= i < len(ranked)]
- if not curated:
- curated = ranked[:max_facts]
- else:
- curated = ranked[:max_facts]
+ detail["candidates"] = len(candidates)
+ detail["paths"] = len(bridge_paths)
+
+ with profile_stage("facts.dedup_rank") as detail:
+ ranked = _dedupe_candidates(candidates)
+ detail["candidates"] = len(candidates)
+ detail["deduped"] = len(ranked)
+
+ if request.use_ppr and seed_hits and ranked:
+ with profile_stage("ppr"):
+ personalization = _seed_personalization(seed_hits)
+ with profile_stage(
+ "ppr.adjacency", seeds=len(personalization)
+ ) as adjacency_detail:
+ adjacency = _build_adjacency_from_seeds(
+ sorted(personalization.keys()),
+ request.brain_id,
+ query=request.text,
+ )
+ adjacency_detail["nodes"] = len(adjacency)
+ with profile_stage("ppr.iterations", iterations=_PPR_ITERS):
+ ppr_scores = personalized_pagerank(
+ adjacency,
+ personalization,
+ damping=_PPR_DAMPING,
+ iterations=_PPR_ITERS,
+ )
+ if ppr_scores:
+ with profile_stage("ppr.reorder", scored=len(ppr_scores)):
+ for candidate in ranked:
+ n, _r, m, _r2, b = candidate["triple"]
+ masses = [
+ ppr_scores.get(str(getattr(node, "uuid", "")), 0.0)
+ for node in (n, m, b)
+ if getattr(node, "uuid", None)
+ ]
+ candidate["ppr_mass"] = max(masses) if masses else 0.0
+ ranked = sorted(
+ ranked, key=lambda c: _ppr_rank_key(c, ppr_scores)
+ )
+
+ max_facts = max(0, request.max_facts)
- text_lines: list[str] = []
- triples: list[GetContextTriple] = []
provenance_ids: list[str] = []
- for candidate in curated:
- n, r, m, r2, b = candidate["triple"]
- text_lines.append(candidate["text"])
- triples.append(
- GetContextTriple(
- identified_entity=candidate["identified_entity"],
- triple=(n, r, m, r2, b),
- )
- )
+ for candidate in ranked:
provenance_ids.extend(candidate.get("chunk_ids") or [])
-
for chunk_id, _, body in passage_hits:
if body and body not in source_passages:
source_passages.append(body)
if chunk_id not in provenance_ids:
provenance_ids.append(chunk_id)
+ chunk_sessions: dict[str, list[str]] = {}
if provenance_ids:
+ with profile_stage(
+ "provenance.chunks",
+ blocking=False,
+ chunk_ids=len(provenance_ids[:80]),
+ ):
+ try:
+ chunks, _ = await asyncio.to_thread(
+ data_adapter.get_text_chunks_by_ids,
+ provenance_ids[:80],
+ False,
+ request.brain_id,
+ )
+ for chunk in chunks:
+ body = (chunk.text or "").strip()
+ chunk_id = str(getattr(chunk, "id", "") or "")
+ if chunk_id and body:
+ sessions = _session_ids_from_text(body)
+ if sessions:
+ chunk_sessions[chunk_id] = sessions
+ if body and body not in source_passages:
+ source_passages.append(body)
+ except Exception:
+ pass
+
+ for candidate in ranked:
+ if not candidate.get("session_ids"):
+ candidate["session_ids"] = _session_ids_for_chunks(
+ list(candidate.get("chunk_ids") or []), chunk_sessions
+ )
+
+ response_topics: list[dict[str, Any]] = []
+ with profile_stage("topics.coarse_to_fine") as topic_detail:
try:
- chunks, _ = await asyncio.to_thread(
- data_adapter.get_text_chunks_by_ids,
- provenance_ids[:40],
- False,
- request.brain_id,
+ from src.core.saving.topic_hyperedges import select_topics_and_sessions
+
+ memberships = graph_adapter.list_topic_memberships(request.brain_id)
+ seed_sessions: list[str] = []
+ seen_seed: set[str] = set()
+ for candidate in ranked:
+ for sid in _candidate_session_ids(candidate):
+ if sid not in seen_seed:
+ seen_seed.add(sid)
+ seed_sessions.append(sid)
+ response_topics, preferred_sessions = select_topics_and_sessions(
+ request.text,
+ memberships,
+ seed_sessions,
+ k_topics=10,
+ k_sessions=10,
)
- for chunk in chunks:
- body = (chunk.text or "").strip()
- if body and body not in source_passages:
- source_passages.append(body)
- except Exception:
- pass
+ preferred = set(preferred_sessions)
+ topic_detail["topics"] = len(response_topics)
+ topic_detail["preferred_sessions"] = len(preferred)
+ if preferred:
+ for candidate in ranked:
+ sessions = set(_candidate_session_ids(candidate))
+ if sessions & preferred:
+ candidate["topic_preferred"] = True
+ except Exception as exc:
+ topic_detail["error"] = str(exc)[:120]
+
+ with profile_stage("facts.diversify", max_facts=max_facts) as detail:
+ if request.apply_fact_filter and fact_filter_adapter is not None and ranked:
+ detail["filter_applied"] = True
+ keep = filter_relevant_facts(
+ request.text,
+ [c["text"] for c in ranked],
+ llm_adapter=fact_filter_adapter,
+ max_keep=max(max_facts, len(ranked)),
+ )
+ filtered = [ranked[i] for i in keep if 0 <= i < len(ranked)]
+ if not filtered:
+ filtered = ranked
+ else:
+ detail["filter_applied"] = False
+ filtered = ranked
+ filtered = _rank_facts_with_completeness(filtered)
+ curated = _diversify_facts(filtered, max_facts=max_facts)
+ detail["candidates"] = len(filtered)
+ detail["curated"] = len(curated)
- max_passages = max(1, int(getattr(request, "max_passages", 8) or 8))
- source_passages = source_passages[:max_passages]
+ temporal_conflicts = _temporal_conflict_meta(curated)
- entity_scores: dict[str, float] = {}
- for _uuid, distance, entity_name in seed_hits:
- name = (entity_name or "").strip()
- if not name:
- continue
- if name.lower() == request.text.strip().lower():
- continue
- prev = entity_scores.get(name)
- if prev is None or distance < prev:
- entity_scores[name] = distance
- top_entities = [
- name
- for name, _ in sorted(entity_scores.items(), key=lambda item: item[1])[
- :_MAX_DOSSIER_ENTITIES
- ]
- ]
+ text_lines, triples, graph_session_ids = _build_fact_channel(
+ curated, chunk_sessions
+ )
- dossier_lines: list[str] = []
- if top_entities:
+ source_passages = source_passages[:max_passages]
- def _run_dossiers() -> list[str]:
- lines: list[str] = []
- retriever = EventSynergyRetriever(request.brain_id)
- for entity_name in top_entities:
+ insufficient = False
+ if request.sufficiency_retry:
+ with profile_stage("sufficiency.check") as detail:
+ insufficient = _context_looks_insufficient(
+ request.text, text_lines, source_passages + historical_context
+ )
+ detail["fired"] = insufficient
+
+ if insufficient:
+ with profile_stage("sufficiency.retry", blocking=False) as detail:
+ followups = _decompose_enumeration_queries(request.text, elements)
+ if not followups:
+ tokens = [
+ t
+ for t in re.findall(r"[A-Za-z][A-Za-z']+", request.text)
+ if len(t) > 3
+ ]
+ followups = [" ".join(tokens[:4])] if tokens else []
+ detail["followups"] = len(followups[:3])
+ extra_passages: list[tuple[str, float, str]] = []
+ extra_variant_lists: list[list[dict[str, Any]]] = []
+ for query in followups[:3]:
+ extra_passages.extend(
+ await asyncio.to_thread(
+ _retrieve_passages,
+ query,
+ request.brain_id,
+ limit=max_passages,
+ )
+ )
+ seeds, local = await asyncio.to_thread(
+ _collect_facts_for_variant, query
+ )
+ seed_hits.extend(seeds)
+ extra_variant_lists.append(local)
+ for chunk_id, _, body in extra_passages:
+ if body and body not in source_passages:
+ source_passages.append(body)
+ if chunk_id not in provenance_ids:
+ provenance_ids.append(chunk_id)
+ source_passages = source_passages[:max_passages]
+ candidates.extend(_merge_variant_candidate_lists(extra_variant_lists))
+ ranked = _dedupe_candidates(candidates)
+ for candidate in ranked:
+ if not candidate.get("session_ids"):
+ candidate["session_ids"] = _session_ids_for_chunks(
+ list(candidate.get("chunk_ids") or []), chunk_sessions
+ )
+ curated = _diversify_facts(
+ _rank_facts_with_completeness(ranked), max_facts=max_facts
+ )
+ temporal_conflicts = _temporal_conflict_meta(curated)
+ for candidate in curated:
+ for chunk_id in candidate.get("chunk_ids") or []:
+ if chunk_id not in provenance_ids:
+ provenance_ids.append(chunk_id)
+ missing_ids = [
+ cid for cid in provenance_ids[:80] if cid not in chunk_sessions
+ ]
+ if missing_ids:
try:
- path = retriever.retrieve_matches(
- entity_name,
- request.text,
- max_depth=_DOSSIER_MAX_DEPTH,
+ chunks, _ = await asyncio.to_thread(
+ data_adapter.get_text_chunks_by_ids,
+ missing_ids,
+ False,
+ request.brain_id,
)
+ for chunk in chunks:
+ body = (chunk.text or "").strip()
+ chunk_id = str(getattr(chunk, "id", "") or "")
+ if chunk_id and body:
+ sessions = _session_ids_from_text(body)
+ if sessions:
+ chunk_sessions[chunk_id] = sessions
except Exception:
- continue
- if path is None or path.target_node is None:
- continue
- flattened = _flatten_match_path(path)
- if not flattened:
- continue
- target_name = getattr(path.target_node, "name", None) or entity_name
- lines.append(f"[dossier:{target_name}] " + " | ".join(flattened))
- return lines
+ pass
+ text_lines, triples, graph_session_ids = _build_fact_channel(
+ curated, chunk_sessions
+ )
- dossier_lines = await asyncio.to_thread(_run_dossiers)
+ response_paths = _paths_for_curated(bridge_paths, curated)
- passage_block = [f"[passage] {p}" for p in source_passages]
- text_context = "\n".join(passage_block + text_lines + dossier_lines)
+ with profile_stage("context.render", passages=len(source_passages)):
+ topic_block = [
+ f"[topic] {t.get('label')}"
+ for t in response_topics
+ if isinstance(t, dict) and t.get("label")
+ ]
+ passage_block = [f"[passage] {p}" for p in source_passages]
+ text_context = "\n".join(topic_block + passage_block + text_lines)
return GetContextResponse(
text_context=text_context,
triples=triples,
historical_context=historical_context,
source_passages=source_passages,
+ graph_session_ids=graph_session_ids or None,
+ temporal_conflicts=temporal_conflicts or None,
+ paths=response_paths or None,
+ topics=response_topics or None,
)
diff --git a/src/utils/vector_search.py b/src/utils/vector_search.py
index 4216938..0856a11 100644
--- a/src/utils/vector_search.py
+++ b/src/utils/vector_search.py
@@ -2,6 +2,34 @@ from src.adapters.embeddings import VectorStoreAdapter
from src.constants.embeddings import Vector
+def vector_stable_id(vector: Vector) -> str:
+ meta = vector.metadata or {}
+ for key in ("uuid", "resource_id"):
+ value = meta.get(key)
+ if value is not None and str(value).strip():
+ return str(value)
+ return str(vector.id or "")
+
+
+def vector_rank_key(vector: Vector) -> tuple[float, str]:
+ distance = (
+ float("inf") if vector.distance is None else float(vector.distance)
+ )
+ return (distance, vector_stable_id(vector))
+
+
+def stable_top_k_vectors(vectors: list[Vector], k: int) -> list[Vector]:
+ if k <= 0:
+ return []
+ return sorted(vectors, key=vector_rank_key)[:k]
+
+
+def ann_overfetch_k(k: int) -> int:
+ if k <= 0:
+ return 0
+ return max(k * 4, k + 32)
+
+
class VectorSearchFacade:
def __init__(self, vector_store: VectorStoreAdapter):
self._vector_store = vector_store
diff --git a/src/workers/tasks/ingestion.py b/src/workers/tasks/ingestion.py
index 2ae1de7..017be65 100644
--- a/src/workers/tasks/ingestion.py
+++ b/src/workers/tasks/ingestion.py
@@ -462,12 +462,31 @@ def _merge_description(
return f"{existing_clean} | {incoming_clean}"
+def _is_event_entity(entity) -> bool:
+ if (getattr(entity, "type", None) or "").strip().lower() == "event":
+ return True
+ return any(
+ (str(label) or "").strip().lower() == "event"
+ for label in (getattr(entity, "labels", None) or [])
+ )
+
+
def _invalidate_superseded_relationships(
relationship: ArchitectAgentRelationship,
*,
brain_id: str,
) -> None:
- """Mark older same-type edges from the same subject as invalid when tip changes."""
+ """Mark older same-type outgoing edges from the same subject as invalid when tip changes.
+
+ Event hub legs are never invalidated: an actor accumulates one leg per event and
+ none of them supersede the others.
+ """
+ if _is_event_entity(relationship.tail) or _is_event_entity(relationship.tip):
+ return
+ if not (relationship.tail.type or "").strip() or not (
+ relationship.tip.type or ""
+ ).strip():
+ return
try:
neighbors = graph_adapter.get_neighbors(
[relationship.tail.uuid], brain_id=brain_id
@@ -481,6 +500,10 @@ def _invalidate_superseded_relationships(
for predicate, neighbor in pairs:
if not predicate or not neighbor:
continue
+ if (getattr(predicate, "direction", None) or "").strip().lower() != "out":
+ continue
+ if _is_event_entity(neighbor):
+ continue
if (predicate.name or "").strip().upper() != (relationship.name or "").strip().upper():
continue
if neighbor.uuid == relationship.tip.uuid:
@@ -1012,6 +1035,15 @@ def process_architect_relationships(self, args: dict):
f"{len(item_errors)} relationship persistence failures"
)
+ try:
+ graph_adapter.rebuild_hub_bridge_index(brain_id)
+ except Exception as e:
+ print(f"[!] Hub bridge index rebuild failed: {e}")
+ try:
+ graph_adapter.rebuild_topic_index(brain_id)
+ except Exception as e:
+ print(f"[!] Topic index rebuild failed: {e}")
+
if session_id:
from src.lib.redis.client import _redis_client
diff --git a/tests/test_ann_seed_stability.py b/tests/test_ann_seed_stability.py
new file mode 100644
index 0000000..73b915c
--- /dev/null
+++ b/tests/test_ann_seed_stability.py
@@ -0,0 +1,210 @@
+import os
+import random
+import sys
+import unittest
+from concurrent.futures import ThreadPoolExecutor
+from unittest.mock import MagicMock, patch
+
+ENV_DEFAULTS = {
+ "BRAINPAT_TOKEN": "test-token",
+ "MODELS_MODE": "local",
+ "EMBEDDINGS_LOCAL_MODEL": "local-model",
+ "EMBEDDINGS_SMALL_MODEL": "small-model",
+ "EMBEDDING_NODES_DIMENSION": "3",
+ "EMBEDDING_TRIPLETS_DIMENSION": "3",
+ "EMBEDDING_OBSERVATIONS_DIMENSION": "3",
+ "EMBEDDING_DATA_DIMENSION": "3",
+ "EMBEDDING_RELATIONSHIPS_DIMENSION": "3",
+ "REDIS_HOST": "localhost",
+ "REDIS_PORT": "6379",
+ "NEO4J_HOST": "localhost",
+ "NEO4J_PORT": "7687",
+ "NEO4J_USERNAME": "neo4j",
+ "NEO4J_PASSWORD": "password",
+ "MILVUS_HOST": "localhost",
+ "MILVUS_PORT": "19530",
+ "MONGO_CONNECTION_STRING": "mongodb://localhost:27017",
+ "CELERY_WORKER_CONCURRENCY": "1",
+ "OLLAMA_HOST": "localhost",
+ "OLLAMA_PORT": "11434",
+ "OLLAMA_LLM_SMALL_MODEL": "small",
+ "OLLAMA_LLM_LARGE_MODEL": "large",
+ "POSTGRES_HOST": "localhost",
+ "POSTGRES_PORT": "5432",
+ "POSTGRES_USERNAME": "postgres",
+ "POSTGRES_PASSWORD": "postgres",
+}
+for key, value in ENV_DEFAULTS.items():
+ os.environ.setdefault(key, value)
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from src.constants.embeddings import Vector
+from src.services.api.controllers import retrieve as retrieve_mod
+from src.utils.vector_search import (
+ ann_overfetch_k,
+ stable_top_k_vectors,
+ vector_rank_key,
+)
+
+
+class StableTopKVectorTests(unittest.TestCase):
+ def test_equal_distance_uses_uuid_total_order(self):
+ hits = [
+ Vector(id="1", metadata={"uuid": "z-node"}, distance=0.1),
+ Vector(id="2", metadata={"uuid": "a-node"}, distance=0.1),
+ Vector(id="3", metadata={"uuid": "m-node"}, distance=0.05),
+ Vector(id="4", metadata={"uuid": "b-node"}, distance=0.1),
+ ]
+ forward = stable_top_k_vectors(hits, 3)
+ reverse = stable_top_k_vectors(list(reversed(hits)), 3)
+ self.assertEqual(
+ [v.metadata["uuid"] for v in forward],
+ [v.metadata["uuid"] for v in reverse],
+ )
+ self.assertEqual(
+ [v.metadata["uuid"] for v in forward],
+ ["m-node", "a-node", "b-node"],
+ )
+
+ def test_boundary_k_stable_under_shuffled_equal_distance_pool(self):
+ pool = [
+ Vector(id=str(i), metadata={"uuid": f"u{i:02d}"}, distance=0.2)
+ for i in range(12)
+ ]
+ expected = [f"u{i:02d}" for i in range(5)]
+ for seed in range(30):
+ rng = random.Random(seed)
+ shuffled = list(pool)
+ rng.shuffle(shuffled)
+ got = [v.metadata["uuid"] for v in stable_top_k_vectors(shuffled, 5)]
+ self.assertEqual(got, expected)
+
+ def test_overfetch_is_at_least_4x(self):
+ self.assertEqual(ann_overfetch_k(25), 100)
+ self.assertEqual(ann_overfetch_k(5), 37)
+ self.assertEqual(ann_overfetch_k(0), 0)
+
+ def test_rank_key_falls_back_to_vector_id(self):
+ v = Vector(id="fallback-id", metadata={}, distance=0.3)
+ self.assertEqual(vector_rank_key(v), (0.3, "fallback-id"))
+
+
+class SeedStabilizationTests(unittest.TestCase):
+ def test_stabilize_seed_hits_dedupes_and_orders(self):
+ seeds = [
+ ("b", 0.2, "B"),
+ ("a", 0.2, "A"),
+ ("b", 0.1, "B-better"),
+ ("c", 0.05, "C"),
+ ("", 0.0, "drop"),
+ ]
+ out = retrieve_mod._stabilize_seed_hits(seeds)
+ self.assertEqual(
+ out,
+ [("c", 0.05, "C"), ("b", 0.1, "B-better"), ("a", 0.2, "A")],
+ )
+
+ def test_quantize_distance_collapses_embed_float_noise(self):
+ a = retrieve_mod._stabilize_seed_hits(
+ [("u", 0.6657524665076711, "A"), ("v", 0.37019320459679717, "B")]
+ )
+ b = retrieve_mod._stabilize_seed_hits(
+ [("v", 0.3702673417792153, "B"), ("u", 0.6657041188926127, "A")]
+ )
+ self.assertEqual([uid for uid, _, _ in a], [uid for uid, _, _ in b])
+ self.assertEqual([d for _, d, _ in a], [d for _, d, _ in b])
+ self.assertEqual(
+ retrieve_mod._quantize_distance(0.6657524665076711),
+ retrieve_mod._quantize_distance(0.6657041188926127),
+ )
+ self.assertEqual(
+ retrieve_mod._quantize_distance(0.37019320459679717),
+ retrieve_mod._quantize_distance(0.3702673417792153),
+ )
+
+ def test_equal_distance_concurrent_seed_paths_identical_uuid_lists(self):
+ equal_hits = [
+ Vector(
+ id=f"n{i}",
+ embeddings=[0.1, 0.2, 0.3],
+ metadata={"uuid": f"node-{chr(ord('z') - i)}", "name": f"N{i}"},
+ distance=0.42,
+ )
+ for i in range(8)
+ ]
+ near = Vector(
+ id="near",
+ embeddings=[0.1, 0.2, 0.3],
+ metadata={"uuid": "node-near", "name": "Near"},
+ distance=0.01,
+ )
+
+ mock_embeddings = MagicMock()
+ mock_embeddings.embed_text.return_value = Vector(
+ id="q", embeddings=[0.1, 0.2, 0.3], metadata={}
+ )
+
+ def run_once(order_seed: int):
+ rng = random.Random(order_seed)
+ node_hits = [near] + list(equal_hits)
+ rng.shuffle(node_hits)
+ rel_hits = [
+ Vector(
+ id="r1",
+ embeddings=[0.1, 0.2, 0.3],
+ metadata={"node_ids": ["tail-b", "tail-a"]},
+ distance=0.42,
+ ),
+ Vector(
+ id="r0",
+ embeddings=[0.1, 0.2, 0.3],
+ metadata={"node_ids": ["tail-c", "tail-a"]},
+ distance=0.42,
+ ),
+ ]
+ rng.shuffle(rel_hits)
+ mock_vs = MagicMock()
+ mock_vs.search_nodes.return_value = node_hits
+ mock_vs.search_relationships.return_value = rel_hits
+ with patch.object(retrieve_mod, "embeddings_adapter", mock_embeddings):
+ with patch.object(retrieve_mod, "vector_search", mock_vs):
+ return [
+ uid
+ for uid, _, _ in retrieve_mod._seed_nodes_for_text(
+ "query", "brain"
+ )
+ ]
+
+ results = []
+ with ThreadPoolExecutor(max_workers=8) as pool:
+ futs = [pool.submit(run_once, seed) for seed in range(24)]
+ for fut in futs:
+ results.append(fut.result())
+
+ self.assertTrue(all(r == results[0] for r in results))
+ self.assertEqual(results[0][0], "node-near")
+ self.assertEqual(
+ results[0],
+ sorted(results[0], key=lambda uid: (0.01 if uid == "node-near" else 0.42, uid)),
+ )
+
+
+class AdapterStableSearchTests(unittest.TestCase):
+ def test_adapter_breaks_distance_ties_by_uuid(self):
+ from src.adapters.embeddings import VectorStoreAdapter
+
+ store = MagicMock()
+ store.search_vectors.return_value = [
+ Vector(id="1", metadata={"uuid": "z"}, distance=0.1),
+ Vector(id="2", metadata={"uuid": "a"}, distance=0.1),
+ Vector(id="3", metadata={"uuid": "m"}, distance=0.05),
+ ]
+ adapter = VectorStoreAdapter()
+ adapter.add_client(store)
+ results = adapter.search_vectors([1.0, 0.0], brain_id="b", store="nodes", k=2)
+ self.assertEqual([v.metadata["uuid"] for v in results], ["m", "a"])
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/tests/test_architecture_refactors.py b/tests/test_architecture_refactors.py
index e96ff10..7b0b78a 100644
--- a/tests/test_architecture_refactors.py
+++ b/tests/test_architecture_refactors.py
@@ -62,11 +62,25 @@ class _StubGraphAdapter:
def get_neighbors(self, *_args, **_kwargs):
return {}
+ def get_nodes_by_uuid(self, *_args, **_kwargs):
+ return []
+
class _StubDataAdapter:
def get_observations_list(self, *_args, **_kwargs):
return []
+ def search(self, *_args, **_kwargs):
+ return types.SimpleNamespace(text_chunks=[], observations=[])
+
+ def get_text_chunks_by_ids(self, *_args, **_kwargs):
+ return ([], [])
+
+
+class _StubKGAgent:
+ def retrieve_neighbors(self, *_args, **_kwargs):
+ return types.SimpleNamespace(neighbors=[])
+
_stub_input_agents = types.ModuleType("src.services.input.agents")
_stub_input_agents.embeddings_adapter = _StubEmbeddingsAdapter()
@@ -76,6 +90,7 @@ _stub_kg_main = types.ModuleType("src.services.kg_agent.main")
_stub_kg_main.graph_adapter = _StubGraphAdapter()
_stub_kg_main.vector_store_adapter = _StubVectorStoreAdapter()
_stub_kg_main.embeddings_adapter = _StubEmbeddingsAdapter()
+_stub_kg_main.kg_agent = _StubKGAgent()
sys.modules.setdefault("src.services.kg_agent.main", _stub_kg_main)
_stub_data_main = types.ModuleType("src.services.data.main")
diff --git a/tests/test_context_defaults_and_fact_filter.py b/tests/test_context_defaults_and_fact_filter.py
new file mode 100644
index 0000000..4124da3
--- /dev/null
+++ b/tests/test_context_defaults_and_fact_filter.py
@@ -0,0 +1,238 @@
+import contextlib
+import inspect
+import os
+import re
+import sys
+import unittest
+from unittest.mock import MagicMock, patch
+
+
+ENV_DEFAULTS = {
+ "BRAINPAT_TOKEN": "test-token",
+ "MODELS_MODE": "local",
+ "EMBEDDINGS_LOCAL_MODEL": "local-model",
+ "EMBEDDINGS_SMALL_MODEL": "small-model",
+ "EMBEDDING_NODES_DIMENSION": "3",
+ "EMBEDDING_TRIPLETS_DIMENSION": "3",
+ "EMBEDDING_OBSERVATIONS_DIMENSION": "3",
+ "EMBEDDING_DATA_DIMENSION": "3",
+ "EMBEDDING_RELATIONSHIPS_DIMENSION": "3",
+ "REDIS_HOST": "localhost",
+ "REDIS_PORT": "6379",
+ "NEO4J_HOST": "localhost",
+ "NEO4J_PORT": "7687",
+ "NEO4J_USERNAME": "neo4j",
+ "NEO4J_PASSWORD": "password",
+ "MILVUS_HOST": "localhost",
+ "MILVUS_PORT": "19530",
+ "MONGO_CONNECTION_STRING": "mongodb://localhost:27017",
+ "CELERY_WORKER_CONCURRENCY": "1",
+ "OLLAMA_HOST": "localhost",
+ "OLLAMA_PORT": "11434",
+ "OLLAMA_LLM_SMALL_MODEL": "small",
+ "OLLAMA_LLM_LARGE_MODEL": "large",
+ "POSTGRES_HOST": "localhost",
+ "POSTGRES_PORT": "5432",
+ "POSTGRES_USERNAME": "postgres",
+ "POSTGRES_PASSWORD": "postgres",
+}
+for key, value in ENV_DEFAULTS.items():
+ os.environ.setdefault(key, value)
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from src.constants.embeddings import Vector
+from src.constants.kg import Node, Predicate
+from src.core.search.fact_filter import FactFilterDecision
+from src.services.api.constants.requests import GetContextRequestBody
+from src.utils.nlp.ner import ExtractElementsResponse
+
+
+def _node(uuid: str, name: str, *, happened_at: str | None = None, labels=None):
+ return Node(
+ uuid=uuid,
+ name=name,
+ labels=labels or ["PERSON"],
+ description=f"desc-{name}",
+ happened_at=happened_at,
+ )
+
+
+def _pred(uuid: str, name: str):
+ return Predicate(uuid=uuid, name=name, description=f"desc-{name}")
+
+
+def _facts(count: int):
+ return [
+ (
+ _node("a", "Alice"),
+ _pred(f"r{i}a", f"ATTENDED_{i}"),
+ _node(
+ f"e{i}",
+ f"Event{i}",
+ happened_at=f"2024-0{i + 1}-01",
+ labels=["EVENT"],
+ ),
+ _pred(f"r{i}b", f"WITH_{i}"),
+ _node(f"b{i}", f"Person{i}"),
+ )
+ for i in range(count)
+ ]
+
+
+def _stubbed_adapters(retrieve_mod, facts, extra=()):
+ mock_embeddings = MagicMock()
+ mock_embeddings.embed_text.return_value = Vector(
+ id="q", embeddings=[0.1, 0.2, 0.3], metadata={}
+ )
+ mock_vs = MagicMock()
+ mock_vs.search_nodes.return_value = [
+ Vector(
+ id="n1",
+ embeddings=[0.1, 0.2, 0.3],
+ metadata={"uuid": "a", "name": "Alice"},
+ distance=0.1,
+ )
+ ]
+ mock_vs.search_relationships.return_value = []
+ mock_graph = MagicMock()
+ mock_graph.get_event_centric_neighbors.return_value = facts
+ mock_data = MagicMock()
+ mock_data.get_last_text_chunks.return_value = []
+ mock_data.get_last_structured_data.return_value = []
+
+ with contextlib.ExitStack() as stack:
+ stack.enter_context(
+ patch.object(retrieve_mod, "embeddings_adapter", mock_embeddings)
+ )
+ stack.enter_context(patch.object(retrieve_mod, "vector_search", mock_vs))
+ stack.enter_context(patch.object(retrieve_mod, "graph_adapter", mock_graph))
+ stack.enter_context(patch.object(retrieve_mod, "data_adapter", mock_data))
+ stack.enter_context(
+ patch.object(
+ retrieve_mod._entity_extractor,
+ "extract_elements",
+ return_value=ExtractElementsResponse(tokens=[], noun_chunks=[]),
+ )
+ )
+ for context in extra:
+ stack.enter_context(context)
+ yield mock_graph
+
+
+class ContextRequestDefaultsTests(unittest.IsolatedAsyncioTestCase):
+ def test_controller_reads_no_declared_field_through_a_getattr_default(self):
+ from src.services.api.controllers import retrieve as retrieve_mod
+
+ source = inspect.getsource(retrieve_mod.get_context)
+ offenders = [
+ field
+ for field in GetContextRequestBody.model_fields
+ if re.search(rf"getattr\(\s*request\s*,\s*[\"']{field}[\"']", source)
+ ]
+ self.assertEqual(offenders, [])
+
+ def test_ppr_is_on_and_sufficiency_retry_is_off_by_default(self):
+ defaults = {
+ name: field.default
+ for name, field in GetContextRequestBody.model_fields.items()
+ }
+ self.assertIs(defaults["use_ppr"], True)
+ self.assertIs(defaults["sufficiency_retry"], False)
+
+ async def test_default_request_runs_ppr(self):
+ from src.services.api.controllers import retrieve as retrieve_mod
+
+ adjacency = MagicMock(return_value={})
+ with _stubbed_adapters(
+ retrieve_mod,
+ _facts(3),
+ extra=[
+ patch.object(retrieve_mod, "_build_adjacency_from_seeds", adjacency)
+ ],
+ ):
+ await retrieve_mod.get_context(
+ GetContextRequestBody(text="Alice", brain_id="brain-a")
+ )
+
+ adjacency.assert_called_once()
+
+ async def test_sufficiency_retry_is_opt_in(self):
+ from src.services.api.controllers import retrieve as retrieve_mod
+
+ for sufficiency_retry in (False, True):
+ with self.subTest(sufficiency_retry=sufficiency_retry):
+ insufficient = MagicMock(return_value=True)
+ passages = MagicMock(return_value=[])
+ with _stubbed_adapters(
+ retrieve_mod,
+ _facts(2),
+ extra=[
+ patch.object(
+ retrieve_mod, "_context_looks_insufficient", insufficient
+ ),
+ patch.object(retrieve_mod, "_retrieve_passages", passages),
+ ],
+ ):
+ await retrieve_mod.get_context(
+ GetContextRequestBody(
+ text="Alice",
+ brain_id="brain-a",
+ sufficiency_retry=sufficiency_retry,
+ )
+ )
+
+ self.assertEqual(insufficient.called, sufficiency_retry)
+
+
+class ContextFactFilterTests(unittest.IsolatedAsyncioTestCase):
+ async def test_context_path_without_an_adapter_keeps_every_ranked_fact(self):
+ from src.services.api.controllers import retrieve as retrieve_mod
+
+ with _stubbed_adapters(retrieve_mod, _facts(3)):
+ response = await retrieve_mod.get_context(
+ GetContextRequestBody(
+ text="Alice", brain_id="brain-a", apply_fact_filter=True
+ )
+ )
+
+ self.assertEqual(len(response.triples), 3)
+
+ async def test_supplied_adapter_actually_filters(self):
+ from src.services.api.controllers import retrieve as retrieve_mod
+
+ adapter = MagicMock()
+ adapter.generate_structured.return_value = FactFilterDecision(keep_indices=[1])
+
+ with _stubbed_adapters(retrieve_mod, _facts(3)):
+ response = await retrieve_mod.get_context(
+ GetContextRequestBody(
+ text="Alice", brain_id="brain-a", apply_fact_filter=True
+ ),
+ fact_filter_adapter=adapter,
+ )
+
+ adapter.generate_structured.assert_called_once()
+ self.assertEqual(len(response.triples), 1)
+
+ async def test_adapter_is_ignored_when_the_caller_disables_filtering(self):
+ from src.services.api.controllers import retrieve as retrieve_mod
+
+ adapter = MagicMock()
+ adapter.generate_structured.return_value = FactFilterDecision(keep_indices=[1])
+
+ with _stubbed_adapters(retrieve_mod, _facts(3)):
+ response = await retrieve_mod.get_context(
+ GetContextRequestBody(
+ text="Alice", brain_id="brain-a", apply_fact_filter=False
+ ),
+ fact_filter_adapter=adapter,
+ )
+
+ adapter.generate_structured.assert_not_called()
+ self.assertEqual(len(response.triples), 3)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/tests/test_context_fact_order_determinism.py b/tests/test_context_fact_order_determinism.py
new file mode 100644
index 0000000..223f64f
--- /dev/null
+++ b/tests/test_context_fact_order_determinism.py
@@ -0,0 +1,274 @@
+import asyncio
+import os
+import random
+import sys
+import unittest
+from concurrent.futures import ThreadPoolExecutor
+
+ENV_DEFAULTS = {
+ "BRAINPAT_TOKEN": "test-token",
+ "MODELS_MODE": "local",
+ "EMBEDDINGS_LOCAL_MODEL": "local-model",
+ "EMBEDDINGS_SMALL_MODEL": "small-model",
+ "EMBEDDING_NODES_DIMENSION": "3",
+ "EMBEDDING_TRIPLETS_DIMENSION": "3",
+ "EMBEDDING_OBSERVATIONS_DIMENSION": "3",
+ "EMBEDDING_DATA_DIMENSION": "3",
+ "EMBEDDING_RELATIONSHIPS_DIMENSION": "3",
+ "REDIS_HOST": "localhost",
+ "REDIS_PORT": "6379",
+ "NEO4J_HOST": "localhost",
+ "NEO4J_PORT": "7687",
+ "NEO4J_USERNAME": "neo4j",
+ "NEO4J_PASSWORD": "password",
+ "MILVUS_HOST": "localhost",
+ "MILVUS_PORT": "19530",
+ "MONGO_CONNECTION_STRING": "mongodb://localhost:27017",
+ "CELERY_WORKER_CONCURRENCY": "1",
+ "OLLAMA_HOST": "localhost",
+ "OLLAMA_PORT": "11434",
+ "OLLAMA_LLM_SMALL_MODEL": "small",
+ "OLLAMA_LLM_LARGE_MODEL": "large",
+ "POSTGRES_HOST": "localhost",
+ "POSTGRES_PORT": "5432",
+ "POSTGRES_USERNAME": "postgres",
+ "POSTGRES_PASSWORD": "postgres",
+}
+for key, value in ENV_DEFAULTS.items():
+ os.environ.setdefault(key, value)
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from src.constants.kg import Node, Predicate
+from src.services.api.controllers import retrieve as retrieve_mod
+
+
+def _node(uuid: str, name: str, *, labels=None):
+ return Node(
+ uuid=uuid,
+ name=name,
+ labels=labels or ["PERSON"],
+ description=f"desc-{name}",
+ properties={},
+ )
+
+
+def _pred(uuid: str, name: str, *, flow_key: str = ""):
+ return Predicate(
+ uuid=uuid,
+ name=name,
+ description=f"desc-{name}",
+ flow_key=flow_key or None,
+ )
+
+
+def _candidate(
+ *,
+ score: float,
+ r_uuid: str,
+ r2_uuid: str,
+ flow_key: str = "flow",
+ n_uuid: str = "n",
+ m_uuid: str = "m",
+ b_uuid: str = "b",
+ entity: str = "q",
+):
+ n = _node(n_uuid, n_uuid)
+ r = _pred(r_uuid, "REL", flow_key=flow_key)
+ m = _node(m_uuid, m_uuid, labels=["EVENT"])
+ r2 = _pred(r2_uuid, "REL2", flow_key=flow_key)
+ b = _node(b_uuid, b_uuid)
+ return {
+ "identified_entity": entity,
+ "triple": (n, r, m, r2, b),
+ "score": score,
+ "key": (r_uuid, r2_uuid),
+ "text": f"{n_uuid}|{r_uuid}|{m_uuid}|{r2_uuid}|{b_uuid}",
+ "chunk_ids": [],
+ }
+
+
+def _ordered_ids(candidates):
+ return [c["key"] for c in candidates]
+
+
+class FactOrderDeterminismTests(unittest.TestCase):
+ def test_equal_scores_use_stable_tiebreakers(self):
+ a = _candidate(score=0.5, r_uuid="r-b", r2_uuid="r2-b", flow_key="f2")
+ b = _candidate(score=0.5, r_uuid="r-a", r2_uuid="r2-a", flow_key="f1")
+ c = _candidate(score=0.1, r_uuid="r-c", r2_uuid="r2-c", flow_key="f0")
+ ranked_forward = retrieve_mod._dedupe_candidates([a, b, c])
+ ranked_reverse = retrieve_mod._dedupe_candidates([c, b, a])
+ self.assertEqual(_ordered_ids(ranked_forward), _ordered_ids(ranked_reverse))
+ self.assertEqual(
+ _ordered_ids(ranked_forward),
+ [("r-c", "r2-c"), ("r-a", "r2-a"), ("r-b", "r2-b")],
+ )
+
+ def test_merge_variant_lists_is_order_stable_under_shuffled_arrival(self):
+ v0 = [
+ _candidate(score=0.2, r_uuid="r0", r2_uuid="x0", entity="v0"),
+ _candidate(score=0.2, r_uuid="r1", r2_uuid="x1", entity="v0"),
+ ]
+ v1 = [
+ _candidate(score=0.2, r_uuid="r2", r2_uuid="x2", entity="v1"),
+ _candidate(score=0.2, r_uuid="r3", r2_uuid="x3", entity="v1"),
+ ]
+ expected = _ordered_ids(
+ retrieve_mod._dedupe_candidates(
+ retrieve_mod._merge_variant_candidate_lists([v0, v1])
+ )
+ )
+
+ def race_once(seed: int):
+ rng = random.Random(seed)
+ buckets = [None, None]
+
+ def worker(idx, payload):
+ buckets[idx] = list(payload)
+ rng.random()
+
+ with ThreadPoolExecutor(max_workers=2) as pool:
+ order = [0, 1]
+ rng.shuffle(order)
+ futs = [
+ pool.submit(worker, idx, [v0, v1][idx]) for idx in order
+ ]
+ for fut in futs:
+ fut.result()
+ merged = retrieve_mod._merge_variant_candidate_lists(
+ [buckets[0], buckets[1]]
+ )
+ return _ordered_ids(retrieve_mod._dedupe_candidates(merged))
+
+ results = [race_once(seed) for seed in range(40)]
+ self.assertTrue(all(r == expected for r in results))
+ self.assertEqual(
+ expected,
+ [("r0", "x0"), ("r1", "x1"), ("r2", "x2"), ("r3", "x3")],
+ )
+
+ def test_nondeterministic_extend_diverges_without_ordered_merge(self):
+ a = _candidate(score=0.2, r_uuid="r0", r2_uuid="x0", entity="v0")
+ b = _candidate(score=0.2, r_uuid="r1", r2_uuid="x1", entity="v1")
+ score_only_ab = [c["key"] for c in sorted([a, b], key=lambda c: c["score"])]
+ score_only_ba = [c["key"] for c in sorted([b, a], key=lambda c: c["score"])]
+ self.assertNotEqual(score_only_ab, score_only_ba)
+
+ fixed = {
+ tuple(
+ _ordered_ids(
+ retrieve_mod._dedupe_candidates(
+ retrieve_mod._merge_variant_candidate_lists(lists)
+ )
+ )
+ )
+ for lists in ([[a], [b]], [[b], [a]])
+ }
+ self.assertEqual(len(fixed), 1)
+ self.assertEqual(next(iter(fixed)), (("r0", "x0"), ("r1", "x1")))
+
+
+class ContextFactOrderIntegrationTests(unittest.IsolatedAsyncioTestCase):
+ async def test_two_identical_get_context_calls_same_triple_order(self):
+ import contextlib
+ from unittest.mock import MagicMock, patch
+
+ from src.constants.embeddings import Vector
+ from src.services.api.constants.requests import GetContextRequestBody
+ from src.utils.nlp.ner import ExtractElementsResponse
+
+ facts = []
+ for i in range(6):
+ facts.append(
+ (
+ _node(f"n{i}", f"N{i}"),
+ _pred(f"r{i}", "REL", flow_key=f"f{i % 3}"),
+ _node(f"e{i}", f"E{i}", labels=["EVENT"]),
+ _pred(f"r2{i}", "REL2", flow_key=f"f{i % 3}"),
+ _node(f"b{i}", f"B{i}"),
+ )
+ )
+
+ mock_embeddings = MagicMock()
+ mock_embeddings.embed_text.return_value = Vector(
+ id="q", embeddings=[0.1, 0.2, 0.3], metadata={}
+ )
+ mock_vs = MagicMock()
+ mock_vs.search_nodes.return_value = [
+ Vector(
+ id="n0",
+ embeddings=[0.1, 0.2, 0.3],
+ metadata={"uuid": "n0", "name": "N0"},
+ distance=0.5,
+ ),
+ Vector(
+ id="n1",
+ embeddings=[0.1, 0.2, 0.3],
+ metadata={"uuid": "n1", "name": "N1"},
+ distance=0.5,
+ ),
+ ]
+ mock_vs.search_relationships.return_value = []
+ mock_vs.search_data.return_value = []
+ mock_graph = MagicMock()
+ mock_graph.get_event_centric_neighbors.return_value = facts
+ mock_data = MagicMock()
+ mock_data.get_last_text_chunks.return_value = []
+ mock_data.get_last_structured_data.return_value = []
+ mock_data.get_text_chunks_by_ids.return_value = ([], None)
+ mock_data.search.return_value = MagicMock(text_chunks=[])
+
+ async def _once():
+ with contextlib.ExitStack() as stack:
+ stack.enter_context(
+ patch.object(retrieve_mod, "embeddings_adapter", mock_embeddings)
+ )
+ stack.enter_context(patch.object(retrieve_mod, "vector_search", mock_vs))
+ stack.enter_context(
+ patch.object(retrieve_mod, "graph_adapter", mock_graph)
+ )
+ stack.enter_context(patch.object(retrieve_mod, "data_adapter", mock_data))
+ stack.enter_context(
+ patch.object(
+ retrieve_mod._entity_extractor,
+ "extract_elements",
+ return_value=ExtractElementsResponse(
+ tokens=[
+ {"text": "Alice", "label": "PERSON"},
+ {"text": "Bob", "label": "PERSON"},
+ ],
+ noun_chunks=["Alice", "Bob"],
+ ),
+ )
+ )
+ stack.enter_context(
+ patch.object(retrieve_mod, "_retrieve_passages", return_value=[])
+ )
+ return await retrieve_mod.get_context(
+ GetContextRequestBody(
+ text="Alice and Bob events",
+ brain_id="brain-a",
+ max_facts=4,
+ use_ppr=False,
+ )
+ )
+
+ first = await _once()
+ second = await _once()
+ first_ids = [(t.triple[1].uuid, t.triple[3].uuid) for t in first.triples]
+ second_ids = [(t.triple[1].uuid, t.triple[3].uuid) for t in second.triples]
+ self.assertEqual(first_ids, second_ids)
+ self.assertEqual(len(first_ids), 4)
+
+ parallel = await asyncio.gather(*[_once() for _ in range(12)])
+ id_sets = [
+ tuple((t.triple[1].uuid, t.triple[3].uuid) for t in resp.triples)
+ for resp in parallel
+ ]
+ self.assertEqual(len(set(id_sets)), 1)
+ self.assertEqual(id_sets[0], tuple(first_ids))
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/tests/test_context_max_facts_and_provenance.py b/tests/test_context_max_facts_and_provenance.py
new file mode 100644
index 0000000..e777d69
--- /dev/null
+++ b/tests/test_context_max_facts_and_provenance.py
@@ -0,0 +1,260 @@
+import contextlib
+import os
+import sys
+import unittest
+from unittest.mock import MagicMock, patch
+
+from pydantic import ValidationError
+
+
+ENV_DEFAULTS = {
+ "BRAINPAT_TOKEN": "test-token",
+ "MODELS_MODE": "local",
+ "EMBEDDINGS_LOCAL_MODEL": "local-model",
+ "EMBEDDINGS_SMALL_MODEL": "small-model",
+ "EMBEDDING_NODES_DIMENSION": "3",
+ "EMBEDDING_TRIPLETS_DIMENSION": "3",
+ "EMBEDDING_OBSERVATIONS_DIMENSION": "3",
+ "EMBEDDING_DATA_DIMENSION": "3",
+ "EMBEDDING_RELATIONSHIPS_DIMENSION": "3",
+ "REDIS_HOST": "localhost",
+ "REDIS_PORT": "6379",
+ "NEO4J_HOST": "localhost",
+ "NEO4J_PORT": "7687",
+ "NEO4J_USERNAME": "neo4j",
+ "NEO4J_PASSWORD": "password",
+ "MILVUS_HOST": "localhost",
+ "MILVUS_PORT": "19530",
+ "MONGO_CONNECTION_STRING": "mongodb://localhost:27017",
+ "CELERY_WORKER_CONCURRENCY": "1",
+ "OLLAMA_HOST": "localhost",
+ "OLLAMA_PORT": "11434",
+ "OLLAMA_LLM_SMALL_MODEL": "small",
+ "OLLAMA_LLM_LARGE_MODEL": "large",
+ "POSTGRES_HOST": "localhost",
+ "POSTGRES_PORT": "5432",
+ "POSTGRES_USERNAME": "postgres",
+ "POSTGRES_PASSWORD": "postgres",
+}
+for key, value in ENV_DEFAULTS.items():
+ os.environ.setdefault(key, value)
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from src.constants.data import TextChunk
+from src.constants.embeddings import Vector
+from src.constants.kg import Node, Predicate
+from src.services.api.constants.requests import GetContextRequestBody
+from src.utils.nlp.ner import ExtractElementsResponse
+
+
+def _node(uuid: str, name: str, *, labels=None, source_chunk_ids=None):
+ props = {}
+ if source_chunk_ids:
+ props["source_chunk_ids"] = list(source_chunk_ids)
+ return Node(
+ uuid=uuid,
+ name=name,
+ labels=labels or ["PERSON"],
+ description=f"desc-{name}",
+ properties=props,
+ )
+
+
+def _pred(uuid: str, name: str):
+ return Predicate(uuid=uuid, name=name, description=f"desc-{name}")
+
+
+def _fact(*, chunk_ids=None):
+ return (
+ _node("a", "Alice", source_chunk_ids=chunk_ids),
+ _pred("r1", "ATTENDED"),
+ _node("e1", "Event1", labels=["EVENT"], source_chunk_ids=chunk_ids),
+ _pred("r2", "WITH"),
+ _node("b", "Bob", source_chunk_ids=chunk_ids),
+ )
+
+
+def _stubbed_adapters(retrieve_mod, facts, *, chunks=None, passages=None, extra=()):
+ mock_embeddings = MagicMock()
+ mock_embeddings.embed_text.return_value = Vector(
+ id="q", embeddings=[0.1, 0.2, 0.3], metadata={}
+ )
+ mock_vs = MagicMock()
+ mock_vs.search_nodes.return_value = [
+ Vector(
+ id="n1",
+ embeddings=[0.1, 0.2, 0.3],
+ metadata={"uuid": "a", "name": "Alice"},
+ distance=0.1,
+ )
+ ]
+ mock_vs.search_relationships.return_value = []
+ mock_vs.search_data.return_value = []
+ mock_graph = MagicMock()
+ mock_graph.get_event_centric_neighbors.return_value = facts
+ mock_data = MagicMock()
+ mock_data.get_last_text_chunks.return_value = []
+ mock_data.get_last_structured_data.return_value = []
+ mock_data.get_text_chunks_by_ids.return_value = (chunks or [], None)
+ mock_data.search.return_value = MagicMock(text_chunks=[])
+
+ with contextlib.ExitStack() as stack:
+ stack.enter_context(
+ patch.object(retrieve_mod, "embeddings_adapter", mock_embeddings)
+ )
+ stack.enter_context(patch.object(retrieve_mod, "vector_search", mock_vs))
+ stack.enter_context(patch.object(retrieve_mod, "graph_adapter", mock_graph))
+ stack.enter_context(patch.object(retrieve_mod, "data_adapter", mock_data))
+ stack.enter_context(
+ patch.object(
+ retrieve_mod._entity_extractor,
+ "extract_elements",
+ return_value=ExtractElementsResponse(tokens=[], noun_chunks=[]),
+ )
+ )
+ if passages is not None:
+ stack.enter_context(
+ patch.object(retrieve_mod, "_retrieve_passages", return_value=passages)
+ )
+ for context in extra:
+ stack.enter_context(context)
+ yield mock_data
+
+
+class MaxFactsZeroTests(unittest.IsolatedAsyncioTestCase):
+ def test_negative_max_facts_rejected_at_schema(self):
+ with self.assertRaises(ValidationError):
+ GetContextRequestBody(text="Alice", brain_id="brain-a", max_facts=-1)
+
+ async def test_max_facts_zero_returns_no_graph_facts(self):
+ from src.services.api.controllers import retrieve as retrieve_mod
+
+ with _stubbed_adapters(
+ retrieve_mod,
+ [_fact(chunk_ids=["chunk-1"]), _fact(chunk_ids=["chunk-2"])],
+ passages=[("p1", 0.1, "Session id: session_9. Passage body.")],
+ ):
+ response = await retrieve_mod.get_context(
+ GetContextRequestBody(
+ text="Alice",
+ brain_id="brain-a",
+ max_facts=0,
+ max_passages=2,
+ )
+ )
+
+ self.assertEqual(response.triples, [])
+ self.assertIsNone(response.graph_session_ids)
+ fact_lines = [
+ line
+ for line in response.text_context.splitlines()
+ if line and not line.startswith("[passage]")
+ ]
+ self.assertEqual(fact_lines, [])
+ self.assertTrue(
+ any("Session id: session_9" in p for p in response.source_passages)
+ )
+
+ async def test_max_facts_one_keeps_a_single_fact(self):
+ from src.services.api.controllers import retrieve as retrieve_mod
+
+ with _stubbed_adapters(
+ retrieve_mod,
+ [_fact(chunk_ids=["chunk-1"]), _fact(chunk_ids=["chunk-2"])],
+ ):
+ response = await retrieve_mod.get_context(
+ GetContextRequestBody(
+ text="Alice", brain_id="brain-a", max_facts=1
+ )
+ )
+
+ self.assertEqual(len(response.triples), 1)
+
+
+class GraphProvenanceTests(unittest.IsolatedAsyncioTestCase):
+ async def test_triples_expose_chunk_and_session_provenance(self):
+ from src.services.api.controllers import retrieve as retrieve_mod
+
+ chunks = [
+ TextChunk(
+ id="chunk-3",
+ text="Session id: session_3. Melanie put the keys in the slipper.",
+ )
+ ]
+ with _stubbed_adapters(
+ retrieve_mod,
+ [_fact(chunk_ids=["chunk-3"])],
+ chunks=chunks,
+ passages=[],
+ ):
+ response = await retrieve_mod.get_context(
+ GetContextRequestBody(
+ text="Alice", brain_id="brain-a", max_facts=5, max_passages=2
+ )
+ )
+
+ self.assertEqual(len(response.triples), 1)
+ triple = response.triples[0]
+ self.assertEqual(triple.source_chunk_ids, ["chunk-3"])
+ self.assertEqual(triple.source_session_ids, ["session_3"])
+ self.assertEqual(response.graph_session_ids, ["session_3"])
+ self.assertTrue(
+ any("(session_3)" in line for line in response.text_context.splitlines())
+ )
+
+ async def test_missing_chunk_provenance_omits_session_ids(self):
+ from src.services.api.controllers import retrieve as retrieve_mod
+
+ with _stubbed_adapters(
+ retrieve_mod,
+ [_fact(chunk_ids=["chunk-missing"])],
+ chunks=[],
+ passages=[],
+ ):
+ response = await retrieve_mod.get_context(
+ GetContextRequestBody(
+ text="Alice", brain_id="brain-a", max_facts=5
+ )
+ )
+
+ self.assertEqual(len(response.triples), 1)
+ self.assertEqual(response.triples[0].source_chunk_ids, ["chunk-missing"])
+ self.assertIsNone(response.triples[0].source_session_ids)
+ self.assertIsNone(response.graph_session_ids)
+
+ async def test_instrumentation_still_records_core_stages(self):
+ from src.services.api.controllers import retrieve as retrieve_mod
+
+ with _stubbed_adapters(
+ retrieve_mod,
+ [_fact(chunk_ids=["chunk-3"])],
+ chunks=[
+ TextChunk(
+ id="chunk-3",
+ text="Session id: session_3. Evidence.",
+ )
+ ],
+ passages=[("p1", 0.2, "Session id: session_9. Passage.")],
+ ):
+ response = await retrieve_mod.get_context(
+ GetContextRequestBody(
+ text="Alice",
+ brain_id="brain-a",
+ max_facts=2,
+ profile_stages=True,
+ )
+ )
+
+ self.assertIsNotNone(response.stage_timings)
+ stage_names = {
+ stage["stage"] for stage in response.stage_timings.get("stages", [])
+ }
+ self.assertFalse(any(name.startswith("dossiers") for name in stage_names))
+ self.assertTrue(any(name.startswith("facts") for name in stage_names))
+ self.assertTrue(any(name.startswith("passages") for name in stage_names))
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/tests/test_context_no_dossier.py b/tests/test_context_no_dossier.py
new file mode 100644
index 0000000..2a070e4
--- /dev/null
+++ b/tests/test_context_no_dossier.py
@@ -0,0 +1,144 @@
+import contextlib
+import os
+import sys
+import unittest
+from unittest.mock import MagicMock, patch
+
+
+ENV_DEFAULTS = {
+ "BRAINPAT_TOKEN": "test-token",
+ "MODELS_MODE": "local",
+ "EMBEDDINGS_LOCAL_MODEL": "local-model",
+ "EMBEDDINGS_SMALL_MODEL": "small-model",
+ "EMBEDDING_NODES_DIMENSION": "3",
+ "EMBEDDING_TRIPLETS_DIMENSION": "3",
+ "EMBEDDING_OBSERVATIONS_DIMENSION": "3",
+ "EMBEDDING_DATA_DIMENSION": "3",
+ "EMBEDDING_RELATIONSHIPS_DIMENSION": "3",
+ "REDIS_HOST": "localhost",
+ "REDIS_PORT": "6379",
+ "NEO4J_HOST": "localhost",
+ "NEO4J_PORT": "7687",
+ "NEO4J_USERNAME": "neo4j",
+ "NEO4J_PASSWORD": "password",
+ "MILVUS_HOST": "localhost",
+ "MILVUS_PORT": "19530",
+ "MONGO_CONNECTION_STRING": "mongodb://localhost:27017",
+ "CELERY_WORKER_CONCURRENCY": "1",
+ "OLLAMA_HOST": "localhost",
+ "OLLAMA_PORT": "11434",
+ "OLLAMA_LLM_SMALL_MODEL": "small",
+ "OLLAMA_LLM_LARGE_MODEL": "large",
+ "POSTGRES_HOST": "localhost",
+ "POSTGRES_PORT": "5432",
+ "POSTGRES_USERNAME": "postgres",
+ "POSTGRES_PASSWORD": "postgres",
+}
+for key, value in ENV_DEFAULTS.items():
+ os.environ.setdefault(key, value)
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from src.constants.embeddings import Vector
+from src.constants.kg import Node, Predicate
+from src.services.api.constants.requests import GetContextRequestBody
+from src.utils.nlp.ner import ExtractElementsResponse
+
+
+def _node(uuid: str, name: str, *, labels=None):
+ return Node(
+ uuid=uuid,
+ name=name,
+ labels=labels or ["PERSON"],
+ description=f"desc-{name}",
+ )
+
+
+def _pred(uuid: str, name: str):
+ return Predicate(uuid=uuid, name=name, description=f"desc-{name}")
+
+
+class ContextPathNoDossierTests(unittest.IsolatedAsyncioTestCase):
+ async def test_get_context_does_not_invoke_dossier_retrieval(self):
+ from src.services.api.controllers import retrieve as retrieve_mod
+
+ self.assertFalse(hasattr(retrieve_mod, "EventSynergyRetriever"))
+ self.assertFalse(hasattr(retrieve_mod, "_flatten_match_path"))
+ self.assertFalse(hasattr(retrieve_mod, "_MAX_DOSSIER_ENTITIES"))
+
+ fact = (
+ _node("a", "Alice"),
+ _pred("r1", "ATTENDED"),
+ _node("e1", "Event1", labels=["EVENT"]),
+ _pred("r2", "WITH"),
+ _node("b", "Bob"),
+ )
+ mock_embeddings = MagicMock()
+ mock_embeddings.embed_text.return_value = Vector(
+ id="q", embeddings=[0.1, 0.2, 0.3], metadata={}
+ )
+ mock_vs = MagicMock()
+ mock_vs.search_nodes.return_value = [
+ Vector(
+ id="n1",
+ embeddings=[0.1, 0.2, 0.3],
+ metadata={"uuid": "a", "name": "Alice"},
+ distance=0.1,
+ )
+ ]
+ mock_vs.search_relationships.return_value = []
+ mock_vs.search_data.return_value = []
+ mock_graph = MagicMock()
+ mock_graph.get_event_centric_neighbors.return_value = [fact]
+ mock_data = MagicMock()
+ mock_data.get_last_text_chunks.return_value = []
+ mock_data.get_last_structured_data.return_value = []
+ mock_data.get_text_chunks_by_ids.return_value = ([], None)
+ mock_data.search.return_value = MagicMock(text_chunks=[])
+
+ with contextlib.ExitStack() as stack:
+ stack.enter_context(
+ patch.object(retrieve_mod, "embeddings_adapter", mock_embeddings)
+ )
+ stack.enter_context(patch.object(retrieve_mod, "vector_search", mock_vs))
+ stack.enter_context(patch.object(retrieve_mod, "graph_adapter", mock_graph))
+ stack.enter_context(patch.object(retrieve_mod, "data_adapter", mock_data))
+ stack.enter_context(
+ patch.object(
+ retrieve_mod._entity_extractor,
+ "extract_elements",
+ return_value=ExtractElementsResponse(
+ tokens=[{"text": "Alice", "lemma": "Alice", "pos": "PROPN"}],
+ noun_chunks=[],
+ ),
+ )
+ )
+ synergy = stack.enter_context(
+ patch("src.core.search.entity_info.EventSynergyRetriever")
+ )
+ response = await retrieve_mod.get_context(
+ GetContextRequestBody(
+ text="What did Alice do with Bob?",
+ brain_id="brain-a",
+ max_facts=5,
+ max_passages=2,
+ profile_stages=True,
+ )
+ )
+
+ synergy.assert_not_called()
+ self.assertFalse(
+ any(
+ line.startswith("[dossier:")
+ for line in response.text_context.splitlines()
+ )
+ )
+ stage_names = {
+ stage["stage"] for stage in (response.stage_timings or {}).get("stages", [])
+ }
+ self.assertFalse(any(name.startswith("dossiers") for name in stage_names))
+ self.assertEqual(len(response.triples), 1)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/tests/test_context_retrieval.py b/tests/test_context_retrieval.py
index 0534fcd..b57f106 100644
--- a/tests/test_context_retrieval.py
+++ b/tests/test_context_retrieval.py
@@ -42,7 +42,7 @@ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from src.constants.embeddings import Vector
from src.constants.kg import Node, Predicate
-from src.core.search.entity_info import EventSynergyRetriever, MatchPath
+from src.core.search.entity_info import EventSynergyRetriever
from src.services.api.constants.requests import GetContextRequestBody
from src.utils.nlp.ner import ExtractElementsResponse
@@ -84,16 +84,18 @@ class CollectQueryVariantsTests(unittest.TestCase):
self.assertIn("Alice", variants)
self.assertIn("closest friend", variants)
self.assertIn("charity event", variants)
- noun_only = [
- v
- for v in variants
- if v
- not in {
- "What did Alice do with her closest friend?",
- "Alice",
- }
+ noun_chunks_included = [
+ chunk
+ for chunk in elements.noun_chunks
+ if chunk in variants and chunk != "Alice"
]
- self.assertLessEqual(len(noun_only), retrieve_mod._MAX_NOUN_CHUNKS)
+ self.assertLessEqual(
+ len(noun_chunks_included), retrieve_mod._MAX_NOUN_CHUNKS
+ )
+ self.assertTrue(
+ any("friend" in v.lower() for v in variants[1:]),
+ "enumeration decomposition should add relation-focused subqueries",
+ )
class FormatHelpersTests(unittest.TestCase):
@@ -187,19 +189,6 @@ class GetContextRetrievalTests(unittest.IsolatedAsyncioTestCase):
"extract_elements",
return_value=elements,
),
- patch.object(
- retrieve_mod,
- "EventSynergyRetriever",
- return_value=MagicMock(
- retrieve_matches=MagicMock(
- return_value=MatchPath(
- target_node=None,
- path=(_pred("", ""), None),
- similarity=0.0,
- )
- )
- ),
- ),
):
response = await retrieve_mod.get_context(
GetContextRequestBody(text="Alice", brain_id="brain-a", max_facts=40)
@@ -255,19 +244,6 @@ class GetContextRetrievalTests(unittest.IsolatedAsyncioTestCase):
"extract_elements",
return_value=elements,
),
- patch.object(
- retrieve_mod,
- "EventSynergyRetriever",
- return_value=MagicMock(
- retrieve_matches=MagicMock(
- return_value=MatchPath(
- target_node=None,
- path=(_pred("", ""), None),
- similarity=0.0,
- )
- )
- ),
- ),
):
response = await retrieve_mod.get_context(
GetContextRequestBody(
@@ -332,19 +308,6 @@ class GetContextRetrievalTests(unittest.IsolatedAsyncioTestCase):
"extract_elements",
return_value=elements,
),
- patch.object(
- retrieve_mod,
- "EventSynergyRetriever",
- return_value=MagicMock(
- retrieve_matches=MagicMock(
- return_value=MatchPath(
- target_node=None,
- path=(_pred("", ""), None),
- similarity=0.0,
- )
- )
- ),
- ),
):
response = await retrieve_mod.get_context(
GetContextRequestBody(text="Alice", brain_id="brain-a", max_facts=1)
@@ -352,7 +315,7 @@ class GetContextRetrievalTests(unittest.IsolatedAsyncioTestCase):
self.assertEqual(len(response.triples), 1)
- async def test_dossier_lines_appended_to_text_context(self):
+ async def test_context_path_does_not_emit_dossier_lines(self):
from src.services.api.controllers import retrieve as retrieve_mod
fact = (
@@ -362,21 +325,6 @@ class GetContextRetrievalTests(unittest.IsolatedAsyncioTestCase):
_pred("r2", "WITH"),
_node("b", "Bob"),
)
- dossier = MatchPath(
- target_node=_node("a", "Alice"),
- path=(_pred("d1", "FRIEND_OF"), _node("b", "Bob")),
- similarity=0.9,
- children=[
- MatchPath(
- target_node=_node("a", "Alice"),
- path=(
- _pred("d2", "LIVES_IN"),
- _node("c", "Chicago", labels=["LOCATION"]),
- ),
- similarity=0.8,
- )
- ],
- )
mock_embeddings = MagicMock()
mock_embeddings.embed_text.return_value = Vector(
@@ -401,8 +349,6 @@ class GetContextRetrievalTests(unittest.IsolatedAsyncioTestCase):
tokens=[{"text": "Alice", "lemma": "Alice", "pos": "PROPN"}],
noun_chunks=[],
)
- retriever = MagicMock()
- retriever.retrieve_matches.return_value = dossier
with (
patch.object(retrieve_mod, "embeddings_adapter", mock_embeddings),
@@ -414,7 +360,9 @@ class GetContextRetrievalTests(unittest.IsolatedAsyncioTestCase):
"extract_elements",
return_value=elements,
),
- patch.object(retrieve_mod, "EventSynergyRetriever", return_value=retriever),
+ patch(
+ "src.core.search.entity_info.EventSynergyRetriever"
+ ) as synergy_cls,
):
response = await retrieve_mod.get_context(
GetContextRequestBody(
@@ -423,8 +371,10 @@ class GetContextRetrievalTests(unittest.IsolatedAsyncioTestCase):
)
)
- self.assertIn("[dossier:Alice]", response.text_context)
- self.assertIn("Chicago", response.text_context)
+ synergy_cls.assert_not_called()
+ self.assertFalse(
+ any(line.startswith("[dossier:") for line in response.text_context.splitlines())
+ )
self.assertEqual(len(response.triples), 1)
diff --git a/tests/test_entity_synergy_recency.py b/tests/test_entity_synergy_recency.py
new file mode 100644
index 0000000..38c3311
--- /dev/null
+++ b/tests/test_entity_synergy_recency.py
@@ -0,0 +1,97 @@
+import os
+import sys
+import unittest
+from datetime import datetime, timedelta, timezone
+from unittest.mock import patch
+
+ENV_DEFAULTS = {
+ "BRAINPAT_TOKEN": "test-token",
+ "MODELS_MODE": "local",
+ "EMBEDDINGS_LOCAL_MODEL": "local-model",
+ "EMBEDDINGS_SMALL_MODEL": "small-model",
+ "EMBEDDING_NODES_DIMENSION": "3",
+ "EMBEDDING_TRIPLETS_DIMENSION": "3",
+ "EMBEDDING_OBSERVATIONS_DIMENSION": "3",
+ "EMBEDDING_DATA_DIMENSION": "3",
+ "EMBEDDING_RELATIONSHIPS_DIMENSION": "3",
+ "REDIS_HOST": "localhost",
+ "REDIS_PORT": "6379",
+ "NEO4J_HOST": "localhost",
+ "NEO4J_PORT": "7687",
+ "NEO4J_USERNAME": "neo4j",
+ "NEO4J_PASSWORD": "password",
+ "MILVUS_HOST": "localhost",
+ "MILVUS_PORT": "19530",
+ "MONGO_CONNECTION_STRING": "mongodb://localhost:27017",
+ "CELERY_WORKER_CONCURRENCY": "1",
+ "OLLAMA_HOST": "localhost",
+ "OLLAMA_PORT": "11434",
+ "OLLAMA_LLM_SMALL_MODEL": "small",
+ "OLLAMA_LLM_LARGE_MODEL": "large",
+ "POSTGRES_HOST": "localhost",
+ "POSTGRES_PORT": "5432",
+ "POSTGRES_USERNAME": "postgres",
+ "POSTGRES_PASSWORD": "postgres",
+}
+for key, value in ENV_DEFAULTS.items():
+ os.environ.setdefault(key, value)
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from src.constants.kg import Node
+from src.core.search.entity_info import _recency_score
+
+
+def _node(happened_at=None):
+ return Node.model_construct(
+ uuid="node-1",
+ name="Dinner with Melanie",
+ labels=["EVENT"],
+ happened_at=happened_at,
+ )
+
+
+class RecencyScoreTests(unittest.TestCase):
+ def test_missing_happened_at_is_neutral(self):
+ self.assertEqual(_recency_score(_node()), 1.0)
+
+ def test_stored_day_first_date_decays(self):
+ score = _recency_score(_node("08/05/2023"))
+ self.assertLess(score, 1.0)
+ self.assertGreater(score, 0.0)
+
+ def test_locomo_timestamp_decays(self):
+ self.assertEqual(
+ _recency_score(_node("1:56 pm on 8 May, 2023")),
+ _recency_score(_node("08/05/2023")),
+ )
+
+ def test_older_event_scores_lower_than_newer_event(self):
+ older = _recency_score(_node("08/05/2013"))
+ newer = _recency_score(_node("08/05/2023"))
+ self.assertLess(older, newer)
+ self.assertLess(newer, 1.0)
+
+ def test_today_is_neutral(self):
+ today = datetime.now().strftime("%d/%m/%Y")
+ self.assertEqual(_recency_score(_node(today)), 1.0)
+
+ def test_datetime_happened_at_is_supported(self):
+ self.assertEqual(
+ _recency_score(_node(datetime(2023, 5, 8))),
+ _recency_score(_node("08/05/2023")),
+ )
+
+ def test_offset_aware_datetime_does_not_raise(self):
+ aware = datetime(2023, 5, 8, tzinfo=timezone(timedelta(hours=2)))
+ self.assertLess(_recency_score(_node(aware)), 1.0)
+
+ def test_unparseable_happened_at_falls_back_to_neutral_and_logs(self):
+ with patch("builtins.print") as printed:
+ self.assertEqual(_recency_score(_node("sometime last summer")), 1.0)
+ self.assertTrue(printed.called)
+ self.assertIn("unparseable happened_at", printed.call_args[0][0])
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/tests/test_event_hub_invalidation.py b/tests/test_event_hub_invalidation.py
new file mode 100644
index 0000000..6eafc4d
--- /dev/null
+++ b/tests/test_event_hub_invalidation.py
@@ -0,0 +1,210 @@
+import os
+import sys
+import unittest
+from unittest.mock import patch
+
+
+ENV_DEFAULTS = {
+ "BRAINPAT_TOKEN": "test-token",
+ "MODELS_MODE": "local",
+ "EMBEDDINGS_LOCAL_MODEL": "local-model",
+ "EMBEDDINGS_SMALL_MODEL": "small-model",
+ "EMBEDDING_NODES_DIMENSION": "3",
+ "EMBEDDING_TRIPLETS_DIMENSION": "3",
+ "EMBEDDING_OBSERVATIONS_DIMENSION": "3",
+ "EMBEDDING_DATA_DIMENSION": "3",
+ "EMBEDDING_RELATIONSHIPS_DIMENSION": "3",
+ "REDIS_HOST": "localhost",
+ "REDIS_PORT": "6379",
+ "NEO4J_HOST": "localhost",
+ "NEO4J_PORT": "7687",
+ "NEO4J_USERNAME": "neo4j",
+ "NEO4J_PASSWORD": "password",
+ "MILVUS_HOST": "localhost",
+ "MILVUS_PORT": "19530",
+ "MONGO_CONNECTION_STRING": "mongodb://localhost:27017",
+ "CELERY_WORKER_CONCURRENCY": "1",
+ "OLLAMA_HOST": "localhost",
+ "OLLAMA_PORT": "11434",
+ "OLLAMA_LLM_SMALL_MODEL": "small",
+ "OLLAMA_LLM_LARGE_MODEL": "large",
+ "PIPELINE_MODE": "accurate",
+}
+for key, value in ENV_DEFAULTS.items():
+ os.environ.setdefault(key, value)
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from src.constants.agents import ArchitectAgentEntity, ArchitectAgentRelationship
+from src.constants.kg import Node, Predicate
+
+
+class _FakeGraphAdapter:
+ def __init__(self):
+ self.nodes = {}
+ self.edges = []
+ self.updates = []
+
+ def add_node(self, uuid, name, node_type):
+ self.nodes[uuid] = Node(uuid=uuid, name=name, labels=[node_type])
+ return self.nodes[uuid]
+
+ def add_edge(self, rel_uuid, tail_uuid, name, tip_uuid):
+ predicate = Predicate(uuid=rel_uuid, name=name, description=f"desc-{name}")
+ self.edges.append((tail_uuid, predicate, tip_uuid))
+ return predicate
+
+ def predicate(self, rel_uuid):
+ for _, predicate, _ in self.edges:
+ if predicate.uuid == rel_uuid:
+ return predicate
+ raise KeyError(rel_uuid)
+
+ def get_neighbors(
+ self,
+ nodes,
+ brain_id="default",
+ same_type_only=False,
+ limit=None,
+ of_types=None,
+ ):
+ uuids = [node if isinstance(node, str) else node.uuid for node in nodes]
+ neighbors = {uuid: [] for uuid in uuids}
+ for tail_uuid, predicate, tip_uuid in self.edges:
+ if tail_uuid in neighbors:
+ neighbors[tail_uuid].append(
+ (
+ predicate.model_copy(update={"direction": "out"}),
+ self.nodes[tip_uuid],
+ )
+ )
+ if tip_uuid in neighbors:
+ neighbors[tip_uuid].append(
+ (
+ predicate.model_copy(update={"direction": "in"}),
+ self.nodes[tail_uuid],
+ )
+ )
+ return neighbors
+
+ def update_properties(self, uuid, kind, brain_id="default", new_properties=None):
+ self.updates.append((uuid, kind, dict(new_properties or {})))
+ predicate = self.predicate(uuid)
+ predicate.properties = {
+ **(predicate.properties or {}),
+ **(new_properties or {}),
+ }
+ if (new_properties or {}).get("deprecated"):
+ predicate.deprecated = True
+
+
+def _entity(uuid, name, entity_type):
+ return ArchitectAgentEntity(uuid=uuid, name=name, type=entity_type)
+
+
+def _relationship(rel_uuid, tail, name, tip, valid_at=None):
+ return ArchitectAgentRelationship(
+ uuid=rel_uuid,
+ flow_key="flow-1",
+ tail=tail,
+ name=name,
+ description=f"desc-{name}",
+ tip=tip,
+ properties={"valid_at": valid_at} if valid_at else {},
+ )
+
+
+class EventHubInvalidationTests(unittest.TestCase):
+ def test_second_event_by_same_actor_keeps_both_attributions(self):
+ from src.services.api.controllers import retrieve as retrieve_mod
+ from src.workers.tasks import ingestion as ingestion_mod
+
+ graph = _FakeGraphAdapter()
+ graph.add_node("alice", "Alice", "PERSON")
+ graph.add_node("event-1", "Purchase", "EVENT")
+ graph.add_node("event-2", "Trip", "EVENT")
+ graph.add_edge("rel-1", "alice", "MADE", "event-1")
+ graph.add_edge("rel-2", "alice", "MADE", "event-2")
+
+ relationship = _relationship(
+ "rel-2",
+ _entity("alice", "Alice", "PERSON"),
+ "MADE",
+ _entity("event-2", "Trip", "EVENT"),
+ valid_at="02/02/2024",
+ )
+
+ with patch.object(ingestion_mod, "graph_adapter", graph):
+ ingestion_mod._invalidate_superseded_relationships(
+ relationship, brain_id="brain-a"
+ )
+
+ self.assertEqual(graph.updates, [])
+ self.assertTrue(retrieve_mod._is_currently_valid(graph.predicate("rel-1")))
+ self.assertTrue(retrieve_mod._is_currently_valid(graph.predicate("rel-2")))
+
+ def test_functional_attribute_edge_is_still_superseded(self):
+ from src.services.api.controllers import retrieve as retrieve_mod
+ from src.workers.tasks import ingestion as ingestion_mod
+
+ graph = _FakeGraphAdapter()
+ graph.add_node("alice", "Alice", "PERSON")
+ graph.add_node("nyc", "New York City", "CITY")
+ graph.add_node("sf", "San Francisco", "CITY")
+ graph.add_edge("rel-1", "alice", "LIVES_IN", "nyc")
+ graph.add_edge("rel-2", "alice", "LIVES_IN", "sf")
+
+ relationship = _relationship(
+ "rel-2",
+ _entity("alice", "Alice", "PERSON"),
+ "LIVES_IN",
+ _entity("sf", "San Francisco", "CITY"),
+ valid_at="02/02/2024",
+ )
+
+ with patch.object(ingestion_mod, "graph_adapter", graph):
+ ingestion_mod._invalidate_superseded_relationships(
+ relationship, brain_id="brain-a"
+ )
+
+ self.assertEqual(
+ graph.updates,
+ [
+ (
+ "rel-1",
+ "relationship",
+ {"invalid_at": "02/02/2024", "deprecated": True},
+ )
+ ],
+ )
+ self.assertFalse(retrieve_mod._is_currently_valid(graph.predicate("rel-1")))
+ self.assertTrue(retrieve_mod._is_currently_valid(graph.predicate("rel-2")))
+
+ def test_inbound_edge_on_the_subject_is_not_superseded(self):
+ from src.workers.tasks import ingestion as ingestion_mod
+
+ graph = _FakeGraphAdapter()
+ graph.add_node("alice", "Alice", "PERSON")
+ graph.add_node("bob", "Bob", "PERSON")
+ graph.add_node("carol", "Carol", "PERSON")
+ graph.add_edge("rel-1", "bob", "MANAGES", "alice")
+ graph.add_edge("rel-2", "alice", "MANAGES", "carol")
+
+ relationship = _relationship(
+ "rel-2",
+ _entity("alice", "Alice", "PERSON"),
+ "MANAGES",
+ _entity("carol", "Carol", "PERSON"),
+ valid_at="02/02/2024",
+ )
+
+ with patch.object(ingestion_mod, "graph_adapter", graph):
+ ingestion_mod._invalidate_superseded_relationships(
+ relationship, brain_id="brain-a"
+ )
+
+ self.assertEqual(graph.updates, [])
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/tests/test_graphrag_upgrade_helpers.py b/tests/test_graphrag_upgrade_helpers.py
index 7a0ae65..60a5d5c 100644
--- a/tests/test_graphrag_upgrade_helpers.py
+++ b/tests/test_graphrag_upgrade_helpers.py
@@ -34,7 +34,11 @@ for key, value in ENV_DEFAULTS.items():
os.environ.setdefault(key, value)
from src.core.saving.identity import merge_source_chunk_ids, stamp_provenance
-from src.core.search.fact_filter import reciprocal_rank_fusion
+from src.core.search.fact_filter import (
+ filter_relevant_facts,
+ personalized_pagerank,
+ reciprocal_rank_fusion,
+)
from src.utils.dates import normalize_date_string, resolve_relative_date
@@ -91,6 +95,32 @@ class RrfTests(unittest.TestCase):
self.assertIn("b", ranked[:2])
+class FactFilterTests(unittest.TestCase):
+ def test_fallback_preserves_all_candidates(self):
+ keep = filter_relevant_facts(
+ "What books has Melanie read?",
+ [f"fact-{i}" for i in range(12)],
+ llm_adapter=None,
+ max_keep=4,
+ )
+ self.assertEqual(keep, list(range(12)))
+
+
+class PersonalizedPageRankTests(unittest.TestCase):
+ def test_ppr_concentrates_on_seed_neighborhood(self):
+ adjacency = {
+ "seed": ["a", "b"],
+ "a": ["seed", "c"],
+ "b": ["seed"],
+ "c": ["a"],
+ "far": ["x"],
+ "x": ["far"],
+ }
+ scores = personalized_pagerank(adjacency, {"seed": 1.0}, iterations=30)
+ self.assertGreater(scores["seed"], scores["far"])
+ self.assertGreater(scores["a"], scores["far"])
+
+
class ScoutChunkingTests(unittest.TestCase):
def test_chunk_text_splits_long_input(self):
from src.utils.text_chunking import chunk_text
diff --git a/tests/test_phase_b_event_graphrag.py b/tests/test_phase_b_event_graphrag.py
new file mode 100644
index 0000000..7ee14ac
--- /dev/null
+++ b/tests/test_phase_b_event_graphrag.py
@@ -0,0 +1,409 @@
+import os
+import sys
+import time
+import unittest
+
+ENV_DEFAULTS = {
+ "BRAINPAT_TOKEN": "test-token",
+ "MODELS_MODE": "local",
+ "EMBEDDINGS_LOCAL_MODEL": "local-model",
+ "EMBEDDINGS_SMALL_MODEL": "small-model",
+ "EMBEDDING_NODES_DIMENSION": "3",
+ "EMBEDDING_TRIPLETS_DIMENSION": "3",
+ "EMBEDDING_OBSERVATIONS_DIMENSION": "3",
+ "EMBEDDING_DATA_DIMENSION": "3",
+ "EMBEDDING_RELATIONSHIPS_DIMENSION": "3",
+ "REDIS_HOST": "localhost",
+ "REDIS_PORT": "6379",
+ "NEO4J_HOST": "localhost",
+ "NEO4J_PORT": "7687",
+ "NEO4J_USERNAME": "neo4j",
+ "NEO4J_PASSWORD": "password",
+ "MILVUS_HOST": "localhost",
+ "MILVUS_PORT": "19530",
+ "MONGO_CONNECTION_STRING": "mongodb://localhost:27017",
+ "CELERY_WORKER_CONCURRENCY": "1",
+ "OLLAMA_HOST": "localhost",
+ "OLLAMA_PORT": "11434",
+ "OLLAMA_LLM_SMALL_MODEL": "small",
+ "OLLAMA_LLM_LARGE_MODEL": "large",
+ "POSTGRES_HOST": "localhost",
+ "POSTGRES_PORT": "5432",
+ "POSTGRES_USERNAME": "postgres",
+ "POSTGRES_PASSWORD": "postgres",
+}
+for key, value in ENV_DEFAULTS.items():
+ os.environ.setdefault(key, value)
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from src.constants.kg import Node, Predicate
+from src.core.search.fact_filter import personalized_pagerank
+from src.services.api.controllers import retrieve as retrieve_mod
+
+
+def _node(uuid: str, name: str, *, labels=None, happened_at=None):
+ return Node(
+ uuid=uuid,
+ name=name,
+ labels=labels or ["PERSON"],
+ description=f"desc-{name}",
+ properties={},
+ happened_at=happened_at,
+ )
+
+
+def _pred(uuid: str, name: str, *, flow_key: str = "", invalid=False, deprecated=False):
+ props = {"invalid_at": "01/01/2020"} if invalid else {}
+ return Predicate(
+ uuid=uuid,
+ name=name,
+ description=f"desc-{name}",
+ flow_key=flow_key or None,
+ properties=props,
+ deprecated=deprecated,
+ )
+
+
+def _candidate(
+ *,
+ score: float,
+ r_uuid: str,
+ r2_uuid: str,
+ flow_key: str = "flow",
+ n_uuid: str = "n",
+ m_uuid: str = "m",
+ b_uuid: str = "b",
+ r_name: str = "MADE",
+ r2_name: str = "TARGETED",
+ n_labels=None,
+ m_labels=None,
+ b_labels=None,
+ session_ids=None,
+ happened_at=None,
+ entity: str = "q",
+):
+ n = _node(n_uuid, n_uuid, labels=n_labels or ["PERSON"])
+ r = _pred(r_uuid, r_name, flow_key=flow_key)
+ m = _node(
+ m_uuid,
+ m_uuid,
+ labels=m_labels or ["EVENT"],
+ happened_at=happened_at,
+ )
+ r2 = _pred(r2_uuid, r2_name, flow_key=flow_key)
+ b = _node(b_uuid, b_uuid, labels=b_labels or ["PERSON"])
+ return {
+ "identified_entity": entity,
+ "triple": (n, r, m, r2, b),
+ "score": score,
+ "key": (r_uuid, r2_uuid),
+ "text": retrieve_mod._format_event_fact(n, r, m, r2, b),
+ "chunk_ids": [],
+ "session_ids": list(session_ids or []),
+ }
+
+
+class DiversificationTests(unittest.TestCase):
+ def test_coverage_beats_pure_truncation_under_max_facts(self):
+ ranked = []
+ for i in range(8):
+ ranked.append(
+ _candidate(
+ score=0.01 * i,
+ r_uuid=f"r-hub0-{i}",
+ r2_uuid=f"r2-hub0-{i}",
+ flow_key="hub-0",
+ m_uuid="event-0",
+ session_ids=["session_1"],
+ )
+ )
+ for i in range(4):
+ ranked.append(
+ _candidate(
+ score=0.5 + 0.01 * i,
+ r_uuid=f"r-hub{i+1}",
+ r2_uuid=f"r2-hub{i+1}",
+ flow_key=f"hub-{i+1}",
+ m_uuid=f"event-{i+1}",
+ session_ids=[f"session_{i+2}"],
+ )
+ )
+ control = ranked[:5]
+ diversified = retrieve_mod._diversify_facts(ranked, max_facts=5)
+ control_hubs = {retrieve_mod._event_hub_id(c) for c in control}
+ control_sessions = {s for c in control for s in c["session_ids"]}
+ div_hubs = {retrieve_mod._event_hub_id(c) for c in diversified}
+ div_sessions = {s for c in diversified for s in c["session_ids"]}
+ self.assertGreaterEqual(len(div_hubs), len(control_hubs))
+ self.assertGreater(len(div_sessions), len(control_sessions))
+ self.assertEqual(len(diversified), 5)
+
+ def test_diversify_is_deterministic(self):
+ ranked = [
+ _candidate(
+ score=0.2,
+ r_uuid=f"r{i}",
+ r2_uuid=f"x{i}",
+ flow_key=f"h{i % 3}",
+ m_uuid=f"e{i % 3}",
+ session_ids=[f"session_{(i % 4) + 1}"],
+ )
+ for i in range(12)
+ ]
+ a = retrieve_mod._diversify_facts(ranked, max_facts=6)
+ b = retrieve_mod._diversify_facts(list(reversed(ranked)), max_facts=6)
+ self.assertEqual([c["key"] for c in a], [c["key"] for c in b])
+
+ def test_diversify_max_facts_zero(self):
+ ranked = [_candidate(score=0.1, r_uuid="r", r2_uuid="x")]
+ self.assertEqual(retrieve_mod._diversify_facts(ranked, max_facts=0), [])
+
+ def test_diversify_latency_under_budget(self):
+ ranked = [
+ _candidate(
+ score=0.001 * i,
+ r_uuid=f"r{i}",
+ r2_uuid=f"x{i}",
+ flow_key=f"h{i % 40}",
+ m_uuid=f"e{i % 40}",
+ session_ids=[f"session_{(i % 20) + 1}"],
+ )
+ for i in range(200)
+ ]
+ retrieve_mod._diversify_facts(ranked, max_facts=50)
+ samples = []
+ for _ in range(5):
+ start = time.perf_counter()
+ out = retrieve_mod._diversify_facts(ranked, max_facts=50)
+ samples.append((time.perf_counter() - start) * 1000)
+ self.assertEqual(len(out), 50)
+ self.assertLess(min(samples), 25.0)
+
+
+class HubCompletenessTests(unittest.TestCase):
+ def test_spine_scores_above_context_only(self):
+ spine = _candidate(
+ score=0.5,
+ r_uuid="r1",
+ r2_uuid="r2",
+ r_name="MADE",
+ r2_name="TARGETED",
+ m_uuid="e1",
+ )
+ context = _candidate(
+ score=0.5,
+ r_uuid="r3",
+ r2_uuid="r4",
+ r_name="RELATED",
+ r2_name="OCCURRED_WITHIN",
+ m_uuid="e2",
+ b_uuid="ctx",
+ b_labels=["CONTEXT"],
+ )
+ self.assertGreater(
+ retrieve_mod._hub_completeness_score(spine),
+ retrieve_mod._hub_completeness_score(context),
+ )
+
+ def test_completeness_prefers_full_triangle_when_scores_tied(self):
+ weak = _candidate(
+ score=0.4,
+ r_uuid="rw",
+ r2_uuid="rw2",
+ r_name="MENTIONED",
+ r2_name="NEAR",
+ m_uuid="ew",
+ m_labels=["THING"],
+ )
+ strong = _candidate(
+ score=0.4,
+ r_uuid="rs",
+ r2_uuid="rs2",
+ r_name="MADE",
+ r2_name="TARGETED",
+ m_uuid="es",
+ )
+ ranked = retrieve_mod._rank_facts_with_completeness([weak, strong])
+ self.assertEqual(ranked[0]["key"], ("rs", "rs2"))
+
+ def test_complementary_legs_kept_under_budget(self):
+ spine = _candidate(
+ score=0.1,
+ r_uuid="r-spine",
+ r2_uuid="r2-spine",
+ flow_key="same-hub",
+ m_uuid="event-same",
+ r_name="MADE",
+ r2_name="TARGETED",
+ session_ids=["session_1"],
+ )
+ context = _candidate(
+ score=0.11,
+ r_uuid="r-ctx",
+ r2_uuid="r2-ctx",
+ flow_key="same-hub",
+ m_uuid="event-same",
+ r_name="MADE",
+ r2_name="OCCURRED_WITHIN",
+ b_uuid="place",
+ b_labels=["LOCATION"],
+ session_ids=["session_1"],
+ )
+ other = _candidate(
+ score=0.12,
+ r_uuid="r-other",
+ r2_uuid="r2-other",
+ flow_key="other-hub",
+ m_uuid="event-other",
+ session_ids=["session_2"],
+ )
+ noise = [
+ _candidate(
+ score=0.8,
+ r_uuid=f"rn{i}",
+ r2_uuid=f"xn{i}",
+ flow_key="noise-hub",
+ m_uuid="event-noise",
+ session_ids=["session_1"],
+ )
+ for i in range(5)
+ ]
+ curated = retrieve_mod._diversify_facts(
+ [spine, context, other] + noise, max_facts=3
+ )
+ keys = {c["key"] for c in curated}
+ self.assertIn(("r-spine", "r2-spine"), keys)
+ self.assertIn(("r-ctx", "r2-ctx"), keys)
+
+
+class TypedPprTests(unittest.TestCase):
+ def test_weighted_ppr_accepts_edge_weights(self):
+ adjacency = {
+ "seed": [("spine", 1.0), ("ctx", 0.2)],
+ "spine": [("seed", 1.0)],
+ "ctx": [("seed", 0.2)],
+ "far": [("x", 1.0)],
+ "x": [("far", 1.0)],
+ }
+ scores = personalized_pagerank(adjacency, {"seed": 1.0}, iterations=40)
+ self.assertGreater(scores["spine"], scores["ctx"])
+ self.assertGreater(scores["seed"], scores["far"])
+
+ def test_adjacency_skips_invalid_legs(self):
+ valid_n = _node("n1", "Alice")
+ valid_r = _pred("r1", "MADE", flow_key="f1")
+ event = _node("e1", "Party", labels=["EVENT"])
+ valid_r2 = _pred("r2", "TARGETED", flow_key="f1")
+ obj = _node("b1", "Bob")
+ bad_r = _pred("rb", "MADE", flow_key="f2", invalid=True)
+ bad_r2 = _pred("rb2", "TARGETED", flow_key="f2")
+ neighbors = [
+ (valid_n, valid_r, event, valid_r2, obj),
+ (valid_n, bad_r, event, bad_r2, obj),
+ ]
+
+ class _Graph:
+ def get_event_centric_neighbors(self, seed_uuids, brain_id="default"):
+ return neighbors
+
+ original = retrieve_mod.graph_adapter
+ retrieve_mod.graph_adapter = _Graph()
+ try:
+ adj = retrieve_mod._build_adjacency_from_seeds(["n1"], "brain")
+ finally:
+ retrieve_mod.graph_adapter = original
+ self.assertIn("n1", adj)
+ self.assertIn("e1", adj)
+ edge_targets = {dst for dst, _w in adj["n1"]}
+ self.assertIn("e1", edge_targets)
+
+ def test_typed_adjacency_weights_spine_over_context(self):
+ actor = _node("n1", "Alice")
+ made = _pred("r1", "MADE", flow_key="f1")
+ event = _node("e1", "Party", labels=["EVENT"])
+ targeted = _pred("r2", "TARGETED", flow_key="f1")
+ obj = _node("b1", "Bob")
+ occurred = _pred("r3", "OCCURRED_WITHIN", flow_key="f1")
+ place = _node("p1", "Park", labels=["LOCATION"])
+ neighbors = [
+ (actor, made, event, targeted, obj),
+ (actor, made, event, occurred, place),
+ ]
+
+ class _Graph:
+ def get_event_centric_neighbors(self, seed_uuids, brain_id="default"):
+ return neighbors
+
+ original = retrieve_mod.graph_adapter
+ retrieve_mod.graph_adapter = _Graph()
+ try:
+ adj = retrieve_mod._build_adjacency_from_seeds(["n1", "e1"], "brain")
+ finally:
+ retrieve_mod.graph_adapter = original
+ e_out = {dst: w for dst, w in adj["e1"]}
+ self.assertGreater(e_out.get("b1", 0), e_out.get("p1", 0))
+
+ def test_typed_ppr_reorder_deterministic(self):
+ a = _candidate(score=0.3, r_uuid="ra", r2_uuid="xa", m_uuid="ea", n_uuid="na")
+ b = _candidate(score=0.3, r_uuid="rb", r2_uuid="xb", m_uuid="eb", n_uuid="nb")
+ ppr = {"na": 0.2, "ea": 0.5, "nb": 0.2, "eb": 0.5}
+ forward = sorted([a, b], key=lambda c: retrieve_mod._ppr_rank_key(c, ppr))
+ reverse = sorted([b, a], key=lambda c: retrieve_mod._ppr_rank_key(c, ppr))
+ self.assertEqual([c["key"] for c in forward], [c["key"] for c in reverse])
+
+
+class PathFormatTests(unittest.TestCase):
+ def test_format_labels_actor_event_target(self):
+ n = _node("a", "Alice")
+ r = _pred("r1", "MADE")
+ m = _node("e", "Wedding", happened_at="2024-05-01", labels=["EVENT"])
+ r2 = _pred("r2", "TARGETED")
+ b = _node("b", "Bob")
+ line = retrieve_mod._format_event_fact(n, r, m, r2, b)
+ self.assertIn("Actor:", line)
+ self.assertIn("Event:", line)
+ self.assertIn("Target:", line)
+ self.assertIn("@2024-05-01", line)
+ self.assertNotIn("hub→hub", line.lower())
+
+ def test_format_labels_context_for_occurred_within(self):
+ n = _node("a", "Alice")
+ r = _pred("r1", "MADE")
+ m = _node("e", "Picnic", labels=["EVENT"])
+ r2 = _pred("r2", "OCCURRED_WITHIN")
+ b = _node("p", "Park", labels=["LOCATION"])
+ line = retrieve_mod._format_event_fact(n, r, m, r2, b)
+ self.assertIn("Context:", line)
+ self.assertNotIn("Target:", line)
+
+
+class TemporalConflictMetaTests(unittest.TestCase):
+ def test_recency_prefer_and_emit_conflicts(self):
+ older = _candidate(
+ score=0.2,
+ r_uuid="ro",
+ r2_uuid="xo",
+ flow_key="h-old",
+ m_uuid="e-old",
+ n_uuid="alice",
+ happened_at="01/01/2020",
+ )
+ newer = _candidate(
+ score=0.2,
+ r_uuid="rn",
+ r2_uuid="xn",
+ flow_key="h-new",
+ m_uuid="e-new",
+ n_uuid="alice",
+ happened_at="01/01/2024",
+ )
+ ranked = retrieve_mod._rank_facts_with_completeness([older, newer])
+ self.assertEqual(ranked[0]["key"], ("rn", "xn"))
+ conflicts = retrieve_mod._temporal_conflict_meta([older, newer])
+ self.assertTrue(conflicts)
+ self.assertEqual(conflicts[0]["entity_uuid"], "alice")
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/tests/test_phase_c_hub_bridges.py b/tests/test_phase_c_hub_bridges.py
new file mode 100644
index 0000000..0c66865
--- /dev/null
+++ b/tests/test_phase_c_hub_bridges.py
@@ -0,0 +1,780 @@
+import os
+import sys
+import unittest
+from unittest.mock import MagicMock, patch
+
+ENV_DEFAULTS = {
+ "BRAINPAT_TOKEN": "test-token",
+ "MODELS_MODE": "local",
+ "EMBEDDINGS_LOCAL_MODEL": "local-model",
+ "EMBEDDINGS_SMALL_MODEL": "small-model",
+ "EMBEDDING_NODES_DIMENSION": "3",
+ "EMBEDDING_TRIPLETS_DIMENSION": "3",
+ "EMBEDDING_OBSERVATIONS_DIMENSION": "3",
+ "EMBEDDING_DATA_DIMENSION": "3",
+ "EMBEDDING_RELATIONSHIPS_DIMENSION": "3",
+ "REDIS_HOST": "localhost",
+ "REDIS_PORT": "6379",
+ "NEO4J_HOST": "localhost",
+ "NEO4J_PORT": "7687",
+ "NEO4J_USERNAME": "neo4j",
+ "NEO4J_PASSWORD": "password",
+ "MILVUS_HOST": "localhost",
+ "MILVUS_PORT": "19530",
+ "MONGO_CONNECTION_STRING": "mongodb://localhost:27017",
+ "CELERY_WORKER_CONCURRENCY": "1",
+ "OLLAMA_HOST": "localhost",
+ "OLLAMA_PORT": "11434",
+ "OLLAMA_LLM_SMALL_MODEL": "small",
+ "OLLAMA_LLM_LARGE_MODEL": "large",
+ "POSTGRES_HOST": "localhost",
+ "POSTGRES_PORT": "5432",
+ "POSTGRES_USERNAME": "postgres",
+ "POSTGRES_PASSWORD": "postgres",
+}
+for key, value in ENV_DEFAULTS.items():
+ os.environ.setdefault(key, value)
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from src.constants.kg import Node, Predicate
+from src.core.saving.hub_bridges import (
+ HubBridge,
+ bridges_from_memberships,
+ canonical_event_pair,
+ entity_event_memberships,
+ select_bridge_neighbors,
+)
+from src.services.api.controllers import retrieve as retrieve_mod
+
+
+def _node(uuid: str, name: str, *, labels=None):
+ return Node(
+ uuid=uuid,
+ name=name,
+ labels=labels or ["PERSON"],
+ description=f"desc-{name}",
+ properties={},
+ )
+
+
+def _pred(uuid: str, name: str, *, flow_key: str = ""):
+ return Predicate(
+ uuid=uuid,
+ name=name,
+ description=f"desc-{name}",
+ flow_key=flow_key or None,
+ properties={},
+ )
+
+
+def _candidate(
+ *,
+ score: float,
+ r_uuid: str,
+ r2_uuid: str,
+ flow_key: str,
+ m_uuid: str,
+ n_uuid: str = "n",
+ b_uuid: str = "b",
+):
+ n = _node(n_uuid, n_uuid)
+ r = _pred(r_uuid, "MADE", flow_key=flow_key)
+ m = _node(m_uuid, m_uuid, labels=["EVENT"])
+ r2 = _pred(r2_uuid, "TARGETED", flow_key=flow_key)
+ b = _node(b_uuid, b_uuid)
+ return {
+ "identified_entity": "q",
+ "triple": (n, r, m, r2, b),
+ "score": score,
+ "key": (r_uuid, r2_uuid),
+ "text": f"fact-{m_uuid}",
+ "chunk_ids": [],
+ }
+
+
+class HubBridgeIndexTests(unittest.TestCase):
+ def test_canonical_pair_orders_lexicographically(self):
+ self.assertEqual(canonical_event_pair("b", "a"), ("a", "b"))
+
+ def test_build_bridges_from_shared_actor(self):
+ rows = [
+ ("event-1", "melanie", "Melanie", "MADE"),
+ ("event-2", "melanie", "Melanie", "MADE"),
+ ("event-1", "park", "Park", "OCCURRED_WITHIN"),
+ ("event-2", "park", "Park", "OCCURRED_WITHIN"),
+ ("event-3", "jon", "Jon", "TARGETED"),
+ ]
+ membership = entity_event_memberships(rows)
+ bridges = bridges_from_memberships(membership)
+ self.assertEqual(len(bridges), 1)
+ self.assertEqual(bridges[0].event_a, "event-1")
+ self.assertEqual(bridges[0].event_b, "event-2")
+ self.assertEqual(bridges[0].shared_entity, "melanie")
+
+ def test_rebuild_idempotent_membership(self):
+ rows = [
+ ("e2", "ent", "Ent", "MADE"),
+ ("e1", "ent", "Ent", "TARGETED"),
+ ("e2", "ent", "Ent", "MADE"),
+ ]
+ first = bridges_from_memberships(entity_event_memberships(rows))
+ second = bridges_from_memberships(entity_event_memberships(rows))
+ self.assertEqual(first, second)
+
+ def test_select_bridge_neighbors_respects_max_and_determinism(self):
+ bridges = [
+ HubBridge("e1", "e2", "a", "A", 1.0),
+ HubBridge("e1", "e3", "b", "B", 2.0),
+ HubBridge("e1", "e4", "c", "C", 1.5),
+ ]
+ a = select_bridge_neighbors(["e1"], bridges, max_per_hub=2)
+ b = select_bridge_neighbors(["e1"], bridges, max_per_hub=2)
+ self.assertEqual(a, b)
+ self.assertEqual([n for n, _ in a], ["e3", "e4"])
+
+ def test_prefer_bridge_to_novel_source_session(self):
+ bridges = [
+ HubBridge("e1", "e_same", "shared", "Shared", 2.0),
+ HubBridge("e1", "e_novel", "shared", "Shared", 1.0),
+ ]
+ chosen = select_bridge_neighbors(
+ ["e1"],
+ bridges,
+ max_per_hub=1,
+ seed_sessions={"session_1"},
+ hub_sessions={
+ "e_same": {"session_1"},
+ "e_novel": {"session_5"},
+ },
+ )
+ self.assertEqual([n for n, _ in chosen], ["e_novel"])
+
+ def test_novel_session_ranking_is_deterministic(self):
+ bridges = [
+ HubBridge("e1", "e_a", "ent", "Ent", 1.0),
+ HubBridge("e1", "e_b", "ent", "Ent", 1.0),
+ HubBridge("e1", "e_c", "ent", "Ent", 1.0),
+ ]
+ kwargs = dict(
+ seed_sessions={"session_1"},
+ hub_sessions={
+ "e_a": {"session_1"},
+ "e_b": {"session_5"},
+ "e_c": {"session_1", "session_7"},
+ },
+ max_per_hub=2,
+ )
+ a = select_bridge_neighbors(["e1"], bridges, **kwargs)
+ b = select_bridge_neighbors(["e1"], bridges, **kwargs)
+ self.assertEqual(a, b)
+ self.assertEqual([n for n, _ in a], ["e_b", "e_c"])
+
+
+class BridgeExpandTests(unittest.TestCase):
+ def test_one_bridge_expand_adds_cross_event_candidates(self):
+ seed = _candidate(
+ score=0.1,
+ r_uuid="r1",
+ r2_uuid="r2",
+ flow_key="f1",
+ m_uuid="event-seed",
+ n_uuid="actor",
+ b_uuid="obj1",
+ )
+ bridged_n = _node("actor2", "actor2")
+ bridged_r = _pred("r3", "MADE", flow_key="f2")
+ bridged_m = _node("event-bridge", "event-bridge", labels=["EVENT"])
+ bridged_r2 = _pred("r4", "TARGETED", flow_key="f2")
+ bridged_b = _node("obj2", "obj2")
+ bridge = HubBridge(
+ event_a="event-bridge",
+ event_b="event-seed",
+ shared_entity="actor",
+ shared_entity_name="actor",
+ weight=1.0,
+ )
+
+ mock_adapter = MagicMock()
+ mock_adapter.get_hub_bridges.return_value = [bridge]
+ mock_adapter.get_event_hub_facts.return_value = [
+ (bridged_n, bridged_r, bridged_m, bridged_r2, bridged_b)
+ ]
+
+ with patch.object(retrieve_mod, "graph_adapter", mock_adapter):
+ expanded, paths = retrieve_mod._expand_cross_event_bridges(
+ [seed],
+ "brain",
+ max_per_hub=3,
+ )
+
+ self.assertEqual(len(expanded), 2)
+ hubs = {retrieve_mod._event_hub_id(c) for c in expanded}
+ self.assertEqual(hubs, {"event-seed", "event-bridge"})
+ self.assertEqual(len(paths), 1)
+ self.assertEqual(sorted(paths[0]["hubs"]), ["event-bridge", "event-seed"])
+
+ def test_paths_for_curated_attaches_legs(self):
+ seed = _candidate(
+ score=0.1,
+ r_uuid="r1",
+ r2_uuid="r2",
+ flow_key="f1",
+ m_uuid="event-seed",
+ )
+ bridged = _candidate(
+ score=0.2,
+ r_uuid="r3",
+ r2_uuid="r4",
+ flow_key="f2",
+ m_uuid="event-bridge",
+ n_uuid="actor2",
+ b_uuid="obj2",
+ )
+ path_meta = [
+ {
+ "hubs": ["event-bridge", "event-seed"],
+ "shared_entity": "actor",
+ "shared_entity_name": "actor",
+ "weight": 1.0,
+ }
+ ]
+ kept = retrieve_mod._paths_for_curated(path_meta, [seed, bridged])
+ self.assertEqual(len(kept), 1)
+ self.assertEqual(kept[0]["legs"], ["fact-event-bridge", "fact-event-seed"])
+ dropped = retrieve_mod._paths_for_curated(path_meta, [seed])
+ self.assertEqual(dropped, [])
+
+ def test_bridge_expand_disabled_when_zero(self):
+ seed = _candidate(
+ score=0.1,
+ r_uuid="r1",
+ r2_uuid="r2",
+ flow_key="f1",
+ m_uuid="event-seed",
+ )
+ expanded, paths = retrieve_mod._expand_cross_event_bridges(
+ [seed],
+ "brain",
+ max_per_hub=0,
+ )
+ self.assertEqual(expanded, [seed])
+ self.assertEqual(paths, [])
+
+ def test_diversify_after_expand_respects_max_facts(self):
+ candidates = [
+ _candidate(
+ score=0.01 * i,
+ r_uuid=f"r{i}",
+ r2_uuid=f"rr{i}",
+ flow_key=f"f{i}",
+ m_uuid=f"event-{i}",
+ n_uuid=f"n{i}",
+ b_uuid=f"b{i}",
+ )
+ for i in range(8)
+ ]
+ curated = retrieve_mod._diversify_facts(candidates, max_facts=3)
+ self.assertEqual(len(curated), 3)
+
+ def test_expand_then_diversify_deterministic(self):
+ seed = _candidate(
+ score=0.2,
+ r_uuid="r1",
+ r2_uuid="r2",
+ flow_key="f1",
+ m_uuid="event-seed",
+ )
+ bridged = []
+ for i in range(3):
+ bridged.append(
+ (
+ _node(f"n{i}", f"n{i}"),
+ _pred(f"br{i}", "MADE", flow_key=f"bf{i}"),
+ _node(f"event-b{i}", f"event-b{i}", labels=["EVENT"]),
+ _pred(f"brr{i}", "TARGETED", flow_key=f"bf{i}"),
+ _node(f"bb{i}", f"bb{i}"),
+ )
+ )
+ bridges = [
+ HubBridge("event-b0", "event-seed", "shared", "Shared", 1.0),
+ HubBridge("event-b1", "event-seed", "shared", "Shared", 1.0),
+ HubBridge("event-b2", "event-seed", "shared", "Shared", 1.0),
+ ]
+ mock_adapter = MagicMock()
+ mock_adapter.get_hub_bridges.return_value = bridges
+ mock_adapter.get_event_hub_facts.return_value = bridged
+
+ with patch.object(retrieve_mod, "graph_adapter", mock_adapter):
+ a, _ = retrieve_mod._expand_cross_event_bridges(
+ [seed], "brain", max_per_hub=2
+ )
+ b, _ = retrieve_mod._expand_cross_event_bridges(
+ [seed], "brain", max_per_hub=2
+ )
+ self.assertEqual(
+ [c["key"] for c in a],
+ [c["key"] for c in b],
+ )
+ curated_a = retrieve_mod._diversify_facts(
+ retrieve_mod._rank_facts_with_completeness(a), max_facts=2
+ )
+ curated_b = retrieve_mod._diversify_facts(
+ retrieve_mod._rank_facts_with_completeness(b), max_facts=2
+ )
+ self.assertEqual(
+ [c["key"] for c in curated_a],
+ [c["key"] for c in curated_b],
+ )
+ self.assertEqual(len(curated_a), 2)
+
+
+ def test_expand_selects_by_weight_without_session_pool(self):
+ seed = _candidate(
+ score=0.1,
+ r_uuid="r1",
+ r2_uuid="r2",
+ flow_key="f1",
+ m_uuid="event-seed",
+ )
+ seed["session_ids"] = ["session_1"]
+ seed["chunk_ids"] = ["chunk-seed"]
+
+ same_n = _node("n-same", "n-same")
+ same_r = _pred("rs", "MADE", flow_key="fs")
+ same_m = _node("event-same", "event-same", labels=["EVENT"])
+ same_r2 = _pred("rrs", "TARGETED", flow_key="fs")
+ same_b = _node("b-same", "b-same")
+ for item in (same_n, same_r, same_m, same_r2, same_b):
+ item.properties = {"source_chunk_ids": ["chunk-same"]}
+
+ novel_n = _node("n-novel", "n-novel")
+ novel_r = _pred("rn", "MADE", flow_key="fn")
+ novel_m = _node("event-novel", "event-novel", labels=["EVENT"])
+ novel_r2 = _pred("rrn", "TARGETED", flow_key="fn")
+ novel_b = _node("b-novel", "b-novel")
+ for item in (novel_n, novel_r, novel_m, novel_r2, novel_b):
+ item.properties = {"source_chunk_ids": ["chunk-novel"]}
+
+ bridges = [
+ HubBridge("event-same", "event-seed", "shared", "Shared", 2.0),
+ HubBridge("event-novel", "event-seed", "shared", "Shared", 1.0),
+ ]
+ mock_adapter = MagicMock()
+ mock_adapter.get_hub_bridges.return_value = bridges
+ mock_adapter.get_event_hub_facts.return_value = [
+ (same_n, same_r, same_m, same_r2, same_b),
+ (novel_n, novel_r, novel_m, novel_r2, novel_b),
+ ]
+
+ chunk_sessions = {
+ "chunk-seed": ["session_1"],
+ "chunk-same": ["session_1"],
+ "chunk-novel": ["session_5"],
+ }
+
+ with patch.object(retrieve_mod, "graph_adapter", mock_adapter):
+ with patch.object(
+ retrieve_mod,
+ "_resolve_chunk_sessions",
+ side_effect=lambda ids, _brain: {
+ cid: chunk_sessions[cid]
+ for cid in ids
+ if cid in chunk_sessions
+ },
+ ):
+ expanded, _paths = retrieve_mod._expand_cross_event_bridges(
+ [seed],
+ "brain",
+ max_per_hub=1,
+ )
+
+ hubs = {retrieve_mod._event_hub_id(c) for c in expanded}
+ self.assertIn("event-seed", hubs)
+ self.assertIn("event-same", hubs)
+ self.assertNotIn("event-novel", hubs)
+ mock_adapter.get_event_hub_facts.assert_called_once()
+ fetched = mock_adapter.get_event_hub_facts.call_args[0][0]
+ self.assertEqual(fetched, ["event-same"])
+
+ def test_reserved_slots_keep_novel_bridge_session_under_max_facts(self):
+ seeds = [
+ _candidate(
+ score=0.01 * i,
+ r_uuid=f"rs{i}",
+ r2_uuid=f"rrs{i}",
+ flow_key=f"fs{i}",
+ m_uuid=f"event-seed-{i}",
+ n_uuid=f"ns{i}",
+ b_uuid=f"bs{i}",
+ )
+ for i in range(8)
+ ]
+ for seed in seeds:
+ seed["session_ids"] = ["session_1"]
+
+ bridges = [
+ _candidate(
+ score=5.0 + 0.01 * i,
+ r_uuid=f"rb{i}",
+ r2_uuid=f"rrb{i}",
+ flow_key=f"fb{i}",
+ m_uuid=f"event-bridge-{i}",
+ n_uuid=f"nb{i}",
+ b_uuid=f"bb{i}",
+ )
+ for i in range(3)
+ ]
+ for bridge in bridges:
+ bridge["session_ids"] = ["session_5"]
+ bridge["bridge"] = {
+ "from_hub": "event-seed-0",
+ "to_hub": bridge["triple"][2].uuid,
+ "shared_entity": "shared",
+ "shared_entity_name": "Shared",
+ }
+
+ ranked = seeds + bridges
+ max_facts = 5
+
+ pool = [retrieve_mod._prepare_diversify_item(c) for c in ranked]
+ pool.sort(key=lambda item: (-item["relevance"], item["tie"]))
+ without_reserve: list[dict] = []
+ selected_meta: list[dict] = []
+ selected_hubs: set[str] = set()
+ selected_sessions: set[str] = set()
+ while len(without_reserve) < max_facts and pool:
+ idx = retrieve_mod._mmr_pick_index(
+ pool, selected_meta, selected_hubs, selected_sessions
+ )
+ chosen = pool.pop(idx)
+ without_reserve.append(chosen["candidate"])
+ selected_meta.append(chosen)
+ selected_hubs.add(chosen["hub"])
+ selected_sessions.update(chosen["sessions"])
+ without_sessions = {
+ s for c in without_reserve for s in (c.get("session_ids") or [])
+ }
+ self.assertNotIn("session_5", without_sessions)
+
+ curated = retrieve_mod._diversify_facts(ranked, max_facts=max_facts)
+ self.assertEqual(len(curated), max_facts)
+ curated_sessions = {
+ s for c in curated for s in (c.get("session_ids") or [])
+ }
+ self.assertIn("session_5", curated_sessions)
+ self.assertTrue(any(c.get("bridge") for c in curated))
+
+ again = retrieve_mod._diversify_facts(
+ list(reversed(ranked)), max_facts=max_facts
+ )
+ self.assertEqual(
+ [c["key"] for c in curated],
+ [c["key"] for c in again],
+ )
+
+ def test_build_fact_channel_keeps_candidate_session_ids(self):
+ seed = _candidate(
+ score=0.1,
+ r_uuid="r1",
+ r2_uuid="r2",
+ flow_key="f1",
+ m_uuid="event-seed",
+ )
+ seed["session_ids"] = ["session_1"]
+ seed["chunk_ids"] = ["chunk-missing-from-map"]
+ bridge = _candidate(
+ score=5.0,
+ r_uuid="rb",
+ r2_uuid="rrb",
+ flow_key="fb",
+ m_uuid="event-bridge",
+ n_uuid="nb",
+ b_uuid="bb",
+ )
+ bridge["session_ids"] = ["session_5"]
+ bridge["chunk_ids"] = ["chunk-bridge-missing"]
+ bridge["bridge"] = {
+ "from_hub": "event-seed",
+ "to_hub": "event-bridge",
+ "shared_entity": "shared",
+ "shared_entity_name": "Shared",
+ }
+ curated = retrieve_mod._diversify_facts([seed, bridge], max_facts=2)
+ text_lines, triples, graph_sessions = retrieve_mod._build_fact_channel(
+ curated, chunk_sessions={}
+ )
+ self.assertIn("session_5", graph_sessions)
+ self.assertTrue(any("session_5" in line for line in text_lines))
+ emitted = {
+ sid
+ for t in triples
+ for sid in (t.source_session_ids or [])
+ }
+ self.assertIn("session_5", emitted)
+
+ def test_rank_bridge_seed_hubs_total_order_ignores_arrival(self):
+ candidates = [
+ _candidate(
+ score=0.4,
+ r_uuid="r1",
+ r2_uuid="x1",
+ flow_key="f1",
+ m_uuid="hub-b",
+ ),
+ _candidate(
+ score=0.1,
+ r_uuid="r2",
+ r2_uuid="x2",
+ flow_key="f2",
+ m_uuid="hub-a",
+ ),
+ _candidate(
+ score=0.1,
+ r_uuid="r3",
+ r2_uuid="x3",
+ flow_key="f3",
+ m_uuid="hub-c",
+ ),
+ _candidate(
+ score=0.2,
+ r_uuid="r4",
+ r2_uuid="x4",
+ flow_key="f4",
+ m_uuid="hub-b",
+ ),
+ ]
+ a = retrieve_mod._rank_bridge_seed_hubs(candidates, cap=3)
+ b = retrieve_mod._rank_bridge_seed_hubs(list(reversed(candidates)), cap=3)
+ self.assertEqual(a, b)
+ self.assertEqual(a, ["hub-a", "hub-c", "hub-b"])
+
+ def test_expand_marks_reserve_ok_only_for_top_reserve_hubs(self):
+ seeds = [
+ _candidate(
+ score=0.01 * i,
+ r_uuid=f"rs{i}",
+ r2_uuid=f"rrs{i}",
+ flow_key=f"fs{i}",
+ m_uuid=f"event-seed-{i:02d}",
+ n_uuid=f"ns{i}",
+ b_uuid=f"bs{i}",
+ )
+ for i in range(10)
+ ]
+ bridges = [
+ HubBridge(
+ f"event-bridge-{i}",
+ f"event-seed-{i:02d}",
+ f"ent-{i}",
+ f"Ent{i}",
+ 1.0,
+ )
+ for i in range(10)
+ ]
+ facts = []
+ for i in range(10):
+ facts.append(
+ (
+ _node(f"nb{i}", f"nb{i}"),
+ _pred(f"rb{i}", "MADE", flow_key=f"fb{i}"),
+ _node(
+ f"event-bridge-{i}",
+ f"event-bridge-{i}",
+ labels=["EVENT"],
+ ),
+ _pred(f"rrb{i}", "TARGETED", flow_key=f"fb{i}"),
+ _node(f"bb{i}", f"bb{i}"),
+ )
+ )
+ mock_adapter = MagicMock()
+ mock_adapter.get_hub_bridges.return_value = bridges
+ mock_adapter.get_event_hub_facts.return_value = facts
+ with patch.object(retrieve_mod, "graph_adapter", mock_adapter):
+ expanded, _ = retrieve_mod._expand_cross_event_bridges(
+ seeds, "brain", max_per_hub=1
+ )
+ by_to = {
+ c["bridge"]["to_hub"]: c["bridge"]
+ for c in expanded
+ if c.get("bridge")
+ }
+ reserve_cap = retrieve_mod._BRIDGE_RESERVE_HUB_CAP
+ for i in range(10):
+ meta = by_to[f"event-bridge-{i}"]
+ self.assertEqual(meta["reserve_ok"], i < reserve_cap)
+
+ def test_reserved_slots_ignore_unstable_tail_hub_bridges(self):
+ seeds = [
+ _candidate(
+ score=0.01 * i,
+ r_uuid=f"rs{i}",
+ r2_uuid=f"rrs{i}",
+ flow_key=f"fs{i}",
+ m_uuid=f"event-seed-{i}",
+ n_uuid=f"ns{i}",
+ b_uuid=f"bs{i}",
+ )
+ for i in range(8)
+ ]
+ for seed in seeds:
+ seed["session_ids"] = ["session_1"]
+
+ stable = _candidate(
+ score=5.0,
+ r_uuid="rb0",
+ r2_uuid="rrb0",
+ flow_key="fb0",
+ m_uuid="event-bridge-stable",
+ n_uuid="nb0",
+ b_uuid="bb0",
+ )
+ stable["session_ids"] = ["session_5"]
+ stable["bridge"] = {
+ "from_hub": "event-seed-0",
+ "to_hub": "event-bridge-stable",
+ "shared_entity": "shared",
+ "shared_entity_name": "Shared",
+ "reserve_ok": True,
+ }
+
+ jitter_a = _candidate(
+ score=5.1,
+ r_uuid="rb1",
+ r2_uuid="rrb1",
+ flow_key="fb1",
+ m_uuid="event-bridge-jitter-a",
+ n_uuid="nb1",
+ b_uuid="bb1",
+ )
+ jitter_a["session_ids"] = ["session_9"]
+ jitter_a["bridge"] = {
+ "from_hub": "event-seed-9",
+ "to_hub": "event-bridge-jitter-a",
+ "shared_entity": "shared",
+ "shared_entity_name": "Shared",
+ "reserve_ok": False,
+ }
+
+ jitter_b = _candidate(
+ score=5.1,
+ r_uuid="rb2",
+ r2_uuid="rrb2",
+ flow_key="fb2",
+ m_uuid="event-bridge-jitter-b",
+ n_uuid="nb2",
+ b_uuid="bb2",
+ )
+ jitter_b["session_ids"] = ["session_13"]
+ jitter_b["bridge"] = {
+ "from_hub": "event-seed-9",
+ "to_hub": "event-bridge-jitter-b",
+ "shared_entity": "shared",
+ "shared_entity_name": "Shared",
+ "reserve_ok": False,
+ }
+
+ curated_a = retrieve_mod._diversify_facts(
+ seeds + [stable, jitter_a], max_facts=5
+ )
+ curated_b = retrieve_mod._diversify_facts(
+ seeds + [stable, jitter_b], max_facts=5
+ )
+ sessions_a = {
+ s for c in curated_a for s in (c.get("session_ids") or [])
+ }
+ sessions_b = {
+ s for c in curated_b for s in (c.get("session_ids") or [])
+ }
+ self.assertEqual(sessions_a, sessions_b)
+ self.assertIn("session_5", sessions_a)
+ self.assertNotIn("session_9", sessions_a)
+ self.assertNotIn("session_13", sessions_b)
+
+ def test_expand_then_diversify_stable_under_tail_seed_jitter(self):
+ core = [
+ _candidate(
+ score=0.01 * i,
+ r_uuid=f"rs{i}",
+ r2_uuid=f"rrs{i}",
+ flow_key=f"fs{i}",
+ m_uuid=f"event-seed-{i:02d}",
+ n_uuid=f"ns{i}",
+ b_uuid=f"bs{i}",
+ )
+ for i in range(retrieve_mod._BRIDGE_RESERVE_HUB_CAP)
+ ]
+ for seed in core:
+ seed["session_ids"] = ["session_1"]
+ seed["chunk_ids"] = ["chunk-seed"]
+
+ def run_with_tail(tail_hub: str, novel_hub: str, novel_session: str):
+ tail = _candidate(
+ score=0.9,
+ r_uuid=f"rt-{tail_hub}",
+ r2_uuid=f"rrt-{tail_hub}",
+ flow_key=f"ft-{tail_hub}",
+ m_uuid=tail_hub,
+ n_uuid=f"nt-{tail_hub}",
+ b_uuid=f"bt-{tail_hub}",
+ )
+ tail["session_ids"] = ["session_1"]
+ tail["chunk_ids"] = ["chunk-seed"]
+ seeds = core + [tail]
+ bridges = [
+ HubBridge("event-bridge-core", "event-seed-00", "ent-c", "C", 1.0),
+ HubBridge(novel_hub, tail_hub, "ent-t", "T", 1.0),
+ ]
+ core_fact = (
+ _node("nbc", "nbc"),
+ _pred("rbc", "MADE", flow_key="fbc"),
+ _node("event-bridge-core", "event-bridge-core", labels=["EVENT"]),
+ _pred("rrbc", "TARGETED", flow_key="fbc"),
+ _node("bbc", "bbc"),
+ )
+ for item in core_fact:
+ item.properties = {"source_chunk_ids": ["chunk-core"]}
+ novel_fact = (
+ _node("nbn", "nbn"),
+ _pred("rbn", "MADE", flow_key="fbn"),
+ _node(novel_hub, novel_hub, labels=["EVENT"]),
+ _pred("rrbn", "TARGETED", flow_key="fbn"),
+ _node("bbn", "bbn"),
+ )
+ for item in novel_fact:
+ item.properties = {"source_chunk_ids": ["chunk-novel"]}
+ mock_adapter = MagicMock()
+ mock_adapter.get_hub_bridges.return_value = bridges
+ mock_adapter.get_event_hub_facts.return_value = [core_fact, novel_fact]
+ chunk_sessions = {
+ "chunk-seed": ["session_1"],
+ "chunk-core": ["session_5"],
+ "chunk-novel": [novel_session],
+ }
+ with patch.object(retrieve_mod, "graph_adapter", mock_adapter):
+ with patch.object(
+ retrieve_mod,
+ "_resolve_chunk_sessions",
+ side_effect=lambda ids, _brain: {
+ cid: chunk_sessions[cid]
+ for cid in ids
+ if cid in chunk_sessions
+ },
+ ):
+ expanded, _ = retrieve_mod._expand_cross_event_bridges(
+ seeds, "brain", max_per_hub=1
+ )
+ return retrieve_mod._diversify_facts(expanded, max_facts=5)
+
+ a = run_with_tail("event-seed-90", "event-bridge-a", "session_9")
+ b = run_with_tail("event-seed-91", "event-bridge-b", "session_13")
+ sessions_a = {s for c in a for s in (c.get("session_ids") or [])}
+ sessions_b = {s for c in b for s in (c.get("session_ids") or [])}
+ self.assertEqual(sessions_a, sessions_b)
+ self.assertIn("session_5", sessions_a)
+ self.assertNotIn("session_9", sessions_a)
+ self.assertNotIn("session_13", sessions_b)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/tests/test_quantity_node_identity.py b/tests/test_quantity_node_identity.py
new file mode 100644
index 0000000..101633f
--- /dev/null
+++ b/tests/test_quantity_node_identity.py
@@ -0,0 +1,143 @@
+import json
+import os
+import re
+import unittest
+
+
+ENV_DEFAULTS = {
+ "BRAINPAT_TOKEN": "test-token",
+ "MODELS_MODE": "local",
+ "EMBEDDINGS_LOCAL_MODEL": "local-model",
+ "EMBEDDINGS_SMALL_MODEL": "small-model",
+ "EMBEDDING_NODES_DIMENSION": "3",
+ "EMBEDDING_TRIPLETS_DIMENSION": "3",
+ "EMBEDDING_OBSERVATIONS_DIMENSION": "3",
+ "EMBEDDING_DATA_DIMENSION": "3",
+ "EMBEDDING_RELATIONSHIPS_DIMENSION": "3",
+ "REDIS_HOST": "localhost",
+ "REDIS_PORT": "6379",
+ "NEO4J_HOST": "localhost",
+ "NEO4J_PORT": "7687",
+ "NEO4J_USERNAME": "neo4j",
+ "NEO4J_PASSWORD": "password",
+ "MILVUS_HOST": "localhost",
+ "MILVUS_PORT": "19530",
+ "MONGO_CONNECTION_STRING": "mongodb://localhost:27017",
+ "CELERY_WORKER_CONCURRENCY": "1",
+ "OLLAMA_HOST": "localhost",
+ "OLLAMA_PORT": "11434",
+ "OLLAMA_LLM_SMALL_MODEL": "small",
+ "OLLAMA_LLM_LARGE_MODEL": "large",
+}
+for key, value in ENV_DEFAULTS.items():
+ os.environ.setdefault(key, value)
+
+from src.constants.prompts import architect_agent, janitor_agent, scout_agent
+from src.core.saving.identity import stable_node_id
+
+
+_PLACEHOLDER_TYPES = {
+ "MONEY",
+ "UNIT",
+ "UNITS",
+ "AMOUNT",
+ "QUANTITY",
+ "COUNT",
+ "CURRENCY",
+ "NUMBER",
+}
+_ENTITY_PATTERN = re.compile(r"\{\{[^{}]*\"type\"\s*:[^{}]*\}\}")
+
+
+def _prompt_constants(module) -> dict[str, str]:
+ return {
+ name: value
+ for name, value in vars(module).items()
+ if name.isupper() and isinstance(value, str)
+ }
+
+
+def _demonstrated_entities(prompt: str) -> list[dict]:
+ entities: list[dict] = []
+ for match in _ENTITY_PATTERN.finditer(prompt):
+ body = match.group(0).replace("{{", "{").replace("}}", "}")
+ try:
+ parsed = json.loads(body)
+ except json.JSONDecodeError:
+ continue
+ if isinstance(parsed, dict) and parsed.get("type") and parsed.get("name"):
+ entities.append(parsed)
+ return entities
+
+
+def _shared_hub_entities(prompt: str) -> list[dict]:
+ shared: list[dict] = []
+ for entity in _demonstrated_entities(prompt):
+ entity_type = str(entity["type"]).strip()
+ name = str(entity["name"]).strip()
+ if entity_type.upper() in _PLACEHOLDER_TYPES:
+ shared.append(entity)
+ elif name.lower() == entity_type.lower():
+ shared.append(entity)
+ return shared
+
+
+def _fact_node_ids(prompt: str, event_name: str, happened_at: str) -> set[str]:
+ ids = {stable_node_id(event_name, "EVENT", happened_at)}
+ for entity in _shared_hub_entities(prompt):
+ ids.add(stable_node_id(entity["name"], entity["type"]))
+ return ids
+
+
+class QuantityNodeIdentityTests(unittest.TestCase):
+ def test_two_monetary_amounts_do_not_share_a_node_id(self):
+ for module in (scout_agent, architect_agent):
+ for name, prompt in _prompt_constants(module).items():
+ acme = _fact_node_ids(prompt, "Raised", "19/01/2026")
+ globex = _fact_node_ids(prompt, "Raised", "03/04/2026")
+ self.assertEqual(
+ acme & globex,
+ set(),
+ f"{module.__name__}.{name} demonstrates a node shared by two "
+ f"unrelated monetary facts",
+ )
+
+ def test_prompts_demonstrate_no_type_named_placeholder_nodes(self):
+ offenders: list[str] = []
+ for module in (scout_agent, architect_agent):
+ for name, prompt in _prompt_constants(module).items():
+ for entity in _shared_hub_entities(prompt):
+ offenders.append(
+ f"{module.__name__}.{name}: "
+ f"{entity['type']}/{entity['name']}"
+ )
+ self.assertEqual(offenders, [])
+
+ def test_prompts_do_not_instruct_unit_entities(self):
+ offenders: list[str] = []
+ for module in (scout_agent, architect_agent):
+ for name, prompt in _prompt_constants(module).items():
+ if re.search(r"\"Unit\".*as an entity", prompt):
+ offenders.append(f"{module.__name__}.{name}")
+ if "quantitative units" in prompt:
+ offenders.append(f"{module.__name__}.{name}")
+ self.assertEqual(offenders, [])
+
+ def test_janitor_still_prescribes_amount_migration(self):
+ prompts = "\n".join(_prompt_constants(janitor_agent).values())
+ self.assertIn("Amount Migration", prompts)
+ self.assertIn("amount-nodes", prompts)
+
+ def test_type_named_identity_is_constant_across_facts(self):
+ self.assertEqual(
+ stable_node_id("Money", "MONEY"),
+ stable_node_id("Money", "MONEY"),
+ )
+ self.assertNotEqual(
+ stable_node_id("Raised", "EVENT", "19/01/2026"),
+ stable_node_id("Raised", "EVENT", "03/04/2026"),
+ )
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/tests/test_topic_hyperedges.py b/tests/test_topic_hyperedges.py
new file mode 100644
index 0000000..b2ec708
--- /dev/null
+++ b/tests/test_topic_hyperedges.py
@@ -0,0 +1,58 @@
+from __future__ import annotations
+
+import unittest
+
+from src.core.saving.topic_hyperedges import (
+ cluster_sessions_into_topics,
+ lexical_topic_score,
+ rrf_fuse,
+ select_topics_and_sessions,
+ session_entity_sets,
+)
+
+
+class TopicHyperedgesTests(unittest.TestCase):
+ def test_session_entity_sets_and_cluster(self) -> None:
+ rows = [
+ ("session_1", "e1", "pottery"),
+ ("session_1", "e2", "Melanie"),
+ ("session_2", "e3", "pottery workshop"),
+ ("session_2", "e2", "Melanie"),
+ ("session_3", "e4", "counseling"),
+ ("session_3", "e5", "Caroline"),
+ ]
+ entities = session_entity_sets(rows)
+ topics = cluster_sessions_into_topics(entities, merge_threshold=0.2)
+ self.assertGreaterEqual(len(topics), 2)
+ session_ids = {t.session_id for t in topics}
+ self.assertEqual(session_ids, {"session_1", "session_2", "session_3"})
+ coverage = len(session_ids) / 3
+ self.assertGreaterEqual(coverage, 0.8)
+
+ def test_select_topics_boosts_query_overlap(self) -> None:
+ from src.core.saving.topic_hyperedges import TopicSession
+
+ memberships = [
+ TopicSession("topic:a", "pottery camping", "session_1"),
+ TopicSession("topic:a", "pottery camping", "session_2"),
+ TopicSession("topic:b", "counseling career", "session_3"),
+ ]
+ topics, sessions = select_topics_and_sessions(
+ "What pottery workshops did Melanie attend?",
+ memberships,
+ seed_sessions=["session_1"],
+ k_topics=2,
+ k_sessions=5,
+ )
+ self.assertTrue(topics)
+ self.assertIn("session_1", sessions)
+ self.assertGreater(lexical_topic_score("pottery", "pottery camping"), 0.0)
+
+ def test_rrf_fuse(self) -> None:
+ fused = rrf_fuse([["a", "b", "c"], ["b", "a", "d"]])
+ self.assertEqual(fused[0], "a")
+ self.assertIn("b", fused[:2])
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/tests/test_vector_similarity_contract.py b/tests/test_vector_similarity_contract.py
index 52c60b8..0559583 100644
--- a/tests/test_vector_similarity_contract.py
+++ b/tests/test_vector_similarity_contract.py
@@ -51,6 +51,20 @@ class VectorSimilarityContractTests(unittest.TestCase):
results = adapter.search_vectors([1.0, 0.0, 0.0], brain_id="b", store="nodes", k=3)
self.assertEqual([v.id for v in results], ["near", "mid", "far"])
+ def test_adapter_equal_distance_tiebreaks_by_uuid(self):
+ from src.adapters.embeddings import VectorStoreAdapter
+ from src.constants.embeddings import Vector
+
+ store = MagicMock()
+ store.search_vectors.return_value = [
+ Vector(id="2", metadata={"uuid": "z"}, distance=0.1),
+ Vector(id="1", metadata={"uuid": "a"}, distance=0.1),
+ ]
+ adapter = VectorStoreAdapter()
+ adapter.add_client(store)
+ results = adapter.search_vectors([1.0], brain_id="b", store="nodes", k=2)
+ self.assertEqual([v.metadata["uuid"] for v in results], ["a", "z"])
+
def test_milvus_converts_similarity_score_to_cosine_distance(self):
from src.constants.embeddings import Vector
from src.lib.milvus.client import MilvusClient