BrainAPI is a knowledge graph–powered AI memory layer that transforms unstructured data into structured knowledge, enabling intelligent search, recommendations, and contextual memory for AI agents and applications.
Python ai-agentsai-memoryknowledge-graphknowledge-managementknowledge-representationreccomendation-system
test(ingestion+retrieval): test improvements after locomo benchmark
Christian82f41d21d9b4
diff --git a/benchmarks/locomo/client.py b/benchmarks/locomo/client.pyindex 1715bb7..8951fea 100644--- a/benchmarks/locomo/client.py+++ b/benchmarks/locomo/client.py@@ -84,6 +84,7 @@ class BrainAPIClient:*,observate_for: list[str] | None = None,preferred_extraction_entities: list[str] | None = None,+ source_timestamp: str | None = None,) -> TimedResult:body = {"data": {"data_type": "text", "text_data": text},@@ -97,6 +98,8 @@ class BrainAPIClient:"preferred_extraction_entities": preferred_extraction_entitiesor ["Person", "Event", "Location", "Date"],}+ if source_timestamp:+ body["source_timestamp"] = source_timestampreturn self._request("POST", "/ingest/", json=body)def get_task(self, task_id: str, brain_id: str) -> TimedResult:diff --git a/benchmarks/locomo/dataset.py b/benchmarks/locomo/dataset.pyindex 2360899..53b130f 100644--- a/benchmarks/locomo/dataset.py+++ b/benchmarks/locomo/dataset.py@@ -128,6 +128,7 @@ def iter_ingest_units("unit_id": key,"session_key": key,"text": format_session(conversation, key),+ "source_timestamp": conversation.get(f"{key}_date_time"),})return units@@ -144,6 +145,7 @@ def iter_ingest_units("session_key": key,"dia_id": dia_id,"text": format_turn_unit(conversation, key, turn),+ "source_timestamp": conversation.get(f"{key}_date_time"),})return unitsdiff --git a/benchmarks/locomo/ingest.py b/benchmarks/locomo/ingest.pyindex 0de2870..4339bfd 100644--- a/benchmarks/locomo/ingest.py+++ b/benchmarks/locomo/ingest.py@@ -135,7 +135,11 @@ def ingest_samples(with BrainAPIClient(settings) as client:try:- submitted = client.ingest_text(unit["text"], job["brain_id"])+ submitted = client.ingest_text(+ unit["text"],+ job["brain_id"],+ source_timestamp=unit.get("source_timestamp"),+ )task_id = (submitted.data or {}).get("task_id")if not task_id:raise RuntimeError(f"No task_id in ingest response: {submitted.data}")diff --git a/benchmarks/locomo/metrics.py b/benchmarks/locomo/metrics.pyindex 0fbe71f..dfecc15 100644--- a/benchmarks/locomo/metrics.py+++ b/benchmarks/locomo/metrics.py@@ -73,6 +73,20 @@ def mean(values: Iterable[float]) -> float | None:return sum(vals) / len(vals)+def wilson_ci(successes: int, n: int, z: float = 1.96) -> tuple[float | None, float | None]:+ if n <= 0:+ return None, None+ p = successes / n+ denom = 1.0 + (z * z) / n+ center = (p + (z * z) / (2.0 * n)) / denom+ margin = (+ z+ * math.sqrt((p * (1.0 - p) / n) + (z * z) / (4.0 * n * n))+ / denom+ )+ return max(0.0, center - margin), min(1.0, center + margin)++def aggregate_answers(rows: list[dict[str, Any]]) -> dict[str, Any]:by_category: dict[int, list[dict[str, Any]]] = {}for row in rows:@@ -84,19 +98,27 @@ def aggregate_answers(rows: list[dict[str, Any]]) -> dict[str, Any]:return {"n": 0,"judge_accuracy": None,+ "judge_accuracy_ci95": None,"mean_f1": None,"mean_bleu1": None,}correct = [1.0 if r.get("judge_correct") else 0.0 for r in items]+ successes = int(sum(correct))+ lo, hi = wilson_ci(successes, len(items))return {"n": len(items),"judge_accuracy": mean(correct),+ "judge_accuracy_ci95": (+ {"low": lo, "high": hi} if lo is not None and hi is not None else None+ ),"mean_f1": mean(float(r.get("f1") or 0.0) for r in items),"mean_bleu1": mean(float(r.get("bleu1") or 0.0) for r in items),}non_adv = [r for r in rows if int(r.get("category") or 0) != 5]all_rows = rows+ non_adv_bucket = _bucket(non_adv)+ all_bucket = _bucket(all_rows)retrieval_latencies = [float(r["retrieve_latency_ms"])for r in rows@@ -111,10 +133,12 @@ def aggregate_answers(rows: list[dict[str, Any]]) -> dict[str, Any]:return {"n_total": len(all_rows),"n_non_adversarial": len(non_adv),- "headline_judge_accuracy": _bucket(non_adv)["judge_accuracy"],- "overall_judge_accuracy": _bucket(all_rows)["judge_accuracy"],- "mean_f1_non_adversarial": _bucket(non_adv)["mean_f1"],- "mean_bleu1_non_adversarial": _bucket(non_adv)["mean_bleu1"],+ "headline_judge_accuracy": non_adv_bucket["judge_accuracy"],+ "headline_judge_accuracy_ci95": non_adv_bucket["judge_accuracy_ci95"],+ "overall_judge_accuracy": all_bucket["judge_accuracy"],+ "overall_judge_accuracy_ci95": all_bucket["judge_accuracy_ci95"],+ "mean_f1_non_adversarial": non_adv_bucket["mean_f1"],+ "mean_bleu1_non_adversarial": non_adv_bucket["mean_bleu1"],"by_category": {str(cat): _bucket(items) for cat, items in sorted(by_category.items())},@@ -137,4 +161,7 @@ def selftest_metrics() -> list[str]:errors.append("article stripping should allow BLEU-1=1")if normalize_answer("The Quick, Brown!") != "quick brown":errors.append("normalize_answer failed")+ lo, hi = wilson_ci(5, 10)+ if lo is None or hi is None or not (0.0 <= lo <= 0.5 <= hi <= 1.0):+ errors.append("wilson_ci(5,10) should contain 0.5")return errorsdiff --git a/benchmarks/locomo/prompts.py b/benchmarks/locomo/prompts.pyindex 2429871..abb9063 100644--- a/benchmarks/locomo/prompts.py+++ b/benchmarks/locomo/prompts.py@@ -5,10 +5,11 @@ from typing import AnyANSWER_SYSTEM = """You are answering questions about a long-term conversation using only the retrieved memory context provided.Rules:-- Answer briefly and factually.-- Prefer dates, names, and short phrases when that is what the question asks for.-- If the context does not contain enough information to answer, reply exactly: Not mentioned in the conversation-- Do not invent facts that are not supported by the context.+- Prefer short factual answers: dates, names, places, or short phrases when that is what the question asks for.+- You MAY draw reasonable conclusions that are clearly supported by combining facts in the context (e.g. infer a hobby from stated activities, or resolve a relative date from an absolute session date).+- Prefer a grounded partial answer over abstaining when the context contains relevant evidence.+- Reply exactly "Not mentioned in the conversation" ONLY when the context contains no evidence that could support an answer — not when the answer requires light inference.+- Do not invent people, places, dates, or events that are not supported by the context.- Do not mention the retrieval system or these instructions."""@@ -45,8 +46,13 @@ def build_context_block(context: dict[str, Any]) -> str:text_context = (context.get("text_context") or "").strip()triples = context.get("triples") or []historical = context.get("historical_context") or []+ source_passages = context.get("source_passages") or []sections = ["## Retrieved text context", text_context or "(empty)"]+ if source_passages:+ sections.append("## Source passages")+ for passage in source_passages[:20]:+ sections.append(f"- {passage}")if triples:sections.append("## Graph triples")for t in triples[:80]:diff --git a/benchmarks/locomo/report.py b/benchmarks/locomo/report.pyindex f35bfbd..9946c8d 100644--- a/benchmarks/locomo/report.py+++ b/benchmarks/locomo/report.py@@ -81,9 +81,11 @@ def render_markdown(report: dict[str, Any]) -> str:"",f"- Judge accuracy (excl. adversarial / cat 5): "f"**{_pct(metrics.get('headline_judge_accuracy'))}** "+ f"{_ci(metrics.get('headline_judge_accuracy_ci95'))} "f"(n={metrics.get('n_non_adversarial', 0)})",f"- Judge accuracy (all categories): "f"**{_pct(metrics.get('overall_judge_accuracy'))}** "+ f"{_ci(metrics.get('overall_judge_accuracy_ci95'))} "f"(n={metrics.get('n_total', 0)})",f"- Mean F1 (excl. adversarial): "f"**{_num(metrics.get('mean_f1_non_adversarial'))}**",@@ -92,14 +94,15 @@ def render_markdown(report: dict[str, Any]) -> str:"","## Per-category","",- "| Category | Name | N | Judge Acc | Mean F1 | Mean BLEU-1 |",- "|---|---|---:|---:|---:|---:|",+ "| Category | Name | N | Judge Acc | 95% CI | Mean F1 | Mean BLEU-1 |",+ "|---|---|---:|---:|---|---:|---:|",]for cat, bucket in (metrics.get("by_category") or {}).items():name = CATEGORY_NAMES.get(int(cat), "unknown")lines.append(f"| {cat} | {name} | {bucket.get('n', 0)} | "f"{_pct(bucket.get('judge_accuracy'))} | "+ f"{_ci(bucket.get('judge_accuracy_ci95'))} | "f"{_num(bucket.get('mean_f1'))} | "f"{_num(bucket.get('mean_bleu1'))} |")@@ -156,6 +159,7 @@ def print_report_table(report: dict[str, Any]) -> None:table.add_column("Name")table.add_column("N", justify="right")table.add_column("Judge Acc", justify="right")+ table.add_column("95% CI")table.add_column("F1", justify="right")table.add_column("BLEU-1", justify="right")for cat, bucket in (metrics.get("by_category") or {}).items():@@ -164,13 +168,15 @@ def print_report_table(report: dict[str, Any]) -> None:CATEGORY_NAMES.get(int(cat), "unknown"),str(bucket.get("n", 0)),_pct(bucket.get("judge_accuracy")),+ _ci(bucket.get("judge_accuracy_ci95")),_num(bucket.get("mean_f1")),_num(bucket.get("mean_bleu1")),)console.print(table)console.print(f"Headline judge accuracy (excl. cat 5): "- f"[bold]{_pct(metrics.get('headline_judge_accuracy'))}[/bold]"+ f"[bold]{_pct(metrics.get('headline_judge_accuracy'))}[/bold] "+ f"{_ci(metrics.get('headline_judge_accuracy_ci95'))}")for warning in report.get("warnings") or []:console.print(f"[yellow]Warning: {warning}[/yellow]")@@ -182,6 +188,12 @@ def _pct(value: float | None) -> str:return f"{value * 100:.1f}%"+def _ci(value: dict[str, float] | None) -> str:+ if not value or value.get("low") is None or value.get("high") is None:+ return "n/a"+ return f"[{_pct(value['low'])}, {_pct(value['high'])}]"++def _num(value: float | None) -> str:if value is None:return "n/a"diff --git a/src/constants/prompts/scout_agent.py b/src/constants/prompts/scout_agent.pyindex d00a77b..dac1db1 100644--- a/src/constants/prompts/scout_agent.py+++ b/src/constants/prompts/scout_agent.py@@ -141,6 +141,8 @@ SCOUT_AGENT_EXTRACT_ENTITIES_PROMPT = """Carefully read the text and extract ALL the entities, unique action instances (events), and quantitative units.{targeting}+{reference_time}+{preferred_entities}Text: {text}@@ -149,6 +151,7 @@ OUTPUT RULES:- Each object must include: "type", "name", and optional "properties" and "description".- Nodes/Entities MUST be atomic and not composite (phrases) (eg: "Went to San Francisco" is not atomic, "Went to" + "San Francisco" is atomic)- Dates must be in the format "DD/MM/YYYY" and be stored as "happened_at" in the properties of the event nodes.+- When the reference date is provided, resolve relative dates (yesterday, last week, last Tuesday, etc.) against it into absolute DD/MM/YYYY values.- You must extract ALL the building blocks without omitting any concepts.Begin!@@ -159,6 +162,8 @@ Carefully read the text and extract the most important entities that can be usedthe unique action instances (events), and the quantitative units.{targeting}+{reference_time}+{preferred_entities}Text: {text}@@ -167,6 +172,7 @@ OUTPUT RULES:- Each object must include: "type", "name", and optional "properties" and "description".- Nodes/Entities MUST be atomic and not composite (phrases) (eg: "Went to San Francisco" is not atomic, "Went to" + "San Francisco" is atomic)- Dates must be in the format "DD/MM/YYYY" and be stored as "happened_at" in the properties of the event nodes.+- When the reference date is provided, resolve relative dates (yesterday, last week, last Tuesday, etc.) against it into absolute DD/MM/YYYY values.- You must extract the most important entities that can be used to reconstruct a meaningful narrative,the unique action instances (events), and the quantitative units without omitting any concepts.diff --git a/src/constants/tasks/ingestion.py b/src/constants/tasks/ingestion.pyindex 506aa13..fee5335 100644--- a/src/constants/tasks/ingestion.py+++ b/src/constants/tasks/ingestion.py@@ -78,3 +78,10 @@ class IngestionTaskArgs(BaseModel):brain_id: str = Field(default="default", description="The brain identifier to store the data in.")+ source_timestamp: Optional[str] = Field(+ default=None,+ description=(+ "Reference timestamp for the source content (e.g. session date). "+ "Used to resolve relative dates like 'yesterday' into absolute dates."+ ),+ )diff --git a/src/core/agents/scout_agent.py b/src/core/agents/scout_agent.pyindex 8a27e56..c9bd161 100644--- a/src/core/agents/scout_agent.py+++ b/src/core/agents/scout_agent.py@@ -40,6 +40,7 @@ from src.constants.prompts.scout_agent import (from src.core.agents.core import parse_structured_from_messages, runtime_agent_factoryfrom src.core.plugins.prompts import prompt_registryfrom src.services.api.constants.requests import IngestionTripleSet+from src.utils.text_chunking import chunk_text as _chunk_textclass _ScoutEntity(BaseModel):@@ -338,26 +339,60 @@ class ScoutAgent:max_retries: int = 3,ingestion_session_id: Optional[str] = None,mode: Literal["granular", "coarse"] = "granular",+ reference_time: Optional[str] = None,+ preferred_extraction_entities: Optional[List[str]] = None,+ max_chars_per_chunk: int = 6000,+ ) -> ScoutAgentResponse:+ chunks = _chunk_text(text, max_chars=max_chars_per_chunk)+ if len(chunks) == 1:+ return self._run_chunk(+ chunks[0],+ targeting=targeting,+ brain_id=brain_id,+ timeout=timeout,+ max_retries=max_retries,+ ingestion_session_id=ingestion_session_id,+ mode=mode,+ reference_time=reference_time,+ preferred_extraction_entities=preferred_extraction_entities,+ )+ merged: dict[str, ScoutEntity] = {}+ for chunk in chunks:+ response = self._run_chunk(+ chunk,+ targeting=targeting,+ brain_id=brain_id,+ timeout=timeout,+ max_retries=max_retries,+ ingestion_session_id=ingestion_session_id,+ mode=mode,+ reference_time=reference_time,+ preferred_extraction_entities=preferred_extraction_entities,+ )+ for entity in response.entities:+ key = (+ (entity.name or "").strip().lower(),+ (entity.type or "").strip().lower(),+ getattr(entity, "happened_at", None) or "",+ )+ if key not in merged:+ merged[key] = entity+ return ScoutAgentResponse(entities=list(merged.values()))++ def _run_chunk(+ self,+ text: str,+ targeting: Optional[Node] = None,+ brain_id: str = "default",+ timeout: int = 300,+ max_retries: int = 3,+ ingestion_session_id: Optional[str] = None,+ mode: Literal["granular", "coarse"] = "granular",+ reference_time: Optional[str] = None,+ preferred_extraction_entities: Optional[List[str]] = None,) -> ScoutAgentResponse:"""Extract entities from the provided text using the Scout agent and return a structured response containing the entities.-- Performs an LLM invocation (with optional targeting context), applies retries with exponential backoff on timeouts, and enforces a per-invocation timeout.-- Parameters:- text: The input text to extract entities from.- targeting: Optional Node providing contextual targeting information (name, description, properties) to bias extraction.- brain_id: Identifier for the agent/brain configuration to use.- timeout: Maximum seconds to wait for a single agent invocation before treating it as a timeout.- max_retries: Maximum number of retry attempts for timed-out invocations using exponential backoff.- ingestion_session_id: Identifier for the ingestion session to use.- mode: Mode to use for the scout agent. "granular" for a more granular extraction, "coarse" to extract the most important entities only.- Returns:- A ScoutAgentResponse containing:- - entities: list of extracted ScoutEntity objects.-- Raises:- TimeoutError: If a single invocation exceeds `timeout`, or if all retry attempts fail due to timeouts."""self._get_agent(output_schema=_ScoutAgentResponse,@@ -374,6 +409,17 @@ class ScoutAgent:if targetingelse "")+ reference_time_str = (+ f"Reference date for resolving relative dates: {reference_time}"+ if reference_time+ else ""+ )+ preferred_str = (+ "Prefer extracting these entity types when present: "+ + ", ".join(preferred_extraction_entities)+ if preferred_extraction_entities+ else ""+ )if mode == "granular":prompt = prompt_registry.get("SCOUT_AGENT_EXTRACT_ENTITIES_PROMPT",@@ -381,6 +427,8 @@ class ScoutAgent:).format(text=text,targeting=targeting_str,+ reference_time=reference_time_str,+ preferred_entities=preferred_str,)elif mode == "coarse":prompt = prompt_registry.get(@@ -389,6 +437,8 @@ class ScoutAgent:).format(text=text,targeting=targeting_str,+ reference_time=reference_time_str,+ preferred_entities=preferred_str,)else:raise ValueError(f"Invalid mode for scout agent: {mode}")@@ -424,15 +474,6 @@ class ScoutAgent:reraise=True,)def _invoke_agent_with_retry():- """- Invoke the agent in a separate thread and enforce the configured timeout.-- Returns:- dict: The agent response dictionary.-- Raises:- TimeoutError: If the agent invocation exceeds the specified timeout.- """try:with ThreadPoolExecutor(max_workers=1) as executor:future = executor.submit(_invoke_agent)diff --git a/src/core/saving/auto_kg.py b/src/core/saving/auto_kg.pyindex 4896969..23ee6ff 100644--- a/src/core/saving/auto_kg.py+++ b/src/core/saving/auto_kg.py@@ -39,7 +39,12 @@ class EnrichmentOrchestrationResult:def enrich_kg_from_input(- input: str, targeting: Optional[Node] = None, brain_id: str = "default"+ input: str,+ targeting: Optional[Node] = None,+ brain_id: str = "default",+ source_chunk_id: Optional[str] = None,+ source_timestamp: Optional[str] = None,+ preferred_extraction_entities: Optional[List[str]] = None,) -> EnrichmentOrchestrationResult:"""Orchestrates enrichment of the knowledge graph from a free-text input.@@ -58,14 +63,32 @@ def enrich_kg_from_input("ingestion_session_id": ingestion_session_id,"brain_id": brain_id,"flow": "enrich_kg_from_input",+ "source_chunk_id": source_chunk_id,+ "source_timestamp": source_timestamp,},):- return _enrich_kg_impl(input, targeting, brain_id, ingestion_session_id)+ return _enrich_kg_impl(+ input,+ targeting,+ brain_id,+ ingestion_session_id,+ source_chunk_id=source_chunk_id,+ source_timestamp=source_timestamp,+ preferred_extraction_entities=preferred_extraction_entities,+ )def _enrich_kg_impl(- input, targeting, brain_id: str, ingestion_session_id: str+ input,+ targeting,+ brain_id: str,+ ingestion_session_id: str,+ source_chunk_id: Optional[str] = None,+ source_timestamp: Optional[str] = None,+ preferred_extraction_entities: Optional[List[str]] = None,) -> EnrichmentOrchestrationResult:+ from src.core.saving.identity import stamp_provenance+ingestion_manager = IngestionManager(embeddings_adapter, vector_store_adapter, graph_adapter)@@ -95,6 +118,8 @@ def _enrich_kg_impl(brain_id=brain_id,ingestion_session_id=ingestion_session_id,mode=("coarse" if mode == "coarse" else "granular"),+ reference_time=source_timestamp,+ preferred_extraction_entities=preferred_extraction_entities,)print("[DEBUG (initial_scout_entities)]: ", entities.entities)architect_agent.run_tooler(@@ -108,9 +133,23 @@ def _enrich_kg_impl()pending = architect_agent.take_pending_relationships()- enrichment_relationships = [- rel.model_dump(mode="json") for rel in pending- ]+ enrichment_relationships = []+ for rel in pending:+ payload = rel.model_dump(mode="json")+ payload["properties"] = stamp_provenance(+ payload.get("properties"),+ source_chunk_id=source_chunk_id,+ source_timestamp=source_timestamp,+ )+ for endpoint_key in ("tail", "tip"):+ endpoint = payload.get(endpoint_key) or {}+ endpoint["properties"] = stamp_provenance(+ endpoint.get("properties"),+ source_chunk_id=source_chunk_id,+ source_timestamp=source_timestamp,+ )+ payload[endpoint_key] = endpoint+ enrichment_relationships.append(payload)should_consolidate = bool(config.pipeline_mode == "accurate"and config.run_graph_consolidatordiff --git a/src/core/saving/identity.py b/src/core/saving/identity.pyindex 28142e6..576b34d 100644--- a/src/core/saving/identity.py+++ b/src/core/saving/identity.py@@ -1,5 +1,5 @@import hashlib-from typing import Optional+from typing import Any, Optionalfrom src.utils.dates import normalize_date_string@@ -59,3 +59,51 @@ def stable_flow_key(event_type,normalize_date_string(happened_at) if happened_at else None,)+++def merge_source_chunk_ids(*sources: Any) -> list[str]:+ seen: set[str] = set()+ merged: list[str] = []+ for source in sources:+ if source is None:+ continue+ if isinstance(source, str):+ values = [source]+ elif isinstance(source, (list, tuple, set)):+ values = list(source)+ else:+ values = [source]+ for value in values:+ chunk_id = str(value).strip() if value is not None else ""+ if not chunk_id or chunk_id in seen:+ continue+ seen.add(chunk_id)+ merged.append(chunk_id)+ return merged+++def stamp_provenance(+ properties: Optional[dict],+ *,+ source_chunk_id: Optional[str] = None,+ source_timestamp: Optional[str] = None,+ existing_properties: Optional[dict] = None,+) -> dict:+ props = dict(properties or {})+ existing = existing_properties or {}+ chunk_ids = merge_source_chunk_ids(+ existing.get("source_chunk_ids"),+ existing.get("source_chunk_id"),+ props.get("source_chunk_ids"),+ props.get("source_chunk_id"),+ source_chunk_id,+ )+ if chunk_ids:+ props["source_chunk_ids"] = chunk_ids+ props.pop("source_chunk_id", None)+ timestamp = source_timestamp or props.get("source_timestamp") or existing.get(+ "source_timestamp"+ )+ if timestamp:+ props["source_timestamp"] = timestamp+ return propsdiff --git a/src/core/search/fact_filter.py b/src/core/search/fact_filter.pynew file mode 100644index 0000000..c5880e4--- /dev/null+++ b/src/core/search/fact_filter.py@@ -0,0 +1,77 @@+from __future__ import annotations++from typing import Any, Optional++from pydantic import BaseModel, Field+++class FactFilterDecision(BaseModel):+ keep_indices: list[int] = Field(default_factory=list)+++def filter_relevant_facts(+ question: str,+ candidates: list[str],+ *,+ llm_adapter: Any = None,+ max_keep: int = 40,+) -> list[int]:+ if not candidates:+ return []+ if len(candidates) <= max_keep and llm_adapter is None:+ return list(range(len(candidates)))++ indexed = "\n".join(f"{i}: {text}" for i, text in enumerate(candidates))+ prompt = (+ "You are a recognition-memory filter for a conversational memory system.\n"+ "Given a question and numbered candidate facts/passages, return the indices "+ "of items that are relevant evidence for answering the question.\n"+ "Prefer recall of useful evidence over aggressive pruning, but drop clear noise.\n"+ f"Keep at most {max_keep} indices.\n\n"+ f"Question: {question}\n\n"+ f"Candidates:\n{indexed}\n\n"+ 'Respond with JSON: {"keep_indices": [0, 2, ...]}'+ )++ if llm_adapter is None:+ return list(range(min(max_keep, len(candidates))))++ try:+ response = llm_adapter.generate_structured(+ prompt,+ FactFilterDecision,+ )+ if isinstance(response, FactFilterDecision):+ indices = response.keep_indices+ elif isinstance(response, dict):+ indices = response.get("keep_indices") or []+ else:+ indices = getattr(response, "keep_indices", []) or []+ cleaned = sorted(+ {+ int(i)+ for i in indices+ if isinstance(i, (int, float, str))+ and str(i).isdigit()+ and 0 <= int(i) < len(candidates)+ }+ )+ if cleaned:+ return cleaned[:max_keep]+ except Exception:+ pass+ return list(range(min(max_keep, len(candidates))))+++def reciprocal_rank_fusion(+ ranked_lists: list[list[str]],+ *,+ k: int = 60,+) -> list[tuple[str, float]]:+ scores: dict[str, float] = {}+ for ranked in ranked_lists:+ for rank, item in enumerate(ranked):+ if not item:+ continue+ scores[item] = scores.get(item, 0.0) + 1.0 / (k + rank + 1)+ return sorted(scores.items(), key=lambda pair: pair[1], reverse=True)diff --git a/src/lib/neo4j/client.py b/src/lib/neo4j/client.pyindex 6214dd9..e932397 100644--- a/src/lib/neo4j/client.py+++ b/src/lib/neo4j/client.py@@ -259,7 +259,24 @@ class Neo4jClient(GraphClient):for key, value in all_properties.items():cypher_key = self._format_property_key(key)escaped_value = self._format_value(value)- property_assignments.append(f"n.{cypher_key} = {escaped_value}")+ if key == "description":+ property_assignments.append(+ f"n.{cypher_key} = CASE "+ f"WHEN n.{cypher_key} IS NULL OR n.{cypher_key} = '' THEN {escaped_value} "+ f"WHEN {escaped_value} IS NULL OR {escaped_value} = '' THEN n.{cypher_key} "+ f"WHEN toLower(n.{cypher_key}) CONTAINS toLower({escaped_value}) THEN n.{cypher_key} "+ f"WHEN toLower({escaped_value}) CONTAINS toLower(n.{cypher_key}) THEN {escaped_value} "+ f"ELSE n.{cypher_key} + ' | ' + {escaped_value} END"+ )+ elif key in {"source_chunk_ids", "aliases"}:+ property_assignments.append(+ f"n.{cypher_key} = CASE "+ f"WHEN n.{cypher_key} IS NULL THEN {escaped_value} "+ f"WHEN {escaped_value} IS NULL THEN n.{cypher_key} "+ f"ELSE n.{cypher_key} + [x IN {escaped_value} WHERE NOT x IN n.{cypher_key}] END"+ )+ else:+ property_assignments.append(f"n.{cypher_key} = {escaped_value}")properties_set = f"{', '.join(property_assignments)}"diff --git a/src/lib/postgresql/graph_store.py b/src/lib/postgresql/graph_store.pyindex dd1bdf1..864eb18 100644--- a/src/lib/postgresql/graph_store.py+++ b/src/lib/postgresql/graph_store.py@@ -95,7 +95,29 @@ class _BrainGraph:merged = {}if match_uuid in self.graph:merged.update(self.node_data(match_uuid))- merged.update(properties)+ incoming = dict(properties)+ if "description" in incoming and merged.get("description"):+ existing_desc = str(merged.get("description") or "").strip()+ incoming_desc = str(incoming.get("description") or "").strip()+ if existing_desc and incoming_desc and incoming_desc.lower() not in existing_desc.lower():+ if existing_desc.lower() not in incoming_desc.lower():+ incoming["description"] = f"{existing_desc} | {incoming_desc}"+ else:+ incoming["description"] = incoming_desc+ elif existing_desc and not incoming_desc:+ incoming["description"] = existing_desc+ for list_key in ("source_chunk_ids", "aliases"):+ if list_key in incoming or list_key in merged:+ seen = []+ for value in list(merged.get(list_key) or []) + list(+ incoming.get(list_key) or []+ ):+ item = str(value).strip()+ if item and item not in seen:+ seen.append(item)+ if seen:+ incoming[list_key] = seen+ merged.update(incoming)if identification.get("name") is not None:merged["name"] = identification.get("name")merged["labels"] = labels or merged.get("labels") or []diff --git a/src/services/api/constants/requests.py b/src/services/api/constants/requests.pyindex 8c0311e..79837c8 100644--- a/src/services/api/constants/requests.py+++ b/src/services/api/constants/requests.py@@ -394,6 +394,8 @@ class GetContextRequestBody(BaseModel):brain_id: str = "default"historical_limit: int = 10max_facts: int = 40+ max_passages: int = 8+ apply_fact_filter: bool = Trueclass GetContextTriple(BaseModel):@@ -409,3 +411,4 @@ class GetContextResponse(BaseModel):text_context: strtriples: List[GetContextTriple]historical_context: List[str] = []+ source_passages: List[str] = []diff --git a/src/services/api/controllers/retrieve.py b/src/services/api/controllers/retrieve.pyindex 94d2457..98d2f1c 100644--- a/src/services/api/controllers/retrieve.py+++ b/src/services/api/controllers/retrieve.py@@ -20,6 +20,7 @@ from src.constants.embeddings import Vectorfrom src.constants.kg import IdentificationParams, Node, Predicatefrom src.core.search.entities import search_entitiesfrom src.core.search.entity_info import EventSynergyRetriever, MatchPath+from src.core.search.fact_filter import filter_relevant_facts, reciprocal_rank_fusionfrom src.core.search.relationships import search_relationshipsfrom src.utils.vector_search import VectorSearchFacadefrom src.services.api.constants.requests import (@@ -41,6 +42,8 @@ vector_search = VectorSearchFacade(vector_store_adapter)_MAX_NOUN_CHUNKS = 5_MAX_DOSSIER_ENTITIES = 5_DOSSIER_MAX_DEPTH = 3+_SEED_K = 20+_PASSAGE_K = 12async def retrieve_data(@@ -405,6 +408,15 @@ def _fact_key(r: Predicate, r2: Predicate) -> tuple[str, str]:return (getattr(r, "uuid", None) or "", getattr(r2, "uuid", None) or "")+def _is_currently_valid(predicate: Predicate) -> bool:+ props = getattr(predicate, "properties", None) or {}+ if props.get("invalid_at"):+ return False+ if getattr(predicate, "deprecated", False):+ return False+ return True++def _collect_query_variants(text: str, elements) -> list[str]:seen: set[str] = set()variants: list[str] = []@@ -446,6 +458,7 @@ def _seed_nodes_for_text(text: str, brain_id: str) -> list[tuple[str, float, strnode_vectors = vector_search.search_nodes(embeddings,brain_id=brain_id,+ k=_SEED_K,)for vector in node_vectors:meta = vector.metadata or {}@@ -461,6 +474,7 @@ def _seed_nodes_for_text(text: str, brain_id: str) -> list[tuple[str, float, strrel_vectors = vector_search.search_relationships(embeddings,brain_id=brain_id,+ k=_SEED_K,)for vector in rel_vectors:meta = vector.metadata or {}@@ -475,6 +489,86 @@ def _seed_nodes_for_text(text: str, brain_id: str) -> list[tuple[str, float, strreturn seeds+def _extract_source_chunk_ids(*items: Any) -> list[str]:+ ids: list[str] = []+ seen: set[str] = set()+ for item in items:+ props = getattr(item, "properties", None) or {}+ for value in props.get("source_chunk_ids") or []:+ chunk_id = str(value).strip()+ if chunk_id and chunk_id not in seen:+ seen.add(chunk_id)+ ids.append(chunk_id)+ single = props.get("source_chunk_id")+ if single:+ chunk_id = str(single).strip()+ if chunk_id and chunk_id not in seen:+ seen.add(chunk_id)+ ids.append(chunk_id)+ return ids+++def _retrieve_passages(+ text: str, brain_id: str, *, limit: int+) -> list[tuple[str, float, str]]:+ """Return (chunk_id, score, text) ranked passages via vector + keyword fusion."""+ text_embeddings = embeddings_adapter.embed_text(text)+ vector_hits = vector_search.search_data(+ text_embeddings.embeddings,+ brain_id=brain_id,+ k=max(limit, _PASSAGE_K),+ )+ vector_ids: list[str] = []+ id_to_text: dict[str, str] = {}+ id_to_distance: dict[str, float] = {}+ for vector in vector_hits:+ meta = vector.metadata or {}+ resource_id = meta.get("resource_id") or getattr(vector, "id", None)+ if not resource_id:+ continue+ resource_id = str(resource_id)+ vector_ids.append(resource_id)+ id_to_distance[resource_id] = (+ float(vector.distance)+ if vector.distance is not None+ else float("inf")+ )++ keyword_ids: list[str] = []+ try:+ search_result = data_adapter.search(text, brain_id)+ for chunk in getattr(search_result, "text_chunks", None) or []:+ chunk_id = str(getattr(chunk, "id", "") or "")+ if not chunk_id:+ continue+ keyword_ids.append(chunk_id)+ id_to_text[chunk_id] = getattr(chunk, "text", "") or ""+ except Exception:+ pass++ fused = reciprocal_rank_fusion([vector_ids, keyword_ids])+ ranked_ids = [item for item, _ in fused[:limit]]+ missing = [cid for cid in ranked_ids if cid not in id_to_text]+ if missing:+ try:+ chunks, _ = data_adapter.get_text_chunks_by_ids(+ missing, False, brain_id+ )+ for chunk in chunks:+ id_to_text[str(chunk.id)] = chunk.text or ""+ except Exception:+ pass++ passages: list[tuple[str, float, str]] = []+ for chunk_id in ranked_ids:+ body = (id_to_text.get(chunk_id) or "").strip()+ if not body:+ continue+ score = id_to_distance.get(chunk_id, 1.0 / (1.0 + len(passages)))+ passages.append((chunk_id, float(score), body))+ return passages++def _flatten_match_path(path: MatchPath) -> list[str]:lines: list[str] = []current: MatchPath | None = path@@ -499,22 +593,16 @@ def _flatten_match_path(path: MatchPath) -> list[str]:async def get_context(request: GetContextRequestBody) -> GetContextResponse:"""Retrieve contextual information for a text.-- Parameters:- request (GetContextRequestBody): Request body containing:- - text: The text to search context for.- - brain_id: The brain/workspace identifier to query.-- Returns:- GetContextResponse: Response containing the context information."""elements = _entity_extractor.extract_elements(request.text)variants = _collect_query_variants(request.text, elements)historical_context: list[str] = []+ source_passages: list[str] = []candidate_lock = threading.Lock()candidates: list[dict[str, Any]] = []seed_hits: list[tuple[str, float, str]] = []+ passage_hits: list[tuple[str, float, str]] = []def _collect_facts_for_variant(text: str) -> None:seeds = _seed_nodes_for_text(text, request.brain_id)@@ -533,6 +621,8 @@ async def get_context(request: GetContextRequestBody) -> GetContextResponse:)local: list[dict[str, Any]] = []for n, r, m, r2, b in neighbors:+ if not _is_currently_valid(r) or not _is_currently_valid(r2):+ continuedistances = [uuid_to_distance[u]for u in (@@ -549,14 +639,33 @@ async def get_context(request: GetContextRequestBody) -> GetContextResponse:"triple": (n, r, m, r2, b),"score": score,"key": _fact_key(r, r2),+ "text": _format_event_fact(n, r, m, r2, b),+ "chunk_ids": _extract_source_chunk_ids(n, r, m, r2, b),})if local:with candidate_lock:candidates.extend(local)+ def _collect_passages() -> None:+ nonlocal passage_hits+ passage_hits = _retrieve_passages(+ request.text,+ request.brain_id,+ limit=max(1, int(getattr(request, "max_passages", 8) or 8)),+ )+async def _get_historical_context():nonlocal historical_context+ passages = await asyncio.to_thread(+ _retrieve_passages,+ request.text,+ request.brain_id,+ limit=max(request.historical_limit, _PASSAGE_K),+ )+ if passages:+ historical_context = [text for _, _, text in passages[: request.historical_limit]]+ return historical_contexttext_chunks, structured_data = await asyncio.gather(asyncio.to_thread(data_adapter.get_last_text_chunks,@@ -579,6 +688,7 @@ async def get_context(request: GetContextRequestBody) -> GetContextResponse:futures = [asyncio.to_thread(_collect_facts_for_variant, variant) for variant in variants]+ futures.append(asyncio.to_thread(_collect_passages))futures.append(_get_historical_context())await asyncio.gather(*futures)@@ -591,19 +701,56 @@ async def get_context(request: GetContextRequestBody) -> GetContextResponse:ranked = sorted(deduped.values(), key=lambda c: c["score"])max_facts = max(1, int(getattr(request, "max_facts", 40) or 40))- curated = ranked[:max_facts]++ if getattr(request, "apply_fact_filter", True) and ranked:+ keep = filter_relevant_facts(+ request.text,+ [c["text"] for c in ranked],+ max_keep=max_facts,+ )+ curated = [ranked[i] for i in keep if 0 <= i < len(ranked)]+ if not curated:+ curated = ranked[:max_facts]+ else:+ curated = ranked[:max_facts]text_lines: list[str] = []triples: list[GetContextTriple] = []+ provenance_ids: list[str] = []for candidate in curated:n, r, m, r2, b = candidate["triple"]- text_lines.append(_format_event_fact(n, r, m, r2, b))+ text_lines.append(candidate["text"])triples.append(GetContextTriple(identified_entity=candidate["identified_entity"],triple=(n, r, m, r2, b),))+ provenance_ids.extend(candidate.get("chunk_ids") or [])++ for chunk_id, _, body in passage_hits:+ if body and body not in source_passages:+ source_passages.append(body)+ if chunk_id not in provenance_ids:+ provenance_ids.append(chunk_id)++ if provenance_ids:+ try:+ chunks, _ = await asyncio.to_thread(+ data_adapter.get_text_chunks_by_ids,+ provenance_ids[:40],+ False,+ request.brain_id,+ )+ for chunk in chunks:+ body = (chunk.text or "").strip()+ if body and body not in source_passages:+ source_passages.append(body)+ except Exception:+ pass++ max_passages = max(1, int(getattr(request, "max_passages", 8) or 8))+ source_passages = source_passages[:max_passages]entity_scores: dict[str, float] = {}for _uuid, distance, entity_name in seed_hits:@@ -648,10 +795,12 @@ async def get_context(request: GetContextRequestBody) -> GetContextResponse:dossier_lines = await asyncio.to_thread(_run_dossiers)- text_context = "\n".join(text_lines + dossier_lines)+ passage_block = [f"[passage] {p}" for p in source_passages]+ text_context = "\n".join(passage_block + text_lines + dossier_lines)return GetContextResponse(text_context=text_context,triples=triples,historical_context=historical_context,+ source_passages=source_passages,)diff --git a/src/utils/dates.py b/src/utils/dates.pyindex 87cc3e0..cfe6cdb 100644--- a/src/utils/dates.py+++ b/src/utils/dates.py@@ -1,5 +1,6 @@-from datetime import datetime+from datetime import datetime, timedeltafrom typing import Optional+import re_DATE_INPUT_FORMATS = ("%d/%m/%Y",@@ -14,14 +15,109 @@ _DATE_INPUT_FORMATS = ("%d %b %Y",)+_RELATIVE_PATTERNS = (+ (re.compile(r"^yesterday$", re.I), lambda ref: ref - timedelta(days=1)),+ (re.compile(r"^tomorrow$", re.I), lambda ref: ref + timedelta(days=1)),+ (re.compile(r"^today$", re.I), lambda ref: ref),+ (re.compile(r"^last\s+week$", re.I), lambda ref: ref - timedelta(days=7)),+ (re.compile(r"^next\s+week$", re.I), lambda ref: ref + timedelta(days=7)),+ (re.compile(r"^last\s+month$", re.I), lambda ref: _shift_months(ref, -1)),+ (re.compile(r"^next\s+month$", re.I), lambda ref: _shift_months(ref, 1)),+ (+ re.compile(r"^(\d+)\s+days?\s+ago$", re.I),+ lambda ref, m: ref - timedelta(days=int(m.group(1))),+ ),+ (+ re.compile(r"^in\s+(\d+)\s+days?$", re.I),+ lambda ref, m: ref + timedelta(days=int(m.group(1))),+ ),+ (+ re.compile(r"^(\d+)\s+weeks?\s+ago$", re.I),+ lambda ref, m: ref - timedelta(weeks=int(m.group(1))),+ ),+)-def normalize_date_string(value: Optional[str]) -> Optional[str]:+_WEEKDAY_NAMES = {+ "monday": 0,+ "tuesday": 1,+ "wednesday": 2,+ "thursday": 3,+ "friday": 4,+ "saturday": 5,+ "sunday": 6,+}+++def _shift_months(ref: datetime, delta: int) -> datetime:+ month = ref.month - 1 + delta+ year = ref.year + month // 12+ month = month % 12 + 1+ day = min(ref.day, [31, 29 if year % 4 == 0 else 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31][month - 1])+ return ref.replace(year=year, month=month, day=day)+++def parse_date_string(value: Optional[str]) -> Optional[datetime]:if not value or not isinstance(value, str):- return value+ return Nonecleaned = value.strip()for fmt in _DATE_INPUT_FORMATS:try:- return datetime.strptime(cleaned, fmt).strftime("%d/%m/%Y")+ return datetime.strptime(cleaned, fmt)except ValueError:continue- return value+ return None+++def normalize_date_string(value: Optional[str]) -> Optional[str]:+ if not value or not isinstance(value, str):+ return value+ parsed = parse_date_string(value)+ if parsed is not None:+ return parsed.strftime("%d/%m/%Y")+ return value.strip()+++def resolve_relative_date(+ value: Optional[str],+ reference_time: Optional[str | datetime] = None,+) -> Optional[str]:+ if not value or not isinstance(value, str):+ return value+ cleaned = value.strip()+ absolute = normalize_date_string(cleaned)+ if absolute != cleaned or parse_date_string(cleaned) is not None:+ return absolute++ if reference_time is None:+ return cleaned+ if isinstance(reference_time, datetime):+ ref = reference_time+ else:+ ref = parse_date_string(str(reference_time))+ if ref is None:+ return cleaned++ for pattern, resolver in _RELATIVE_PATTERNS:+ match = pattern.match(cleaned)+ if not match:+ continue+ if match.lastindex:+ resolved = resolver(ref, match)+ else:+ resolved = resolver(ref)+ return resolved.strftime("%d/%m/%Y")++ weekday_match = re.match(r"^(last|this|next)\s+(monday|tuesday|wednesday|thursday|friday|saturday|sunday)$", cleaned, re.I)+ if weekday_match:+ mode = weekday_match.group(1).lower()+ target = _WEEKDAY_NAMES[weekday_match.group(2).lower()]+ current = ref.weekday()+ if mode == "this":+ delta = (target - current) % 7+ elif mode == "next":+ delta = (target - current) % 7 or 7+ else:+ delta = -((current - target) % 7 or 7)+ return (ref + timedelta(days=delta)).strftime("%d/%m/%Y")++ return cleaneddiff --git a/src/utils/text_chunking.py b/src/utils/text_chunking.pynew file mode 100644index 0000000..5282c4e--- /dev/null+++ b/src/utils/text_chunking.py@@ -0,0 +1,27 @@+def chunk_text(text: str, max_chars: int = 6000) -> list[str]:+ cleaned = (text or "").strip()+ if not cleaned or len(cleaned) <= max_chars:+ return [cleaned] if cleaned else [""]+ paragraphs = [p.strip() for p in cleaned.split("\n\n") if p.strip()]+ if not paragraphs:+ paragraphs = [cleaned]+ chunks: list[str] = []+ current = ""+ for paragraph in paragraphs:+ candidate = f"{current}\n\n{paragraph}".strip() if current else paragraph+ if len(candidate) <= max_chars:+ current = candidate+ continue+ if current:+ chunks.append(current)+ if len(paragraph) <= max_chars:+ current = paragraph+ continue+ for i in range(0, len(paragraph), max_chars):+ piece = paragraph[i : i + max_chars].strip()+ if piece:+ chunks.append(piece)+ current = ""+ if current:+ chunks.append(current)+ return chunks or [cleaned]diff --git a/src/workers/tasks/ingestion.py b/src/workers/tasks/ingestion.pyindex 460d5f6..2ae1de7 100644--- a/src/workers/tasks/ingestion.py+++ b/src/workers/tasks/ingestion.py@@ -29,6 +29,7 @@ from uuid import uuid4from src.core.saving.identity import (stable_node_id,stable_relationship_id,+ stamp_provenance,)from pydantic import BaseModel@@ -190,17 +191,25 @@ def _is_same_graph_node(graph_node: Node, entity) -> bool:def _normalize_relationship_dates(relationships: List[ArchitectAgentRelationship],+ reference_time: Optional[str] = None,) -> None:+ from src.utils.dates import resolve_relative_date+for relationship in relationships:for entity in (relationship.tail, relationship.tip):if (entity.type or "").strip().upper() == "DATE":- entity.name = normalize_date_string(entity.name)+ entity.name = resolve_relative_date(entity.name, reference_time)if getattr(entity, "happened_at", None):- entity.happened_at = normalize_date_string(entity.happened_at)+ entity.happened_at = resolve_relative_date(+ entity.happened_at, reference_time+ )def _resolve_relationship_entities(- relationships: List[ArchitectAgentRelationship], brain_id: str+ relationships: List[ArchitectAgentRelationship],+ brain_id: str,+ source_chunk_id: Optional[str] = None,+ source_timestamp: Optional[str] = None,) -> None:unique_entities = {}for relationship in relationships:@@ -306,6 +315,25 @@ def _resolve_relationship_entities(entity.name = node.nameif getattr(node, "happened_at", None) and not entity.happened_at:entity.happened_at = node.happened_at+ existing_props = getattr(node, "properties", None) or {}+ entity.properties = stamp_provenance(+ entity.properties,+ source_chunk_id=source_chunk_id,+ source_timestamp=source_timestamp,+ existing_properties=existing_props,+ )+ if getattr(node, "description", None):+ entity.description = _merge_description(+ node.description, entity.description+ )+ aliases = list(existing_props.get("aliases") or [])+ if entity.name and entity.name not in aliases and entity.name != node.name:+ aliases.append(entity.name)+ if aliases:+ entity.properties = {+ **(entity.properties or {}),+ "aliases": aliases,+ }_apply_resolutions()resolved_uuids = {node.uuid for node in resolutions.values()}@@ -382,7 +410,22 @@ def _resolve_relationship_entities(_apply_resolutions()for relationship in relationships:+ relationship.properties = stamp_provenance(+ relationship.properties,+ source_chunk_id=source_chunk_id,+ source_timestamp=source_timestamp,+ )+ if source_timestamp and not (relationship.properties or {}).get("valid_at"):+ relationship.properties = {+ **(relationship.properties or {}),+ "valid_at": source_timestamp,+ }for entity in (relationship.tail, relationship.tip):+ entity.properties = stamp_provenance(+ entity.properties,+ source_chunk_id=source_chunk_id,+ source_timestamp=source_timestamp,+ )if resolutions.get(_entity_key(entity)):continueentity.uuid = stable_node_id(@@ -402,6 +445,65 @@ def _resolve_relationship_entities(relationship.flow_key,)++def _merge_description(+ existing: Optional[str], incoming: Optional[str]+) -> Optional[str]:+ existing_clean = (existing or "").strip()+ incoming_clean = (incoming or "").strip()+ if not existing_clean:+ return incoming_clean or None+ if not incoming_clean:+ return existing_clean+ if incoming_clean.lower() in existing_clean.lower():+ return existing_clean+ if existing_clean.lower() in incoming_clean.lower():+ return incoming_clean+ return f"{existing_clean} | {incoming_clean}"+++def _invalidate_superseded_relationships(+ relationship: ArchitectAgentRelationship,+ *,+ brain_id: str,+) -> None:+ """Mark older same-type edges from the same subject as invalid when tip changes."""+ try:+ neighbors = graph_adapter.get_neighbors(+ [relationship.tail.uuid], brain_id=brain_id+ )+ except Exception:+ return+ pairs = (neighbors or {}).get(relationship.tail.uuid) or []+ valid_at = (relationship.properties or {}).get("valid_at") or (+ relationship.properties or {}+ ).get("source_timestamp")+ for predicate, neighbor in pairs:+ if not predicate or not neighbor:+ continue+ if (predicate.name or "").strip().upper() != (relationship.name or "").strip().upper():+ continue+ if neighbor.uuid == relationship.tip.uuid:+ continue+ if getattr(predicate, "uuid", None) == relationship.uuid:+ continue+ props = getattr(predicate, "properties", None) or {}+ if props.get("invalid_at"):+ continue+ try:+ graph_adapter.update_properties(+ predicate.uuid,+ "relationship",+ brain_id=brain_id,+ new_properties={+ "invalid_at": valid_at+ or datetime.datetime.utcnow().strftime("%d/%m/%Y"),+ "deprecated": True,+ },+ )+ except Exception as exc:+ print(f"[!] Failed to invalidate relationship {predicate.uuid}: {exc}")+def format_textual_data(data: dict, include_keys: bool = True) -> str:def format_value(v):if isinstance(v, str):@@ -511,7 +613,13 @@ def ingest_data(self, args: dict):brain_id=payload.brain_id,)- enrich_result = enrich_kg_from_input(source_text, brain_id=payload.brain_id)+ enrich_result = enrich_kg_from_input(+ source_text,+ brain_id=payload.brain_id,+ source_chunk_id=text_chunk.id,+ source_timestamp=payload.source_timestamp,+ preferred_extraction_entities=payload.preferred_extraction_entities,+ )parent_task_id = self.request.idsteps = []if enrich_result.enrichment_relationships:@@ -643,8 +751,25 @@ def process_architect_relationships(self, args: dict):relationships = [ArchitectAgentRelationship(**rel_data) for rel_data in relationships_data]- _normalize_relationship_dates(relationships)- _resolve_relationship_entities(relationships, brain_id)+ reference_time = None+ source_chunk_id = None+ for rel in relationships:+ props = rel.properties or {}+ if not reference_time:+ reference_time = props.get("source_timestamp") or props.get("valid_at")+ if not source_chunk_id:+ ids = props.get("source_chunk_ids") or []+ if ids:+ source_chunk_id = ids[-1]+ elif props.get("source_chunk_id"):+ source_chunk_id = props.get("source_chunk_id")+ _normalize_relationship_dates(relationships, reference_time=reference_time)+ _resolve_relationship_entities(+ relationships,+ brain_id,+ source_chunk_id=source_chunk_id,+ source_timestamp=reference_time,+ )item_errors: List[dict] = []with ThreadPoolExecutor(max_workers=10) as io_executor:@@ -844,6 +969,10 @@ def process_architect_relationships(self, args: dict):),brain_id=brain_id,)+ _invalidate_superseded_relationships(+ relationship,+ brain_id=brain_id,+ )except FutureTimeoutError:rel_name = getattr(relationship, "name", "unknown")item_errors.append(diff --git a/tests/test_graph_upsert_contract.py b/tests/test_graph_upsert_contract.pyindex ddd2626..25017e8 100644--- a/tests/test_graph_upsert_contract.py+++ b/tests/test_graph_upsert_contract.py@@ -52,7 +52,7 @@ class NetworkXUpsertContractTests(unittest.TestCase):self.assertEqual(second, "alice-1")self.assertEqual(len(list(brain.graph.nodes)), 1)self.assertEqual(brain.node_data("alice-1")["uuid"], "alice-1")- self.assertEqual(brain.node_data("alice-1")["description"], "v2")+ self.assertEqual(brain.node_data("alice-1")["description"], "v1 | v2")def test_same_name_different_uuid_does_not_overwrite_existing_uuid(self):from src.lib.postgresql.graph_store import _BrainGraphdiff --git a/tests/test_graphrag_upgrade_helpers.py b/tests/test_graphrag_upgrade_helpers.pynew file mode 100644index 0000000..7a0ae65--- /dev/null+++ b/tests/test_graphrag_upgrade_helpers.py@@ -0,0 +1,105 @@+import os+import unittest++ENV_DEFAULTS = {+ "BRAINPAT_TOKEN": "test-token",+ "MODELS_MODE": "local",+ "EMBEDDINGS_LOCAL_MODEL": "local-model",+ "EMBEDDINGS_SMALL_MODEL": "small-model",+ "EMBEDDING_NODES_DIMENSION": "3",+ "EMBEDDING_TRIPLETS_DIMENSION": "3",+ "EMBEDDING_OBSERVATIONS_DIMENSION": "3",+ "EMBEDDING_DATA_DIMENSION": "3",+ "EMBEDDING_RELATIONSHIPS_DIMENSION": "3",+ "REDIS_HOST": "localhost",+ "REDIS_PORT": "6379",+ "NEO4J_HOST": "localhost",+ "NEO4J_PORT": "7687",+ "NEO4J_USERNAME": "neo4j",+ "NEO4J_PASSWORD": "password",+ "MILVUS_HOST": "localhost",+ "MILVUS_PORT": "19530",+ "MONGO_CONNECTION_STRING": "mongodb://localhost:27017",+ "CELERY_WORKER_CONCURRENCY": "1",+ "OLLAMA_HOST": "localhost",+ "OLLAMA_PORT": "11434",+ "OLLAMA_LLM_SMALL_MODEL": "small",+ "OLLAMA_LLM_LARGE_MODEL": "large",+ "POSTGRES_HOST": "localhost",+ "POSTGRES_PORT": "5432",+ "POSTGRES_USERNAME": "postgres",+ "POSTGRES_PASSWORD": "postgres",+}+for key, value in ENV_DEFAULTS.items():+ os.environ.setdefault(key, value)++from src.core.saving.identity import merge_source_chunk_ids, stamp_provenance+from src.core.search.fact_filter import reciprocal_rank_fusion+from src.utils.dates import normalize_date_string, resolve_relative_date+++class ProvenanceHelpersTests(unittest.TestCase):+ def test_merge_source_chunk_ids_dedupes(self):+ merged = merge_source_chunk_ids(+ ["a", "b"],+ "b",+ ["c"],+ None,+ "a",+ )+ self.assertEqual(merged, ["a", "b", "c"])++ def test_stamp_provenance_accumulates(self):+ props = stamp_provenance(+ {"source_chunk_ids": ["chunk-1"]},+ source_chunk_id="chunk-2",+ source_timestamp="01/05/2023",+ existing_properties={"source_chunk_ids": ["chunk-0"]},+ )+ self.assertEqual(props["source_chunk_ids"], ["chunk-0", "chunk-1", "chunk-2"])+ self.assertEqual(props["source_timestamp"], "01/05/2023")+++class RelativeDateTests(unittest.TestCase):+ def test_normalize_absolute(self):+ self.assertEqual(normalize_date_string("2023-05-15"), "15/05/2023")++ def test_resolve_yesterday(self):+ self.assertEqual(+ resolve_relative_date("yesterday", "15/05/2023"),+ "14/05/2023",+ )++ def test_resolve_last_tuesday(self):+ # 15/05/2023 is a Monday+ self.assertEqual(+ resolve_relative_date("last Tuesday", "15/05/2023"),+ "09/05/2023",+ )+++class RrfTests(unittest.TestCase):+ def test_rrf_prefers_items_ranked_high_in_multiple_lists(self):+ fused = reciprocal_rank_fusion(+ [+ ["a", "b", "c"],+ ["b", "a", "d"],+ ]+ )+ ranked = [item for item, _ in fused]+ self.assertEqual(ranked[0], "a")+ self.assertIn("b", ranked[:2])+++class ScoutChunkingTests(unittest.TestCase):+ def test_chunk_text_splits_long_input(self):+ from src.utils.text_chunking import chunk_text++ text = ("paragraph one.\n\n" * 50) + ("x" * 7000)+ chunks = chunk_text(text, max_chars=1000)+ self.assertGreater(len(chunks), 1)+ self.assertTrue(all(len(c) <= 1000 or c.count("x") > 0 for c in chunks))+++if __name__ == "__main__":+ unittest.main()