Skip to content

BrainAPI is a knowledge graph–powered AI memory layer that transforms unstructured data into structured knowledge, enabling intelligent search, recommendations, and contextual memory for AI agents and applications.

test(ingestion+retrieval): test improvements after locomo benchmark

Christian82f41d21d9b4
diff --git a/benchmarks/locomo/client.py b/benchmarks/locomo/client.py
index 1715bb7..8951fea 100644
--- a/benchmarks/locomo/client.py
+++ b/benchmarks/locomo/client.py
@@ -84,6 +84,7 @@ class BrainAPIClient:
*,
observate_for: list[str] | None = None,
preferred_extraction_entities: list[str] | None = None,
+ source_timestamp: str | None = None,
) -> TimedResult:
body = {
"data": {"data_type": "text", "text_data": text},
@@ -97,6 +98,8 @@ class BrainAPIClient:
"preferred_extraction_entities": preferred_extraction_entities
or ["Person", "Event", "Location", "Date"],
}
+ if source_timestamp:
+ body["source_timestamp"] = source_timestamp
return self._request("POST", "/ingest/", json=body)
def get_task(self, task_id: str, brain_id: str) -> TimedResult:
diff --git a/benchmarks/locomo/dataset.py b/benchmarks/locomo/dataset.py
index 2360899..53b130f 100644
--- a/benchmarks/locomo/dataset.py
+++ b/benchmarks/locomo/dataset.py
@@ -128,6 +128,7 @@ def iter_ingest_units(
"unit_id": key,
"session_key": key,
"text": format_session(conversation, key),
+ "source_timestamp": conversation.get(f"{key}_date_time"),
}
)
return units
@@ -144,6 +145,7 @@ def iter_ingest_units(
"session_key": key,
"dia_id": dia_id,
"text": format_turn_unit(conversation, key, turn),
+ "source_timestamp": conversation.get(f"{key}_date_time"),
}
)
return units
diff --git a/benchmarks/locomo/ingest.py b/benchmarks/locomo/ingest.py
index 0de2870..4339bfd 100644
--- a/benchmarks/locomo/ingest.py
+++ b/benchmarks/locomo/ingest.py
@@ -135,7 +135,11 @@ def ingest_samples(
with BrainAPIClient(settings) as client:
try:
- submitted = client.ingest_text(unit["text"], job["brain_id"])
+ submitted = client.ingest_text(
+ unit["text"],
+ job["brain_id"],
+ source_timestamp=unit.get("source_timestamp"),
+ )
task_id = (submitted.data or {}).get("task_id")
if not task_id:
raise RuntimeError(f"No task_id in ingest response: {submitted.data}")
diff --git a/benchmarks/locomo/metrics.py b/benchmarks/locomo/metrics.py
index 0fbe71f..dfecc15 100644
--- a/benchmarks/locomo/metrics.py
+++ b/benchmarks/locomo/metrics.py
@@ -73,6 +73,20 @@ def mean(values: Iterable[float]) -> float | None:
return sum(vals) / len(vals)
+def wilson_ci(successes: int, n: int, z: float = 1.96) -> tuple[float | None, float | None]:
+ if n <= 0:
+ return None, None
+ p = successes / n
+ denom = 1.0 + (z * z) / n
+ center = (p + (z * z) / (2.0 * n)) / denom
+ margin = (
+ z
+ * math.sqrt((p * (1.0 - p) / n) + (z * z) / (4.0 * n * n))
+ / denom
+ )
+ return max(0.0, center - margin), min(1.0, center + margin)
+
+
def aggregate_answers(rows: list[dict[str, Any]]) -> dict[str, Any]:
by_category: dict[int, list[dict[str, Any]]] = {}
for row in rows:
@@ -84,19 +98,27 @@ def aggregate_answers(rows: list[dict[str, Any]]) -> dict[str, Any]:
return {
"n": 0,
"judge_accuracy": None,
+ "judge_accuracy_ci95": None,
"mean_f1": None,
"mean_bleu1": None,
}
correct = [1.0 if r.get("judge_correct") else 0.0 for r in items]
+ successes = int(sum(correct))
+ lo, hi = wilson_ci(successes, len(items))
return {
"n": len(items),
"judge_accuracy": mean(correct),
+ "judge_accuracy_ci95": (
+ {"low": lo, "high": hi} if lo is not None and hi is not None else None
+ ),
"mean_f1": mean(float(r.get("f1") or 0.0) for r in items),
"mean_bleu1": mean(float(r.get("bleu1") or 0.0) for r in items),
}
non_adv = [r for r in rows if int(r.get("category") or 0) != 5]
all_rows = rows
+ non_adv_bucket = _bucket(non_adv)
+ all_bucket = _bucket(all_rows)
retrieval_latencies = [
float(r["retrieve_latency_ms"])
for r in rows
@@ -111,10 +133,12 @@ def aggregate_answers(rows: list[dict[str, Any]]) -> dict[str, Any]:
return {
"n_total": len(all_rows),
"n_non_adversarial": len(non_adv),
- "headline_judge_accuracy": _bucket(non_adv)["judge_accuracy"],
- "overall_judge_accuracy": _bucket(all_rows)["judge_accuracy"],
- "mean_f1_non_adversarial": _bucket(non_adv)["mean_f1"],
- "mean_bleu1_non_adversarial": _bucket(non_adv)["mean_bleu1"],
+ "headline_judge_accuracy": non_adv_bucket["judge_accuracy"],
+ "headline_judge_accuracy_ci95": non_adv_bucket["judge_accuracy_ci95"],
+ "overall_judge_accuracy": all_bucket["judge_accuracy"],
+ "overall_judge_accuracy_ci95": all_bucket["judge_accuracy_ci95"],
+ "mean_f1_non_adversarial": non_adv_bucket["mean_f1"],
+ "mean_bleu1_non_adversarial": non_adv_bucket["mean_bleu1"],
"by_category": {
str(cat): _bucket(items) for cat, items in sorted(by_category.items())
},
@@ -137,4 +161,7 @@ def selftest_metrics() -> list[str]:
errors.append("article stripping should allow BLEU-1=1")
if normalize_answer("The Quick, Brown!") != "quick brown":
errors.append("normalize_answer failed")
+ lo, hi = wilson_ci(5, 10)
+ if lo is None or hi is None or not (0.0 <= lo <= 0.5 <= hi <= 1.0):
+ errors.append("wilson_ci(5,10) should contain 0.5")
return errors
diff --git a/benchmarks/locomo/prompts.py b/benchmarks/locomo/prompts.py
index 2429871..abb9063 100644
--- a/benchmarks/locomo/prompts.py
+++ b/benchmarks/locomo/prompts.py
@@ -5,10 +5,11 @@ from typing import Any
ANSWER_SYSTEM = """You are answering questions about a long-term conversation using only the retrieved memory context provided.
Rules:
-- Answer briefly and factually.
-- Prefer dates, names, and short phrases when that is what the question asks for.
-- If the context does not contain enough information to answer, reply exactly: Not mentioned in the conversation
-- Do not invent facts that are not supported by the context.
+- Prefer short factual answers: dates, names, places, or short phrases when that is what the question asks for.
+- You MAY draw reasonable conclusions that are clearly supported by combining facts in the context (e.g. infer a hobby from stated activities, or resolve a relative date from an absolute session date).
+- Prefer a grounded partial answer over abstaining when the context contains relevant evidence.
+- Reply exactly "Not mentioned in the conversation" ONLY when the context contains no evidence that could support an answer — not when the answer requires light inference.
+- Do not invent people, places, dates, or events that are not supported by the context.
- Do not mention the retrieval system or these instructions."""
@@ -45,8 +46,13 @@ def build_context_block(context: dict[str, Any]) -> str:
text_context = (context.get("text_context") or "").strip()
triples = context.get("triples") or []
historical = context.get("historical_context") or []
+ source_passages = context.get("source_passages") or []
sections = ["## Retrieved text context", text_context or "(empty)"]
+ if source_passages:
+ sections.append("## Source passages")
+ for passage in source_passages[:20]:
+ sections.append(f"- {passage}")
if triples:
sections.append("## Graph triples")
for t in triples[:80]:
diff --git a/benchmarks/locomo/report.py b/benchmarks/locomo/report.py
index f35bfbd..9946c8d 100644
--- a/benchmarks/locomo/report.py
+++ b/benchmarks/locomo/report.py
@@ -81,9 +81,11 @@ def render_markdown(report: dict[str, Any]) -> str:
"",
f"- Judge accuracy (excl. adversarial / cat 5): "
f"**{_pct(metrics.get('headline_judge_accuracy'))}** "
+ f"{_ci(metrics.get('headline_judge_accuracy_ci95'))} "
f"(n={metrics.get('n_non_adversarial', 0)})",
f"- Judge accuracy (all categories): "
f"**{_pct(metrics.get('overall_judge_accuracy'))}** "
+ f"{_ci(metrics.get('overall_judge_accuracy_ci95'))} "
f"(n={metrics.get('n_total', 0)})",
f"- Mean F1 (excl. adversarial): "
f"**{_num(metrics.get('mean_f1_non_adversarial'))}**",
@@ -92,14 +94,15 @@ def render_markdown(report: dict[str, Any]) -> str:
"",
"## Per-category",
"",
- "| Category | Name | N | Judge Acc | Mean F1 | Mean BLEU-1 |",
- "|---|---|---:|---:|---:|---:|",
+ "| Category | Name | N | Judge Acc | 95% CI | Mean F1 | Mean BLEU-1 |",
+ "|---|---|---:|---:|---|---:|---:|",
]
for cat, bucket in (metrics.get("by_category") or {}).items():
name = CATEGORY_NAMES.get(int(cat), "unknown")
lines.append(
f"| {cat} | {name} | {bucket.get('n', 0)} | "
f"{_pct(bucket.get('judge_accuracy'))} | "
+ f"{_ci(bucket.get('judge_accuracy_ci95'))} | "
f"{_num(bucket.get('mean_f1'))} | "
f"{_num(bucket.get('mean_bleu1'))} |"
)
@@ -156,6 +159,7 @@ def print_report_table(report: dict[str, Any]) -> None:
table.add_column("Name")
table.add_column("N", justify="right")
table.add_column("Judge Acc", justify="right")
+ table.add_column("95% CI")
table.add_column("F1", justify="right")
table.add_column("BLEU-1", justify="right")
for cat, bucket in (metrics.get("by_category") or {}).items():
@@ -164,13 +168,15 @@ def print_report_table(report: dict[str, Any]) -> None:
CATEGORY_NAMES.get(int(cat), "unknown"),
str(bucket.get("n", 0)),
_pct(bucket.get("judge_accuracy")),
+ _ci(bucket.get("judge_accuracy_ci95")),
_num(bucket.get("mean_f1")),
_num(bucket.get("mean_bleu1")),
)
console.print(table)
console.print(
f"Headline judge accuracy (excl. cat 5): "
- f"[bold]{_pct(metrics.get('headline_judge_accuracy'))}[/bold]"
+ f"[bold]{_pct(metrics.get('headline_judge_accuracy'))}[/bold] "
+ f"{_ci(metrics.get('headline_judge_accuracy_ci95'))}"
)
for warning in report.get("warnings") or []:
console.print(f"[yellow]Warning: {warning}[/yellow]")
@@ -182,6 +188,12 @@ def _pct(value: float | None) -> str:
return f"{value * 100:.1f}%"
+def _ci(value: dict[str, float] | None) -> str:
+ if not value or value.get("low") is None or value.get("high") is None:
+ return "n/a"
+ return f"[{_pct(value['low'])}, {_pct(value['high'])}]"
+
+
def _num(value: float | None) -> str:
if value is None:
return "n/a"
diff --git a/src/constants/prompts/scout_agent.py b/src/constants/prompts/scout_agent.py
index d00a77b..dac1db1 100644
--- a/src/constants/prompts/scout_agent.py
+++ b/src/constants/prompts/scout_agent.py
@@ -141,6 +141,8 @@ SCOUT_AGENT_EXTRACT_ENTITIES_PROMPT = """
Carefully read the text and extract ALL the entities, unique action instances (events), and quantitative units.
{targeting}
+{reference_time}
+{preferred_entities}
Text: {text}
@@ -149,6 +151,7 @@ OUTPUT RULES:
- Each object must include: "type", "name", and optional "properties" and "description".
- Nodes/Entities MUST be atomic and not composite (phrases) (eg: "Went to San Francisco" is not atomic, "Went to" + "San Francisco" is atomic)
- Dates must be in the format "DD/MM/YYYY" and be stored as "happened_at" in the properties of the event nodes.
+- When the reference date is provided, resolve relative dates (yesterday, last week, last Tuesday, etc.) against it into absolute DD/MM/YYYY values.
- You must extract ALL the building blocks without omitting any concepts.
Begin!
@@ -159,6 +162,8 @@ Carefully read the text and extract the most important entities that can be used
the unique action instances (events), and the quantitative units.
{targeting}
+{reference_time}
+{preferred_entities}
Text: {text}
@@ -167,6 +172,7 @@ OUTPUT RULES:
- Each object must include: "type", "name", and optional "properties" and "description".
- Nodes/Entities MUST be atomic and not composite (phrases) (eg: "Went to San Francisco" is not atomic, "Went to" + "San Francisco" is atomic)
- Dates must be in the format "DD/MM/YYYY" and be stored as "happened_at" in the properties of the event nodes.
+- When the reference date is provided, resolve relative dates (yesterday, last week, last Tuesday, etc.) against it into absolute DD/MM/YYYY values.
- You must extract the most important entities that can be used to reconstruct a meaningful narrative,
the unique action instances (events), and the quantitative units without omitting any concepts.
diff --git a/src/constants/tasks/ingestion.py b/src/constants/tasks/ingestion.py
index 506aa13..fee5335 100644
--- a/src/constants/tasks/ingestion.py
+++ b/src/constants/tasks/ingestion.py
@@ -78,3 +78,10 @@ class IngestionTaskArgs(BaseModel):
brain_id: str = Field(
default="default", description="The brain identifier to store the data in."
)
+ source_timestamp: Optional[str] = Field(
+ default=None,
+ description=(
+ "Reference timestamp for the source content (e.g. session date). "
+ "Used to resolve relative dates like 'yesterday' into absolute dates."
+ ),
+ )
diff --git a/src/core/agents/scout_agent.py b/src/core/agents/scout_agent.py
index 8a27e56..c9bd161 100644
--- a/src/core/agents/scout_agent.py
+++ b/src/core/agents/scout_agent.py
@@ -40,6 +40,7 @@ from src.constants.prompts.scout_agent import (
from src.core.agents.core import parse_structured_from_messages, runtime_agent_factory
from src.core.plugins.prompts import prompt_registry
from src.services.api.constants.requests import IngestionTripleSet
+from src.utils.text_chunking import chunk_text as _chunk_text
class _ScoutEntity(BaseModel):
@@ -338,26 +339,60 @@ class ScoutAgent:
max_retries: int = 3,
ingestion_session_id: Optional[str] = None,
mode: Literal["granular", "coarse"] = "granular",
+ reference_time: Optional[str] = None,
+ preferred_extraction_entities: Optional[List[str]] = None,
+ max_chars_per_chunk: int = 6000,
+ ) -> ScoutAgentResponse:
+ chunks = _chunk_text(text, max_chars=max_chars_per_chunk)
+ if len(chunks) == 1:
+ return self._run_chunk(
+ chunks[0],
+ targeting=targeting,
+ brain_id=brain_id,
+ timeout=timeout,
+ max_retries=max_retries,
+ ingestion_session_id=ingestion_session_id,
+ mode=mode,
+ reference_time=reference_time,
+ preferred_extraction_entities=preferred_extraction_entities,
+ )
+ merged: dict[str, ScoutEntity] = {}
+ for chunk in chunks:
+ response = self._run_chunk(
+ chunk,
+ targeting=targeting,
+ brain_id=brain_id,
+ timeout=timeout,
+ max_retries=max_retries,
+ ingestion_session_id=ingestion_session_id,
+ mode=mode,
+ reference_time=reference_time,
+ preferred_extraction_entities=preferred_extraction_entities,
+ )
+ for entity in response.entities:
+ key = (
+ (entity.name or "").strip().lower(),
+ (entity.type or "").strip().lower(),
+ getattr(entity, "happened_at", None) or "",
+ )
+ if key not in merged:
+ merged[key] = entity
+ return ScoutAgentResponse(entities=list(merged.values()))
+
+ def _run_chunk(
+ self,
+ text: str,
+ targeting: Optional[Node] = None,
+ brain_id: str = "default",
+ timeout: int = 300,
+ max_retries: int = 3,
+ ingestion_session_id: Optional[str] = None,
+ mode: Literal["granular", "coarse"] = "granular",
+ reference_time: Optional[str] = None,
+ preferred_extraction_entities: Optional[List[str]] = None,
) -> ScoutAgentResponse:
"""
Extract entities from the provided text using the Scout agent and return a structured response containing the entities.
-
- Performs an LLM invocation (with optional targeting context), applies retries with exponential backoff on timeouts, and enforces a per-invocation timeout.
-
- Parameters:
- text: The input text to extract entities from.
- targeting: Optional Node providing contextual targeting information (name, description, properties) to bias extraction.
- brain_id: Identifier for the agent/brain configuration to use.
- timeout: Maximum seconds to wait for a single agent invocation before treating it as a timeout.
- max_retries: Maximum number of retry attempts for timed-out invocations using exponential backoff.
- ingestion_session_id: Identifier for the ingestion session to use.
- mode: Mode to use for the scout agent. "granular" for a more granular extraction, "coarse" to extract the most important entities only.
- Returns:
- A ScoutAgentResponse containing:
- - entities: list of extracted ScoutEntity objects.
-
- Raises:
- TimeoutError: If a single invocation exceeds `timeout`, or if all retry attempts fail due to timeouts.
"""
self._get_agent(
output_schema=_ScoutAgentResponse,
@@ -374,6 +409,17 @@ class ScoutAgent:
if targeting
else ""
)
+ reference_time_str = (
+ f"Reference date for resolving relative dates: {reference_time}"
+ if reference_time
+ else ""
+ )
+ preferred_str = (
+ "Prefer extracting these entity types when present: "
+ + ", ".join(preferred_extraction_entities)
+ if preferred_extraction_entities
+ else ""
+ )
if mode == "granular":
prompt = prompt_registry.get(
"SCOUT_AGENT_EXTRACT_ENTITIES_PROMPT",
@@ -381,6 +427,8 @@ class ScoutAgent:
).format(
text=text,
targeting=targeting_str,
+ reference_time=reference_time_str,
+ preferred_entities=preferred_str,
)
elif mode == "coarse":
prompt = prompt_registry.get(
@@ -389,6 +437,8 @@ class ScoutAgent:
).format(
text=text,
targeting=targeting_str,
+ reference_time=reference_time_str,
+ preferred_entities=preferred_str,
)
else:
raise ValueError(f"Invalid mode for scout agent: {mode}")
@@ -424,15 +474,6 @@ class ScoutAgent:
reraise=True,
)
def _invoke_agent_with_retry():
- """
- Invoke the agent in a separate thread and enforce the configured timeout.
-
- Returns:
- dict: The agent response dictionary.
-
- Raises:
- TimeoutError: If the agent invocation exceeds the specified timeout.
- """
try:
with ThreadPoolExecutor(max_workers=1) as executor:
future = executor.submit(_invoke_agent)
diff --git a/src/core/saving/auto_kg.py b/src/core/saving/auto_kg.py
index 4896969..23ee6ff 100644
--- a/src/core/saving/auto_kg.py
+++ b/src/core/saving/auto_kg.py
@@ -39,7 +39,12 @@ class EnrichmentOrchestrationResult:
def enrich_kg_from_input(
- input: str, targeting: Optional[Node] = None, brain_id: str = "default"
+ input: str,
+ targeting: Optional[Node] = None,
+ brain_id: str = "default",
+ source_chunk_id: Optional[str] = None,
+ source_timestamp: Optional[str] = None,
+ preferred_extraction_entities: Optional[List[str]] = None,
) -> EnrichmentOrchestrationResult:
"""
Orchestrates enrichment of the knowledge graph from a free-text input.
@@ -58,14 +63,32 @@ def enrich_kg_from_input(
"ingestion_session_id": ingestion_session_id,
"brain_id": brain_id,
"flow": "enrich_kg_from_input",
+ "source_chunk_id": source_chunk_id,
+ "source_timestamp": source_timestamp,
},
):
- return _enrich_kg_impl(input, targeting, brain_id, ingestion_session_id)
+ return _enrich_kg_impl(
+ input,
+ targeting,
+ brain_id,
+ ingestion_session_id,
+ source_chunk_id=source_chunk_id,
+ source_timestamp=source_timestamp,
+ preferred_extraction_entities=preferred_extraction_entities,
+ )
def _enrich_kg_impl(
- input, targeting, brain_id: str, ingestion_session_id: str
+ input,
+ targeting,
+ brain_id: str,
+ ingestion_session_id: str,
+ source_chunk_id: Optional[str] = None,
+ source_timestamp: Optional[str] = None,
+ preferred_extraction_entities: Optional[List[str]] = None,
) -> EnrichmentOrchestrationResult:
+ from src.core.saving.identity import stamp_provenance
+
ingestion_manager = IngestionManager(
embeddings_adapter, vector_store_adapter, graph_adapter
)
@@ -95,6 +118,8 @@ def _enrich_kg_impl(
brain_id=brain_id,
ingestion_session_id=ingestion_session_id,
mode=("coarse" if mode == "coarse" else "granular"),
+ reference_time=source_timestamp,
+ preferred_extraction_entities=preferred_extraction_entities,
)
print("[DEBUG (initial_scout_entities)]: ", entities.entities)
architect_agent.run_tooler(
@@ -108,9 +133,23 @@ def _enrich_kg_impl(
)
pending = architect_agent.take_pending_relationships()
- enrichment_relationships = [
- rel.model_dump(mode="json") for rel in pending
- ]
+ enrichment_relationships = []
+ for rel in pending:
+ payload = rel.model_dump(mode="json")
+ payload["properties"] = stamp_provenance(
+ payload.get("properties"),
+ source_chunk_id=source_chunk_id,
+ source_timestamp=source_timestamp,
+ )
+ for endpoint_key in ("tail", "tip"):
+ endpoint = payload.get(endpoint_key) or {}
+ endpoint["properties"] = stamp_provenance(
+ endpoint.get("properties"),
+ source_chunk_id=source_chunk_id,
+ source_timestamp=source_timestamp,
+ )
+ payload[endpoint_key] = endpoint
+ enrichment_relationships.append(payload)
should_consolidate = bool(
config.pipeline_mode == "accurate"
and config.run_graph_consolidator
diff --git a/src/core/saving/identity.py b/src/core/saving/identity.py
index 28142e6..576b34d 100644
--- a/src/core/saving/identity.py
+++ b/src/core/saving/identity.py
@@ -1,5 +1,5 @@
import hashlib
-from typing import Optional
+from typing import Any, Optional
from src.utils.dates import normalize_date_string
@@ -59,3 +59,51 @@ def stable_flow_key(
event_type,
normalize_date_string(happened_at) if happened_at else None,
)
+
+
+def merge_source_chunk_ids(*sources: Any) -> list[str]:
+ seen: set[str] = set()
+ merged: list[str] = []
+ for source in sources:
+ if source is None:
+ continue
+ if isinstance(source, str):
+ values = [source]
+ elif isinstance(source, (list, tuple, set)):
+ values = list(source)
+ else:
+ values = [source]
+ for value in values:
+ chunk_id = str(value).strip() if value is not None else ""
+ if not chunk_id or chunk_id in seen:
+ continue
+ seen.add(chunk_id)
+ merged.append(chunk_id)
+ return merged
+
+
+def stamp_provenance(
+ properties: Optional[dict],
+ *,
+ source_chunk_id: Optional[str] = None,
+ source_timestamp: Optional[str] = None,
+ existing_properties: Optional[dict] = None,
+) -> dict:
+ props = dict(properties or {})
+ existing = existing_properties or {}
+ chunk_ids = merge_source_chunk_ids(
+ existing.get("source_chunk_ids"),
+ existing.get("source_chunk_id"),
+ props.get("source_chunk_ids"),
+ props.get("source_chunk_id"),
+ source_chunk_id,
+ )
+ if chunk_ids:
+ props["source_chunk_ids"] = chunk_ids
+ props.pop("source_chunk_id", None)
+ timestamp = source_timestamp or props.get("source_timestamp") or existing.get(
+ "source_timestamp"
+ )
+ if timestamp:
+ props["source_timestamp"] = timestamp
+ return props
diff --git a/src/core/search/fact_filter.py b/src/core/search/fact_filter.py
new file mode 100644
index 0000000..c5880e4
--- /dev/null
+++ b/src/core/search/fact_filter.py
@@ -0,0 +1,77 @@
+from __future__ import annotations
+
+from typing import Any, Optional
+
+from pydantic import BaseModel, Field
+
+
+class FactFilterDecision(BaseModel):
+ keep_indices: list[int] = Field(default_factory=list)
+
+
+def filter_relevant_facts(
+ question: str,
+ candidates: list[str],
+ *,
+ llm_adapter: Any = None,
+ max_keep: int = 40,
+) -> list[int]:
+ if not candidates:
+ return []
+ if len(candidates) <= max_keep and llm_adapter is None:
+ return list(range(len(candidates)))
+
+ indexed = "\n".join(f"{i}: {text}" for i, text in enumerate(candidates))
+ prompt = (
+ "You are a recognition-memory filter for a conversational memory system.\n"
+ "Given a question and numbered candidate facts/passages, return the indices "
+ "of items that are relevant evidence for answering the question.\n"
+ "Prefer recall of useful evidence over aggressive pruning, but drop clear noise.\n"
+ f"Keep at most {max_keep} indices.\n\n"
+ f"Question: {question}\n\n"
+ f"Candidates:\n{indexed}\n\n"
+ 'Respond with JSON: {"keep_indices": [0, 2, ...]}'
+ )
+
+ if llm_adapter is None:
+ return list(range(min(max_keep, len(candidates))))
+
+ try:
+ response = llm_adapter.generate_structured(
+ prompt,
+ FactFilterDecision,
+ )
+ if isinstance(response, FactFilterDecision):
+ indices = response.keep_indices
+ elif isinstance(response, dict):
+ indices = response.get("keep_indices") or []
+ else:
+ indices = getattr(response, "keep_indices", []) or []
+ cleaned = sorted(
+ {
+ int(i)
+ for i in indices
+ if isinstance(i, (int, float, str))
+ and str(i).isdigit()
+ and 0 <= int(i) < len(candidates)
+ }
+ )
+ if cleaned:
+ return cleaned[:max_keep]
+ except Exception:
+ pass
+ return list(range(min(max_keep, len(candidates))))
+
+
+def reciprocal_rank_fusion(
+ ranked_lists: list[list[str]],
+ *,
+ k: int = 60,
+) -> list[tuple[str, float]]:
+ scores: dict[str, float] = {}
+ for ranked in ranked_lists:
+ for rank, item in enumerate(ranked):
+ if not item:
+ continue
+ scores[item] = scores.get(item, 0.0) + 1.0 / (k + rank + 1)
+ return sorted(scores.items(), key=lambda pair: pair[1], reverse=True)
diff --git a/src/lib/neo4j/client.py b/src/lib/neo4j/client.py
index 6214dd9..e932397 100644
--- a/src/lib/neo4j/client.py
+++ b/src/lib/neo4j/client.py
@@ -259,7 +259,24 @@ class Neo4jClient(GraphClient):
for key, value in all_properties.items():
cypher_key = self._format_property_key(key)
escaped_value = self._format_value(value)
- property_assignments.append(f"n.{cypher_key} = {escaped_value}")
+ if key == "description":
+ property_assignments.append(
+ f"n.{cypher_key} = CASE "
+ f"WHEN n.{cypher_key} IS NULL OR n.{cypher_key} = '' THEN {escaped_value} "
+ f"WHEN {escaped_value} IS NULL OR {escaped_value} = '' THEN n.{cypher_key} "
+ f"WHEN toLower(n.{cypher_key}) CONTAINS toLower({escaped_value}) THEN n.{cypher_key} "
+ f"WHEN toLower({escaped_value}) CONTAINS toLower(n.{cypher_key}) THEN {escaped_value} "
+ f"ELSE n.{cypher_key} + ' | ' + {escaped_value} END"
+ )
+ elif key in {"source_chunk_ids", "aliases"}:
+ property_assignments.append(
+ f"n.{cypher_key} = CASE "
+ f"WHEN n.{cypher_key} IS NULL THEN {escaped_value} "
+ f"WHEN {escaped_value} IS NULL THEN n.{cypher_key} "
+ f"ELSE n.{cypher_key} + [x IN {escaped_value} WHERE NOT x IN n.{cypher_key}] END"
+ )
+ else:
+ property_assignments.append(f"n.{cypher_key} = {escaped_value}")
properties_set = f"{', '.join(property_assignments)}"
diff --git a/src/lib/postgresql/graph_store.py b/src/lib/postgresql/graph_store.py
index dd1bdf1..864eb18 100644
--- a/src/lib/postgresql/graph_store.py
+++ b/src/lib/postgresql/graph_store.py
@@ -95,7 +95,29 @@ class _BrainGraph:
merged = {}
if match_uuid in self.graph:
merged.update(self.node_data(match_uuid))
- merged.update(properties)
+ incoming = dict(properties)
+ if "description" in incoming and merged.get("description"):
+ existing_desc = str(merged.get("description") or "").strip()
+ incoming_desc = str(incoming.get("description") or "").strip()
+ if existing_desc and incoming_desc and incoming_desc.lower() not in existing_desc.lower():
+ if existing_desc.lower() not in incoming_desc.lower():
+ incoming["description"] = f"{existing_desc} | {incoming_desc}"
+ else:
+ incoming["description"] = incoming_desc
+ elif existing_desc and not incoming_desc:
+ incoming["description"] = existing_desc
+ for list_key in ("source_chunk_ids", "aliases"):
+ if list_key in incoming or list_key in merged:
+ seen = []
+ for value in list(merged.get(list_key) or []) + list(
+ incoming.get(list_key) or []
+ ):
+ item = str(value).strip()
+ if item and item not in seen:
+ seen.append(item)
+ if seen:
+ incoming[list_key] = seen
+ merged.update(incoming)
if identification.get("name") is not None:
merged["name"] = identification.get("name")
merged["labels"] = labels or merged.get("labels") or []
diff --git a/src/services/api/constants/requests.py b/src/services/api/constants/requests.py
index 8c0311e..79837c8 100644
--- a/src/services/api/constants/requests.py
+++ b/src/services/api/constants/requests.py
@@ -394,6 +394,8 @@ class GetContextRequestBody(BaseModel):
brain_id: str = "default"
historical_limit: int = 10
max_facts: int = 40
+ max_passages: int = 8
+ apply_fact_filter: bool = True
class GetContextTriple(BaseModel):
@@ -409,3 +411,4 @@ class GetContextResponse(BaseModel):
text_context: str
triples: List[GetContextTriple]
historical_context: List[str] = []
+ source_passages: List[str] = []
diff --git a/src/services/api/controllers/retrieve.py b/src/services/api/controllers/retrieve.py
index 94d2457..98d2f1c 100644
--- a/src/services/api/controllers/retrieve.py
+++ b/src/services/api/controllers/retrieve.py
@@ -20,6 +20,7 @@ from src.constants.embeddings import Vector
from src.constants.kg import IdentificationParams, Node, Predicate
from src.core.search.entities import search_entities
from src.core.search.entity_info import EventSynergyRetriever, MatchPath
+from src.core.search.fact_filter import filter_relevant_facts, reciprocal_rank_fusion
from src.core.search.relationships import search_relationships
from src.utils.vector_search import VectorSearchFacade
from src.services.api.constants.requests import (
@@ -41,6 +42,8 @@ vector_search = VectorSearchFacade(vector_store_adapter)
_MAX_NOUN_CHUNKS = 5
_MAX_DOSSIER_ENTITIES = 5
_DOSSIER_MAX_DEPTH = 3
+_SEED_K = 20
+_PASSAGE_K = 12
async def retrieve_data(
@@ -405,6 +408,15 @@ def _fact_key(r: Predicate, r2: Predicate) -> tuple[str, str]:
return (getattr(r, "uuid", None) or "", getattr(r2, "uuid", None) or "")
+def _is_currently_valid(predicate: Predicate) -> bool:
+ props = getattr(predicate, "properties", None) or {}
+ if props.get("invalid_at"):
+ return False
+ if getattr(predicate, "deprecated", False):
+ return False
+ return True
+
+
def _collect_query_variants(text: str, elements) -> list[str]:
seen: set[str] = set()
variants: list[str] = []
@@ -446,6 +458,7 @@ def _seed_nodes_for_text(text: str, brain_id: str) -> list[tuple[str, float, str
node_vectors = vector_search.search_nodes(
embeddings,
brain_id=brain_id,
+ k=_SEED_K,
)
for vector in node_vectors:
meta = vector.metadata or {}
@@ -461,6 +474,7 @@ def _seed_nodes_for_text(text: str, brain_id: str) -> list[tuple[str, float, str
rel_vectors = vector_search.search_relationships(
embeddings,
brain_id=brain_id,
+ k=_SEED_K,
)
for vector in rel_vectors:
meta = vector.metadata or {}
@@ -475,6 +489,86 @@ def _seed_nodes_for_text(text: str, brain_id: str) -> list[tuple[str, float, str
return seeds
+def _extract_source_chunk_ids(*items: Any) -> list[str]:
+ ids: list[str] = []
+ seen: set[str] = set()
+ for item in items:
+ props = getattr(item, "properties", None) or {}
+ for value in props.get("source_chunk_ids") or []:
+ chunk_id = str(value).strip()
+ if chunk_id and chunk_id not in seen:
+ seen.add(chunk_id)
+ ids.append(chunk_id)
+ single = props.get("source_chunk_id")
+ if single:
+ chunk_id = str(single).strip()
+ if chunk_id and chunk_id not in seen:
+ seen.add(chunk_id)
+ ids.append(chunk_id)
+ return ids
+
+
+def _retrieve_passages(
+ text: str, brain_id: str, *, limit: int
+) -> list[tuple[str, float, str]]:
+ """Return (chunk_id, score, text) ranked passages via vector + keyword fusion."""
+ text_embeddings = embeddings_adapter.embed_text(text)
+ vector_hits = vector_search.search_data(
+ text_embeddings.embeddings,
+ brain_id=brain_id,
+ k=max(limit, _PASSAGE_K),
+ )
+ vector_ids: list[str] = []
+ id_to_text: dict[str, str] = {}
+ id_to_distance: dict[str, float] = {}
+ for vector in vector_hits:
+ meta = vector.metadata or {}
+ resource_id = meta.get("resource_id") or getattr(vector, "id", None)
+ if not resource_id:
+ continue
+ resource_id = str(resource_id)
+ vector_ids.append(resource_id)
+ id_to_distance[resource_id] = (
+ float(vector.distance)
+ if vector.distance is not None
+ else float("inf")
+ )
+
+ keyword_ids: list[str] = []
+ try:
+ search_result = data_adapter.search(text, brain_id)
+ for chunk in getattr(search_result, "text_chunks", None) or []:
+ chunk_id = str(getattr(chunk, "id", "") or "")
+ if not chunk_id:
+ continue
+ keyword_ids.append(chunk_id)
+ id_to_text[chunk_id] = getattr(chunk, "text", "") or ""
+ except Exception:
+ pass
+
+ fused = reciprocal_rank_fusion([vector_ids, keyword_ids])
+ ranked_ids = [item for item, _ in fused[:limit]]
+ missing = [cid for cid in ranked_ids if cid not in id_to_text]
+ if missing:
+ try:
+ chunks, _ = data_adapter.get_text_chunks_by_ids(
+ missing, False, brain_id
+ )
+ for chunk in chunks:
+ id_to_text[str(chunk.id)] = chunk.text or ""
+ except Exception:
+ pass
+
+ passages: list[tuple[str, float, str]] = []
+ for chunk_id in ranked_ids:
+ body = (id_to_text.get(chunk_id) or "").strip()
+ if not body:
+ continue
+ score = id_to_distance.get(chunk_id, 1.0 / (1.0 + len(passages)))
+ passages.append((chunk_id, float(score), body))
+ return passages
+
+
def _flatten_match_path(path: MatchPath) -> list[str]:
lines: list[str] = []
current: MatchPath | None = path
@@ -499,22 +593,16 @@ def _flatten_match_path(path: MatchPath) -> list[str]:
async def get_context(request: GetContextRequestBody) -> GetContextResponse:
"""
Retrieve contextual information for a text.
-
- Parameters:
- request (GetContextRequestBody): Request body containing:
- - text: The text to search context for.
- - brain_id: The brain/workspace identifier to query.
-
- Returns:
- GetContextResponse: Response containing the context information.
"""
elements = _entity_extractor.extract_elements(request.text)
variants = _collect_query_variants(request.text, elements)
historical_context: list[str] = []
+ source_passages: list[str] = []
candidate_lock = threading.Lock()
candidates: list[dict[str, Any]] = []
seed_hits: list[tuple[str, float, str]] = []
+ passage_hits: list[tuple[str, float, str]] = []
def _collect_facts_for_variant(text: str) -> None:
seeds = _seed_nodes_for_text(text, request.brain_id)
@@ -533,6 +621,8 @@ async def get_context(request: GetContextRequestBody) -> GetContextResponse:
)
local: list[dict[str, Any]] = []
for n, r, m, r2, b in neighbors:
+ if not _is_currently_valid(r) or not _is_currently_valid(r2):
+ continue
distances = [
uuid_to_distance[u]
for u in (
@@ -549,14 +639,33 @@ async def get_context(request: GetContextRequestBody) -> GetContextResponse:
"triple": (n, r, m, r2, b),
"score": score,
"key": _fact_key(r, r2),
+ "text": _format_event_fact(n, r, m, r2, b),
+ "chunk_ids": _extract_source_chunk_ids(n, r, m, r2, b),
}
)
if local:
with candidate_lock:
candidates.extend(local)
+ def _collect_passages() -> None:
+ nonlocal passage_hits
+ passage_hits = _retrieve_passages(
+ request.text,
+ request.brain_id,
+ limit=max(1, int(getattr(request, "max_passages", 8) or 8)),
+ )
+
async def _get_historical_context():
nonlocal historical_context
+ passages = await asyncio.to_thread(
+ _retrieve_passages,
+ request.text,
+ request.brain_id,
+ limit=max(request.historical_limit, _PASSAGE_K),
+ )
+ if passages:
+ historical_context = [text for _, _, text in passages[: request.historical_limit]]
+ return historical_context
text_chunks, structured_data = await asyncio.gather(
asyncio.to_thread(
data_adapter.get_last_text_chunks,
@@ -579,6 +688,7 @@ async def get_context(request: GetContextRequestBody) -> GetContextResponse:
futures = [
asyncio.to_thread(_collect_facts_for_variant, variant) for variant in variants
]
+ futures.append(asyncio.to_thread(_collect_passages))
futures.append(_get_historical_context())
await asyncio.gather(*futures)
@@ -591,19 +701,56 @@ async def get_context(request: GetContextRequestBody) -> GetContextResponse:
ranked = sorted(deduped.values(), key=lambda c: c["score"])
max_facts = max(1, int(getattr(request, "max_facts", 40) or 40))
- curated = ranked[:max_facts]
+
+ if getattr(request, "apply_fact_filter", True) and ranked:
+ keep = filter_relevant_facts(
+ request.text,
+ [c["text"] for c in ranked],
+ max_keep=max_facts,
+ )
+ curated = [ranked[i] for i in keep if 0 <= i < len(ranked)]
+ if not curated:
+ curated = ranked[:max_facts]
+ else:
+ curated = ranked[:max_facts]
text_lines: list[str] = []
triples: list[GetContextTriple] = []
+ provenance_ids: list[str] = []
for candidate in curated:
n, r, m, r2, b = candidate["triple"]
- text_lines.append(_format_event_fact(n, r, m, r2, b))
+ text_lines.append(candidate["text"])
triples.append(
GetContextTriple(
identified_entity=candidate["identified_entity"],
triple=(n, r, m, r2, b),
)
)
+ provenance_ids.extend(candidate.get("chunk_ids") or [])
+
+ for chunk_id, _, body in passage_hits:
+ if body and body not in source_passages:
+ source_passages.append(body)
+ if chunk_id not in provenance_ids:
+ provenance_ids.append(chunk_id)
+
+ if provenance_ids:
+ try:
+ chunks, _ = await asyncio.to_thread(
+ data_adapter.get_text_chunks_by_ids,
+ provenance_ids[:40],
+ False,
+ request.brain_id,
+ )
+ for chunk in chunks:
+ body = (chunk.text or "").strip()
+ if body and body not in source_passages:
+ source_passages.append(body)
+ except Exception:
+ pass
+
+ max_passages = max(1, int(getattr(request, "max_passages", 8) or 8))
+ source_passages = source_passages[:max_passages]
entity_scores: dict[str, float] = {}
for _uuid, distance, entity_name in seed_hits:
@@ -648,10 +795,12 @@ async def get_context(request: GetContextRequestBody) -> GetContextResponse:
dossier_lines = await asyncio.to_thread(_run_dossiers)
- text_context = "\n".join(text_lines + dossier_lines)
+ passage_block = [f"[passage] {p}" for p in source_passages]
+ text_context = "\n".join(passage_block + text_lines + dossier_lines)
return GetContextResponse(
text_context=text_context,
triples=triples,
historical_context=historical_context,
+ source_passages=source_passages,
)
diff --git a/src/utils/dates.py b/src/utils/dates.py
index 87cc3e0..cfe6cdb 100644
--- a/src/utils/dates.py
+++ b/src/utils/dates.py
@@ -1,5 +1,6 @@
-from datetime import datetime
+from datetime import datetime, timedelta
from typing import Optional
+import re
_DATE_INPUT_FORMATS = (
"%d/%m/%Y",
@@ -14,14 +15,109 @@ _DATE_INPUT_FORMATS = (
"%d %b %Y",
)
+_RELATIVE_PATTERNS = (
+ (re.compile(r"^yesterday$", re.I), lambda ref: ref - timedelta(days=1)),
+ (re.compile(r"^tomorrow$", re.I), lambda ref: ref + timedelta(days=1)),
+ (re.compile(r"^today$", re.I), lambda ref: ref),
+ (re.compile(r"^last\s+week$", re.I), lambda ref: ref - timedelta(days=7)),
+ (re.compile(r"^next\s+week$", re.I), lambda ref: ref + timedelta(days=7)),
+ (re.compile(r"^last\s+month$", re.I), lambda ref: _shift_months(ref, -1)),
+ (re.compile(r"^next\s+month$", re.I), lambda ref: _shift_months(ref, 1)),
+ (
+ re.compile(r"^(\d+)\s+days?\s+ago$", re.I),
+ lambda ref, m: ref - timedelta(days=int(m.group(1))),
+ ),
+ (
+ re.compile(r"^in\s+(\d+)\s+days?$", re.I),
+ lambda ref, m: ref + timedelta(days=int(m.group(1))),
+ ),
+ (
+ re.compile(r"^(\d+)\s+weeks?\s+ago$", re.I),
+ lambda ref, m: ref - timedelta(weeks=int(m.group(1))),
+ ),
+)
-def normalize_date_string(value: Optional[str]) -> Optional[str]:
+_WEEKDAY_NAMES = {
+ "monday": 0,
+ "tuesday": 1,
+ "wednesday": 2,
+ "thursday": 3,
+ "friday": 4,
+ "saturday": 5,
+ "sunday": 6,
+}
+
+
+def _shift_months(ref: datetime, delta: int) -> datetime:
+ month = ref.month - 1 + delta
+ year = ref.year + month // 12
+ month = month % 12 + 1
+ day = min(ref.day, [31, 29 if year % 4 == 0 else 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31][month - 1])
+ return ref.replace(year=year, month=month, day=day)
+
+
+def parse_date_string(value: Optional[str]) -> Optional[datetime]:
if not value or not isinstance(value, str):
- return value
+ return None
cleaned = value.strip()
for fmt in _DATE_INPUT_FORMATS:
try:
- return datetime.strptime(cleaned, fmt).strftime("%d/%m/%Y")
+ return datetime.strptime(cleaned, fmt)
except ValueError:
continue
- return value
+ return None
+
+
+def normalize_date_string(value: Optional[str]) -> Optional[str]:
+ if not value or not isinstance(value, str):
+ return value
+ parsed = parse_date_string(value)
+ if parsed is not None:
+ return parsed.strftime("%d/%m/%Y")
+ return value.strip()
+
+
+def resolve_relative_date(
+ value: Optional[str],
+ reference_time: Optional[str | datetime] = None,
+) -> Optional[str]:
+ if not value or not isinstance(value, str):
+ return value
+ cleaned = value.strip()
+ absolute = normalize_date_string(cleaned)
+ if absolute != cleaned or parse_date_string(cleaned) is not None:
+ return absolute
+
+ if reference_time is None:
+ return cleaned
+ if isinstance(reference_time, datetime):
+ ref = reference_time
+ else:
+ ref = parse_date_string(str(reference_time))
+ if ref is None:
+ return cleaned
+
+ for pattern, resolver in _RELATIVE_PATTERNS:
+ match = pattern.match(cleaned)
+ if not match:
+ continue
+ if match.lastindex:
+ resolved = resolver(ref, match)
+ else:
+ resolved = resolver(ref)
+ return resolved.strftime("%d/%m/%Y")
+
+ weekday_match = re.match(r"^(last|this|next)\s+(monday|tuesday|wednesday|thursday|friday|saturday|sunday)$", cleaned, re.I)
+ if weekday_match:
+ mode = weekday_match.group(1).lower()
+ target = _WEEKDAY_NAMES[weekday_match.group(2).lower()]
+ current = ref.weekday()
+ if mode == "this":
+ delta = (target - current) % 7
+ elif mode == "next":
+ delta = (target - current) % 7 or 7
+ else:
+ delta = -((current - target) % 7 or 7)
+ return (ref + timedelta(days=delta)).strftime("%d/%m/%Y")
+
+ return cleaned
diff --git a/src/utils/text_chunking.py b/src/utils/text_chunking.py
new file mode 100644
index 0000000..5282c4e
--- /dev/null
+++ b/src/utils/text_chunking.py
@@ -0,0 +1,27 @@
+def chunk_text(text: str, max_chars: int = 6000) -> list[str]:
+ cleaned = (text or "").strip()
+ if not cleaned or len(cleaned) <= max_chars:
+ return [cleaned] if cleaned else [""]
+ paragraphs = [p.strip() for p in cleaned.split("\n\n") if p.strip()]
+ if not paragraphs:
+ paragraphs = [cleaned]
+ chunks: list[str] = []
+ current = ""
+ for paragraph in paragraphs:
+ candidate = f"{current}\n\n{paragraph}".strip() if current else paragraph
+ if len(candidate) <= max_chars:
+ current = candidate
+ continue
+ if current:
+ chunks.append(current)
+ if len(paragraph) <= max_chars:
+ current = paragraph
+ continue
+ for i in range(0, len(paragraph), max_chars):
+ piece = paragraph[i : i + max_chars].strip()
+ if piece:
+ chunks.append(piece)
+ current = ""
+ if current:
+ chunks.append(current)
+ return chunks or [cleaned]
diff --git a/src/workers/tasks/ingestion.py b/src/workers/tasks/ingestion.py
index 460d5f6..2ae1de7 100644
--- a/src/workers/tasks/ingestion.py
+++ b/src/workers/tasks/ingestion.py
@@ -29,6 +29,7 @@ from uuid import uuid4
from src.core.saving.identity import (
stable_node_id,
stable_relationship_id,
+ stamp_provenance,
)
from pydantic import BaseModel
@@ -190,17 +191,25 @@ def _is_same_graph_node(graph_node: Node, entity) -> bool:
def _normalize_relationship_dates(
relationships: List[ArchitectAgentRelationship],
+ reference_time: Optional[str] = None,
) -> None:
+ from src.utils.dates import resolve_relative_date
+
for relationship in relationships:
for entity in (relationship.tail, relationship.tip):
if (entity.type or "").strip().upper() == "DATE":
- entity.name = normalize_date_string(entity.name)
+ entity.name = resolve_relative_date(entity.name, reference_time)
if getattr(entity, "happened_at", None):
- entity.happened_at = normalize_date_string(entity.happened_at)
+ entity.happened_at = resolve_relative_date(
+ entity.happened_at, reference_time
+ )
def _resolve_relationship_entities(
- relationships: List[ArchitectAgentRelationship], brain_id: str
+ relationships: List[ArchitectAgentRelationship],
+ brain_id: str,
+ source_chunk_id: Optional[str] = None,
+ source_timestamp: Optional[str] = None,
) -> None:
unique_entities = {}
for relationship in relationships:
@@ -306,6 +315,25 @@ def _resolve_relationship_entities(
entity.name = node.name
if getattr(node, "happened_at", None) and not entity.happened_at:
entity.happened_at = node.happened_at
+ existing_props = getattr(node, "properties", None) or {}
+ entity.properties = stamp_provenance(
+ entity.properties,
+ source_chunk_id=source_chunk_id,
+ source_timestamp=source_timestamp,
+ existing_properties=existing_props,
+ )
+ if getattr(node, "description", None):
+ entity.description = _merge_description(
+ node.description, entity.description
+ )
+ aliases = list(existing_props.get("aliases") or [])
+ if entity.name and entity.name not in aliases and entity.name != node.name:
+ aliases.append(entity.name)
+ if aliases:
+ entity.properties = {
+ **(entity.properties or {}),
+ "aliases": aliases,
+ }
_apply_resolutions()
resolved_uuids = {node.uuid for node in resolutions.values()}
@@ -382,7 +410,22 @@ def _resolve_relationship_entities(
_apply_resolutions()
for relationship in relationships:
+ relationship.properties = stamp_provenance(
+ relationship.properties,
+ source_chunk_id=source_chunk_id,
+ source_timestamp=source_timestamp,
+ )
+ if source_timestamp and not (relationship.properties or {}).get("valid_at"):
+ relationship.properties = {
+ **(relationship.properties or {}),
+ "valid_at": source_timestamp,
+ }
for entity in (relationship.tail, relationship.tip):
+ entity.properties = stamp_provenance(
+ entity.properties,
+ source_chunk_id=source_chunk_id,
+ source_timestamp=source_timestamp,
+ )
if resolutions.get(_entity_key(entity)):
continue
entity.uuid = stable_node_id(
@@ -402,6 +445,65 @@ def _resolve_relationship_entities(
relationship.flow_key,
)
+
+def _merge_description(
+ existing: Optional[str], incoming: Optional[str]
+) -> Optional[str]:
+ existing_clean = (existing or "").strip()
+ incoming_clean = (incoming or "").strip()
+ if not existing_clean:
+ return incoming_clean or None
+ if not incoming_clean:
+ return existing_clean
+ if incoming_clean.lower() in existing_clean.lower():
+ return existing_clean
+ if existing_clean.lower() in incoming_clean.lower():
+ return incoming_clean
+ return f"{existing_clean} | {incoming_clean}"
+
+
+def _invalidate_superseded_relationships(
+ relationship: ArchitectAgentRelationship,
+ *,
+ brain_id: str,
+) -> None:
+ """Mark older same-type edges from the same subject as invalid when tip changes."""
+ try:
+ neighbors = graph_adapter.get_neighbors(
+ [relationship.tail.uuid], brain_id=brain_id
+ )
+ except Exception:
+ return
+ pairs = (neighbors or {}).get(relationship.tail.uuid) or []
+ valid_at = (relationship.properties or {}).get("valid_at") or (
+ relationship.properties or {}
+ ).get("source_timestamp")
+ for predicate, neighbor in pairs:
+ if not predicate or not neighbor:
+ continue
+ if (predicate.name or "").strip().upper() != (relationship.name or "").strip().upper():
+ continue
+ if neighbor.uuid == relationship.tip.uuid:
+ continue
+ if getattr(predicate, "uuid", None) == relationship.uuid:
+ continue
+ props = getattr(predicate, "properties", None) or {}
+ if props.get("invalid_at"):
+ continue
+ try:
+ graph_adapter.update_properties(
+ predicate.uuid,
+ "relationship",
+ brain_id=brain_id,
+ new_properties={
+ "invalid_at": valid_at
+ or datetime.datetime.utcnow().strftime("%d/%m/%Y"),
+ "deprecated": True,
+ },
+ )
+ except Exception as exc:
+ print(f"[!] Failed to invalidate relationship {predicate.uuid}: {exc}")
+
def format_textual_data(data: dict, include_keys: bool = True) -> str:
def format_value(v):
if isinstance(v, str):
@@ -511,7 +613,13 @@ def ingest_data(self, args: dict):
brain_id=payload.brain_id,
)
- enrich_result = enrich_kg_from_input(source_text, brain_id=payload.brain_id)
+ enrich_result = enrich_kg_from_input(
+ source_text,
+ brain_id=payload.brain_id,
+ source_chunk_id=text_chunk.id,
+ source_timestamp=payload.source_timestamp,
+ preferred_extraction_entities=payload.preferred_extraction_entities,
+ )
parent_task_id = self.request.id
steps = []
if enrich_result.enrichment_relationships:
@@ -643,8 +751,25 @@ def process_architect_relationships(self, args: dict):
relationships = [
ArchitectAgentRelationship(**rel_data) for rel_data in relationships_data
]
- _normalize_relationship_dates(relationships)
- _resolve_relationship_entities(relationships, brain_id)
+ reference_time = None
+ source_chunk_id = None
+ for rel in relationships:
+ props = rel.properties or {}
+ if not reference_time:
+ reference_time = props.get("source_timestamp") or props.get("valid_at")
+ if not source_chunk_id:
+ ids = props.get("source_chunk_ids") or []
+ if ids:
+ source_chunk_id = ids[-1]
+ elif props.get("source_chunk_id"):
+ source_chunk_id = props.get("source_chunk_id")
+ _normalize_relationship_dates(relationships, reference_time=reference_time)
+ _resolve_relationship_entities(
+ relationships,
+ brain_id,
+ source_chunk_id=source_chunk_id,
+ source_timestamp=reference_time,
+ )
item_errors: List[dict] = []
with ThreadPoolExecutor(max_workers=10) as io_executor:
@@ -844,6 +969,10 @@ def process_architect_relationships(self, args: dict):
),
brain_id=brain_id,
)
+ _invalidate_superseded_relationships(
+ relationship,
+ brain_id=brain_id,
+ )
except FutureTimeoutError:
rel_name = getattr(relationship, "name", "unknown")
item_errors.append(
diff --git a/tests/test_graph_upsert_contract.py b/tests/test_graph_upsert_contract.py
index ddd2626..25017e8 100644
--- a/tests/test_graph_upsert_contract.py
+++ b/tests/test_graph_upsert_contract.py
@@ -52,7 +52,7 @@ class NetworkXUpsertContractTests(unittest.TestCase):
self.assertEqual(second, "alice-1")
self.assertEqual(len(list(brain.graph.nodes)), 1)
self.assertEqual(brain.node_data("alice-1")["uuid"], "alice-1")
- self.assertEqual(brain.node_data("alice-1")["description"], "v2")
+ self.assertEqual(brain.node_data("alice-1")["description"], "v1 | v2")
def test_same_name_different_uuid_does_not_overwrite_existing_uuid(self):
from src.lib.postgresql.graph_store import _BrainGraph
diff --git a/tests/test_graphrag_upgrade_helpers.py b/tests/test_graphrag_upgrade_helpers.py
new file mode 100644
index 0000000..7a0ae65
--- /dev/null
+++ b/tests/test_graphrag_upgrade_helpers.py
@@ -0,0 +1,105 @@
+import os
+import unittest
+
+ENV_DEFAULTS = {
+ "BRAINPAT_TOKEN": "test-token",
+ "MODELS_MODE": "local",
+ "EMBEDDINGS_LOCAL_MODEL": "local-model",
+ "EMBEDDINGS_SMALL_MODEL": "small-model",
+ "EMBEDDING_NODES_DIMENSION": "3",
+ "EMBEDDING_TRIPLETS_DIMENSION": "3",
+ "EMBEDDING_OBSERVATIONS_DIMENSION": "3",
+ "EMBEDDING_DATA_DIMENSION": "3",
+ "EMBEDDING_RELATIONSHIPS_DIMENSION": "3",
+ "REDIS_HOST": "localhost",
+ "REDIS_PORT": "6379",
+ "NEO4J_HOST": "localhost",
+ "NEO4J_PORT": "7687",
+ "NEO4J_USERNAME": "neo4j",
+ "NEO4J_PASSWORD": "password",
+ "MILVUS_HOST": "localhost",
+ "MILVUS_PORT": "19530",
+ "MONGO_CONNECTION_STRING": "mongodb://localhost:27017",
+ "CELERY_WORKER_CONCURRENCY": "1",
+ "OLLAMA_HOST": "localhost",
+ "OLLAMA_PORT": "11434",
+ "OLLAMA_LLM_SMALL_MODEL": "small",
+ "OLLAMA_LLM_LARGE_MODEL": "large",
+ "POSTGRES_HOST": "localhost",
+ "POSTGRES_PORT": "5432",
+ "POSTGRES_USERNAME": "postgres",
+ "POSTGRES_PASSWORD": "postgres",
+}
+for key, value in ENV_DEFAULTS.items():
+ os.environ.setdefault(key, value)
+
+from src.core.saving.identity import merge_source_chunk_ids, stamp_provenance
+from src.core.search.fact_filter import reciprocal_rank_fusion
+from src.utils.dates import normalize_date_string, resolve_relative_date
+
+
+class ProvenanceHelpersTests(unittest.TestCase):
+ def test_merge_source_chunk_ids_dedupes(self):
+ merged = merge_source_chunk_ids(
+ ["a", "b"],
+ "b",
+ ["c"],
+ None,
+ "a",
+ )
+ self.assertEqual(merged, ["a", "b", "c"])
+
+ def test_stamp_provenance_accumulates(self):
+ props = stamp_provenance(
+ {"source_chunk_ids": ["chunk-1"]},
+ source_chunk_id="chunk-2",
+ source_timestamp="01/05/2023",
+ existing_properties={"source_chunk_ids": ["chunk-0"]},
+ )
+ self.assertEqual(props["source_chunk_ids"], ["chunk-0", "chunk-1", "chunk-2"])
+ self.assertEqual(props["source_timestamp"], "01/05/2023")
+
+
+class RelativeDateTests(unittest.TestCase):
+ def test_normalize_absolute(self):
+ self.assertEqual(normalize_date_string("2023-05-15"), "15/05/2023")
+
+ def test_resolve_yesterday(self):
+ self.assertEqual(
+ resolve_relative_date("yesterday", "15/05/2023"),
+ "14/05/2023",
+ )
+
+ def test_resolve_last_tuesday(self):
+ # 15/05/2023 is a Monday
+ self.assertEqual(
+ resolve_relative_date("last Tuesday", "15/05/2023"),
+ "09/05/2023",
+ )
+
+
+class RrfTests(unittest.TestCase):
+ def test_rrf_prefers_items_ranked_high_in_multiple_lists(self):
+ fused = reciprocal_rank_fusion(
+ [
+ ["a", "b", "c"],
+ ["b", "a", "d"],
+ ]
+ )
+ ranked = [item for item, _ in fused]
+ self.assertEqual(ranked[0], "a")
+ self.assertIn("b", ranked[:2])
+
+
+class ScoutChunkingTests(unittest.TestCase):
+ def test_chunk_text_splits_long_input(self):
+ from src.utils.text_chunking import chunk_text
+
+ text = ("paragraph one.\n\n" * 50) + ("x" * 7000)
+ chunks = chunk_text(text, max_chars=1000)
+ self.assertGreater(len(chunks), 1)
+ self.assertTrue(all(len(c) <= 1000 or c.count("x") > 0 for c in chunks))
+
+
+if __name__ == "__main__":
+ unittest.main()