Skip to content

BrainAPI is a knowledge graph–powered AI memory layer that transforms unstructured data into structured knowledge, enabling intelligent search, recommendations, and contextual memory for AI agents and applications.

feat(lib/llm+ingestion): ingestion strength + support for deepseek

Christian2e11ea111641
diff --git a/.env.example b/.env.example
index e8f133c..f08cb1c 100644
--- a/.env.example
+++ b/.env.example
@@ -26,6 +26,12 @@ OPENAI_BASE_URL=""
OPENAI_SMALL_LLM_MODEL="gpt-4o-mini"
OPENAI_LARGE_LLM_MODEL="gpt-4o"
OPENAI_EMBEDDING_MODEL="text-embedding-3-large"
+# DeepSeek chat (OpenAI-compatible). No embeddings — keep EMBEDDINGS_PROVIDER on azure/openai/gcp_vertex/etc.
+# LLM_SMALL_PROVIDER="deepseek"
+# LLM_LARGE_PROVIDER="deepseek"
+DEEPSEEK_API_KEY="sk-..."
+DEEPSEEK_SMALL_LLM_MODEL="deepseek-v4-flash"
+DEEPSEEK_LARGE_LLM_MODEL="deepseek-v4-pro"
# Embedding envs
# The following are just examples of the adapter implementation, you can have completely different envs
diff --git a/src/adapters/embeddings.py b/src/adapters/embeddings.py
index aa44725..474fc92 100644
--- a/src/adapters/embeddings.py
+++ b/src/adapters/embeddings.py
@@ -169,7 +169,12 @@ class VectorStoreAdapter:
Search vectors in the vector store and return the top k vectors.
"""
vectors = self.vector_store.search_vectors(data_vector, brain_id, store, k)
- return sorted(vectors, key=lambda x: x.distance, reverse=True)
+ return sorted(
+ vectors,
+ key=lambda x: (
+ float("inf") if x.distance is None else float(x.distance)
+ ),
+ )
def get_by_ids(
self, ids: list[str], store: str, brain_id: str = "default"
diff --git a/src/config.py b/src/config.py
index 38525d0..27d838d 100644
--- a/src/config.py
+++ b/src/config.py
@@ -33,6 +33,14 @@ def _normalize_openai_base_url(raw: str | None) -> str | None:
return normalized
+def validate_pipeline_mode(mode: str | None) -> Literal["lightweight", "accurate"]:
+ if mode not in ("lightweight", "accurate"):
+ raise ValueError(
+ f"Invalid PIPELINE_MODE: {mode!r}. Expected 'lightweight' or 'accurate'."
+ )
+ return mode
+
+
class AzureConfig:
"""
Configuration class for the Azure configuration.
@@ -113,6 +121,26 @@ class AnthropicConfig:
raise ValueError("Anthropic large LLM model is not set")
+class DeepSeekConfig:
+ def __init__(self):
+ self.api_key = os.getenv("DEEPSEEK_API_KEY")
+ self.base_url = "https://api.deepseek.com"
+ self.small_llm_model = os.getenv(
+ "DEEPSEEK_SMALL_LLM_MODEL", "deepseek-v4-flash"
+ )
+ self.large_llm_model = os.getenv(
+ "DEEPSEEK_LARGE_LLM_MODEL", "deepseek-v4-pro"
+ )
+
+ def validate_llm(self, require_large: bool):
+ if not self.api_key:
+ raise ValueError("DeepSeek API key is not set")
+ if not self.small_llm_model:
+ raise ValueError("DeepSeek small LLM model is not set")
+ if require_large and not self.large_llm_model:
+ raise ValueError("DeepSeek large LLM model is not set")
+
+
class GCPConfig:
"""
Configuration class for the GCP configuration.
@@ -395,7 +423,15 @@ class SpacyConfig:
_MODES = ("local", "remote")
-_PROVIDERS = ("ollama", "azure", "openai", "anthropic", "gcp_vertex", "amazon_bedrock")
+_PROVIDERS = (
+ "ollama",
+ "azure",
+ "openai",
+ "anthropic",
+ "deepseek",
+ "gcp_vertex",
+ "amazon_bedrock",
+)
class Config:
@@ -461,6 +497,11 @@ class Config:
or self.llm_large_provider == "anthropic"
)
use_anthropic_large = self.llm_large_provider == "anthropic"
+ use_deepseek_llm = (
+ self.llm_small_provider == "deepseek"
+ or self.llm_large_provider == "deepseek"
+ )
+ use_deepseek_large = self.llm_large_provider == "deepseek"
use_ollama = (
self.llm_small_provider == "ollama"
or self.llm_large_provider == "ollama"
@@ -499,6 +540,10 @@ class Config:
if self.anthropic is not None:
self.anthropic.validate_llm(require_large=use_anthropic_large)
+ self.deepseek = DeepSeekConfig() if use_deepseek_llm else None
+ if self.deepseek is not None:
+ self.deepseek.validate_llm(require_large=use_deepseek_large)
+
self.ollama = OllamaConfig() if use_ollama else None
self.embeddings = EmbeddingsConfig(mode=self.models_mode)
@@ -520,8 +565,8 @@ class Config:
self.docparser_endpoint = os.getenv("DOCPARSER_ENDPOINT")
self.docparser_token = os.getenv("DOCPARSER_TOKEN")
self.app_host = os.getenv("APP_HOST")
- self.pipeline_mode: Literal["lightweight", "accurate"] = os.getenv(
- "PIPELINE_MODE"
+ self.pipeline_mode: Literal["lightweight", "accurate"] = validate_pipeline_mode(
+ os.getenv("PIPELINE_MODE", "accurate")
)
self.ocr_mode: Literal["docling", "docparser"] = os.getenv(
"OCR_MODE", "docling"
diff --git a/src/constants/embeddings.py b/src/constants/embeddings.py
index e43d732..8c42169 100644
--- a/src/constants/embeddings.py
+++ b/src/constants/embeddings.py
@@ -39,7 +39,7 @@ class Vector(BaseModel):
distance: Optional[float] = Field(
default=None,
description=(
- "The distance between the query and the result. "
+ "Cosine distance between the query and the result where lower is nearer. "
"This is only available for search results."
),
)
diff --git a/src/core/agents/architect_agent.py b/src/core/agents/architect_agent.py
index dac8e60..f4b02b4 100644
--- a/src/core/agents/architect_agent.py
+++ b/src/core/agents/architect_agent.py
@@ -64,9 +64,15 @@ from src.core.agents.tools.architect_agent.ArchitectAgentMarkEntitiesAsUsedTool
ArchitectAgentMarkEntitiesAsUsedTool,
)
from src.core.plugins.prompts import prompt_registry
+from src.core.saving.identity import (
+ stable_flow_key,
+ stable_node_id,
+ stable_relationship_id,
+)
from src.core.saving.ingestion_manager import IngestionManager
from src.services.api.constants.requests import IngestionTripleSet
from src.utils.cleanup import strip_properties
+from src.utils.dates import normalize_date_string
from src.utils.similarity.vectors import cosine_similarity
# from src.core.agents.tools.kg_agent import (
@@ -80,11 +86,18 @@ MAX_RECURSION_LIMIT = 100
def _ingestion_partial_node_entity(node) -> ArchitectAgentEntity:
return ArchitectAgentEntity(
- uuid=node.uuid or str(uuid.uuid4()),
+ uuid=stable_node_id(
+ node.name,
+ node.type,
+ getattr(node, "happened_at", None),
+ getattr(node, "uuid", None),
+ ),
name=node.name,
type=node.type,
description=node.description,
properties=node.properties or {},
+ happened_at=getattr(node, "happened_at", None),
+ polarity=getattr(node, "polarity", None) or "neutral",
)
@@ -97,10 +110,140 @@ def _to_architect_relationship(relationship) -> ArchitectAgentRelationship:
else relationship.model_dump(mode="json")
)
if not data.get("flow_key"):
- data["flow_key"] = str(uuid.uuid4())
+ data["flow_key"] = stable_flow_key(
+ event_uuid=None,
+ event_name=(data.get("tip") or {}).get("name")
+ if isinstance(data.get("tip"), dict)
+ else getattr(data.get("tip"), "name", None),
+ event_type=(data.get("tip") or {}).get("type")
+ if isinstance(data.get("tip"), dict)
+ else getattr(data.get("tip"), "type", None),
+ happened_at=(data.get("tip") or {}).get("happened_at")
+ if isinstance(data.get("tip"), dict)
+ else getattr(data.get("tip"), "happened_at", None),
+ )
+ if not data.get("uuid"):
+ tail = data.get("tail") or {}
+ tip = data.get("tip") or {}
+ tail_uuid = (
+ tail.get("uuid")
+ if isinstance(tail, dict)
+ else getattr(tail, "uuid", "")
+ )
+ tip_uuid = (
+ tip.get("uuid") if isinstance(tip, dict) else getattr(tip, "uuid", "")
+ )
+ data["uuid"] = stable_relationship_id(
+ tail_uuid or "",
+ data.get("name") or "",
+ tip_uuid or "",
+ data.get("flow_key"),
+ )
return ArchitectAgentRelationship(**data)
+def ingestion_triples_to_relationships(
+ current_triples: List[IngestionTripleSet],
+ partial_triples: List[IngestionTripleSet],
+) -> Tuple[List[ArchitectAgentRelationship], Dict[Tuple, ArchitectAgentEntity]]:
+ triple_entity_registry: Dict[Tuple, ArchitectAgentEntity] = {}
+
+ def _registry_key(node) -> Tuple:
+ name = (node.name or "").strip().lower()
+ entity_type = (node.type or "").strip().lower()
+ if entity_type == "event":
+ return (
+ name,
+ entity_type,
+ normalize_date_string(getattr(node, "happened_at", None)) or "",
+ )
+ return (name, entity_type)
+
+ def _triple_entity(node) -> ArchitectAgentEntity:
+ key = _registry_key(node)
+ if key not in triple_entity_registry:
+ triple_entity_registry[key] = _ingestion_partial_node_entity(node)
+ elif getattr(node, "uuid", None):
+ existing = triple_entity_registry[key]
+ if node.uuid and existing.uuid != node.uuid:
+ triple_entity_registry[key] = _ingestion_partial_node_entity(node)
+ return triple_entity_registry[key]
+
+ triple_relationships: List[ArchitectAgentRelationship] = []
+ for cr in current_triples:
+ if not cr.subject or not cr.subj_event:
+ continue
+ subject = _triple_entity(cr.subject)
+ event = _triple_entity(cr.event)
+ obj = _triple_entity(cr.object)
+ flow_key = stable_flow_key(
+ event_uuid=event.uuid,
+ event_name=event.name,
+ event_type=event.type,
+ happened_at=event.happened_at,
+ supplied=cr.subj_event.uuid or cr.event_obj.uuid,
+ )
+ subj_event_uuid = cr.subj_event.uuid or stable_relationship_id(
+ subject.uuid, cr.subj_event.name, event.uuid, flow_key
+ )
+ event_obj_uuid = cr.event_obj.uuid or stable_relationship_id(
+ event.uuid, cr.event_obj.name, obj.uuid, flow_key
+ )
+ triple_relationships.extend(
+ [
+ ArchitectAgentRelationship(
+ tail=subject,
+ name=cr.subj_event.name,
+ tip=event,
+ description=cr.subj_event.description,
+ amount=cr.subj_event.amount,
+ properties=cr.subj_event.properties or {},
+ uuid=subj_event_uuid,
+ flow_key=flow_key,
+ ),
+ ArchitectAgentRelationship(
+ tail=event,
+ name=cr.event_obj.name,
+ tip=obj,
+ description=cr.event_obj.description,
+ amount=cr.event_obj.amount,
+ properties=cr.event_obj.properties or {},
+ uuid=event_obj_uuid,
+ flow_key=flow_key,
+ ),
+ ]
+ )
+ for pt in partial_triples:
+ event = _triple_entity(pt.event)
+ obj = _triple_entity(pt.object)
+ if pt.subject:
+ _triple_entity(pt.subject)
+ flow_key = stable_flow_key(
+ event_uuid=event.uuid,
+ event_name=event.name,
+ event_type=event.type,
+ happened_at=event.happened_at,
+ supplied=pt.event_obj.uuid,
+ )
+ event_obj_uuid = pt.event_obj.uuid or stable_relationship_id(
+ event.uuid, pt.event_obj.name, obj.uuid, flow_key
+ )
+ triple_relationships.append(
+ ArchitectAgentRelationship(
+ tail=event,
+ name=pt.event_obj.name,
+ tip=obj,
+ description=pt.event_obj.description,
+ amount=pt.event_obj.amount,
+ properties=pt.event_obj.properties or {},
+ uuid=event_obj_uuid,
+ flow_key=flow_key,
+ ),
+ )
+
+ return triple_relationships, triple_entity_registry
+
+
class ArchitectAgent:
"""
Architect agent.
@@ -155,6 +298,7 @@ class ArchitectAgent:
self.embeddings = embeddings
self.agent = None
self.relationships_set: List[ArchitectAgentRelationship] = []
+ self.pending_persistence_batches: List[List[ArchitectAgentRelationship]] = []
self.used_entities_dict = {}
self.ingestion_manager = ingestion_manager
self.session_id: Optional[str] = None
@@ -162,6 +306,23 @@ class ArchitectAgent:
self._janitor_agent_brain_id = None
# self.database_desc = database_desc
+ def take_pending_relationships(self) -> List[ArchitectAgentRelationship]:
+ pending: List[ArchitectAgentRelationship] = []
+ for batch in self.pending_persistence_batches:
+ pending.extend(batch)
+ self.pending_persistence_batches.clear()
+ if pending:
+ return pending
+ return list(self.relationships_set)
+
+ def queue_relationships_for_persistence(
+ self, relationships: List[ArchitectAgentRelationship]
+ ) -> None:
+ if not relationships:
+ return
+ self.pending_persistence_batches.append(list(relationships))
+ self.relationships_set.extend(relationships)
+
def _get_tools(
self,
text: Optional[str] = None,
@@ -759,36 +920,10 @@ class ArchitectAgent:
for rel in output_rels
if isinstance(rel, ArchitectAgentRelationship)
]
- if relationships_data:
- from src.lib.redis.client import _redis_client
- from src.workers.tasks.ingestion import process_architect_relationships
-
- if self.session_id:
- _redis_client.client.incr(
- f"{brain_id}:session:{self.session_id}:pending_tasks"
- )
-
- process_architect_relationships.delay(
- {
- "relationships": relationships_data,
- "brain_id": brain_id,
- "session_id": self.session_id,
- }
- )
-
- self.relationships_set.extend(output_rels)
-
- if self.session_id and output_rels:
- from src.lib.redis.client import _redis_client
-
- _redis_client.set(
- f"session:{self.session_id}:relationships",
- json.dumps(
- [rel.model_dump(mode="json") for rel in self.relationships_set]
- ),
- brain_id=brain_id,
- expires_in=3600,
- )
+ if output_rels:
+ self.queue_relationships_for_persistence(output_rels)
+ elif relationships_data:
+ pass
wrong_relationships = (
getattr(janitor_response, "wrong_relationships", []) or []
@@ -806,6 +941,7 @@ class ArchitectAgent:
ingestion_session_id: Optional[str] = None,
partial_triples: List[IngestionTripleSet] = [],
current_triples: List[IngestionTripleSet] = [],
+ persist_submitted: bool = True,
) -> ArchitectAgentResponse:
"""
Given the provided text and partial/full triples, discover relationships and new nodes for the provided entities.
@@ -826,62 +962,9 @@ class ArchitectAgent:
- relationships: list of relationships the agent created between entities or new nodes.
"""
- triple_entity_registry: Dict[Tuple[str, str], ArchitectAgentEntity] = {}
-
- def _triple_entity(node) -> ArchitectAgentEntity:
- key = (node.name.strip().lower(), (node.type or "").strip().lower())
- if key not in triple_entity_registry:
- triple_entity_registry[key] = _ingestion_partial_node_entity(node)
- return triple_entity_registry[key]
-
- triple_relationships: List[ArchitectAgentRelationship] = []
- for cr in current_triples:
- if not cr.subject or not cr.subj_event:
- continue
- flow_key = str(uuid.uuid4())
- subject = _triple_entity(cr.subject)
- event = _triple_entity(cr.event)
- obj = _triple_entity(cr.object)
- triple_relationships.extend(
- [
- ArchitectAgentRelationship(
- tail=subject,
- name=cr.subj_event.name,
- tip=event,
- description=cr.subj_event.description,
- amount=cr.subj_event.amount,
- properties=cr.subj_event.properties or {},
- uuid=str(uuid.uuid4()),
- flow_key=flow_key,
- ),
- ArchitectAgentRelationship(
- tail=event,
- name=cr.event_obj.name,
- tip=obj,
- description=cr.event_obj.description,
- amount=cr.event_obj.amount,
- properties=cr.event_obj.properties or {},
- uuid=str(uuid.uuid4()),
- flow_key=flow_key,
- ),
- ]
- )
- for pt in partial_triples:
- flow_key = str(uuid.uuid4())
- event = _triple_entity(pt.event)
- obj = _triple_entity(pt.object)
- triple_relationships.append(
- ArchitectAgentRelationship(
- tail=event,
- name=pt.event_obj.name,
- tip=obj,
- description=pt.event_obj.description,
- amount=pt.event_obj.amount,
- properties=pt.event_obj.properties or {},
- uuid=str(uuid.uuid4()),
- flow_key=flow_key,
- ),
- )
+ triple_relationships, triple_entity_registry = ingestion_triples_to_relationships(
+ current_triples, partial_triples
+ )
triple_keys = set(triple_entity_registry.keys())
entities = [
@@ -892,6 +975,8 @@ class ArchitectAgent:
type=te.type,
description=te.description,
properties=te.properties or {},
+ happened_at=te.happened_at,
+ polarity=te.polarity,
)
for te in triple_entity_registry.values()
],
@@ -907,6 +992,15 @@ class ArchitectAgent:
self.entities = entities_dict
self.session_id = str(uuid.uuid4())
self.relationships_set.clear()
+ self.pending_persistence_batches.clear()
+
+ if triple_relationships and persist_submitted:
+ self._persist_relationships(
+ triple_relationships,
+ text=text or "",
+ brain_id=brain_id,
+ targeting=targeting,
+ )
self._get_agent(
output_schema=_ArchitectAgentResponse,
@@ -1259,6 +1353,7 @@ class ArchitectAgent:
self.session_id = str(uuid.uuid4())
self.relationships_set.clear()
+ self.pending_persistence_batches.clear()
entities_dict = {
entity.uuid: strip_properties([entity.model_dump(mode="json")])[0]
diff --git a/src/core/agents/scout_agent.py b/src/core/agents/scout_agent.py
index 100bdd4..8a27e56 100644
--- a/src/core/agents/scout_agent.py
+++ b/src/core/agents/scout_agent.py
@@ -51,6 +51,10 @@ class _ScoutEntity(BaseModel):
name: str
properties: Optional[dict] = Field(default_factory=dict)
description: Optional[str] = None
+ happened_at: Optional[str] = Field(
+ default=None,
+ description="The date and time the entity happened at if known otherwise None.",
+ )
polarity: Optional[Literal["positive", "negative", "neutral"]] = Field(
default="neutral",
description="The polarity of the entity.",
diff --git a/src/core/agents/tools/architect_agent/ArchitectAgentCreateRelationshipTool.py b/src/core/agents/tools/architect_agent/ArchitectAgentCreateRelationshipTool.py
index 1bf23dc..cfe4b19 100644
--- a/src/core/agents/tools/architect_agent/ArchitectAgentCreateRelationshipTool.py
+++ b/src/core/agents/tools/architect_agent/ArchitectAgentCreateRelationshipTool.py
@@ -548,47 +548,8 @@ class ArchitectAgentCreateRelationshipTool(BaseTool):
"[DEBUG (architect_agent_create_relationship)]: Relationships data: ",
relationships_data,
)
- if relationships_data:
- from src.workers.tasks.ingestion import process_architect_relationships
- from src.lib.redis.client import _redis_client
-
- print(
- "[DEBUG (architect_agent_create_relationship)]: Sending relationships to ingestion task"
- )
-
- session_id = getattr(self.architect_agent, "session_id", None)
- if session_id:
- _redis_client.client.incr(
- f"{self.brain_id}:session:{session_id}:pending_tasks"
- )
-
- task_result = process_architect_relationships.delay(
- {
- "relationships": relationships_data,
- "brain_id": self.brain_id,
- "session_id": session_id,
- }
- )
- print(
- f"[DEBUG (architect_agent_create_relationship)]: Task {task_result.id} queued for session {session_id}"
- )
-
- self.architect_agent.relationships_set.extend(output_rels)
-
- session_id = getattr(self.architect_agent, "session_id", None)
- if session_id and output_rels:
- from src.lib.redis.client import _redis_client
-
- relationships_data = [
- rel.model_dump(mode="json")
- for rel in self.architect_agent.relationships_set
- ]
- _redis_client.set(
- f"session:{session_id}:relationships",
- json.dumps(relationships_data),
- brain_id=self.brain_id,
- expires_in=3600,
- )
+ if output_rels:
+ self.architect_agent.queue_relationships_for_persistence(output_rels)
wrong_relationships = []
if getattr(janitor_response, "wrong_relationships", []):
@@ -612,10 +573,4 @@ class ArchitectAgentCreateRelationshipTool(BaseTool):
),
}
- # for entity_uuid in used_entity_uuids:
- # if entity_uuid in self.architect_agent.entities:
- # del self.architect_agent.entities[entity_uuid]
-
- # return natural_lang
-
return json.dumps({"status": "success"})
diff --git a/src/core/instances.py b/src/core/instances.py
index 27e0e18..f0a77c4 100644
--- a/src/core/instances.py
+++ b/src/core/instances.py
@@ -23,6 +23,10 @@ def _build_small_llm(provider: str):
from src.lib.llm.client_anthropic import _llm_small_client_anthropic
return _llm_small_client_anthropic
+ if provider == "deepseek":
+ from src.lib.llm.client_deepseek import _llm_small_client_deepseek
+
+ return _llm_small_client_deepseek
if provider == "gcp_vertex":
from src.lib.llm.client_small import _llm_small_client
@@ -51,6 +55,10 @@ def _build_large_llm(provider: str):
from src.lib.llm.client_anthropic import _llm_large_client_anthropic
return _llm_large_client_anthropic
+ if provider == "deepseek":
+ from src.lib.llm.client_deepseek import _llm_large_client_deepseek
+
+ return _llm_large_client_deepseek
if provider == "gcp_vertex":
from src.lib.llm.client_vertex import _llm_large_client_vertex
diff --git a/src/core/saving/auto_kg.py b/src/core/saving/auto_kg.py
index 307a5fb..4896969 100644
--- a/src/core/saving/auto_kg.py
+++ b/src/core/saving/auto_kg.py
@@ -10,7 +10,8 @@ Modified By: Christian Nonis <[email protected]>
import os
import uuid
-from typing import Optional
+from dataclasses import dataclass, field
+from typing import List, Optional
from src.config import config
from src.constants.kg import Node
@@ -28,19 +29,23 @@ from src.services.input.agents import (
import langsmith
+@dataclass
+class EnrichmentOrchestrationResult:
+ session_id: Optional[str]
+ ingestion_session_id: str
+ brain_id: str
+ enrichment_relationships: List[dict] = field(default_factory=list)
+ should_consolidate: bool = False
+
+
def enrich_kg_from_input(
input: str, targeting: Optional[Node] = None, brain_id: str = "default"
-) -> str:
+) -> EnrichmentOrchestrationResult:
"""
Orchestrates enrichment of the knowledge graph from a free-text input.
- Runs the scout and architect agents to extract entities and relationships from the provided input, and optionally consolidates the resulting relationships into the graph. Side effects include updating the knowledge graph and printing debug summaries to standard output.
-
- Parameters:
- input (str): Free-text content to process and ingest into the knowledge graph.
- targeting (Optional[Node]): Optional target node guiding where or how the input should be applied within the graph.
- brain_id (str): Identifier for the processing context/brain to use for agent operations and consolidation.
-
+ Runs the scout and architect agents to extract entities and relationships from the provided input
+ and returns the collected persistence batch for the caller to write.
"""
ingestion_session_id = str(uuid.uuid4())
@@ -55,10 +60,12 @@ def enrich_kg_from_input(
"flow": "enrich_kg_from_input",
},
):
- _enrich_kg_impl(input, targeting, brain_id, ingestion_session_id)
+ return _enrich_kg_impl(input, targeting, brain_id, ingestion_session_id)
-def _enrich_kg_impl(input: str, targeting, brain_id: str, ingestion_session_id: str):
+def _enrich_kg_impl(
+ input, targeting, brain_id: str, ingestion_session_id: str
+) -> EnrichmentOrchestrationResult:
ingestion_manager = IngestionManager(
embeddings_adapter, vector_store_adapter, graph_adapter
)
@@ -79,97 +86,36 @@ def _enrich_kg_impl(input: str, targeting, brain_id: str, ingestion_session_id:
ingestion_manager=ingestion_manager,
)
- if config.pipeline_mode == "lightweight":
- print("[DEBUG (enrich_kg_from_input)]: Lightweight pipeline mode selected")
-
- entities = scout_agent.run(
- input,
- targeting=targeting,
- brain_id=brain_id,
- ingestion_session_id=ingestion_session_id,
- mode="coarse",
- )
- print("[DEBUG (initial_scout_entities)]: ", entities.entities)
- architect_agent.run_tooler(
- input,
- entities.entities,
- targeting=targeting,
- brain_id=brain_id,
- timeout=20000,
- ingestion_session_id=ingestion_session_id,
- mode="coarse",
- )
-
- return
-
- if config.pipeline_mode == "accurate":
- print(f"[DEBUG (ingestion_session_id)]: {ingestion_session_id}")
-
- entities = scout_agent.run(
- input,
- targeting=targeting,
- brain_id=brain_id,
- ingestion_session_id=ingestion_session_id,
- )
-
- architect_agent.run_tooler(
- input,
- entities.entities,
- targeting=targeting,
- brain_id=brain_id,
- timeout=20000,
- ingestion_session_id=ingestion_session_id,
- )
-
- if config.run_graph_consolidator and architect_agent.session_id:
- from src.lib.redis.client import _redis_client
- import time
-
- print(
- f"[DEBUG (enrich_kg_from_input)]: Waiting for async tasks to complete for session {architect_agent.session_id}"
- )
-
- max_wait_time = 300
- check_interval = 2
- elapsed_time = 0
-
- while elapsed_time < max_wait_time:
- pending_count = _redis_client.client.get(
- f"{brain_id}:session:{architect_agent.session_id}:pending_tasks"
- )
-
- if pending_count is None or int(pending_count) == 0:
- print(
- f"[DEBUG (enrich_kg_from_input)]: All async tasks completed after {elapsed_time}s"
- )
- break
-
- print(
- f"[DEBUG (enrich_kg_from_input)]: {pending_count} tasks still pending, waiting..."
- )
- time.sleep(check_interval)
- elapsed_time += check_interval
- else:
- print(
- f"[DEBUG (enrich_kg_from_input)]: Timeout waiting for tasks to complete after {max_wait_time}s"
- )
-
- print(
- "[DEBUG (consolidate_graph)]: Consolidating graph with ",
- len(architect_agent.relationships_set),
- " relationships",
- )
-
- from src.workers.tasks.ingestion import consolidate_graph_async
-
- task_result = consolidate_graph_async.delay(
- session_id=architect_agent.session_id,
- brain_id=brain_id,
- ingestion_session_id=ingestion_session_id,
- )
- print(
- f"[DEBUG (enrich_kg_from_input)]: Consolidation task {task_result.id} queued"
- )
+ mode = "coarse" if config.pipeline_mode == "lightweight" else "granular"
+ print(f"[DEBUG (enrich_kg_from_input)]: {config.pipeline_mode} pipeline mode selected")
+
+ entities = scout_agent.run(
+ input,
+ targeting=targeting,
+ brain_id=brain_id,
+ ingestion_session_id=ingestion_session_id,
+ mode=("coarse" if mode == "coarse" else "granular"),
+ )
+ print("[DEBUG (initial_scout_entities)]: ", entities.entities)
+ architect_agent.run_tooler(
+ input,
+ entities.entities,
+ targeting=targeting,
+ brain_id=brain_id,
+ timeout=20000,
+ ingestion_session_id=ingestion_session_id,
+ mode=("coarse" if mode == "coarse" else "granular"),
+ )
+
+ pending = architect_agent.take_pending_relationships()
+ enrichment_relationships = [
+ rel.model_dump(mode="json") for rel in pending
+ ]
+ should_consolidate = bool(
+ config.pipeline_mode == "accurate"
+ and config.run_graph_consolidator
+ and enrichment_relationships
+ )
try:
from langchain_core.tracers.langchain import wait_for_all_tracers
@@ -177,3 +123,11 @@ def _enrich_kg_impl(input: str, targeting, brain_id: str, ingestion_session_id:
wait_for_all_tracers()
except ImportError:
pass
+
+ return EnrichmentOrchestrationResult(
+ session_id=architect_agent.session_id,
+ ingestion_session_id=ingestion_session_id,
+ brain_id=brain_id,
+ enrichment_relationships=enrichment_relationships,
+ should_consolidate=should_consolidate,
+ )
diff --git a/src/core/saving/identity.py b/src/core/saving/identity.py
new file mode 100644
index 0000000..28142e6
--- /dev/null
+++ b/src/core/saving/identity.py
@@ -0,0 +1,61 @@
+import hashlib
+from typing import Optional
+
+from src.utils.dates import normalize_date_string
+
+
+def stable_uuid(*parts: Optional[str]) -> str:
+ material = "|".join((part or "").strip().lower() for part in parts)
+ digest = hashlib.sha256(material.encode("utf-8")).hexdigest()
+ return f"{digest[:8]}-{digest[8:12]}-{digest[12:16]}-{digest[16:20]}-{digest[20:32]}"
+
+
+def stable_node_id(
+ name: Optional[str],
+ entity_type: Optional[str],
+ happened_at: Optional[str] = None,
+ supplied_uuid: Optional[str] = None,
+) -> str:
+ if supplied_uuid and str(supplied_uuid).strip():
+ return str(supplied_uuid).strip()
+ normalized_type = (entity_type or "").strip().lower()
+ parts = [(name or "").strip().lower(), normalized_type]
+ if normalized_type == "event":
+ parts.append(normalize_date_string(happened_at) or "")
+ return stable_uuid("node", *parts)
+
+
+def stable_relationship_id(
+ tail_uuid: str,
+ predicate: str,
+ tip_uuid: str,
+ flow_key: Optional[str] = None,
+) -> str:
+ parts = [
+ "rel",
+ tail_uuid or "",
+ (predicate or "").strip().upper(),
+ tip_uuid or "",
+ ]
+ if flow_key:
+ parts.append(flow_key)
+ return stable_uuid(*parts)
+
+
+def stable_flow_key(
+ event_uuid: Optional[str] = None,
+ event_name: Optional[str] = None,
+ event_type: Optional[str] = None,
+ happened_at: Optional[str] = None,
+ supplied: Optional[str] = None,
+) -> str:
+ if supplied and str(supplied).strip():
+ return str(supplied).strip()
+ if event_uuid and str(event_uuid).strip():
+ return str(event_uuid).strip()
+ return stable_uuid(
+ "flow",
+ event_name,
+ event_type,
+ normalize_date_string(happened_at) if happened_at else None,
+ )
diff --git a/src/core/saving/ingestion_manager.py b/src/core/saving/ingestion_manager.py
index 44835a2..b3a9887 100644
--- a/src/core/saving/ingestion_manager.py
+++ b/src/core/saving/ingestion_manager.py
@@ -14,6 +14,20 @@ from src.adapters.graph import GraphAdapter
from src.adapters.embeddings import VectorStoreAdapter
from src.core.agents.scout_agent import ScoutEntity
from src.core.agents.architect_agent import ArchitectAgentRelationship
+from src.core.saving.identity import (
+ stable_flow_key,
+ stable_node_id,
+ stable_relationship_id,
+ stable_uuid,
+)
+
+__all__ = [
+ "IngestionManager",
+ "stable_flow_key",
+ "stable_node_id",
+ "stable_relationship_id",
+ "stable_uuid",
+]
class IngestionManager:
diff --git a/src/lib/llm/client_deepseek.py b/src/lib/llm/client_deepseek.py
new file mode 100644
index 0000000..49e8e6a
--- /dev/null
+++ b/src/lib/llm/client_deepseek.py
@@ -0,0 +1,104 @@
+import json
+import os
+import threading
+from typing import Any
+
+from langchain_openai import ChatOpenAI
+from openai import OpenAI
+
+from src.adapters.interfaces.llm import LLM
+from src.config import config
+
+_THINKING_DISABLED = {"thinking": {"type": "disabled"}}
+
+
+def _deepseek_client_kwargs() -> dict[str, Any]:
+ return {
+ "api_key": config.deepseek.api_key,
+ "base_url": config.deepseek.base_url,
+ "timeout": 120.0,
+ "max_retries": 3,
+ }
+
+
+class _DeepSeekLLMBase(LLM):
+ def __init__(self, model: str):
+ self.model = model
+ self._client: OpenAI | None = None
+ self._langchain_model: ChatOpenAI | None = None
+ self._lock = threading.Lock()
+ self._pid = os.getpid()
+
+ def _check_fork(self) -> None:
+ if os.getpid() != self._pid:
+ self._client = None
+ self._langchain_model = None
+ self._pid = os.getpid()
+
+ @property
+ def client(self) -> OpenAI:
+ self._check_fork()
+ if self._client is None:
+ with self._lock:
+ if self._client is None:
+ self._client = OpenAI(**_deepseek_client_kwargs())
+ return self._client
+
+ @property
+ def langchain_model(self) -> ChatOpenAI:
+ self._check_fork()
+ if self._langchain_model is None:
+ with self._lock:
+ if self._langchain_model is None:
+ self._langchain_model = ChatOpenAI(
+ model=self.model,
+ **_deepseek_client_kwargs(),
+ extra_body=_THINKING_DISABLED,
+ )
+ return self._langchain_model
+
+ def generate_text(self, prompt: str, max_new_tokens: int = None) -> str:
+ response = self.client.chat.completions.create(
+ model=self.model,
+ messages=[{"role": "user", "content": prompt}],
+ **({"max_tokens": max_new_tokens} if max_new_tokens else {}),
+ extra_body=_THINKING_DISABLED,
+ )
+ return response.choices[0].message.content
+
+ def generate_json(
+ self, prompt: str, max_new_tokens: int = None, max_retries: int = 3
+ ) -> dict:
+ response = None
+ while max_retries > 0 and response is None:
+ try:
+ _response = self.client.chat.completions.create(
+ model=self.model,
+ messages=[{"role": "user", "content": prompt}],
+ **({"max_tokens": max_new_tokens} if max_new_tokens else {}),
+ response_format={"type": "json_object"},
+ extra_body=_THINKING_DISABLED,
+ )
+ _response = (
+ _response.choices[0].message.content.strip("```json").strip("```")
+ )
+ response = json.loads(_response)
+ except Exception as e:
+ max_retries -= 1
+ if max_retries <= 0:
+ raise e
+ return response
+
+
+class DeepSeekLLMClientSmall(_DeepSeekLLMBase):
+ def __init__(self):
+ super().__init__(config.deepseek.small_llm_model)
+
+
+class DeepSeekLLMClientLarge(_DeepSeekLLMBase):
+ def __init__(self):
+ super().__init__(config.deepseek.large_llm_model)
+
+
+_llm_small_client_deepseek = DeepSeekLLMClientSmall()
+_llm_large_client_deepseek = DeepSeekLLMClientLarge()
diff --git a/src/lib/milvus/client.py b/src/lib/milvus/client.py
index 05c8850..cc17785 100644
--- a/src/lib/milvus/client.py
+++ b/src/lib/milvus/client.py
@@ -235,6 +235,7 @@ class MilvusClient(VectorStoreClient):
results = []
for query_results in _results:
for result in query_results:
+ raw_score = float(result.get("distance", 0.0))
results.append(
Vector(
id=str(result["id"]),
@@ -243,10 +244,14 @@ class MilvusClient(VectorStoreClient):
for k, v in result.get("entity").items()
if k not in ["id", "embeddings", "distance"]
},
- distance=result["distance"],
+ distance=1.0 - raw_score,
)
)
- results.sort(key=lambda x: x.distance)
+ results.sort(
+ key=lambda x: (
+ float("inf") if x.distance is None else float(x.distance)
+ )
+ )
return results
def get_by_ids(self, ids: list[str], store: str, brain_id: str) -> list[Vector]:
@@ -339,7 +344,9 @@ class MilvusClient(VectorStoreClient):
r_entity = r.get("entity", {})
if r_entity.get("uuid") == origin_uuid:
continue
- sim = 1.0 - float(r.get("distance", 0.0))
+ raw_score = float(r.get("distance", 0.0))
+ distance = 1.0 - raw_score
+ sim = 1.0 - distance
if sim >= min_similarity:
collected.append(
Vector(
@@ -349,7 +356,7 @@ class MilvusClient(VectorStoreClient):
for k, v in r.get("entity", {}).items()
if k not in ["id", "embeddings", "distance"]
},
- distance=r["distance"],
+ distance=distance,
)
)
if len(collected) >= limit:
diff --git a/src/lib/neo4j/client.py b/src/lib/neo4j/client.py
index 4190524..6214dd9 100644
--- a/src/lib/neo4j/client.py
+++ b/src/lib/neo4j/client.py
@@ -224,27 +224,21 @@ class Neo4jClient(GraphClient):
self.ensure_database(brain_id)
for node in nodes:
- identification_dict = {"name": node.name}
+ if not node.uuid:
+ raise ValueError("Node uuid is required for upsert")
merged_metadata = {**(metadata or {}), **(node.metadata or {})}
all_properties = {
**(node.properties or {}),
"metadata": merged_metadata or None,
+ "name": node.name,
}
if identification_params:
for key, value in identification_params.items():
normalized_key = self._clean_property_key(key)
- if normalized_key != "name":
- identification_dict[normalized_key] = value
-
- identification_items = []
- for key, value in identification_dict.items():
- cypher_key = self._format_property_key(key)
- escaped_value = self._format_value(value)
- identification_items.append(f"{cypher_key}: {escaped_value}")
-
- identification_set_str = f"{{{', '.join(identification_items)}}}"
+ if normalized_key not in {"uuid", "name"}:
+ all_properties[normalized_key] = value
attributes = [
"description",
@@ -267,13 +261,11 @@ class Neo4jClient(GraphClient):
escaped_value = self._format_value(value)
property_assignments.append(f"n.{cypher_key} = {escaped_value}")
- property_assignments.append(f"n['uuid'] = '{node.uuid}'")
-
properties_set = f"{', '.join(property_assignments)}"
labels_expression = ":".join(self._clean_labels(node.labels))
cypher_query = f"""
- MERGE (n:{labels_expression} {identification_set_str})
+ MERGE (n:{labels_expression} {{uuid: {self._format_value(node.uuid)}}})
SET {properties_set}
RETURN n
"""
@@ -328,6 +320,9 @@ class Neo4jClient(GraphClient):
"amount",
]
+ if not subject.uuid or not to_object.uuid or not predicate.uuid:
+ raise ValueError("Subject, object, and predicate uuids are required for upsert")
+
extra_ops = ""
for obj in objects:
for attr in attributes:
@@ -337,17 +332,16 @@ class Neo4jClient(GraphClient):
SET r['{attr}'] = {self._format_value(value)}
"""
+ rel_type = ":".join(self._clean_labels([predicate.name]))
cypher_query = f"""
- MATCH (a:{":".join(self._clean_labels(subject.labels))}) WHERE a['name'] = {self._format_value(subject.name)}
- MATCH (b:{":".join(self._clean_labels(to_object.labels))}) WHERE b['name'] = {self._format_value(to_object.name)}
- MERGE (a)-[r:{":".join(self._clean_labels([predicate.name]))}]->(b)
- ON CREATE
- SET r['description'] = {self._format_value(predicate.description)},
- r['uuid'] = {self._format_value(predicate.uuid)},
- r['v_id'] = {self._format_value(predicate.properties.get("v_id"))},
- r['flow_key'] = {self._format_value(predicate.flow_key)}
+ MATCH (a) WHERE a['uuid'] = {self._format_value(subject.uuid)}
+ MATCH (b) WHERE b['uuid'] = {self._format_value(to_object.uuid)}
+ MERGE (a)-[r:{rel_type} {{uuid: {self._format_value(predicate.uuid)}}}]->(b)
+ SET r['description'] = {self._format_value(predicate.description)},
+ r['v_id'] = {self._format_value((predicate.properties or {}).get("v_id"))},
+ r['flow_key'] = {self._format_value(predicate.flow_key)}
{extra_ops}
- RETURN a, b
+ RETURN a, b, r
"""
self.ensure_database(brain_id)
diff --git a/src/lib/postgresql/graph_store.py b/src/lib/postgresql/graph_store.py
index 27a5ff7..2ed7d39 100644
--- a/src/lib/postgresql/graph_store.py
+++ b/src/lib/postgresql/graph_store.py
@@ -77,32 +77,15 @@ class _BrainGraph:
def upsert_node(self, labels: list[str], identification: dict, properties: dict) -> str:
node_uuid = properties.get("uuid") or identification.get("uuid")
- match_uuid = None
- for candidate_uuid, data in self.graph.nodes(data=True):
- if identification.get("uuid") and candidate_uuid != identification["uuid"]:
- continue
- if identification.get("name") and data.get("name") != identification.get("name"):
- continue
- if labels and not set(labels).issubset(set(data.get("labels") or [])):
- continue
- extra_match = True
- for key, value in identification.items():
- if key in {"uuid", "name"}:
- continue
- if data.get(key) != value:
- extra_match = False
- break
- if extra_match:
- match_uuid = candidate_uuid
- break
- if match_uuid is None:
- match_uuid = node_uuid or identification.get("uuid")
- if match_uuid is None:
+ if not node_uuid:
raise ValueError("Unable to resolve node uuid for merge")
+ match_uuid = node_uuid if node_uuid in self.graph else node_uuid
merged = {}
if match_uuid in self.graph:
merged.update(self.node_data(match_uuid))
merged.update(properties)
+ if identification.get("name") is not None:
+ merged["name"] = identification.get("name")
merged["labels"] = labels or merged.get("labels") or []
merged["uuid"] = match_uuid
merged.setdefault("name", identification.get("name"))
@@ -398,16 +381,32 @@ class PostgreSQLGraphStore:
object_name: Any,
rel_type: str,
rel_props: dict[str, Any],
+ subject_uuid: Optional[str] = None,
+ object_uuid: Optional[str] = None,
) -> Optional[tuple[dict, dict]]:
brain = self._load_brain(brain_id)
- source_uuid = self.resolve_node_by_name_labels(brain, subject_labels, subject_name)
- target_uuid = self.resolve_node_by_name_labels(brain, object_labels, object_name)
+ source_uuid = subject_uuid if subject_uuid and subject_uuid in brain.graph else None
+ target_uuid = object_uuid if object_uuid and object_uuid in brain.graph else None
+ if not source_uuid:
+ source_uuid = self.resolve_node_by_name_labels(
+ brain, subject_labels, subject_name
+ )
+ if not target_uuid:
+ target_uuid = self.resolve_node_by_name_labels(
+ brain, object_labels, object_name
+ )
if not source_uuid or not target_uuid:
return None
props = dict(rel_props)
props["rel_type"] = rel_type
rel_uuid = props.get("uuid") or f"{source_uuid}-{rel_type}-{target_uuid}"
props["uuid"] = rel_uuid
+ if brain.graph.has_edge(source_uuid, target_uuid, key=rel_uuid):
+ existing = dict(brain.graph.get_edge_data(source_uuid, target_uuid, key=rel_uuid))
+ existing.update(props)
+ props = existing
+ props["uuid"] = rel_uuid
+ props["rel_type"] = rel_type
brain.graph.add_edge(source_uuid, target_uuid, key=rel_uuid, **props)
self._persist_relationship(
brain_id, rel_uuid, rel_type, source_uuid, target_uuid, props
diff --git a/src/lib/postgresql/networkx_client.py b/src/lib/postgresql/networkx_client.py
index 55cabb9..2a6e92d 100644
--- a/src/lib/postgresql/networkx_client.py
+++ b/src/lib/postgresql/networkx_client.py
@@ -124,7 +124,9 @@ class NetworkXGraphClient(GraphClient):
) -> list[Node] | str:
self._store.ensure_database(brain_id)
for node in nodes:
- identification_dict: dict[str, Any] = {"name": node.name}
+ if not node.uuid:
+ raise ValueError("Node uuid is required for upsert")
+ identification_dict: dict[str, Any] = {"uuid": node.uuid, "name": node.name}
merged_metadata = {**(metadata or {}), **(node.metadata or {})}
all_properties: dict[str, Any] = {
**(node.properties or {}),
@@ -133,7 +135,7 @@ class NetworkXGraphClient(GraphClient):
if identification_params:
for key, value in identification_params.items():
normalized_key = self._clean_property_key(key)
- if normalized_key != "name":
+ if normalized_key not in {"uuid", "name"}:
identification_dict[normalized_key] = value
for attr in (
"description",
@@ -194,6 +196,8 @@ class NetworkXGraphClient(GraphClient):
to_object.name,
self._clean_labels([predicate.name])[0],
rel_props,
+ subject_uuid=subject.uuid,
+ object_uuid=to_object.uuid,
)
return "ok"
diff --git a/src/services/api/constants/requests.py b/src/services/api/constants/requests.py
index 51aafce..f1cb571 100644
--- a/src/services/api/constants/requests.py
+++ b/src/services/api/constants/requests.py
@@ -10,7 +10,7 @@ Modified By: Christian Nonis <[email protected]>
from typing import Any, List, Optional, Tuple, Union
-from pydantic import BaseModel, Field, field_serializer
+from pydantic import BaseModel, Field, field_serializer, field_validator, model_validator
from src.constants.data import Observation, PartialNode, PartialPredicate, TextChunk
from src.constants.kg import (
@@ -60,14 +60,38 @@ class IngestionStructuredDataElement(BaseModel):
class RequestPartialNode(PartialNode):
uuid: Optional[str] = Field(None, description="The id of the node.")
type: str = Field(description="The type of the node.")
+ labels: Optional[List[str]] = Field(
+ default=None,
+ description="Optional labels; defaults to [type] when omitted.",
+ )
+ description: Optional[str] = Field(
+ default=None,
+ description="The description of the node.",
+ )
+ properties: Optional[dict] = Field(
+ default_factory=dict,
+ description="The properties of the node.",
+ )
happened_at: Optional[str] = Field(
None,
description="The date and time the node happened at if known otherwise None. Mostly used for event nodes.",
)
+ @model_validator(mode="after")
+ def default_labels_from_type(self):
+ if not self.labels:
+ self.labels = [self.type]
+ if self.properties is None:
+ self.properties = {}
+ return self
+
class RequestPartialPredicate(PartialPredicate):
uuid: Optional[str] = Field(None, description="The id of the relationship.")
+ description: Optional[str] = Field(
+ default=None,
+ description="The description of the relationship.",
+ )
amount: Optional[Union[int, float]] = Field(
None,
description="The amount of the relationship, if it is a quantitative relationship.",
@@ -110,6 +134,16 @@ class PartialNodeFilter(BaseModel):
description="An extra description that explains what this node is about, helpful for entity resolution.",
)
+ @model_validator(mode="after")
+ def require_uuid_or_name_and_type(self):
+ if self.uuid:
+ return self
+ if self.name and self.type:
+ return self
+ raise ValueError(
+ "Anchor must include either uuid, or both name and type."
+ )
+
class IngestionStructuredRequestBody(BaseModel):
"""
@@ -120,18 +154,27 @@ class IngestionStructuredRequestBody(BaseModel):
...,
description="The list of event-centric information triples to ingest.",
)
- anchor: Optional[Union[str, PartialNodeFilter]] = Field(
+ anchor: Optional[PartialNodeFilter] = Field(
None,
description="The related information to connect the structured data to.",
)
text: Optional[str] = Field(
- ...,
+ default=None,
description="Additional text context for the structured data.",
)
brain_id: str = Field(
default="default", description="The brain identifier to store the data in."
)
+ @field_validator("anchor", mode="before")
+ @classmethod
+ def reject_string_anchor(cls, value):
+ if isinstance(value, str):
+ raise ValueError(
+ "Anchor must be an object with uuid or name+type, not a string."
+ )
+ return value
+
class RetrieveRequestResponse(BaseModel):
"""
diff --git a/src/services/api/routes/ingest.py b/src/services/api/routes/ingest.py
index 2f30879..51ed4a8 100644
--- a/src/services/api/routes/ingest.py
+++ b/src/services/api/routes/ingest.py
@@ -31,6 +31,7 @@ from src.workers.tasks.ingestion import ingest_file as ingest_file_task
from src.workers.tasks.ingestion import (
ingest_structured_data as ingest_structured_data_task,
)
+from src.workers.tasks.ingestion import set_ingestion_task_status
MAX_TASK_RETRIES = 3
RETRY_DELAY_BASE = 0.1
@@ -38,78 +39,83 @@ RETRY_DELAY_BASE = 0.1
ingest_router = APIRouter(prefix="/ingest", tags=["ingest"])
-@ingest_router.post(path="/")
+@ingest_router.post(path="/", status_code=202)
async def ingest_data(
data: IngestionRequestBody,
request: Request,
brain_id: str = Depends(get_brain_id),
):
"""
- Ingest data to the processing pipeline and save to the memory.
+ Accept data for asynchronous ingestion.
"""
data.brain_id = brain_id
print("[Ingest] received task for brain: ", brain_id)
- flow_task_identifier = None
+ flow_task_identifier = request.headers.get("Task-Identifier")
+ task_id = flow_task_identifier or str(uuid4())
- if request.headers.get("Task-Identifier"):
- flow_task_identifier = request.headers.get("Task-Identifier")
+ set_ingestion_task_status(task_id, data.brain_id, "queued", stage="queued")
- task = None
for attempt in range(MAX_TASK_RETRIES):
try:
- kwargs = {"args": [data.model_dump()]}
- if flow_task_identifier:
- kwargs["task_id"] = flow_task_identifier
-
- task = ingest_data_task.apply_async(**kwargs)
+ ingest_data_task.apply_async(
+ args=[data.model_dump()],
+ task_id=task_id,
+ )
break
except OperationalError:
if attempt == MAX_TASK_RETRIES - 1:
+ set_ingestion_task_status(
+ task_id,
+ data.brain_id,
+ "failed",
+ stage="queue",
+ error="Task queue unavailable",
+ )
raise HTTPException(status_code=503, detail="Task queue unavailable")
await asyncio.sleep(RETRY_DELAY_BASE * (attempt + 1))
- cache_adapter.set(
- key=f"task:{task.id}",
- value=json.dumps({"status": "queued", "task_id": task.id}),
- brain_id=data.brain_id,
- expires_in=3600 * 24 * 7,
- )
-
return JSONResponse(
- content={"message": "Data ingested successfully", "task_id": task.id}
+ status_code=202,
+ content={"message": "Ingestion accepted", "task_id": task_id},
)
-@ingest_router.post(path="/structured")
+@ingest_router.post(path="/structured", status_code=202)
async def ingest_structured_data(
data: IngestionStructuredRequestBody,
brain_id: str = Depends(get_brain_id),
):
"""
- Ingest structured data to the processing pipeline and save to the memory.
+ Accept structured data for asynchronous ingestion.
"""
data.brain_id = brain_id
print("[IngestStructured] received task for brain: ", brain_id)
- task = None
+ task_id = str(uuid4())
+ set_ingestion_task_status(task_id, data.brain_id, "queued", stage="queued")
+
for attempt in range(MAX_TASK_RETRIES):
try:
- task = ingest_structured_data_task.delay(data.model_dump())
+ ingest_structured_data_task.apply_async(
+ args=[data.model_dump()],
+ task_id=task_id,
+ )
break
except OperationalError:
if attempt == MAX_TASK_RETRIES - 1:
+ set_ingestion_task_status(
+ task_id,
+ data.brain_id,
+ "failed",
+ stage="queue",
+ error="Task queue unavailable",
+ )
raise HTTPException(status_code=503, detail="Task queue unavailable")
await asyncio.sleep(RETRY_DELAY_BASE * (attempt + 1))
- cache_adapter.set(
- key=f"task:{task.id}",
- value=json.dumps({"status": "queued", "task_id": task.id}),
- brain_id=data.brain_id,
- expires_in=3600 * 24 * 7,
- )
-
return JSONResponse(
- content={"message": "Structured data ingested successfully", "task_id": task.id}
+ status_code=202,
+ content={"message": "Structured ingestion accepted", "task_id": task_id},
)
@@ -132,6 +138,7 @@ async def ingest_file(
if config.ocr_mode == "docling":
content_b64 = base64.b64encode(file.file.read()).decode("ascii")
filename = file.filename or "file"
+ set_ingestion_task_status(task, brain_id, "queued", stage="queued")
for attempt in range(MAX_TASK_RETRIES):
try:
ingest_file_task.apply_async(
@@ -145,14 +152,8 @@ async def ingest_file(
status_code=503, detail="Task queue unavailable"
)
await asyncio.sleep(RETRY_DELAY_BASE * (attempt + 1))
- cache_adapter.set(
- key=f"task:{task}",
- value=json.dumps({"status": "queued", "task_id": task}),
- brain_id=brain_id,
- expires_in=3600 * 24 * 7,
- )
response_content = {
- "message": "File ingested successfully",
+ "message": "File ingestion accepted",
"task_id": task,
}
else:
@@ -166,12 +167,7 @@ async def ingest_file(
)
app_host = f"{forwarded_proto}://{forwarded_host}".rstrip("/")
print(f"app_host: {app_host}")
- cache_adapter.set(
- key=f"task:{task}",
- value=json.dumps({"status": "queued", "task_id": task}),
- brain_id=brain_id,
- expires_in=3600 * 24 * 7,
- )
+ set_ingestion_task_status(task, brain_id, "queued", stage="queued")
response = requests.post(
f"{config.docparser_endpoint}/ingest",
files={"file": (file.filename or "file", file.file, file.content_type)},
@@ -185,7 +181,7 @@ async def ingest_file(
if response.status_code != 200:
raise HTTPException(status_code=500, detail=response.text)
response_content = {
- "message": "File ingested successfully",
+ "message": "File ingestion accepted",
"task_id": task,
}
except HTTPException:
@@ -193,4 +189,4 @@ async def ingest_file(
except Exception as e:
raise HTTPException(status_code=500, detail=str(e))
- return JSONResponse(content=response_content)
+ return JSONResponse(status_code=202, content=response_content)
diff --git a/src/services/api/routes/tasks.py b/src/services/api/routes/tasks.py
index 7745b6f..536d2ab 100644
--- a/src/services/api/routes/tasks.py
+++ b/src/services/api/routes/tasks.py
@@ -10,7 +10,7 @@ Modified By: the developer formerly known as Christian Nonis at <alch.infoemail@
import json
from src.utils.logging import log
-from fastapi import APIRouter, Depends
+from fastapi import APIRouter, Depends, HTTPException
from src.services.api.dependencies import get_brain_id
from src.services.kg_agent.main import cache_adapter
@@ -52,11 +52,7 @@ async def get_task(task_id: str, brain_id: str = Depends(get_brain_id)):
try:
str_result = cache_adapter.get_task(task_id, brain_id=brain_id)
if str_result is None:
- return {
- "task_id": task_id,
- "status": "pending",
- "result": {"message": "Task is still processing or not found"},
- }
+ raise HTTPException(status_code=404, detail="Task not found")
if isinstance(str_result, bytes):
result = json.loads(str_result.decode("utf-8"))
else:
@@ -66,6 +62,8 @@ async def get_task(task_id: str, brain_id: str = Depends(get_brain_id)):
"task_id": task_id,
"status": result.get("status", "unknown"),
}
+ except HTTPException:
+ raise
except Exception as e:
log(f"Error in get_task: {type(e).__name__}: {str(e)}")
return {
diff --git a/src/workers/app.py b/src/workers/app.py
index 2ed4497..e4d33b4 100644
--- a/src/workers/app.py
+++ b/src/workers/app.py
@@ -67,6 +67,7 @@ TASK_QUEUES = (
),
Queue("ingest_structured_data", routing_key="ingest_structured_data"),
Queue("consolidate_graph", routing_key="consolidate_graph"),
+ Queue("finalize_ingestion", routing_key="finalize_ingestion"),
Queue("ingest_file", routing_key="ingest_file"),
) + _PLUGIN_QUEUES
@@ -81,6 +82,9 @@ TASK_ROUTES = {
"src.workers.tasks.ingestion.consolidate_graph_async": {
"queue": "consolidate_graph"
},
+ "src.workers.tasks.ingestion.finalize_ingestion_task": {
+ "queue": "finalize_ingestion"
+ },
"src.workers.tasks.ingestion.ingest_file": {"queue": "ingest_file"},
**_PLUGIN_ROUTES,
}
diff --git a/src/workers/tasks/ingestion.py b/src/workers/tasks/ingestion.py
index a357e92..aad19ae 100644
--- a/src/workers/tasks/ingestion.py
+++ b/src/workers/tasks/ingestion.py
@@ -26,6 +26,11 @@ from pathlib import Path
from typing import List, Optional, Tuple, TypedDict
from uuid import uuid4
+from src.core.saving.identity import (
+ stable_node_id,
+ stable_relationship_id,
+)
+
from pydantic import BaseModel
from src.config import config
@@ -77,17 +82,104 @@ with open(PYPROJECT_PATH, "rb") as f:
NODE_RESOLUTION_SIMILARITY = 0.9
EVENT_RESOLUTION_NAME_SIMILARITY = 0.7
+RELATIONSHIP_DEDUP_MAX_DISTANCE = 0.1
+
+TASK_STATUS_RANK = {
+ "queued": 0,
+ "started": 1,
+ "persisting": 2,
+ "consolidating": 3,
+ "completed": 4,
+ "partial_failed": 4,
+ "failed": 4,
+}
+TERMINAL_TASK_STATUSES = {"completed", "failed", "partial_failed"}
+
+
+def set_ingestion_task_status(
+ task_id: str,
+ brain_id: str,
+ status: str,
+ *,
+ stage: Optional[str] = None,
+ error: Optional[str] = None,
+ errors: Optional[list] = None,
+ counts: Optional[dict] = None,
+) -> dict:
+ key = f"task:{task_id}"
+ existing_raw = cache_adapter.get(key, brain_id=brain_id)
+ existing: dict = {}
+ if existing_raw:
+ try:
+ if isinstance(existing_raw, bytes):
+ existing_raw = existing_raw.decode("utf-8")
+ existing = json.loads(existing_raw) if isinstance(existing_raw, str) else dict(existing_raw)
+ except Exception:
+ existing = {}
+ previous = existing.get("status")
+ if previous in TERMINAL_TASK_STATUSES and status not in TERMINAL_TASK_STATUSES:
+ return existing
+ if previous and TASK_STATUS_RANK.get(status, -1) < TASK_STATUS_RANK.get(previous, -1):
+ return existing
+ payload = {
+ **existing,
+ "status": status,
+ "task_id": task_id,
+ }
+ if stage is not None:
+ payload["stage"] = stage
+ if counts is not None:
+ payload["counts"] = counts
+ if error is not None:
+ payload["error"] = str(error)[:2000]
+ if errors is not None:
+ payload["errors"] = [
+ {
+ "message": str(item.get("message", item))[:500],
+ **(
+ {"relationship": item.get("relationship")}
+ if isinstance(item, dict) and item.get("relationship")
+ else {}
+ ),
+ }
+ for item in errors[:50]
+ ]
+ cache_adapter.set(
+ key=key,
+ value=json.dumps(payload),
+ brain_id=brain_id,
+ expires_in=3600 * 24 * 7,
+ )
+ return payload
+
+
+def nearest_existing_vector(candidates, exclude_id) -> Optional[object]:
+ for candidate in candidates or []:
+ if exclude_id is not None and str(candidate.id) == str(exclude_id):
+ continue
+ return candidate
+ return None
-def _entity_key(entity) -> Tuple[str, str]:
+def should_dedup_relationship(candidate, max_distance: float = RELATIONSHIP_DEDUP_MAX_DISTANCE) -> bool:
return (
+ candidate is not None
+ and candidate.distance is not None
+ and float(candidate.distance) < max_distance
+ )
+
+
+def _entity_key(entity) -> Tuple[str, str, str]:
+ entity_uuid = (getattr(entity, "uuid", None) or "").strip()
+ return (
+ entity_uuid,
(entity.name or "").strip().lower(),
(entity.type or "").strip().lower(),
)
def _is_same_graph_node(graph_node: Node, entity) -> bool:
- if graph_node.uuid and graph_node.uuid == entity.uuid:
+ if graph_node.uuid and graph_node.uuid == getattr(entity, "uuid", None):
return True
same_name = (graph_node.name or "").strip().lower() == (
entity.name or ""
@@ -115,6 +207,16 @@ def _resolve_relationship_entities(
for entity in (relationship.tail, relationship.tip):
unique_entities.setdefault(_entity_key(entity), entity)
+ pre_stable_rel_ids = {
+ id(relationship): stable_relationship_id(
+ relationship.tail.uuid,
+ relationship.name,
+ relationship.tip.uuid,
+ relationship.flow_key,
+ )
+ for relationship in relationships
+ }
+
embeddings_cache: dict = {}
def _name_embedding(name: str):
@@ -127,6 +229,15 @@ def _resolve_relationship_entities(
embeddings_cache[key] = None
return embeddings_cache[key]
+ def _uuid_match(entity) -> Optional[Node]:
+ entity_uuid = (getattr(entity, "uuid", None) or "").strip()
+ if not entity_uuid:
+ return None
+ try:
+ return graph_adapter.get_by_uuid(entity_uuid, brain_id=brain_id)
+ except Exception:
+ return None
+
def _exact_match(entity) -> Optional[Node]:
try:
return graph_adapter.get_by_identification_params(
@@ -148,6 +259,7 @@ def _resolve_relationship_entities(
except Exception:
return None
entity_type = (entity.type or "").strip().lower()
+ best = None
for candidate in candidates:
metadata = candidate.metadata or {}
candidate_uuid = metadata.get("uuid")
@@ -164,14 +276,22 @@ def _resolve_relationship_entities(
similarity = cosine_similarity(embedding, candidate_vectors[0].embeddings)
if similarity < NODE_RESOLUTION_SIMILARITY:
continue
+ if best is not None and similarity < best[0]:
+ continue
+ if best is not None and abs(similarity - best[0]) < 0.02:
+ return None
node = graph_adapter.get_by_uuid(candidate_uuid, brain_id=brain_id)
if node:
- return node
- return None
+ best = (similarity, node)
+ return best[1] if best else None
resolutions: dict = {}
for key, entity in unique_entities.items():
- if key[1] == "event":
+ uuid_hit = _uuid_match(entity)
+ if uuid_hit:
+ resolutions[key] = uuid_hit
+ continue
+ if key[2] == "event":
continue
node = _exact_match(entity) or _vector_match(entity)
if node:
@@ -184,12 +304,18 @@ def _resolve_relationship_entities(
if node:
entity.uuid = node.uuid
entity.name = node.name
+ if getattr(node, "happened_at", None) and not entity.happened_at:
+ entity.happened_at = node.happened_at
_apply_resolutions()
resolved_uuids = {node.uuid for node in resolutions.values()}
for key, event in unique_entities.items():
- if key[1] != "event" or key in resolutions:
+ if key[2] != "event" or key in resolutions:
+ continue
+ uuid_hit = _uuid_match(event)
+ if uuid_hit:
+ resolutions[key] = uuid_hit
continue
anchor_uuids = set()
for relationship in relationships:
@@ -245,6 +371,9 @@ def _resolve_relationship_entities(
similarity = cosine_similarity(event_embedding, candidate_embedding)
if similarity < EVENT_RESOLUTION_NAME_SIMILARITY:
continue
+ if best is not None and abs(similarity - best[0]) < 0.02:
+ best = None
+ break
if best is None or similarity > best[0]:
best = (similarity, candidate)
if best:
@@ -252,6 +381,26 @@ def _resolve_relationship_entities(
_apply_resolutions()
+ for relationship in relationships:
+ for entity in (relationship.tail, relationship.tip):
+ if resolutions.get(_entity_key(entity)):
+ continue
+ entity.uuid = stable_node_id(
+ entity.name,
+ entity.type,
+ getattr(entity, "happened_at", None),
+ getattr(entity, "uuid", None),
+ )
+ current_uuid = getattr(relationship, "uuid", None)
+ pre_stable = pre_stable_rel_ids.get(id(relationship))
+ if current_uuid and pre_stable and current_uuid != pre_stable:
+ continue
+ relationship.uuid = stable_relationship_id(
+ relationship.tail.uuid,
+ relationship.name,
+ relationship.tip.uuid,
+ relationship.flow_key,
+ )
def format_textual_data(data: dict, include_keys: bool = True) -> str:
def format_value(v):
@@ -266,6 +415,12 @@ def format_textual_data(data: dict, include_keys: bool = True) -> str:
return "\n".join(format_value(v) for v in data.values())
+def source_text_from_payload(payload: IngestionTaskArgs) -> str:
+ if payload.data.data_type == IngestionTaskDataType.TEXT.value:
+ return payload.data.text_data
+ return json.dumps(payload.data.json_data)
+
+
@ingestion_app.task(bind=True)
def ingest_data(self, args: dict):
"""
@@ -281,12 +436,15 @@ def ingest_data(self, args: dict):
payload = None
try:
payload = IngestionTaskArgs(**args)
+ from src.config import validate_pipeline_mode
- cache_adapter.set(
- key=f"task:{self.request.id}",
- value=json.dumps({"status": "started", "task_id": self.request.id}),
- brain_id=payload.brain_id,
- expires_in=3600 * 24 * 7,
+ validate_pipeline_mode(config.pipeline_mode)
+
+ set_ingestion_task_status(
+ self.request.id,
+ payload.brain_id,
+ "started",
+ stage="source",
)
payload.meta_keys = (
@@ -309,16 +467,11 @@ def ingest_data(self, args: dict):
else None
)
- # ================================================
- # --------------- Data Saving --------------------
- # ================================================
+ source_text = source_text_from_payload(payload)
+
text_chunk = data_adapter.save_text_chunk(
TextChunk(
- text=(
- payload.data.text_data
- if payload.data.data_type == IngestionTaskDataType.TEXT.value
- else json.dumps(payload.data.json_data)
- ),
+ text=source_text,
metadata=payload.meta_keys,
brain_version=BRAIN_VERSION,
),
@@ -340,16 +493,10 @@ def ingest_data(self, args: dict):
print("[DEBUG (ingest_data)]: Lightweight pipeline mode selected")
if config.pipeline_mode == "accurate":
- # ================================================
- # --------------- Observations -------------------
- # ================================================
observations = observations_agent.observe(
- text=(
- payload.data.text_data
- if payload.data.data_type == IngestionTaskDataType.TEXT.value
- else json.dumps(payload.data.json_data)
- ),
+ text=source_text,
observate_for=payload.observate_for,
+ context=None,
)
data_adapter.save_observations(
@@ -360,42 +507,94 @@ def ingest_data(self, args: dict):
resource_id=text_chunk.id,
)
for observation in observations
- ]
+ ],
+ brain_id=payload.brain_id,
)
- # ================================================
- # ------------ Triplet Extraction ----------------
- # ================================================
- enrich_kg_from_input(payload.data.text_data, brain_id=payload.brain_id)
-
- cache_adapter.set(
- key=f"task:{self.request.id}",
- value=json.dumps({"status": "completed", "task_id": self.request.id}),
- brain_id=payload.brain_id,
- expires_in=3600 * 24 * 7,
+ enrich_result = enrich_kg_from_input(source_text, brain_id=payload.brain_id)
+ parent_task_id = self.request.id
+ steps = []
+ if enrich_result.enrichment_relationships:
+ set_ingestion_task_status(
+ parent_task_id,
+ payload.brain_id,
+ "persisting",
+ stage="persisting",
+ counts={
+ "relationships": len(enrich_result.enrichment_relationships),
+ },
+ )
+ steps.append(
+ process_architect_relationships.si(
+ {
+ "relationships": enrich_result.enrichment_relationships,
+ "brain_id": enrich_result.brain_id,
+ "session_id": enrich_result.session_id,
+ "parent_task_id": parent_task_id,
+ }
+ )
+ )
+ if enrich_result.should_consolidate:
+ set_ingestion_task_status(
+ parent_task_id,
+ payload.brain_id,
+ "consolidating",
+ stage="consolidating",
+ )
+ steps.append(
+ consolidate_graph_async.si(
+ enrich_result.session_id,
+ enrich_result.brain_id,
+ enrich_result.ingestion_session_id,
+ enrich_result.enrichment_relationships,
+ )
+ )
+ steps.append(
+ finalize_ingestion_task.si(
+ parent_task_id,
+ payload.brain_id,
+ "completed",
+ )
)
+ from celery import chain
- return self.request.id
+ chain(*steps).apply_async()
+ return parent_task_id
except Exception as e:
brain_id = payload.brain_id if payload else args.get("brain_id", "default")
- error_result = {
- "status": "failed",
- "task_id": self.request.id,
- "error": str(e),
- "payload": payload.model_dump(mode="json") if payload else args,
- }
-
- cache_adapter.set(
- key=f"task:{self.request.id}",
- value=json.dumps(error_result),
- brain_id=brain_id,
- expires_in=3600 * 24 * 7,
+ set_ingestion_task_status(
+ self.request.id,
+ brain_id,
+ "failed",
+ stage="failed",
+ error=str(e),
)
-
raise
+@ingestion_app.task(bind=True)
+def finalize_ingestion_task(
+ self,
+ parent_task_id: str,
+ brain_id: str = "default",
+ status: str = "completed",
+ error: Optional[str] = None,
+ errors: Optional[list] = None,
+ counts: Optional[dict] = None,
+):
+ set_ingestion_task_status(
+ parent_task_id,
+ brain_id,
+ status,
+ stage=status,
+ error=error,
+ errors=errors,
+ counts=counts,
+ )
+ return parent_task_id
+
+
@ingestion_app.task(bind=True)
def process_architect_relationships(self, args: dict):
"""
@@ -425,19 +624,16 @@ def process_architect_relationships(self, args: dict):
relationships_data: List[dict] = args.get("relationships", [])
brain_id: str = args.get("brain_id", "default")
session_id: Optional[str] = args.get("session_id")
+ parent_task_id: Optional[str] = args.get("parent_task_id")
+ status_task_id = parent_task_id or self.request.id
try:
- cache_adapter.set(
- key=f"task:{self.request.id}",
- value=json.dumps(
- {
- "status": "started",
- "task_id": self.request.id,
- "total_relationships": len(relationships_data),
- }
- ),
- brain_id=brain_id,
- expires_in=3600 * 24 * 7,
+ set_ingestion_task_status(
+ status_task_id,
+ brain_id,
+ "persisting",
+ stage="persisting",
+ counts={"relationships": len(relationships_data)},
)
ingestion_manager = IngestionManager(
@@ -450,6 +646,7 @@ def process_architect_relationships(self, args: dict):
_normalize_relationship_dates(relationships)
_resolve_relationship_entities(relationships, brain_id)
+ item_errors: List[dict] = []
with ThreadPoolExecutor(max_workers=10) as io_executor:
rel_embedding_futures: List[Tuple[Future, ArchitectAgentRelationship]] = []
for relationship in relationships:
@@ -457,6 +654,12 @@ def process_architect_relationships(self, args: dict):
print(
f"[!] Skipping invalid relationship type: {type(relationship)}"
)
+ item_errors.append(
+ {
+ "relationship": str(type(relationship)),
+ "message": "invalid relationship type",
+ }
+ )
continue
if relationship.tail.uuid == relationship.tip.uuid:
print(
@@ -501,9 +704,12 @@ def process_architect_relationships(self, args: dict):
store="relationships",
k=10,
)
- if similar_v_rels and similar_v_rels[0].distance > 0.9:
+ nearest_existing = nearest_existing_vector(
+ similar_v_rels, v_rel_id
+ )
+ if should_dedup_relationship(nearest_existing):
similar_rel = graph_adapter.get_triples_by_uuid(
- [similar_v_rels[0].metadata.get("uuid")],
+ [nearest_existing.metadata.get("uuid")],
brain_id=brain_id,
)
if similar_rel:
@@ -640,23 +846,38 @@ def process_architect_relationships(self, args: dict):
)
except FutureTimeoutError:
rel_name = getattr(relationship, "name", "unknown")
- print(
- f"[!] Relationship embedding future timed out for {rel_name}, skipping"
+ item_errors.append(
+ {
+ "relationship": rel_name,
+ "message": "relationship embedding timed out",
+ }
)
- continue
except Exception as e:
rel_name = getattr(relationship, "name", "unknown")
- print(
- f"[!] Relationship embedding future failed for {rel_name}: {e}"
+ item_errors.append(
+ {"relationship": rel_name, "message": str(e)}
)
- continue
- cache_adapter.set(
- key=f"task:{self.request.id}",
- value=json.dumps({"status": "completed", "task_id": self.request.id}),
- brain_id=brain_id,
- expires_in=3600 * 24 * 7,
- )
+ if item_errors:
+ status = (
+ "partial_failed"
+ if len(item_errors) < len(relationships_data)
+ else "failed"
+ )
+ set_ingestion_task_status(
+ status_task_id,
+ brain_id,
+ status,
+ stage="persisting",
+ errors=item_errors,
+ counts={
+ "relationships": len(relationships_data),
+ "failed": len(item_errors),
+ },
+ )
+ raise RuntimeError(
+ f"{len(item_errors)} relationship persistence failures"
+ )
if session_id:
from src.lib.redis.client import _redis_client
@@ -671,25 +892,28 @@ def process_architect_relationships(self, args: dict):
return self.request.id
except Exception as e:
- error_result = {
- "status": "failed",
- "task_id": self.request.id,
- "error": str(e),
- "args": args,
- }
-
- cache_adapter.set(
- key=f"task:{self.request.id}",
- value=json.dumps(error_result),
- brain_id=brain_id,
- expires_in=3600 * 24 * 7,
- )
-
+ current = cache_adapter.get(f"task:{status_task_id}", brain_id=brain_id)
+ current_status = None
+ if current:
+ try:
+ parsed = json.loads(
+ current.decode("utf-8") if isinstance(current, bytes) else current
+ )
+ current_status = parsed.get("status")
+ except Exception:
+ current_status = None
+ if current_status not in TERMINAL_TASK_STATUSES:
+ set_ingestion_task_status(
+ status_task_id,
+ brain_id,
+ "failed",
+ stage="persisting",
+ error=str(e),
+ )
if session_id:
from src.lib.redis.client import _redis_client
_redis_client.client.decr(f"{brain_id}:session:{session_id}:pending_tasks")
-
raise
@@ -714,17 +938,12 @@ def ingest_structured_data(self, args: dict):
try:
payload = IngestionStructuredRequestBody(**args)
- cache_adapter.set(
- key=f"task:{self.request.id}",
- value=json.dumps(
- {
- "status": "started",
- "task_id": self.request.id,
- "total_elements": len(payload.data),
- }
- ),
- brain_id=payload.brain_id,
- expires_in=3600 * 24 * 7,
+ set_ingestion_task_status(
+ self.request.id,
+ payload.brain_id,
+ "started",
+ stage="source",
+ counts={"triples": len(payload.data)},
)
anchor = None
@@ -732,30 +951,29 @@ def ingest_structured_data(self, args: dict):
embeddings_adapter, vector_store_adapter, graph_adapter
)
- if isinstance(payload.anchor, PartialNodeFilter) and payload.anchor.uuid:
+ if payload.anchor is None:
+ pass
+ elif payload.anchor.uuid:
anchor = graph_adapter.get_by_uuid(
payload.anchor.uuid, brain_id=payload.brain_id
)
if not anchor:
- cache_adapter.set(
- key=f"task:{self.request.id}",
- value=json.dumps(
- {
- "status": "failed",
- "task_id": self.request.id,
- "error": f"Anchor node with uuid {payload.anchor.uuid} not found",
- }
- ),
- brain_id=payload.brain_id,
- expires_in=3600 * 24 * 7,
+ error = f"Anchor node with uuid {payload.anchor.uuid} not found"
+ set_ingestion_task_status(
+ self.request.id,
+ payload.brain_id,
+ "failed",
+ stage="anchor",
+ error=error,
)
- return
-
+ raise ValueError(error)
else:
anchor = graph_adapter.get_by_identification_params(
IdentificationParams(
name=payload.anchor.name, entity_types=[payload.anchor.type]
- )
+ ),
+ brain_id=payload.brain_id,
+ entity_types=[payload.anchor.type] if payload.anchor.type else None,
)
if not anchor:
txt = (
@@ -789,7 +1007,7 @@ def ingest_structured_data(self, args: dict):
)
if kg_agent_anchor_result.exists:
anchor = kg_agent_anchor_result.node
- elif payload.anchor:
+ else:
anchor_entity = ScoutEntity(
name=payload.anchor.name,
type=payload.anchor.type,
@@ -812,15 +1030,44 @@ def ingest_structured_data(self, args: dict):
)
anchor = added_nodes[0]
- if payload.text:
- current_triples: List[IngestionTripleSet] = []
- partial_triples: List[IngestionTripleSet] = []
- for triple in payload.data:
- if triple.subject and triple.subj_event:
- current_triples.append(triple)
- else:
- partial_triples.append(triple)
+ data_adapter.save_structured_data(
+ StructuredData(
+ id=self.request.id,
+ data={
+ "triples": [t.model_dump(mode="json") for t in payload.data],
+ "text": payload.text,
+ "anchor": (
+ payload.anchor.model_dump(mode="json")
+ if payload.anchor
+ else None
+ ),
+ },
+ types=["ingestion_structured"],
+ metadata={"task_id": self.request.id},
+ brain_version=BRAIN_VERSION,
+ ),
+ brain_id=payload.brain_id,
+ )
+ current_triples: List[IngestionTripleSet] = []
+ partial_triples: List[IngestionTripleSet] = []
+ for triple in payload.data:
+ if triple.subject and triple.subj_event:
+ current_triples.append(triple)
+ else:
+ partial_triples.append(triple)
+
+ from src.core.agents.architect_agent import ingestion_triples_to_relationships
+
+ required_relationships, _ = ingestion_triples_to_relationships(
+ current_triples, partial_triples
+ )
+ persistence_batches = [
+ rel.model_dump(mode="json") for rel in required_relationships
+ ]
+ session_id = None
+
+ if payload.text:
scout_agent = ScoutAgent(
llm_adapter=llm_small_adapter,
cache_adapter=cache_adapter,
@@ -837,7 +1084,7 @@ def ingest_structured_data(self, args: dict):
ingestion_manager=ingestion_manager,
)
scout_agent_response = scout_agent.run_structured(
- text=payload.text if payload.text else None,
+ text=payload.text,
brain_id=payload.brain_id,
timeout=180,
max_retries=3,
@@ -846,7 +1093,7 @@ def ingest_structured_data(self, args: dict):
current_triples=current_triples,
)
architect_agent.run_structured(
- text=payload.text if payload.text else None,
+ text=payload.text,
entities=scout_agent_response.entities,
targeting=anchor,
brain_id=payload.brain_id,
@@ -855,39 +1102,55 @@ def ingest_structured_data(self, args: dict):
ingestion_session_id=self.request.id,
partial_triples=partial_triples,
current_triples=current_triples,
+ persist_submitted=False,
+ )
+ enrichment = architect_agent.take_pending_relationships()
+ session_id = architect_agent.session_id
+ persistence_batches.extend(
+ rel.model_dump(mode="json") for rel in enrichment
)
- # TODO [SIR]: Add the triples to the knowledge graph + save the data + create the embeddings and save them -- save the text if present and find a way to save the rels (maybe isn't necessary?)
-
- # TODO [SIR]: Create the observations ??
- # NOTE [SIR]: structured is by definition ment to save precise information so the pipeline will only be granular/accurate
- # NOTE [SIR]: for the text ingestion the lightweight can be converted into ingestion of only observations instead whole text
-
- cache_adapter.set(
- key=f"task:{self.request.id}",
- value=json.dumps({"status": "completed", "task_id": self.request.id}),
- brain_id=payload.brain_id,
- expires_in=3600 * 24 * 7,
+ parent_task_id = self.request.id
+ steps = []
+ if persistence_batches:
+ set_ingestion_task_status(
+ parent_task_id,
+ payload.brain_id,
+ "persisting",
+ stage="persisting",
+ counts={"relationships": len(persistence_batches)},
+ )
+ steps.append(
+ process_architect_relationships.si(
+ {
+ "relationships": persistence_batches,
+ "brain_id": payload.brain_id,
+ "session_id": session_id,
+ "parent_task_id": parent_task_id,
+ }
+ )
+ )
+ steps.append(
+ finalize_ingestion_task.si(
+ parent_task_id,
+ payload.brain_id,
+ "completed",
+ )
)
+ from celery import chain
- return self.request.id
+ chain(*steps).apply_async()
+ return parent_task_id
except Exception as e:
brain_id = payload.brain_id if payload else args.get("brain_id", "default")
- error_result = {
- "status": "failed",
- "task_id": self.request.id,
- "error": str(e),
- "payload": payload.model_dump(mode="json") if payload else args,
- }
-
- cache_adapter.set(
- key=f"task:{self.request.id}",
- value=json.dumps(error_result),
- brain_id=brain_id,
- expires_in=3600 * 24 * 7,
+ set_ingestion_task_status(
+ self.request.id,
+ brain_id,
+ "failed",
+ stage="failed",
+ error=str(e),
)
-
raise
@@ -897,6 +1160,7 @@ def consolidate_graph_async(
session_id: str,
brain_id: str = "default",
ingestion_session_id: str = None,
+ relationships: Optional[List[dict]] = None,
):
"""
Consolidate graph after all processing tasks complete.
@@ -921,22 +1185,25 @@ def consolidate_graph_async(
)
return
- relationships_data_str = _redis_client.get(
- f"session:{session_id}:relationships", brain_id=brain_id
- )
- if not relationships_data_str:
+ relationships_data = relationships
+ if not relationships_data:
+ relationships_data_str = _redis_client.get(
+ f"session:{session_id}:relationships", brain_id=brain_id
+ )
+ if relationships_data_str:
+ relationships_data = json.loads(relationships_data_str)
+ if not relationships_data:
print(
f"[DEBUG (consolidate_graph_async)]: No relationships data found for session {session_id}"
)
return
- relationships_data = json.loads(relationships_data_str)
- relationships = [
+ relationship_models = [
ArchitectAgentRelationship(**rel_data) for rel_data in relationships_data
]
print(
- f"[DEBUG (consolidate_graph_async)]: Consolidating graph with {len(relationships)} relationships"
+ f"[DEBUG (consolidate_graph_async)]: Consolidating graph with {len(relationship_models)} relationships"
)
project_name = os.getenv("LANGSMITH_PROJECT", "brainapi")
@@ -951,7 +1218,7 @@ def consolidate_graph_async(
tags=["consolidate_graph", "janitor", "kg_agent"],
metadata=tracing_metadata,
):
- consolidate_graph(relationships, brain_id=brain_id)
+ consolidate_graph(relationship_models, brain_id=brain_id)
try:
from langchain_core.tracers.langchain import wait_for_all_tracers
@@ -963,11 +1230,7 @@ def consolidate_graph_async(
)
_redis_client.delete(f"session:{session_id}:relationships", brain_id=brain_id)
_redis_client.client.delete(f"{brain_id}:session:{session_id}:pending_tasks")
- return (
- consolidation_response.model_dump(mode="json")
- if hasattr(consolidation_response, "model_dump")
- else None
- )
+ return None
except Exception as e:
print(
f"[DEBUG (consolidate_graph_async)]: Consolidation failed for session {session_id}: {e}"
diff --git a/tests/test_graph_upsert_contract.py b/tests/test_graph_upsert_contract.py
new file mode 100644
index 0000000..9999516
--- /dev/null
+++ b/tests/test_graph_upsert_contract.py
@@ -0,0 +1,187 @@
+import os
+import unittest
+from unittest.mock import MagicMock, patch
+
+
+ENV_DEFAULTS = {
+ "BRAINPAT_TOKEN": "test-token",
+ "MODELS_MODE": "local",
+ "EMBEDDINGS_LOCAL_MODEL": "local-model",
+ "EMBEDDINGS_SMALL_MODEL": "small-model",
+ "EMBEDDING_NODES_DIMENSION": "3",
+ "EMBEDDING_TRIPLETS_DIMENSION": "3",
+ "EMBEDDING_OBSERVATIONS_DIMENSION": "3",
+ "EMBEDDING_DATA_DIMENSION": "3",
+ "EMBEDDING_RELATIONSHIPS_DIMENSION": "3",
+ "REDIS_HOST": "localhost",
+ "REDIS_PORT": "6379",
+ "NEO4J_HOST": "localhost",
+ "NEO4J_PORT": "7687",
+ "NEO4J_USERNAME": "neo4j",
+ "NEO4J_PASSWORD": "password",
+ "MILVUS_HOST": "localhost",
+ "MILVUS_PORT": "19530",
+ "MONGO_CONNECTION_STRING": "mongodb://localhost:27017",
+ "CELERY_WORKER_CONCURRENCY": "1",
+ "OLLAMA_HOST": "localhost",
+ "OLLAMA_PORT": "11434",
+ "OLLAMA_LLM_SMALL_MODEL": "small",
+ "OLLAMA_LLM_LARGE_MODEL": "large",
+ "PIPELINE_MODE": "accurate",
+}
+for key, value in ENV_DEFAULTS.items():
+ os.environ.setdefault(key, value)
+
+
+class NetworkXUpsertContractTests(unittest.TestCase):
+ def test_same_uuid_upsert_keeps_one_node_and_preserves_uuid(self):
+ from src.lib.postgresql.graph_store import _BrainGraph
+
+ brain = _BrainGraph("brain-a")
+ first = brain.upsert_node(
+ ["PERSON"],
+ {"uuid": "alice-1", "name": "Alice"},
+ {"uuid": "alice-1", "name": "Alice", "description": "v1"},
+ )
+ second = brain.upsert_node(
+ ["PERSON"],
+ {"uuid": "alice-1", "name": "Alice"},
+ {"uuid": "alice-1", "name": "Alice", "description": "v2"},
+ )
+ self.assertEqual(first, "alice-1")
+ self.assertEqual(second, "alice-1")
+ self.assertEqual(len(list(brain.graph.nodes)), 1)
+ self.assertEqual(brain.node_data("alice-1")["uuid"], "alice-1")
+ self.assertEqual(brain.node_data("alice-1")["description"], "v2")
+
+ def test_same_name_different_uuid_does_not_overwrite_existing_uuid(self):
+ from src.lib.postgresql.graph_store import _BrainGraph
+
+ brain = _BrainGraph("brain-a")
+ brain.upsert_node(
+ ["PERSON"],
+ {"uuid": "alice-1", "name": "Alice"},
+ {"uuid": "alice-1", "name": "Alice"},
+ )
+ brain.upsert_node(
+ ["PERSON"],
+ {"uuid": "alice-2", "name": "Alice"},
+ {"uuid": "alice-2", "name": "Alice"},
+ )
+ self.assertEqual(len(list(brain.graph.nodes)), 2)
+ self.assertEqual(brain.node_data("alice-1")["uuid"], "alice-1")
+ self.assertEqual(brain.node_data("alice-2")["uuid"], "alice-2")
+
+ def test_relationship_upsert_by_uuid_is_idempotent(self):
+ from src.lib.postgresql.graph_store import PostgreSQLGraphStore
+
+ store = PostgreSQLGraphStore.__new__(PostgreSQLGraphStore)
+ from src.lib.postgresql.graph_store import _BrainGraph
+
+ brain = _BrainGraph("brain-a")
+ brain.upsert_node(["PERSON"], {"uuid": "a"}, {"uuid": "a", "name": "Alice"})
+ brain.upsert_node(["PRODUCT"], {"uuid": "b"}, {"uuid": "b", "name": "Widget"})
+ store._brains = {"brain-a": brain}
+ store._persist_relationship = MagicMock()
+
+ first = store.merge_relationship(
+ "brain-a",
+ ["PERSON"],
+ "Alice",
+ ["PRODUCT"],
+ "Widget",
+ "OWNS",
+ {"uuid": "rel-1", "description": "v1"},
+ subject_uuid="a",
+ object_uuid="b",
+ )
+ second = store.merge_relationship(
+ "brain-a",
+ ["PERSON"],
+ "Alice",
+ ["PRODUCT"],
+ "Widget",
+ "OWNS",
+ {"uuid": "rel-1", "description": "v2"},
+ subject_uuid="a",
+ object_uuid="b",
+ )
+ self.assertIsNotNone(first)
+ self.assertIsNotNone(second)
+ edges = list(brain.graph.edges(keys=True, data=True))
+ self.assertEqual(len(edges), 1)
+ self.assertEqual(edges[0][2], "rel-1")
+ self.assertEqual(edges[0][3]["description"], "v2")
+
+
+class Neo4jUpsertContractTests(unittest.TestCase):
+ def test_add_nodes_merges_by_uuid_not_name(self):
+ from src.constants.kg import Node
+ from src.lib.neo4j.client import Neo4jClient
+
+ client = Neo4jClient.__new__(Neo4jClient)
+ client.ensure_database = MagicMock()
+ queries = []
+
+ def _capture(query, brain_id):
+ queries.append(query)
+ return "ok"
+
+ client._execute_query_with_retry = _capture
+ client._clean_labels = lambda labels: [str(l).upper() for l in labels]
+ client._format_property_key = lambda key: key
+ client._format_value = lambda value: (
+ "null"
+ if value is None
+ else f"'{value}'"
+ if isinstance(value, str)
+ else str(value)
+ )
+ client._clean_property_key = lambda key: key
+
+ node = Node(
+ uuid="alice-1",
+ name="Alice",
+ labels=["PERSON"],
+ description="v1",
+ properties={},
+ )
+ client.add_nodes([node], "brain-a")
+ self.assertEqual(len(queries), 1)
+ self.assertIn("MERGE (n:PERSON {uuid: 'alice-1'})", queries[0])
+ self.assertNotIn("{name:", queries[0])
+
+ def test_add_relationship_merges_by_endpoint_and_relationship_uuid(self):
+ from src.constants.kg import Node, Predicate
+ from src.lib.neo4j.client import Neo4jClient
+
+ client = Neo4jClient.__new__(Neo4jClient)
+ client.ensure_database = MagicMock()
+ client.driver = MagicMock()
+ client._clean_labels = lambda labels: [str(l).upper() for l in labels]
+ client._format_value = lambda value: (
+ "null"
+ if value is None
+ else f"'{value}'"
+ if isinstance(value, str)
+ else str(value)
+ )
+
+ subject = Node(uuid="a", name="Alice", labels=["PERSON"], properties={})
+ obj = Node(uuid="b", name="Widget", labels=["PRODUCT"], properties={})
+ predicate = Predicate(
+ uuid="rel-1",
+ name="OWNS",
+ description="owns",
+ properties={"v_id": "v1"},
+ flow_key="flow-1",
+ )
+ client.add_relationship(subject, predicate, obj, "brain-a")
+ query = client.driver.execute_query.call_args.args[0]
+ self.assertIn("WHERE a['uuid'] = 'a'", query)
+ self.assertIn("WHERE b['uuid'] = 'b'", query)
+ self.assertIn("MERGE (a)-[r:OWNS {uuid: 'rel-1'}]->(b)", query)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/tests/test_ingestion_identity.py b/tests/test_ingestion_identity.py
new file mode 100644
index 0000000..724dcaf
--- /dev/null
+++ b/tests/test_ingestion_identity.py
@@ -0,0 +1,228 @@
+import os
+import unittest
+from unittest.mock import MagicMock, patch
+
+
+ENV_DEFAULTS = {
+ "BRAINPAT_TOKEN": "test-token",
+ "MODELS_MODE": "local",
+ "EMBEDDINGS_LOCAL_MODEL": "local-model",
+ "EMBEDDINGS_SMALL_MODEL": "small-model",
+ "EMBEDDING_NODES_DIMENSION": "3",
+ "EMBEDDING_TRIPLETS_DIMENSION": "3",
+ "EMBEDDING_OBSERVATIONS_DIMENSION": "3",
+ "EMBEDDING_DATA_DIMENSION": "3",
+ "EMBEDDING_RELATIONSHIPS_DIMENSION": "3",
+ "REDIS_HOST": "localhost",
+ "REDIS_PORT": "6379",
+ "NEO4J_HOST": "localhost",
+ "NEO4J_PORT": "7687",
+ "NEO4J_USERNAME": "neo4j",
+ "NEO4J_PASSWORD": "password",
+ "MILVUS_HOST": "localhost",
+ "MILVUS_PORT": "19530",
+ "MONGO_CONNECTION_STRING": "mongodb://localhost:27017",
+ "CELERY_WORKER_CONCURRENCY": "1",
+ "OLLAMA_HOST": "localhost",
+ "OLLAMA_PORT": "11434",
+ "OLLAMA_LLM_SMALL_MODEL": "small",
+ "OLLAMA_LLM_LARGE_MODEL": "large",
+ "PIPELINE_MODE": "accurate",
+}
+for key, value in ENV_DEFAULTS.items():
+ os.environ.setdefault(key, value)
+
+
+def _triple(happened_at="01/01/2024", subject_uuid=None, event_uuid=None):
+ subject = {"name": "Alice", "type": "PERSON"}
+ if subject_uuid:
+ subject["uuid"] = subject_uuid
+ event = {
+ "name": "Purchase",
+ "type": "EVENT",
+ "happened_at": happened_at,
+ }
+ if event_uuid:
+ event["uuid"] = event_uuid
+ return {
+ "subject": subject,
+ "subj_event": {"name": "MADE"},
+ "event": event,
+ "event_obj": {"name": "TARGETED"},
+ "object": {"name": "Widget", "type": "PRODUCT"},
+ }
+
+
+class StableIdentityHelpersTests(unittest.TestCase):
+ def test_stable_node_id_is_deterministic(self):
+ from src.core.saving.identity import stable_node_id
+
+ first = stable_node_id("Alice", "PERSON")
+ second = stable_node_id("alice", "person")
+ self.assertEqual(first, second)
+ self.assertNotEqual(
+ stable_node_id("Purchase", "EVENT", "01/01/2024"),
+ stable_node_id("Purchase", "EVENT", "02/01/2024"),
+ )
+
+ def test_supplied_uuid_is_authoritative(self):
+ from src.core.saving.identity import stable_node_id
+
+ self.assertEqual(
+ stable_node_id("Alice", "PERSON", supplied_uuid="keep-me"),
+ "keep-me",
+ )
+
+
+class TripleReplayIdentityTests(unittest.TestCase):
+ def test_identical_structured_conversion_replays_same_ids(self):
+ from src.core.agents.architect_agent import ingestion_triples_to_relationships
+ from src.services.api.constants.requests import IngestionTripleSet
+
+ triple = IngestionTripleSet(**_triple())
+ first_rels, first_entities = ingestion_triples_to_relationships([triple], [])
+ second_rels, second_entities = ingestion_triples_to_relationships([triple], [])
+
+ self.assertEqual(
+ [r.uuid for r in first_rels],
+ [r.uuid for r in second_rels],
+ )
+ self.assertEqual(
+ {k: v.uuid for k, v in first_entities.items()},
+ {k: v.uuid for k, v in second_entities.items()},
+ )
+ self.assertEqual(first_rels[0].tip.uuid, first_rels[1].tail.uuid)
+
+ def test_same_named_events_with_different_dates_stay_separate(self):
+ from src.core.agents.architect_agent import ingestion_triples_to_relationships
+ from src.services.api.constants.requests import IngestionTripleSet
+
+ first = IngestionTripleSet(**_triple(happened_at="01/01/2024"))
+ second = IngestionTripleSet(**_triple(happened_at="02/01/2024"))
+ relationships, entities = ingestion_triples_to_relationships(
+ [first, second], []
+ )
+
+ event_uuids = {
+ rel.tip.uuid if rel.name == "MADE" else rel.tail.uuid
+ for rel in relationships
+ if rel.name in ("MADE", "TARGETED")
+ }
+ purchase_events = [
+ entity
+ for key, entity in entities.items()
+ if len(key) >= 2 and key[0] == "purchase" and key[1] == "event"
+ ]
+ self.assertEqual(len(purchase_events), 2)
+ self.assertEqual(len({e.uuid for e in purchase_events}), 2)
+ self.assertEqual(len(event_uuids), 2)
+
+
+class ResolutionIdentityTests(unittest.TestCase):
+ def test_existing_uuid_is_reused_and_not_overwritten(self):
+ from src.constants.agents import ArchitectAgentEntity, ArchitectAgentRelationship
+ from src.constants.kg import Node
+ from src.workers.tasks import ingestion as ingestion_mod
+
+ existing = Node(
+ uuid="existing-alice",
+ name="Alice",
+ labels=["PERSON"],
+ properties={},
+ )
+ alice = ArchitectAgentEntity(
+ uuid="existing-alice",
+ name="Alice",
+ type="PERSON",
+ )
+ widget = ArchitectAgentEntity(
+ uuid="widget-1",
+ name="Widget",
+ type="PRODUCT",
+ )
+ relationships = [
+ ArchitectAgentRelationship(
+ tail=alice,
+ name="OWNS",
+ tip=widget,
+ flow_key="flow-1",
+ uuid="rel-supplied",
+ )
+ ]
+
+ with patch.object(
+ ingestion_mod.graph_adapter,
+ "get_by_uuid",
+ side_effect=lambda uuid, brain_id="default": (
+ existing if uuid == "existing-alice" else None
+ ),
+ ), patch.object(
+ ingestion_mod.graph_adapter,
+ "get_by_identification_params",
+ return_value=None,
+ ), patch.object(
+ ingestion_mod.vector_store_adapter,
+ "search_vectors",
+ return_value=[],
+ ):
+ ingestion_mod._resolve_relationship_entities(relationships, "brain-a")
+
+ self.assertEqual(relationships[0].tail.uuid, "existing-alice")
+ self.assertEqual(relationships[0].uuid, "rel-supplied")
+
+ def test_ambiguous_semantic_matches_do_not_force_merge(self):
+ from src.constants.agents import ArchitectAgentEntity, ArchitectAgentRelationship
+ from src.constants.embeddings import Vector
+ from src.constants.kg import Node
+ from src.workers.tasks import ingestion as ingestion_mod
+
+ alice = ArchitectAgentEntity(uuid="new-alice", name="Alice", type="PERSON")
+ widget = ArchitectAgentEntity(uuid="widget-1", name="Widget", type="PRODUCT")
+ relationships = [
+ ArchitectAgentRelationship(
+ tail=alice,
+ name="OWNS",
+ tip=widget,
+ flow_key="flow-1",
+ )
+ ]
+
+ candidate_a = MagicMock()
+ candidate_a.id = "va"
+ candidate_a.metadata = {"uuid": "a1", "labels": ["PERSON"]}
+ candidate_b = MagicMock()
+ candidate_b.id = "vb"
+ candidate_b.metadata = {"uuid": "a2", "labels": ["PERSON"]}
+
+ with patch.object(
+ ingestion_mod.graph_adapter, "get_by_uuid", return_value=None
+ ), patch.object(
+ ingestion_mod.graph_adapter,
+ "get_by_identification_params",
+ return_value=None,
+ ), patch.object(
+ ingestion_mod.embeddings_adapter,
+ "embed_text",
+ return_value=Vector(id="q", embeddings=[1.0, 0.0, 0.0], metadata={}),
+ ), patch.object(
+ ingestion_mod.vector_store_adapter,
+ "search_vectors",
+ return_value=[candidate_a, candidate_b],
+ ), patch.object(
+ ingestion_mod.vector_store_adapter,
+ "get_by_ids",
+ side_effect=lambda ids, store="nodes", brain_id="default": [
+ Vector(id=ids[0], embeddings=[1.0, 0.0, 0.0], metadata={})
+ ],
+ ), patch.object(
+ ingestion_mod,
+ "cosine_similarity",
+ return_value=0.95,
+ ):
+ ingestion_mod._resolve_relationship_entities(relationships, "brain-a")
+
+ self.assertEqual(relationships[0].tail.uuid, "new-alice")
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/tests/test_ingestion_orchestration_collection.py b/tests/test_ingestion_orchestration_collection.py
new file mode 100644
index 0000000..e3d0089
--- /dev/null
+++ b/tests/test_ingestion_orchestration_collection.py
@@ -0,0 +1,130 @@
+import os
+import unittest
+from unittest.mock import MagicMock, patch
+
+
+ENV_DEFAULTS = {
+ "BRAINPAT_TOKEN": "test-token",
+ "MODELS_MODE": "local",
+ "EMBEDDINGS_LOCAL_MODEL": "local-model",
+ "EMBEDDINGS_SMALL_MODEL": "small-model",
+ "EMBEDDING_NODES_DIMENSION": "3",
+ "EMBEDDING_TRIPLETS_DIMENSION": "3",
+ "EMBEDDING_OBSERVATIONS_DIMENSION": "3",
+ "EMBEDDING_DATA_DIMENSION": "3",
+ "EMBEDDING_RELATIONSHIPS_DIMENSION": "3",
+ "REDIS_HOST": "localhost",
+ "REDIS_PORT": "6379",
+ "NEO4J_HOST": "localhost",
+ "NEO4J_PORT": "7687",
+ "NEO4J_USERNAME": "neo4j",
+ "NEO4J_PASSWORD": "password",
+ "MILVUS_HOST": "localhost",
+ "MILVUS_PORT": "19530",
+ "MONGO_CONNECTION_STRING": "mongodb://localhost:27017",
+ "CELERY_WORKER_CONCURRENCY": "1",
+ "OLLAMA_HOST": "localhost",
+ "OLLAMA_PORT": "11434",
+ "OLLAMA_LLM_SMALL_MODEL": "small",
+ "OLLAMA_LLM_LARGE_MODEL": "large",
+ "PIPELINE_MODE": "accurate",
+}
+for key, value in ENV_DEFAULTS.items():
+ os.environ.setdefault(key, value)
+
+
+class IngestionOrchestrationCollectionTests(unittest.TestCase):
+ def test_architect_queues_relationships_instead_of_celery_delay(self):
+ from src.constants.agents import ArchitectAgentEntity, ArchitectAgentRelationship
+ from src.core.agents.architect_agent import ArchitectAgent
+
+ agent = ArchitectAgent.__new__(ArchitectAgent)
+ agent.relationships_set = []
+ agent.pending_persistence_batches = []
+ agent.session_id = "session-1"
+
+ rel = ArchitectAgentRelationship(
+ uuid="rel-1",
+ flow_key="flow-1",
+ name="OWNS",
+ tail=ArchitectAgentEntity(uuid="a", name="Alice", type="PERSON"),
+ tip=ArchitectAgentEntity(uuid="b", name="Widget", type="PRODUCT"),
+ )
+ with patch(
+ "src.workers.tasks.ingestion.process_architect_relationships"
+ ) as mocked_task:
+ agent.queue_relationships_for_persistence([rel])
+ mocked_task.delay.assert_not_called()
+
+ pending = agent.take_pending_relationships()
+ self.assertEqual(len(pending), 1)
+ self.assertEqual(pending[0].uuid, "rel-1")
+ self.assertEqual(agent.pending_persistence_batches, [])
+
+ def test_enrich_kg_returns_complete_persistence_batch(self):
+ from src.constants.agents import ArchitectAgentEntity, ArchitectAgentRelationship
+ from src.core.saving.auto_kg import EnrichmentOrchestrationResult
+ from src.core.saving import auto_kg
+
+ rel = ArchitectAgentRelationship(
+ uuid="rel-1",
+ flow_key="flow-1",
+ name="OWNS",
+ description="owns",
+ tail=ArchitectAgentEntity(uuid="a", name="Alice", type="PERSON"),
+ tip=ArchitectAgentEntity(uuid="b", name="Widget", type="PRODUCT"),
+ )
+
+ scout = MagicMock()
+ scout.run.return_value = MagicMock(entities=[])
+ architect = MagicMock()
+ architect.session_id = "sess-1"
+ architect.take_pending_relationships.return_value = [rel]
+
+ with patch.object(auto_kg, "ScoutAgent", return_value=scout), patch.object(
+ auto_kg, "ArchitectAgent", return_value=architect
+ ), patch.object(auto_kg, "IngestionManager", return_value=MagicMock()), patch.object(
+ auto_kg.config, "pipeline_mode", "accurate"
+ ), patch.object(
+ auto_kg.config, "run_graph_consolidator", True
+ ), patch(
+ "src.workers.tasks.ingestion.consolidate_graph_async"
+ ) as consolidate_task:
+ result = auto_kg.enrich_kg_from_input("Alice owns a widget", brain_id="brain-a")
+ consolidate_task.delay.assert_not_called()
+
+ self.assertIsInstance(result, EnrichmentOrchestrationResult)
+ self.assertEqual(result.session_id, "sess-1")
+ self.assertEqual(result.brain_id, "brain-a")
+ self.assertEqual(len(result.enrichment_relationships), 1)
+ self.assertTrue(result.should_consolidate)
+
+ def test_create_relationship_tool_does_not_publish_celery_tasks(self):
+ from src.constants.agents import ArchitectAgentEntity, ArchitectAgentRelationship
+ from src.core.agents.architect_agent import ArchitectAgent
+
+ agent = ArchitectAgent.__new__(ArchitectAgent)
+ agent.relationships_set = []
+ agent.pending_persistence_batches = []
+ agent.session_id = "sess-1"
+
+ output_rels = [
+ ArchitectAgentRelationship(
+ uuid="rel-1",
+ flow_key="flow-1",
+ name="OWNS",
+ tail=ArchitectAgentEntity(uuid="a", name="Alice", type="PERSON"),
+ tip=ArchitectAgentEntity(uuid="b", name="Widget", type="PRODUCT"),
+ )
+ ]
+
+ with patch(
+ "src.workers.tasks.ingestion.process_architect_relationships"
+ ) as mocked_task:
+ agent.queue_relationships_for_persistence(output_rels)
+ mocked_task.delay.assert_not_called()
+
+ self.assertEqual(len(agent.take_pending_relationships()), 1)
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/tests/test_ingestion_standard.py b/tests/test_ingestion_standard.py
new file mode 100644
index 0000000..47a483c
--- /dev/null
+++ b/tests/test_ingestion_standard.py
@@ -0,0 +1,196 @@
+import ast
+import json
+import os
+import unittest
+from pathlib import Path
+from unittest.mock import MagicMock, patch
+
+
+ROOT = Path(__file__).resolve().parent.parent
+
+
+ENV_DEFAULTS = {
+ "BRAINPAT_TOKEN": "test-token",
+ "MODELS_MODE": "local",
+ "EMBEDDINGS_LOCAL_MODEL": "local-model",
+ "EMBEDDINGS_SMALL_MODEL": "small-model",
+ "EMBEDDING_NODES_DIMENSION": "3",
+ "EMBEDDING_TRIPLETS_DIMENSION": "3",
+ "EMBEDDING_OBSERVATIONS_DIMENSION": "3",
+ "EMBEDDING_DATA_DIMENSION": "3",
+ "EMBEDDING_RELATIONSHIPS_DIMENSION": "3",
+ "REDIS_HOST": "localhost",
+ "REDIS_PORT": "6379",
+ "NEO4J_HOST": "localhost",
+ "NEO4J_PORT": "7687",
+ "NEO4J_USERNAME": "neo4j",
+ "NEO4J_PASSWORD": "password",
+ "MILVUS_HOST": "localhost",
+ "MILVUS_PORT": "19530",
+ "MONGO_CONNECTION_STRING": "mongodb://localhost:27017",
+ "CELERY_WORKER_CONCURRENCY": "1",
+ "OLLAMA_HOST": "localhost",
+ "OLLAMA_PORT": "11434",
+ "OLLAMA_LLM_SMALL_MODEL": "small",
+ "OLLAMA_LLM_LARGE_MODEL": "large",
+ "PIPELINE_MODE": "accurate",
+}
+for key, value in ENV_DEFAULTS.items():
+ os.environ.setdefault(key, value)
+
+
+class SourceTextHelperTests(unittest.TestCase):
+ def test_source_text_from_text_payload(self):
+ from src.constants.tasks.ingestion import IngestionTaskArgs
+ from src.workers.tasks.ingestion import source_text_from_payload
+
+ payload = IngestionTaskArgs(
+ data={"data_type": "text", "text_data": "hello world"},
+ brain_id="brain1",
+ )
+ self.assertEqual(source_text_from_payload(payload), "hello world")
+
+ def test_source_text_from_json_payload(self):
+ from src.constants.tasks.ingestion import IngestionTaskArgs
+ from src.workers.tasks.ingestion import source_text_from_payload
+
+ payload = IngestionTaskArgs(
+ data={"data_type": "json", "json_data": {"a": 1, "b": "x"}},
+ brain_id="brain1",
+ )
+ self.assertEqual(
+ source_text_from_payload(payload),
+ json.dumps({"a": 1, "b": "x"}),
+ )
+
+
+class PipelineModeValidationTests(unittest.TestCase):
+ def test_invalid_pipeline_mode_raises(self):
+ from src.config import validate_pipeline_mode
+
+ with self.assertRaises(ValueError):
+ validate_pipeline_mode(None)
+ with self.assertRaises(ValueError):
+ validate_pipeline_mode("turbo")
+ self.assertEqual(validate_pipeline_mode("accurate"), "accurate")
+ self.assertEqual(validate_pipeline_mode("lightweight"), "lightweight")
+
+
+class ConsolidateGraphAsyncReturnTests(unittest.TestCase):
+ def test_consolidate_graph_async_has_no_undefined_consolidation_response(self):
+ source = (ROOT / "src/workers/tasks/ingestion.py").read_text(encoding="utf-8")
+ tree = ast.parse(source)
+ for node in ast.walk(tree):
+ if isinstance(node, ast.Name) and node.id == "consolidation_response":
+ self.fail(
+ "consolidation_response is still referenced in ingestion.py"
+ )
+
+
+class IngestDataObservationAndEnrichmentTests(unittest.TestCase):
+ def _make_request(self):
+ request = MagicMock()
+ request.id = "task-123"
+ return request
+
+ def test_json_ingest_passes_serialized_text_to_enrichment_and_observations(self):
+ from src.constants.data import TextChunk
+ from src.constants.embeddings import Vector
+ from src.workers.tasks import ingestion as ingestion_mod
+
+ saved_observations = []
+ enrich_calls = []
+ observe_calls = []
+
+ def fake_save_text_chunk(chunk, brain_id="default"):
+ return chunk
+
+ def fake_save_observations(observations, brain_id="default"):
+ saved_observations.append({"observations": observations, "brain_id": brain_id})
+ return observations
+
+ def fake_observe(*, text, observate_for, context=None):
+ observe_calls.append(
+ {"text": text, "observate_for": observate_for, "context": context}
+ )
+ return ["obs-1"]
+
+ def fake_enrich(text, targeting=None, brain_id="default"):
+ from src.core.saving.auto_kg import EnrichmentOrchestrationResult
+
+ enrich_calls.append({"text": text, "brain_id": brain_id})
+ return EnrichmentOrchestrationResult(
+ session_id=None,
+ ingestion_session_id="ingest-1",
+ brain_id=brain_id,
+ enrichment_relationships=[],
+ should_consolidate=False,
+ )
+
+ task = ingestion_mod.ingest_data
+ request = self._make_request()
+ args = {
+ "data": {"data_type": "json", "json_data": {"name": "Alice"}},
+ "brain_id": "tenant-a",
+ "observate_for": ["people"],
+ }
+
+ with (
+ patch.object(ingestion_mod, "data_adapter") as data_adapter,
+ patch.object(ingestion_mod, "embeddings_adapter") as embeddings_adapter,
+ patch.object(ingestion_mod, "vector_store_adapter") as vector_store_adapter,
+ patch.object(ingestion_mod, "cache_adapter"),
+ patch.object(ingestion_mod, "observations_agent") as observations_agent,
+ patch.object(ingestion_mod, "enrich_kg_from_input", side_effect=fake_enrich),
+ patch.object(ingestion_mod.config, "pipeline_mode", "accurate"),
+ patch.object(ingestion_mod, "set_ingestion_task_status"),
+ patch("celery.chain") as chain_mock,
+ ):
+ chain_mock.return_value.apply_async = MagicMock()
+ data_adapter.save_text_chunk.side_effect = fake_save_text_chunk
+ data_adapter.save_observations.side_effect = fake_save_observations
+ embeddings_adapter.embed_text.return_value = Vector(
+ id="v1", embeddings=[0.1, 0.2], metadata={}
+ )
+ observations_agent.observe.side_effect = fake_observe
+ vector_store_adapter.add_vectors.return_value = ["v1"]
+
+ result = task.run.__func__(
+ type("Bound", (), {"request": request})(),
+ args,
+ )
+
+ expected_text = json.dumps({"name": "Alice"})
+ self.assertEqual(result, "task-123")
+ self.assertEqual(len(observe_calls), 1)
+ self.assertEqual(observe_calls[0]["text"], expected_text)
+ self.assertIsNone(observe_calls[0]["context"])
+ self.assertEqual(len(saved_observations), 1)
+ self.assertEqual(saved_observations[0]["brain_id"], "tenant-a")
+ self.assertEqual(len(enrich_calls), 1)
+ self.assertEqual(enrich_calls[0]["text"], expected_text)
+ self.assertEqual(enrich_calls[0]["brain_id"], "tenant-a")
+
+ def test_invalid_pipeline_mode_fails_ingest(self):
+ from src.workers.tasks import ingestion as ingestion_mod
+
+ task = ingestion_mod.ingest_data
+ request = self._make_request()
+ args = {
+ "data": {"data_type": "text", "text_data": "hello"},
+ "brain_id": "tenant-a",
+ }
+
+ with (
+ patch.object(ingestion_mod, "cache_adapter"),
+ patch.object(ingestion_mod.config, "pipeline_mode", None),
+ ):
+ with self.assertRaises(ValueError):
+ task.run.__func__(
+ type("Bound", (), {"request": request})(),
+ args,
+ )
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/tests/test_ingestion_structured_contract.py b/tests/test_ingestion_structured_contract.py
new file mode 100644
index 0000000..0c8124a
--- /dev/null
+++ b/tests/test_ingestion_structured_contract.py
@@ -0,0 +1,177 @@
+import os
+import unittest
+from unittest.mock import MagicMock, patch
+
+from pydantic import ValidationError
+
+
+ENV_DEFAULTS = {
+ "BRAINPAT_TOKEN": "test-token",
+ "MODELS_MODE": "local",
+ "EMBEDDINGS_LOCAL_MODEL": "local-model",
+ "EMBEDDINGS_SMALL_MODEL": "small-model",
+ "EMBEDDING_NODES_DIMENSION": "3",
+ "EMBEDDING_TRIPLETS_DIMENSION": "3",
+ "EMBEDDING_OBSERVATIONS_DIMENSION": "3",
+ "EMBEDDING_DATA_DIMENSION": "3",
+ "EMBEDDING_RELATIONSHIPS_DIMENSION": "3",
+ "REDIS_HOST": "localhost",
+ "REDIS_PORT": "6379",
+ "NEO4J_HOST": "localhost",
+ "NEO4J_PORT": "7687",
+ "NEO4J_USERNAME": "neo4j",
+ "NEO4J_PASSWORD": "password",
+ "MILVUS_HOST": "localhost",
+ "MILVUS_PORT": "19530",
+ "MONGO_CONNECTION_STRING": "mongodb://localhost:27017",
+ "CELERY_WORKER_CONCURRENCY": "1",
+ "OLLAMA_HOST": "localhost",
+ "OLLAMA_PORT": "11434",
+ "OLLAMA_LLM_SMALL_MODEL": "small",
+ "OLLAMA_LLM_LARGE_MODEL": "large",
+ "PIPELINE_MODE": "accurate",
+}
+for key, value in ENV_DEFAULTS.items():
+ os.environ.setdefault(key, value)
+
+
+def _minimal_triple():
+ return {
+ "event": {"name": "Purchase", "type": "EVENT"},
+ "event_obj": {"name": "TARGETED"},
+ "object": {"name": "Widget", "type": "PRODUCT"},
+ }
+
+
+class StructuredRequestValidationTests(unittest.TestCase):
+ def test_triple_only_without_anchor_or_text_validates(self):
+ from src.services.api.constants.requests import IngestionStructuredRequestBody
+
+ body = IngestionStructuredRequestBody(data=[_minimal_triple()])
+ self.assertIsNone(body.anchor)
+ self.assertIsNone(body.text)
+
+ def test_string_anchor_is_rejected(self):
+ from src.services.api.constants.requests import IngestionStructuredRequestBody
+
+ with self.assertRaises(ValidationError):
+ IngestionStructuredRequestBody(
+ data=[_minimal_triple()],
+ anchor="John",
+ )
+
+ def test_anchor_requires_uuid_or_name_and_type(self):
+ from src.services.api.constants.requests import IngestionStructuredRequestBody
+
+ with self.assertRaises(ValidationError):
+ IngestionStructuredRequestBody(
+ data=[_minimal_triple()],
+ anchor={"name": "John"},
+ )
+ with self.assertRaises(ValidationError):
+ IngestionStructuredRequestBody(
+ data=[_minimal_triple()],
+ anchor={"type": "PERSON"},
+ )
+
+ body = IngestionStructuredRequestBody(
+ data=[_minimal_triple()],
+ anchor={"uuid": "abc-123"},
+ )
+ self.assertEqual(body.anchor.uuid, "abc-123")
+
+ body = IngestionStructuredRequestBody(
+ data=[_minimal_triple()],
+ anchor={"name": "John", "type": "PERSON"},
+ )
+ self.assertEqual(body.anchor.name, "John")
+
+
+class StructuredAnchorRuntimeTests(unittest.TestCase):
+ def test_missing_uuid_anchor_raises_and_marks_failed(self):
+ from src.workers.tasks import ingestion as ingestion_mod
+
+ request = MagicMock()
+ request.id = "task-anchor-miss"
+ cache_writes = []
+
+ def fake_cache_set(key, value, brain_id="default", expires_in=None):
+ cache_writes.append({"key": key, "value": value, "brain_id": brain_id})
+ return True
+
+ args = {
+ "data": [_minimal_triple()],
+ "anchor": {"uuid": "missing-uuid"},
+ "brain_id": "tenant-a",
+ }
+
+ with (
+ patch.object(ingestion_mod, "cache_adapter") as cache_adapter,
+ patch.object(ingestion_mod, "graph_adapter") as graph_adapter,
+ patch.object(ingestion_mod, "IngestionManager"),
+ ):
+ cache_adapter.set.side_effect = fake_cache_set
+ graph_adapter.get_by_uuid.return_value = None
+
+ with self.assertRaises(ValueError) as ctx:
+ ingestion_mod.ingest_structured_data.run.__func__(
+ type("Bound", (), {"request": request})(),
+ args,
+ )
+
+ self.assertIn("missing-uuid", str(ctx.exception))
+ failed = [w for w in cache_writes if '"status": "failed"' in w["value"]]
+ self.assertTrue(failed)
+ self.assertEqual(failed[-1]["brain_id"], "tenant-a")
+ graph_adapter.get_by_uuid.assert_called_once_with(
+ "missing-uuid", brain_id="tenant-a"
+ )
+
+ def test_name_type_anchor_lookup_passes_brain_id(self):
+ from src.constants.kg import Node
+ from src.workers.tasks import ingestion as ingestion_mod
+
+ request = MagicMock()
+ request.id = "task-anchor-name"
+ captured = {}
+
+ def fake_get_by_identification_params(*args, **kwargs):
+ captured["args"] = args
+ captured["kwargs"] = kwargs
+ return Node(
+ uuid="n1",
+ name="John",
+ labels=["PERSON"],
+ description="",
+ properties={},
+ )
+
+ args = {
+ "data": [_minimal_triple()],
+ "anchor": {"name": "John", "type": "PERSON"},
+ "brain_id": "tenant-b",
+ }
+
+ with (
+ patch.object(ingestion_mod, "cache_adapter"),
+ patch.object(ingestion_mod, "graph_adapter") as graph_adapter,
+ patch.object(ingestion_mod, "IngestionManager"),
+ patch.object(ingestion_mod, "data_adapter"),
+ patch.object(ingestion_mod, "set_ingestion_task_status"),
+ patch("celery.chain") as chain_mock,
+ ):
+ chain_mock.return_value.apply_async = MagicMock()
+ graph_adapter.get_by_identification_params.side_effect = (
+ fake_get_by_identification_params
+ )
+ result = ingestion_mod.ingest_structured_data.run.__func__(
+ type("Bound", (), {"request": request})(),
+ args,
+ )
+
+ self.assertEqual(result, "task-anchor-name")
+ self.assertEqual(captured["kwargs"].get("brain_id"), "tenant-b")
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/tests/test_ingestion_task_lifecycle.py b/tests/test_ingestion_task_lifecycle.py
new file mode 100644
index 0000000..6b12b40
--- /dev/null
+++ b/tests/test_ingestion_task_lifecycle.py
@@ -0,0 +1,222 @@
+import json
+import os
+import unittest
+from unittest.mock import MagicMock, patch
+
+
+ENV_DEFAULTS = {
+ "BRAINPAT_TOKEN": "test-token",
+ "MODELS_MODE": "local",
+ "EMBEDDINGS_LOCAL_MODEL": "local-model",
+ "EMBEDDINGS_SMALL_MODEL": "small-model",
+ "EMBEDDING_NODES_DIMENSION": "3",
+ "EMBEDDING_TRIPLETS_DIMENSION": "3",
+ "EMBEDDING_OBSERVATIONS_DIMENSION": "3",
+ "EMBEDDING_DATA_DIMENSION": "3",
+ "EMBEDDING_RELATIONSHIPS_DIMENSION": "3",
+ "REDIS_HOST": "localhost",
+ "REDIS_PORT": "6379",
+ "NEO4J_HOST": "localhost",
+ "NEO4J_PORT": "7687",
+ "NEO4J_USERNAME": "neo4j",
+ "NEO4J_PASSWORD": "password",
+ "MILVUS_HOST": "localhost",
+ "MILVUS_PORT": "19530",
+ "MONGO_CONNECTION_STRING": "mongodb://localhost:27017",
+ "CELERY_WORKER_CONCURRENCY": "1",
+ "OLLAMA_HOST": "localhost",
+ "OLLAMA_PORT": "11434",
+ "OLLAMA_LLM_SMALL_MODEL": "small",
+ "OLLAMA_LLM_LARGE_MODEL": "large",
+ "PIPELINE_MODE": "accurate",
+}
+for key, value in ENV_DEFAULTS.items():
+ os.environ.setdefault(key, value)
+
+
+class TaskStatusMonotonicTests(unittest.TestCase):
+ def test_queued_cannot_overwrite_started_or_terminal(self):
+ from src.workers.tasks import ingestion as ingestion_mod
+
+ store = {}
+
+ def fake_get(key, brain_id="default"):
+ return store.get((key, brain_id))
+
+ def fake_set(key, value, brain_id="default", expires_in=None):
+ store[(key, brain_id)] = value
+ return True
+
+ with patch.object(ingestion_mod, "cache_adapter") as cache_adapter:
+ cache_adapter.get.side_effect = fake_get
+ cache_adapter.set.side_effect = fake_set
+
+ ingestion_mod.set_ingestion_task_status("t1", "b1", "queued")
+ ingestion_mod.set_ingestion_task_status("t1", "b1", "started", stage="source")
+ ingestion_mod.set_ingestion_task_status("t1", "b1", "queued")
+ started = json.loads(store[("task:t1", "b1")])
+ self.assertEqual(started["status"], "started")
+
+ ingestion_mod.set_ingestion_task_status("t1", "b1", "completed")
+ ingestion_mod.set_ingestion_task_status("t1", "b1", "started")
+ completed = json.loads(store[("task:t1", "b1")])
+ self.assertEqual(completed["status"], "completed")
+
+ def test_finalizer_writes_completed(self):
+ from src.workers.tasks import ingestion as ingestion_mod
+
+ store = {}
+
+ def fake_get(key, brain_id="default"):
+ return store.get((key, brain_id))
+
+ def fake_set(key, value, brain_id="default", expires_in=None):
+ store[(key, brain_id)] = value
+ return True
+
+ with patch.object(ingestion_mod, "cache_adapter") as cache_adapter:
+ cache_adapter.get.side_effect = fake_get
+ cache_adapter.set.side_effect = fake_set
+ ingestion_mod.set_ingestion_task_status("t2", "b1", "persisting")
+ result = ingestion_mod.finalize_ingestion_task.run.__func__(
+ type("Bound", (), {"request": MagicMock(id="child")})(),
+ "t2",
+ "b1",
+ "completed",
+ )
+ self.assertEqual(result, "t2")
+ self.assertEqual(json.loads(store[("task:t2", "b1")])["status"], "completed")
+
+
+class IngestApiLifecycleTests(unittest.TestCase):
+ def test_ingest_records_queued_before_publish_and_returns_202(self):
+ from fastapi.testclient import TestClient
+
+ events = []
+
+ def fake_set_status(task_id, brain_id, status, **kwargs):
+ events.append(("status", status, task_id))
+ return {"status": status, "task_id": task_id}
+
+ def fake_apply_async(*args, **kwargs):
+ events.append(("publish", kwargs.get("task_id")))
+ return MagicMock(id=kwargs.get("task_id"))
+
+ with patch(
+ "src.services.api.routes.ingest.set_ingestion_task_status",
+ side_effect=fake_set_status,
+ ), patch(
+ "src.services.api.routes.ingest.ingest_data_task.apply_async",
+ side_effect=fake_apply_async,
+ ):
+ from src.services.api.routes.ingest import ingest_router
+ from src.services.api.dependencies import get_brain_id
+ from fastapi import FastAPI
+
+ app = FastAPI()
+ app.dependency_overrides[get_brain_id] = lambda: "brain-a"
+ app.include_router(ingest_router)
+ client = TestClient(app)
+ response = client.post(
+ "/ingest/",
+ json={
+ "data": {"data_type": "text", "text_data": "hello"},
+ },
+ )
+
+ self.assertEqual(response.status_code, 202)
+ body = response.json()
+ self.assertEqual(body["message"], "Ingestion accepted")
+ self.assertIn("task_id", body)
+ self.assertEqual(events[0][0], "status")
+ self.assertEqual(events[0][1], "queued")
+ self.assertEqual(events[1][0], "publish")
+ self.assertEqual(events[0][2], events[1][1])
+
+
+class TasksNotFoundTests(unittest.TestCase):
+ def test_unknown_task_returns_not_found(self):
+ from fastapi import FastAPI
+ from fastapi.testclient import TestClient
+
+ with patch(
+ "src.services.api.routes.tasks.cache_adapter"
+ ) as cache_adapter:
+ cache_adapter.get_task.return_value = None
+ from src.services.api.routes.tasks import tasks_router
+ from src.services.api.dependencies import get_brain_id
+ from fastapi import FastAPI
+
+ app = FastAPI()
+ app.dependency_overrides[get_brain_id] = lambda: "brain-a"
+ app.include_router(tasks_router)
+ client = TestClient(app)
+ response = client.get("/tasks/missing-id")
+
+ self.assertEqual(response.status_code, 404)
+
+
+class ParentCompletionChainTests(unittest.TestCase):
+ def test_parent_does_not_mark_completed_before_chain(self):
+ from src.core.saving.auto_kg import EnrichmentOrchestrationResult
+ from src.workers.tasks import ingestion as ingestion_mod
+
+ request = MagicMock()
+ request.id = "parent-1"
+ statuses = []
+
+ def fake_set_status(task_id, brain_id, status, **kwargs):
+ statuses.append(status)
+ return {"status": status, "task_id": task_id}
+
+ chained = []
+
+ class FakeChain:
+ def __init__(self, *steps):
+ chained.extend(steps)
+
+ def apply_async(self):
+ return MagicMock(id="chain-1")
+
+ with patch.object(
+ ingestion_mod, "set_ingestion_task_status", side_effect=fake_set_status
+ ), patch.object(
+ ingestion_mod,
+ "enrich_kg_from_input",
+ return_value=EnrichmentOrchestrationResult(
+ session_id="s1",
+ ingestion_session_id="i1",
+ brain_id="brain-a",
+ enrichment_relationships=[{"uuid": "r1"}],
+ should_consolidate=False,
+ ),
+ ), patch.object(ingestion_mod, "data_adapter") as data_adapter, patch.object(
+ ingestion_mod, "embeddings_adapter"
+ ) as embeddings_adapter, patch.object(
+ ingestion_mod, "vector_store_adapter"
+ ), patch.object(
+ ingestion_mod.config, "pipeline_mode", "lightweight"
+ ), patch(
+ "celery.chain", FakeChain
+ ):
+ from src.constants.embeddings import Vector
+
+ data_adapter.save_text_chunk.side_effect = lambda chunk, brain_id="default": chunk
+ embeddings_adapter.embed_text.return_value = Vector(
+ id="v1", embeddings=[0.1], metadata={}
+ )
+ result = ingestion_mod.ingest_data.run.__func__(
+ type("Bound", (), {"request": request})(),
+ {
+ "data": {"data_type": "text", "text_data": "hello"},
+ "brain_id": "brain-a",
+ },
+ )
+
+ self.assertEqual(result, "parent-1")
+ self.assertNotIn("completed", statuses)
+ self.assertTrue(chained)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/tests/test_structured_triple_persistence.py b/tests/test_structured_triple_persistence.py
new file mode 100644
index 0000000..3dc8bf5
--- /dev/null
+++ b/tests/test_structured_triple_persistence.py
@@ -0,0 +1,192 @@
+import os
+import unittest
+from unittest.mock import MagicMock, patch
+
+
+ENV_DEFAULTS = {
+ "BRAINPAT_TOKEN": "test-token",
+ "MODELS_MODE": "local",
+ "EMBEDDINGS_LOCAL_MODEL": "local-model",
+ "EMBEDDINGS_SMALL_MODEL": "small-model",
+ "EMBEDDING_NODES_DIMENSION": "3",
+ "EMBEDDING_TRIPLETS_DIMENSION": "3",
+ "EMBEDDING_OBSERVATIONS_DIMENSION": "3",
+ "EMBEDDING_DATA_DIMENSION": "3",
+ "EMBEDDING_RELATIONSHIPS_DIMENSION": "3",
+ "REDIS_HOST": "localhost",
+ "REDIS_PORT": "6379",
+ "NEO4J_HOST": "localhost",
+ "NEO4J_PORT": "7687",
+ "NEO4J_USERNAME": "neo4j",
+ "NEO4J_PASSWORD": "password",
+ "MILVUS_HOST": "localhost",
+ "MILVUS_PORT": "19530",
+ "MONGO_CONNECTION_STRING": "mongodb://localhost:27017",
+ "CELERY_WORKER_CONCURRENCY": "1",
+ "OLLAMA_HOST": "localhost",
+ "OLLAMA_PORT": "11434",
+ "OLLAMA_LLM_SMALL_MODEL": "small",
+ "OLLAMA_LLM_LARGE_MODEL": "large",
+ "PIPELINE_MODE": "accurate",
+}
+for key, value in ENV_DEFAULTS.items():
+ os.environ.setdefault(key, value)
+
+
+def _immediate_chain_factory(persist_calls):
+ class ImmediateChain:
+ def __init__(self, *steps):
+ self.steps = steps
+
+ def apply_async(self, *args, **kwargs):
+ for step in self.steps:
+ task_name = getattr(step, "task", None) or getattr(step, "name", "")
+ args_ = getattr(step, "args", ()) or ()
+ if "process_architect_relationships" in str(task_name) and args_:
+ persist_calls.append(args_[0])
+ return MagicMock(id="chain")
+
+ return ImmediateChain
+
+
+
+def _full_triple():
+ return {
+ "subject": {"name": "Alice", "type": "PERSON", "uuid": "sub-1"},
+ "subj_event": {"name": "MADE", "uuid": "rel-1"},
+ "event": {
+ "name": "Purchase",
+ "type": "EVENT",
+ "uuid": "evt-1",
+ "happened_at": "01/01/2024",
+ },
+ "event_obj": {"name": "TARGETED", "uuid": "rel-2", "amount": 3},
+ "object": {"name": "Widget", "type": "PRODUCT", "uuid": "obj-1"},
+ }
+
+
+class TripleConversionTests(unittest.TestCase):
+ def test_full_triple_produces_subject_event_and_event_object_edges(self):
+ from src.core.agents.architect_agent import ingestion_triples_to_relationships
+ from src.services.api.constants.requests import IngestionTripleSet
+
+ triple = IngestionTripleSet(**_full_triple())
+ relationships, entities = ingestion_triples_to_relationships([triple], [])
+
+ self.assertEqual(len(relationships), 2)
+ self.assertEqual(relationships[0].tail.uuid, "sub-1")
+ self.assertEqual(relationships[0].name, "MADE")
+ self.assertEqual(relationships[0].tip.uuid, "evt-1")
+ self.assertEqual(relationships[0].tip.happened_at, "01/01/2024")
+ self.assertEqual(relationships[1].tail.uuid, "evt-1")
+ self.assertEqual(relationships[1].name, "TARGETED")
+ self.assertEqual(relationships[1].tip.uuid, "obj-1")
+ self.assertEqual(relationships[1].amount, 3)
+ self.assertIn(("alice", "person"), entities)
+ self.assertIn(("purchase", "event", "01/01/2024"), entities)
+
+
+class StructuredTriplePersistenceTests(unittest.TestCase):
+ def test_triple_only_persists_without_llm(self):
+ from src.workers.tasks import ingestion as ingestion_mod
+
+ request = MagicMock()
+ request.id = "task-struct-1"
+ persist_calls = []
+ saved_structured = []
+ scout_called = []
+ architect_called = []
+
+ args = {
+ "data": [_full_triple()],
+ "brain_id": "tenant-a",
+ "text": None,
+ }
+
+ with (
+ patch.object(ingestion_mod, "cache_adapter"),
+ patch.object(ingestion_mod, "graph_adapter"),
+ patch.object(ingestion_mod, "IngestionManager"),
+ patch.object(ingestion_mod, "data_adapter") as data_adapter,
+ patch.object(ingestion_mod, "ScoutAgent") as ScoutAgent,
+ patch.object(ingestion_mod, "ArchitectAgent") as ArchitectAgent,
+ patch("celery.chain", _immediate_chain_factory(persist_calls)),
+ patch.object(ingestion_mod, "set_ingestion_task_status"),
+ patch.object(ingestion_mod, "finalize_ingestion_task"),
+ ):
+ data_adapter.save_structured_data.side_effect = (
+ lambda data, brain_id="default": saved_structured.append(
+ {"data": data, "brain_id": brain_id}
+ )
+ or data
+ )
+ ScoutAgent.side_effect = lambda **kwargs: scout_called.append(True)
+ ArchitectAgent.side_effect = lambda **kwargs: architect_called.append(True)
+
+ result = ingestion_mod.ingest_structured_data.run.__func__(
+ type("Bound", (), {"request": request})(),
+ args,
+ )
+
+ self.assertEqual(result, "task-struct-1")
+ self.assertEqual(len(persist_calls), 1)
+ relationships = persist_calls[0]["relationships"]
+ self.assertEqual(len(relationships), 2)
+ self.assertEqual(relationships[0]["name"], "MADE")
+ self.assertEqual(relationships[1]["name"], "TARGETED")
+ self.assertEqual(persist_calls[0]["brain_id"], "tenant-a")
+ self.assertEqual(len(saved_structured), 1)
+ self.assertEqual(saved_structured[0]["brain_id"], "tenant-a")
+ self.assertEqual(saved_structured[0]["data"].id, "task-struct-1")
+ self.assertEqual(scout_called, [])
+ self.assertEqual(architect_called, [])
+
+ def test_text_enrichment_does_not_repersist_submitted_triples(self):
+ from src.core.agents.architect_agent import ArchitectAgentResponse
+ from src.core.agents.scout_agent import ScoutAgentResponse
+ from src.workers.tasks import ingestion as ingestion_mod
+
+ request = MagicMock()
+ request.id = "task-struct-2"
+ persist_calls = []
+ run_structured_kwargs = []
+
+ args = {
+ "data": [_full_triple()],
+ "brain_id": "tenant-a",
+ "text": "Alice bought a widget",
+ }
+
+ scout = MagicMock()
+ scout.run_structured.return_value = ScoutAgentResponse(entities=[])
+ architect = MagicMock()
+ architect.session_id = "sess-enrich"
+ architect.take_pending_relationships.return_value = []
+ architect.run_structured.side_effect = (
+ lambda **kwargs: run_structured_kwargs.append(kwargs)
+ or ArchitectAgentResponse(new_nodes=[], relationships=[])
+ )
+
+ with (
+ patch.object(ingestion_mod, "cache_adapter"),
+ patch.object(ingestion_mod, "graph_adapter"),
+ patch.object(ingestion_mod, "IngestionManager"),
+ patch.object(ingestion_mod, "data_adapter"),
+ patch.object(ingestion_mod, "ScoutAgent", return_value=scout),
+ patch.object(ingestion_mod, "ArchitectAgent", return_value=architect),
+ patch("celery.chain", _immediate_chain_factory(persist_calls)),
+ patch.object(ingestion_mod, "set_ingestion_task_status"),
+ patch.object(ingestion_mod, "finalize_ingestion_task"),
+ ):
+ ingestion_mod.ingest_structured_data.run.__func__(
+ type("Bound", (), {"request": request})(),
+ args,
+ )
+
+ self.assertEqual(len(persist_calls), 1)
+ self.assertEqual(len(run_structured_kwargs), 1)
+ self.assertFalse(run_structured_kwargs[0]["persist_submitted"])
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/tests/test_vector_similarity_contract.py b/tests/test_vector_similarity_contract.py
new file mode 100644
index 0000000..52c60b8
--- /dev/null
+++ b/tests/test_vector_similarity_contract.py
@@ -0,0 +1,104 @@
+import os
+import unittest
+from types import SimpleNamespace
+from unittest.mock import MagicMock
+
+
+ENV_DEFAULTS = {
+ "BRAINPAT_TOKEN": "test-token",
+ "MODELS_MODE": "local",
+ "EMBEDDINGS_LOCAL_MODEL": "local-model",
+ "EMBEDDINGS_SMALL_MODEL": "small-model",
+ "EMBEDDING_NODES_DIMENSION": "3",
+ "EMBEDDING_TRIPLETS_DIMENSION": "3",
+ "EMBEDDING_OBSERVATIONS_DIMENSION": "3",
+ "EMBEDDING_DATA_DIMENSION": "3",
+ "EMBEDDING_RELATIONSHIPS_DIMENSION": "3",
+ "REDIS_HOST": "localhost",
+ "REDIS_PORT": "6379",
+ "NEO4J_HOST": "localhost",
+ "NEO4J_PORT": "7687",
+ "NEO4J_USERNAME": "neo4j",
+ "NEO4J_PASSWORD": "password",
+ "MILVUS_HOST": "localhost",
+ "MILVUS_PORT": "19530",
+ "MONGO_CONNECTION_STRING": "mongodb://localhost:27017",
+ "CELERY_WORKER_CONCURRENCY": "1",
+ "OLLAMA_HOST": "localhost",
+ "OLLAMA_PORT": "11434",
+ "OLLAMA_LLM_SMALL_MODEL": "small",
+ "OLLAMA_LLM_LARGE_MODEL": "large",
+ "PIPELINE_MODE": "accurate",
+}
+for key, value in ENV_DEFAULTS.items():
+ os.environ.setdefault(key, value)
+
+
+class VectorSimilarityContractTests(unittest.TestCase):
+ def test_adapter_sorts_nearest_first_by_cosine_distance(self):
+ from src.adapters.embeddings import VectorStoreAdapter
+ from src.constants.embeddings import Vector
+
+ store = MagicMock()
+ store.search_vectors.return_value = [
+ Vector(id="far", metadata={}, distance=0.8),
+ Vector(id="near", metadata={}, distance=0.05),
+ Vector(id="mid", metadata={}, distance=0.3),
+ ]
+ adapter = VectorStoreAdapter()
+ adapter.add_client(store)
+
+ results = adapter.search_vectors([1.0, 0.0, 0.0], brain_id="b", store="nodes", k=3)
+ self.assertEqual([v.id for v in results], ["near", "mid", "far"])
+
+ def test_milvus_converts_similarity_score_to_cosine_distance(self):
+ from src.constants.embeddings import Vector
+ from src.lib.milvus.client import MilvusClient
+
+ client = MilvusClient.__new__(MilvusClient)
+ milvus = MagicMock()
+ milvus.search.return_value = [
+ [
+ {
+ "id": 1,
+ "distance": 0.95,
+ "entity": {"uuid": "near"},
+ },
+ {
+ "id": 2,
+ "distance": 0.2,
+ "entity": {"uuid": "far"},
+ },
+ ]
+ ]
+ client._get_client = MagicMock(return_value=milvus)
+ client._ensure_store = MagicMock()
+
+ results = MilvusClient.search_vectors(
+ client, [1.0, 0.0, 0.0], "brain-a", "nodes", k=2
+ )
+ self.assertEqual(results[0].id, "1")
+ self.assertAlmostEqual(results[0].distance, 0.05)
+ self.assertEqual(results[1].id, "2")
+ self.assertAlmostEqual(results[1].distance, 0.8)
+ self.assertTrue(isinstance(results[0], Vector))
+
+ def test_relationship_dedup_skips_self_and_uses_distance_threshold(self):
+ from src.workers.tasks.ingestion import (
+ nearest_existing_vector,
+ should_dedup_relationship,
+ )
+
+ self_hit = SimpleNamespace(id="v-self", distance=0.0, metadata={"uuid": "self"})
+ near = SimpleNamespace(id="v-near", distance=0.05, metadata={"uuid": "near"})
+ far = SimpleNamespace(id="v-far", distance=0.5, metadata={"uuid": "far"})
+
+ nearest = nearest_existing_vector([self_hit, near, far], "v-self")
+ self.assertEqual(nearest.id, "v-near")
+ self.assertTrue(should_dedup_relationship(nearest))
+ self.assertFalse(should_dedup_relationship(far))
+ self.assertIsNone(nearest_existing_vector([self_hit], "v-self"))
+
+
+if __name__ == "__main__":
+ unittest.main()