BrainAPI is a knowledge graph–powered AI memory layer that transforms unstructured data into structured knowledge, enabling intelligent search, recommendations, and contextual memory for AI agents and applications.
Python ai-agentsai-memoryknowledge-graphknowledge-managementknowledge-representationreccomendation-system
feat(ingest): batch Architect extract with scratchpad, grounding, and cost tracking
Christiandd4e9e35f040
diff --git a/.env.example b/.env.exampleindex f08cb1c..83262bf 100644--- a/.env.example+++ b/.env.example@@ -135,12 +135,38 @@ SPACY_KEEP_MODELS_IN_MEMORY="true"# MODELS_MODE => local = uses local models for llm and embeddings, remote = uses remote models for llm and embeddings# OCR_MODE => docling = uses local docling ocr engine, docparser = uses remote docparser ocr pipeline engine# AGENTIC_ARCHITECTURE => custom = uses the custom agentic architecture (faster and token efficient), langchain = uses the langchain agentic architecture-# RUN_GRAPH_CONSOLIDATOR => true = consolidates the graph after ingestion of new data with a high level overview of the graph, false = does not consolidate the graph+# RUN_GRAPH_CONSOLIDATOR => true = consolidates after ingest (LLM mutations; keep false until audited)+# RUN_OBSERVATIONS => true = ObservationsAgent LLM notes (does not write the graph)+# JANITOR_BATCH_SIZE => relationships per Janitor LLM call when batching (default 20)+# INGEST_JANITOR_MAX_LLM_CALLS => hard cap on Janitor LLM batches/session (default 2); remainder dropped with audit+# INGEST_ARCHITECT_PER_UNIT => true = Architect sees Scout chunks (+ prior window); false = full text (legacy)+# INGEST_DEFER_JANITOR => true = batched post-Architect Janitor; false = per-create Janitor (legacy)+# INGEST_ARCHITECT_MODE => batch (default; pure tools=[] extract; escalate to tooler on fail) | schema | tooler (ReAct rollback)+# INGEST_ARCHITECT_MAX_SCHEMA_CALLS => max LLM calls/unit in schema|batch before escalate (default 3)+# INGEST_ARCHITECT_ESCALATE => true (default) allow tooler escalate on empty schema; false = partial/empty unit+# INGEST_ARCHITECT_ESCALATE_MAX_TURNS => max tool turns on escalate tooler path (default 3)+# INGEST_ARCHITECT_DENSE_ENTITY_THRESHOLD => rechunk architect unit when Scout entities >= N (default 12)+# INGEST_ARCHITECT_DENSE_MAX_CHARS => max chars per dense rechunk part (default 1200)+# INGEST_ARCHITECT_PRIOR_CONTEXT => auto (default; scratchpad for batch/schema, raw for tooler) | scratchpad | raw+# INGEST_ARCHITECT_SCRATCHPAD_TOKEN_CAP => max reference tokens for prior scratchpad (default 500)PIPELINE_MODE="accurate"MODELS_MODE="remote"OCR_MODE="docparser"AGENTIC_ARCHITECTURE="custom"-RUN_GRAPH_CONSOLIDATOR="true"+RUN_GRAPH_CONSOLIDATOR="false"+RUN_OBSERVATIONS="false"+JANITOR_BATCH_SIZE="20"+INGEST_JANITOR_MAX_LLM_CALLS="2"+INGEST_ARCHITECT_PER_UNIT="true"+INGEST_DEFER_JANITOR="true"+INGEST_ARCHITECT_MODE="batch"+INGEST_ARCHITECT_MAX_SCHEMA_CALLS="3"+INGEST_ARCHITECT_ESCALATE="true"+INGEST_ARCHITECT_ESCALATE_MAX_TURNS="3"+INGEST_ARCHITECT_DENSE_ENTITY_THRESHOLD="12"+INGEST_ARCHITECT_DENSE_MAX_CHARS="1200"+INGEST_ARCHITECT_PRIOR_CONTEXT="auto"+INGEST_ARCHITECT_SCRATCHPAD_TOKEN_CAP="500"GRAPH_DB="networkx"DATA_DB="postgresql"VECTOR_DB="postgresql"diff --git a/src/config.py b/src/config.pyindex 27d838d..74bcc03 100644--- a/src/config.py+++ b/src/config.py@@ -41,6 +41,36 @@ def validate_pipeline_mode(mode: str | None) -> Literal["lightweight", "accuratereturn mode+def validate_ingest_architect_mode(+ mode: str | None,+) -> Literal["tooler", "schema", "batch"]:+ normalized = (mode or "batch").strip().lower()+ aliases = {+ "tooler": "tooler",+ "current": "tooler",+ "schema": "schema",+ "batch": "batch",+ }+ if normalized not in aliases:+ raise ValueError(+ f"Invalid INGEST_ARCHITECT_MODE: {mode!r}. "+ "Expected 'tooler', 'schema', or 'batch'."+ )+ return aliases[normalized] # type: ignore[return-value]+++def validate_ingest_architect_prior_context(+ mode: str | None,+) -> Literal["auto", "scratchpad", "raw"]:+ normalized = (mode or "auto").strip().lower()+ if normalized in ("auto", "scratchpad", "raw"):+ return normalized # type: ignore[return-value]+ raise ValueError(+ f"Invalid INGEST_ARCHITECT_PRIOR_CONTEXT: {mode!r}. "+ "Expected 'auto', 'scratchpad', or 'raw'."+ )++class AzureConfig:"""Configuration class for the Azure configuration.@@ -560,7 +590,48 @@ class Config:self.spacy = SpacyConfig()self.run_graph_consolidator = (- os.getenv("RUN_GRAPH_CONSOLIDATOR", "true") == "true"+ os.getenv("RUN_GRAPH_CONSOLIDATOR", "false") == "true"+ )+ self.run_observations = os.getenv("RUN_OBSERVATIONS", "false") == "true"+ self.janitor_batch_size = int(os.getenv("JANITOR_BATCH_SIZE", "20"))+ self.ingest_janitor_max_llm_calls = int(+ os.getenv("INGEST_JANITOR_MAX_LLM_CALLS", "2")+ )+ # Accurate-mode hotpath: default to cheap/fast. Set false to restore legacy+ # Architect-full-text + per-create Janitor for A/B measurement.+ self.ingest_architect_per_unit = (+ os.getenv("INGEST_ARCHITECT_PER_UNIT", "true") == "true"+ )+ self.ingest_defer_janitor = (+ os.getenv("INGEST_DEFER_JANITOR", "true") == "true"+ )+ self.ingest_architect_mode: Literal["tooler", "schema", "batch"] = (+ validate_ingest_architect_mode(+ os.getenv("INGEST_ARCHITECT_MODE", "batch")+ )+ )+ self.ingest_architect_max_schema_calls = int(+ os.getenv("INGEST_ARCHITECT_MAX_SCHEMA_CALLS", "3")+ )+ self.ingest_architect_escalate_max_turns = int(+ os.getenv("INGEST_ARCHITECT_ESCALATE_MAX_TURNS", "3")+ )+ self.ingest_architect_escalate = (+ os.getenv("INGEST_ARCHITECT_ESCALATE", "true") == "true"+ )+ self.ingest_architect_dense_entity_threshold = int(+ os.getenv("INGEST_ARCHITECT_DENSE_ENTITY_THRESHOLD", "12")+ )+ self.ingest_architect_dense_max_chars = int(+ os.getenv("INGEST_ARCHITECT_DENSE_MAX_CHARS", "1200")+ )+ self.ingest_architect_prior_context: Literal[+ "auto", "scratchpad", "raw"+ ] = validate_ingest_architect_prior_context(+ os.getenv("INGEST_ARCHITECT_PRIOR_CONTEXT", "auto")+ )+ self.ingest_architect_scratchpad_token_cap = int(+ os.getenv("INGEST_ARCHITECT_SCRATCHPAD_TOKEN_CAP", "500"))self.docparser_endpoint = os.getenv("DOCPARSER_ENDPOINT")self.docparser_token = os.getenv("DOCPARSER_TOKEN")diff --git a/src/constants/agents.py b/src/constants/agents.pyindex 90b2f54..2a3fe54 100644--- a/src/constants/agents.py+++ b/src/constants/agents.py@@ -115,11 +115,13 @@ class AtomicJanitorAgentWrongRelationship(BaseModel):class AtomicJanitorAgentInputOutput(BaseModel):- status: Literal["OK", "ERROR"] = Field(default="ERROR")+ status: Literal["OK", "ERROR", "REJECT"] = Field(default="ERROR")fixed_relationships: Optional[List[_ArchitectAgentRelationship]] = Nonewrong_relationships: Optional[List[AtomicJanitorAgentWrongRelationship]] = Nonerequired_new_nodes: Optional[List[ArchitectAgentNew]] = None+ # Explicit veto notes when status == REJECT (audit); edges are not approved.+ veto_reasons: Optional[List[str]] = Noneclass GraphConsolidatorOutput(BaseModel):diff --git a/src/constants/prompts/architect_agent.py b/src/constants/prompts/architect_agent.pyindex bdfb285..4d545b3 100644--- a/src/constants/prompts/architect_agent.py+++ b/src/constants/prompts/architect_agent.py@@ -213,6 +213,55 @@ STRICT RULES:Begin!"""+BATCH_ARCHITECT_EXTRACT_PROMPT = """+Role: Graph Structural Architect (batch extract).+Task: Emit ALL event-hub relationships for this unit in ONE JSON response. No tools.++{targeting}++Source Text:+{text}++Entities Found by Scout (reuse these UUIDs):+{entities}++STRICT RULES:+- Prefer Scout entity UUIDs. Only add new_nodes when an EVENT hub is clearly missing from Scout.+- Never invent type-named placeholders (name must not equal type, e.g. no name="PERSON" type="PERSON").+- Open-vocabulary predicates are allowed (do not restrict to a closed ontology).+- Every relationship MUST include source_span: an exact contiguous quote from the Source Text that supports the edge.+- Optional span_start/span_end are 0-based character offsets into Source Text for that quote; if unsure, omit offsets rather than guessing.+- Every EVENT endpoint (tail or tip with type EVENT, and every new EVENT node) MUST include a non-empty description: one short sentence grounded in the Source Text (reuse Scout description when present).+- For EVENT hubs include actor and object/content legs when the text supports them.+- Include amount and happened_at when stated in the text.+- "tail" is source/origin; "tip" is destination/target. Never invert them.+- Do not emit bookkeeping commentary — only the structured response.++Begin!+"""++BATCH_ARCHITECT_REPAIR_PROMPT = """+Your previous batch extract had validation errors. Repair ONLY the invalid items.+Keep valid relationships unchanged conceptually; re-emit a complete corrected batch.++Validation errors:+{errors}++Source Text:+{text}++Entities Found by Scout (reuse these UUIDs):+{entities}++STRICT RULES:+- Fix missing/ungrounded source_span, mismatched span offsets (prefer correct offsets or omit them), type-named placeholders, unknown endpoints, malformed endpoints, and missing_event_description.+- Every EVENT endpoint/new EVENT node needs a non-empty grounded description (reuse Scout description when present).+- Prefer Scout UUIDs. No type==name placeholders.+- Re-emit the full relationships list for this unit after repairs.++Begin!+"""+ARCHITECT_AGENT_TOOLER_SYSTEM_PROMPT_UNCOMPRESSED = """You are a "Structural Graph Architect." Your goal is to map information into an Active Vector Graph.diff --git a/src/core/agents/architect_agent.py b/src/core/agents/architect_agent.pyindex a6a79a1..e07cf34 100644--- a/src/core/agents/architect_agent.py+++ b/src/core/agents/architect_agent.py@@ -13,6 +13,7 @@ import osimport uuidfrom concurrent.futures import ThreadPoolExecutorfrom concurrent.futures import TimeoutError as FutureTimeoutError+from src.core.saving.ingest_cost import submit_with_contextfrom typing import Callable, Dict, List, Literal, Optional, Tuplefrom langchain.tools import BaseTool@@ -46,6 +47,8 @@ from src.constants.prompts.architect_agent import (ARCHITECT_AGENT_TOOLER_COARSE_SYSTEM_PROMPT,ARCHITECT_AGENT_TOOLER_CREATE_RELATIONSHIPS_PROMPT,ARCHITECT_AGENT_TOOLER_SYSTEM_PROMPT,+ BATCH_ARCHITECT_EXTRACT_PROMPT,+ BATCH_ARCHITECT_REPAIR_PROMPT,STRUCTURED_ARCHITECT_AGENT_CREATE_RELATIONSHIPS_PROMPT,STRUCTURED_ARCHITECT_AGENT_FIX_RELATIONSHIPS_PROMPT,)@@ -122,6 +125,13 @@ def _to_architect_relationship(relationship) -> ArchitectAgentRelationship:if isinstance(data.get("tip"), dict)else getattr(data.get("tip"), "happened_at", None),)+ # Entity endpoints must have string uuids for ArchitectAgentEntity.+ for end_key in ("tail", "tip"):+ end = data.get(end_key)+ if isinstance(end, dict) and not end.get("uuid"):+ end = dict(end)+ end["uuid"] = str(uuid.uuid4())+ data[end_key] = endif not data.get("uuid"):tail = data.get("tail") or {}tip = data.get("tip") or {}@@ -304,6 +314,7 @@ class ArchitectAgent:self.session_id: Optional[str] = Noneself.janitor_agent = Noneself._janitor_agent_brain_id = None+ self.defer_janitor = True# self.database_desc = database_descdef take_pending_relationships(self) -> List[ArchitectAgentRelationship]:@@ -323,6 +334,184 @@ class ArchitectAgent:self.pending_persistence_batches.append(list(relationships))self.relationships_set.extend(relationships)+ def run_batched_janitor(+ self,+ *,+ text: str,+ brain_id: str = "default",+ targeting: Optional[Node] = None,+ batch_size: int = 20,+ cost_ledger=None,+ ) -> None:+ """+ Run Janitor once per batch over pending relationships (post-Architect),+ with cheap grounding triage so LLM Janitor stays exception-only.+ """+ from src.core.agents.janitor_agent import JanitorAgent+ from src.core.saving.grounding import triage_relationships_for_janitor+ from src.services.input.agents import (+ embeddings_adapter,+ graph_adapter,+ llm_small_adapter,+ vector_store_adapter,+ )++ pending = []+ for batch in self.pending_persistence_batches:+ pending.extend(batch)+ if not pending:+ pending = list(self.relationships_set)+ if not pending:+ return++ triage = triage_relationships_for_janitor(pending, text)+ if cost_ledger is not None:+ cost_ledger.janitor_skipped += len(triage.accept)+ cost_ledger.janitor_rejected += len(triage.reject)+ cost_ledger.janitor_ran += 0+ for decision in triage.reject:+ cost_ledger.janitor_drop_reasons.append(decision.reason)++ if triage.reject:+ print(+ "[DEBUG (run_batched_janitor)]: deterministic rejects "+ f"count={len(triage.reject)} "+ f"reasons={[d.reason for d in triage.reject]}"+ )++ cleaned: List[ArchitectAgentRelationship] = list(triage.accept)+ need = list(triage.ambiguous)+ if cost_ledger is not None:+ cost_ledger.janitor_ambiguous += len(need)+ if not need:+ self.pending_persistence_batches = [cleaned] if cleaned else []+ self.relationships_set = list(cleaned)+ return++ janitor_agent = self.janitor_agent+ if janitor_agent is None or self._janitor_agent_brain_id != brain_id:+ janitor_agent = JanitorAgent(+ llm_small_adapter,+ kg=graph_adapter,+ vector_store=vector_store_adapter,+ embeddings=embeddings_adapter,+ database_desc=graph_adapter.graphdb_description,+ )+ self.janitor_agent = janitor_agent+ self._janitor_agent_brain_id = brain_id++ size = max(1, int(batch_size or 20))+ max_llm_calls = max(0, int(getattr(config, "ingest_janitor_max_llm_calls", 2)))+ llm_calls = 0+ for i in range(0, len(need), size):+ if llm_calls >= max_llm_calls:+ remaining = len(need) - i+ print(+ "[DEBUG (run_batched_janitor)]: janitor LLM budget exhausted "+ f"max_calls={max_llm_calls}; dropping remaining ambiguous={remaining}"+ )+ if cost_ledger is not None:+ cost_ledger.janitor_drop_reasons.append(+ f"janitor_budget_exhausted:{remaining}"+ )+ break+ batch = need[i : i + size]+ input_rels = [+ _ArchitectAgentRelationship(+ tip=rel.tip,+ tail=rel.tail,+ name=rel.name,+ description=rel.description,+ properties=getattr(rel, "properties", {}) or {},+ **(+ {"amount": rel.amount}+ if getattr(rel, "amount", None)+ else {}+ ),+ )+ for rel in batch+ ]+ response = janitor_agent.run_atomic_janitor(+ input_relationships=input_rels,+ text=text,+ targeting=targeting,+ brain_id=brain_id,+ timeout=300,+ max_retries=3,+ )+ llm_calls += 1+ if cost_ledger is not None:+ cost_ledger.janitor_ran += 1++ # Parse / provider failure must not silently approve edges.+ if response is None:+ print(+ "[DEBUG (run_batched_janitor)]: janitor parse/provider failure; "+ f"dropping ambiguous batch size={len(batch)}"+ )+ if cost_ledger is not None:+ cost_ledger.janitor_drop_reasons.append("janitor_parse_failure")+ continue++ if response == "OK":+ cleaned.extend(batch)+ continue++ status = getattr(response, "status", None)+ if status == "REJECT":+ veto = getattr(response, "veto_reasons", None) or []+ print(+ "[DEBUG (run_batched_janitor)]: janitor veto "+ f"count={len(batch)} reasons={veto}"+ )+ if cost_ledger is not None:+ cost_ledger.janitor_drop_reasons.extend(+ list(veto) or ["janitor_veto"]+ )+ continue++ fixed = getattr(response, "fixed_relationships", None) or []+ wrong = getattr(response, "wrong_relationships", None) or []+ wrong_keys = set()+ for w in wrong:+ target = getattr(w, "relationship", None) or w+ wrong_keys.add(+ (+ getattr(getattr(target, "tail", None), "uuid", None),+ getattr(getattr(target, "tip", None), "uuid", None),+ getattr(target, "name", None),+ )+ )+ if cost_ledger is not None and wrong:+ for w in wrong:+ cost_ledger.janitor_drop_reasons.append(+ getattr(w, "reason", None) or "janitor_wrong"+ )+ for rel in batch:+ key = (rel.tail.uuid, rel.tip.uuid, rel.name)+ if key in wrong_keys:+ continue+ cleaned.append(rel)+ for rel in fixed:+ cleaned.append(+ ArchitectAgentRelationship(+ flow_key=str(uuid.uuid4()),+ tip=rel.tip,+ tail=rel.tail,+ name=rel.name,+ description=rel.description,+ properties=getattr(rel, "properties", {}) or {},+ **(+ {"amount": getattr(rel, "amount", None)}+ if getattr(rel, "amount", None)+ else {}+ ),+ )+ )++ self.pending_persistence_batches = [cleaned] if cleaned else []+ self.relationships_set = list(cleaned)+def _get_tools(self,text: Optional[str] = None,@@ -654,7 +843,7 @@ class ArchitectAgent:"""try:with ThreadPoolExecutor(max_workers=1) as executor:- future = executor.submit(+ future = submit_with_context(executor,_invoke_agent,unconnected_entities_list,all_relationships,@@ -1170,7 +1359,7 @@ class ArchitectAgent:"""try:with ThreadPoolExecutor(max_workers=1) as executor:- future = executor.submit(+ future = submit_with_context(executor,_invoke_agent,unconnected_entities_list,all_relationships,@@ -1316,6 +1505,256 @@ class ArchitectAgent:relationships=relationships_to_persist,)+ def run_batch_extract(+ self,+ text: str,+ entities: List[ScoutEntity],+ targeting: Optional[Node] = None,+ brain_id: str = "default",+ timeout: int = 120,+ max_retries: int = 2,+ ingestion_session_id: Optional[str] = None,+ mode: Literal["granular", "coarse"] = "granular",+ reset: bool = True,+ cost_ledger=None,+ escalate: bool = True,+ ) -> List[ArchitectAgentRelationship]:+ """+ Pure tools=[] schema extract: one primary call, at most one repair, then+ optional escalate to run_tooler. No graph side effects before validation.+ """+ from src.core.agents.core.parsing import parse_structured_from_messages+ from src.core.saving.architect_batch import (+ BatchExtractResponse,+ event_leg_incomplete,+ validate_batch_extract,+ )++ if reset or not self.session_id:+ self.session_id = str(uuid.uuid4())+ self.relationships_set.clear()+ self.pending_persistence_batches.clear()++ entities_dict = {entity.uuid: entity for entity in entities}+ self.entities = {+ uuid_: strip_properties([entity.model_dump(mode="json")])[0]+ for uuid_, entity in entities_dict.items()+ }++ self._get_agent(+ output_schema=BatchExtractResponse,+ brain_id=brain_id,+ targeting=targeting,+ type_="single",+ mode=mode,+ )++ entity_payload = [+ strip_properties([entity.model_dump(mode="json")])[0]+ for entity in entities+ ]+ targeting_block = (+ f"""+The information is related to the following node; connect relationships to it+when relevant:+Name: {targeting.name}+UUID: {targeting.uuid}+Type: {targeting.labels}+Description: {targeting.description}+{targeting.properties}+"""+ if targeting+ else ""+ )++ schema_calls = 0+ repair_calls = 0+ max_schema_calls = max(1, int(config.ingest_architect_max_schema_calls))++ def _invoke(prompt: str) -> BatchExtractResponse:+ nonlocal schema_calls+ messages_list = [+ {+ "role": "user",+ "content": prompt,+ }+ ]+ metadata = {+ "agent": "architect_agent",+ "brain_id": brain_id,+ "loop_origin": "architect_batch",+ }+ if ingestion_session_id:+ metadata["ingestion_session_id"] = ingestion_session_id++ def _call():+ return self.agent.invoke(+ {"messages": messages_list},+ config={+ "tags": ["architect_agent", "architect_batch"],+ "metadata": metadata,+ },+ )++ @retry(+ stop=stop_after_attempt(max_retries),+ wait=wait_exponential(multiplier=1, min=1, max=10),+ retry=retry_if_exception_type(TimeoutError),+ reraise=True,+ )+ def _with_timeout():+ try:+ with ThreadPoolExecutor(max_workers=1) as executor:+ future = submit_with_context(executor, _call)+ return future.result(timeout=timeout)+ except FutureTimeoutError:+ raise TimeoutError(+ f"Architect batch extract timed out after {timeout}s"+ )++ response = _with_timeout()+ schema_calls += 1+ structured = response.get("structured_response")+ if isinstance(structured, dict):+ try:+ structured = BatchExtractResponse.model_validate(structured)+ except Exception:+ structured = None+ if structured is None:+ fallback = parse_structured_from_messages(+ response.get("messages", []), BatchExtractResponse+ )+ structured = fallback+ if structured is None:+ structured = BatchExtractResponse(relationships=[], new_nodes=[])+ return structured++ primary_prompt = prompt_registry.get(+ "BATCH_ARCHITECT_EXTRACT_PROMPT",+ BATCH_ARCHITECT_EXTRACT_PROMPT,+ ).format(+ text=text,+ entities=entity_payload,+ targeting=targeting_block,+ )+ extracted = _invoke(primary_prompt)+ validation = validate_batch_extract(+ extracted, source_text=text, scout_entities=entities+ )+ primary_validation = validation++ if validation.rejected and schema_calls < max_schema_calls:+ repair_prompt = prompt_registry.get(+ "BATCH_ARCHITECT_REPAIR_PROMPT",+ BATCH_ARCHITECT_REPAIR_PROMPT,+ ).format(+ errors="\n".join(validation.reasons),+ text=text,+ entities=entity_payload,+ )+ extracted = _invoke(repair_prompt)+ repair_calls += 1+ repaired_validation = validate_batch_extract(+ extracted, source_text=text, scout_entities=entities+ )+ # Keep the more usable extract; do not discard a good primary+ # when repair returns fewer accepted edges.+ if len(repaired_validation.accepted) > len(primary_validation.accepted):+ validation = repaired_validation+ elif (+ len(repaired_validation.accepted) == len(primary_validation.accepted)+ and len(repaired_validation.rejected) < len(primary_validation.rejected)+ ):+ validation = repaired_validation+ else:+ validation = primary_validation++ escalate_reason = None+ if not validation.usable and entities:+ escalate_reason = "schema_empty_or_all_rejected"+ elif validation.rejected:+ # Partial but usable: keep accepted edges, drop rejects with audit.+ # Do not tooler-escalate — that was discarding happy-path extracts.+ print(+ "[DEBUG (run_batch_extract)]: dropping rejected items "+ f"count={len(validation.rejected)} reasons={validation.reasons}"+ )+ if (+ validation.usable+ and event_leg_incomplete(validation.accepted, entities)+ ):+ print(+ "[DEBUG (run_batch_extract)]: event_leg_incomplete after "+ "schema extract; persisting accepted without escalate"+ )++ accepted_rels: List[ArchitectAgentRelationship] = []+ if validation.accepted:+ for rel in validation.accepted:+ props = dict(rel.properties or {})+ if rel.source_span and not props.get("source_span"):+ props["source_span"] = rel.source_span+ if rel.span_start is not None:+ props["span_start"] = rel.span_start+ if rel.span_end is not None:+ props["span_end"] = rel.span_end+ if rel.happened_at and not props.get("happened_at"):+ props["happened_at"] = rel.happened_at+ accepted_rels.append(+ _to_architect_relationship(+ {+ "tail": rel.tail.model_dump(mode="json"),+ "tip": rel.tip.model_dump(mode="json"),+ "name": rel.name,+ "description": rel.description,+ "amount": rel.amount,+ "properties": props,+ }+ )+ )+ if accepted_rels and escalate_reason is None:+ self.queue_relationships_for_persistence(accepted_rels)++ should_escalate = bool(escalate and escalate_reason)+ if cost_ledger is not None:+ cost_ledger.record_architect_unit(+ escalated=should_escalate,+ reason=escalate_reason,+ schema_calls=schema_calls,+ repair_calls=repair_calls,+ )++ if should_escalate:+ max_turns = max(0, int(config.ingest_architect_escalate_max_turns))+ if max_turns <= 0:+ print(+ "[DEBUG (run_batch_extract)]: escalate suppressed "+ f"reason={escalate_reason} (max_turns=0); partial/empty unit"+ )+ if cost_ledger is not None:+ cost_ledger.janitor_drop_reasons.append(+ "escalate_budget_exhausted"+ )+ return accepted_rels+ print(+ f"[DEBUG (run_batch_extract)]: escalating unit reason={escalate_reason} "+ f"max_turns={max_turns}"+ )+ return self.run_tooler(+ text,+ entities,+ targeting=targeting,+ brain_id=brain_id,+ timeout=timeout,+ max_retries=max_retries,+ ingestion_session_id=ingestion_session_id,+ mode=mode,+ reset=False,+ max_tool_turns=max_turns,+ )++ return list(self.relationships_set) if self.relationships_set else accepted_rels+def run_tooler(self,text: str,@@ -1326,6 +1765,8 @@ class ArchitectAgent:max_retries: int = 3,ingestion_session_id: Optional[str] = None,mode: Literal["granular", "coarse"] = "granular",+ reset: bool = True,+ max_tool_turns: Optional[int] = None,) -> List[ArchitectAgentRelationship]:"""Drive the architect agent in "tooler" mode to iteratively discover relationships using available tools and collect the results.@@ -1339,6 +1780,8 @@ class ArchitectAgent:brain_id (str): Identifier for the knowledge brain/context to use.timeout (int): Maximum seconds to wait for a single agent invocation before raising a timeout.max_retries (int): Maximum number of retry attempts for timed or retried invocations.+ reset (bool): When True, clear prior session relationships before running.+ max_tool_turns (Optional[int]): Hard cap on custom-backend tool loop turns (escalate budget).Returns:List[ArchitectAgentRelationship]: The relationships discovered and collected by the agent during this run.@@ -1350,15 +1793,21 @@ class ArchitectAgent:from src.lib.redis.client import _redis_client- self.session_id = str(uuid.uuid4())- self.relationships_set.clear()- self.pending_persistence_batches.clear()+ if reset or not self.session_id:+ self.session_id = str(uuid.uuid4())+ self.relationships_set.clear()+ self.pending_persistence_batches.clear()entities_dict = {entity.uuid: strip_properties([entity.model_dump(mode="json")])[0]for entity in entities}- self.entities = entities_dict+ if reset:+ self.entities = entities_dict+ else:+ existing = getattr(self, "entities", None) or {}+ existing.update(entities_dict)+ self.entities = existingself._get_agent(type_="tooler",@@ -1437,13 +1886,16 @@ class ArchitectAgent:}if ingestion_session_id:metadata["ingestion_session_id"] = ingestion_session_id+ invoke_config = {+ "recursion_limit": MAX_RECURSION_LIMIT,+ "tags": ["architect_agent", "architect_tooler"],+ "metadata": metadata,+ }+ if max_tool_turns is not None:+ invoke_config["max_tool_turns"] = int(max_tool_turns)return self.agent.invoke({"messages": messages_list},- config={- "recursion_limit": MAX_RECURSION_LIMIT,- "tags": ["architect_agent", "architect_tooler"],- "metadata": metadata,- },+ config=invoke_config,)@retry(@@ -1467,7 +1919,7 @@ class ArchitectAgent:"""try:with ThreadPoolExecutor(max_workers=1) as executor:- future = executor.submit(_invoke_agent, previous_messages)+ future = submit_with_context(executor, _invoke_agent, previous_messages)response = future.result(timeout=timeout)return responseexcept FutureTimeoutError:diff --git a/src/core/agents/core/invoke_loop.py b/src/core/agents/core/invoke_loop.pyindex a27c96e..730f664 100644--- a/src/core/agents/core/invoke_loop.py+++ b/src/core/agents/core/invoke_loop.py@@ -60,6 +60,12 @@ def _log_token_usage(response) -> None:f"out={usage.get('output_tokens', 0)}"+ (f" cached={cached}" if cached else ""))+ try:+ from src.core.saving.ingest_cost import record_usage_from_response++ record_usage_from_response(response)+ except Exception:+ passdef _trace_metadata(agent, config) -> dict:@@ -425,6 +431,16 @@ def _run_invoke_loop_impl(_last_called_tool_name = tool_namewhile True:tool_loop_count += 1+ max_tool_turns = config.get("max_tool_turns")+ if max_tool_turns is not None and tool_loop_count > int(+ max_tool_turns+ ):+ if agent.debug:+ print(+ "[DEBUG (agent_base)]: max_tool_turns reached: ",+ max_tool_turns,+ )+ breaktracer.expensive_loop("agent.invoke_loop.tool_loop",tool_loop_count,diff --git a/src/core/agents/janitor_agent.py b/src/core/agents/janitor_agent.pyindex 226cdce..8279d01 100644--- a/src/core/agents/janitor_agent.py+++ b/src/core/agents/janitor_agent.py@@ -16,6 +16,7 @@ from langchain.agents.structured_output import ToolStrategyfrom langchain.tools import BaseToolfrom pydantic import BaseModelfrom concurrent.futures import ThreadPoolExecutor, TimeoutError as FutureTimeoutError+from src.core.saving.ingest_cost import submit_with_contextfrom tenacity import (retry,stop_after_attempt,@@ -356,7 +357,7 @@ class JanitorAgent:"""try:with ThreadPoolExecutor(max_workers=1) as executor:- future = executor.submit(_invoke_agent, previous_messages)+ future = submit_with_context(executor, _invoke_agent, previous_messages)response = future.result(timeout=timeout)return responseexcept FutureTimeoutError:@@ -507,7 +508,7 @@ class JanitorAgent:"""try:with ThreadPoolExecutor(max_workers=1) as executor:- future = executor.submit(_invoke_agent, previous_messages)+ future = submit_with_context(executor, _invoke_agent, previous_messages)response = future.result(timeout=timeout)return responseexcept FutureTimeoutError:@@ -674,7 +675,7 @@ class JanitorAgent:"""try:with ThreadPoolExecutor(max_workers=1) as executor:- future = executor.submit(_invoke_agent, previous_messages)+ future = submit_with_context(executor, _invoke_agent, previous_messages)response = future.result(timeout=timeout)return responseexcept FutureTimeoutError:diff --git a/src/core/agents/scout_agent.py b/src/core/agents/scout_agent.pyindex c9bd161..8f543a8 100644--- a/src/core/agents/scout_agent.py+++ b/src/core/agents/scout_agent.py@@ -12,6 +12,7 @@ import osimport uuidfrom concurrent.futures import ThreadPoolExecutorfrom concurrent.futures import TimeoutError as FutureTimeoutError+from src.core.saving.ingest_cost import submit_with_contextfrom typing import Dict, List, Literal, Optionalfrom langchain.tools import BaseTool@@ -289,7 +290,7 @@ class ScoutAgent:"""try:with ThreadPoolExecutor(max_workers=1) as executor:- future = executor.submit(_invoke_agent)+ future = submit_with_context(executor, _invoke_agent)response = future.result(timeout=timeout)return responseexcept FutureTimeoutError:@@ -476,7 +477,7 @@ class ScoutAgent:def _invoke_agent_with_retry():try:with ThreadPoolExecutor(max_workers=1) as executor:- future = executor.submit(_invoke_agent)+ future = submit_with_context(executor, _invoke_agent)response = future.result(timeout=timeout)return responseexcept FutureTimeoutError:diff --git a/src/core/agents/tools/architect_agent/ArchitectAgentCreateRelationshipTool.py b/src/core/agents/tools/architect_agent/ArchitectAgentCreateRelationshipTool.pyindex 4286db3..7d3603d 100644--- a/src/core/agents/tools/architect_agent/ArchitectAgentCreateRelationshipTool.py+++ b/src/core/agents/tools/architect_agent/ArchitectAgentCreateRelationshipTool.py@@ -380,7 +380,9 @@ class ArchitectAgentCreateRelationshipTool(BaseTool):fixed_rels_sets = set()janitor_response = None- if self.mode == "granular":+ if self.mode == "granular" and not getattr(+ self.architect_agent, "defer_janitor", False+ ):from src.core.agents.janitor_agent import JanitorAgentjanitor_agent = getattr(self.architect_agent, "janitor_agent", None)diff --git a/src/core/saving/architect_batch.py b/src/core/saving/architect_batch.pynew file mode 100644index 0000000..845ce31--- /dev/null+++ b/src/core/saving/architect_batch.py@@ -0,0 +1,488 @@+from __future__ import annotations++from dataclasses import dataclass, field+from typing import Any, Optional, Sequence++from pydantic import BaseModel, Field++from src.core.saving.grounding import (+ align_span,+ endpoints_well_formed,+ find_span_offsets,+ realign_span_offsets,+)+++class BatchEndpoint(BaseModel):+ uuid: Optional[str] = None+ name: str+ type: str+ description: Optional[str] = None+ happened_at: Optional[str] = None+ polarity: Optional[str] = "neutral"+ properties: Optional[dict] = Field(default_factory=dict)+++class BatchRelationship(BaseModel):+ tail: BatchEndpoint+ tip: BatchEndpoint+ name: str+ description: Optional[str] = None+ amount: Optional[float] = None+ source_span: str = ""+ span_start: Optional[int] = None+ span_end: Optional[int] = None+ happened_at: Optional[str] = None+ properties: Optional[dict] = Field(default_factory=dict)+++class BatchExtractResponse(BaseModel):+ new_nodes: list[BatchEndpoint] = Field(default_factory=list)+ relationships: list[BatchRelationship] = Field(default_factory=list)+++@dataclass+class BatchValidationIssue:+ index: int+ reason: str+ relationship: Optional[BatchRelationship] = None+++@dataclass+class BatchValidationResult:+ accepted: list[BatchRelationship] = field(default_factory=list)+ rejected: list[BatchValidationIssue] = field(default_factory=list)+ new_nodes: list[BatchEndpoint] = field(default_factory=list)++ @property+ def ok(self) -> bool:+ return bool(self.accepted) and not self.rejected++ @property+ def usable(self) -> bool:+ """True when at least one relationship survived validation."""+ return bool(self.accepted)++ @property+ def reasons(self) -> list[str]:+ return [f"[{i.index}] {i.reason}" for i in self.rejected]+++def _is_type_named_placeholder(endpoint: Any) -> bool:+ name = (getattr(endpoint, "name", None) or "").strip()+ typ = (getattr(endpoint, "type", None) or "").strip()+ if not name or not typ:+ return True+ return name.lower() == typ.lower()+++def _is_event_endpoint(endpoint: Any) -> bool:+ return (getattr(endpoint, "type", None) or "").strip().lower() == "event"+++def _endpoint_exists(+ endpoint: BatchEndpoint,+ *,+ known_uuids: set[str],+ known_keys: set[tuple[str, str]],+) -> bool:+ if endpoint.uuid and endpoint.uuid in known_uuids:+ return True+ key = (+ (endpoint.name or "").strip().lower(),+ (endpoint.type or "").strip().lower(),+ )+ return key in known_keys+++def _scout_lookup(+ scout_entities: Sequence[Any],+) -> tuple[dict[str, Any], dict[tuple[str, str], Any]]:+ by_uuid: dict[str, Any] = {}+ by_key: dict[tuple[str, str], Any] = {}+ for entity in scout_entities:+ uuid = getattr(entity, "uuid", None) or (+ entity.get("uuid") if isinstance(entity, dict) else None+ )+ name = getattr(entity, "name", None) or (+ entity.get("name") if isinstance(entity, dict) else None+ )+ typ = getattr(entity, "type", None) or (+ entity.get("type") if isinstance(entity, dict) else None+ )+ if uuid:+ by_uuid[str(uuid)] = entity+ if name and typ:+ by_key[(str(name).strip().lower(), str(typ).strip().lower())] = entity+ return by_uuid, by_key+++def _entity_description(entity: Any) -> str:+ if entity is None:+ return ""+ if isinstance(entity, dict):+ return (entity.get("description") or "").strip()+ return (getattr(entity, "description", None) or "").strip()+++def _fill_event_description(+ endpoint: BatchEndpoint,+ *,+ rel: BatchRelationship,+ scout_by_uuid: dict[str, Any],+ scout_by_key: dict[tuple[str, str], Any],+) -> bool:+ """+ Ensure EVENT endpoint has a non-empty description.+ Prefers Scout description, then relationship description, then source_span,+ then a non-placeholder endpoint name.+ Returns True if description is non-empty after fill.+ """+ if not _is_event_endpoint(endpoint):+ return True+ if (endpoint.description or "").strip():+ return True++ scout = None+ if endpoint.uuid and endpoint.uuid in scout_by_uuid:+ scout = scout_by_uuid[endpoint.uuid]+ else:+ key = (+ (endpoint.name or "").strip().lower(),+ (endpoint.type or "").strip().lower(),+ )+ scout = scout_by_key.get(key)++ name = (endpoint.name or "").strip()+ for candidate in (+ _entity_description(scout),+ (rel.description or "").strip(),+ (rel.source_span or "").strip(),+ name if name and not _is_type_named_placeholder(endpoint) else "",+ ):+ if candidate:+ endpoint.description = candidate+ return True+ return False+++def _ensure_endpoint_uuid(+ endpoint: BatchEndpoint,+ *,+ scout_by_uuid: dict[str, Any],+ scout_by_key: dict[tuple[str, str], Any],+) -> None:+ """Fill missing endpoint uuid from scout match or mint a fresh one."""+ if endpoint.uuid:+ return+ key = (+ (endpoint.name or "").strip().lower(),+ (endpoint.type or "").strip().lower(),+ )+ scout = scout_by_key.get(key)+ if scout is not None:+ scout_uuid = getattr(scout, "uuid", None) or (+ scout.get("uuid") if isinstance(scout, dict) else None+ )+ if scout_uuid:+ endpoint.uuid = str(scout_uuid)+ return+ import uuid as _uuid++ endpoint.uuid = str(_uuid.uuid4())+++def _admit_endpoint(+ endpoint: BatchEndpoint,+ *,+ known_uuids: set[str],+ known_keys: set[tuple[str, str]],+ admitted: list[BatchEndpoint],+ scout_by_uuid: dict[str, Any] | None = None,+ scout_by_key: dict[tuple[str, str], Any] | None = None,+) -> bool:+ """+ Soft-register a well-formed unknown relationship endpoint so dense extracts+ stay on the schema path instead of all-reject → tooler escalate.+ """+ if _is_type_named_placeholder(endpoint) or not (endpoint.name or "").strip():+ return False+ typ = (endpoint.type or "").strip()+ if not typ:+ return False+ key = (endpoint.name.strip().lower(), typ.lower())+ _ensure_endpoint_uuid(+ endpoint,+ scout_by_uuid=scout_by_uuid or {},+ scout_by_key=scout_by_key or {},+ )+ if endpoint.uuid:+ known_uuids.add(endpoint.uuid)+ if key not in known_keys:+ known_keys.add(key)+ admitted.append(endpoint)+ return True+++def ensure_event_descriptions(+ rel: BatchRelationship,+ *,+ scout_by_uuid: dict[str, Any],+ scout_by_key: dict[tuple[str, str], Any],+) -> tuple[bool, str]:+ for endpoint in (rel.tail, rel.tip):+ if not _is_event_endpoint(endpoint):+ continue+ if not _fill_event_description(+ endpoint,+ rel=rel,+ scout_by_uuid=scout_by_uuid,+ scout_by_key=scout_by_key,+ ):+ return False, "missing_event_description"+ return True, ""+++def _span_in_bounds(+ rel: BatchRelationship,+ source_text: str,+) -> tuple[bool, str]:+ text = source_text or ""+ span = (rel.source_span or "").strip()+ if not span:+ return False, "missing_source_span"++ ok, new_start, new_end, reason = realign_span_offsets(+ span=span,+ source_text=text,+ span_start=rel.span_start,+ span_end=rel.span_end,+ )+ if not ok:+ return False, reason or "source_span_not_grounded"++ rel.span_start = new_start+ rel.span_end = new_end+ # Prefer exact quote from source when offsets were repaired.+ if new_start is not None and new_end is not None:+ sliced = text[new_start:new_end]+ if sliced.strip():+ rel.source_span = sliced+ return True, ""+++def normalize_loose(value: str) -> str:+ return " ".join((value or "").split()).strip().lower()+++def validate_batch_extract(+ payload: BatchExtractResponse | dict[str, Any],+ *,+ source_text: str,+ scout_entities: Sequence[Any],+) -> BatchValidationResult:+ if isinstance(payload, dict):+ payload = BatchExtractResponse.model_validate(payload)++ known_uuids: set[str] = set()+ known_keys: set[tuple[str, str]] = set()+ scout_by_uuid, scout_by_key = _scout_lookup(scout_entities)+ for entity in scout_entities:+ uuid = getattr(entity, "uuid", None) or (entity.get("uuid") if isinstance(entity, dict) else None)+ name = getattr(entity, "name", None) or (entity.get("name") if isinstance(entity, dict) else None)+ typ = getattr(entity, "type", None) or (entity.get("type") if isinstance(entity, dict) else None)+ if uuid:+ known_uuids.add(str(uuid))+ if name and typ:+ known_keys.add((str(name).strip().lower(), str(typ).strip().lower()))++ kept_nodes: list[BatchEndpoint] = []+ result = BatchValidationResult(new_nodes=[])+ for node in payload.new_nodes or []:+ if _is_type_named_placeholder(node):+ result.rejected.append(+ BatchValidationIssue(+ index=-1,+ reason=f"new_node_type_named_placeholder:{node.name}",+ )+ )+ continue+ if _is_event_endpoint(node) and not (node.description or "").strip():+ scout = None+ if node.uuid and node.uuid in scout_by_uuid:+ scout = scout_by_uuid[node.uuid]+ else:+ scout = scout_by_key.get(+ (+ (node.name or "").strip().lower(),+ (node.type or "").strip().lower(),+ )+ )+ filled = _entity_description(scout)+ if filled:+ node.description = filled+ else:+ result.rejected.append(+ BatchValidationIssue(+ index=-1,+ reason=f"new_node_missing_event_description:{node.name}",+ )+ )+ continue+ if not node.uuid:+ import uuid as _uuid++ node.uuid = str(_uuid.uuid4())+ kept_nodes.append(node)+ if node.uuid:+ known_uuids.add(node.uuid)+ known_keys.add(+ ((node.name or "").strip().lower(), (node.type or "").strip().lower())+ )+ result.new_nodes = kept_nodes++ for idx, rel in enumerate(payload.relationships or []):+ if _is_type_named_placeholder(rel.tail) or _is_type_named_placeholder(rel.tip):+ result.rejected.append(+ BatchValidationIssue(+ index=idx,+ reason="type_named_placeholder_endpoint",+ relationship=rel,+ )+ )+ continue+ if not endpoints_well_formed(rel):+ result.rejected.append(+ BatchValidationIssue(+ index=idx,+ reason="malformed_endpoints",+ relationship=rel,+ )+ )+ continue+ if not (rel.name or "").strip():+ result.rejected.append(+ BatchValidationIssue(+ index=idx,+ reason="missing_predicate",+ relationship=rel,+ )+ )+ continue+ if not _endpoint_exists(rel.tail, known_uuids=known_uuids, known_keys=known_keys):+ if not _admit_endpoint(+ rel.tail,+ known_uuids=known_uuids,+ known_keys=known_keys,+ admitted=kept_nodes,+ scout_by_uuid=scout_by_uuid,+ scout_by_key=scout_by_key,+ ):+ result.rejected.append(+ BatchValidationIssue(+ index=idx,+ reason="unknown_tail_endpoint",+ relationship=rel,+ )+ )+ continue+ if not _endpoint_exists(rel.tip, known_uuids=known_uuids, known_keys=known_keys):+ if not _admit_endpoint(+ rel.tip,+ known_uuids=known_uuids,+ known_keys=known_keys,+ admitted=kept_nodes,+ scout_by_uuid=scout_by_uuid,+ scout_by_key=scout_by_key,+ ):+ result.rejected.append(+ BatchValidationIssue(+ index=idx,+ reason="unknown_tip_endpoint",+ relationship=rel,+ )+ )+ continue+ ok_span, span_reason = _span_in_bounds(rel, source_text)+ if not ok_span:+ result.rejected.append(+ BatchValidationIssue(+ index=idx,+ reason=span_reason,+ relationship=rel,+ )+ )+ continue+ ok_event, event_reason = ensure_event_descriptions(+ rel,+ scout_by_uuid=scout_by_uuid,+ scout_by_key=scout_by_key,+ )+ if not ok_event:+ result.rejected.append(+ BatchValidationIssue(+ index=idx,+ reason=event_reason,+ relationship=rel,+ )+ )+ continue+ _ensure_endpoint_uuid(+ rel.tail, scout_by_uuid=scout_by_uuid, scout_by_key=scout_by_key+ )+ _ensure_endpoint_uuid(+ rel.tip, scout_by_uuid=scout_by_uuid, scout_by_key=scout_by_key+ )+ result.accepted.append(rel)++ return result+++def event_leg_incomplete(+ relationships: Sequence[BatchRelationship],+ scout_entities: Sequence[Any],+) -> bool:+ event_uuids = set()+ for entity in scout_entities:+ typ = getattr(entity, "type", None) or (entity.get("type") if isinstance(entity, dict) else None)+ uuid = getattr(entity, "uuid", None) or (entity.get("uuid") if isinstance(entity, dict) else None)+ if typ and str(typ).strip().lower() == "event" and uuid:+ event_uuids.add(str(uuid))+ if not event_uuids:+ return False+ touched: set[str] = set()+ for rel in relationships:+ for endpoint in (rel.tail, rel.tip):+ if endpoint.uuid and endpoint.uuid in event_uuids:+ touched.add(endpoint.uuid)+ elif (endpoint.type or "").strip().lower() == "event":+ for entity in scout_entities:+ name = getattr(entity, "name", None) or (+ entity.get("name") if isinstance(entity, dict) else None+ )+ uuid = getattr(entity, "uuid", None) or (+ entity.get("uuid") if isinstance(entity, dict) else None+ )+ if (+ uuid+ and name+ and normalize_loose(name) == normalize_loose(endpoint.name)+ ):+ touched.add(str(uuid))+ return bool(event_uuids - touched)+++# Re-export for callers/tests that previously imported align helpers here.+__all__ = [+ "BatchEndpoint",+ "BatchRelationship",+ "BatchExtractResponse",+ "BatchValidationIssue",+ "BatchValidationResult",+ "validate_batch_extract",+ "event_leg_incomplete",+ "ensure_event_descriptions",+ "normalize_loose",+ "find_span_offsets",+ "align_span",+]diff --git a/src/core/saving/architect_scratchpad.py b/src/core/saving/architect_scratchpad.pynew file mode 100644index 0000000..32f4c5e--- /dev/null+++ b/src/core/saving/architect_scratchpad.py@@ -0,0 +1,338 @@+"""Compact prior-state scratchpad for Architect batch extract.++Replaces replaying raw prior unit text with a token-capped summary of+entities, open event hubs, dates, and recent predicate/span pointers.+"""++from __future__ import annotations++from dataclasses import dataclass, field+from typing import Any, Optional, Sequence++from src.core.saving.ingest_cost import count_source_tokens++DEFAULT_SCRATCHPAD_TOKEN_CAP = 500+DEFAULT_PRIOR_UNIT_WINDOW = 4+_SPAN_POINTER_CHARS = 72+_DESC_CHARS = 60+_MAX_ENTITIES = 40+_MAX_HUBS = 16+_MAX_PREDICATES = 24+_MAX_DATES = 12+++@dataclass+class ScratchpadEntity:+ uuid: str+ name: str+ type: str+ happened_at: Optional[str] = None+++@dataclass+class ScratchpadEventHub:+ uuid: str+ name: str+ happened_at: Optional[str] = None+ description: Optional[str] = None+ actors: list[str] = field(default_factory=list)+ objects: list[str] = field(default_factory=list)+++@dataclass+class ScratchpadPredicate:+ predicate: str+ tail: str+ tip: str+ span_pointer: Optional[str] = None+++@dataclass+class ArchitectScratchpad:+ entities: list[ScratchpadEntity] = field(default_factory=list)+ event_hubs: list[ScratchpadEventHub] = field(default_factory=list)+ recent_predicates: list[ScratchpadPredicate] = field(default_factory=list)+ dates: list[str] = field(default_factory=list)+ raw_span_fetches: int = 0+++def _attr(obj: Any, key: str, default: Any = None) -> Any:+ if obj is None:+ return default+ if isinstance(obj, dict):+ return obj.get(key, default)+ return getattr(obj, key, default)+++def _truncate(text: Optional[str], limit: int) -> str:+ value = (text or "").strip()+ if len(value) <= limit:+ return value+ return value[: max(0, limit - 1)].rstrip() + "…"+++def _is_event(typ: Optional[str]) -> bool:+ return (typ or "").strip().lower() == "event"+++def _entity_from_any(obj: Any) -> Optional[ScratchpadEntity]:+ name = (_attr(obj, "name") or "").strip()+ typ = (_attr(obj, "type") or "").strip()+ if not name or not typ:+ return None+ if name.lower() == typ.lower():+ return None+ uuid_ = str(_attr(obj, "uuid") or "").strip()+ happened = _attr(obj, "happened_at")+ return ScratchpadEntity(+ uuid=uuid_,+ name=name,+ type=typ,+ happened_at=(str(happened).strip() if happened else None) or None,+ )+++def _span_pointer_from_rel(rel: Any) -> Optional[str]:+ props = _attr(rel, "properties") or {}+ if not isinstance(props, dict):+ props = {}+ span = props.get("source_span") or _attr(rel, "source_span") or ""+ span = str(span).strip()+ if not span:+ return None+ return _truncate(span, _SPAN_POINTER_CHARS)+++def build_scratchpad(+ prior_entities: Sequence[Any],+ prior_relationships: Sequence[Any] = (),+ *,+ max_entities: int = _MAX_ENTITIES,+ max_hubs: int = _MAX_HUBS,+ max_predicates: int = _MAX_PREDICATES,+) -> ArchitectScratchpad:+ """Build a compact prior-state scratchpad from Scout entities + Architect rels."""+ entities: list[ScratchpadEntity] = []+ seen_keys: set[tuple[str, str]] = set()+ seen_uuids: set[str] = set()+ dates: list[str] = []+ seen_dates: set[str] = set()++ for raw in prior_entities:+ ent = _entity_from_any(raw)+ if ent is None:+ continue+ key = (ent.name.lower(), ent.type.lower())+ if ent.uuid and ent.uuid in seen_uuids:+ continue+ if key in seen_keys:+ continue+ seen_keys.add(key)+ if ent.uuid:+ seen_uuids.add(ent.uuid)+ entities.append(ent)+ if ent.happened_at and ent.happened_at not in seen_dates:+ seen_dates.add(ent.happened_at)+ dates.append(ent.happened_at)++ entities = entities[-max_entities:]++ hubs_by_key: dict[str, ScratchpadEventHub] = {}+ predicates: list[ScratchpadPredicate] = []++ for rel in prior_relationships:+ tail = _entity_from_any(_attr(rel, "tail"))+ tip = _entity_from_any(_attr(rel, "tip"))+ pred = (_attr(rel, "name") or "").strip()+ if not pred or tail is None or tip is None:+ continue++ for endpoint in (tail, tip):+ if endpoint.happened_at and endpoint.happened_at not in seen_dates:+ seen_dates.add(endpoint.happened_at)+ dates.append(endpoint.happened_at)+ props = _attr(_attr(rel, "tail" if endpoint is tail else "tip"), "properties")+ if isinstance(props, dict):+ ha = props.get("happened_at")+ if ha and str(ha) not in seen_dates:+ seen_dates.add(str(ha))+ dates.append(str(ha))++ span_ptr = _span_pointer_from_rel(rel)+ predicates.append(+ ScratchpadPredicate(+ predicate=pred,+ tail=tail.name,+ tip=tip.name,+ span_pointer=span_ptr,+ )+ )++ for endpoint, other, role in (+ (tip, tail, "actor"),+ (tail, tip, "object"),+ ):+ if not _is_event(endpoint.type):+ continue+ hub_key = endpoint.uuid or f"{endpoint.name.lower()}|{endpoint.type.lower()}"+ hub = hubs_by_key.get(hub_key)+ if hub is None:+ desc = _attr(+ _attr(rel, "tip" if endpoint is tip else "tail"),+ "description",+ )+ hub = ScratchpadEventHub(+ uuid=endpoint.uuid,+ name=endpoint.name,+ happened_at=endpoint.happened_at,+ description=_truncate(desc, _DESC_CHARS) or None,+ )+ hubs_by_key[hub_key] = hub+ if role == "actor":+ if other.name not in hub.actors:+ hub.actors.append(other.name)+ else:+ if other.name not in hub.objects:+ hub.objects.append(other.name)++ predicates = predicates[-max_predicates:]+ hubs = list(hubs_by_key.values())[-max_hubs:]+ dates = dates[-_MAX_DATES:]++ return ArchitectScratchpad(+ entities=entities,+ event_hubs=hubs,+ recent_predicates=predicates,+ dates=dates,+ )+++def serialize_scratchpad(+ pad: ArchitectScratchpad,+ *,+ token_cap: int = DEFAULT_SCRATCHPAD_TOKEN_CAP,+) -> tuple[str, int]:+ """Serialize scratchpad under a reference-token cap. Returns (text, tokens)."""+ cap = max(32, int(token_cap or DEFAULT_SCRATCHPAD_TOKEN_CAP))+ sections: list[str] = []++ if pad.entities:+ lines = ["Entities:"]+ for ent in pad.entities:+ bits = [ent.name, ent.type]+ if ent.uuid:+ bits.append(ent.uuid[:8])+ if ent.happened_at:+ bits.append(f"@{ent.happened_at}")+ lines.append("- " + " | ".join(bits))+ sections.append("\n".join(lines))++ if pad.event_hubs:+ lines = ["Open event hubs:"]+ for hub in pad.event_hubs:+ head = hub.name+ if hub.happened_at:+ head += f" @{hub.happened_at}"+ detail_parts = []+ if hub.actors:+ detail_parts.append("actors=" + ", ".join(hub.actors[:4]))+ if hub.objects:+ detail_parts.append("objects=" + ", ".join(hub.objects[:4]))+ if hub.description:+ detail_parts.append(hub.description)+ line = f"- {head}"+ if detail_parts:+ line += " (" + "; ".join(detail_parts) + ")"+ lines.append(line)+ sections.append("\n".join(lines))++ if pad.dates:+ sections.append("Dates: " + ", ".join(pad.dates))++ if pad.recent_predicates:+ lines = ["Recent predicates:"]+ for pred in pad.recent_predicates:+ line = f"- {pred.tail} --{pred.predicate}--> {pred.tip}"+ if pred.span_pointer:+ line += f' [{pred.span_pointer}]'+ lines.append(line)+ sections.append("\n".join(lines))++ if not sections:+ return "", 0++ # Drop trailing sections until under cap (prefer keep entities + hubs).+ while sections:+ text = "\n".join(sections)+ tokens, _, _ = count_source_tokens(text)+ if tokens <= cap:+ return text, tokens+ if len(sections) == 1:+ # Hard-trim the last remaining section by lines.+ lines = sections[0].split("\n")+ while len(lines) > 1:+ lines.pop()+ text = "\n".join(lines)+ tokens, _, _ = count_source_tokens(text)+ if tokens <= cap:+ return text, tokens+ return "", 0+ sections.pop()++ return "", 0+++def fetch_prior_span(+ prior_chunks: Sequence[str],+ needle: str,+ *,+ window_chars: int = 200,+) -> Optional[str]:+ """On-demand: locate `needle` in prior unit text and return a small window."""+ query = (needle or "").strip()+ if not query or not prior_chunks:+ return None+ window = max(40, int(window_chars or 200))+ for chunk in reversed(list(prior_chunks)):+ text = chunk or ""+ idx = text.find(query)+ if idx < 0:+ idx = text.lower().find(query.lower())+ if idx < 0:+ continue+ start = max(0, idx - window // 4)+ end = min(len(text), idx + len(query) + window // 2)+ return text[start:end].strip()+ return None+++def format_unit_with_scratchpad(current_chunk: str, scratchpad_text: str) -> str:+ """Compose Architect unit prompt text: scratchpad preamble + current unit."""+ if not (scratchpad_text or "").strip():+ return current_chunk+ return (+ f"[Prior scratchpad]\n{scratchpad_text.strip()}\n\n"+ f"[Current unit]\n{current_chunk}"+ )+++def resolve_prior_context_mode(+ configured: str | None,+ architect_mode: str,+) -> str:+ """+ Resolve prior-context strategy.++ auto (default): scratchpad for batch/schema, raw for tooler.+ """+ value = (configured or "auto").strip().lower()+ if value in ("scratchpad", "raw"):+ return value+ if value in ("auto", ""):+ if (architect_mode or "").strip().lower() in ("batch", "schema"):+ return "scratchpad"+ return "raw"+ raise ValueError(+ f"Invalid INGEST_ARCHITECT_PRIOR_CONTEXT: {configured!r}. "+ "Expected 'auto', 'scratchpad', or 'raw'."+ )diff --git a/src/core/saving/auto_kg.py b/src/core/saving/auto_kg.pyindex 23ee6ff..14506b2 100644--- a/src/core/saving/auto_kg.py+++ b/src/core/saving/auto_kg.py@@ -15,9 +15,16 @@ from typing import List, Optionalfrom src.config import configfrom src.constants.kg import Node-from src.core.agents.scout_agent import ScoutAgent+from src.core.agents.scout_agent import ScoutAgent, ScoutEntityfrom src.core.agents.architect_agent import ArchitectAgent+from src.core.saving.architect_scratchpad import (+ build_scratchpad,+ format_unit_with_scratchpad,+ resolve_prior_context_mode,+ serialize_scratchpad,+)from src.core.saving.ingestion_manager import IngestionManager+from src.core.saving.ingest_cost import IngestCostLedger, track_stagefrom src.services.input.agents import (cache_adapter,embeddings_adapter,@@ -25,9 +32,12 @@ from src.services.input.agents import (llm_small_adapter,vector_store_adapter,)+from src.utils.text_chunking import chunk_textimport langsmith+_PRIOR_UNIT_WINDOW = 4+@dataclassclass EnrichmentOrchestrationResult:@@ -36,6 +46,94 @@ class EnrichmentOrchestrationResult:brain_id: strenrichment_relationships: List[dict] = field(default_factory=list)should_consolidate: bool = False+ cost: dict = field(default_factory=dict)+++def _entity_merge_key(entity: ScoutEntity) -> tuple:+ return (+ (entity.name or "").strip().lower(),+ (entity.type or "").strip().lower(),+ getattr(entity, "happened_at", None) or "",+ )+++def _merge_entities(chunks_entities: List[List[ScoutEntity]]) -> List[ScoutEntity]:+ merged: dict[tuple, ScoutEntity] = {}+ for entities in chunks_entities:+ for entity in entities:+ key = _entity_merge_key(entity)+ if key not in merged:+ merged[key] = entity+ else:+ existing = merged[key]+ if (entity.description or "") and len(entity.description or "") > len(+ existing.description or ""+ ):+ existing.description = entity.description+ return list(merged.values())+++def _architect_unit_text(chunk: str, prior_chunks: List[str]) -> str:+ if not prior_chunks:+ return chunk+ prior = prior_chunks[-_PRIOR_UNIT_WINDOW:]+ preamble = "\n\n".join(+ f"[Prior context {i + 1}]\n{p}" for i, p in enumerate(prior)+ )+ return f"{preamble}\n\n[Current unit]\n{chunk}"+++def _architect_unit_text_scratchpad(+ chunk: str,+ *,+ prior_entities: List[ScoutEntity],+ prior_relationships: list,+ token_cap: int,+) -> str:+ if not prior_entities and not prior_relationships:+ return chunk+ pad = build_scratchpad(prior_entities, prior_relationships)+ pad_text, _tokens = serialize_scratchpad(pad, token_cap=token_cap)+ return format_unit_with_scratchpad(chunk, pad_text)+++def _entity_name(entity: ScoutEntity) -> str:+ return (getattr(entity, "name", None) or "").strip()+++def _filter_entities_for_text(+ entities: List[ScoutEntity], text: str+) -> List[ScoutEntity]:+ """Keep entities whose names appear in the unit; fall back to all if none match."""+ if not entities or not text:+ return list(entities or [])+ hay = text.lower()+ matched = [e for e in entities if _entity_name(e) and _entity_name(e).lower() in hay]+ return matched or list(entities)+++def _dense_architect_parts(+ chunk: str,+ entities: List[ScoutEntity],+ *,+ entity_threshold: int,+ dense_max_chars: int,+) -> List[str]:+ """+ Split entity-dense dialogue units into smaller schema extracts so the batch+ path stays populated instead of empty→tooler escalate storms.+ """+ cleaned = (chunk or "").strip()+ if not cleaned:+ return [cleaned]+ threshold = max(1, int(entity_threshold or 12))+ max_chars = max(200, int(dense_max_chars or 1200))+ if len(entities) < threshold:+ return [cleaned]+ if len(cleaned) <= max_chars and cleaned.count("\n") < 4:+ return [cleaned]+ parts = chunk_text(cleaned, max_chars=max_chars)+ return parts if parts else [cleaned]def enrich_kg_from_input(@@ -89,6 +187,8 @@ def _enrich_kg_impl() -> EnrichmentOrchestrationResult:from src.core.saving.identity import stamp_provenance+ cost_ledger = IngestCostLedger()+ cost_ledger.set_source_text(input if isinstance(input, str) else str(input or ""))ingestion_manager = IngestionManager(embeddings_adapter, vector_store_adapter, graph_adapter)@@ -110,27 +210,138 @@ def _enrich_kg_impl()mode = "coarse" if config.pipeline_mode == "lightweight" else "granular"- print(f"[DEBUG (enrich_kg_from_input)]: {config.pipeline_mode} pipeline mode selected")-- entities = scout_agent.run(- input,- targeting=targeting,- brain_id=brain_id,- ingestion_session_id=ingestion_session_id,- mode=("coarse" if mode == "coarse" else "granular"),- reference_time=source_timestamp,- preferred_extraction_entities=preferred_extraction_entities,+ architect_mode = config.ingest_architect_mode+ prior_context_mode = resolve_prior_context_mode(+ config.ingest_architect_prior_context,+ architect_mode,)- print("[DEBUG (initial_scout_entities)]: ", entities.entities)- architect_agent.run_tooler(- input,- entities.entities,- targeting=targeting,- brain_id=brain_id,- timeout=20000,- ingestion_session_id=ingestion_session_id,- mode=("coarse" if mode == "coarse" else "granular"),+ scratchpad_token_cap = int(config.ingest_architect_scratchpad_token_cap or 500)+ print(+ f"[DEBUG (enrich_kg_from_input)]: {config.pipeline_mode} pipeline mode selected; "+ f"architect_mode={architect_mode}; prior_context={prior_context_mode}")+ architect_agent.defer_janitor = bool(config.ingest_defer_janitor)++ def _run_architect_unit(+ unit_text: str,+ unit_entities: List[ScoutEntity],+ *,+ reset: bool,+ ) -> None:+ if architect_mode in ("schema", "batch"):+ architect_agent.run_batch_extract(+ unit_text,+ unit_entities,+ targeting=targeting,+ brain_id=brain_id,+ timeout=20000,+ ingestion_session_id=ingestion_session_id,+ mode=("coarse" if mode == "coarse" else "granular"),+ reset=reset,+ cost_ledger=cost_ledger,+ escalate=bool(config.ingest_architect_escalate),+ )+ else:+ architect_agent.run_tooler(+ unit_text,+ unit_entities,+ targeting=targeting,+ brain_id=brain_id,+ timeout=20000,+ ingestion_session_id=ingestion_session_id,+ mode=("coarse" if mode == "coarse" else "granular"),+ reset=reset,+ )+ cost_ledger.record_architect_unit(escalated=False)++ if config.ingest_architect_per_unit:+ chunks = chunk_text(input, max_chars=6000)+ per_chunk_entities: List[List[ScoutEntity]] = []+ with track_stage(cost_ledger, "scout"):+ for chunk in chunks:+ response = scout_agent._run_chunk(+ chunk,+ targeting=targeting,+ brain_id=brain_id,+ ingestion_session_id=ingestion_session_id,+ mode=("coarse" if mode == "coarse" else "granular"),+ reference_time=source_timestamp,+ preferred_extraction_entities=preferred_extraction_entities,+ )+ per_chunk_entities.append(list(response.entities))++ entities = _merge_entities(per_chunk_entities)+ print("[DEBUG (initial_scout_entities)]: ", entities)++ with track_stage(cost_ledger, "architect"):+ first_unit = True+ for idx, chunk in enumerate(chunks):+ chunk_entities = (+ per_chunk_entities[idx] if idx < len(per_chunk_entities) else []+ )+ if not chunk_entities and not entities:+ continue+ unit_entities = chunk_entities if chunk_entities else entities+ architect_parts = _dense_architect_parts(+ chunk,+ unit_entities,+ entity_threshold=config.ingest_architect_dense_entity_threshold,+ dense_max_chars=config.ingest_architect_dense_max_chars,+ )+ if len(architect_parts) > 1:+ print(+ "[DEBUG (enrich_kg_from_input)]: dense architect rechunk "+ f"entities={len(unit_entities)} parts={len(architect_parts)} "+ f"max_chars={config.ingest_architect_dense_max_chars}"+ )+ for part in architect_parts:+ part_entities = _filter_entities_for_text(unit_entities, part)+ if prior_context_mode == "scratchpad" and not first_unit:+ window_start = max(0, idx - _PRIOR_UNIT_WINDOW)+ prior_ents: List[ScoutEntity] = []+ for prior_idx in range(window_start, idx):+ prior_ents.extend(per_chunk_entities[prior_idx])+ unit_text = _architect_unit_text_scratchpad(+ part,+ prior_entities=prior_ents,+ prior_relationships=list(+ getattr(architect_agent, "relationships_set", []) or []+ ),+ token_cap=scratchpad_token_cap,+ )+ elif prior_context_mode == "raw" and not first_unit:+ unit_text = _architect_unit_text(part, chunks[:idx])+ else:+ unit_text = part+ _run_architect_unit(+ unit_text, part_entities, reset=first_unit+ )+ first_unit = False+ else:+ with track_stage(cost_ledger, "scout"):+ scout_response = scout_agent.run(+ input,+ targeting=targeting,+ brain_id=brain_id,+ ingestion_session_id=ingestion_session_id,+ mode=("coarse" if mode == "coarse" else "granular"),+ reference_time=source_timestamp,+ preferred_extraction_entities=preferred_extraction_entities,+ )+ entities = list(scout_response.entities)+ print("[DEBUG (initial_scout_entities)]: ", entities)+ with track_stage(cost_ledger, "architect"):+ _run_architect_unit(input, entities, reset=True)++ if mode == "granular" and config.ingest_defer_janitor:+ with track_stage(cost_ledger, "janitor"):+ architect_agent.run_batched_janitor(+ text=input,+ brain_id=brain_id,+ targeting=targeting,+ batch_size=config.janitor_batch_size,+ cost_ledger=cost_ledger,+ )pending = architect_agent.take_pending_relationships()enrichment_relationships = []@@ -169,4 +380,5 @@ def _enrich_kg_impl(brain_id=brain_id,enrichment_relationships=enrichment_relationships,should_consolidate=should_consolidate,+ cost=cost_ledger.to_dict(),)diff --git a/src/core/saving/grounding.py b/src/core/saving/grounding.pynew file mode 100644index 0000000..76fb328--- /dev/null+++ b/src/core/saving/grounding.py@@ -0,0 +1,362 @@+from __future__ import annotations++import re+from dataclasses import dataclass, field+from typing import Any, Literal, Optional, Sequence+++_WHITESPACE_RE = re.compile(r"\s+")++GroundingVerdict = Literal["accept", "reject", "ambiguous"]+++def normalize_span(text: str) -> str:+ return _WHITESPACE_RE.sub(" ", (text or "").strip().lower())+++def extract_source_span(+ rel: Any,+ source_text: str,+ *,+ allow_synthetic: bool = True,+) -> str:+ props = getattr(rel, "properties", None) or {}+ if isinstance(props, dict):+ for key in ("source_span", "span", "evidence", "quote"):+ value = props.get(key)+ if isinstance(value, str) and value.strip():+ return value.strip()+ explicit = getattr(rel, "source_span", None)+ if isinstance(explicit, str) and explicit.strip():+ return explicit.strip()+ description = getattr(rel, "description", None) or ""+ if description and source_text:+ score, aligned = align_span(description, source_text)+ if score >= 0.55 and aligned:+ return aligned+ if description.strip():+ return description.strip()+ if not allow_synthetic:+ return ""+ name = getattr(rel, "name", None) or ""+ tail = getattr(getattr(rel, "tail", None), "name", "") or ""+ tip = getattr(getattr(rel, "tip", None), "name", "") or ""+ return " ".join(p for p in (tail, name, tip) if p).strip()+++def extract_explicit_evidence(rel: Any, source_text: str) -> str:+ """Evidence quote/description only — never invent from endpoint names."""+ return extract_source_span(rel, source_text, allow_synthetic=False)+++def align_span(span: str, source_text: str) -> tuple[float, str]:+ """+ Cheap string alignment: exact / containment / token Jaccard against windows.+ Returns (score in [0,1], best matching source window or "").+ """+ needle = normalize_span(span)+ hay = normalize_span(source_text)+ if not needle or not hay:+ return 0.0, ""+ if needle in hay:+ return 1.0, span.strip()+ needle_tokens = [t for t in needle.split(" ") if t]+ if not needle_tokens:+ return 0.0, ""+ window = max(len(needle_tokens) + 4, 8)+ hay_tokens = hay.split(" ")+ best_score = 0.0+ best_window = ""+ needle_set = set(needle_tokens)+ for i in range(0, max(1, len(hay_tokens) - window + 1)):+ chunk = hay_tokens[i : i + window]+ chunk_set = set(chunk)+ if not chunk_set:+ continue+ score = len(needle_set & chunk_set) / len(needle_set | chunk_set)+ if score > best_score:+ best_score = score+ best_window = " ".join(chunk)+ return best_score, best_window+++def find_span_offsets(+ span: str,+ source_text: str,+) -> Optional[tuple[int, int]]:+ """+ Locate `span` inside `source_text` with exact then whitespace-tolerant match.+ Returns (start, end) character offsets into the original source, or None.+ """+ text = source_text or ""+ needle = (span or "").strip()+ if not needle or not text:+ return None+ exact = text.find(needle)+ if exact >= 0:+ return exact, exact + len(needle)++ # Whitespace-tolerant: map normalized needle back onto original offsets.+ norm_needle = normalize_span(needle)+ if not norm_needle:+ return None+ lower = text.lower()+ # Build map from normalized index -> original index+ norm_chars: list[str] = []+ norm_to_orig: list[int] = []+ prev_space = False+ for i, ch in enumerate(lower):+ if ch.isspace():+ if norm_chars and not prev_space:+ norm_chars.append(" ")+ norm_to_orig.append(i)+ prev_space = True+ continue+ if not norm_chars and ch.isspace():+ continue+ norm_chars.append(ch)+ norm_to_orig.append(i)+ prev_space = False+ norm_hay = "".join(norm_chars).strip()+ # Trim leading spaces recorded above+ while norm_to_orig and norm_hay and norm_hay[0] == " ":+ norm_hay = norm_hay[1:]+ norm_to_orig = norm_to_orig[1:]+ idx = norm_hay.find(norm_needle)+ if idx < 0:+ return None+ start_orig = norm_to_orig[idx]+ end_idx = idx + len(norm_needle) - 1+ if end_idx >= len(norm_to_orig):+ return None+ end_orig = norm_to_orig[end_idx] + 1+ return start_orig, end_orig+++def realign_span_offsets(+ *,+ span: str,+ source_text: str,+ span_start: Optional[int] = None,+ span_end: Optional[int] = None,+ min_score: float = 0.45,+) -> tuple[bool, Optional[int], Optional[int], str]:+ """+ Validate or repair character offsets for a source span.++ Returns (ok, new_start, new_end, reason). On success with repaired or+ cleared offsets, reason is "". On failure, reason is a machine-readable code.+ """+ text = source_text or ""+ needle = (span or "").strip()+ if not needle:+ return False, span_start, span_end, "missing_source_span"++ score, _ = align_span(needle, text)+ if score < min_score:+ return False, span_start, span_end, "source_span_not_grounded"++ if span_start is None or span_end is None:+ found = find_span_offsets(needle, text)+ if found:+ return True, found[0], found[1], ""+ return True, None, None, ""++ if span_start < 0 or span_end > len(text) or span_start >= span_end:+ found = find_span_offsets(needle, text)+ if found:+ return True, found[0], found[1], ""+ # Offsets unusable but span text is grounded — clear offsets.+ return True, None, None, ""++ sliced = text[span_start:span_end]+ if sliced.strip() and normalize_span(sliced) == normalize_span(needle):+ return True, span_start, span_end, ""++ slice_score, _ = align_span(needle, sliced)+ if slice_score >= min_score:+ return True, span_start, span_end, ""++ found = find_span_offsets(needle, text)+ if found:+ return True, found[0], found[1], ""++ # Soften: keep the grounded quote, drop bad offsets instead of rejecting.+ return True, None, None, ""+++def relationship_looks_grounded(+ rel: Any,+ source_text: str,+ *,+ min_score: float = 0.45,+) -> tuple[bool, float, str]:+ span = extract_explicit_evidence(rel, source_text)+ score, aligned = align_span(span, source_text)+ if not span:+ return False, 0.0, ""+ return score >= min_score, score, aligned or span+++def endpoints_well_formed(rel: Any) -> bool:+ for endpoint in (getattr(rel, "tail", None), getattr(rel, "tip", None)):+ if endpoint is None:+ return False+ name = (getattr(endpoint, "name", None) or "").strip()+ typ = (getattr(endpoint, "type", None) or "").strip()+ if not name or not typ:+ return False+ if name.lower() == typ.lower():+ return False+ predicate = (getattr(rel, "name", None) or "").strip()+ return bool(predicate)+++def _endpoint_is_type_named(endpoint: Any) -> bool:+ if endpoint is None:+ return True+ name = (getattr(endpoint, "name", None) or "").strip()+ typ = (getattr(endpoint, "type", None) or "").strip()+ if not name or not typ:+ return True+ return name.lower() == typ.lower()+++@dataclass+class GroundingDecision:+ decision: GroundingVerdict+ reason: str+ score: float = 0.0+ span: str = ""+ relationship: Any = None+++@dataclass+class JanitorTriageResult:+ accept: list[Any] = field(default_factory=list)+ reject: list[GroundingDecision] = field(default_factory=list)+ ambiguous: list[Any] = field(default_factory=list)+++def decide_relationship_grounding(+ rel: Any,+ source_text: str,+ *,+ min_score: float = 0.45,+ reject_below: float = 0.15,+) -> GroundingDecision:+ """+ Classify one relationship as accept / reject / ambiguous for Janitor triage.+ """+ if _endpoint_is_type_named(getattr(rel, "tail", None)) or _endpoint_is_type_named(+ getattr(rel, "tip", None)+ ):+ return GroundingDecision(+ decision="reject",+ reason="type_named_placeholder_endpoint",+ relationship=rel,+ )+ if not endpoints_well_formed(rel):+ return GroundingDecision(+ decision="reject",+ reason="malformed_endpoints",+ relationship=rel,+ )+ predicate = (getattr(rel, "name", None) or "").strip()+ if not predicate:+ return GroundingDecision(+ decision="reject",+ reason="missing_predicate",+ relationship=rel,+ )++ span = extract_explicit_evidence(rel, source_text)+ score, aligned = align_span(span, source_text) if span else (0.0, "")+ if not span:+ return GroundingDecision(+ decision="ambiguous",+ reason="missing_source_span",+ score=0.0,+ relationship=rel,+ )+ if score >= min_score:+ if isinstance(getattr(rel, "properties", None), dict) and aligned:+ props = rel.properties or {}+ if not props.get("source_span"):+ try:+ rel.properties = {+ **props,+ "source_span": aligned,+ "grounding_score": score,+ }+ except Exception:+ pass+ return GroundingDecision(+ decision="accept",+ reason="grounded_endpoints_ok",+ score=score,+ span=aligned or span,+ relationship=rel,+ )+ if score < reject_below:+ return GroundingDecision(+ decision="reject",+ reason="source_span_not_grounded",+ score=score,+ span=span,+ relationship=rel,+ )+ return GroundingDecision(+ decision="ambiguous",+ reason="weak_grounding",+ score=score,+ span=span,+ relationship=rel,+ )+++def triage_relationships_for_janitor(+ relationships: Sequence[Any],+ source_text: str,+ *,+ min_grounding: float = 0.45,+ reject_below: float = 0.15,+) -> JanitorTriageResult:+ """+ Split relationships into accept (skip LLM), reject (drop), ambiguous (LLM).+ """+ result = JanitorTriageResult()+ for rel in relationships:+ decision = decide_relationship_grounding(+ rel,+ source_text,+ min_score=min_grounding,+ reject_below=reject_below,+ )+ if decision.decision == "accept":+ result.accept.append(rel)+ elif decision.decision == "reject":+ result.reject.append(decision)+ else:+ result.ambiguous.append(rel)+ return result+++def cheap_janitor_precheck(+ relationships: Sequence[Any],+ source_text: str,+ *,+ min_grounding: float = 0.45,+) -> tuple[list[Any], list[Any]]:+ """+ Split relationships into (skip_janitor, need_janitor).++ Rejects from triage are folded into need_janitor for backward compatibility;+ prefer triage_relationships_for_janitor for explicit drop auditing.+ """+ triage = triage_relationships_for_janitor(+ relationships, source_text, min_grounding=min_grounding+ )+ need = list(triage.ambiguous)+ need.extend(d.relationship for d in triage.reject if d.relationship is not None)+ return list(triage.accept), needdiff --git a/src/core/saving/ingest_cost.py b/src/core/saving/ingest_cost.pynew file mode 100644index 0000000..442bdd1--- /dev/null+++ b/src/core/saving/ingest_cost.py@@ -0,0 +1,414 @@+from __future__ import annotations++import time+from contextlib import contextmanager+from contextvars import ContextVar+from dataclasses import dataclass, field+from typing import Any, Iterator, Optional++SOURCE_TOKENIZER_ID = "tiktoken/cl100k_base"+_SOURCE_ENCODING = None+_SOURCE_ENCODING_FAILED = False+++def _get_cl100k_encoding():+ global _SOURCE_ENCODING, _SOURCE_ENCODING_FAILED+ if _SOURCE_ENCODING is not None:+ return _SOURCE_ENCODING+ if _SOURCE_ENCODING_FAILED:+ return None+ try:+ import tiktoken++ _SOURCE_ENCODING = tiktoken.get_encoding("cl100k_base")+ return _SOURCE_ENCODING+ except Exception:+ _SOURCE_ENCODING_FAILED = True+ return None+++def count_source_tokens(text: str) -> tuple[int, str, bool]:+ """+ Count source tokens with the pinned cross-provider tokenizer.++ Returns (token_count, tokenizer_id, estimated).+ """+ if not text:+ return 0, SOURCE_TOKENIZER_ID, False+ enc = _get_cl100k_encoding()+ if enc is not None:+ return len(enc.encode(text)), SOURCE_TOKENIZER_ID, False+ return max(1, len(text) // 4), "estimate/chars_div_4", True+++@dataclass+class StageCost:+ input_tokens: int = 0+ output_tokens: int = 0+ cached_tokens: int = 0+ latency_ms: float = 0.0+ calls: int = 0++ def add_usage(+ self,+ *,+ input_tokens: int = 0,+ output_tokens: int = 0,+ cached_tokens: int = 0,+ latency_ms: float = 0.0,+ calls: int = 1,+ ) -> None:+ self.input_tokens += int(input_tokens or 0)+ self.output_tokens += int(output_tokens or 0)+ self.cached_tokens += int(cached_tokens or 0)+ self.latency_ms += float(latency_ms or 0.0)+ self.calls += int(calls or 0)++ def merge(self, other: Optional["StageCost"]) -> None:+ if other is None:+ return+ self.add_usage(+ input_tokens=other.input_tokens,+ output_tokens=other.output_tokens,+ cached_tokens=other.cached_tokens,+ latency_ms=other.latency_ms,+ calls=other.calls,+ )++ @property+ def total_tokens(self) -> int:+ return self.input_tokens + self.output_tokens++ def to_dict(self) -> dict[str, Any]:+ return {+ "input_tokens": self.input_tokens,+ "output_tokens": self.output_tokens,+ "cached_tokens": self.cached_tokens,+ "total_tokens": self.total_tokens,+ "latency_ms": round(self.latency_ms, 3),+ "calls": self.calls,+ }+++@dataclass+class IngestCostLedger:+ scout: StageCost = field(default_factory=StageCost)+ architect: StageCost = field(default_factory=StageCost)+ janitor: StageCost = field(default_factory=StageCost)+ observations: StageCost = field(default_factory=StageCost)+ consolidation: StageCost = field(default_factory=StageCost)+ embed: StageCost = field(default_factory=StageCost)+ index_rebuild: StageCost = field(default_factory=StageCost)+ janitor_skipped: int = 0+ janitor_ran: int = 0+ janitor_rejected: int = 0+ janitor_ambiguous: int = 0+ janitor_drop_reasons: list[str] = field(default_factory=list)+ er_adjudications: int = 0+ source_chars: int = 0+ source_tokens: int = 0+ source_tokenizer: str = SOURCE_TOKENIZER_ID+ source_tokens_estimated: bool = False+ architect_units: int = 0+ architect_escalations: int = 0+ architect_schema_calls: int = 0+ architect_repair_calls: int = 0+ escalate_reasons: list[str] = field(default_factory=list)++ def stage(self, name: str) -> StageCost:+ return getattr(self, name)++ def set_source_text(self, text: str) -> None:+ self.source_chars = len(text or "")+ tokens, tokenizer_id, estimated = count_source_tokens(text or "")+ self.source_tokens = tokens+ self.source_tokenizer = tokenizer_id+ self.source_tokens_estimated = estimated++ def record_architect_unit(+ self,+ *,+ escalated: bool = False,+ reason: Optional[str] = None,+ schema_calls: int = 0,+ repair_calls: int = 0,+ ) -> None:+ self.architect_units += 1+ self.architect_schema_calls += int(schema_calls or 0)+ self.architect_repair_calls += int(repair_calls or 0)+ if escalated:+ self.architect_escalations += 1+ if reason:+ self.escalate_reasons.append(reason)++ @property+ def llm_source_multiplier(self) -> Optional[float]:+ if self.source_tokens <= 0:+ return None+ return round(self.total_llm_tokens / float(self.source_tokens), 3)++ @property+ def escalate_rate(self) -> Optional[float]:+ if self.architect_units <= 0:+ return None+ return round(self.architect_escalations / float(self.architect_units), 4)++ @property+ def janitor_skip_rate(self) -> Optional[float]:+ """Share of edges auto-accepted without LLM Janitor."""+ if self.janitor_ambiguous or self.janitor_rejected:+ total = (+ self.janitor_skipped+ + self.janitor_rejected+ + self.janitor_ambiguous+ )+ else:+ total = self.janitor_skipped + self.janitor_ran+ if total <= 0:+ return None+ return round(self.janitor_skipped / float(total), 4)++ @property+ def total_llm_tokens(self) -> int:+ return (+ self.scout.total_tokens+ + self.architect.total_tokens+ + self.janitor.total_tokens+ + self.observations.total_tokens+ + self.consolidation.total_tokens+ )++ def merge(self, other: Optional["IngestCostLedger"]) -> None:+ if other is None:+ return+ self.scout.merge(other.scout)+ self.architect.merge(other.architect)+ self.janitor.merge(other.janitor)+ self.observations.merge(other.observations)+ self.consolidation.merge(other.consolidation)+ self.embed.merge(other.embed)+ self.index_rebuild.merge(other.index_rebuild)+ self.janitor_skipped += other.janitor_skipped+ self.janitor_ran += other.janitor_ran+ self.janitor_rejected += other.janitor_rejected+ self.janitor_ambiguous += other.janitor_ambiguous+ self.janitor_drop_reasons.extend(other.janitor_drop_reasons)+ self.er_adjudications += other.er_adjudications+ self.source_chars = max(self.source_chars, other.source_chars)+ if other.source_tokens and (+ not self.source_tokens or other.source_tokens > self.source_tokens+ ):+ self.source_tokens = other.source_tokens+ self.source_tokenizer = other.source_tokenizer+ self.source_tokens_estimated = other.source_tokens_estimated+ self.architect_units += other.architect_units+ self.architect_escalations += other.architect_escalations+ self.architect_schema_calls += other.architect_schema_calls+ self.architect_repair_calls += other.architect_repair_calls+ self.escalate_reasons.extend(other.escalate_reasons)++ def to_dict(self) -> dict[str, Any]:+ stages = {+ "scout": self.scout.to_dict(),+ "architect": self.architect.to_dict(),+ "janitor": self.janitor.to_dict(),+ "observations": self.observations.to_dict(),+ "consolidation": self.consolidation.to_dict(),+ "embed": self.embed.to_dict(),+ "index_rebuild": self.index_rebuild.to_dict(),+ }+ total_in = sum(s["input_tokens"] for s in stages.values())+ total_out = sum(s["output_tokens"] for s in stages.values())+ total_cached = sum(s["cached_tokens"] for s in stages.values())+ multiplier = self.llm_source_multiplier+ return {+ **{f"{k}_{field}": v[field] for k, v in stages.items() for field in v},+ "stages": stages,+ "total_input_tokens": total_in,+ "total_output_tokens": total_out,+ "total_cached_tokens": total_cached,+ "total_llm_tokens": total_in + total_out,+ "janitor_skipped": self.janitor_skipped,+ "janitor_ran": self.janitor_ran,+ "janitor_rejected": self.janitor_rejected,+ "janitor_ambiguous": self.janitor_ambiguous,+ "janitor_drop_reasons": list(self.janitor_drop_reasons),+ "janitor_skip_rate": self.janitor_skip_rate,+ "er_adjudications": self.er_adjudications,+ "source_chars": self.source_chars,+ "source_tokens": self.source_tokens,+ "source_tokenizer": self.source_tokenizer,+ "source_tokens_estimated": self.source_tokens_estimated,+ "llm_source_multiplier": multiplier,+ "architect_units": self.architect_units,+ "architect_escalations": self.architect_escalations,+ "architect_schema_calls": self.architect_schema_calls,+ "architect_repair_calls": self.architect_repair_calls,+ "escalate_rate": self.escalate_rate,+ "escalate_reasons": list(self.escalate_reasons),+ "calls_per_unit": (+ round(self.architect.calls / float(self.architect_units), 3)+ if self.architect_units+ else None+ ),+ }++ @classmethod+ def from_dict(cls, data: Optional[dict[str, Any]]) -> "IngestCostLedger":+ ledger = cls()+ if not data:+ return ledger+ stages = data.get("stages") or {}+ for name in (+ "scout",+ "architect",+ "janitor",+ "observations",+ "consolidation",+ "embed",+ "index_rebuild",+ ):+ raw = stages.get(name) or {}+ stage = getattr(ledger, name)+ stage.input_tokens = int(raw.get("input_tokens") or 0)+ stage.output_tokens = int(raw.get("output_tokens") or 0)+ stage.cached_tokens = int(raw.get("cached_tokens") or 0)+ stage.latency_ms = float(raw.get("latency_ms") or 0.0)+ stage.calls = int(raw.get("calls") or 0)+ ledger.janitor_skipped = int(data.get("janitor_skipped") or 0)+ ledger.janitor_ran = int(data.get("janitor_ran") or 0)+ ledger.janitor_rejected = int(data.get("janitor_rejected") or 0)+ ledger.janitor_ambiguous = int(data.get("janitor_ambiguous") or 0)+ ledger.janitor_drop_reasons = list(data.get("janitor_drop_reasons") or [])+ ledger.er_adjudications = int(data.get("er_adjudications") or 0)+ ledger.source_chars = int(data.get("source_chars") or 0)+ ledger.source_tokens = int(data.get("source_tokens") or 0)+ ledger.source_tokenizer = str(+ data.get("source_tokenizer") or SOURCE_TOKENIZER_ID+ )+ ledger.source_tokens_estimated = bool(+ data.get("source_tokens_estimated") or False+ )+ ledger.architect_units = int(data.get("architect_units") or 0)+ ledger.architect_escalations = int(data.get("architect_escalations") or 0)+ ledger.architect_schema_calls = int(data.get("architect_schema_calls") or 0)+ ledger.architect_repair_calls = int(data.get("architect_repair_calls") or 0)+ reasons = data.get("escalate_reasons") or []+ if isinstance(reasons, list):+ ledger.escalate_reasons = [str(r) for r in reasons]+ return ledger+++_active_stage: ContextVar[Optional[StageCost]] = ContextVar(+ "ingest_cost_active_stage", default=None+)+_active_ledger: ContextVar[Optional[IngestCostLedger]] = ContextVar(+ "ingest_cost_active_ledger", default=None+)+++def get_active_ledger() -> Optional[IngestCostLedger]:+ return _active_ledger.get()+++def get_active_stage() -> Optional[StageCost]:+ return _active_stage.get()+++def record_usage_from_response(response: Any, latency_ms: float = 0.0) -> None:+ stage = _active_stage.get()+ if stage is None:+ return+ usage = getattr(response, "usage_metadata", None)+ if usage is None and isinstance(response, dict):+ usage = response.get("usage_metadata")+ if not usage and isinstance(response, dict):+ rm = response.get("response_metadata") or {}+ if isinstance(rm, dict):+ usage = rm.get("usage_metadata") or rm.get("token_usage")+ if not usage:+ rm = getattr(response, "response_metadata", None) or {}+ if isinstance(rm, dict):+ usage = rm.get("usage_metadata") or rm.get("token_usage")+ if not usage:+ return+ if not isinstance(usage, dict):+ usage = {+ "input_tokens": getattr(usage, "input_tokens", None)+ or getattr(usage, "prompt_tokens", 0),+ "output_tokens": getattr(usage, "output_tokens", None)+ or getattr(usage, "completion_tokens", 0),+ }+ details = usage.get("input_token_details") or {}+ cached = (+ details.get("cache_read", 0)+ or details.get("cached_tokens", 0)+ or usage.get("cached_tokens", 0)+ or 0+ )+ input_tokens = int(+ usage.get("input_tokens")+ or usage.get("prompt_tokens")+ or 0+ )+ output_tokens = int(+ usage.get("output_tokens")+ or usage.get("completion_tokens")+ or 0+ )+ stage.add_usage(+ input_tokens=input_tokens,+ output_tokens=output_tokens,+ cached_tokens=int(cached or 0),+ latency_ms=latency_ms,+ calls=1,+ )+++def submit_with_context(executor, fn, *args, **kwargs):+ """Propagate ContextVars (ingest cost stage) into ThreadPoolExecutor workers."""+ import contextvars++ ctx = contextvars.copy_context()+ return executor.submit(ctx.run, fn, *args, **kwargs)+++def estimate_tokens(text: str) -> int:+ """Legacy char/4 estimate. Prefer count_source_tokens for ledger denominators."""+ if not text:+ return 0+ return max(1, len(text) // 4)+++@contextmanager+def track_stage(ledger: IngestCostLedger, name: str) -> Iterator[StageCost]:+ stage = ledger.stage(name)+ token_stage = _active_stage.set(stage)+ token_ledger = _active_ledger.set(ledger)+ started = time.perf_counter()+ try:+ yield stage+ finally:+ stage.latency_ms += (time.perf_counter() - started) * 1000.0+ _active_stage.reset(token_stage)+ _active_ledger.reset(token_ledger)+++@contextmanager+def bind_ledger(ledger: IngestCostLedger) -> Iterator[IngestCostLedger]:+ token = _active_ledger.set(ledger)+ try:+ yield ledger+ finally:+ _active_ledger.reset(token)+++def merge_cost_into_status_payload(+ existing: dict[str, Any], ledger: Optional[IngestCostLedger]+) -> dict[str, Any]:+ if ledger is None:+ return existing+ prior = IngestCostLedger.from_dict(existing.get("cost"))+ prior.merge(ledger)+ existing = {**existing, "cost": prior.to_dict()}+ return existingdiff --git a/src/utils/text_chunking.py b/src/utils/text_chunking.pyindex 5282c4e..c4961a6 100644--- a/src/utils/text_chunking.py+++ b/src/utils/text_chunking.py@@ -2,23 +2,37 @@ def chunk_text(text: str, max_chars: int = 6000) -> list[str]:cleaned = (text or "").strip()if not cleaned or len(cleaned) <= max_chars:return [cleaned] if cleaned else [""]- paragraphs = [p.strip() for p in cleaned.split("\n\n") if p.strip()]- if not paragraphs:- paragraphs = [cleaned]++ # Prefer blank-line paragraphs, then single newlines (dialogue turns),+ # else hard character slices.+ if "\n\n" in cleaned:+ parts = [p.strip() for p in cleaned.split("\n\n") if p.strip()]+ joiner = "\n\n"+ elif "\n" in cleaned:+ parts = [p.strip() for p in cleaned.split("\n") if p.strip()]+ joiner = "\n"+ else:+ parts = [cleaned]+ joiner = "\n"++ if not parts:+ parts = [cleaned]+chunks: list[str] = []current = ""- for paragraph in paragraphs:- candidate = f"{current}\n\n{paragraph}".strip() if current else paragraph+ for part in parts:+ candidate = f"{current}{joiner}{part}".strip() if current else partif len(candidate) <= max_chars:current = candidatecontinueif current:chunks.append(current)- if len(paragraph) <= max_chars:- current = paragraph+ current = ""+ if len(part) <= max_chars:+ current = partcontinue- for i in range(0, len(paragraph), max_chars):- piece = paragraph[i : i + max_chars].strip()+ for i in range(0, len(part), max_chars):+ piece = part[i : i + max_chars].strip()if piece:chunks.append(piece)current = ""diff --git a/src/workers/tasks/ingestion.py b/src/workers/tasks/ingestion.pyindex 017be65..06e176d 100644--- a/src/workers/tasks/ingestion.py+++ b/src/workers/tasks/ingestion.py@@ -106,7 +106,10 @@ def set_ingestion_task_status(error: Optional[str] = None,errors: Optional[list] = None,counts: Optional[dict] = None,+ cost: Optional[dict] = None,) -> dict:+ from src.core.saving.ingest_cost import IngestCostLedger, merge_cost_into_status_payload+key = f"task:{task_id}"existing_raw = cache_adapter.get(key, brain_id=brain_id)existing: dict = {}@@ -145,6 +148,10 @@ def set_ingestion_task_status(}for item in errors[:50]]+ if cost is not None:+ payload = merge_cost_into_status_payload(+ payload, IngestCostLedger.from_dict(cost)+ )cache_adapter.set(key=key,value=json.dumps(payload),@@ -268,7 +275,7 @@ def _resolve_relationship_entities(except Exception:return Noneentity_type = (entity.type or "").strip().lower()- best = None+ ranked = []for candidate in candidates:metadata = candidate.metadata or {}candidate_uuid = metadata.get("uuid")@@ -285,14 +292,40 @@ def _resolve_relationship_entities(similarity = cosine_similarity(embedding, candidate_vectors[0].embeddings)if similarity < NODE_RESOLUTION_SIMILARITY:continue- if best is not None and similarity < best[0]:- continue- if best is not None and abs(similarity - best[0]) < 0.02:- return Nonenode = graph_adapter.get_by_uuid(candidate_uuid, brain_id=brain_id)if node:- best = (similarity, node)- return best[1] if best else None+ ranked.append((similarity, node))+ if not ranked:+ return None+ ranked.sort(key=lambda item: item[0], reverse=True)+ if len(ranked) >= 2 and abs(ranked[0][0] - ranked[1][0]) < 0.02:+ pool_nodes = [node for _, node in ranked[:5]]+ try:+ from src.core.agents.kg_agent import KGAgent++ kg_agent = KGAgent(+ llm_adapter=llm_small_adapter,+ cache_adapter=cache_adapter,+ kg=graph_adapter,+ vector_store=vector_store_adapter,+ embeddings=embeddings_adapter,+ database_desc=graph_adapter.graphdb_description,+ )+ adjudication = kg_agent.verify_entity_existence(+ entity_name=entity.name,+ entity_types=[entity.type] if entity.type else [],+ entity_meta_description=getattr(entity, "description", None),+ pool_nodes=pool_nodes,+ brain_id=brain_id,+ )+ if getattr(adjudication, "exists", False) and getattr(+ adjudication, "node", None+ ):+ return adjudication.node+ except Exception as exc:+ print(f"[!] Entity adjudication failed for {entity.name}: {exc}")+ return None+ return ranked[0][1]resolutions: dict = {}for key, entity in unique_entities.items():@@ -617,12 +650,24 @@ def ingest_data(self, args: dict):if config.pipeline_mode == "lightweight":print("[DEBUG (ingest_data)]: Lightweight pipeline mode selected")- if config.pipeline_mode == "accurate":- observations = observations_agent.observe(- text=source_text,- observate_for=payload.observate_for,- context=None,- )+ from src.core.saving.ingest_cost import IngestCostLedger, estimate_tokens, track_stage++ cost_ledger = IngestCostLedger()+ with track_stage(cost_ledger, "embed"):+ cost_ledger.embed.add_usage(calls=1)++ if config.pipeline_mode == "accurate" and config.run_observations:+ with track_stage(cost_ledger, "observations"):+ observations = observations_agent.observe(+ text=source_text,+ observate_for=payload.observate_for,+ context=None,+ )+ cost_ledger.observations.add_usage(+ input_tokens=estimate_tokens(source_text),+ output_tokens=estimate_tokens("\n".join(observations or [])),+ calls=1,+ )data_adapter.save_observations([@@ -643,7 +688,17 @@ def ingest_data(self, args: dict):source_timestamp=payload.source_timestamp,preferred_extraction_entities=payload.preferred_extraction_entities,)+ cost_ledger.merge(+ IngestCostLedger.from_dict(getattr(enrich_result, "cost", None))+ )parent_task_id = self.request.id+ set_ingestion_task_status(+ parent_task_id,+ payload.brain_id,+ "started",+ stage="enriched",+ cost=cost_ledger.to_dict(),+ )steps = []if enrich_result.enrichment_relationships:set_ingestion_task_status(@@ -654,6 +709,7 @@ def ingest_data(self, args: dict):counts={"relationships": len(enrich_result.enrichment_relationships),},+ cost=cost_ledger.to_dict(),)steps.append(process_architect_relationships.si(@@ -671,6 +727,7 @@ def ingest_data(self, args: dict):payload.brain_id,"consolidating",stage="consolidating",+ cost=cost_ledger.to_dict(),)steps.append(consolidate_graph_async.si(@@ -685,6 +742,10 @@ def ingest_data(self, args: dict):parent_task_id,payload.brain_id,"completed",+ None,+ None,+ None,+ cost_ledger.to_dict(),))from celery import chain@@ -713,7 +774,12 @@ def finalize_ingestion_task(error: Optional[str] = None,errors: Optional[list] = None,counts: Optional[dict] = None,+ cost: Optional[dict] = None,):+ try:+ graph_adapter.rebuild_topic_index(brain_id)+ except Exception as e:+ print(f"[!] Topic index rebuild failed: {e}")set_ingestion_task_status(parent_task_id,brain_id,@@ -722,6 +788,7 @@ def finalize_ingestion_task(error=error,errors=errors,counts=counts,+ cost=cost,)return parent_task_id@@ -1036,13 +1103,36 @@ def process_architect_relationships(self, args: dict):)try:- graph_adapter.rebuild_hub_bridge_index(brain_id)+ import time as _time++ touched_entities: set[str] = set()+ for relationship in relationships:+ for endpoint in (relationship.tail, relationship.tip):+ typ = (getattr(endpoint, "type", None) or "").strip().upper()+ if typ and typ != "EVENT" and getattr(endpoint, "uuid", None):+ touched_entities.add(str(endpoint.uuid))+ started = _time.perf_counter()+ if touched_entities:+ graph_adapter.refresh_hub_bridges_for_entities(+ list(touched_entities), brain_id=brain_id+ )+ else:+ graph_adapter.rebuild_hub_bridge_index(brain_id)+ index_ms = (_time.perf_counter() - started) * 1000.0+ if parent_task_id:+ from src.core.saving.ingest_cost import IngestCostLedger++ ledger = IngestCostLedger()+ ledger.index_rebuild.add_usage(latency_ms=index_ms, calls=1)+ set_ingestion_task_status(+ parent_task_id,+ brain_id,+ "persisting",+ stage="indexing",+ cost=ledger.to_dict(),+ )except Exception as e:print(f"[!] Hub bridge index rebuild failed: {e}")- try:- graph_adapter.rebuild_topic_index(brain_id)- except Exception as e:- print(f"[!] Topic index rebuild failed: {e}")if session_id:from src.lib.redis.client import _redis_clientdiff --git a/tests/test_architect_batch_extract.py b/tests/test_architect_batch_extract.pynew file mode 100644index 0000000..3b801c1--- /dev/null+++ b/tests/test_architect_batch_extract.py@@ -0,0 +1,274 @@+import os++ENV_DEFAULTS = {+ "BRAINPAT_TOKEN": "test-token",+ "MODELS_MODE": "local",+ "PIPELINE_MODE": "accurate",+ "EMBEDDINGS_LOCAL_MODEL": "local-model",+ "EMBEDDINGS_SMALL_MODEL": "small-model",+ "EMBEDDING_NODES_DIMENSION": "3",+ "EMBEDDING_TRIPLETS_DIMENSION": "3",+ "EMBEDDING_OBSERVATIONS_DIMENSION": "3",+ "EMBEDDING_DATA_DIMENSION": "3",+ "EMBEDDING_RELATIONSHIPS_DIMENSION": "3",+ "REDIS_HOST": "localhost",+ "REDIS_PORT": "6379",+ "NEO4J_HOST": "localhost",+ "NEO4J_PORT": "7687",+ "NEO4J_USERNAME": "neo4j",+ "NEO4J_PASSWORD": "password",+ "MILVUS_HOST": "localhost",+ "MILVUS_PORT": "19530",+ "MONGO_CONNECTION_STRING": "mongodb://localhost:27017",+ "CELERY_WORKER_CONCURRENCY": "1",+ "OLLAMA_HOST": "localhost",+ "OLLAMA_PORT": "11434",+ "OLLAMA_LLM_SMALL_MODEL": "small",+ "OLLAMA_LLM_LARGE_MODEL": "large",+}+for key, value in ENV_DEFAULTS.items():+ os.environ.setdefault(key, value)++from unittest.mock import MagicMock, patch++from src.core.agents.architect_agent import ArchitectAgent+from src.core.agents.scout_agent import ScoutEntity+from src.core.saving.architect_batch import (+ BatchEndpoint,+ BatchExtractResponse,+ BatchRelationship,+)+from src.core.saving.ingest_cost import IngestCostLedger+++def test_run_batch_extract_queues_valid_without_escalate():+ agent = ArchitectAgent(+ llm_adapter=MagicMock(),+ cache_adapter=MagicMock(),+ kg=MagicMock(),+ vector_store=MagicMock(),+ embeddings=MagicMock(),+ ingestion_manager=MagicMock(),+ )+ source = "Alice hired Bob as a contractor."+ entities = [+ ScoutEntity(uuid="u1", name="Alice", type="PERSON"),+ ScoutEntity(uuid="u2", name="Hiring", type="EVENT"),+ ScoutEntity(uuid="u3", name="Bob", type="PERSON"),+ ]+ payload = BatchExtractResponse(+ relationships=[+ BatchRelationship(+ tail=BatchEndpoint(uuid="u1", name="Alice", type="PERSON"),+ tip=BatchEndpoint(uuid="u2", name="Hiring", type="EVENT"),+ name="MADE",+ source_span="Alice hired Bob as a contractor",+ ),+ BatchRelationship(+ tail=BatchEndpoint(uuid="u2", name="Hiring", type="EVENT"),+ tip=BatchEndpoint(uuid="u3", name="Bob", type="PERSON"),+ name="TARGETED",+ source_span="Alice hired Bob as a contractor",+ ),+ ]+ )++ with patch.object(agent, "_get_agent"), patch.object(+ agent, "run_tooler"+ ) as tooler:+ agent.agent = MagicMock()+ agent.agent.invoke.return_value = {+ "structured_response": payload,+ "messages": [],+ }+ ledger = IngestCostLedger()+ rels = agent.run_batch_extract(+ source,+ entities,+ cost_ledger=ledger,+ escalate=True,+ timeout=5,+ )+ tooler.assert_not_called()+ assert len(rels) == 2+ assert ledger.architect_units == 1+ assert ledger.architect_escalations == 0+ assert ledger.architect_schema_calls == 1+++def test_run_batch_extract_repairs_then_escalates_on_failure():+ agent = ArchitectAgent(+ llm_adapter=MagicMock(),+ cache_adapter=MagicMock(),+ kg=MagicMock(),+ vector_store=MagicMock(),+ embeddings=MagicMock(),+ ingestion_manager=MagicMock(),+ )+ source = "Alice met Bob."+ entities = [+ ScoutEntity(uuid="u1", name="Alice", type="PERSON"),+ ScoutEntity(uuid="u2", name="Bob", type="PERSON"),+ ]+ bad = BatchExtractResponse(+ relationships=[+ BatchRelationship(+ tail=BatchEndpoint(uuid="u1", name="PERSON", type="PERSON"),+ tip=BatchEndpoint(uuid="u2", name="Bob", type="PERSON"),+ name="MET",+ source_span="Alice met Bob",+ )+ ]+ )++ with patch.object(agent, "_get_agent"), patch.object(+ agent, "run_tooler", return_value=[]+ ) as tooler:+ agent.agent = MagicMock()+ agent.agent.invoke.return_value = {+ "structured_response": bad,+ "messages": [],+ }+ ledger = IngestCostLedger()+ agent.run_batch_extract(+ source,+ entities,+ cost_ledger=ledger,+ escalate=True,+ timeout=5,+ max_retries=1,+ )+ assert agent.agent.invoke.call_count == 2+ tooler.assert_called_once()+ assert ledger.architect_escalations == 1+ assert ledger.escalate_rate == 1.0+ assert ledger.architect_repair_calls == 1+++def test_run_batch_extract_keeps_partial_without_escalate():+ agent = ArchitectAgent(+ llm_adapter=MagicMock(),+ cache_adapter=MagicMock(),+ kg=MagicMock(),+ vector_store=MagicMock(),+ embeddings=MagicMock(),+ ingestion_manager=MagicMock(),+ )+ source = "Alice hired Bob as a contractor."+ entities = [+ ScoutEntity(uuid="u1", name="Alice", type="PERSON"),+ ScoutEntity(uuid="u2", name="Hiring", type="EVENT"),+ ScoutEntity(uuid="u3", name="Bob", type="PERSON"),+ ]+ mixed = BatchExtractResponse(+ relationships=[+ BatchRelationship(+ tail=BatchEndpoint(uuid="u1", name="Alice", type="PERSON"),+ tip=BatchEndpoint(uuid="u2", name="Hiring", type="EVENT"),+ name="MADE",+ source_span="Alice hired Bob as a contractor",+ ),+ BatchRelationship(+ tail=BatchEndpoint(uuid="u2", name="Hiring", type="EVENT"),+ tip=BatchEndpoint(uuid="u3", name="Bob", type="PERSON"),+ name="TARGETED",+ source_span="totally fabricated zebra claim",+ ),+ BatchRelationship(+ tail=BatchEndpoint(uuid="u1", name="PERSON", type="PERSON"),+ tip=BatchEndpoint(uuid="u3", name="Bob", type="PERSON"),+ name="MET",+ source_span="Alice hired Bob",+ ),+ ]+ )++ with patch.object(agent, "_get_agent"), patch.object(+ agent, "run_tooler"+ ) as tooler:+ agent.agent = MagicMock()+ agent.agent.invoke.return_value = {+ "structured_response": mixed,+ "messages": [],+ }+ ledger = IngestCostLedger()+ rels = agent.run_batch_extract(+ source,+ entities,+ cost_ledger=ledger,+ escalate=True,+ timeout=5,+ max_retries=1,+ )+ assert agent.agent.invoke.call_count == 2+ tooler.assert_not_called()+ assert len(rels) == 1+ assert rels[0].name == "MADE"+ assert ledger.architect_escalations == 0+ assert ledger.escalate_rate == 0.0+ assert ledger.architect_repair_calls == 1+++def test_run_batch_extract_prefers_primary_when_repair_worse():+ agent = ArchitectAgent(+ llm_adapter=MagicMock(),+ cache_adapter=MagicMock(),+ kg=MagicMock(),+ vector_store=MagicMock(),+ embeddings=MagicMock(),+ ingestion_manager=MagicMock(),+ )+ source = "Alice hired Bob as a contractor."+ entities = [+ ScoutEntity(uuid="u1", name="Alice", type="PERSON"),+ ScoutEntity(uuid="u2", name="Hiring", type="EVENT"),+ ScoutEntity(uuid="u3", name="Bob", type="PERSON"),+ ]+ primary = BatchExtractResponse(+ relationships=[+ BatchRelationship(+ tail=BatchEndpoint(uuid="u1", name="Alice", type="PERSON"),+ tip=BatchEndpoint(uuid="u2", name="Hiring", type="EVENT"),+ name="MADE",+ source_span="Alice hired Bob as a contractor",+ ),+ BatchRelationship(+ tail=BatchEndpoint(uuid="u1", name="PERSON", type="PERSON"),+ tip=BatchEndpoint(uuid="u3", name="Bob", type="PERSON"),+ name="MET",+ source_span="Alice hired Bob",+ ),+ ]+ )+ worse_repair = BatchExtractResponse(+ relationships=[+ BatchRelationship(+ tail=BatchEndpoint(uuid="u1", name="PERSON", type="PERSON"),+ tip=BatchEndpoint(uuid="u3", name="Bob", type="PERSON"),+ name="MET",+ source_span="Alice hired Bob",+ )+ ]+ )++ with patch.object(agent, "_get_agent"), patch.object(+ agent, "run_tooler"+ ) as tooler:+ agent.agent = MagicMock()+ agent.agent.invoke.side_effect = [+ {"structured_response": primary, "messages": []},+ {"structured_response": worse_repair, "messages": []},+ ]+ ledger = IngestCostLedger()+ rels = agent.run_batch_extract(+ source,+ entities,+ cost_ledger=ledger,+ escalate=True,+ timeout=5,+ max_retries=1,+ )+ tooler.assert_not_called()+ assert len(rels) == 1+ assert rels[0].name == "MADE"+ assert ledger.architect_escalations == 0diff --git a/tests/test_architect_batch_schema.py b/tests/test_architect_batch_schema.pynew file mode 100644index 0000000..a1381d0--- /dev/null+++ b/tests/test_architect_batch_schema.py@@ -0,0 +1,196 @@+from src.core.saving.architect_batch import (+ BatchEndpoint,+ BatchExtractResponse,+ BatchRelationship,+ event_leg_incomplete,+ validate_batch_extract,+)+++def _entity(uuid: str, name: str, type_: str):+ return type("E", (), {"uuid": uuid, "name": name, "type": type_})()+++def test_validate_accepts_grounded_event_hub():+ source = "Alice hired Bob as a contractor in May 2024 for $5000."+ entities = [+ _entity("u1", "Alice", "PERSON"),+ _entity("u2", "Hiring", "EVENT"),+ _entity("u3", "Bob", "PERSON"),+ ]+ payload = BatchExtractResponse(+ relationships=[+ BatchRelationship(+ tail=BatchEndpoint(uuid="u1", name="Alice", type="PERSON"),+ tip=BatchEndpoint(uuid="u2", name="Hiring", type="EVENT"),+ name="MADE",+ description="Alice hired Bob",+ amount=5000,+ source_span="Alice hired Bob as a contractor",+ happened_at="2024-05",+ ),+ BatchRelationship(+ tail=BatchEndpoint(uuid="u2", name="Hiring", type="EVENT"),+ tip=BatchEndpoint(uuid="u3", name="Bob", type="PERSON"),+ name="TARGETED",+ description="Alice hired Bob",+ amount=5000,+ source_span="Alice hired Bob as a contractor in May 2024",+ ),+ ]+ )+ result = validate_batch_extract(+ payload, source_text=source, scout_entities=entities+ )+ assert len(result.accepted) == 2+ assert result.rejected == []+ assert not event_leg_incomplete(result.accepted, entities)+++def test_validate_rejects_type_named_and_ungrounded():+ source = "Alice met Bob at the park."+ entities = [+ _entity("u1", "Alice", "PERSON"),+ _entity("u2", "Bob", "PERSON"),+ ]+ payload = BatchExtractResponse(+ relationships=[+ BatchRelationship(+ tail=BatchEndpoint(uuid="u1", name="PERSON", type="PERSON"),+ tip=BatchEndpoint(uuid="u2", name="Bob", type="PERSON"),+ name="MET",+ source_span="Alice met Bob",+ ),+ BatchRelationship(+ tail=BatchEndpoint(uuid="u1", name="Alice", type="PERSON"),+ tip=BatchEndpoint(uuid="u2", name="Bob", type="PERSON"),+ name="MET",+ source_span="totally fabricated zebra claim",+ ),+ BatchRelationship(+ # well-formed unknown endpoint is soft-admitted (no escalate)+ tail=BatchEndpoint(uuid="missing", name="Carol", type="PERSON"),+ tip=BatchEndpoint(uuid="u2", name="Bob", type="PERSON"),+ name="MET",+ source_span="Alice met Bob at the park",+ ),+ ]+ )+ result = validate_batch_extract(+ payload, source_text=source, scout_entities=entities+ )+ reasons = {issue.reason for issue in result.rejected}+ assert "type_named_placeholder_endpoint" in reasons+ assert "source_span_not_grounded" in reasons+ assert "unknown_tail_endpoint" not in reasons+ assert len(result.accepted) == 1+ assert result.accepted[0].tail.name == "Carol"+ assert any(n.name == "Carol" for n in result.new_nodes)+++def test_validate_admits_unknown_well_formed_endpoints():+ source = "Alice introduced Carol to Bob at the park."+ entities = [+ _entity("u1", "Alice", "PERSON"),+ _entity("u2", "Bob", "PERSON"),+ ]+ payload = BatchExtractResponse(+ relationships=[+ BatchRelationship(+ tail=BatchEndpoint(name="Carol", type="PERSON"),+ tip=BatchEndpoint(uuid="u2", name="Bob", type="PERSON"),+ name="MET",+ source_span="Alice introduced Carol to Bob at the park",+ ),+ ]+ )+ result = validate_batch_extract(+ payload, source_text=source, scout_entities=entities+ )+ assert result.usable+ assert len(result.accepted) == 1+ assert result.rejected == []+++def test_validate_span_offsets_and_scout_uuid_reuse():+ source = "On Tuesday Alice paid Bob 12 dollars."+ entities = [_entity("a", "Alice", "PERSON"), _entity("b", "Bob", "PERSON")]+ span = "Alice paid Bob 12 dollars"+ start = source.index(span)+ payload = {+ "new_nodes": [],+ "relationships": [+ {+ "tail": {"uuid": "a", "name": "Alice", "type": "PERSON"},+ "tip": {"uuid": "b", "name": "Bob", "type": "PERSON"},+ "name": "PAID",+ "amount": 12,+ "source_span": span,+ "span_start": start,+ "span_end": start + len(span),+ }+ ],+ }+ result = validate_batch_extract(+ payload, source_text=source, scout_entities=entities+ )+ assert len(result.accepted) == 1+ assert result.accepted[0].amount == 12+++def test_event_leg_incomplete_detects_untouched_event():+ entities = [+ _entity("e1", "Trip", "EVENT"),+ _entity("p1", "Alice", "PERSON"),+ ]+ rels = [+ BatchRelationship(+ tail=BatchEndpoint(uuid="p1", name="Alice", type="PERSON"),+ tip=BatchEndpoint(uuid="p1", name="Alice", type="PERSON"),+ name="SELF",+ source_span="Alice",+ )+ ]+ assert event_leg_incomplete(rels, entities)+++def test_validate_partial_keeps_usable_and_drops_bad_new_nodes():+ source = "Alice hired Bob as a contractor in May 2024."+ entities = [+ _entity("u1", "Alice", "PERSON"),+ _entity("u2", "Hiring", "EVENT"),+ _entity("u3", "Bob", "PERSON"),+ ]+ payload = BatchExtractResponse(+ new_nodes=[+ BatchEndpoint(name="PERSON", type="PERSON"),+ BatchEndpoint(name="Contract", type="DOCUMENT"),+ ],+ relationships=[+ BatchRelationship(+ tail=BatchEndpoint(uuid="u1", name="Alice", type="PERSON"),+ tip=BatchEndpoint(uuid="u2", name="Hiring", type="EVENT"),+ name="MADE",+ source_span="Alice hired Bob as a contractor",+ ),+ BatchRelationship(+ tail=BatchEndpoint(uuid="u2", name="Hiring", type="EVENT"),+ tip=BatchEndpoint(uuid="missing", name="Carol", type="PERSON"),+ name="TARGETED",+ source_span="Alice hired Bob as a contractor in May 2024",+ ),+ ],+ )+ result = validate_batch_extract(+ payload, source_text=source, scout_entities=entities+ )+ assert result.usable+ assert not result.ok+ assert len(result.accepted) == 2+ assert {r.name for r in result.accepted} == {"MADE", "TARGETED"}+ assert len(result.new_nodes) >= 2+ assert any(n.name == "Contract" for n in result.new_nodes)+ assert any(n.name == "Carol" for n in result.new_nodes)+ reasons = {issue.reason for issue in result.rejected}+ assert "unknown_tip_endpoint" not in reasons+ assert any(r.startswith("new_node_type_named_placeholder:") for r in reasons)diff --git a/tests/test_architect_escalation.py b/tests/test_architect_escalation.pynew file mode 100644index 0000000..0bbd052--- /dev/null+++ b/tests/test_architect_escalation.py@@ -0,0 +1,232 @@+import os++ENV_DEFAULTS = {+ "BRAINPAT_TOKEN": "test-token",+ "MODELS_MODE": "local",+ "PIPELINE_MODE": "accurate",+ "EMBEDDINGS_LOCAL_MODEL": "local-model",+ "EMBEDDINGS_SMALL_MODEL": "small-model",+ "EMBEDDING_NODES_DIMENSION": "3",+ "EMBEDDING_TRIPLETS_DIMENSION": "3",+ "EMBEDDING_OBSERVATIONS_DIMENSION": "3",+ "EMBEDDING_DATA_DIMENSION": "3",+ "EMBEDDING_RELATIONSHIPS_DIMENSION": "3",+ "REDIS_HOST": "localhost",+ "REDIS_PORT": "6379",+ "NEO4J_HOST": "localhost",+ "NEO4J_PORT": "7687",+ "NEO4J_USERNAME": "neo4j",+ "NEO4J_PASSWORD": "password",+ "MILVUS_HOST": "localhost",+ "MILVUS_PORT": "19530",+ "MONGO_CONNECTION_STRING": "mongodb://localhost:27017",+ "CELERY_WORKER_CONCURRENCY": "1",+ "OLLAMA_HOST": "localhost",+ "OLLAMA_PORT": "11434",+ "OLLAMA_LLM_SMALL_MODEL": "small",+ "OLLAMA_LLM_LARGE_MODEL": "large",+}+for key, value in ENV_DEFAULTS.items():+ os.environ.setdefault(key, value)++from unittest.mock import MagicMock, patch++from src.core.agents.architect_agent import ArchitectAgent+from src.core.agents.scout_agent import ScoutEntity+from src.core.saving.architect_batch import BatchExtractResponse+from src.core.saving.auto_kg import _dense_architect_parts, _filter_entities_for_text+from src.core.saving.ingest_cost import IngestCostLedger+from src.utils.text_chunking import chunk_text+++def _entity(uuid: str, name: str, type_: str = "Event"):+ return ScoutEntity(uuid=uuid, name=name, type=type_)+++def test_chunk_text_splits_dialogue_on_single_newlines():+ lines = [f"Speaker: turn number {i} with some filler text." for i in range(20)]+ text = "\n".join(lines)+ parts = chunk_text(text, max_chars=200)+ assert len(parts) > 1+ assert all(len(p) <= 200 for p in parts)+ joined = "\n".join(parts)+ assert "Speaker: turn number 0" in joined+ assert "Speaker: turn number 19" in joined+++def test_dense_architect_parts_splits_entity_heavy_dialogue():+ lines = [+ "Caroline: I applied to adoption agencies this week.",+ "Melanie: That is huge. How are Oscar and Bailey?",+ "Caroline: Oscar hid a bone. Bailey is new.",+ "Melanie: Show me Oliver too.",+ "Caroline: I also painted a horse with dad.",+ "Melanie: Proud of you.",+ ]+ text = "\n".join(lines)+ entities = [+ _entity(f"e{i}", name)+ for i, name in enumerate(+ [+ "Caroline",+ "Melanie",+ "Applied to adoption agencies",+ "Oscar",+ "Bailey",+ "Oliver",+ "Hid a bone",+ "Painted a horse",+ "Horseback riding",+ "Self-portrait",+ "Goodbyes",+ "Support",+ "Conversation",+ ]+ )+ ]+ assert len(entities) >= 12+ parts = _dense_architect_parts(+ text, entities, entity_threshold=12, dense_max_chars=120+ )+ assert len(parts) > 1+ filtered = _filter_entities_for_text(entities, parts[0])+ assert filtered+ assert len(filtered) <= len(entities)+++def test_run_batch_extract_empty_with_escalate_disabled_stays_partial():+ agent = ArchitectAgent(+ llm_adapter=MagicMock(),+ cache_adapter=MagicMock(),+ kg=MagicMock(),+ vector_store=MagicMock(),+ embeddings=MagicMock(),+ ingestion_manager=MagicMock(),+ )+ source = "Alice met Bob."+ entities = [+ ScoutEntity(uuid="u1", name="Alice", type="PERSON"),+ ScoutEntity(uuid="u2", name="Bob", type="PERSON"),+ ]+ empty = BatchExtractResponse(relationships=[], new_nodes=[])++ with patch.object(agent, "_get_agent"), patch.object(+ agent, "run_tooler"+ ) as tooler:+ agent.agent = MagicMock()+ agent.agent.invoke.return_value = {+ "structured_response": empty,+ "messages": [],+ }+ ledger = IngestCostLedger()+ rels = agent.run_batch_extract(+ source,+ entities,+ cost_ledger=ledger,+ escalate=False,+ timeout=5,+ max_retries=1,+ )+ tooler.assert_not_called()+ assert rels == []+ assert ledger.architect_escalations == 0+++def test_run_batch_extract_escalate_respects_zero_turn_budget():+ agent = ArchitectAgent(+ llm_adapter=MagicMock(),+ cache_adapter=MagicMock(),+ kg=MagicMock(),+ vector_store=MagicMock(),+ embeddings=MagicMock(),+ ingestion_manager=MagicMock(),+ )+ source = "Alice met Bob."+ entities = [+ ScoutEntity(uuid="u1", name="Alice", type="PERSON"),+ ScoutEntity(uuid="u2", name="Bob", type="PERSON"),+ ]+ empty = BatchExtractResponse(relationships=[], new_nodes=[])++ with patch.object(agent, "_get_agent"), patch.object(+ agent, "run_tooler"+ ) as tooler, patch(+ "src.core.agents.architect_agent.config"+ ) as mock_config:+ mock_config.ingest_architect_max_schema_calls = 2+ mock_config.ingest_architect_escalate_max_turns = 0+ agent.agent = MagicMock()+ agent.agent.invoke.return_value = {+ "structured_response": empty,+ "messages": [],+ }+ ledger = IngestCostLedger()+ rels = agent.run_batch_extract(+ source,+ entities,+ cost_ledger=ledger,+ escalate=True,+ timeout=5,+ max_retries=1,+ )+ tooler.assert_not_called()+ assert rels == []+ assert ledger.architect_escalations == 1+ assert "escalate_budget_exhausted" in ledger.janitor_drop_reasons+++def test_run_batched_janitor_caps_llm_calls():+ from src.constants.agents import ArchitectAgentEntity, ArchitectAgentRelationship++ agent = ArchitectAgent(+ llm_adapter=MagicMock(),+ cache_adapter=MagicMock(),+ kg=MagicMock(),+ vector_store=MagicMock(),+ embeddings=MagicMock(),+ ingestion_manager=MagicMock(),+ )++ pending = [+ ArchitectAgentRelationship(+ flow_key=f"f{i}",+ tail=ArchitectAgentEntity(uuid=f"t{i}", name=f"Tail{i}", type="PERSON"),+ tip=ArchitectAgentEntity(uuid=f"p{i}", name=f"Tip{i}", type="PERSON"),+ name="RELATED",+ description="not in source text at all",+ properties={},+ )+ for i in range(5)+ ]+ agent.pending_persistence_batches = [pending]+ agent.relationships_set = list(pending)++ janitor = MagicMock()+ janitor.run_atomic_janitor.return_value = "OK"+ agent.janitor_agent = janitor+ agent._janitor_agent_brain_id = "default"++ ledger = IngestCostLedger()+ with patch(+ "src.core.agents.architect_agent.config"+ ) as mock_config, patch(+ "src.core.saving.grounding.triage_relationships_for_janitor"+ ) as triage_mock:+ mock_config.ingest_janitor_max_llm_calls = 1+ triage = MagicMock()+ triage.accept = []+ triage.reject = []+ triage.ambiguous = pending+ triage_mock.return_value = triage+ agent.run_batched_janitor(+ text="Alice met Bob.",+ brain_id="default",+ batch_size=2,+ cost_ledger=ledger,+ )++ assert janitor.run_atomic_janitor.call_count == 1+ assert any(+ str(r).startswith("janitor_budget_exhausted")+ for r in ledger.janitor_drop_reasons+ )diff --git a/tests/test_architect_grounding_decisions.py b/tests/test_architect_grounding_decisions.pynew file mode 100644index 0000000..392d1db--- /dev/null+++ b/tests/test_architect_grounding_decisions.py@@ -0,0 +1,142 @@+from types import SimpleNamespace++from src.core.saving.grounding import (+ decide_relationship_grounding,+ triage_relationships_for_janitor,+)+from src.core.saving.architect_batch import (+ BatchEndpoint,+ BatchExtractResponse,+ BatchRelationship,+ validate_batch_extract,+)+++def _entity(uuid: str, name: str, type_: str, description: str = ""):+ return type(+ "E",+ (),+ {"uuid": uuid, "name": name, "type": type_, "description": description},+ )()+++def test_validate_realigns_mismatched_span_offsets():+ source = "On Tuesday Alice paid Bob 12 dollars."+ entities = [_entity("a", "Alice", "PERSON"), _entity("b", "Bob", "PERSON")]+ span = "Alice paid Bob 12 dollars"+ payload = BatchExtractResponse(+ relationships=[+ BatchRelationship(+ tail=BatchEndpoint(uuid="a", name="Alice", type="PERSON"),+ tip=BatchEndpoint(uuid="b", name="Bob", type="PERSON"),+ name="PAID",+ source_span=span,+ span_start=0,+ span_end=3,+ )+ ]+ )+ result = validate_batch_extract(+ payload, source_text=source, scout_entities=entities+ )+ assert len(result.accepted) == 1+ accepted = result.accepted[0]+ assert accepted.span_start is not None+ assert source[accepted.span_start : accepted.span_end] == span+++def test_validate_fills_event_description_from_scout():+ source = "Alice hired Bob as a contractor in May 2024."+ entities = [+ _entity("u1", "Alice", "PERSON"),+ _entity(+ "u2",+ "Hiring",+ "EVENT",+ description="Alice hired Bob as a contractor in May 2024",+ ),+ _entity("u3", "Bob", "PERSON"),+ ]+ payload = BatchExtractResponse(+ relationships=[+ BatchRelationship(+ tail=BatchEndpoint(uuid="u1", name="Alice", type="PERSON"),+ tip=BatchEndpoint(uuid="u2", name="Hiring", type="EVENT"),+ name="MADE",+ source_span="Alice hired Bob as a contractor",+ )+ ]+ )+ result = validate_batch_extract(+ payload, source_text=source, scout_entities=entities+ )+ assert len(result.accepted) == 1+ event = result.accepted[0].tip+ assert (event.description or "").strip()+ assert "hired" in event.description.lower()+++def test_validate_fills_event_description_from_source_span():+ source = "Alice hired Bob as a contractor in May 2024."+ entities = [+ _entity("u1", "Alice", "PERSON"),+ _entity("u2", "Hiring", "EVENT"),+ _entity("u3", "Bob", "PERSON"),+ ]+ payload = BatchExtractResponse(+ relationships=[+ BatchRelationship(+ tail=BatchEndpoint(uuid="u1", name="Alice", type="PERSON"),+ tip=BatchEndpoint(uuid="u2", name="Hiring", type="EVENT"),+ name="MADE",+ source_span="Alice hired Bob as a contractor",+ )+ ]+ )+ result = validate_batch_extract(+ payload, source_text=source, scout_entities=entities+ )+ assert len(result.accepted) == 1+ assert result.accepted[0].tip.description == "Alice hired Bob as a contractor"+++def test_validate_rejects_new_event_node_without_description():+ source = "Alice hired Bob as a contractor."+ entities = [+ _entity("u1", "Alice", "PERSON"),+ _entity("u3", "Bob", "PERSON"),+ ]+ payload = BatchExtractResponse(+ new_nodes=[BatchEndpoint(name="Hiring", type="EVENT")],+ relationships=[+ BatchRelationship(+ tail=BatchEndpoint(uuid="u1", name="Alice", type="PERSON"),+ tip=BatchEndpoint(name="Hiring", type="EVENT"),+ name="MADE",+ source_span="Alice hired Bob as a contractor",+ )+ ],+ )+ result = validate_batch_extract(+ payload, source_text=source, scout_entities=entities+ )+ reasons = {issue.reason for issue in result.rejected}+ assert any(r.startswith("new_node_missing_event_description:") for r in reasons)+++def test_grounding_decision_machine_readable():+ source = "Alice met Bob at the park."+ rel = SimpleNamespace(+ name="MET",+ description="Alice met Bob at the park",+ properties={"source_span": "Alice met Bob at the park"},+ tip=SimpleNamespace(name="Bob", type="PERSON", uuid="1"),+ tail=SimpleNamespace(name="Alice", type="PERSON", uuid="2"),+ )+ decision = decide_relationship_grounding(rel, source)+ assert decision.decision == "accept"+ assert decision.reason == "grounded_endpoints_ok"+ triage = triage_relationships_for_janitor([rel], source)+ assert len(triage.accept) == 1+ assert triage.reject == []+ assert triage.ambiguous == []diff --git a/tests/test_architect_prior_context.py b/tests/test_architect_prior_context.pynew file mode 100644index 0000000..2b3c5f5--- /dev/null+++ b/tests/test_architect_prior_context.py@@ -0,0 +1,303 @@+import os++ENV_DEFAULTS = {+ "BRAINPAT_TOKEN": "test-token",+ "MODELS_MODE": "local",+ "PIPELINE_MODE": "accurate",+ "EMBEDDINGS_LOCAL_MODEL": "local-model",+ "EMBEDDINGS_SMALL_MODEL": "small-model",+ "EMBEDDING_NODES_DIMENSION": "3",+ "EMBEDDING_TRIPLETS_DIMENSION": "3",+ "EMBEDDING_OBSERVATIONS_DIMENSION": "3",+ "EMBEDDING_DATA_DIMENSION": "3",+ "EMBEDDING_RELATIONSHIPS_DIMENSION": "3",+ "REDIS_HOST": "localhost",+ "REDIS_PORT": "6379",+ "NEO4J_HOST": "localhost",+ "NEO4J_PORT": "7687",+ "NEO4J_USERNAME": "neo4j",+ "NEO4J_PASSWORD": "password",+ "MILVUS_HOST": "localhost",+ "MILVUS_PORT": "19530",+ "MONGO_CONNECTION_STRING": "mongodb://localhost:27017",+ "CELERY_WORKER_CONCURRENCY": "1",+ "OLLAMA_HOST": "localhost",+ "OLLAMA_PORT": "11434",+ "OLLAMA_LLM_SMALL_MODEL": "small",+ "OLLAMA_LLM_LARGE_MODEL": "large",+}+for key, value in ENV_DEFAULTS.items():+ os.environ.setdefault(key, value)++from unittest.mock import MagicMock, patch++from src.constants.agents import ArchitectAgentEntity, ArchitectAgentRelationship+from src.core.agents.scout_agent import ScoutEntity+from src.core.saving.architect_scratchpad import (+ build_scratchpad,+ fetch_prior_span,+ format_unit_with_scratchpad,+ resolve_prior_context_mode,+ serialize_scratchpad,+)+from src.core.saving.auto_kg import (+ _architect_unit_text,+ _architect_unit_text_scratchpad,+)+from src.core.saving.ingest_cost import count_source_tokens+++def test_resolve_prior_context_defaults_scratchpad_for_batch():+ assert resolve_prior_context_mode("auto", "batch") == "scratchpad"+ assert resolve_prior_context_mode("auto", "schema") == "scratchpad"+ assert resolve_prior_context_mode("auto", "tooler") == "raw"+ assert resolve_prior_context_mode("raw", "batch") == "raw"+ assert resolve_prior_context_mode("scratchpad", "tooler") == "scratchpad"+++def test_build_and_serialize_scratchpad_includes_hubs_and_predicates():+ entities = [+ ScoutEntity(uuid="u1", name="Alice", type="PERSON"),+ ScoutEntity(+ uuid="u2",+ name="Hiring",+ type="EVENT",+ happened_at="2024-01-15",+ description="Alice hired Bob",+ ),+ ScoutEntity(uuid="u3", name="Bob", type="PERSON"),+ ]+ rels = [+ ArchitectAgentRelationship(+ uuid="r1",+ flow_key="fk1",+ name="MADE",+ tail=ArchitectAgentEntity(uuid="u1", name="Alice", type="PERSON"),+ tip=ArchitectAgentEntity(+ uuid="u2",+ name="Hiring",+ type="EVENT",+ happened_at="2024-01-15",+ description="Alice hired Bob",+ ),+ properties={"source_span": "Alice hired Bob as a contractor yesterday"},+ ),+ ArchitectAgentRelationship(+ uuid="r2",+ flow_key="fk2",+ name="TARGETED",+ tail=ArchitectAgentEntity(+ uuid="u2", name="Hiring", type="EVENT", happened_at="2024-01-15"+ ),+ tip=ArchitectAgentEntity(uuid="u3", name="Bob", type="PERSON"),+ properties={"source_span": "hired Bob as a contractor"},+ ),+ ]++ pad = build_scratchpad(entities, rels)+ assert any(e.name == "Alice" for e in pad.entities)+ assert any(h.name == "Hiring" for h in pad.event_hubs)+ hub = next(h for h in pad.event_hubs if h.name == "Hiring")+ assert "Alice" in hub.actors+ assert "Bob" in hub.objects+ assert "2024-01-15" in pad.dates+ assert any(p.predicate == "MADE" for p in pad.recent_predicates)++ text, tokens = serialize_scratchpad(pad, token_cap=500)+ assert tokens > 0+ assert tokens <= 500+ assert "Hiring" in text+ assert "Alice" in text+ assert "MADE" in text+ # Span pointers are truncated quotes, not full prior unit bodies.+ assert "Alice hired Bob" in text+++def test_serialize_scratchpad_obeys_token_cap():+ entities = [+ ScoutEntity(+ uuid=f"u{i}",+ name=f"PersonName{i}WithExtraDetail",+ type="PERSON",+ happened_at=f"2024-01-{(i % 28) + 1:02d}",+ )+ for i in range(80)+ ]+ rels = [+ ArchitectAgentRelationship(+ uuid=f"r{i}",+ flow_key=f"fk{i}",+ name="MADE",+ tail=ArchitectAgentEntity(+ uuid=f"u{i}", name=f"PersonName{i}WithExtraDetail", type="PERSON"+ ),+ tip=ArchitectAgentEntity(+ uuid=f"e{i}",+ name=f"EventHub{i}LongLabel",+ type="EVENT",+ happened_at=f"2024-02-{(i % 28) + 1:02d}",+ ),+ properties={+ "source_span": (+ f"PersonName{i}WithExtraDetail did something notable "+ f"during EventHub{i}LongLabel on a long day of narrative text."+ )+ },+ )+ for i in range(40)+ ]+ pad = build_scratchpad(entities, rels)+ text, tokens = serialize_scratchpad(pad, token_cap=120)+ assert tokens <= 120+ counted, _, _ = count_source_tokens(text)+ assert counted <= 120+++def test_scratchpad_unit_text_excludes_full_prior_bodies():+ prior_body = (+ "Caroline told Melanie she started a new job at Google last Monday "+ "and also mentioned her sister's wedding plans for June in detail. "+ * 8+ )+ prior_entities = [+ ScoutEntity(uuid="u1", name="Caroline", type="PERSON"),+ ScoutEntity(+ uuid="u2",+ name="Started new job",+ type="EVENT",+ happened_at="2023-05-08",+ ),+ ]+ prior_rels = [+ ArchitectAgentRelationship(+ uuid="r1",+ flow_key="fk1",+ name="MADE",+ tail=ArchitectAgentEntity(uuid="u1", name="Caroline", type="PERSON"),+ tip=ArchitectAgentEntity(+ uuid="u2",+ name="Started new job",+ type="EVENT",+ happened_at="2023-05-08",+ ),+ properties={"source_span": "Caroline told Melanie she started a new job"},+ )+ ]+ current = "She said the salary was better than before."+ scratch = _architect_unit_text_scratchpad(+ current,+ prior_entities=prior_entities,+ prior_relationships=prior_rels,+ token_cap=500,+ )+ raw = _architect_unit_text(current, [prior_body])++ assert "[Prior scratchpad]" in scratch+ assert "[Current unit]" in scratch+ assert current in scratch+ assert "Caroline" in scratch+ assert "Started new job" in scratch+ # Must not replay the full prior narrative body.+ assert prior_body not in scratch+ assert prior_body in raw+ assert "sister's wedding plans for June in detail" not in scratch+++def test_fetch_prior_span_returns_window():+ prior = [+ "Intro filler. Alice met Bob at the cafe on Tuesday afternoon. More filler."+ ]+ span = fetch_prior_span(prior, "Alice met Bob", window_chars=80)+ assert span is not None+ assert "Alice met Bob" in span+ assert len(span) < len(prior[0])+++def test_format_unit_with_scratchpad_empty_passthrough():+ assert format_unit_with_scratchpad("only current", "") == "only current"+++def test_enrich_loop_uses_scratchpad_not_raw_prior_for_batch():+ """Integration-ish: second unit prompt must not contain prior chunk body."""+ from src.core.saving import auto_kg++ prior_chunk = (+ "UNIQUE_PRIOR_MARKER_XYZ Caroline started working at Google last week "+ "and talked about her sister wedding plans extensively across many sentences."+ )+ current_chunk = "She said the pay was higher."++ scout = MagicMock()+ first_entities = [+ ScoutEntity(uuid="u1", name="Caroline", type="PERSON"),+ ScoutEntity(uuid="u2", name="Started job", type="EVENT"),+ ]+ second_entities = [+ ScoutEntity(uuid="u1", name="Caroline", type="PERSON"),+ ]++ class _Resp:+ def __init__(self, entities):+ self.entities = entities++ scout._run_chunk.side_effect = [+ _Resp(first_entities),+ _Resp(second_entities),+ ]++ architect = MagicMock()+ architect.session_id = "s1"+ architect.relationships_set = [+ ArchitectAgentRelationship(+ uuid="r1",+ flow_key="fk1",+ name="MADE",+ tail=ArchitectAgentEntity(uuid="u1", name="Caroline", type="PERSON"),+ tip=ArchitectAgentEntity(uuid="u2", name="Started job", type="EVENT"),+ properties={"source_span": "Caroline started working at Google"},+ )+ ]+ architect.pending_persistence_batches = [list(architect.relationships_set)]+ architect.take_pending_relationships.return_value = list(+ architect.relationships_set+ )+ architect.defer_janitor = True++ captured_texts: list[str] = []++ def _capture_batch(text, entities, **kwargs):+ captured_texts.append(text)+ return list(architect.relationships_set)++ architect.run_batch_extract.side_effect = _capture_batch++ with patch.object(auto_kg, "ScoutAgent", return_value=scout), patch.object(+ auto_kg, "ArchitectAgent", return_value=architect+ ), patch.object(auto_kg, "IngestionManager", return_value=MagicMock()), patch.object(+ auto_kg, "chunk_text", return_value=[prior_chunk, current_chunk]+ ), patch.object(+ auto_kg.config, "pipeline_mode", "accurate"+ ), patch.object(+ auto_kg.config, "ingest_architect_mode", "batch"+ ), patch.object(+ auto_kg.config, "ingest_architect_prior_context", "auto"+ ), patch.object(+ auto_kg.config, "ingest_architect_per_unit", True+ ), patch.object(+ auto_kg.config, "ingest_defer_janitor", False+ ), patch.object(+ auto_kg.config, "run_graph_consolidator", False+ ), patch.object(+ auto_kg.config, "ingest_architect_scratchpad_token_cap", 500+ ):+ auto_kg.enrich_kg_from_input(+ prior_chunk + "\n" + current_chunk, brain_id="brain-scratch"+ )++ assert len(captured_texts) == 2+ second = captured_texts[1]+ assert "[Prior scratchpad]" in second+ assert "UNIQUE_PRIOR_MARKER_XYZ" not in second+ assert "wedding plans extensively" not in second+ assert current_chunk in second+ assert "Caroline" in seconddiff --git a/tests/test_grounding.py b/tests/test_grounding.pynew file mode 100644index 0000000..a91f849--- /dev/null+++ b/tests/test_grounding.py@@ -0,0 +1,158 @@+from types import SimpleNamespace++from src.core.saving.grounding import (+ align_span,+ cheap_janitor_precheck,+ decide_relationship_grounding,+ endpoints_well_formed,+ find_span_offsets,+ realign_span_offsets,+ relationship_looks_grounded,+ triage_relationships_for_janitor,+)+++def test_align_span_exact_and_fuzzy():+ source = "Alice met Bob at the park on Tuesday afternoon."+ score, _ = align_span("Alice met Bob at the park", source)+ assert score == 1.0+ score2, window = align_span("Bob at the park on Tuesday", source)+ assert score2 >= 0.45+ assert "bob" in window or "park" in window+++def test_find_span_offsets_exact_and_whitespace():+ source = "On Tuesday Alice paid Bob 12 dollars."+ span = "Alice paid Bob 12 dollars"+ start, end = find_span_offsets(span, source)+ assert source[start:end] == span++ spaced = "Alice paid Bob 12 dollars"+ found = find_span_offsets(spaced, source)+ assert found is not None+ assert "alice" in source[found[0] : found[1]].lower()+++def test_realign_span_offsets_repairs_mismatch():+ source = "On Tuesday Alice paid Bob 12 dollars."+ span = "Alice paid Bob 12 dollars"+ ok, start, end, reason = realign_span_offsets(+ span=span,+ source_text=source,+ span_start=0,+ span_end=5,+ )+ assert ok and reason == ""+ assert start is not None and end is not None+ assert source[start:end] == span+++def test_realign_clears_bad_offsets_when_span_grounded():+ source = "Alice hired Bob as a contractor in May."+ span = "Alice hired Bob as a contractor"+ # Offsets point nowhere useful and span cannot be exact-found if we+ # mutate — use a grounded span with impossible offsets that find can fix.+ ok, start, end, reason = realign_span_offsets(+ span=span,+ source_text=source,+ span_start=-1,+ span_end=9999,+ )+ assert ok and reason == ""+ assert start is not None+ assert source[start:end] == span+++def test_endpoints_reject_type_named_placeholders():+ good = SimpleNamespace(+ name="MET",+ tip=SimpleNamespace(name="Bob", type="PERSON"),+ tail=SimpleNamespace(name="Alice", type="PERSON"),+ )+ bad = SimpleNamespace(+ name="PAID",+ tip=SimpleNamespace(name="Money", type="MONEY"),+ tail=SimpleNamespace(name="Alice", type="PERSON"),+ )+ assert endpoints_well_formed(good)+ assert not endpoints_well_formed(bad)+++def test_cheap_janitor_precheck_splits():+ source = "Alice hired Bob as a contractor in May."+ grounded = SimpleNamespace(+ name="HIRED",+ description="Alice hired Bob as a contractor",+ properties={},+ tip=SimpleNamespace(name="Bob", type="PERSON", uuid="1"),+ tail=SimpleNamespace(name="Alice", type="PERSON", uuid="2"),+ )+ ungrounded = SimpleNamespace(+ name="HIRED",+ description="totally fabricated claim about zebras",+ properties={},+ tip=SimpleNamespace(name="Bob", type="PERSON", uuid="1"),+ tail=SimpleNamespace(name="Alice", type="PERSON", uuid="2"),+ )+ skip, need = cheap_janitor_precheck([grounded, ungrounded], source)+ assert grounded in skip+ assert ungrounded in need+ ok, score, _ = relationship_looks_grounded(grounded, source)+ assert ok and score >= 0.45+++def test_triage_accept_reject_ambiguous():+ source = "Alice hired Bob as a contractor in May."+ accept_rel = SimpleNamespace(+ name="HIRED",+ description="Alice hired Bob as a contractor",+ properties={"source_span": "Alice hired Bob as a contractor"},+ tip=SimpleNamespace(name="Bob", type="PERSON", uuid="1"),+ tail=SimpleNamespace(name="Alice", type="PERSON", uuid="2"),+ )+ reject_placeholder = SimpleNamespace(+ name="HIRED",+ description="Alice hired Bob",+ properties={},+ tip=SimpleNamespace(name="PERSON", type="PERSON", uuid="1"),+ tail=SimpleNamespace(name="Alice", type="PERSON", uuid="2"),+ )+ reject_ungrounded = SimpleNamespace(+ name="HIRED",+ description="zebras invented a spaceship yesterday",+ properties={"source_span": "zebras invented a spaceship yesterday"},+ tip=SimpleNamespace(name="Bob", type="PERSON", uuid="1"),+ tail=SimpleNamespace(name="Alice", type="PERSON", uuid="2"),+ )+ ambiguous = SimpleNamespace(+ name="HIRED",+ description="",+ properties={},+ tip=SimpleNamespace(name="Bob", type="PERSON", uuid="1"),+ tail=SimpleNamespace(name="Alice", type="PERSON", uuid="2"),+ )+ triage = triage_relationships_for_janitor(+ [accept_rel, reject_placeholder, reject_ungrounded, ambiguous],+ source,+ )+ assert accept_rel in triage.accept+ reject_reasons = {d.reason for d in triage.reject}+ assert "type_named_placeholder_endpoint" in reject_reasons+ assert "source_span_not_grounded" in reject_reasons+ assert ambiguous in triage.ambiguous+++def test_decide_relationship_grounding_accept_sets_span():+ source = "Alice hired Bob as a contractor in May."+ rel = SimpleNamespace(+ name="HIRED",+ description="Alice hired Bob as a contractor",+ properties={},+ tip=SimpleNamespace(name="Bob", type="PERSON", uuid="1"),+ tail=SimpleNamespace(name="Alice", type="PERSON", uuid="2"),+ )+ decision = decide_relationship_grounding(rel, source)+ assert decision.decision == "accept"+ assert decision.score >= 0.45+ assert rel.properties.get("source_span")+ assert "grounding_score" in rel.propertiesdiff --git a/tests/test_ingest_cost.py b/tests/test_ingest_cost.pynew file mode 100644index 0000000..df8700f--- /dev/null+++ b/tests/test_ingest_cost.py@@ -0,0 +1,95 @@+from src.core.saving.ingest_cost import (+ SOURCE_TOKENIZER_ID,+ IngestCostLedger,+ StageCost,+ count_source_tokens,+ merge_cost_into_status_payload,+ record_usage_from_response,+ track_stage,+)+++def test_stage_cost_accumulates():+ stage = StageCost()+ stage.add_usage(input_tokens=10, output_tokens=5, cached_tokens=2, calls=1)+ stage.add_usage(input_tokens=3, output_tokens=1, calls=1)+ assert stage.input_tokens == 13+ assert stage.output_tokens == 6+ assert stage.cached_tokens == 2+ assert stage.calls == 2+ assert stage.total_tokens == 19+++def test_ledger_roundtrip_and_merge():+ a = IngestCostLedger()+ a.scout.add_usage(input_tokens=100, output_tokens=20, calls=2)+ a.janitor_skipped = 3+ payload = a.to_dict()+ assert payload["total_llm_tokens"] == 120+ assert payload["stages"]["scout"]["calls"] == 2++ b = IngestCostLedger.from_dict(payload)+ b.architect.add_usage(input_tokens=50, output_tokens=10, calls=1)+ a.merge(b)+ assert a.scout.input_tokens == 200+ assert a.architect.input_tokens == 50+ assert a.janitor_skipped == 6+++def test_track_stage_records_usage_from_response():+ ledger = IngestCostLedger()+ with track_stage(ledger, "architect"):+ record_usage_from_response(+ {"usage_metadata": {"input_tokens": 40, "output_tokens": 8}}+ )+ assert ledger.architect.input_tokens == 40+ assert ledger.architect.output_tokens == 8+ assert ledger.architect.calls == 1+ assert ledger.architect.latency_ms >= 0+++def test_merge_cost_into_status_payload():+ existing = {"status": "started"}+ ledger = IngestCostLedger()+ ledger.scout.add_usage(input_tokens=1, output_tokens=1, calls=1)+ merged = merge_cost_into_status_payload(existing, ledger)+ assert merged["cost"]["stages"]["scout"]["input_tokens"] == 1+ again = merge_cost_into_status_payload(+ merged, IngestCostLedger.from_dict(merged["cost"])+ )+ assert again["cost"]["stages"]["scout"]["input_tokens"] == 2+++def test_source_token_multiplier_uses_pinned_tokenizer():+ ledger = IngestCostLedger()+ text = "Alice hired Bob as a contractor in May."+ ledger.set_source_text(text)+ tokens, tokenizer_id, estimated = count_source_tokens(text)+ assert tokens == ledger.source_tokens+ assert tokenizer_id == ledger.source_tokenizer+ assert not estimated+ assert ledger.source_tokenizer == SOURCE_TOKENIZER_ID+ assert ledger.source_chars == len(text)++ ledger.architect.add_usage(input_tokens=90, output_tokens=10, calls=2)+ payload = ledger.to_dict()+ assert payload["llm_source_multiplier"] == round(100 / float(tokens), 3)+ assert payload["source_tokenizer"] == SOURCE_TOKENIZER_ID+ assert payload["source_tokens"] == tokens+++def test_escalate_and_janitor_rates():+ ledger = IngestCostLedger()+ ledger.record_architect_unit(escalated=False, schema_calls=1)+ ledger.record_architect_unit(+ escalated=True, reason="schema_empty_or_all_rejected", schema_calls=2, repair_calls=1+ )+ ledger.janitor_skipped = 7+ ledger.janitor_ran = 3+ payload = ledger.to_dict()+ assert payload["architect_units"] == 2+ assert payload["architect_escalations"] == 1+ assert payload["escalate_rate"] == 0.5+ assert payload["janitor_skip_rate"] == 0.7+ assert payload["architect_repair_calls"] == 1+ assert "schema_empty_or_all_rejected" in payload["escalate_reasons"]diff --git a/tests/test_janitor_outcomes.py b/tests/test_janitor_outcomes.pynew file mode 100644index 0000000..e57fdcf--- /dev/null+++ b/tests/test_janitor_outcomes.py@@ -0,0 +1,268 @@+from types import SimpleNamespace+from unittest.mock import MagicMock, patch++import os++ENV_DEFAULTS = {+ "BRAINPAT_TOKEN": "test-token",+ "MODELS_MODE": "local",+ "PIPELINE_MODE": "accurate",+ "EMBEDDINGS_LOCAL_MODEL": "local-model",+ "EMBEDDINGS_SMALL_MODEL": "small-model",+ "EMBEDDING_NODES_DIMENSION": "3",+ "EMBEDDING_TRIPLETS_DIMENSION": "3",+ "EMBEDDING_OBSERVATIONS_DIMENSION": "3",+ "EMBEDDING_DATA_DIMENSION": "3",+ "EMBEDDING_RELATIONSHIPS_DIMENSION": "3",+ "REDIS_HOST": "localhost",+ "REDIS_PORT": "6379",+ "NEO4J_HOST": "localhost",+ "NEO4J_PORT": "7687",+ "NEO4J_USERNAME": "neo4j",+ "NEO4J_PASSWORD": "password",+ "MILVUS_HOST": "localhost",+ "MILVUS_PORT": "19530",+ "MONGO_CONNECTION_STRING": "mongodb://localhost:27017",+ "CELERY_WORKER_CONCURRENCY": "1",+ "OLLAMA_HOST": "localhost",+ "OLLAMA_PORT": "11434",+ "OLLAMA_LLM_SMALL_MODEL": "small",+ "OLLAMA_LLM_LARGE_MODEL": "large",+}+for key, value in ENV_DEFAULTS.items():+ os.environ.setdefault(key, value)++from src.constants.agents import (+ ArchitectAgentEntity,+ ArchitectAgentRelationship,+ AtomicJanitorAgentInputOutput,+ AtomicJanitorAgentWrongRelationship,+ _ArchitectAgentRelationship,+)+from src.core.agents.architect_agent import ArchitectAgent+from src.core.saving.ingest_cost import IngestCostLedger+++def _rel(name="MET", grounded=True):+ props = {}+ if grounded:+ props["source_span"] = "Alice met Bob at the park"+ return ArchitectAgentRelationship(+ flow_key="fk",+ uuid="r1",+ name=name,+ description="Alice met Bob at the park" if grounded else "zebras invented ships",+ properties=props,+ tail=ArchitectAgentEntity(+ uuid="a", name="Alice", type="PERSON", description="", properties={}+ ),+ tip=ArchitectAgentEntity(+ uuid="b", name="Bob", type="PERSON", description="", properties={}+ ),+ )+++def test_janitor_ok_status_keeps_batch():+ agent = ArchitectAgent(+ llm_adapter=MagicMock(),+ cache_adapter=MagicMock(),+ kg=MagicMock(),+ vector_store=MagicMock(),+ embeddings=MagicMock(),+ ingestion_manager=MagicMock(),+ )+ # Force ambiguous path: weak/missing span so triage sends to LLM.+ rel = _rel(grounded=False)+ rel.properties = {}+ rel.description = ""+ agent.pending_persistence_batches = [[rel]]+ ledger = IngestCostLedger()++ mock_janitor = MagicMock()+ mock_janitor.run_atomic_janitor.return_value = "OK"+ with patch(+ "src.core.agents.janitor_agent.JanitorAgent", return_value=mock_janitor+ ), patch("src.services.input.agents.llm_small_adapter"), patch(+ "src.services.input.agents.graph_adapter"+ ) as ga, patch(+ "src.services.input.agents.vector_store_adapter"+ ), patch(+ "src.services.input.agents.embeddings_adapter"+ ):+ ga.graphdb_description = "db"+ agent.run_batched_janitor(+ text="Alice met Bob at the park.",+ cost_ledger=ledger,+ )+ pending = agent.take_pending_relationships()+ assert len(pending) == 1+ assert ledger.janitor_ran == 1+ assert ledger.janitor_ambiguous == 1+++def test_janitor_reject_status_vetoes_without_persist():+ agent = ArchitectAgent(+ llm_adapter=MagicMock(),+ cache_adapter=MagicMock(),+ kg=MagicMock(),+ vector_store=MagicMock(),+ embeddings=MagicMock(),+ ingestion_manager=MagicMock(),+ )+ rel = _rel(grounded=False)+ rel.properties = {}+ rel.description = ""+ agent.pending_persistence_batches = [[rel]]+ ledger = IngestCostLedger()++ mock_janitor = MagicMock()+ mock_janitor.run_atomic_janitor.return_value = AtomicJanitorAgentInputOutput(+ status="REJECT",+ veto_reasons=["fabricated_edge"],+ )+ with patch(+ "src.core.agents.janitor_agent.JanitorAgent", return_value=mock_janitor+ ), patch("src.services.input.agents.llm_small_adapter"), patch(+ "src.services.input.agents.graph_adapter"+ ) as ga, patch(+ "src.services.input.agents.vector_store_adapter"+ ), patch(+ "src.services.input.agents.embeddings_adapter"+ ):+ ga.graphdb_description = "db"+ agent.run_batched_janitor(+ text="Alice met Bob at the park.",+ cost_ledger=ledger,+ )+ assert agent.take_pending_relationships() == []+ assert "fabricated_edge" in ledger.janitor_drop_reasons+++def test_janitor_parse_failure_does_not_approve():+ agent = ArchitectAgent(+ llm_adapter=MagicMock(),+ cache_adapter=MagicMock(),+ kg=MagicMock(),+ vector_store=MagicMock(),+ embeddings=MagicMock(),+ ingestion_manager=MagicMock(),+ )+ rel = _rel(grounded=False)+ rel.properties = {}+ rel.description = ""+ agent.pending_persistence_batches = [[rel]]+ ledger = IngestCostLedger()++ mock_janitor = MagicMock()+ mock_janitor.run_atomic_janitor.return_value = None+ with patch(+ "src.core.agents.janitor_agent.JanitorAgent", return_value=mock_janitor+ ), patch("src.services.input.agents.llm_small_adapter"), patch(+ "src.services.input.agents.graph_adapter"+ ) as ga, patch(+ "src.services.input.agents.vector_store_adapter"+ ), patch(+ "src.services.input.agents.embeddings_adapter"+ ):+ ga.graphdb_description = "db"+ agent.run_batched_janitor(+ text="Alice met Bob at the park.",+ cost_ledger=ledger,+ )+ assert agent.take_pending_relationships() == []+ assert "janitor_parse_failure" in ledger.janitor_drop_reasons+++def test_janitor_wrong_relationship_dropped_fixed_kept():+ agent = ArchitectAgent(+ llm_adapter=MagicMock(),+ cache_adapter=MagicMock(),+ kg=MagicMock(),+ vector_store=MagicMock(),+ embeddings=MagicMock(),+ ingestion_manager=MagicMock(),+ )+ bad = _rel(grounded=False)+ bad.properties = {}+ bad.description = ""+ agent.pending_persistence_batches = [[bad]]+ ledger = IngestCostLedger()++ fixed = _ArchitectAgentRelationship(+ tip=bad.tip,+ tail=bad.tail,+ name="MET",+ description="Alice met Bob at the park",+ properties={"source_span": "Alice met Bob at the park"},+ )+ wrong = AtomicJanitorAgentWrongRelationship(+ relationship=_ArchitectAgentRelationship(+ tip=bad.tip,+ tail=bad.tail,+ name=bad.name,+ description=bad.description or "",+ properties={},+ ),+ reason="ungrounded",+ instructions="drop",+ )+ mock_janitor = MagicMock()+ mock_janitor.run_atomic_janitor.return_value = AtomicJanitorAgentInputOutput(+ status="ERROR",+ fixed_relationships=[fixed],+ wrong_relationships=[wrong],+ )+ with patch(+ "src.core.agents.janitor_agent.JanitorAgent", return_value=mock_janitor+ ), patch("src.services.input.agents.llm_small_adapter"), patch(+ "src.services.input.agents.graph_adapter"+ ) as ga, patch(+ "src.services.input.agents.vector_store_adapter"+ ), patch(+ "src.services.input.agents.embeddings_adapter"+ ):+ ga.graphdb_description = "db"+ agent.run_batched_janitor(+ text="Alice met Bob at the park.",+ cost_ledger=ledger,+ )+ pending = agent.take_pending_relationships()+ assert len(pending) == 1+ assert pending[0].name == "MET"+ assert "ungrounded" in ledger.janitor_drop_reasons+++def test_deterministic_reject_skips_llm():+ agent = ArchitectAgent(+ llm_adapter=MagicMock(),+ cache_adapter=MagicMock(),+ kg=MagicMock(),+ vector_store=MagicMock(),+ embeddings=MagicMock(),+ ingestion_manager=MagicMock(),+ )+ placeholder = ArchitectAgentRelationship(+ flow_key="fk",+ uuid="r1",+ name="MET",+ description="Alice met Bob",+ properties={"source_span": "Alice met Bob"},+ tail=ArchitectAgentEntity(+ uuid="a", name="PERSON", type="PERSON", description="", properties={}+ ),+ tip=ArchitectAgentEntity(+ uuid="b", name="Bob", type="PERSON", description="", properties={}+ ),+ )+ agent.pending_persistence_batches = [[placeholder]]+ ledger = IngestCostLedger()+ with patch("src.core.agents.janitor_agent.JanitorAgent") as jan_cls:+ agent.run_batched_janitor(+ text="Alice met Bob at the park.",+ cost_ledger=ledger,+ )+ jan_cls.assert_not_called()+ assert agent.take_pending_relationships() == []+ assert ledger.janitor_rejected == 1+ assert ledger.janitor_ran == 0+ assert "type_named_placeholder_endpoint" in ledger.janitor_drop_reasons