Python
"""Question-blind exact-L2 supplement for the Ultra persona catalog.
The supplement is generated after the balanced catalog. It selects only from
active/latest L2 facts not already represented by that catalog, applies direct
and semantic forget closure, and persists a special internal L6 node. The node
never enters ordinary Schema recall and consumes no profile slot; Ultra appends
it to the unique visible L0 chain after the balanced catalog.
Raw and ordinary S1 records are never read or mutated by this module.
"""
from __future__ import annotations
import hashlib
import json
import os
import re
import uuid
from datetime import datetime
from typing import Any, Dict, Iterable, List, Optional, Sequence
from ..agent.llm_provider import LLMProvider
from ..config import MemoryConfig
from ..data.graph_store_base import GraphStoreBase
from ..data.vector_store_base import VectorStoreBase
from ..models.memory import MemoryLayer, MemoryNode, MemoryStatus, SourceType
from ..utils.source_filter import merge_sources
from .balanced_catalog import (
CATALOG_DOMAINS,
catalog_evidence_fingerprint,
extract_forget_targets,
compact_verified_catalog_entries,
get_active_balanced_catalog,
normalize_catalog_selection,
parse_custom,
source_bank_fingerprint,
suppress_forget_targets,
)
SUPPLEMENT_SCHEMA_KIND = "specific_persona_supplement"
SUPPLEMENT_VERSION = 1
SUPPLEMENT_MAX_PER_DOMAIN = 2
SUPPLEMENT_MAX_TOTAL = 16
SUPPLEMENT_MARKER = "System2 specific supplement (exact stored evidence):"
SUPPLEMENT_MATERIALIZATION_VERSION = "exact-evidence-artifact-v1"
_SUPPLEMENT_ITEM_RE = re.compile(r"^- \[([^\]]+)\] (.*)$")
_FORGET_RE = re.compile(
r"(?i)\b(?:forget|forgotten|do not remember|asked (?:the )?assistant to forget|"
r"requested (?:that )?.* be forgotten)\b"
)
_ASSISTANT_ADVICE_RE = re.compile(
r"(?i)^\s*(?:the )?assistant (?:advised|recommended|suggested|explained|outlined)\b"
)
_GENERIC_QUESTION_RE = re.compile(
r"(?i)^\s*the user (?:asked|wants to know|is curious|wondered)\b"
)
_CURIOUS_RE = re.compile(r"(?i)^\s*the user is curious\b")
_ABBREVIATIONS = ("Dr.", "Mr.", "Mrs.", "Ms.", "Prof.", "St.", "U.S.", "e.g.", "i.e.")
_SPECIFIC_PROMPT = """Build a balanced specific-personal-evidence supplement from numbered recorded facts.
Return strict JSON only:
{"domains":{"identity_relationships":[1],"health_accessibility":[2]}}
Use exactly these domain keys:
- identity_relationships: identity and the user's own family/relationship structure
- health_accessibility: explicit health, sensory, mobility, ergonomic constraints
- preferences_collections: explicit durable likes, dislikes, tastes, collections
- habits_items: recurring habits and personally owned/carried/used items
- work_education: durable role, expertise, education, recurring responsibilities
- culture_creative_experience: concrete cultural/creative interests and strong experiences
- values_coping: durable values and personally practiced coping resources
- current_constraints_goals: ongoing burdens, constraints, or durable active goals
Select at most 2 entry numbers per domain and at most 16 unique entries total.
You may select fewer; do not fill a quota.
Prefer atomic facts whose wording independently states the useful personal
constraint. Strong examples include a named family relationship, a diagnosed or
experienced health/accessibility constraint, a clearly stated durable preference,
a recurring routine, a personally owned/carried item, or an ongoing goal.
Penalize broad biography, occupation/location alone, generic topical interest,
temporary events, and compound records whose useful claim is not self-contained.
Reject forget requests, assistant advice, generic questions or curiosity without
a personal fact, temporary logistics, one-off writing/translation requests, and
facts mainly about another person. Do not infer, summarize, rewrite, or create facts.
An entry may appear in at most one domain. Output entry numbers only."""
_CLAUSE_PROMPT = """Select explicit personal claims from deterministic exact source candidates.
Return strict JSON only:
{"selected":[{"parent":1,"candidates":[2]}]}
You receive question-shaped stored memory parents and numbered exact contiguous
candidate spans generated by code. For each parent select 0-2 candidates only
when they explicitly state a user-owned fact that actually happened or
persists independently of the question framing.
Valid evidence includes an experienced health/accessibility symptom, explicit
preference or dislike, recurring habit or owned item, the user's own family or
relationship structure, concrete personal experience, durable value shaped by
experience, or an ongoing state/constraint.
Reject pure curiosity or a hypothetical premise, assistant advice, generic
requests for tips, facts mainly about another person, temporary unrelated
logistics, inferred traits, and text that merely repeats what the user asked.
Preserve negation, ownership, person, time, and current-vs-past state. Select
only supplied parent/candidate numbers. Do not generate or rewrite text."""
_SEMANTIC_FORGET_PROMPT = """Identify specific supplement entries that must be suppressed because the user explicitly asked to forget the same personal fact.
Return strict JSON only: {"remove":[1,3]}.
You receive atomic forget targets and numbered exact stored supplement entries.
Remove an entry only when it restates, paraphrases, clearly entails, contains,
or directly reveals one of the forgotten personal facts. This includes a
specific episode that makes the forgotten fact evident even with different wording.
Do not remove an entry merely because it shares a topic, person, occupation,
location, broad theme, health domain, hobby domain, or generic emotional tone.
Preserve unrelated facts. Ignore vague targets such as "the fact" or targets
with no specific personal content. Do not create, rewrite, or summarize facts.
Output supplied entry numbers only and no explanation."""
def _env_bool(name: str, default: str = "false") -> bool:
return os.getenv(name, default).strip().lower() in {"1", "true", "yes", "on"}
def specific_supplement_generation_enabled() -> bool:
return _env_bool("MEMORY_EVOLING_SPECIFIC_SUPPLEMENT_ENABLED")
def specific_supplement_user_profile_enabled() -> bool:
value = os.getenv("MEMORY_USER_PROFILE_SPECIFIC_SUPPLEMENT_ENABLED")
if value is not None:
return value.strip().lower() in {"1", "true", "yes", "on"}
return _env_bool("MEMORY_HYBRID_V2_SPECIFIC_SUPPLEMENT_L0_ENABLED")
def specific_supplement_l0_enabled() -> bool:
"""Backward-compatible alias for the pre-profile-split helper name."""
return specific_supplement_user_profile_enabled()
def is_specific_supplement_custom(custom: Optional[Dict[str, Any]]) -> bool:
return str((custom or {}).get("schema_kind") or "") == SUPPLEMENT_SCHEMA_KIND
def is_specific_supplement_hit(hit: Dict[str, Any]) -> bool:
return is_specific_supplement_custom(parse_custom(
hit.get("custom_json") or hit.get("custom") or {}
))
def is_internal_profile_chain_custom(custom: Optional[Dict[str, Any]]) -> bool:
from .balanced_catalog import is_balanced_catalog_custom
return is_balanced_catalog_custom(custom) or is_specific_supplement_custom(custom)
def is_internal_profile_chain_hit(hit: Dict[str, Any]) -> bool:
return is_internal_profile_chain_custom(parse_custom(
hit.get("custom_json") or hit.get("custom") or {}
))
def _parse_json_object(text: str) -> Optional[Dict[str, Any]]:
value = str(text or "").strip()
fenced = re.search(r"```(?:json)?\s*(\{.*?\})\s*```", value, re.S | re.I)
if fenced:
value = fenced.group(1)
else:
obj = re.search(r"\{.*\}", value, re.S)
value = obj.group(0) if obj else value
try:
parsed = json.loads(value)
except json.JSONDecodeError:
try:
from json_repair import repair_json
parsed = repair_json(value, return_objects=True)
except Exception:
return None
return parsed if isinstance(parsed, dict) else None
def _clean(value: Any) -> str:
return " ".join(str(value or "").split()).strip()
def supplement_reject_reason(content: str) -> str:
value = _clean(content)
if not value:
return "empty"
if _FORGET_RE.search(value):
return "forget"
if _ASSISTANT_ADVICE_RE.search(value):
return "assistant_advice"
if _GENERIC_QUESTION_RE.search(value):
return "generic_question"
return ""
def eligible_supplement_facts(nodes: Iterable[Any]) -> List[Any]:
output = []
for node in nodes:
if getattr(node, "layer", None) != MemoryLayer.L2_FACT:
continue
if getattr(node, "status", None) != MemoryStatus.ACTIVE:
continue
if not bool(getattr(node, "is_latest", True)):
continue
if (getattr(node, "custom", None) or {}).get("memory_kind") == "canonical_state":
continue
output.append(node)
def stable_key(node: Any) -> tuple:
content = _clean(getattr(node, "content", "")).casefold()
tags = tuple(
sorted(_clean(value).casefold() for value in getattr(node, "tags", None) or [])
)
event_time = (
getattr(node, "memory_at", None)
or getattr(node, "gmt_created", None)
or getattr(node, "valid_from", None)
)
event_key = event_time.isoformat() if isinstance(event_time, datetime) else ""
# node_id is a final tie-break only. Clone remaps it, while the fields
# above remain stable and determine every prompt-visible index.
return (
content,
tags,
event_key,
str(getattr(node, "node_id", "") or ""),
)
return sorted(output, key=stable_key)
def collect_supplement_forget_targets(nodes: Iterable[Any]) -> List[str]:
"""Match the frozen offline supplement inventory.
The supplement experiment intentionally started from every L2 containing a
forget-related marker, then relied on exact/full-anchor and semantic review
to avoid broad-topic deletion. This differs from the catalog32 lifecycle's
conservative explicit-marker detector and must remain local to this module.
"""
targets: List[str] = []
seen = set()
for node in nodes:
content = str(getattr(node, "content", "") or "")
if not _FORGET_RE.search(content):
continue
for target in extract_forget_targets(content):
key = target.casefold()
if key not in seen:
seen.add(key)
targets.append(target)
return targets
def prepare_specific_entries(
domains: Dict[str, List[int]],
facts: Sequence[Any],
) -> tuple[List[Dict[str, Any]], List[Dict[str, Any]]]:
selected: List[Dict[str, Any]] = []
rejected: List[Dict[str, Any]] = []
for domain in CATALOG_DOMAINS:
for index in domains.get(domain) or []:
node = facts[index - 1]
content = _clean(getattr(node, "content", ""))
record = {
"domain": domain,
"source_index": index,
"node_id": str(getattr(node, "node_id", "") or ""),
"content": content,
"tags": [str(value) for value in getattr(node, "tags", None) or []],
}
reason = supplement_reject_reason(content)
if reason in {"forget", "assistant_advice"}:
rejected.append({**record, "reason": reason})
else:
# Generic/question-shaped parents are preserved temporarily so
# deterministic exact-clause salvage can inspect them.
selected.append(record)
return selected, rejected
def _trim_candidate(value: str) -> str:
candidate = _clean(value).strip(" ,;:-")
candidate = re.sub(
r"(?i),?\s+and\s+the\s+assistant\s+"
r"(?:recommended|suggested|explained|advised|outlined)\b.*$",
"", candidate,
).strip(" ,;:-")
candidate = re.sub(
r"(?i);?\s+the\s+assistant\s+"
r"(?:recommended|suggested|explained|advised|outlined)\b.*$",
"", candidate,
).strip(" ,;:-")
return candidate
def _bounded_parts(value: str) -> List[str]:
candidate = _trim_candidate(value)
if len(candidate) < 35:
return []
if len(candidate) <= 360:
return [candidate]
parts = [
_trim_candidate(part)
for part in re.split(
r"(?<=[;.!?])\s+|,\s+(?=(?:and|but|while|which|who|their|the user)\b)",
candidate,
)
if _trim_candidate(part)
]
return [part for part in parts if 35 <= len(part) <= 360]
def _split_sentences_exact(text: str) -> List[str]:
value = _clean(text)
protected = value
replacements: Dict[str, str] = {}
for index, abbreviation in enumerate(_ABBREVIATIONS):
marker = f"<DOT{index}>"
replacements[marker] = abbreviation
protected = protected.replace(abbreviation, abbreviation.replace(".", marker))
parts = [
part.strip() for part in re.split(r"(?<=[.!?])\s+", protected)
if part.strip()
]
output = []
for part in parts:
for marker, abbreviation in replacements.items():
part = part.replace(abbreviation.replace(".", marker), abbreviation)
output.append(part)
return output or ([value] if value else [])
def deterministic_claim_candidates(parent: str) -> List[str]:
source = _clean(parent)
candidates: List[str] = []
def add(value: str) -> None:
for candidate in _bounded_parts(value):
if candidate not in source:
continue
if supplement_reject_reason(candidate) or _CURIOUS_RE.search(candidate):
continue
if candidate not in candidates:
candidates.append(candidate)
for sentence in _split_sentences_exact(source):
add(sentence)
for marker in (
r"(?i)\bwhy\s+", r"(?i)\babout\s+", r"(?i)\bindicating\s+",
r"(?i)\bpreferred\s+", r"(?i)\baccepted\s+",
):
match = re.search(marker, sentence)
if match:
add(sentence[match.end():])
for marker in (
r"(?i)\bthe user was\s+", r"(?i)\bthe user has\s+",
r"(?i)\bthe user had\s+", r"(?i)\bthe user is\s+",
r"(?i)\btheir\s+",
):
for match in re.finditer(marker, sentence):
add(sentence[match.start():])
for part in sentence.split(";")[1:]:
add(part)
return candidates
def normalize_clause_selection(
raw: Optional[Dict[str, Any]],
parents: Sequence[Dict[str, Any]],
) -> Dict[str, int]:
if not isinstance(raw, dict) or not isinstance(raw.get("selected"), list):
return {}
output: Dict[str, int] = {}
seen_parents = set()
for item in raw["selected"]:
if not isinstance(item, dict):
continue
try:
parent_index = int(item.get("parent"))
except (TypeError, ValueError):
continue
if not 1 <= parent_index <= len(parents) or parent_index in seen_parents:
continue
seen_parents.add(parent_index)
parent = parents[parent_index - 1]
for value in item.get("candidates") or []:
try:
candidate_index = int(value)
except (TypeError, ValueError):
continue
if 1 <= candidate_index <= len(parent["candidates"]):
output[str(parent["node_id"])] = candidate_index
break
return output
def specific_forget_targets(values: Iterable[str]) -> List[str]:
output: List[str] = []
seen = set()
for value in values:
text = _clean(value).strip(" .")
lowered = text.casefold()
terms = re.findall(r"[a-z0-9]+", lowered)
if lowered in {"the fact", "fact", "it", "this"} or len(terms) < 3:
continue
if lowered not in seen:
seen.add(lowered)
output.append(text)
return output
def normalize_remove_selection(
raw: Optional[Dict[str, Any]], count: int,
) -> List[int]:
if not isinstance(raw, dict) or not isinstance(raw.get("remove"), list):
return []
output = []
for value in raw["remove"]:
try:
index = int(value)
except (TypeError, ValueError):
continue
if 1 <= index <= count and index not in output:
output.append(index)
return sorted(output)
def render_specific_supplement(entries: Iterable[Dict[str, Any]]) -> str:
return SUPPLEMENT_MARKER + "\n" + "\n".join(
f"- [{entry['domain']}] {entry['content']}" for entry in entries
)
def parse_rendered_specific_entries(content: str) -> List[Dict[str, str]]:
"""Parse the persisted exact-evidence lines without interpreting them."""
output: List[Dict[str, str]] = []
for line in str(content or "").splitlines():
match = _SUPPLEMENT_ITEM_RE.match(line.strip())
if not match:
continue
output.append({
"domain": match.group(1).strip(),
"content": _clean(match.group(2)),
})
return output
def _supplement_node_id(
user_id: str, source_fingerprint: str, evidence_fingerprint: str,
) -> str:
value = (
f"specific-supplement:v{SUPPLEMENT_VERSION}:user-profile:{user_id}:"
f"{source_fingerprint}:{evidence_fingerprint}"
)
return f"catalog_supplement_{uuid.uuid5(uuid.NAMESPACE_URL, value).hex}"
async def _active_supplements(
graph_store: GraphStoreBase, *, user_id: str,
) -> List[MemoryNode]:
canonical = await graph_store.get_all_nodes(
isolation_key=MemoryNode.build_isolation_key(user_id, ""),
layer=MemoryLayer.L6_SCHEMA,
status=MemoryStatus.ACTIVE,
limit=200,
)
nodes_by_id = {node.node_id: node for node in canonical}
get_by_user = getattr(graph_store, "get_nodes_by_user", None)
if callable(get_by_user):
for node in await get_by_user(
user_id=user_id,
layer=MemoryLayer.L6_SCHEMA,
status=MemoryStatus.ACTIVE,
limit=1000,
):
nodes_by_id[node.node_id] = node
return [
node for node in nodes_by_id.values()
if is_specific_supplement_custom(node.custom)
]
async def _retire_nodes(
graph_store: GraphStoreBase, nodes: Sequence[MemoryNode], keep_id: str = "",
) -> tuple[bool, List[str]]:
retired: List[str] = []
for node in nodes:
if node.node_id == keep_id:
continue
ok = await graph_store.update_node(node.node_id, {
"status": MemoryStatus.SUPERSEDED.value,
"is_latest": False,
"valid_until": datetime.now(),
})
if not ok:
for node_id in retired:
await graph_store.update_node(node_id, {
"status": MemoryStatus.ACTIVE.value,
"is_latest": True,
"valid_until": None,
})
return False, retired
retired.append(node.node_id)
return True, retired
async def _adopt_cloned_specific_supplement(
*, current: Sequence[MemoryNode], facts: Sequence[Any],
combined_source_fingerprint: str, catalog_node_id: str,
user_id: str, vector_store: VectorStoreBase,
graph_store: GraphStoreBase,
) -> Optional[Dict[str, Any]]:
"""Adopt a clone-remapped supplement without re-sampling the selector.
Clone already remaps ``evidence_chain`` and DERIVED_FROM edges, but custom
domain ids and source fingerprints still refer to the source namespace.
Adoption is therefore allowed only for an unmistakable clone state:
* every recorded custom evidence id is stale in the current L2 bank;
* every remapped evidence-chain id is a current active/latest residual L2;
* each visible line equals, or is an exact contiguous clause of, its paired
evidence parent (the latter preserves deterministic clause salvage);
* line, evidence, and recorded selected counts are identical.
A normal same-namespace bank change leaves at least one custom evidence id
valid and cannot enter this path. Any ambiguity falls through to the
existing fail-closed generation path.
"""
fact_by_id = {
str(getattr(node, "node_id", "") or ""): node for node in facts
}
current_ids = set(fact_by_id)
for old in current:
custom = dict(old.custom or {})
if int(custom.get("specific_supplement_version") or 0) != SUPPLEMENT_VERSION:
continue
expected_count = int(custom.get("selected_count") or 0)
evidence_ids = [str(value) for value in old.evidence_chain or [] if value]
parsed = parse_rendered_specific_entries(old.content)
if (
bool(custom.get("artifact_empty"))
and expected_count == 0
and not evidence_ids
and not parsed
and str(custom.get("base_catalog_node_id") or "") != catalog_node_id
):
evidence_fingerprint = catalog_evidence_fingerprint([])
node_id = _supplement_node_id(
user_id, combined_source_fingerprint,
evidence_fingerprint,
)
node = MemoryNode(
node_id=node_id, user_id=user_id, agent_id="",
source=merge_sources(getattr(fact, "source", None) for fact in facts),
layer=MemoryLayer.L6_SCHEMA, content=SUPPLEMENT_MARKER,
confidence=1.0, status=MemoryStatus.ACTIVE,
source_type=SourceType.COMPOSITE,
memory_at=datetime.now(), tags=["specific-persona-supplement"],
evidence_chain=[],
custom={
**custom,
"schema_type": "internal_profile_chain",
"schema_kind": SUPPLEMENT_SCHEMA_KIND,
"specific_supplement_version": SUPPLEMENT_VERSION,
"source_bank_fingerprint": combined_source_fingerprint,
"base_catalog_node_id": catalog_node_id,
"evidence_fingerprint": evidence_fingerprint,
"selected_count": 0,
"domains": {domain: [] for domain in CATALOG_DOMAINS},
"artifact_empty": True,
"clone_adoption_version": "exact-evidence-remap-v1",
"profile_scope": "user",
"retrieval_text": "",
},
)
created_new = await graph_store.get_node(node_id) is None
if created_new:
await graph_store.upsert_memory_node(node)
ok, retired = await _retire_nodes(
graph_store, current, keep_id=node_id,
)
if not ok:
if created_new:
await graph_store.delete_node(node_id)
return None
return {
"success": True, "enabled": True, "skipped": True,
"reason": "cloned_specific_supplement_adopted",
"created": [{
"node_id": node_id,
"evidence_list": [],
"selected_count": 0,
}] if created_new else [],
"supplement_node_id": node_id,
"source_count": len(facts),
"selected_count": 0,
"retired_supplement_ids": retired,
"prompt_tokens": 0,
"completion_tokens": 0,
}
recorded_ids = {
str(node_id)
for values in (custom.get("domains") or {}).values()
for node_id in values or []
if node_id
}
legacy_agent_scope = bool(str(getattr(old, "agent_id", "") or "").strip())
if not recorded_ids:
continue
if not legacy_agent_scope and any(
node_id in current_ids for node_id in recorded_ids
):
continue
if (
not parsed
or len(parsed) != expected_count
or len(evidence_ids) != expected_count
or len(set(evidence_ids)) != expected_count
):
continue
mapped: List[Dict[str, Any]] = []
valid = True
for entry, evidence_id in zip(parsed, evidence_ids):
fact = fact_by_id.get(evidence_id)
visible = _clean(entry["content"])
parent = _clean(getattr(fact, "content", "")) if fact is not None else ""
if (
fact is None
or entry["domain"] not in CATALOG_DOMAINS
or not visible
or visible not in parent
):
valid = False
break
mapped.append({
"domain": entry["domain"],
"node_id": evidence_id,
"content": visible,
"tags": [str(value) for value in getattr(fact, "tags", None) or []],
})
if not valid:
continue
evidence_fingerprint = catalog_evidence_fingerprint(mapped)
node_id = _supplement_node_id(
user_id, combined_source_fingerprint, evidence_fingerprint,
)
node = MemoryNode(
node_id=node_id, user_id=user_id, agent_id="",
source=merge_sources(
getattr(fact_by_id.get(evidence_id), "source", None)
for evidence_id in evidence_ids
),
layer=MemoryLayer.L6_SCHEMA,
content=str(old.content or "").strip(),
confidence=1.0, status=MemoryStatus.ACTIVE,
source_type=SourceType.COMPOSITE,
memory_at=datetime.now(), tags=["specific-persona-supplement"],
evidence_chain=evidence_ids,
custom={
**custom,
"schema_type": "internal_profile_chain",
"schema_kind": SUPPLEMENT_SCHEMA_KIND,
"specific_supplement_version": SUPPLEMENT_VERSION,
"source_bank_fingerprint": combined_source_fingerprint,
"base_catalog_node_id": catalog_node_id,
"evidence_fingerprint": evidence_fingerprint,
"selected_count": len(mapped),
"domains": {
domain: [
entry["node_id"] for entry in mapped
if entry["domain"] == domain
]
for domain in CATALOG_DOMAINS
},
"clone_adoption_version": "exact-evidence-remap-v1",
"profile_scope": "user",
"retrieval_text": "",
},
)
created_new = await graph_store.get_node(node_id) is None
if created_new:
await graph_store.upsert_memory_node(node)
try:
for evidence_id in evidence_ids:
evidence_node = await vector_store.get_by_id(evidence_id)
if evidence_node is None:
raise RuntimeError(
f"adopted supplement evidence missing: {evidence_id}"
)
await graph_store.ensure_vdbref(evidence_id, evidence_node.layer.value)
if await graph_store.add_derived_from(node_id, evidence_id) is False:
raise RuntimeError(
f"adopted supplement link failed: {evidence_id}"
)
except Exception:
if created_new:
await graph_store.delete_node(node_id)
continue
ok, retired = await _retire_nodes(graph_store, current, keep_id=node_id)
if not ok:
if created_new:
await graph_store.delete_node(node_id)
return None
return {
"success": True, "enabled": True, "skipped": True,
"reason": "cloned_specific_supplement_adopted",
"created": [{
"node_id": node_id,
"evidence_list": evidence_ids,
"selected_count": len(mapped),
}] if created_new else [],
"supplement_node_id": node_id,
"source_count": len(facts),
"selected_count": len(mapped),
"retired_supplement_ids": retired,
"prompt_tokens": 0,
"completion_tokens": 0,
}
return None
async def _materialize_specific_supplement_unlocked(
*, user_id: str,
entries: Sequence[Dict[str, Any]],
vector_store: VectorStoreBase,
graph_store: Optional[GraphStoreBase],
materialization_source: str = "",
) -> Dict[str, Any]:
"""Persist a precomputed exact-evidence System2 supplement safely.
This is a generic lifecycle operation for importing a previously generated
question-blind System2 artifact. Every visible line is re-grounded against
the current user's active/latest residual L2 bank. The operation never
reads raw and performs no LLM call. Any invalid, ambiguous, duplicated, or
catalog-overlapping parent fails the whole transaction before a new node is
made visible.
An empty artifact is persisted as an internal empty state. It prevents a
later evolve from randomly regenerating forgotten/removed content, while
``get_active_specific_supplement`` keeps it invisible to Search.
"""
if graph_store is None or not hasattr(vector_store, "list_by_user"):
return {
"success": False,
"error": "specific supplement storage unavailable",
"created": [],
}
catalog = await get_active_balanced_catalog(
graph_store, user_id=user_id,
)
if catalog is None:
return {
"success": False,
"error": "balanced catalog required before artifact materialization",
"created": [],
}
all_active_l2 = list(await vector_store.list_by_user(
user_id=user_id,
status_filter=[MemoryStatus.ACTIVE],
layers=[MemoryLayer.L2_FACT],
))
catalog_ids = {str(value) for value in catalog.evidence_chain or []}
facts = [
node for node in eligible_supplement_facts(all_active_l2)
if str(node.node_id) not in catalog_ids
]
fact_by_id = {str(node.node_id): node for node in facts}
source_fingerprint = source_bank_fingerprint(facts)
combined_source_fingerprint = hashlib.sha256(
f"{catalog.node_id}:{source_fingerprint}".encode("utf-8")
).hexdigest()
if len(entries) > SUPPLEMENT_MAX_TOTAL:
return {
"success": False,
"error": f"artifact total overflow: {len(entries)}",
"created": [],
}
mapped: List[Dict[str, Any]] = []
seen_parents = set()
domain_counts: Dict[str, int] = {}
for index, entry in enumerate(entries, 1):
domain = str(entry.get("domain") or "")
parent_id = str(
entry.get("parent_node_id") or entry.get("node_id") or ""
)
visible = _clean(entry.get("content") or entry.get("visible_content"))
parent = fact_by_id.get(parent_id)
if domain not in CATALOG_DOMAINS:
return {
"success": False,
"error": f"artifact entry {index} invalid domain: {domain}",
"created": [],
}
if parent is None:
return {
"success": False,
"error": f"artifact entry {index} parent outside residual L2: {parent_id}",
"created": [],
}
parent_content = _clean(parent.content)
if not visible or visible not in parent_content:
return {
"success": False,
"error": f"artifact entry {index} is not exact parent evidence",
"created": [],
}
reject = supplement_reject_reason(visible)
if reject:
return {
"success": False,
"error": f"artifact entry {index} rejected: {reject}",
"created": [],
}
if parent_id in seen_parents:
return {
"success": False,
"error": f"artifact duplicate parent: {parent_id}",
"created": [],
}
seen_parents.add(parent_id)
domain_counts[domain] = domain_counts.get(domain, 0) + 1
if domain_counts[domain] > SUPPLEMENT_MAX_PER_DOMAIN:
return {
"success": False,
"error": f"artifact domain overflow: {domain}",
"created": [],
}
mapped.append({
"domain": domain,
"node_id": parent_id,
"content": visible,
"tags": [str(value) for value in parent.tags or []],
})
forget_targets = collect_supplement_forget_targets(all_active_l2)
_kept, direct_suppressed = suppress_forget_targets(mapped, forget_targets)
if direct_suppressed:
return {
"success": False,
"error": "artifact exposes a direct forget target",
"created": [],
"suppressed_forget_target_entries": direct_suppressed,
}
evidence_fingerprint = catalog_evidence_fingerprint(mapped)
node_id = _supplement_node_id(
user_id, combined_source_fingerprint, evidence_fingerprint,
)
evidence_ids = [entry["node_id"] for entry in mapped]
current = await _active_supplements(
graph_store, user_id=user_id,
)
node = MemoryNode(
node_id=node_id,
user_id=user_id,
agent_id="",
source=merge_sources(
getattr(fact_by_id.get(evidence_id), "source", None)
for evidence_id in evidence_ids
),
layer=MemoryLayer.L6_SCHEMA,
content=(render_specific_supplement(mapped) if mapped else SUPPLEMENT_MARKER),
confidence=1.0,
status=MemoryStatus.ACTIVE,
source_type=SourceType.COMPOSITE,
memory_at=datetime.now(),
tags=["specific-persona-supplement"],
evidence_chain=evidence_ids,
custom={
"schema_type": "internal_profile_chain",
"schema_kind": SUPPLEMENT_SCHEMA_KIND,
"specific_supplement_version": SUPPLEMENT_VERSION,
"source_bank_fingerprint": combined_source_fingerprint,
"base_catalog_node_id": catalog.node_id,
"evidence_fingerprint": evidence_fingerprint,
"selected_count": len(mapped),
"max_per_domain": SUPPLEMENT_MAX_PER_DOMAIN,
"max_total": SUPPLEMENT_MAX_TOTAL,
"direct_forget_suppressed_count": 0,
"semantic_forget_removed_count": 0,
"clause_salvage_count": sum(
entry["content"] != _clean(fact_by_id[entry["node_id"]].content)
for entry in mapped
),
"domains": {
domain: [
entry["node_id"] for entry in mapped
if entry["domain"] == domain
]
for domain in CATALOG_DOMAINS
},
"materialization_version": SUPPLEMENT_MATERIALIZATION_VERSION,
"materialization_source": str(materialization_source or "")[:240],
"artifact_empty": not mapped,
"profile_scope": "user",
"retrieval_text": "",
},
)
created_new = await graph_store.get_node(node_id) is None
if created_new:
await graph_store.upsert_memory_node(node)
try:
for evidence_id in evidence_ids:
evidence_node = await vector_store.get_by_id(evidence_id)
if evidence_node is None:
raise RuntimeError(
f"materialized supplement evidence missing: {evidence_id}"
)
if (
evidence_node.user_id != user_id
or evidence_node.layer != MemoryLayer.L2_FACT
or evidence_node.status != MemoryStatus.ACTIVE
or not bool(evidence_node.is_latest)
):
raise RuntimeError(
f"materialized evidence outside active user scope: {evidence_id}"
)
await graph_store.ensure_vdbref(evidence_id, evidence_node.layer.value)
if await graph_store.add_derived_from(node_id, evidence_id) is False:
raise RuntimeError(
f"materialized supplement link failed: {evidence_id}"
)
except Exception as exc:
if created_new:
await graph_store.delete_node(node_id)
return {
"success": False,
"error": str(exc),
"created": [],
}
ok, retired = await _retire_nodes(graph_store, current, keep_id=node_id)
if not ok:
if created_new:
await graph_store.delete_node(node_id)
return {
"success": False,
"error": "failed to retire old specific supplement",
"created": [],
}
return {
"success": True,
"reason": "specific_supplement_artifact_materialized",
"created": [{
"node_id": node_id,
"evidence_list": evidence_ids,
"selected_count": len(mapped),
}] if created_new else [],
"supplement_node_id": node_id,
"source_count": len(facts),
"selected_count": len(mapped),
"retired_supplement_ids": retired,
"materialization_version": SUPPLEMENT_MATERIALIZATION_VERSION,
"prompt_tokens": 0,
"completion_tokens": 0,
}
async def _run_specific_supplement_generation_unlocked(
*, user_id: str, vector_store: VectorStoreBase,
graph_store: Optional[GraphStoreBase], config: MemoryConfig,
) -> Dict[str, Any]:
if not specific_supplement_generation_enabled():
return {"success": True, "enabled": False, "created": [],
"prompt_tokens": 0, "completion_tokens": 0}
if graph_store is None or not hasattr(vector_store, "list_by_user"):
return {"success": False, "enabled": True,
"error": "specific supplement storage unavailable", "created": [],
"prompt_tokens": 0, "completion_tokens": 0}
catalog = await get_active_balanced_catalog(
graph_store, user_id=user_id,
)
if catalog is None:
return {"success": False, "enabled": True,
"error": "balanced catalog required before specific supplement",
"created": [], "prompt_tokens": 0, "completion_tokens": 0}
all_active_l2 = list(await vector_store.list_by_user(
user_id=user_id,
status_filter=[MemoryStatus.ACTIVE], layers=[MemoryLayer.L2_FACT],
))
forget_targets = collect_supplement_forget_targets(all_active_l2)
catalog_ids = set(str(value) for value in catalog.evidence_chain or [])
facts = [
node for node in eligible_supplement_facts(all_active_l2)
if node.node_id not in catalog_ids
]
source_fingerprint = source_bank_fingerprint(facts)
combined_source_fingerprint = hashlib.sha256(
f"{catalog.node_id}:{source_fingerprint}".encode("utf-8")
).hexdigest()
current = await _active_supplements(
graph_store, user_id=user_id,
)
matching = [
node for node in current
if not str(getattr(node, "agent_id", "") or "").strip()
if int((node.custom or {}).get("specific_supplement_version") or 0)
== SUPPLEMENT_VERSION
and str((node.custom or {}).get("source_bank_fingerprint") or "")
== combined_source_fingerprint
]
if matching:
return {
"success": True, "enabled": True, "skipped": True,
"reason": "current_specific_supplement_exists", "created": [],
"supplement_node_id": matching[0].node_id,
"source_count": len(facts),
"selected_count": int((matching[0].custom or {}).get("selected_count") or 0),
"prompt_tokens": 0, "completion_tokens": 0,
}
adopted = await _adopt_cloned_specific_supplement(
current=current,
facts=facts,
combined_source_fingerprint=combined_source_fingerprint,
catalog_node_id=catalog.node_id,
user_id=user_id,
vector_store=vector_store,
graph_store=graph_store,
)
if adopted is not None:
return adopted
if not facts:
ok, retired = await _retire_nodes(graph_store, current)
return {
"success": ok, "enabled": True, "skipped": True,
"reason": "empty_residual_bank", "created": [],
"retired_supplement_ids": retired,
"source_count": 0, "selected_count": 0,
"prompt_tokens": 0, "completion_tokens": 0,
}
provider = LLMProvider(config)
prompt_lines = ["Recorded facts:"]
for index, node in enumerate(facts[:120], 1):
tags = ", ".join(str(value) for value in list(node.tags or [])[:4])
prefix = f"[{tags}] " if tags else ""
prompt_lines.append(f"{index}. {prefix}{_clean(node.content)[:180]}")
total_prompt_tokens = 0
total_completion_tokens = 0
try:
response = await provider.complete_messages(
messages=[
{"role": "system", "content": _SPECIFIC_PROMPT},
{"role": "user", "content": "\n".join(prompt_lines)},
],
max_tokens=512,
)
total_prompt_tokens += int(response.prompt_tokens or 0)
total_completion_tokens += int(response.completion_tokens or 0)
except Exception as exc:
return {"success": False, "enabled": True, "error": str(exc),
"created": [], "source_count": len(facts),
"prompt_tokens": total_prompt_tokens,
"completion_tokens": total_completion_tokens}
parsed = _parse_json_object(response.content or "")
if parsed is None:
return {"success": False, "enabled": True,
"error": "invalid specific supplement JSON", "created": [],
"source_count": len(facts), "prompt_tokens": total_prompt_tokens,
"completion_tokens": total_completion_tokens}
domains = normalize_catalog_selection(
parsed, candidate_count=min(len(facts), 120),
max_per_domain=SUPPLEMENT_MAX_PER_DOMAIN,
max_total=SUPPLEMENT_MAX_TOTAL,
)
entries, rejected = prepare_specific_entries(domains, facts[:120])
entries, direct_suppressed = suppress_forget_targets(entries, forget_targets)
entries = compact_verified_catalog_entries(
entries, max_per_domain=SUPPLEMENT_MAX_PER_DOMAIN,
max_total=SUPPLEMENT_MAX_TOTAL,
)
generic = [
entry for entry in entries
if supplement_reject_reason(entry["content"]) == "generic_question"
]
salvaged_by_parent: Dict[str, Dict[str, Any]] = {}
if generic:
parents = [{**entry, "candidates": deterministic_claim_candidates(entry["content"])}
for entry in generic]
if any(parent["candidates"] for parent in parents):
lines = ["Question-shaped parents and deterministic exact candidates:"]
for parent_index, parent in enumerate(parents, 1):
lines.append(f"P{parent_index}. parent={parent['node_id']} | {parent['content']}")
for candidate_index, candidate in enumerate(parent["candidates"], 1):
lines.append(f" P{parent_index}C{candidate_index}. {candidate}")
try:
clause_response = await provider.complete_messages(
messages=[
{"role": "system", "content": _CLAUSE_PROMPT},
{"role": "user", "content": "\n".join(lines)},
],
max_tokens=512,
)
total_prompt_tokens += int(clause_response.prompt_tokens or 0)
total_completion_tokens += int(clause_response.completion_tokens or 0)
except Exception as exc:
return {"success": False, "enabled": True, "error": str(exc),
"created": [], "source_count": len(facts),
"prompt_tokens": total_prompt_tokens,
"completion_tokens": total_completion_tokens}
clause_parsed = _parse_json_object(clause_response.content or "")
if clause_parsed is None:
return {"success": False, "enabled": True,
"error": "invalid specific clause JSON", "created": [],
"source_count": len(facts),
"prompt_tokens": total_prompt_tokens,
"completion_tokens": total_completion_tokens}
chosen = normalize_clause_selection(clause_parsed, parents)
for parent in parents:
candidate_index = chosen.get(parent["node_id"])
if not candidate_index:
continue
quote = parent["candidates"][candidate_index - 1]
if quote not in parent["content"] or supplement_reject_reason(quote):
return {"success": False, "enabled": True,
"error": f"invalid grounded supplement clause {parent['node_id']}",
"created": [], "source_count": len(facts),
"prompt_tokens": total_prompt_tokens,
"completion_tokens": total_completion_tokens}
salvaged_by_parent[parent["node_id"]] = {
**parent,
"parent_node_id": parent["node_id"],
"content": quote,
"salvage": "deterministic-claim-candidate-v1",
}
# Preserve the specific selector's frozen order. A question-shaped parent
# is either replaced in-place by its one grounded exact clause or omitted;
# ordinary exact L2 entries remain in their original relative positions.
rebuilt_entries: List[Dict[str, Any]] = []
for entry in entries:
reason = supplement_reject_reason(entry["content"])
if reason == "generic_question":
salvaged = salvaged_by_parent.get(entry["node_id"])
if salvaged is not None:
rebuilt_entries.append(salvaged)
elif reason == "":
rebuilt_entries.append(entry)
entries = rebuilt_entries
semantic_removed: List[Dict[str, Any]] = []
targets = specific_forget_targets(forget_targets)
if entries and targets:
lines = ["Atomic forget targets:"]
for index, target in enumerate(targets, 1):
lines.append(f"F{index}. {target}")
lines.append("Specific supplement entries:")
for index, entry in enumerate(entries, 1):
lines.append(f"{index}. [{entry['domain']}] {entry['content']}")
try:
semantic_response = await provider.complete_messages(
messages=[
{"role": "system", "content": _SEMANTIC_FORGET_PROMPT},
{"role": "user", "content": "\n".join(lines)},
],
max_tokens=256,
)
total_prompt_tokens += int(semantic_response.prompt_tokens or 0)
total_completion_tokens += int(semantic_response.completion_tokens or 0)
except Exception as exc:
return {"success": False, "enabled": True, "error": str(exc),
"created": [], "source_count": len(facts),
"prompt_tokens": total_prompt_tokens,
"completion_tokens": total_completion_tokens}
semantic_parsed = _parse_json_object(semantic_response.content or "")
if semantic_parsed is None:
return {"success": False, "enabled": True,
"error": "invalid semantic forget JSON", "created": [],
"source_count": len(facts),
"prompt_tokens": total_prompt_tokens,
"completion_tokens": total_completion_tokens}
remove = set(normalize_remove_selection(semantic_parsed, len(entries)))
semantic_removed = [
{"node_id": str(entry.get("parent_node_id") or entry["node_id"]),
"domain": entry["domain"]}
for index, entry in enumerate(entries, 1) if index in remove
]
entries = [entry for index, entry in enumerate(entries, 1) if index not in remove]
# One visible entry per source parent, preserving frozen order.
deduped: List[Dict[str, Any]] = []
seen_parents = set()
for entry in entries:
parent_id = str(entry.get("parent_node_id") or entry["node_id"]).split("#", 1)[0]
if parent_id in seen_parents:
continue
seen_parents.add(parent_id)
deduped.append(entry)
entries = deduped
if not entries:
ok, retired = await _retire_nodes(graph_store, current)
return {
"success": ok, "enabled": True, "skipped": True,
"reason": "no_selected_specific_supplement", "created": [],
"retired_supplement_ids": retired,
"source_count": len(facts), "selected_count": 0,
"rejected": rejected,
"suppressed_forget_target_entries": direct_suppressed,
"semantic_removed_entries": semantic_removed,
"prompt_tokens": total_prompt_tokens,
"completion_tokens": total_completion_tokens,
}
evidence_entries = [{**entry, "node_id": str(entry.get("parent_node_id") or entry["node_id"]).split("#", 1)[0]}
for entry in entries]
evidence_fingerprint = catalog_evidence_fingerprint(evidence_entries)
node_id = _supplement_node_id(
user_id, combined_source_fingerprint, evidence_fingerprint,
)
evidence_ids = [entry["node_id"] for entry in evidence_entries]
fact_by_id = {
str(getattr(fact, "node_id", "") or ""): fact for fact in facts
}
node = MemoryNode(
node_id=node_id, user_id=user_id, agent_id="",
source=merge_sources(
getattr(fact_by_id.get(evidence_id), "source", None)
for evidence_id in evidence_ids
),
layer=MemoryLayer.L6_SCHEMA,
content=render_specific_supplement(entries),
confidence=1.0, status=MemoryStatus.ACTIVE,
source_type=SourceType.COMPOSITE,
memory_at=datetime.now(), tags=["specific-persona-supplement"],
evidence_chain=evidence_ids,
custom={
"schema_type": "internal_profile_chain",
"schema_kind": SUPPLEMENT_SCHEMA_KIND,
"specific_supplement_version": SUPPLEMENT_VERSION,
"source_bank_fingerprint": combined_source_fingerprint,
"base_catalog_node_id": catalog.node_id,
"evidence_fingerprint": evidence_fingerprint,
"selected_count": len(entries),
"max_per_domain": SUPPLEMENT_MAX_PER_DOMAIN,
"max_total": SUPPLEMENT_MAX_TOTAL,
"direct_forget_suppressed_count": len(direct_suppressed),
"semantic_forget_removed_count": len(semantic_removed),
"clause_salvage_count": len(salvaged_by_parent),
"domains": {
domain: [entry["node_id"] for entry in evidence_entries
if entry["domain"] == domain]
for domain in CATALOG_DOMAINS
},
"profile_scope": "user",
"retrieval_text": "",
},
)
created_new = await graph_store.get_node(node_id) is None
if created_new:
await graph_store.upsert_memory_node(node)
try:
for evidence_id in evidence_ids:
evidence_node = await vector_store.get_by_id(evidence_id)
if evidence_node is None:
raise RuntimeError(f"specific supplement evidence missing: {evidence_id}")
if (
evidence_node.user_id != user_id
or evidence_node.layer != MemoryLayer.L2_FACT
or evidence_node.status != MemoryStatus.ACTIVE
or not bool(evidence_node.is_latest)
):
raise RuntimeError(
f"specific supplement evidence outside active user scope: {evidence_id}"
)
await graph_store.ensure_vdbref(evidence_id, evidence_node.layer.value)
if await graph_store.add_derived_from(node_id, evidence_id) is False:
raise RuntimeError(f"specific supplement evidence link failed: {evidence_id}")
except Exception as exc:
if created_new:
await graph_store.delete_node(node_id)
return {"success": False, "enabled": True, "error": str(exc),
"created": [], "source_count": len(facts),
"selected_count": len(entries),
"prompt_tokens": total_prompt_tokens,
"completion_tokens": total_completion_tokens}
ok, retired = await _retire_nodes(graph_store, current, keep_id=node_id)
if not ok:
if created_new:
await graph_store.delete_node(node_id)
return {"success": False, "enabled": True,
"error": "failed to retire old specific supplement", "created": [],
"prompt_tokens": total_prompt_tokens,
"completion_tokens": total_completion_tokens}
return {
"success": True, "enabled": True,
"created": [{"node_id": node_id, "evidence_list": evidence_ids,
"selected_count": len(entries)}] if created_new else [],
"supplement_node_id": node_id,
"source_count": len(facts), "selected_count": len(entries),
"rejected": rejected,
"suppressed_forget_target_entries": direct_suppressed,
"semantic_removed_entries": semantic_removed,
"retired_supplement_ids": retired,
"prompt_tokens": total_prompt_tokens,
"completion_tokens": total_completion_tokens,
}
async def materialize_specific_supplement(
*, user_id: str, entries: Sequence[Dict[str, Any]],
vector_store: VectorStoreBase, graph_store: Optional[GraphStoreBase],
materialization_source: str = "",
) -> Dict[str, Any]:
"""Materialize the one user-level supplement, serialized by user_id."""
from .user_profile_scope import user_profile_lock
async with user_profile_lock(user_id):
return await _materialize_specific_supplement_unlocked(
user_id=user_id,
entries=entries,
vector_store=vector_store,
graph_store=graph_store,
materialization_source=materialization_source,
)
async def run_specific_supplement_generation(
*, user_id: str, vector_store: VectorStoreBase,
graph_store: Optional[GraphStoreBase], config: MemoryConfig,
) -> Dict[str, Any]:
"""Generate the one user-level supplement, serialized by user_id."""
from .user_profile_scope import user_profile_lock
async with user_profile_lock(user_id):
return await _run_specific_supplement_generation_unlocked(
user_id=user_id,
vector_store=vector_store,
graph_store=graph_store,
config=config,
)
async def get_active_specific_supplement(
graph_store: Optional[GraphStoreBase], *, user_id: str,
) -> Optional[MemoryNode]:
if not specific_supplement_l0_enabled() or graph_store is None or not user_id:
return None
nodes = await _active_supplements(
graph_store, user_id=user_id,
)
nodes = [
node for node in nodes
if int((node.custom or {}).get("selected_count") or 0) > 0
and bool(parse_rendered_specific_entries(node.content))
]
if not nodes:
return None
nodes.sort(
key=lambda node: (
not bool(str(getattr(node, "agent_id", "") or "").strip()),
getattr(node, "gmt_created", None) or datetime.min, node.node_id,
),
reverse=True,
)
return nodes[0]
def attach_specific_supplement_to_l0(
l0_results: List[Dict[str, Any]], supplement: Optional[MemoryNode],
) -> tuple[List[Dict[str, Any]], Dict[str, Any]]:
results = [dict(item) for item in l0_results]
if supplement is None:
return results, {
"enabled": specific_supplement_l0_enabled(), "attached": False,
}
if len(results) != 1:
return results, {
"enabled": True, "attached": False,
"reason": f"l0_count={len(results)}",
"supplement_node_id": supplement.node_id,
}
item = dict(results[0])
item["_specific_supplement_suffix"] = str(supplement.content or "").strip()
item["_specific_supplement_node_id"] = supplement.node_id
item["_specific_supplement_count"] = int(
(supplement.custom or {}).get("selected_count") or 0
)
results[0] = item
return results, {
"enabled": True, "attached": True,
"supplement_node_id": supplement.node_id,
"selected_count": item["_specific_supplement_count"],
"supplement_chars": len(str(supplement.content or "")),
}