Skip to content
Python
38.7 KB
Raw Blame
"""Balanced, exact-L2 persona catalog for Ultra/System2.

The catalog is generated question-blind during ``evolve``.  It stores exact
active/latest L2 text grouped into a fixed domain taxonomy and keeps evidence
links to every selected L2.  It is a special L6 node, but it never participates
in ordinary L6 retrieval or consumes a schema slot.  Ultra deterministically
appends its content to the one visible L0 chain; Pro never enables this path.

Raw and S1 records are never modified by this module.
"""

from __future__ import annotations

import hashlib
import json
import os
import re
import uuid
from datetime import datetime
from typing import Any, Dict, Iterable, List, Optional, Sequence

from ..agent.llm_provider import LLMProvider
from ..config import MemoryConfig
from ..data.graph_store_base import GraphStoreBase
from ..data.vector_store_base import VectorStoreBase
from ..models.memory import (
    MemoryLayer,
    MemoryNode,
    MemoryStatus,
    SourceType,
)
from ..utils.source_filter import merge_sources


CATALOG_SCHEMA_KIND = "balanced_persona_catalog"
CATALOG_VERSION = 4
CATALOG_MAX_PER_DOMAIN = 4
CATALOG_MAX_TOTAL = 32
CATALOG_DOMAINS = (
    "identity_relationships",
    "health_accessibility",
    "preferences_collections",
    "habits_items",
    "work_education",
    "culture_creative_experience",
    "values_coping",
    "current_constraints_goals",
)
_CATALOG_ITEM_RE = re.compile(r"^- \[([^\]]+)\] (.*)$")

_FORGET_RE = re.compile(
    r"(?i)\b(?:forget|forgotten|do not remember|asked (?:the )?assistant to forget|"
    r"requested (?:that )?.* be forgotten)\b"
)
_ASSISTANT_ADVICE_RE = re.compile(
    r"(?i)^\s*(?:the )?assistant (?:advised|recommended|suggested|explained|outlined)\b"
)
_GENERIC_QUESTION_RE = re.compile(
    r"(?i)^\s*the user (?:asked|wants to know|is curious|wondered)\b"
)
_EXPLICIT_FORGET_EN_RE = re.compile(
    r"(?:"
    r"\b(?:asked|told|requested)\b.{0,100}\bforget\s+"
    r"(?:that\s+(?:they|the user)\b|(?:their|the user's)\b|"
    r"(?:the\s+)?(?:detail|information|preference)\s+about\b)"
    r"|"
    r"\brequested\b.{0,40}\b(?:their|the user's)\b.{0,120}\bbe\s+forgotten\b"
    r")",
    re.I,
)
_EXPLICIT_FORGET_ZH_RE = re.compile(
    r"用户.{0,30}(?:请|要求|让).{0,20}忘记.{0,10}(?:自己|其|他们|她|他)"
)


def is_explicit_forget_marker(text: str) -> bool:
    """Conservative detector copied from the validated direct-forget path."""
    value = str(text or "").strip()
    if not value:
        return False
    lowered = value.lower()
    if not (lowered.startswith("the user ") or value.startswith("用户")):
        return False
    return bool(
        _EXPLICIT_FORGET_EN_RE.search(value)
        or _EXPLICIT_FORGET_ZH_RE.search(value)
    )


def extract_forget_targets(marker_text: str) -> List[str]:
    """Extract atomic forget targets using the validated direct-forget grammar."""
    text = " ".join(str(marker_text or "").split()).strip()
    if not text:
        return []
    sentences = [
        part.strip()
        for part in re.split(r"(?<=[.!?。!?])\s+", text)
        if part.strip()
    ]
    targets: List[str] = []
    seen = set()
    for sentence in sentences:
        target = sentence
        embedded_passive = re.search(
            r"(?i)\b(?:and\s+)?requested\s+that\s+(.+?)\s+be\s+forgotten\.?$",
            target,
        )
        embedded_active = re.search(
            r"(?i)\b(?:and\s+)?asked(?:\s+the\s+assistant)?\s+to\s+forget\s+"
            r"(?:that\s+)?(.+?)\.?$",
            target,
        )
        if embedded_passive:
            target = embedded_passive.group(1).strip()
        elif embedded_active:
            target = embedded_active.group(1).strip()
        passive = re.sub(
            r"(?i)^the user\s+requested\s+that\s+", "", target, count=1,
        )
        passive = re.sub(r"(?i)\s+be\s+forgotten\.?$", "", passive).strip()
        if passive != target:
            target = passive
        for pattern in (
            r"(?i)^the user\s+(?:asked|told|requested)(?:\s+the assistant)?"
            r"(?:\s+that)?\s+to\s+forget\s+(?:that\s+)?",
            r"(?i)^the user\s+requested\s+that\s+the assistant\s+forget\s+"
            r"(?:that\s+)?",
            r"(?i)^(?:please\s+)?forget\s+(?:that\s+)?",
        ):
            updated = re.sub(pattern, "", target, count=1)
            if updated != target:
                target = updated
                break
        target = re.sub(
            r"(?i)^the assistant\s+forget\s+(?:that\s+)?", "", target, count=1,
        )
        target = re.sub(
            r"(?i),?\s+and\s+the\s+assistant\s+"
            r"(?:confirmed|acknowledged|agreed|said|stated)\b.*$",
            "", target, count=1,
        ).strip()
        for atomic_target in re.split(
            r"(?i),?\s+(?:and\s+)?that\s+"
            r"(?=(?:they|their|the user|i|my|me)\b)",
            target,
        ):
            atomic_target = re.sub(
                r"^(?:请)?忘记(?:我|我的|本人)?", "", atomic_target,
            ).strip()
            atomic_target = re.sub(
                r"(?i)\s+from your memory\.?$", "", atomic_target,
            ).strip(" .")
            atomic_target = re.sub(
                r"(?i)^(?:their|the user's)\s+preference\s+for\s+",
                "", atomic_target,
            )
            if not atomic_target:
                continue
            key = atomic_target.casefold()
            if key not in seen:
                seen.add(key)
                targets.append(atomic_target)
    return targets


def _forget_terms(text: str) -> set[str]:
    stop = {
        "the", "user", "asked", "assistant", "please", "forget", "that",
        "they", "their", "this", "from", "memory", "about", "with",
        "and", "for", "into", "instead", "daily", "occasionally", "regularly",
        "prefer", "prefers", "preference", "enjoy", "enjoys", "like", "likes",
        "love", "loves", "use", "uses", "using", "have", "has", "had",
        "feel", "feels", "felt", "managed", "manage", "seeking", "valuing",
        "take", "takes", "keep", "keeps", "get", "gets", "visit", "visits",
    }
    output = set()
    for token in re.findall(r"[a-z0-9][a-z0-9_-]+", str(text or "").lower()):
        if len(token) < 3 or token in stop:
            continue
        if token == "apps":
            token = "app"
        elif token.endswith("ies") and len(token) > 5:
            token = token[:-3] + "y"
        elif token.endswith("s") and len(token) > 4 and not token.endswith("ss"):
            token = token[:-1]
        output.add(token)
    return output


def text_matches_forget_targets(text: str, targets: Iterable[str]) -> bool:
    """Full-anchor matcher shared with direct-forget; weak overlap is rejected."""
    text_terms = _forget_terms(text)
    for target in targets:
        anchors = _forget_terms(target)
        if not anchors:
            continue
        needed = len(anchors) if len(anchors) <= 3 else max(
            3, (len(anchors) * 3 + 4) // 5,
        )
        if len(anchors & text_terms) >= needed:
            return True
    return False


def collect_forget_targets(facts: Iterable[Any]) -> List[str]:
    targets: List[str] = []
    seen = set()
    for node in facts:
        content = str(getattr(node, "content", "") or "")
        if not is_explicit_forget_marker(content):
            continue
        for target in extract_forget_targets(content):
            key = target.casefold()
            if key not in seen:
                seen.add(key)
                targets.append(target)
    return targets


def suppress_forget_targets(
    entries: Iterable[Dict[str, Any]],
    targets: Iterable[str],
) -> tuple[List[Dict[str, Any]], List[Dict[str, Any]]]:
    target_list = list(targets)
    kept: List[Dict[str, Any]] = []
    suppressed: List[Dict[str, Any]] = []
    for entry in entries:
        if text_matches_forget_targets(str(entry.get("content") or ""), target_list):
            # Never persist target plaintext.  Node/domain IDs are sufficient
            # for internal audit and rebuilding.
            suppressed.append({
                "node_id": str(entry.get("node_id") or ""),
                "domain": str(entry.get("domain") or ""),
            })
        else:
            kept.append(entry)
    return kept, suppressed


def _env_bool(name: str, default: str = "false") -> bool:
    return os.getenv(name, default).strip().lower() in {"1", "true", "yes", "on"}


def balanced_catalog_generation_enabled() -> bool:
    return _env_bool("MEMORY_EVOLING_BALANCED_CATALOG_ENABLED")


def balanced_catalog_user_profile_enabled() -> bool:
    value = os.getenv("MEMORY_USER_PROFILE_BALANCED_CATALOG_ENABLED")
    if value is not None:
        return value.strip().lower() in {"1", "true", "yes", "on"}
    return _env_bool("MEMORY_HYBRID_V2_BALANCED_CATALOG_L0_ENABLED")


def balanced_catalog_l0_enabled() -> bool:
    """Backward-compatible alias for the pre-profile-split helper name."""
    return balanced_catalog_user_profile_enabled()


def is_balanced_catalog_custom(custom: Optional[Dict[str, Any]]) -> bool:
    return str((custom or {}).get("schema_kind") or "") == CATALOG_SCHEMA_KIND


def parse_custom(value: Any) -> Dict[str, Any]:
    if isinstance(value, dict):
        return dict(value)
    if isinstance(value, str):
        try:
            parsed = json.loads(value or "{}")
            return parsed if isinstance(parsed, dict) else {}
        except (TypeError, json.JSONDecodeError):
            return {}
    return {}


def is_balanced_catalog_hit(hit: Dict[str, Any]) -> bool:
    return is_balanced_catalog_custom(parse_custom(
        hit.get("custom_json") or hit.get("custom") or {}
    ))


def catalog_reject_reason(content: str) -> str:
    value = " ".join(str(content or "").split()).strip()
    if not value:
        return "empty"
    if _FORGET_RE.search(value):
        return "forget"
    if _ASSISTANT_ADVICE_RE.search(value):
        return "assistant_advice"
    if _GENERIC_QUESTION_RE.search(value):
        return "generic_question"
    return ""


def _is_user_subject_fact(node: Any) -> bool:
    owner = str(getattr(node, "owner", "") or "").strip().lower()
    if owner in {"agent", "assistant"}:
        return False
    content = str(getattr(node, "content", "") or "").strip().lower()
    return bool(re.match(r"^(?:the\s+user|user)(?:\b|\s|[,:])|^用户", content))


def eligible_catalog_facts(nodes: Iterable[Any]) -> List[Any]:
    output = []
    for node in nodes:
        if getattr(node, "layer", None) != MemoryLayer.L2_FACT:
            continue
        if getattr(node, "status", None) != MemoryStatus.ACTIVE:
            continue
        if not bool(getattr(node, "is_latest", True)):
            continue
        if not _is_user_subject_fact(node):
            continue
        if (getattr(node, "custom", None) or {}).get("memory_kind") == "canonical_state":
            continue
        output.append(node)
    return sorted(output, key=lambda node: str(getattr(node, "node_id", "") or ""))


def source_bank_fingerprint(facts: Iterable[Any]) -> str:
    rows = [
        {
            "node_id": str(getattr(node, "node_id", "") or ""),
            "content": " ".join(str(getattr(node, "content", "") or "").split()),
            "tags": [str(value) for value in getattr(node, "tags", None) or []],
        }
        for node in facts
    ]
    encoded = json.dumps(rows, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
    return hashlib.sha256(encoded.encode("utf-8")).hexdigest()


def catalog_prompt(max_per_domain: int = 4, max_total: int = 32) -> str:
    return f"""Build a balanced long-term persona catalog from numbered recorded facts.
Return strict JSON only:
{{"domains":{{"identity_relationships":[1],"health_accessibility":[2]}}}}

Use exactly these domain keys:
- identity_relationships: identity and the user's own family/relationship structure
- health_accessibility: explicit health, sensory, mobility, ergonomic constraints
- preferences_collections: explicit durable likes, dislikes, tastes, collections
- habits_items: recurring habits and personally owned/carried/used items
- work_education: durable role, expertise, education, recurring responsibilities
- culture_creative_experience: concrete cultural/creative interests and strong experiences
- values_coping: durable values and personally practiced coping resources
- current_constraints_goals: ongoing burdens, constraints, or durable active goals

Select at most {max_per_domain} entry numbers per domain and at most {max_total}
unique entries total. Prefer specific, discriminative, user-owned facts that
could materially personalize future answers. Preserve diversity within and
across domains. A family member or partner may be selected only when it
establishes the user's relationship/family structure or a constraint on the user.

Reject forget requests, assistant advice, generic questions or curiosity without
a personal fact, temporary logistics, one-off writing/translation requests, and
facts mainly about another person. Do not infer, summarize, rewrite, or create
facts. An entry may appear in at most one domain. Output entry numbers only."""


def _parse_json_object(text: str) -> Optional[Dict[str, Any]]:
    value = str(text or "").strip()
    match = re.search(r"```(?:json)?\s*(\{.*?\})\s*```", value, re.S | re.I)
    payload = match.group(1) if match else value
    try:
        parsed = json.loads(payload)
        return parsed if isinstance(parsed, dict) else None
    except json.JSONDecodeError:
        try:
            from json_repair import repair_json
            parsed = repair_json(payload, return_objects=True)
            return parsed if isinstance(parsed, dict) else None
        except Exception:
            return None


def normalize_catalog_selection(
    raw: Optional[Dict[str, Any]],
    *,
    candidate_count: int,
    max_per_domain: int = 4,
    max_total: int = 32,
) -> Dict[str, List[int]]:
    source = raw.get("domains") if isinstance(raw, dict) else None
    if not isinstance(source, dict) and isinstance(raw, dict):
        if any(domain in raw for domain in CATALOG_DOMAINS):
            source = raw
    if not isinstance(source, dict):
        source = {}
    output: Dict[str, List[int]] = {domain: [] for domain in CATALOG_DOMAINS}
    used = set()
    for domain in CATALOG_DOMAINS:
        for value in source.get(domain) or []:
            try:
                index = int(value)
            except (TypeError, ValueError):
                continue
            if not 1 <= index <= candidate_count or index in used:
                continue
            used.add(index)
            output[domain].append(index)
            if len(output[domain]) >= max_per_domain or len(used) >= max_total:
                break
    return output


def prepare_catalog_entries(
    domains: Dict[str, List[int]],
    facts: List[Any],
) -> tuple[List[Dict[str, Any]], List[Dict[str, Any]]]:
    selected: List[Dict[str, Any]] = []
    rejected: List[Dict[str, Any]] = []
    for domain in CATALOG_DOMAINS:
        for index in domains.get(domain) or []:
            node = facts[index - 1]
            content = " ".join(str(getattr(node, "content", "") or "").split()).strip()
            reason = catalog_reject_reason(content)
            record = {
                "domain": domain,
                "source_index": index,
                "node_id": str(getattr(node, "node_id", "") or ""),
                "content": content,
                "tags": [str(value) for value in getattr(node, "tags", None) or []],
            }
            if reason:
                rejected.append({**record, "reason": reason})
            else:
                selected.append(record)
    return selected, rejected


def compact_verified_catalog_entries(
    entries: Iterable[Dict[str, Any]],
    *,
    max_per_domain: int = CATALOG_MAX_PER_DOMAIN,
    max_total: int = CATALOG_MAX_TOTAL,
) -> List[Dict[str, Any]]:
    """Keep the verified model order, capped uniformly across fixed domains.

    Compaction intentionally happens after deterministic reject/forget checks.
    This lets a later valid model nominee backfill an invalid earlier nominee,
    matching the validated dev322 offline mechanism.
    """
    grouped = {domain: [] for domain in CATALOG_DOMAINS}
    for entry in entries:
        domain = str(entry.get("domain") or "")
        if domain in grouped:
            grouped[domain].append(dict(entry))
    output: List[Dict[str, Any]] = []
    for domain in CATALOG_DOMAINS:
        for entry in grouped[domain][:max(0, int(max_per_domain))]:
            if len(output) >= max(0, int(max_total)):
                return output
            output.append(entry)
    return output


def render_catalog_content(entries: Iterable[Dict[str, Any]]) -> str:
    return "System2 verified persona catalog:\n" + "\n".join(
        f"- [{entry['domain']}] {entry['content']}" for entry in entries
    )


def parse_rendered_catalog_entries(content: str) -> List[Dict[str, str]]:
    """Parse this module's exact line-oriented visible catalog format."""
    marker = "System2 verified persona catalog:"
    value = str(content or "")
    if marker not in value:
        return []
    output: List[Dict[str, str]] = []
    for line in value.split(marker, 1)[1].strip().splitlines():
        match = _CATALOG_ITEM_RE.match(line.strip())
        if not match:
            continue
        output.append({"domain": match.group(1), "content": match.group(2)})
    return output


def catalog_evidence_fingerprint(entries: Iterable[Dict[str, Any]]) -> str:
    values = [
        (str(entry.get("domain") or ""), str(entry.get("node_id") or ""))
        for entry in entries
    ]
    encoded = json.dumps(values, ensure_ascii=False, separators=(",", ":"))
    return hashlib.sha256(encoded.encode("utf-8")).hexdigest()


def _catalog_node_id(
    user_id: str,
    source_fingerprint: str,
    evidence_fingerprint: str,
) -> str:
    value = (
        f"balanced-catalog:v{CATALOG_VERSION}:user-profile:{user_id}:"
        f"{source_fingerprint}:{evidence_fingerprint}"
    )
    return f"catalog_{uuid.uuid5(uuid.NAMESPACE_URL, value).hex}"


async def _active_catalogs(
    graph_store: GraphStoreBase,
    *,
    user_id: str,
) -> List[MemoryNode]:
    canonical = await graph_store.get_all_nodes(
        isolation_key=MemoryNode.build_isolation_key(user_id, ""),
        layer=MemoryLayer.L6_SCHEMA,
        status=MemoryStatus.ACTIVE,
        limit=200,
    )
    nodes_by_id = {node.node_id: node for node in canonical}
    get_by_user = getattr(graph_store, "get_nodes_by_user", None)
    if callable(get_by_user):
        for node in await get_by_user(
            user_id=user_id,
            layer=MemoryLayer.L6_SCHEMA,
            status=MemoryStatus.ACTIVE,
            limit=1000,
        ):
            nodes_by_id[node.node_id] = node
    return [
        node for node in nodes_by_id.values()
        if is_balanced_catalog_custom(node.custom)
    ]


async def _adopt_cloned_catalog(
    *,
    current: Sequence[MemoryNode],
    facts: List[Any],
    source_fingerprint: str,
    user_id: str,
    vector_store: VectorStoreBase,
    graph_store: GraphStoreBase,
) -> Optional[Dict[str, Any]]:
    """Adopt a correctly remapped clone without re-running catalog selection.

    Clone remaps VDB/Graph node ids and DERIVED_FROM edges, but historical
    ``custom.domains`` values and the source fingerprint still reference the
    source namespace.  A normal same-namespace bank change keeps those custom
    evidence ids valid, so adoption is allowed only when *all* recorded custom
    evidence ids are stale while every visible exact catalog line maps uniquely
    to a current active/latest L2 fact.  This preserves the frozen catalog
    selection across clone namespaces without masking real bank changes.
    """
    fact_by_content: Dict[str, List[Any]] = {}
    current_ids = {str(getattr(node, "node_id", "") or "") for node in facts}
    for fact in facts:
        content = " ".join(str(getattr(fact, "content", "") or "").split()).strip()
        fact_by_content.setdefault(content, []).append(fact)

    for old in current:
        custom = dict(old.custom or {})
        if int(custom.get("balanced_catalog_version") or 0) != CATALOG_VERSION:
            continue
        recorded_ids = {
            str(node_id)
            for values in (custom.get("domains") or {}).values()
            for node_id in values or []
            if node_id
        }
        legacy_agent_scope = bool(str(getattr(old, "agent_id", "") or "").strip())
        if not recorded_ids:
            continue
        if not legacy_agent_scope and any(
            node_id in current_ids for node_id in recorded_ids
        ):
            continue
        parsed = parse_rendered_catalog_entries(old.content)
        expected_count = int(custom.get("selected_count") or 0)
        if not parsed or len(parsed) != expected_count:
            continue
        mapped: List[Dict[str, Any]] = []
        valid = True
        for entry in parsed:
            matches = fact_by_content.get(entry["content"]) or []
            if len(matches) != 1 or entry["domain"] not in CATALOG_DOMAINS:
                valid = False
                break
            fact = matches[0]
            mapped.append({
                "domain": entry["domain"],
                "node_id": str(fact.node_id),
                "content": entry["content"],
                "tags": [str(value) for value in fact.tags or []],
            })
        if not valid:
            continue
        evidence_fingerprint = catalog_evidence_fingerprint(mapped)
        node_id = _catalog_node_id(
            user_id, source_fingerprint, evidence_fingerprint,
        )
        mapped_ids = {entry["node_id"] for entry in mapped}
        node = MemoryNode(
            node_id=node_id,
            user_id=user_id,
            agent_id="",
            source=merge_sources(
                getattr(fact, "source", None)
                for fact in facts
                if str(getattr(fact, "node_id", "") or "") in mapped_ids
            ),
            layer=MemoryLayer.L6_SCHEMA,
            content=str(old.content or "").strip(),
            confidence=1.0,
            status=MemoryStatus.ACTIVE,
            source_type=SourceType.COMPOSITE,
            memory_at=datetime.now(),
            tags=["balanced-persona-catalog"],
            evidence_chain=[entry["node_id"] for entry in mapped],
            custom={
                **custom,
                "schema_type": "internal_profile_chain",
                "schema_kind": CATALOG_SCHEMA_KIND,
                "balanced_catalog_version": CATALOG_VERSION,
                "source_bank_fingerprint": source_fingerprint,
                "evidence_fingerprint": evidence_fingerprint,
                "selected_count": len(mapped),
                "domains": {
                    domain: [
                        entry["node_id"] for entry in mapped
                        if entry["domain"] == domain
                    ]
                    for domain in CATALOG_DOMAINS
                },
                "clone_adoption_version": "exact-content-remap-v1",
                "profile_scope": "user",
                "retrieval_text": "",
            },
        )
        created_new = await graph_store.get_node(node_id) is None
        if created_new:
            await graph_store.upsert_memory_node(node)
        try:
            for evidence_id in node.evidence_chain or []:
                evidence_node = await vector_store.get_by_id(evidence_id)
                if evidence_node is None:
                    raise RuntimeError(f"adopted catalog evidence missing: {evidence_id}")
                await graph_store.ensure_vdbref(evidence_id, evidence_node.layer.value)
                if await graph_store.add_derived_from(node_id, evidence_id) is False:
                    raise RuntimeError(f"adopted catalog link failed: {evidence_id}")
        except Exception:
            if created_new:
                await graph_store.delete_node(node_id)
            continue

        retired: List[str] = []
        retirement_failed = False
        for candidate in current:
            if candidate.node_id == node_id:
                continue
            ok = await graph_store.update_node(candidate.node_id, {
                "status": MemoryStatus.SUPERSEDED.value,
                "is_latest": False,
                "valid_until": datetime.now(),
            })
            if not ok:
                retirement_failed = True
                break
            retired.append(candidate.node_id)
        if retirement_failed:
            for retired_id in retired:
                await graph_store.update_node(retired_id, {
                    "status": MemoryStatus.ACTIVE.value,
                    "is_latest": True,
                    "valid_until": None,
                })
            if created_new:
                await graph_store.delete_node(node_id)
            return None
        return {
            "success": True,
            "enabled": True,
            "skipped": True,
            "reason": "cloned_catalog_adopted",
            "created": [{
                "node_id": node_id,
                "evidence_list": list(node.evidence_chain or []),
                "selected_count": len(mapped),
            }] if created_new else [],
            "catalog_node_id": node_id,
            "source_count": len(facts),
            "selected_count": len(mapped),
            "retired_catalog_ids": retired,
            "prompt_tokens": 0,
            "completion_tokens": 0,
        }
    return None


async def _run_balanced_catalog_generation_unlocked(
    *,
    user_id: str,
    vector_store: VectorStoreBase,
    graph_store: Optional[GraphStoreBase],
    config: MemoryConfig,
) -> Dict[str, Any]:
    if not balanced_catalog_generation_enabled():
        return {
            "success": True,
            "enabled": False,
            "created": [],
            "prompt_tokens": 0,
            "completion_tokens": 0,
        }
    if graph_store is None or not hasattr(vector_store, "list_by_user"):
        return {
            "success": False,
            "enabled": True,
            "error": "catalog storage unavailable",
            "created": [],
            "prompt_tokens": 0,
            "completion_tokens": 0,
        }

    all_active_l2 = list(await vector_store.list_by_user(
        user_id=user_id,
        status_filter=[MemoryStatus.ACTIVE],
        layers=[MemoryLayer.L2_FACT],
    ))
    forget_targets = collect_forget_targets(all_active_l2)
    facts = eligible_catalog_facts(all_active_l2)
    if not facts:
        return {
            "success": True,
            "enabled": True,
            "skipped": True,
            "reason": "no_eligible_facts",
            "created": [],
            "source_count": 0,
            "prompt_tokens": 0,
            "completion_tokens": 0,
        }

    source_fingerprint = source_bank_fingerprint(facts)
    current = await _active_catalogs(
        graph_store, user_id=user_id,
    )
    matching = [
        node for node in current
        if not str(getattr(node, "agent_id", "") or "").strip()
        if int((node.custom or {}).get("balanced_catalog_version") or 0) == CATALOG_VERSION
        and str((node.custom or {}).get("source_bank_fingerprint") or "") == source_fingerprint
    ]
    if matching:
        return {
            "success": True,
            "enabled": True,
            "skipped": True,
            "reason": "current_catalog_exists",
            "created": [],
            "catalog_node_id": matching[0].node_id,
            "source_count": len(facts),
            "selected_count": int((matching[0].custom or {}).get("selected_count") or 0),
            "prompt_tokens": 0,
            "completion_tokens": 0,
        }
    adopted = await _adopt_cloned_catalog(
        current=current,
        facts=facts,
        source_fingerprint=source_fingerprint,
        user_id=user_id,
        vector_store=vector_store,
        graph_store=graph_store,
    )
    if adopted is not None:
        return adopted

    max_source = max(1, int(os.getenv(
        "MEMORY_EVOLING_BALANCED_CATALOG_MAX_FACTS", "120"
    )))
    candidate_facts = facts[:max_source]
    content_chars = max(100, int(os.getenv(
        "MEMORY_EVOLING_BALANCED_CATALOG_CONTENT_CHARS", "180"
    )))
    prompt_lines = ["Recorded facts:"]
    for index, node in enumerate(candidate_facts, 1):
        tags = ", ".join(str(value) for value in list(node.tags or [])[:4])
        prefix = f"[{tags}] " if tags else ""
        content = " ".join(str(node.content or "").split())[:content_chars]
        prompt_lines.append(f"{index}. {prefix}{content}")
    provider = LLMProvider(config)
    try:
        response = await provider.complete_messages(
            messages=[
                {"role": "system", "content": catalog_prompt()},
                {"role": "user", "content": "\n".join(prompt_lines)},
            ],
            max_tokens=max(512, int(os.getenv(
                "MEMORY_EVOLING_BALANCED_CATALOG_MAX_TOKENS", "512"
            ))),
        )
    except Exception as exc:
        return {
            "success": False,
            "enabled": True,
            "error": str(exc),
            "created": [],
            "source_count": len(facts),
            "prompt_tokens": 0,
            "completion_tokens": 0,
        }
    parsed = _parse_json_object(response.content or "")
    if parsed is None:
        return {
            "success": False,
            "enabled": True,
            "error": "invalid balanced catalog JSON",
            "created": [],
            "source_count": len(facts),
            "prompt_tokens": response.prompt_tokens or 0,
            "completion_tokens": response.completion_tokens or 0,
        }
    domains = normalize_catalog_selection(parsed, candidate_count=len(candidate_facts))
    entries, rejected = prepare_catalog_entries(domains, candidate_facts)
    entries, suppressed = suppress_forget_targets(entries, forget_targets)
    entries = compact_verified_catalog_entries(entries)
    if not entries:
        return {
            "success": False,
            "enabled": True,
            "error": "balanced catalog empty after verification",
            "created": [],
            "source_count": len(facts),
            "rejected": rejected,
            "suppressed_forget_target_entries": suppressed,
            "prompt_tokens": response.prompt_tokens or 0,
            "completion_tokens": response.completion_tokens or 0,
        }

    evidence_fingerprint = catalog_evidence_fingerprint(entries)
    node_id = _catalog_node_id(
        user_id, source_fingerprint, evidence_fingerprint,
    )
    content = render_catalog_content(entries)
    evidence_ids = [entry["node_id"] for entry in entries]
    node = MemoryNode(
        node_id=node_id,
        user_id=user_id,
        agent_id="",
        source=merge_sources(
            getattr(fact, "source", None)
            for fact in facts
            if str(getattr(fact, "node_id", "") or "") in set(evidence_ids)
        ),
        layer=MemoryLayer.L6_SCHEMA,
        content=content,
        confidence=1.0,
        status=MemoryStatus.ACTIVE,
        source_type=SourceType.COMPOSITE,
        memory_at=datetime.now(),
        tags=["balanced-persona-catalog"],
        evidence_chain=evidence_ids,
        custom={
            "schema_type": "internal_profile_chain",
            "schema_kind": CATALOG_SCHEMA_KIND,
            "balanced_catalog_version": CATALOG_VERSION,
            "source_bank_fingerprint": source_fingerprint,
            "evidence_fingerprint": evidence_fingerprint,
            "selected_count": len(entries),
            "catalog_max_per_domain": CATALOG_MAX_PER_DOMAIN,
            "catalog_max_total": CATALOG_MAX_TOTAL,
            "catalog_compaction_stage": "after_verifier_and_forget_suppression",
            "forget_suppression_version": "direct-forget-full-anchor-v1",
            "suppressed_forget_target_count": len(suppressed),
            "profile_scope": "user",
            "domains": {
                domain: [
                    entry["node_id"] for entry in entries if entry["domain"] == domain
                ]
                for domain in CATALOG_DOMAINS
            },
            # Empty by design: the node must never enter ordinary semantic L6 recall.
            "retrieval_text": "",
        },
    )

    created_new = await graph_store.get_node(node_id) is None
    if created_new:
        await graph_store.upsert_memory_node(node)
    try:
        for evidence_id in evidence_ids:
            evidence_node = await vector_store.get_by_id(evidence_id)
            if evidence_node is None:
                raise RuntimeError(f"catalog evidence missing: {evidence_id}")
            if (
                evidence_node.user_id != user_id
                or evidence_node.layer != MemoryLayer.L2_FACT
                or evidence_node.status != MemoryStatus.ACTIVE
                or not bool(evidence_node.is_latest)
            ):
                raise RuntimeError(f"catalog evidence outside active user scope: {evidence_id}")
            await graph_store.ensure_vdbref(evidence_id, evidence_node.layer.value)
            linked = await graph_store.add_derived_from(node_id, evidence_id)
            if linked is False:
                raise RuntimeError(f"catalog evidence link failed: {evidence_id}")
    except Exception as exc:
        if created_new:
            await graph_store.delete_node(node_id)
        return {
            "success": False,
            "enabled": True,
            "error": str(exc),
            "created": [],
            "source_count": len(facts),
            "selected_count": len(entries),
            "prompt_tokens": response.prompt_tokens or 0,
            "completion_tokens": response.completion_tokens or 0,
        }

    retired = []
    for old in current:
        if old.node_id == node_id:
            continue
        ok = await graph_store.update_node(old.node_id, {
            "status": MemoryStatus.SUPERSEDED.value,
            "is_latest": False,
            "valid_until": datetime.now(),
        })
        if not ok:
            for retired_id in retired:
                await graph_store.update_node(retired_id, {
                    "status": MemoryStatus.ACTIVE.value,
                    "is_latest": True,
                    "valid_until": None,
                })
            if created_new:
                await graph_store.delete_node(node_id)
            return {
                "success": False,
                "enabled": True,
                "error": f"failed to retire old catalog {old.node_id}",
                "created": [],
                "prompt_tokens": response.prompt_tokens or 0,
                "completion_tokens": response.completion_tokens or 0,
            }
        retired.append(old.node_id)

    return {
        "success": True,
        "enabled": True,
        "created": [{
            "node_id": node_id,
            "evidence_list": evidence_ids,
            "selected_count": len(entries),
        }] if created_new else [],
        "catalog_node_id": node_id,
        "source_count": len(facts),
        "selected_count": len(entries),
        "rejected": rejected,
        "suppressed_forget_target_entries": suppressed,
        "retired_catalog_ids": retired,
        "prompt_tokens": response.prompt_tokens or 0,
        "completion_tokens": response.completion_tokens or 0,
    }


async def run_balanced_catalog_generation(
    *, user_id: str, vector_store: VectorStoreBase,
    graph_store: Optional[GraphStoreBase], config: MemoryConfig,
) -> Dict[str, Any]:
    """Generate the one user-level catalog, serialized by user_id."""
    from .user_profile_scope import user_profile_lock

    async with user_profile_lock(user_id):
        return await _run_balanced_catalog_generation_unlocked(
            user_id=user_id,
            vector_store=vector_store,
            graph_store=graph_store,
            config=config,
        )


async def get_active_balanced_catalog(
    graph_store: Optional[GraphStoreBase],
    *,
    user_id: str,
) -> Optional[MemoryNode]:
    if not balanced_catalog_l0_enabled() or graph_store is None or not user_id:
        return None
    nodes = await _active_catalogs(
        graph_store, user_id=user_id,
    )
    if not nodes:
        return None
    nodes.sort(
        key=lambda node: (
            not bool(str(getattr(node, "agent_id", "") or "").strip()),
            getattr(node, "gmt_created", None) or datetime.min,
            node.node_id,
        ),
        reverse=True,
    )
    return nodes[0]


def attach_balanced_catalog_to_l0(
    l0_results: List[Dict[str, Any]],
    catalog: Optional[MemoryNode],
) -> tuple[List[Dict[str, Any]], Dict[str, Any]]:
    results = [dict(item) for item in l0_results]
    if catalog is None:
        return results, {"enabled": balanced_catalog_l0_enabled(), "attached": False}
    if not results:
        # Some legitimate banks have no extracted L0 basic-profile node.  The
        # catalog is itself a verified high-level memory, so expose it as the
        # one synthetic L0 instead of dropping the entire System2 contribution.
        # This row is read-only and is never persisted back to VDB/Graph.
        synthetic = {
            "node_id": catalog.node_id,
            "node": None,
            "content": str(catalog.content or "").strip(),
            "layer": MemoryLayer.L0_BASIC_INFO.value,
            "score": float(getattr(catalog, "confidence", 1.0) or 1.0),
            "source": "balanced_catalog_synthetic_l0",
            "_user_profile_channel": True,
            "_balanced_catalog_node_id": catalog.node_id,
            "_balanced_catalog_count": int(
                (catalog.custom or {}).get("selected_count") or 0
            ),
        }
        return [synthetic], {
            "enabled": True,
            "attached": True,
            "synthetic_l0": True,
            "catalog_node_id": catalog.node_id,
            "selected_count": synthetic["_balanced_catalog_count"],
            "catalog_chars": len(str(catalog.content or "")),
        }
    if len(results) != 1:
        return results, {
            "enabled": True,
            "attached": False,
            "reason": f"l0_count={len(results)}",
            "catalog_node_id": catalog.node_id,
        }
    item = dict(results[0])
    node = item.get("node")
    original = str(item.get("content") or getattr(node, "content", "") or "").rstrip()
    # Do not mutate ``content`` here. Evolved L0 nodes are rendered later from
    # ``evolution_chain`` and that formatter would overwrite an early content
    # change. Carry an explicit suffix through Stage 12 and append it only after
    # the complete Current/History capsule has been rendered.
    item["_balanced_catalog_suffix"] = str(catalog.content or "").strip()
    item["_balanced_catalog_original_content"] = original
    item["_balanced_catalog_node_id"] = catalog.node_id
    item["_balanced_catalog_count"] = int(
        (catalog.custom or {}).get("selected_count") or 0
    )
    results[0] = item
    return results, {
        "enabled": True,
        "attached": True,
        "synthetic_l0": False,
        "catalog_node_id": catalog.node_id,
        "selected_count": item["_balanced_catalog_count"],
        "catalog_chars": len(str(catalog.content or "")),
    }