Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

19.9 KB
Raw Blame
# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
# SPDX-License-Identifier: AGPL-3.0
"""
Core domain data classes for memory system.
"""

import json
import re
from datetime import datetime
from enum import Enum
from typing import (
    Annotated,
    Any,
    Dict,
    List,
    Optional,
    Protocol,
    TypeVar,
    Union,
    get_args,
    get_origin,
    get_type_hints,
)

from pydantic import BaseModel, Field, PrivateAttr, WithJsonSchema, model_validator

from openviking.session.memory.merge_op.base import (
    FieldType,
    MergeOp,
)
from openviking.session.memory.utils.template_utils import TemplateUtils

T = TypeVar("T")


# ============================================================================
# Link Type and Link Models
# ============================================================================


LINK_TYPE_DEFAULT = "related_to"
_LINK_TYPE_RE = re.compile(r"^[a-z]+(?:_[a-z]+){0,2}$")


class LinkType(str, Enum):
    """Legacy predefined link labels kept for compatibility in tests/call sites."""

    RELATED_TO = LINK_TYPE_DEFAULT
    BELONGS_TO = "belongs_to"
    CAUSED_BY = "caused_by"
    DERIVED_FROM = "derived_from"
    CONTRADICTS = "contradicts"
    EVOLVED_FROM = "evolved_from"


class WikiLink(BaseModel):
    """Link output by LLM during extraction, using temporary page_ids.

    f and t use WithJsonSchema to appear as required int in the JSON schema
    sent to the LLM, but parse as Optional[int] to tolerate null values.
    Invalid links (null f/t) are filtered in _resolve_links.
    """

    @model_validator(mode="before")
    @classmethod
    def normalize_link_fields(cls, data: Any) -> Any:
        if isinstance(data, dict):
            raw_link_type = data.get("link_type")
            if raw_link_type is not None:
                normalized = str(raw_link_type).strip().lower().replace("-", "_").replace(" ", "_")
                if _LINK_TYPE_RE.fullmatch(normalized):
                    data["link_type"] = normalized
                else:
                    data["link_type"] = LINK_TYPE_DEFAULT

            raw_weight = data.get("weight")
            if raw_weight is not None:
                try:
                    data["weight"] = min(1.0, max(0.0, float(raw_weight)))
                except (TypeError, ValueError):
                    data["weight"] = 0.5
        return data

    f: Annotated[Optional[int], WithJsonSchema({"type": "integer"})] = Field(
        ..., description="From page_id. Use the page_id from the item's 'page_id' field."
    )
    t: Annotated[Optional[int], WithJsonSchema({"type": "integer"})] = Field(
        ..., description="To page_id. Use the page_id from the item's 'page_id' field."
    )
    link_type: str = Field(
        LINK_TYPE_DEFAULT,
        description=(
            "A short relation label describing how the source relates to the target. "
            "Prefer one of these lowercase snake_case values: belongs_to, related_to, "
            "derived_from, caused_by, contradicts, evolved_from. "
            "Use belongs_to for part-of/profile membership, related_to for general association, "
            "derived_from for extracted/summary facts, caused_by for direct causation, "
            "contradicts for mutually inconsistent facts, and evolved_from for time-based changes. "
            "Do not invent new link_type values unless absolutely necessary."
        ),
    )
    weight: float = Field(
        0.5,
        description=(
            "Relative ranking score from 0 to 1; use higher values for the best link "
            "when multiple links compete for the same anchor or attention."
        ),
    )
    match_text: Annotated[
        Optional[str],
        WithJsonSchema({"anyOf": [{"type": "string"}, {"type": "null"}]}),
    ] = Field(
        ...,
        description=(
            "A single WORD from the original conversation to be linkified. "
            "This field must always be included in link output. "
            "Use a single exact word from the original conversation whenever possible; "
            "only use null when no valid single-word anchor exists. "
            "Rules: (1) must be a single word only (NOT a phrase or multi-word text); "
            "(2) must exist verbatim in the original conversation messages; "
            "(3) pick the most specific/identifying word"
        ),
    )
    description: str = Field("", description="Brief explanation of the relationship")


class StoredLink(BaseModel):
    """Persisted link in MEMORY_FIELDS, with URIs instead of page_ids."""

    from_uri: str
    to_uri: str
    link_type: str = LINK_TYPE_DEFAULT
    weight: float = 0.5
    match_text: Optional[str] = None  # single word, must exist verbatim in conversation
    description: str = ""
    created_at: str = ""


class DeleteId(BaseModel):
    """Delete request by page_id, optionally remapped to a replacement page_id."""

    delete_page_id: Annotated[Optional[int], WithJsonSchema({"type": "integer"})] = Field(
        ..., description="Page_id of the memory item to delete."
    )
    replacement_page_id: Annotated[
        Optional[int],
        WithJsonSchema({"anyOf": [{"type": "integer"}, {"type": "null"}]}),
    ] = Field(
        ...,
        description=(
            "Replacement page_id that should inherit this deleted page's existing links/backlinks; "
            "use null for a pure delete."
        ),
    )


class MemoryOperationSource(BaseModel):
    """Runtime and persisted provenance for one extracted memory operation.

    ``extraction_id`` identifies one archive/commit extraction run, not an
    entire chat session.  Streaming memory merge uses it to detect batches that
    combine patches produced from different extraction snapshots.
    """

    extraction_id: Optional[str] = None
    session_id: Optional[str] = None
    archive_uri: Optional[str] = None
    task_id: Optional[str] = None
    trace_id: Optional[str] = None
    extracted_at: Optional[str] = None


class MemoryOperationSkipCode(str, Enum):
    """Stable reason codes for intentionally skipped memory operations."""

    MEMORY_TYPE_FILTERED = "memory_type_filtered"
    SELF_MEMORY_DISABLED = "self_memory_disabled"
    PEER_MEMORY_DISABLED = "peer_memory_disabled"
    INVALID_PEER_ID = "invalid_peer_id"
    PEER_NOT_ALLOWED = "peer_not_allowed"
    INVALID_RANGES = "invalid_ranges"
    AMBIGUOUS_TARGET = "ambiguous_target"
    NO_WRITABLE_TARGET = "no_writable_target"
    PAGE_ID_TYPE_MISMATCH = "page_id_type_mismatch"
    INVALID_URI_TEMPLATE_VARS = "invalid_uri_template_vars"


class MemoryOperationSkip(BaseModel):
    """Internal policy/validation decision explaining why no URI was produced."""

    reason_code: MemoryOperationSkipCode
    reason: str


class SkippedMemoryOperation(BaseModel):
    """Structured, task-visible record for one intentionally skipped operation."""

    memory_type: str
    page_id: Optional[int] = None
    uri: Optional[str] = None
    reason_code: MemoryOperationSkipCode
    reason: str
    # Source is used only to scope shared streaming-batch results back to the
    # submitting commit. It must never be serialized into the public task result.
    source: Optional[MemoryOperationSource] = Field(default=None, exclude=True)


# ============================================================================
# Memory Field and Schema Definitions
# ============================================================================


class MemoryField(BaseModel):
    """Memory field definition."""

    name: str = Field(..., description="Field name")
    field_type: FieldType = Field(..., description="Field type")
    description: str = Field("", description="Field description")
    merge_op: MergeOp = Field(MergeOp.PATCH, description="Merge strategy")
    init_value: Optional[str] = Field(None, description="Initial value for this field")
    required: bool = Field(True, description="Required in complete Python SDK create/set calls")


class MemoryTypeSchema(BaseModel):
    """Memory type schema definition."""

    # True only for account bodies differing from server-owned deployment defaults.
    # Recomputed by the loader, never serialized or accepted as a client trust flag.
    _account_content_template: bool = PrivateAttr(default=False)

    memory_type: str = Field(..., description="Memory type name")
    description: str = Field("", description="Type description")
    fields: List[MemoryField] = Field(default_factory=list, description="Field definitions")
    filename_template: str = Field("", description="Filename template")
    content_template: Optional[str] = Field(
        None, description="Content template (for template mode)"
    )
    embedding_template: Optional[str] = Field(None, description="Embedding text template")
    directory: str = Field("", description="Directory path")
    enabled: bool = Field(True, description="Whether this memory type is enabled")
    operation_mode: str = Field(
        "upsert", description="Operation mode: 'upsert' (default) or 'add_only'"
    )
    stage: str = Field(
        "user",
        description="Extraction stage: 'user' for long-term user memory, 'agent' for execution-derived memory.",
    )
    peer_enabled: bool = Field(
        True,
        description="Whether this memory type is stored separately under peer directories.",
    )
    overview_template: Optional[str] = Field(
        None, description="Overview template for auto-generating .overview.md files"
    )

    def filename_has_variables(self):
        return "{{" in self.filename_template and "}}" in self.filename_template

    def identity_fields(self, *, include_peer_id: bool = True) -> tuple[str, ...]:
        """Return fields whose values determine the memory object's URI identity."""
        identity = ["peer_id"] if include_peer_id and self.peer_enabled else []
        uri_template = f"{self.directory}/{self.filename_template}"
        referenced = TemplateUtils.referenced_variables(uri_template)
        identity.extend(field.name for field in self.fields if field.name in referenced)
        return tuple(dict.fromkeys(identity))


class MemoryData(BaseModel):
    """Dynamic memory data."""

    memory_type: str = Field(..., description="Memory type name")
    uri: Optional[str] = Field(None, description="Memory URI (for updates)")
    fields: dict[str, Any] = Field(default_factory=dict, description="Dynamic field data")
    abstract: Optional[str] = Field(None, description="L0 abstract")
    overview: Optional[str] = Field(None, description="L1 overview")
    content: Optional[str] = Field(None, description="L2 content")
    name: Optional[str] = Field(None, description="Memory name")
    tags: List[str] = Field(default_factory=list, description="Tags")
    created_at: Optional[datetime] = Field(None, description="Created time")
    updated_at: Optional[datetime] = Field(None, description="Updated time")

    def get_field(self, field_name: str) -> Any:
        """Get field value."""
        return self.fields.get(field_name)

    def set_field(self, field_name: str, value: Any) -> None:
        """Set field value."""
        self.fields[field_name] = value


class MemoryFile(BaseModel):
    """Typed representation of a memory file's parsed content."""

    uri: Optional[str] = None
    content: str = ""
    links: List[Dict[str, Any]] = []
    backlinks: List[Dict[str, Any]] = []
    memory_type: Optional[str] = None
    extra_fields: Dict[str, Any] = {}

    def plain_content(self) -> str:
        from openviking.session.memory.utils.link_renderer import LinkRenderer

        return LinkRenderer.strip_links(self.content)

    @classmethod
    def from_parsed(cls, uri: Optional[str] = None, parsed: Dict[str, Any] = None) -> "MemoryFile":
        """Build from parse_memory_file_with_fields result."""
        if parsed is None:
            parsed = {}
        content = parsed.pop("content", "")
        links = parsed.pop("links", []) or []
        backlinks = parsed.pop("backlinks", []) or []
        memory_type = parsed.pop("memory_type", None)
        # Remaining keys are dynamic schema fields + system fields
        return cls(
            uri=uri,
            content=content,
            links=links,
            backlinks=backlinks,
            memory_type=memory_type,
            extra_fields=parsed,
        )

    def to_metadata(self) -> Dict[str, Any]:
        """Flatten to a dict suitable for serialize_with_metadata."""
        metadata = {
            key: value
            for key, value in dict(self.extra_fields).items()
            if key not in {"user_id", "user_ids", "_uri"}
        }
        metadata.setdefault("version", 1)
        metadata["content"] = self.content
        if self.links:
            metadata["links"] = self.links
        if self.backlinks:
            metadata["backlinks"] = self.backlinks
        if self.memory_type:
            metadata["memory_type"] = self.memory_type
        return metadata


class ResolvedOperation(BaseModel):
    old_memory_file_content: Optional[MemoryFile] = None
    memory_fields: Dict
    memory_type: str  # The memory type (e.g., 'tools', 'skills', 'events')
    uris: List[str]
    page_id: Optional[int] = None  # Temporary page_id for link resolution (not persisted)
    source: Optional[MemoryOperationSource] = None
    # Runtime-only resolution decision. It is deliberately excluded from model
    # serialization so it cannot enter later LLM merge prompts or memory files.
    resolution_skip: Optional[MemoryOperationSkip] = Field(default=None, exclude=True)
    # Custom scalar tags (already normalized as "key=value") to attach to this
    # operation's memories in the vector index. None means "no tags"; used by
    # event-memory auto-tagging. Not persisted in the memory file content.
    search_tags: Optional[List[str]] = None

    def is_edit(self):
        return self.old_memory_file_content is not None


class ResolvedOperations(BaseModel):
    upsert_operations: List[ResolvedOperation]
    delete_file_contents: List[MemoryFile]
    errors: List[str]
    resolved_links: List[StoredLink] = Field(default_factory=list)
    delete_replacements: Dict[str, str] = Field(default_factory=dict)

    def has_errors(self) -> bool:
        return len(self.errors) > 0


# ============================================================================
# Fault Tolerant Base Model (参考 vikingdb BaseModelCompat)
# ============================================================================


class FaultTolerantBaseModel(BaseModel):
    """
    支持验证前自动容错的 BaseModel,类似 vikingdb 的 BaseModelCompat。

    在 model_validator(mode='before') 中对所有字段做类型容错处理,
    使得模型可以接受 LLM 输出的不标准格式数据。
    """

    @model_validator(mode="before")
    @classmethod
    def values_fault_tolerance(cls, data: Dict[str, Any]) -> Dict[str, Any]:
        """在验证前对所有字段做容错处理"""
        if isinstance(data, dict):
            field_types = get_type_hints(cls)
            for field_name, value in data.items():
                if field_name in field_types:
                    data[field_name] = cls.value_fault_tolerance(field_types[field_name], value)
            return data
        return {}

    @classmethod
    def get_origin_type(cls, annotation) -> type:
        """从 Optional 或 Union 类型中提取基础类型"""
        origin = get_origin(annotation)
        if origin is Union:
            args = get_args(annotation)
            if len(args) == 2 and args[1] is type(None):
                return cls.get_origin_type(args[0])
        elif origin is list:
            return list
        return annotation

    @classmethod
    def get_arg_type(cls, annotation) -> type:
        """从 List annotation 中提取元素类型"""
        origin = get_origin(annotation)
        if origin is Union:
            args = get_args(annotation)
            if len(args) == 2 and args[1] is type(None):
                return cls.get_arg_type(args[0])
        elif origin is list:
            args = get_args(annotation)
            if args:
                return args[0]
        return None

    @classmethod
    def any_to_str(cls, value) -> str:
        """将任意值转换为字符串"""
        if value is None:
            return ""
        if isinstance(value, list):
            return ",".join(map(str, value))
        elif isinstance(value, dict):
            return json.dumps(value, ensure_ascii=False)
        elif isinstance(value, (int, bool, float)):
            return f"{value}"
        return str(value)

    @classmethod
    def value_fault_tolerance(cls, field_type, value):
        """
        字段级别的容错处理:
        - 'None' -> None (非 str 类型)
        - list/dict/number -> str (目标是 str)
        - str -> int/float (目标是数字)
        - str/dict -> list (目标是 list)
        - list 元素类型容错
        - 非法 LinkType -> related_to
        """
        origin_type = cls.get_origin_type(field_type)

        # json_repair 会把 None 转换成 'None'
        if value == "None" and origin_type is not str:
            return None

        if isinstance(origin_type, type) and issubclass(origin_type, Enum):
            if origin_type is LinkType:
                try:
                    return origin_type(value)
                except (ValueError, TypeError):
                    return LinkType.RELATED_TO

        if origin_type is str:
            return cls.any_to_str(value)
        elif origin_type is int:
            if isinstance(value, str):
                if value is None or value == "None":
                    return 0
                try:
                    return int(value)
                except (ValueError, TypeError):
                    pass
        elif origin_type is float:
            if isinstance(value, str):
                if value is None or value == "None":
                    return 0.0
                try:
                    return float(value)
                except (ValueError, TypeError):
                    pass
        elif origin_type is list:
            if isinstance(value, str):
                return [value]
            elif isinstance(value, dict):
                return [value]
            elif isinstance(value, list):
                arg_type = cls.get_arg_type(field_type)
                if arg_type is str:
                    return [cls.any_to_str(v) for v in value]
        return value


# ============================================================================
# Memory Operations
# ============================================================================


class MemoryOperationsProtocol(Protocol):
    """Protocol for memory operations (for type checking)."""

    reasoning: str
    write_uris: List[Any]
    edit_uris: List[Any]
    delete_ids: List[DeleteId]

    def is_empty(self) -> bool: ...


class StructuredMemoryOperations(FaultTolerantBaseModel):
    """
    Fallback memory operations model with fault tolerance.

    Use SchemaModelGenerator.create_structured_operations_model() to get
    the actual type-safe implementation with per-memory_type fields.
    """

    reasoning: str = Field(
        "",
        description="reasoning",
    )
    write_uris: List[Any] = Field(
        default_factory=list,
        description="Write operations with flat data format",
    )
    edit_uris: List[Any] = Field(
        default_factory=list,
        description="Edit operations with flat data format",
    )
    delete_ids: List[DeleteId] = Field(
        default_factory=list,
        description="Delete operations by page_id, with optional replacement_page_id",
    )

    def is_empty(self) -> bool:
        """Check if there are any operations."""
        return len(self.write_uris) == 0 and len(self.edit_uris) == 0 and len(self.delete_ids) == 0

    def to_legacy_operations(self) -> Dict[str, Any]:
        """Convert to legacy format (identity for fallback)."""
        return {
            "write_uris": self.write_uris,
            "edit_uris": self.edit_uris,
            "delete_ids": self.delete_ids,
        }

    model_config = {"extra": "ignore"}


# Backward compatibility alias
MemoryOperations = StructuredMemoryOperations