Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

12.2 KB
Raw Blame
# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
# SPDX-License-Identifier: AGPL-3.0
"""SemanticMsg: Semantic extraction queue message dataclass."""

import json
from dataclasses import asdict, dataclass, field
from datetime import datetime
from typing import Any, Dict, List, Optional, Sequence
from uuid import uuid4

from openviking.storage.context_update_plan import SemanticPlan
from openviking.utils.ingest_options import IngestOptions


def build_semantic_coalesce_key(
    *,
    context_type: str,
    uri: str,
    account_id: str = "default",
    user_id: str = "default",
    peer_id: str = "default",
) -> str:
    return "|".join([context_type, account_id, user_id, peer_id, uri.rstrip("/")])


@dataclass
class SemanticMsg:
    """Semantic extraction queue message.

    Attributes:
        id: Unique identifier (UUID)
        uri: Directory URI to process
        context_type: Type of context (resource, memory, skill, session)
        status: Processing status (pending/processing/completed)
        timestamp: Creation timestamp
        recursive: Whether to recursively process subdirectories.
                   When True, the processor will collect all subdirectory info and
                   enqueue them for processing (bottom-up order).
                   When False, only the specified directory will be processed.
        use_hierarchical_aggregation: Route memory directories through the
                   shared semantic tree instead of the specialized flat memory
                   update path.
        propagate_to_parent: Whether a completed directory refresh may enqueue
                   a freshness refresh for its parent.
    """

    id: str  # UUID
    uri: str  # Directory URI
    context_type: str  # resource, memory, skill, session
    status: str = "pending"  # pending/processing/completed
    timestamp: int = field(default_factory=lambda: int(datetime.now().timestamp()))
    queue_enqueued_at: float = 0.0
    failure_requeue_count: int = 0
    recursive: bool = True  # Whether to recursively process subdirectories
    account_id: str = "default"
    user_id: str = "default"
    group_ids: List[str] = field(default_factory=list)
    peer_id: str = "default"
    role: str = "root"
    # Additional flags
    skip_vectorization: bool = False
    telemetry_id: str = ""
    task_id: str = ""
    _task_work_id: str = ""
    target_uri: str = ""
    lock_handoff: Optional[Dict[str, Any]] = None
    lock_handoff_retry: int = 0
    is_code_repo: bool = False
    ingest_options: IngestOptions = field(default_factory=IngestOptions)
    coalesce_key: str = ""
    coalesce_version: int = 0
    changes: Optional[Dict[str, List[str]]] = (
        None  # {"added": [...], "modified": [...], "deleted": [...]}
    )
    source: Optional[Dict[str, str]] = None
    generation_trigger: str = "semantic_refresh"
    aggregate_directory: bool = True
    use_hierarchical_aggregation: bool = False
    propagate_to_parent: bool = True
    copy_source_uri: str = ""
    # Per-file md5 of final stored bytes, keyed by target URI. Supplied by the
    # local incremental apply so the tree executor's re-vectorization writes a fresh
    # fingerprint; empty for all other flows.
    file_md5s: Dict[str, str] = field(default_factory=dict)
    artifact_ref: Optional[Dict[str, Any]] = None
    artifact_files: List[str] = field(default_factory=list)
    file_abstracts: Dict[str, str] = field(default_factory=dict)
    file_vector_actions: Dict[str, str] = field(default_factory=dict)
    file_ingest_options: Dict[str, IngestOptions] = field(default_factory=dict)
    plan: Optional[SemanticPlan] = None

    def __init__(
        self,
        uri: str,
        context_type: str,
        recursive: bool = True,
        account_id: str = "default",
        user_id: str = "default",
        group_ids: Optional[Sequence[str]] = None,
        peer_id: str = "default",
        role: str = "root",
        skip_vectorization: bool = False,
        telemetry_id: str = "",
        task_id: str = "",
        _task_work_id: str = "",
        target_uri: str = "",
        lock_handoff: Optional[Dict[str, Any]] = None,
        is_code_repo: bool = False,
        ingest_options: IngestOptions | Dict[str, Any] | None = None,
        coalesce_key: str = "",
        coalesce_version: int = 0,
        changes: Optional[Dict[str, List[str]]] = None,
        source: Optional[Dict[str, str]] = None,
        generation_trigger: str = "semantic_refresh",
        aggregate_directory: bool = True,
        use_hierarchical_aggregation: bool = False,
        propagate_to_parent: bool = True,
        copy_source_uri: str = "",
        file_md5s: Optional[Dict[str, str]] = None,
        artifact_ref: Optional[Dict[str, Any]] = None,
        artifact_files: Optional[List[str]] = None,
        file_abstracts: Optional[Dict[str, str]] = None,
        file_vector_actions: Optional[Dict[str, str]] = None,
        file_ingest_options: Optional[Dict[str, IngestOptions | Dict[str, Any]]] = None,
        plan: SemanticPlan | Dict[str, Any] | None = None,
        queue_enqueued_at: float = 0.0,
        failure_requeue_count: int = 0,
        lock_handoff_retry: int = 0,
    ):
        self.id = str(uuid4())
        self.timestamp = int(datetime.now().timestamp())
        self.queue_enqueued_at = max(float(queue_enqueued_at or 0.0), 0.0)
        self.failure_requeue_count = max(int(failure_requeue_count or 0), 0)
        self.uri = uri
        self.context_type = context_type
        self.recursive = recursive
        self.account_id = account_id
        self.user_id = user_id
        self.group_ids = list(group_ids or [])
        self.peer_id = peer_id
        self.role = role
        self.skip_vectorization = skip_vectorization
        self.telemetry_id = telemetry_id
        self.task_id = task_id
        self._task_work_id = _task_work_id
        self.target_uri = target_uri
        self.lock_handoff = lock_handoff
        try:
            self.lock_handoff_retry = max(0, int(lock_handoff_retry or 0))
        except (TypeError, ValueError):
            self.lock_handoff_retry = 0
        self.is_code_repo = is_code_repo
        self.ingest_options = IngestOptions.from_value(ingest_options)
        self.coalesce_key = coalesce_key
        self.coalesce_version = coalesce_version
        self.changes = changes
        self.source = dict(source) if source else None
        self.generation_trigger = generation_trigger
        self.aggregate_directory = bool(aggregate_directory)
        self.use_hierarchical_aggregation = bool(use_hierarchical_aggregation)
        self.propagate_to_parent = bool(propagate_to_parent)
        self.copy_source_uri = copy_source_uri
        self.file_md5s = dict(file_md5s or {})
        self.artifact_ref = dict(artifact_ref) if artifact_ref else None
        self.artifact_files = list(artifact_files or [])
        self.file_abstracts = dict(file_abstracts or {})
        self.file_vector_actions = {}
        for uri, action in (file_vector_actions or {}).items():
            normalized_action = str(action)
            if normalized_action not in {"upsert", "merge"}:
                raise ValueError(f"unsupported file vector action: {normalized_action}")
            self.file_vector_actions[str(uri)] = normalized_action
        self.file_ingest_options = {
            str(uri): IngestOptions.from_value(options)
            for uri, options in (file_ingest_options or {}).items()
        }
        self.plan = (
            plan
            if isinstance(plan, SemanticPlan)
            else SemanticPlan.from_dict(plan)
            if isinstance(plan, dict)
            else None
        )

    def to_dict(self) -> Dict[str, Any]:
        """Convert object to dictionary."""
        data = asdict(self)
        data["ingest_options"] = self.ingest_options.to_dict()
        data["file_ingest_options"] = {
            uri: options.to_dict() for uri, options in self.file_ingest_options.items()
        }
        if self.plan is not None:
            data["plan"] = self.plan.to_dict()
        return data

    def to_json(self) -> str:
        """Convert object to JSON string."""
        return json.dumps(self.to_dict())

    @classmethod
    def from_dict(cls, data: Dict[str, Any]) -> "SemanticMsg":
        """Safely create object from dictionary, filtering extra fields and handling missing fields."""
        if not data:
            raise ValueError("Data dictionary is empty")

        uri = data.get("uri")
        context_type = data.get("context_type")
        account_id = data.get("account_id")

        if not uri or not context_type:
            missing = []
            if not uri:
                missing.append("uri")
            if not context_type:
                missing.append("context_type")
            raise ValueError(f"Missing required fields: {missing}")
        if not isinstance(account_id, str) or not account_id.strip():
            raise ValueError("Missing required fields: ['account_id']")

        obj = cls(
            uri=uri,
            context_type=context_type,
            recursive=data.get("recursive", True),
            account_id=account_id,
            user_id=data.get("user_id", "default"),
            group_ids=data.get("group_ids") if isinstance(data.get("group_ids"), list) else None,
            peer_id=data.get("peer_id", "default"),
            role=data.get("role", "root"),
            skip_vectorization=data.get("skip_vectorization", False),
            telemetry_id=data.get("telemetry_id", ""),
            task_id=data.get("task_id", ""),
            _task_work_id=data.get("_task_work_id", ""),
            target_uri=data.get("target_uri", ""),
            lock_handoff=data.get("lock_handoff"),
            lock_handoff_retry=data.get("lock_handoff_retry", 0),
            is_code_repo=data.get("is_code_repo", False),
            ingest_options=(
                data.get("ingest_options")
                or {
                    "search_tags": data.get("search_tags"),
                    "search_tag_mode": data.get("search_tag_mode", "replace"),
                }
            ),
            coalesce_key=data.get("coalesce_key", ""),
            coalesce_version=data.get("coalesce_version", 0),
            changes=data.get("changes"),
            source=data.get("source"),
            generation_trigger=data.get("generation_trigger", "semantic_refresh"),
            aggregate_directory=data.get("aggregate_directory", True),
            use_hierarchical_aggregation=data.get("use_hierarchical_aggregation", False),
            propagate_to_parent=data.get("propagate_to_parent", True),
            copy_source_uri=data.get("copy_source_uri", ""),
            file_md5s=(data.get("file_md5s") if isinstance(data.get("file_md5s"), dict) else None),
            artifact_ref=(
                data.get("artifact_ref") if isinstance(data.get("artifact_ref"), dict) else None
            ),
            artifact_files=(
                data.get("artifact_files") if isinstance(data.get("artifact_files"), list) else None
            ),
            file_abstracts=(
                data.get("file_abstracts") if isinstance(data.get("file_abstracts"), dict) else None
            ),
            file_vector_actions=(
                data.get("file_vector_actions")
                if isinstance(data.get("file_vector_actions"), dict)
                else None
            ),
            file_ingest_options=(
                data.get("file_ingest_options")
                if isinstance(data.get("file_ingest_options"), dict)
                else None
            ),
            plan=data.get("plan") if isinstance(data.get("plan"), dict) else None,
            queue_enqueued_at=data.get("queue_enqueued_at", 0.0),
            failure_requeue_count=data.get("failure_requeue_count", 0),
        )
        if "id" in data and data["id"]:
            obj.id = data["id"]
        if "status" in data:
            obj.status = data["status"]
        if "timestamp" in data:
            obj.timestamp = data["timestamp"]
        return obj

    @classmethod
    def from_json(cls, json_str: str) -> "SemanticMsg":
        """Create object from JSON string."""
        try:
            data = json.loads(json_str)
            return cls.from_dict(data)
        except json.JSONDecodeError as e:
            raise ValueError(f"Invalid JSON string: {e}")