Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

14.1 KB
Raw Blame
# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
# SPDX-License-Identifier: AGPL-3.0
"""In-process memory consolidation for `ov compile --skill memory`.

This runs the memory framework (ExtractLoop + MemoryUpdater) directly inside
OpenViking core, without the VikingBot agent path. It borrows the lightweight
`session.commit` task shape: create a tracked task, run one background
``asyncio.Task``, and record running/completed/failed via the task tracker.
There is no QueueFS re-delivery; a crashed run leaves the task failed.
"""

from __future__ import annotations

import asyncio
from typing import Any, Optional

from openviking.core.namespace import classify_uri, uri_parts
from openviking.core.path_variables import resolve_path_variables
from openviking.core.uri_validation import validate_request_viking_uri
from openviking.server.identity import RequestContext
from openviking.service.fs_service import FSService
from openviking.service.task_domain import TaskPatch
from openviking.service.task_tracker import TaskRecord, get_task_tracker
from openviking.service.task_work_index import bind_task_context
from openviking.session.memory.consolidation_context_provider import (
    ConsolidationExtractContextProvider,
    build_consolidation_isolation_handler,
)
from openviking.session.memory.extract_loop import ExtractLoop
from openviking.session.memory.memory_type_registry import MemoryTypeRegistry, get_default_registry
from openviking.session.memory.memory_updater import MemoryUpdater
from openviking.session.memory.streaming_memory_updater import (
    acquire_memory_operation_lease,
)
from openviking.session.memory.utils.uri import render_template
from openviking.telemetry import tracer
from openviking.telemetry.span_models import create_root_span_attributes
from openviking_cli.exceptions import InvalidArgumentError, NotFoundError
from openviking_cli.utils import get_logger
from openviking_cli.utils.config import get_openviking_config

try:
    from openviking.observability.context import (
        bind_root_observability_context,
        reset_root_observability_context,
    )
except ImportError:  # pragma: no cover - observability optional
    bind_root_observability_context = None
    reset_root_observability_context = None

logger = get_logger(__name__)


def _peer_id_from_memory_uri(uri: str) -> Optional[str]:
    """Return the peer_id when a memory URI lives under a peer space, else None."""
    parts = uri_parts(uri)
    if len(parts) >= 5 and parts[0] == "user" and parts[2] == "peers":
        return parts[3]
    return None


def _memory_type_from_target(uri: str) -> Optional[str]:
    classification = classify_uri(uri)
    parts = uri_parts(uri)
    if classification.context_type != "memory" or classification.content_index is None:
        raise InvalidArgumentError(
            "Memory-mode Compile target must be a memories root or inside a memory type directory"
        )
    if classification.is_memory_root:
        return None
    return parts[classification.content_index + 1]


async def _discover_memory_types(
    target: str,
    registry: MemoryTypeRegistry,
    viking_fs: Any,
    ctx: RequestContext,
) -> list[str]:
    parts = uri_parts(target)
    user_space = "/".join(parts[1:-1])
    peer_id = _peer_id_from_memory_uri(target)
    memory_types = []
    for schema in registry.list_all():
        if not schema.directory or (peer_id and not schema.peer_enabled):
            continue
        directory = render_template(schema.directory, {"user_space": user_space}).rstrip("/")
        if directory != target and not directory.startswith(f"{target}/"):
            continue
        path = directory
        is_directory = schema.filename_has_variables()
        if not is_directory:
            path = f"{directory}/{schema.filename_template.lstrip('/')}"
        try:
            stat = await viking_fs.stat(path, ctx=ctx, skip_count=True)
        except NotFoundError:
            continue
        if bool(stat.get("isDir")) == is_directory:
            memory_types.append(schema.memory_type)
    return sorted(memory_types)


class MemoryCompileRunner:
    """Own the validation and background execution of memory-mode Compile tasks."""

    task_type = "compile"
    task_id_prefix = "cmp_"

    def __init__(self, fs: FSService, vikingdb: Any = None) -> None:
        self._fs = fs
        self._vikingdb = vikingdb
        # Keep strong references so background tasks are not garbage collected.
        self._running: set[asyncio.Task[None]] = set()

    def set_vikingdb(self, vikingdb: Any) -> None:
        self._vikingdb = vikingdb

    async def create(
        self,
        *,
        target: str,
        instruction: Optional[str],
        ctx: RequestContext,
    ) -> TaskRecord:
        canonical_target, memory_type, peer_id = await self._normalize_target(target, ctx)
        tracker = get_task_tracker()
        task = await tracker.create(
            self.task_type,
            resource_id=canonical_target,
            account_id=ctx.account_id,
            user_id=ctx.user.user_id,
            meta={
                "request": {
                    "to": canonical_target,
                    "skill": "memory",
                    "memory_type": memory_type,
                    "peer_id": peer_id,
                    "instruction": instruction,
                }
            },
        )
        await tracker.patch_task(
            task.task_id,
            TaskPatch(stage="queued"),
            account_id=ctx.account_id,
            user_id=ctx.user.user_id,
        )
        background = asyncio.create_task(
            self._run(
                task_id=task.task_id,
                target=canonical_target,
                memory_type=memory_type,
                peer_id=peer_id,
                instruction=instruction,
                ctx=ctx,
            )
        )
        self._running.add(background)
        background.add_done_callback(self._running.discard)
        current = await tracker.get(
            task.task_id,
            account_id=ctx.account_id,
            user_id=ctx.user.user_id,
        )
        return current or task

    async def _normalize_target(
        self,
        target: str,
        ctx: RequestContext,
    ) -> tuple[str, Optional[str], Optional[str]]:
        uri = validate_request_viking_uri(
            resolve_path_variables(target),
            ctx,
            field_name="to",
        ).rstrip("/")
        memory_type = _memory_type_from_target(uri)
        await self._fs.ensure_write_access(uri, ctx)
        stat = await self._fs.stat(uri, ctx)
        if not stat.get("isDir"):
            raise InvalidArgumentError("Memory-mode Compile target must be a directory")
        canonical = str(stat.get("uri") or uri).rstrip("/")
        return canonical, memory_type, _peer_id_from_memory_uri(canonical)

    async def _run(
        self,
        *,
        task_id: str,
        target: str,
        memory_type: Optional[str],
        peer_id: Optional[str],
        instruction: Optional[str],
        ctx: RequestContext,
    ) -> None:
        tracker = get_task_tracker()
        running_work_id = await tracker.register_running_task(
            task_id,
            account_id=ctx.account_id,
            user_id=ctx.user.user_id,
        )
        root_token = None
        if bind_root_observability_context is not None:
            root_attrs = create_root_span_attributes(
                http_method="TASK",
                http_route="/compile/memory",
                request_id=task_id,
                url_path=target,
            )
            root_attrs.account_id = ctx.account_id
            root_attrs.user_id = ctx.user.user_id
            root_token = bind_root_observability_context(root_attrs)
        try:
            with tracer.start_as_current_span(name="compile.memory.consolidate"):
                trace_id = tracer.get_trace_id() or ""
                await tracker.start(
                    task_id,
                    account_id=ctx.account_id,
                    user_id=ctx.user.user_id,
                    stage="consolidating",
                )
                with bind_task_context(
                    task_id,
                    ctx.account_id,
                    ctx.user.user_id,
                    work_id=running_work_id,
                ):
                    result = await self._consolidate(
                        target=target,
                        memory_type=memory_type,
                        peer_id=peer_id,
                        instruction=instruction,
                        ctx=ctx,
                    )
                result = {"to": target, "skill": "memory", "trace_id": trace_id, **result}
                if result["errors"]:
                    if not any(result[key] for key in ("adds", "updates", "deletes")):
                        await tracker.fail(
                            task_id,
                            "MEMORY_CONSOLIDATION_FAILED: No memory changes succeeded; "
                            "see result.errors for details",
                            account_id=ctx.account_id,
                            user_id=ctx.user.user_id,
                            result=result,
                        )
                        return
                    logger.warning(
                        "Memory-mode compile %s partially succeeded: %s",
                        task_id,
                        result["errors"],
                    )
                await tracker.complete(
                    task_id,
                    result,
                    account_id=ctx.account_id,
                    user_id=ctx.user.user_id,
                )
        except asyncio.CancelledError:
            return
        except Exception as exc:  # noqa: BLE001 - surface any failure as task error
            tracer.error(f"Memory-mode compile failed: {exc}")
            await tracker.fail(
                task_id,
                f"MEMORY_CONSOLIDATION_FAILED: {exc}",
                account_id=ctx.account_id,
                user_id=ctx.user.user_id,
            )
        finally:
            if root_token is not None and reset_root_observability_context is not None:
                reset_root_observability_context(root_token)
            await tracker.unregister_running_task(task_id)

    async def _consolidate(
        self,
        *,
        target: str,
        memory_type: Optional[str],
        peer_id: Optional[str],
        instruction: Optional[str],
        ctx: RequestContext,
    ) -> dict[str, Any]:
        default_registry = get_default_registry()
        viking_fs = self._fs._ensure_initialized()
        from openviking.session.memory.account_templates import resolve_account_memory_registry

        registry = await resolve_account_memory_registry(
            viking_fs, ctx.account_id, default_registry
        )
        memory_types = (
            [memory_type]
            if memory_type is not None
            else await _discover_memory_types(target, registry, viking_fs, ctx)
        )
        result = {
            "memory_type": memory_type,
            "memory_types": memory_types,
            "adds": [],
            "updates": [],
            "deletes": [],
            "total_adds": 0,
            "total_updates": 0,
            "total_deletes": 0,
            "errors": [],
        }
        if not memory_types:
            return result
        config = get_openviking_config()
        vlm = config.vlm.get_vlm_instance()
        provider = ConsolidationExtractContextProvider(
            memory_types=memory_types,
            target_directory=target,
            instruction=instruction,
            memory_registry=registry,
            ctx=ctx,
            viking_fs=viking_fs,
        )

        extract_context = provider.get_extract_context()
        isolation_handler = build_consolidation_isolation_handler(
            ctx,
            extract_context,
            memory_types=memory_types,
            peer_id=peer_id,
        )
        isolation_handler.prepare_messages()
        provider.bind_isolation_handler(isolation_handler)
        await provider.prepare_extraction_messages()

        orchestrator = ExtractLoop(
            vlm=vlm,
            viking_fs=viking_fs,
            ctx=ctx,
            context_provider=provider,
            isolation_handler=isolation_handler,
            # Compile is offline and user-initiated; allow a longer agentic loop
            # so the model can ls/search/read across the directory before writing.
            max_iterations=10,
        )
        operations, _tools_used = await orchestrator.run()
        if operations is None:
            return result

        lease = await acquire_memory_operation_lease(operations, viking_fs, ctx)
        try:
            updater = MemoryUpdater(
                registry=registry,
                vikingdb=self._vikingdb,
                transaction_handle=lease,
            )
            apply_result = await updater.apply_operations(
                operations,
                ctx,
                extract_context=extract_context,
                isolation_handler=isolation_handler,
            )
        finally:
            if lease is not None:
                await viking_fs._async_agfs.pathlock_release(lease)

        # Classify each touched URI as add vs update using the files the model
        # actually read. ExtractLoop's write-before-read guard guarantees any
        # modified pre-existing file is in read_file_contents, so a written URI
        # absent from it is a genuinely new file.
        before_uris = set(provider.read_file_contents.keys())
        adds: list[str] = []
        updates: list[str] = []
        for uri in apply_result.written_uris:
            (updates if uri in before_uris else adds).append(uri)
        for uri in apply_result.edited_uris:
            if uri not in updates:
                updates.append(uri)
        deletes = list(apply_result.deleted_uris)
        return {
            **result,
            "adds": adds,
            "updates": updates,
            "deletes": deletes,
            "total_adds": len(adds),
            "total_updates": len(updates),
            "total_deletes": len(deletes),
            "errors": [str(err) for _uri, err in apply_result.errors],
        }


__all__ = ["MemoryCompileRunner"]