Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

20.5 KB
Raw Blame
# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
# SPDX-License-Identifier: AGPL-3.0
"""Async adapter for the synchronous AGFS client."""

from __future__ import annotations

import asyncio
from collections.abc import Iterator
from typing import Any, BinaryIO, Dict, List, Union

from openviking.service.task_tracker_concurrency import run_to_completion
from openviking.storage.errors import InvalidPathLockRequestError

from .leader import _LOCAL_LEADER_BACKEND, LeaderCandidate, LeaderMode, leader_candidate
from .protocols import AGFSSyncClientProtocol

_SYSTEM_ACCOUNT_ID = "_system"


def fs_ctx_from_agfs_path(path: str) -> Dict[str, str]:
    """Derive a stable FsContext from an absolute AGFS path.

    `/local/{account_id}/...` paths use their path-scoped account to match
    VikingFS URI conversion. Plugin/system paths do not encode a tenant, so
    they use the reserved system account key instead of running without ctx.
    """
    parts = path.strip("/").split("/")
    if len(parts) >= 2 and parts[0] == "local" and parts[1]:
        return {"account_id": parts[1]}
    return {"account_id": _SYSTEM_ACCOUNT_ID}


def _fs_ctx_or_default(path: str, fs_ctx: Dict[str, str] | None) -> Dict[str, str]:
    """Return explicit FsContext when present, otherwise derive it from path."""
    return fs_ctx if fs_ctx is not None else fs_ctx_from_agfs_path(path)


def _fs_ctx_with_auto_pathlock(
    path: str,
    fs_ctx: Dict[str, str] | None,
    auto_pathlock: bool,
) -> Dict[str, str]:
    """Return FsContext with optional auto PathLock bypass for unlocked calls.

    Args:
        path: AGFS path used to derive a default context when ``fs_ctx`` is absent.
        fs_ctx: Optional caller-provided context.
        auto_pathlock: Whether PathLockWrappedFS should acquire locks automatically.

    Returns:
        A copied FsContext. Existing ``lease_ref`` takes precedence and keeps wrapper
        lease validation enabled; otherwise ``auto_pathlock=False`` disables auto-locking.
    """
    ctx = dict(_fs_ctx_or_default(path, fs_ctx))
    if ctx.get("lease_ref"):
        ctx.pop("disable_auto_pathlock", None)
    elif not auto_pathlock:
        ctx["disable_auto_pathlock"] = "true"
    return ctx


def local_account_id_from_agfs_path(path: str) -> str | None:
    """Extract the account_id from `/local/{account_id}/...`, or None for non-local paths."""
    parts = path.strip("/").split("/")
    if len(parts) >= 2 and parts[0] == "local" and parts[1]:
        return parts[1]
    return None


def encryption_account_id_from_agfs_path(path: str) -> str:
    """Return the account_id used by the encryption layer for this AGFS path."""
    return fs_ctx_from_agfs_path(path)["account_id"]


def ensure_same_encryption_account(src_path: str, dst_path: str) -> None:
    """Reject AGFS moves/copies across encryption-account domains."""
    src_account = encryption_account_id_from_agfs_path(src_path)
    dst_account = encryption_account_id_from_agfs_path(dst_path)
    if src_account != dst_account:
        raise ValueError(
            f"cross-account AGFS move/copy is not supported: {src_account!r} -> {dst_account!r}"
        )


class AsyncAGFSClient:
    """Run blocking AGFS binding client operations off the event loop.

    This is a thin adapter over ``RAGFSBindingClient``. Blocking methods use a
    worker thread; native async methods, currently PathLock acquisition, run
    without occupying that shared executor.
    """

    def __init__(self, client: AGFSSyncClientProtocol):
        self._client = client

    def leader_candidate(
        self,
        key: str,
        holder_id: str,
        *,
        ttl_secs: float,
        renew_interval_secs: float,
        retry_interval_secs: float,
        mode: LeaderMode = "distributed",
    ) -> LeaderCandidate:
        """Create a leader candidate whose terms expose only a stop event."""
        return leader_candidate(
            self._client,
            _LOCAL_LEADER_BACKEND,
            key=key,
            holder_id=holder_id,
            ttl_secs=ttl_secs,
            renew_interval_secs=renew_interval_secs,
            retry_interval_secs=retry_interval_secs,
            mode=mode,
        )

    async def run(self, method_name: str, /, *args: Any, **kwargs: Any) -> Any:
        """Run a sync method off-thread, settling I/O before propagating cancellation."""
        method = getattr(self._client, method_name)
        try:
            return await run_to_completion(lambda: asyncio.to_thread(method, *args, **kwargs))
        except TypeError as exc:
            message = str(exc)
            if "ctx" not in kwargs or "unexpected keyword argument 'ctx'" not in message:
                raise
            kwargs.pop("ctx")
            return await run_to_completion(lambda: asyncio.to_thread(method, *args, **kwargs))

    async def ls(
        self,
        path: str = "/",
        *,
        offset: int = 0,
        limit: int | None = None,
        sort_by: str | None = None,
        sort_order: str = "asc",
        fs_ctx: Dict[str, str] | None = None,
    ) -> List[Dict[str, Any]]:
        """Return a sorted directory range."""
        kwargs: Dict[str, Any] = {}
        if offset:
            kwargs["offset"] = offset
        if limit is not None:
            kwargs["limit"] = limit
        if sort_by is not None:
            kwargs["sort_by"] = sort_by
            kwargs["sort_order"] = sort_order
        return await self.run("ls", path, **kwargs, ctx=_fs_ctx_or_default(path, fs_ctx))

    async def read(
        self,
        path: str,
        offset: int = 0,
        size: int = -1,
        stream: bool = False,
        *,
        fs_ctx: Dict[str, str] | None = None,
    ) -> Any:
        kwargs: Dict[str, Any] = {}
        if offset != 0:
            kwargs["offset"] = offset
        if size != -1:
            kwargs["size"] = size
        if stream:
            kwargs["stream"] = stream
        return await self.run("read", path, **kwargs, ctx=_fs_ctx_or_default(path, fs_ctx))

    async def cat(
        self,
        path: str,
        offset: int = 0,
        size: int = -1,
        stream: bool = False,
        *,
        fs_ctx: Dict[str, str] | None = None,
    ) -> Any:
        kwargs: Dict[str, Any] = {}
        if offset != 0:
            kwargs["offset"] = offset
        if size != -1:
            kwargs["size"] = size
        if stream:
            kwargs["stream"] = stream
        return await self.run("cat", path, **kwargs, ctx=_fs_ctx_or_default(path, fs_ctx))

    async def write(
        self,
        path: str,
        data: Union[bytes, Iterator[bytes], BinaryIO],
        max_retries: int = 3,
        *,
        fs_ctx: Dict[str, str] | None = None,
        auto_pathlock: bool = True,
    ) -> str:
        """Write file content to AGFS.

        1. Automatic PathLock is enabled by default to prevent data conflicts in concurrent scenarios.
        2. If automatic PathLock is disabled, the caller must assess the possible data conflict risk.
        """
        ctx = _fs_ctx_with_auto_pathlock(path, fs_ctx, auto_pathlock)
        if max_retries == 3:
            return await self.run("write", path, data, ctx=ctx)
        return await self.run("write", path, data, max_retries=max_retries, ctx=ctx)

    async def mkdir(
        self, path: str, mode: str = "755", *, fs_ctx: Dict[str, str] | None = None
    ) -> Dict[str, Any]:
        if mode == "755":
            return await self.run("mkdir", path, ctx=_fs_ctx_or_default(path, fs_ctx))
        return await self.run("mkdir", path, mode=mode, ctx=_fs_ctx_or_default(path, fs_ctx))

    async def ensure_parent_dirs(
        self, path: str, mode: str = "755", *, fs_ctx: Dict[str, str] | None = None
    ) -> Dict[str, Any]:
        if mode == "755":
            return await self.run("ensure_parent_dirs", path, ctx=_fs_ctx_or_default(path, fs_ctx))
        return await self.run(
            "ensure_parent_dirs", path, mode=mode, ctx=_fs_ctx_or_default(path, fs_ctx)
        )

    async def rm(
        self,
        path: str,
        recursive: bool = False,
        force: bool = True,
        *,
        fs_ctx: Dict[str, str] | None = None,
        auto_pathlock: bool = True,
    ) -> Dict[str, Any]:
        """Remove a file or directory from AGFS.

        1. Automatic PathLock is enabled by default to prevent data conflicts in concurrent scenarios.
        2. If automatic PathLock is disabled, the caller must assess the possible data conflict risk.
        """
        kwargs: Dict[str, Any] = {}
        if recursive:
            kwargs["recursive"] = recursive
        if not force:
            kwargs["force"] = force
        return await self.run(
            "rm",
            path,
            **kwargs,
            ctx=_fs_ctx_with_auto_pathlock(path, fs_ctx, auto_pathlock),
        )

    async def stat(
        self, path: str, *, fs_ctx: Dict[str, str] | None = None, bypass_cache: bool = False
    ) -> Dict[str, Any]:
        ctx = dict(_fs_ctx_or_default(path, fs_ctx))
        if bypass_cache:
            # Force plugin-local metadata caches (e.g. S3FS stat cache) to serve
            # fresh backend state so pollers observe writer-side changes.
            ctx["bypass_cache"] = "true"
        return await self.run("stat", path, ctx=ctx)

    async def mv(
        self,
        old_path: str,
        new_path: str,
        *,
        fs_ctx: Dict[str, str] | None = None,
        auto_pathlock: bool = True,
    ) -> Dict[str, Any]:
        """Move or rename a path inside AGFS.

        1. Automatic PathLock is enabled by default to prevent data conflicts in concurrent scenarios.
        2. If automatic PathLock is disabled, the caller must assess the possible data conflict risk.
        """
        ensure_same_encryption_account(old_path, new_path)
        return await self.run(
            "mv",
            old_path,
            new_path,
            ctx=_fs_ctx_with_auto_pathlock(old_path, fs_ctx, auto_pathlock),
        )

    async def cp(
        self,
        src_path: str,
        dst_path: str,
        recursive: bool = False,
        *,
        fs_ctx: Dict[str, str] | None = None,
        auto_pathlock: bool = True,
        allow_same_mount_fast_path: bool = False,
    ) -> Any:
        """Copy a path within AGFS while preserving the caller's FsContext.

        1. Automatic PathLock is enabled by default to prevent data conflicts in concurrent scenarios.
        2. If automatic PathLock is disabled, the caller must assess the possible data conflict risk.
        """
        from .helpers import cp

        return await run_to_completion(
            lambda: asyncio.to_thread(
                cp,
                self._client,
                src_path,
                dst_path,
                recursive=recursive,
                stream=True,
                fs_ctx=_fs_ctx_with_auto_pathlock(src_path, fs_ctx, auto_pathlock),
                allow_same_mount_fast_path=allow_same_mount_fast_path,
            )
        )

    async def grep(self, **kwargs: Any) -> Dict[str, Any]:
        if "ctx" not in kwargs or kwargs["ctx"] is None:
            path = kwargs.get("path")
            if isinstance(path, str):
                kwargs["ctx"] = fs_ctx_from_agfs_path(path)
        return await self.run("grep", **kwargs)

    async def tree_directory(
        self,
        path: str,
        show_hidden: bool = False,
        node_limit: int | None = None,
        level_limit: int | None = None,
        *,
        offset: int = 0,
        sort_by: str | None = None,
        sort_order: str = "asc",
        directories_only: bool = False,
        fs_ctx: Dict[str, str] | None = None,
    ) -> list[Dict[str, Any]]:
        """Return a sorted range from a recursive directory traversal."""
        kwargs: Dict[str, Any] = {
            "show_hidden": show_hidden,
            "node_limit": node_limit,
            "level_limit": level_limit,
        }
        if directories_only:
            kwargs["directories_only"] = True
        if offset:
            kwargs["offset"] = offset
        if sort_by is not None:
            kwargs["sort_by"] = sort_by
            kwargs["sort_order"] = sort_order
        return await self.run(
            "tree_directory",
            path,
            **kwargs,
            ctx=_fs_ctx_or_default(path, fs_ctx),
        )

    async def glob_directory(
        self,
        path: str,
        pattern: str,
        show_hidden: bool = False,
        page_size: int | None = None,
        level_limit: int | None = None,
        continuation_token: str | None = None,
        *,
        fs_ctx: Dict[str, str] | None = None,
    ) -> Dict[str, Any]:
        return await self.run(
            "glob_directory",
            path,
            pattern,
            show_hidden=show_hidden,
            page_size=page_size,
            level_limit=level_limit,
            continuation_token=continuation_token,
            ctx=_fs_ctx_or_default(path, fs_ctx),
        )

    # -- pathlock async wrappers ------------------------------------------------

    async def pathlock_acquire_exact(
        self,
        path: str,
        timeout_secs: float = 0.0,
        owner_lease_ref: Dict[str, Any] | None = None,
        *,
        fs_ctx: Dict[str, str] | None = None,
    ) -> Dict[str, Any]:
        """Acquire an exact lock on a single path."""
        return await self.pathlock_acquire_batch(
            [{"path": path, "kind": "exact"}],
            timeout_secs,
            owner_lease_ref,
            fs_ctx=fs_ctx,
        )

    async def pathlock_acquire_exact_batch(
        self,
        paths: list[str],
        timeout_secs: float = 0.0,
        owner_lease_ref: Dict[str, Any] | None = None,
        *,
        fs_ctx: Dict[str, str] | None = None,
    ) -> Dict[str, Any]:
        """Acquire exact locks on multiple paths."""
        return await self.pathlock_acquire_batch(
            [{"path": path, "kind": "exact"} for path in paths],
            timeout_secs,
            owner_lease_ref,
            fs_ctx=fs_ctx,
        )

    async def pathlock_acquire_tree(
        self,
        path: str,
        timeout_secs: float = 0.0,
        owner_lease_ref: Dict[str, Any] | None = None,
        *,
        fs_ctx: Dict[str, str] | None = None,
    ) -> Dict[str, Any]:
        """Acquire a tree lock on a single path."""
        return await self.pathlock_acquire_batch(
            [{"path": path, "kind": "tree"}],
            timeout_secs,
            owner_lease_ref,
            fs_ctx=fs_ctx,
        )

    async def pathlock_acquire_tree_batch(
        self,
        paths: list[str],
        timeout_secs: float = 0.0,
        owner_lease_ref: Dict[str, Any] | None = None,
        *,
        fs_ctx: Dict[str, str] | None = None,
    ) -> Dict[str, Any]:
        """Acquire tree locks on multiple paths."""
        return await self.pathlock_acquire_batch(
            [{"path": path, "kind": "tree"} for path in paths],
            timeout_secs,
            owner_lease_ref,
            fs_ctx=fs_ctx,
        )

    async def pathlock_acquire_exact_tree_batch(
        self,
        exact_paths: list[str],
        tree_paths: list[str],
        timeout_secs: float = 0.0,
        owner_lease_ref: Dict[str, Any] | None = None,
        *,
        fs_ctx: Dict[str, str] | None = None,
    ) -> Dict[str, Any]:
        """Acquire a mixed batch of exact and tree locks."""
        requests = [{"path": path, "kind": "exact"} for path in exact_paths]
        requests.extend({"path": path, "kind": "tree"} for path in tree_paths)
        return await self.pathlock_acquire_batch(
            requests,
            timeout_secs,
            owner_lease_ref,
            fs_ctx=fs_ctx,
        )

    async def pathlock_acquire_batch(
        self,
        requests: list[Dict[str, str]],
        timeout_secs: float = 0.0,
        owner_lease_ref: Dict[str, Any] | None = None,
        *,
        fs_ctx: Dict[str, str] | None = None,
    ) -> Dict[str, Any]:
        """Acquire a batch of locks from a list of request dicts."""
        if not requests:
            raise ValueError("pathlock request batch must not be empty")
        for request in requests:
            path = request.get("path")
            if not path or not path.startswith("/"):
                raise ValueError("pathlock request.path must be an absolute path")
            if request.get("kind") not in {"exact", "tree"}:
                raise ValueError("pathlock request.kind must be 'exact' or 'tree'")
        fs_ctx = _fs_ctx_or_default(requests[0]["path"], fs_ctx)
        future = asyncio.ensure_future(
            self._client.pathlock_acquire_batch_async(
                fs_ctx, requests, timeout_secs, owner_lease_ref
            )
        )
        try:
            return await asyncio.shield(future)
        except asyncio.CancelledError as cancelled:

            async def release_acquired() -> None:
                await self.pathlock_release(await future, fs_ctx=fs_ctx)

            try:
                await run_to_completion(release_acquired)
            finally:
                raise cancelled

    async def pathlock_as_borrowed(
        self,
        owned_lease_ref: Dict[str, Any],
        *,
        fs_ctx: Dict[str, str] | None = None,
    ) -> Dict[str, Any]:
        """Create a borrowed view of an owned lease."""
        return await self.run(
            "pathlock_as_borrowed",
            _fs_ctx_or_default("/", fs_ctx),
            owned_lease_ref,
        )

    async def pathlock_refresh(
        self,
        owned_lease_ref: Dict[str, Any],
        *,
        fs_ctx: Dict[str, str] | None = None,
    ) -> str:
        """Refresh an owned lease. Returns 'refreshed', 'lost', or 'failed'."""
        return await self.run(
            "pathlock_refresh",
            _fs_ctx_or_default("/", fs_ctx),
            owned_lease_ref,
        )

    async def pathlock_release(
        self,
        owned_lease_ref: Dict[str, Any],
        *,
        fs_ctx: Dict[str, str] | None = None,
    ) -> None:
        """Release an owned lease."""
        await self.run(
            "pathlock_release",
            _fs_ctx_or_default("/", fs_ctx),
            owned_lease_ref,
        )

    async def pathlock_release_selected(
        self,
        owned_lease_ref: Dict[str, Any],
        lock_paths: list[str],
        *,
        fs_ctx: Dict[str, str] | None = None,
    ) -> None:
        """Release selected lock paths from an owned lease."""
        await self.run(
            "pathlock_release_selected",
            _fs_ctx_or_default("/", fs_ctx),
            owned_lease_ref,
            lock_paths,
        )

    async def pathlock_to_handoff(
        self,
        owned_lease_ref: Dict[str, Any],
        *,
        fs_ctx: Dict[str, str] | None = None,
    ) -> Dict[str, Any]:
        """Export a handoff ref from an owned lease."""
        return await self.run(
            "pathlock_to_handoff",
            _fs_ctx_or_default("/", fs_ctx),
            owned_lease_ref,
        )

    async def pathlock_handoff(
        self,
        owned_lease_ref: Dict[str, Any],
        *,
        fs_ctx: Dict[str, str] | None = None,
    ) -> None:
        """Mark an owned lease as handed off."""
        await self.run(
            "pathlock_handoff",
            _fs_ctx_or_default("/", fs_ctx),
            owned_lease_ref,
        )

    async def pathlock_adopt(
        self,
        handoff_ref: Dict[str, Any],
        *,
        fs_ctx: Dict[str, str] | None = None,
    ) -> Dict[str, Any]:
        """Adopt a handoff ref, returning a new owned lease."""
        try:
            return await self.run(
                "pathlock_adopt",
                _fs_ctx_or_default("/", fs_ctx),
                handoff_ref,
            )
        except ValueError as exc:
            # The native binding maps PathLockError::InvalidRequest to ValueError.
            # Give queue consumers a typed, non-retryable error without changing
            # unrelated ValueError contracts from legacy binding clients.
            if str(exc).startswith("invalid lock request:"):
                raise InvalidPathLockRequestError(str(exc)) from exc
            raise

    async def pathlock_is_locked(
        self,
        path: str,
        ignore_stale: bool = True,
        *,
        fs_ctx: Dict[str, str] | None = None,
    ) -> bool:
        """Check if a path is locked."""
        return await self.run(
            "pathlock_is_locked",
            _fs_ctx_or_default(path, fs_ctx),
            path,
            ignore_stale,
        )

    async def pathlock_observe(
        self,
        *,
        fs_ctx: Dict[str, str] | None = None,
    ) -> Dict[str, Any]:
        """Return an observability snapshot of current lock state."""
        return await self.run(
            "pathlock_observe",
            _fs_ctx_or_default("/", fs_ctx),
        )