Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

24.7 KB
Raw Blame
# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
# SPDX-License-Identifier: AGPL-3.0
"""
Playback module for IORecorder.

Replays recorded IO operations and compares performance across different backends.
"""

import asyncio
import json
import time
from dataclasses import dataclass
from typing import Any, Dict, List, Optional

from openviking.eval.recorder import IORecord, IOType
from openviking_cli.utils.config import OPENVIKING_CONFIG_ENV
from openviking_cli.utils.logger import get_logger

logger = get_logger(__name__)


@dataclass
class PlaybackResult:
    """
    Result of a single playback operation.

    Attributes:
        record: Original IO record
        playback_latency_ms: Latency during playback
        playback_success: Whether playback succeeded
        playback_error: Error message if failed
        response_match: Whether response matches original
    """

    record: IORecord
    playback_latency_ms: float = 0.0
    playback_success: bool = True
    playback_error: Optional[str] = None
    response_match: Optional[bool] = None


@dataclass
class PlaybackStats:
    """
    Statistics for playback session.

    Attributes:
        total_records: Total number of records played
        success_count: Number of successful operations
        error_count: Number of failed operations
        total_original_latency_ms: Total original latency
        total_playback_latency_ms: Total playback latency
        fs_stats: Statistics for FS operations
        vikingdb_stats: Statistics for VikingDB operations
        viking_fs_success_count: VikingFS operation success count
        viking_fs_error_count: VikingFS operation error count
        agfs_fs_success_count: AGFS FS operation success count
        agfs_fs_error_count: AGFS FS operation error count
        total_agfs_calls: Total number of AGFS calls across all VikingFS operations
        total_viking_fs_operations: Total number of VikingFS operations with AGFS calls
    """

    total_records: int = 0
    success_count: int = 0
    error_count: int = 0
    total_original_latency_ms: float = 0.0
    total_playback_latency_ms: float = 0.0
    fs_stats: Dict[str, Dict[str, Any]] = None
    vikingdb_stats: Dict[str, Dict[str, Any]] = None
    viking_fs_success_count: int = 0
    viking_fs_error_count: int = 0
    agfs_fs_success_count: int = 0
    agfs_fs_error_count: int = 0
    total_agfs_calls: int = 0
    total_viking_fs_operations: int = 0

    def __post_init__(self):
        if self.fs_stats is None:
            self.fs_stats = {}
        if self.vikingdb_stats is None:
            self.vikingdb_stats = {}

    def to_dict(self) -> Dict[str, Any]:
        """Convert to dictionary."""
        viking_fs_success_rate = (
            self.viking_fs_success_count / self.total_viking_fs_operations * 100
            if self.total_viking_fs_operations > 0
            else 0
        )
        agfs_fs_total = self.agfs_fs_success_count + self.agfs_fs_error_count
        agfs_fs_success_rate = (
            self.agfs_fs_success_count / agfs_fs_total * 100 if agfs_fs_total > 0 else 0
        )
        avg_agfs_calls = (
            self.total_agfs_calls / self.total_viking_fs_operations
            if self.total_viking_fs_operations > 0
            else 0
        )

        return {
            "total_records": self.total_records,
            "success_count": self.success_count,
            "error_count": self.error_count,
            "total_original_latency_ms": self.total_original_latency_ms,
            "total_playback_latency_ms": self.total_playback_latency_ms,
            "speedup_ratio": (
                self.total_original_latency_ms / self.total_playback_latency_ms
                if self.total_playback_latency_ms > 0
                else 0
            ),
            "viking_fs_stats": {
                "success_count": self.viking_fs_success_count,
                "error_count": self.viking_fs_error_count,
                "success_rate_percent": viking_fs_success_rate,
                "total_operations": self.total_viking_fs_operations,
                "avg_agfs_calls_per_operation": avg_agfs_calls,
            },
            "agfs_fs_stats": {
                "success_count": self.agfs_fs_success_count,
                "error_count": self.agfs_fs_error_count,
                "success_rate_percent": agfs_fs_success_rate,
                "total_calls": agfs_fs_total,
            },
            "fs_stats": self.fs_stats,
            "vikingdb_stats": self.vikingdb_stats,
        }


class _AGFSCallCollector:
    """
    Helper class to collect AGFS calls during playback for comparison.
    """

    def __init__(self, agfs_client: Any):
        self._agfs = agfs_client
        self.calls: List[Dict[str, Any]] = []

    def __getattr__(self, name: str):
        original_attr = getattr(self._agfs, name)
        if not callable(original_attr):
            return original_attr

        def wrapped(*args, **kwargs):
            call_record = {
                "operation": name,
                "request": {"args": args, "kwargs": kwargs},
                "success": True,
                "error": None,
            }
            try:
                response = original_attr(*args, **kwargs)
                call_record["response"] = response
                return response
            except Exception as e:
                call_record["success"] = False
                call_record["error"] = str(e)
                raise
            finally:
                self.calls.append(call_record)

        return wrapped


class IOPlayback:
    """
    Playback recorded IO operations.

    Replays recorded operations against a target backend and measures performance.

    Usage:
        playback = IOPlayback(config_file="./ov.conf")
        stats = await playback.play(record_file="./records/io_recorder_20260214.jsonl")
        print(stats.to_dict())
    """

    def __init__(
        self,
        config_file: Optional[str] = None,
        compare_response: bool = False,
        fail_fast: bool = False,
        enable_fs: bool = True,
        enable_vikingdb: bool = True,
        check_agfs_calls: bool = True,
    ):
        """
        Initialize IOPlayback.

        Args:
            config_file: Path to OpenViking config file (ov.conf)
            compare_response: Whether to compare playback response with original
            fail_fast: Stop on first error
            enable_fs: Whether to play FS operations
            enable_vikingdb: Whether to play VikingDB operations
            check_agfs_calls: Whether to check AGFS calls match recorded calls
        """
        self.config_file = config_file
        self.compare_response = compare_response
        self.fail_fast = fail_fast
        self.enable_fs = enable_fs
        self.enable_vikingdb = enable_vikingdb
        self.check_agfs_calls = check_agfs_calls
        self._viking_fs = None
        self._vector_store = None

    def _path_to_uri(self, path: str) -> str:
        """Convert AGFS path to VikingFS URI."""
        return self._viking_fs._path_to_uri(path)

    def _init_backends(self) -> None:
        """Initialize backend clients from config."""
        if self.config_file:
            import os

            os.environ[OPENVIKING_CONFIG_ENV] = self.config_file

        from openviking.config.vlm import ClusterVLMResolver
        from openviking.storage.viking_fs import init_viking_fs
        from openviking.storage.viking_vector_index_backend import VikingVectorIndexBackend
        from openviking.utils.agfs_utils import RagfsBindingConfig, create_agfs_client
        from openviking_cli.utils.config import get_openviking_config
        from openviking_cli.utils.config.vectordb_config import VectorDBBackendConfig

        config = get_openviking_config()
        agfs_config = config.storage.agfs

        # Create RAGFS client using utility
        agfs_client = create_agfs_client(RagfsBindingConfig(agfs=agfs_config))

        vector_store = None
        if self.enable_vikingdb:
            vectordb_config = config.storage.vectordb
            backend_config = VectorDBBackendConfig(
                backend=vectordb_config.backend or "local",
                path=vectordb_config.path or "./data/vectordb",
                url=vectordb_config.url,
                dimension=config.embedding.dimension,
            )
            if vectordb_config.volcengine:
                backend_config.volcengine = vectordb_config.volcengine
            vector_store = VikingVectorIndexBackend(config=backend_config)

        if self.enable_fs:
            # Use init_viking_fs which handles mode (HTTP/Binding) automatically based on agfs_config
            self._viking_fs = init_viking_fs(
                agfs=agfs_client,
                vector_store=vector_store,
                vlm_resolver=ClusterVLMResolver(lambda: config),
            )
        self._vector_store = vector_store

        logger.info(
            f"[IOPlayback] Initialized with config: {self.config_file}, "
            f"fs={self.enable_fs}, vikingdb={self.enable_vikingdb}"
        )

    async def _play_fs_operation(self, record: IORecord) -> PlaybackResult:
        """Play a single FS operation."""
        result = PlaybackResult(record=record)
        start_time = time.time()
        args0 = None

        try:
            operation = record.operation
            request = record.request

            if "args" in request or "kwargs" in request:
                args = request.get("args", [])
                kwargs = request.get("kwargs", {})
                args0 = args[0] if args else kwargs.get("path", kwargs.get("uri", ""))
            else:
                args = []
                kwargs = request
                args0 = kwargs.get("path", kwargs.get("uri", ""))

            collector = None
            original_agfs = None
            if self.check_agfs_calls and hasattr(record, "agfs_calls") and record.agfs_calls:
                collector = _AGFSCallCollector(self._viking_fs.agfs)
                original_agfs = self._viking_fs.agfs
                self._viking_fs.agfs = collector

            def process_arg(arg: Any) -> Any:
                if isinstance(arg, dict) and "__bytes__" in arg:
                    return arg["__bytes__"].encode("utf-8")
                if isinstance(arg, dict):
                    return {k: process_arg(v) for k, v in arg.items()}
                if isinstance(arg, list):
                    return [process_arg(item) for item in arg]
                return arg

            processed_args = [process_arg(arg) for arg in args]
            processed_kwargs = {k: process_arg(v) for k, v in kwargs.items()}

            method = getattr(self._viking_fs, operation)
            await method(*processed_args, **processed_kwargs)

            if collector and original_agfs:
                self._viking_fs.agfs = original_agfs
                result.response_match = self._compare_agfs_calls(record.agfs_calls, collector.calls)
                if not result.response_match:
                    result.playback_error = "AGFS calls mismatch"

            result.playback_latency_ms = (time.time() - start_time) * 1000
            result.playback_success = True

        except Exception as e:
            if original_agfs:
                self._viking_fs.agfs = original_agfs
            result.playback_latency_ms = (time.time() - start_time) * 1000
            playback_error = str(e)

            if record.error and self._errors_match(playback_error, record.error):
                result.playback_success = True
                result.playback_error = f"Matched original error: {playback_error}"
            else:
                result.playback_success = False
                result.playback_error = playback_error
                logger.error(f"[IOPlayback] FS {operation} on {args0} failed: {e}")

        return result

    def _compare_agfs_calls(
        self, recorded_calls: List[Any], actual_calls: List[Dict[str, Any]]
    ) -> bool:
        """
        Compare recorded AGFS calls with actual AGFS calls.

        Args:
            recorded_calls: List of recorded AGFS calls (AGFSCallRecord or dict)
            actual_calls: List of actual AGFS calls (dicts)

        Returns:
            True if calls match, False otherwise
        """
        if len(recorded_calls) != len(actual_calls):
            logger.warning(
                f"AGFS call count mismatch: recorded {len(recorded_calls)}, actual {len(actual_calls)}"
            )
            return False

        for recorded_call, actual_call in zip(recorded_calls, actual_calls, strict=True):
            if isinstance(recorded_call, dict):
                recorded_op = recorded_call.get("operation")
                recorded_req = recorded_call.get("request")
                recorded_success = recorded_call.get("success", True)
            else:
                recorded_op = recorded_call.operation
                recorded_req = recorded_call.request
                recorded_success = recorded_call.success

            if recorded_op != actual_call["operation"]:
                logger.warning(
                    f"AGFS operation mismatch: recorded {recorded_op}, actual {actual_call['operation']}"
                )
                return False
            if recorded_req != actual_call["request"]:
                logger.warning(f"AGFS request mismatch for operation {recorded_op}")
                return False
            if recorded_success != actual_call["success"]:
                logger.warning(f"AGFS success status mismatch for operation {recorded_op}")
                return False

        return True

    def _errors_match(self, playback_error: str, record_error: str) -> bool:
        """Check if playback error matches original record error."""
        playback_lower = playback_error.lower()
        record_lower = record_error.lower()

        if playback_lower == record_lower:
            return True

        error_type_patterns = [
            (
                "no such file",
                ["no such file", "not found", "does not exist", "no such file or directory"],
            ),
            ("not a directory", ["not a directory", "not directory"]),
            ("is a directory", ["is a directory", "is directory"]),
            ("permission denied", ["permission denied", "access denied"]),
            ("already exists", ["already exists", "file exists", "directory already exists"]),
            ("directory not empty", ["directory not empty", "not empty"]),
            ("connection refused", ["connection refused", "server not running"]),
            ("timeout", ["timeout", "timed out"]),
            ("failed to stat", ["failed to stat", "stat failed"]),
        ]

        for _error_type, patterns in error_type_patterns:
            playback_match = any(p in playback_lower for p in patterns)
            record_match = any(p in record_lower for p in patterns)
            if playback_match and record_match:
                return True

        return False

    async def _play_vikingdb_operation(self, record: IORecord) -> PlaybackResult:
        """Play a single VikingDB operation."""
        result = PlaybackResult(record=record)
        start_time = time.time()

        try:
            operation = record.operation
            request = record.request

            args = request.get("args", [])
            kwargs = request.get("kwargs", {})

            if operation == "insert":
                if args:
                    payload = args[-1]
                else:
                    payload = kwargs.get("data", request.get("data", {}))
                await self._vector_store.upsert(payload)
            elif operation == "update":
                if len(args) >= 3:
                    record_id = args[-2]
                    payload = args[-1]
                elif len(args) == 2:
                    record_id = args[0]
                    payload = args[1]
                else:
                    record_id = kwargs.get("id", request.get("id"))
                    payload = kwargs.get("data", request.get("data", {}))
                existing = await self._vector_store.get([record_id])
                if existing:
                    merged = {**existing[0], **payload, "id": record_id}
                    await self._vector_store.upsert(merged)
            elif operation == "upsert":
                if args:
                    payload = args[-1]
                else:
                    payload = kwargs.get("data", request.get("data", {}))
                await self._vector_store.upsert(payload)
            elif operation == "delete":
                if args:
                    ids = args[-1]
                else:
                    ids = kwargs.get("ids", request.get("ids", []))
                await self._vector_store.delete(ids)
            elif operation == "get":
                if args:
                    ids = args[-1]
                else:
                    ids = kwargs.get("ids", request.get("ids", []))
                await self._vector_store.get(ids)
            elif operation == "exists":
                if len(args) >= 2:
                    record_id = args[-1]
                elif len(args) == 1:
                    record_id = args[0]
                else:
                    record_id = kwargs.get("id", request.get("id"))
                await self._vector_store.exists(record_id)
            elif operation == "search":
                if len(args) >= 4:
                    query_vector = args[1]
                    limit = args[2]
                    where = args[3]
                elif args:
                    query_vector = args[0]
                    limit = kwargs.get("top_k", kwargs.get("limit", 10))
                    where = kwargs.get("filter")
                else:
                    query_vector = kwargs.get("vector", kwargs.get("query_vector"))
                    limit = kwargs.get("top_k", kwargs.get("limit", request.get("top_k", 10)))
                    where = kwargs.get("filter", request.get("filter"))
                await self._vector_store.search(
                    query_vector=query_vector, filter=where, limit=limit
                )
            elif operation == "filter":
                if len(args) >= 4:
                    where = args[1]
                    limit = args[2]
                    offset = args[3]
                elif args:
                    where = args[0]
                    limit = kwargs.get("limit", 100)
                    offset = kwargs.get("offset", 0)
                else:
                    where = kwargs.get("filter", request.get("filter", {}))
                    limit = kwargs.get("limit", request.get("limit", 100))
                    offset = kwargs.get("offset", request.get("offset", 0))
                await self._vector_store.filter(filter=where, limit=limit, offset=offset)
            elif operation == "create_collection":
                await self._vector_store.create_collection(*args, **kwargs)
            elif operation == "drop_collection":
                await self._vector_store.drop_collection()
            elif operation == "collection_exists":
                await self._vector_store.collection_exists()
            else:
                raise ValueError(f"Unknown VikingDB operation: {operation}")

            result.playback_latency_ms = (time.time() - start_time) * 1000
            result.playback_success = True

        except Exception as e:
            result.playback_latency_ms = (time.time() - start_time) * 1000
            playback_error = str(e)

            if record.error and self._errors_match(playback_error, record.error):
                result.playback_success = True
                result.playback_error = f"Matched original error: {playback_error}"
            else:
                result.playback_success = False
                result.playback_error = playback_error
                logger.error(f"[IOPlayback] VikingDB {operation} failed: {e}")

        return result

    async def play_record(self, record: IORecord) -> PlaybackResult:
        """Play a single record."""
        if record.io_type == IOType.FS.value:
            if not self.enable_fs:
                return PlaybackResult(record=record, playback_success=True)
            return await self._play_fs_operation(record)
        else:
            if not self.enable_vikingdb:
                return PlaybackResult(record=record, playback_success=True)
            return await self._play_vikingdb_operation(record)

    async def play(
        self,
        record_file: str,
        limit: Optional[int] = None,
        offset: int = 0,
        io_type: Optional[str] = None,
        operation: Optional[str] = None,
    ) -> PlaybackStats:
        """
        Play all records from a record file.

        Args:
            record_file: Path to the record JSONL file
            limit: Maximum number of records to play
            offset: Number of records to skip
            io_type: Filter by IO type (fs or vikingdb)
            operation: Filter by operation name

        Returns:
            PlaybackStats with playback results
        """

        need_fs = self.enable_fs and (io_type is None or io_type == "fs")
        need_vikingdb = self.enable_vikingdb and (io_type is None or io_type == "vikingdb")

        if need_fs or need_vikingdb:
            self._init_backends()

        records = []
        with open(record_file, "r", encoding="utf-8") as f:
            for line in f:
                line = line.strip()
                if line:
                    records.append(IORecord.from_dict(json.loads(line)))

        filtered_records = []
        for r in records:
            if io_type and r.io_type != io_type:
                continue
            if operation and r.operation != operation:
                continue
            if r.io_type == IOType.FS.value and not self.enable_fs:
                continue
            if r.io_type == IOType.VIKINGDB.value and not self.enable_vikingdb:
                continue
            filtered_records.append(r)

        records = filtered_records[offset:]
        if limit:
            records = records[:limit]

        stats = PlaybackStats(total_records=len(records))
        logger.info(f"[IOPlayback] Playing {len(records)} records from {record_file}")

        for i, record in enumerate(records):
            result = await self.play_record(record)

            stats.total_original_latency_ms += record.latency_ms
            stats.total_playback_latency_ms += result.playback_latency_ms

            if result.playback_success:
                stats.success_count += 1
            else:
                stats.error_count += 1

            op_key = f"{record.io_type}.{record.operation}"
            if record.io_type == IOType.FS.value:
                if op_key not in stats.fs_stats:
                    stats.fs_stats[op_key] = {
                        "count": 0,
                        "total_original_latency_ms": 0.0,
                        "total_playback_latency_ms": 0.0,
                    }
                stats.fs_stats[op_key]["count"] += 1
                stats.fs_stats[op_key]["total_original_latency_ms"] += record.latency_ms
                stats.fs_stats[op_key]["total_playback_latency_ms"] += result.playback_latency_ms

                if hasattr(record, "agfs_calls") and record.agfs_calls:
                    stats.total_viking_fs_operations += 1
                    if result.playback_success:
                        stats.viking_fs_success_count += 1
                    else:
                        stats.viking_fs_error_count += 1

                    stats.total_agfs_calls += len(record.agfs_calls)
                    for call in record.agfs_calls:
                        if call.success:
                            stats.agfs_fs_success_count += 1
                        else:
                            stats.agfs_fs_error_count += 1
            else:
                if op_key not in stats.vikingdb_stats:
                    stats.vikingdb_stats[op_key] = {
                        "count": 0,
                        "total_original_latency_ms": 0.0,
                        "total_playback_latency_ms": 0.0,
                    }
                stats.vikingdb_stats[op_key]["count"] += 1
                stats.vikingdb_stats[op_key]["total_original_latency_ms"] += record.latency_ms
                stats.vikingdb_stats[op_key]["total_playback_latency_ms"] += (
                    result.playback_latency_ms
                )

            if (i + 1) % 100 == 0:
                logger.info(f"[IOPlayback] Progress: {i + 1}/{len(records)}")

            if self.fail_fast and not result.playback_success:
                logger.error(f"[IOPlayback] Stopping due to error at record {i + 1}")
                break

        logger.info(
            f"[IOPlayback] Completed: {stats.success_count}/{stats.total_records} successful"
        )
        return stats

    def play_sync(self, **kwargs) -> PlaybackStats:
        """Synchronous wrapper for play method."""
        return asyncio.run(self.play(**kwargs))