Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

13.9 KB
Raw Blame
#!/usr/bin/env python3
from __future__ import annotations

import argparse
import asyncio
import inspect
import json
import os
import time
from concurrent.futures import ProcessPoolExecutor
from pathlib import Path
from typing import Any

from benchmark.memory_organization.autonomous_cases import AutonomousCase, build_cases
from benchmark.memory_organization.autonomous_grader import (
    autonomous_metrics,
    grade_autonomous_result,
)
from benchmark.memory_organization.run_ab import _call_count
from openviking.session.memory.session_extract_context_provider import (
    SessionExtractContextProvider,
)

ROOT = Path(__file__).resolve().parent
Job = tuple[int, str, str, int]


class AutonomousProvider(SessionExtractContextProvider):
    def __init__(self, case: AutonomousCase, *, output_language: str = "") -> None:
        super().__init__(messages=list(case.messages), viking_fs=object())
        self.case = case
        self._benchmark_output_language = output_language.strip()
        self._read_file_contents = dict(case.initial_files)

    def get_output_language(self) -> str:
        return self._benchmark_output_language or super().get_output_language()

    async def prefetch(self) -> list[dict[str, Any]]:
        from openviking.session.memory.tools import (
            add_tool_call_pair_to_messages,
            memory_maintenance_notice,
        )

        # Keep the production conversation wrapper and add only the read results
        # that the real prefetch/search path would have selected. No benchmark
        # maintenance instruction or expected organization is shown to the model.
        messages: list[dict[str, Any]] = [self._build_conversation_message()]
        for call_id, (uri, memory_file) in enumerate(self.read_file_contents.items(), start=1):
            page_id = self._extract_context.page_id_map.get_page_id(uri)
            result = memory_file.to_metadata()
            result["page_id"] = page_id
            maintenance_notice = memory_maintenance_notice(
                memory_file.plain_content() or "",
                review_after_tokens=self.case.maintenance_review_tokens,
            )
            if maintenance_notice is not None:
                result["memory_maintenance_notice"] = maintenance_notice
            add_tool_call_pair_to_messages(
                messages,
                call_id=call_id,
                tool_name="read",
                params={"uri": uri},
                result=result,
            )
        return messages

    async def execute_tool(self, tool_call: Any) -> dict[str, str]:
        from openviking.session.memory.tools import memory_maintenance_notice

        uri = str(tool_call.arguments.get("uri", ""))
        memory_file = self.read_file_contents.get(uri)
        if memory_file is None:
            return {"error": f"File not found: {uri}"}
        result = memory_file.to_metadata()
        result["page_id"] = self.get_extract_context().page_id_map.get_page_id(uri)
        maintenance_notice = memory_maintenance_notice(
            memory_file.plain_content() or "",
            review_after_tokens=self.case.maintenance_review_tokens,
        )
        if maintenance_notice is not None:
            result["memory_maintenance_notice"] = maintenance_notice
        return result


class AutonomousIsolationHandler:
    def get_read_scope(self) -> Any:
        from openviking.session.memory.memory_isolation_handler import RoleScope

        return RoleScope(user_ids=[], peer_ids=[])

    def fill_identity_fields(self, item_dict: dict[str, Any], **kwargs: Any) -> None:
        del kwargs
        item_dict.pop("peer_id", None)

    def calculate_memory_uris(
        self,
        *,
        memory_type_schema: Any,
        operation: Any,
        extract_context: Any,
    ) -> list[str]:
        from openviking.session.memory.utils.template_utils import TemplateUtils

        path = f"{memory_type_schema.directory}/{memory_type_schema.filename_template}"
        fields = {"user_space": "default", **operation.memory_fields}
        return [TemplateUtils.render(path, fields, extract_context)]


async def materialize(
    case: AutonomousCase, operations: Any
) -> tuple[dict[str, tuple[str, str]], dict[str, str]]:
    from openviking.session.memory.merge_op import FieldType, PatchOp

    files = {
        uri: (str(memory_file.memory_type), memory_file.content)
        for uri, memory_file in case.initial_files.items()
    }
    for memory_file in operations.delete_file_contents:
        if memory_file.uri:
            files.pop(memory_file.uri, None)
    patch_op = PatchOp(FieldType.STRING)
    for operation in operations.upsert_operations:
        if not operation.uris:
            continue
        uri = operation.uris[0]
        value = operation.memory_fields.get("content")
        if operation.old_memory_file_content is not None:
            value = await patch_op.apply(operation.old_memory_file_content.content, value)
        files[uri] = (operation.memory_type, str(value or ""))
    return files, dict(operations.delete_replacements)


async def run_one(
    case: AutonomousCase,
    protocol: str,
    repeat_index: int,
    config: Any,
    *,
    output_language: str = "",
) -> dict:
    from openviking.session.memory.extract_loop import ExtractLoop

    provider = AutonomousProvider(case, output_language=output_language)
    config.memory.extraction_output_format = protocol
    config.memory.link_enabled = False
    vlm = config.vlm.get_vlm_instance()
    vlm.reset_token_usage()
    raw_responses: list[dict[str, Any]] = []
    original_get_completion_async = vlm.get_completion_async

    async def recording_get_completion_async(*args: Any, **kwargs: Any) -> Any:
        response = await original_get_completion_async(*args, **kwargs)
        if isinstance(response, str):
            raw_responses.append({"content": response, "tool_calls": []})
        else:
            raw_responses.append(
                {
                    "content": response.content or "",
                    "tool_calls": [
                        {
                            "id": tool_call.id,
                            "name": tool_call.name,
                            "arguments": tool_call.arguments,
                        }
                        for tool_call in response.tool_calls
                    ],
                }
            )
        return response

    vlm.get_completion_async = recording_get_completion_async
    loop = ExtractLoop(
        vlm=vlm,
        viking_fs=object(),
        context_provider=provider,
        isolation_handler=AutonomousIsolationHandler(),
        max_iterations=3,
        thinking=False,
    )
    started = time.perf_counter()
    try:
        try:
            operations, _ = await loop.run()
            files, replacements = await materialize(case, operations)
            grade = grade_autonomous_result(case, files, replacements)
            decision_metrics = autonomous_metrics(case, files, replacements, grade)
            terminal_error = "; ".join(operations.errors)
        except Exception as exc:
            files = {
                uri: (str(memory_file.memory_type), memory_file.content)
                for uri, memory_file in case.initial_files.items()
            }
            replacements = {}
            grade = grade_autonomous_result(case, files, replacements)
            decision_metrics = autonomous_metrics(case, files, replacements, grade)
            terminal_error = f"{type(exc).__name__}: {exc}"
    finally:
        vlm.get_completion_async = original_get_completion_async
    usage_report = vlm.get_token_usage()
    usage = usage_report.get("total_usage", {})
    calls = _call_count(usage_report)
    return {
        "case_id": case.case_id,
        "category": case.category,
        "protocol": protocol,
        "repeat_index": repeat_index,
        "model": {
            "provider": vlm.provider,
            "name": vlm.model,
            "temperature": vlm.temperature,
            "thinking": vlm.thinking,
            "max_tokens": vlm.max_tokens,
        },
        "grade": grade.to_dict(),
        "autonomous_metrics": decision_metrics,
        "actual_files": {
            uri: {"memory_type": memory_type, "content": content}
            for uri, (memory_type, content) in files.items()
        },
        "actual_replacements": replacements,
        "raw_responses": raw_responses,
        "calls": calls,
        "retries": max(0, calls - 1),
        "prompt_tokens": int(usage.get("prompt_tokens", 0)),
        "completion_tokens": int(usage.get("completion_tokens", 0)),
        "total_tokens": int(usage.get("total_tokens", 0)),
        "duration_seconds": round(time.perf_counter() - started, 3),
        "terminal_error": terminal_error,
    }


def build_jobs(cases: list[AutonomousCase], repeat: int) -> list[Job]:
    """Build counterbalanced JSON/Python jobs with stable unique indices."""
    protocols = ("json", "python")
    jobs: list[Job] = []
    for repeat_index in range(repeat):
        for case_index, case in enumerate(cases):
            ordered = (
                protocols if (repeat_index + case_index) % 2 == 0 else tuple(reversed(protocols))
            )
            for protocol in ordered:
                jobs.append((len(jobs), case.case_id, protocol, repeat_index))
    return jobs


def _run_job_in_process(
    config_path: str,
    case_id: str,
    protocol: str,
    repeat_index: int,
    output_language: str,
) -> dict[str, Any]:
    """Run one job in a process-isolated config/VLM environment."""
    from openviking_cli.utils.config.open_viking_config import OpenVikingConfigSingleton

    config = OpenVikingConfigSingleton.initialize(config_path=config_path)
    case = next(case for case in build_cases() if case.case_id == case_id)

    async def execute() -> dict[str, Any]:
        try:
            return await run_one(
                case,
                protocol,
                repeat_index,
                config,
                output_language=output_language,
            )
        finally:
            await _close_vlm_async_clients(config.vlm._vlm_instance)

    return asyncio.run(execute())


async def _close_vlm_async_clients(vlm: Any) -> None:
    """Close loop-bound clients before a process worker's event loop exits."""
    if vlm is None:
        return
    cache = getattr(vlm, "_async_client_cache", None)
    if cache is not None:
        for client in cache.pop_all():
            close = getattr(client, "close", None) or getattr(client, "aclose", None)
            if close is None:
                continue
            result = close()
            if inspect.isawaitable(result):
                await result
    for child_name in ("primary", "backup", "vlm_instances", "_vlm_instances"):
        children = getattr(vlm, child_name, None)
        if children is None:
            continue
        if not isinstance(children, (list, tuple)):
            children = [children]
        for child in children:
            await _close_vlm_async_clients(child)


def _append_result(output: Path, row: dict[str, Any], done: int, total: int) -> None:
    with output.open("a") as fp:
        fp.write(json.dumps(row, ensure_ascii=False) + "\n")
    metrics = row["autonomous_metrics"]
    action = "PASS" if metrics["organization_action_success"] else "FAIL"
    integrity = "PASS" if metrics["information_integrity"] else "FAIL"
    print(
        f"[{done}/{total}] {row['protocol']} {row['case_id']}: "
        f"action={action} integrity={integrity}",
        flush=True,
    )


async def main_async(args: argparse.Namespace) -> None:
    from openviking_cli.utils.config.open_viking_config import OpenVikingConfigSingleton

    config = OpenVikingConfigSingleton.initialize(config_path=args.config)
    cases = build_cases()
    selected = set(args.case or [])
    if selected:
        cases = [case for case in cases if case.case_id in selected]
    output = Path(args.output)
    output.parent.mkdir(parents=True, exist_ok=True)
    output.write_text("")
    jobs = build_jobs(cases, args.repeat)
    total = len(jobs)
    if args.parallel == 1:
        for done, (_index, case_id, protocol, repeat_index) in enumerate(jobs, start=1):
            case = next(case for case in cases if case.case_id == case_id)
            row = await run_one(
                case,
                protocol,
                repeat_index,
                config,
                output_language=args.output_language,
            )
            _append_result(output, row, done, total)
        return

    loop = asyncio.get_running_loop()
    with ProcessPoolExecutor(max_workers=args.parallel) as executor:
        futures = [
            loop.run_in_executor(
                executor,
                _run_job_in_process,
                args.config,
                case_id,
                protocol,
                repeat_index,
                args.output_language,
            )
            for _index, case_id, protocol, repeat_index in jobs
        ]
        for done, future in enumerate(asyncio.as_completed(futures), start=1):
            _append_result(output, await future, done, total)


def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
    parser = argparse.ArgumentParser()
    parser.add_argument("--config", default=os.path.expanduser("~/.openviking/ov.conf"))
    parser.add_argument("--case", action="append")
    parser.add_argument("--repeat", type=int, default=1)
    parser.add_argument(
        "--parallel",
        type=int,
        default=1,
        help="Maximum process-isolated jobs to run concurrently (default: 1)",
    )
    parser.add_argument(
        "--output-language",
        default="",
        help="Optional production output-language override for this benchmark run",
    )
    parser.add_argument("--output", default=str(ROOT / "result" / "autonomous_results.jsonl"))
    args = parser.parse_args(argv)
    if args.repeat < 1:
        parser.error("--repeat must be >= 1")
    if args.parallel < 1:
        parser.error("--parallel must be >= 1")
    return args


if __name__ == "__main__":
    asyncio.run(main_async(parse_args()))