Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

11.0 KB
Raw Blame
#!/usr/bin/env python3
from __future__ import annotations

import argparse
import asyncio
import json
import os
import time
from pathlib import Path
from typing import Any

from benchmark.memory_organization.grader import grade_result
from benchmark.memory_organization.models import OrganizationCase, load_cases, topic_from_uri

ROOT = Path(__file__).resolve().parent
DEFAULT_CASES = ROOT / "cases" / "organization_cases.json"


class FixtureProvider:
    def __init__(self, case: OrganizationCase, schema: Any, memory_files: dict[str, Any]) -> None:
        from openviking.session.memory.memory_updater import ExtractContext
        from openviking.session.memory.page_id_map import PageIdMap

        self.case = case
        self.schema = schema
        self.read_file_contents = memory_files
        self._extract_context = ExtractContext([])
        self._extract_context.page_id_map = PageIdMap()

    def instruction(self) -> str:
        return (
            "Maintain the existing memory collection according to its schema. Decide whether any "
            "files should remain unchanged, be merged, or be split; the number and names of final "
            "files are your decision. Preserve every complete [Fxx] fact line exactly once and do "
            "not invent facts. Delete superseded source files. When multiple files merge into one "
            "unambiguous successor, use that survivor as the replacement so links can be inherited; "
            "a source split across multiple successors has no replacement. Choose concise lowercase "
            "snake_case topics. An empty operation plan is valid when the collection already satisfies "
            "the schema. Output only the required final operation protocol."
        )

    def get_memory_schemas(self, ctx: Any) -> list[Any]:
        del ctx
        return [self.schema]

    def get_output_language(self) -> str:
        return "en"

    def get_tools(self) -> list[str]:
        return []

    def get_extract_context(self) -> Any:
        return self._extract_context

    async def prefetch(self) -> list[dict[str, Any]]:
        from openviking.session.memory.tools import add_tool_call_pair_to_messages

        messages: list[dict[str, Any]] = [
            {
                "role": "user",
                "content": (
                    f"Memory maintenance case {self.case.case_id}. Review all files below as one "
                    "atomic collection. Fact markers [Fxx] are immutable grading anchors."
                ),
            }
        ]
        for call_id, (uri, memory_file) in enumerate(self.read_file_contents.items(), start=1):
            page_id = self._extract_context.page_id_map.get_page_id(uri)
            result = memory_file.to_metadata()
            result["page_id"] = page_id
            add_tool_call_pair_to_messages(
                messages,
                call_id=call_id,
                tool_name="read",
                params={"uri": uri},
                result=result,
            )
        return messages

    async def execute_tool(self, tool_call: Any) -> dict[str, str]:
        return {"error": f"File not found: {tool_call.arguments.get('uri', '')}"}


class FixtureIsolationHandler:
    def get_read_scope(self) -> Any:
        from openviking.session.memory.memory_isolation_handler import RoleScope

        return RoleScope(user_ids=[], peer_ids=[])

    def fill_identity_fields(self, item_dict: dict[str, Any], **kwargs: Any) -> None:
        del kwargs
        item_dict.pop("peer_id", None)

    def calculate_memory_uris(
        self,
        *,
        memory_type_schema: Any,
        operation: Any,
        extract_context: Any,
    ) -> list[str]:
        del extract_context
        topic = str(operation.memory_fields["topic"])
        return [f"{memory_type_schema.directory}/{topic}.md"]


def build_schema(case: OrganizationCase) -> Any:
    from openviking.session.memory.dataclass import MemoryField, MemoryTypeSchema
    from openviking.session.memory.merge_op import FieldType, MergeOp

    size_rule = (
        f" A file may contain at most {case.max_fact_lines_per_file} [Fxx] fact lines. "
        "If it exceeds that limit, reorganize it into coherent files that each fit the limit."
        if case.max_fact_lines_per_file is not None
        else ""
    )
    return MemoryTypeSchema(
        memory_type="records",
        description=(
            "A collection of topic-focused memory files. Each file must cover exactly one "
            "coherent topic. Merge aliases and split mixed files. Preserve [Fxx] markers."
            + size_rule
        ),
        directory="viking://user/default/memories/records",
        filename_template="{{ topic }}.md",
        peer_enabled=False,
        fields=[
            MemoryField(
                name="topic",
                field_type=FieldType.STRING,
                merge_op=MergeOp.IMMUTABLE,
                description="Canonical lowercase snake_case topic used as the filename.",
            ),
            MemoryField(
                name="content",
                field_type=FieldType.STRING,
                merge_op=MergeOp.REPLACE,
                description=(
                    "Complete content for exactly one topic. Preserve every source [Fxx] marker "
                    "verbatim and include each marker once."
                ),
            ),
        ],
    )


def build_memory_files(case: OrganizationCase, schema: Any) -> dict[str, Any]:
    from openviking.session.memory.dataclass import MemoryFile

    return {
        f"{schema.directory}/{topic}.md": MemoryFile(
            uri=f"{schema.directory}/{topic}.md",
            memory_type=schema.memory_type,
            content=content,
            extra_fields={"topic": topic},
        )
        for topic, content in case.initial_files.items()
    }


def materialize(
    case: OrganizationCase,
    operations: Any,
) -> tuple[dict[str, str], dict[str, str]]:
    files = dict(case.initial_files)
    for memory_file in operations.delete_file_contents:
        if memory_file.uri:
            files.pop(topic_from_uri(memory_file.uri), None)
    for operation in operations.upsert_operations:
        topic = str(operation.memory_fields["topic"])
        files[topic] = str(operation.memory_fields.get("content") or "")
    replacements = {
        topic_from_uri(source): topic_from_uri(target)
        for source, target in operations.delete_replacements.items()
    }
    return files, replacements


def _call_count(usage: dict[str, Any]) -> int:
    """Sum model call counts because TokenUsageTracker.total_usage omits call_count."""
    return sum(
        int(model.get("total_usage", {}).get("call_count", 0))
        for model in usage.get("usage_by_model", {}).values()
    )


async def run_one(case: OrganizationCase, protocol: str, repeat_index: int, config: Any) -> dict:
    from openviking.session.memory.extract_loop import ExtractLoop

    schema = build_schema(case)
    memory_files = build_memory_files(case, schema)
    provider = FixtureProvider(case, schema, memory_files)
    isolation = FixtureIsolationHandler()
    config.memory.extraction_output_format = protocol
    config.memory.link_enabled = False
    vlm = config.vlm.get_vlm_instance()
    vlm.reset_token_usage()
    loop = ExtractLoop(
        vlm=vlm,
        viking_fs=object(),
        context_provider=provider,
        isolation_handler=isolation,
        max_iterations=3,
        thinking=False,
    )
    started = time.perf_counter()
    terminal_error = ""
    try:
        operations, _ = await loop.run()
        files, replacements = materialize(case, operations)
        grade = grade_result(case, files, replacements)
        terminal_error = "; ".join(operations.errors)
    except Exception as exc:
        files, replacements = dict(case.initial_files), {}
        grade = grade_result(case, files, replacements)
        terminal_error = f"{type(exc).__name__}: {exc}"
    duration = time.perf_counter() - started
    usage_report = vlm.get_token_usage()
    usage = usage_report.get("total_usage", {})
    return {
        "case_id": case.case_id,
        "category": case.category,
        "protocol": protocol,
        "repeat_index": repeat_index,
        "model": {
            "provider": vlm.provider,
            "name": vlm.model,
            "temperature": vlm.temperature,
            "thinking": vlm.thinking,
            "max_tokens": vlm.max_tokens,
        },
        "grade": grade.to_dict(),
        "actual_files": files,
        "actual_replacements": replacements,
        "calls": _call_count(usage_report),
        "retries": max(0, _call_count(usage_report) - 1),
        "prompt_tokens": int(usage.get("prompt_tokens", 0)),
        "completion_tokens": int(usage.get("completion_tokens", 0)),
        "total_tokens": int(usage.get("total_tokens", 0)),
        "duration_seconds": round(duration, 3),
        "terminal_error": terminal_error,
    }


async def main_async(args: argparse.Namespace) -> None:
    from openviking_cli.utils.config.open_viking_config import OpenVikingConfigSingleton

    config = OpenVikingConfigSingleton.initialize(config_path=args.config)
    cases = load_cases(Path(args.cases))
    selected = set(args.case or [])
    if selected:
        cases = [case for case in cases if case.case_id in selected]
    protocols = tuple(dict.fromkeys(args.protocol))
    output = Path(args.output)
    output.parent.mkdir(parents=True, exist_ok=True)
    output.write_text("")
    total = len(cases) * args.repeat * len(protocols)
    done = 0
    for repeat_index in range(args.repeat):
        for case_index, case in enumerate(cases):
            # Counterbalance temporal/provider drift instead of always giving one
            # protocol the first call in every pair.
            pair_protocols = (
                protocols if (repeat_index + case_index) % 2 == 0 else tuple(reversed(protocols))
            )
            for protocol in pair_protocols:
                row = await run_one(case, protocol, repeat_index, config)
                with output.open("a") as fp:
                    fp.write(json.dumps(row, ensure_ascii=False) + "\n")
                done += 1
                status = "PASS" if row["grade"]["organization_success"] else "FAIL"
                print(f"[{done}/{total}] {protocol} {case.case_id}: {status}", flush=True)


def parse_args() -> argparse.Namespace:
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--config", default=os.path.expanduser("~/.openviking/ov.conf"))
    parser.add_argument("--cases", default=str(DEFAULT_CASES))
    parser.add_argument("--case", action="append", help="Run one case id (repeatable)")
    parser.add_argument("--protocol", action="append", choices=("json", "python"), default=[])
    parser.add_argument("--repeat", type=int, default=1)
    parser.add_argument("--output", default=str(ROOT / "result" / "ab_results.jsonl"))
    args = parser.parse_args()
    args.protocol = args.protocol or ["json", "python"]
    if args.repeat < 1:
        parser.error("--repeat must be >= 1")
    return args


if __name__ == "__main__":
    asyncio.run(main_async(parse_args()))