Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

24.6 KB
Raw Blame
import argparse
import csv
import hashlib
import json
import sys
import threading
import time
from concurrent.futures import ThreadPoolExecutor, as_completed
from datetime import datetime
from pathlib import Path
from typing import Any

from openviking_cli.client.sync_http import SyncHTTPClient

try:
    from benchmark.longmemeval.openviking.longmemeval_prompts import (
        get_answer_generation_prompt,
    )
except ModuleNotFoundError:
    sys.path.insert(0, str(Path(__file__).resolve().parent))
    from longmemeval_prompts import get_answer_generation_prompt


LONGMEMEVAL_TIME_FORMAT = "%Y/%m/%d (%a) %H:%M"
DEFAULT_SINGLE_SEARCH_CONTEXT_LIMIT = 10
DEFAULT_SINGLE_SEARCH_RERANK_LIMIT = 10
DEFAULT_SINGLE_SEARCH_MAX_CONTEXT_CHARS = 4000
SINGLE_SEARCH_EXCLUDED_BASENAMES = {".abstract.md", ".overview.md"}


def get_token_encoding_name(model_name: str | None = None) -> str:
    try:
        import tiktoken

        if model_name:
            candidates = [model_name]
            if "/" in model_name:
                candidates.append(model_name.rsplit("/", 1)[-1])
            for candidate in candidates:
                try:
                    return tiktoken.encoding_for_model(candidate).name
                except KeyError:
                    continue
    except Exception:
        pass

    normalized = (model_name or "").lower()
    if normalized.startswith(("gpt-4o", "gpt-5", "o1", "o3", "o4")):
        return "o200k_base"
    if normalized.startswith(("gpt-4", "gpt-3.5")):
        return "cl100k_base"
    return "approx_chars_div_4"


def count_text_tokens(text: str, model_name: str | None = None) -> int:
    encoding_name = get_token_encoding_name(model_name)
    if encoding_name == "approx_chars_div_4":
        return max(0, len(text or "") // 4)

    try:
        import tiktoken

        encoding = tiktoken.get_encoding(encoding_name)
        return len(encoding.encode(text or ""))
    except Exception:
        return max(0, len(text or "") // 4)


def parse_longmemeval_datetime(date_str: str) -> datetime | None:
    try:
        return datetime.strptime(date_str.strip(), LONGMEMEVAL_TIME_FORMAT)
    except ValueError:
        return None


def build_sample_agent_id(sample_id: str | int) -> str:
    """Return the agent_id used for one sample eval."""
    digest = hashlib.md5(str(sample_id).encode("utf-8")).hexdigest()[:12]
    return f"lm_{digest}"


def build_sample_user_id(sample_id: str | int) -> str:
    """Return the user_id used for one sample eval."""
    digest = hashlib.md5(f"user:{sample_id}".encode("utf-8")).hexdigest()[:12]
    return f"lm_user_{digest}"


def load_csv_qa(input_path: str, count: int | None = None) -> list[dict]:
    qa_list = []
    with open(input_path, "r", encoding="utf-8", newline="") as f:
        reader = csv.DictReader(f)
        for row in reader:
            qa_list.append(
                {
                    "sample_id": row.get("sample_id", row.get("question_id", "")),
                    "question": row.get("question", ""),
                    "answer": row.get("answer", ""),
                    "question_type": row.get("question_type", ""),
                    "question_time": row.get("question_time", ""),
                }
            )
    if count is not None:
        qa_list = qa_list[:count]
    return qa_list


def load_longmemeval_qa(
    input_path: str,
    sample_index: int | None = None,
    count: int | None = None,
) -> list[dict]:
    if input_path.lower().endswith(".csv"):
        return load_csv_qa(input_path, count)

    with open(input_path, "r", encoding="utf-8") as f:
        data = json.load(f)

    if sample_index is not None:
        if sample_index < 0 or sample_index >= len(data):
            raise ValueError(f"sample index {sample_index} out of range (0-{len(data) - 1})")
        samples = [data[sample_index]]
    else:
        samples = data

    qa_list = []
    for sample in samples:
        question_dt = parse_longmemeval_datetime(sample.get("question_date", ""))
        question_time = question_dt.strftime("%Y-%m-%d") if question_dt else ""
        qa_list.append(
            {
                "sample_id": sample.get("question_id", ""),
                "question": sample.get("question", ""),
                "answer": sample.get("answer", ""),
                "question_type": sample.get("question_type", ""),
                "question_time": question_time,
            }
        )

    if count is not None:
        qa_list = qa_list[:count]
    return qa_list


def _iter_search_contexts(search_result: Any) -> list[Any]:
    if search_result is None:
        return []
    if isinstance(search_result, list):
        return search_result

    contexts = []
    for attr in ("memories", "resources", "skills"):
        contexts.extend(getattr(search_result, attr, []) or [])
    if contexts:
        return contexts

    try:
        return list(search_result)
    except TypeError:
        return []


def is_single_search_excluded_uri(uri: str) -> bool:
    basename = str(uri or "").rstrip("/").rsplit("/", 1)[-1]
    return basename in SINGLE_SEARCH_EXCLUDED_BASENAMES


def select_single_search_contexts(
    search_result: Any,
    limit: int = DEFAULT_SINGLE_SEARCH_CONTEXT_LIMIT,
) -> list[dict[str, Any]]:
    selected = []
    for raw_rank, context in enumerate(_iter_search_contexts(search_result), start=1):
        if raw_rank > limit:
            break
        uri = getattr(context, "uri", "")
        if not uri or is_single_search_excluded_uri(uri):
            continue
        selected.append(
            {
                "raw_rank": raw_rank,
                "uri": uri,
                "score": getattr(context, "score", 0.0),
                "abstract": getattr(context, "abstract", ""),
            }
        )
    return selected


def build_single_search_context_prompt(
    question: str,
    question_type: str | None,
    question_time: str | None,
    contexts: list[dict[str, Any]],
) -> str:
    search_results = build_single_search_prompt_search_results(contexts)

    prompt = get_answer_generation_prompt(
        question=question,
        search_results=search_results,
        question_date=question_time or "unknown",
    )
    if question_type:
        return f"Question Type: {question_type}\n\n{prompt}"
    return prompt


def build_single_search_prompt_search_results(
    contexts: list[dict[str, Any]],
) -> list[dict[str, Any]]:
    return [
        {
            "memory": str(context.get("content", "")),
            "score": context.get("score", 0.0),
            "raw_rank": context.get("raw_rank"),
        }
        for context in contexts
    ]


def count_retrieved_memory_content_tokens(
    contexts: list[dict[str, Any]],
    model_name: str | None = None,
) -> tuple[int, int, str]:
    memory_texts = [str(context.get("content", "")) for context in contexts]
    tokenizer = get_token_encoding_name(model_name)
    return (
        sum(count_text_tokens(memory_text, model_name) for memory_text in memory_texts),
        sum(len(memory_text) for memory_text in memory_texts),
        tokenizer,
    )


def filter_contexts_by_char_budget(
    contexts: list[dict[str, Any]],
    max_chars: int = DEFAULT_SINGLE_SEARCH_MAX_CONTEXT_CHARS,
) -> tuple[list[dict[str, Any]], list[str], int]:
    if max_chars <= 0:
        return contexts, [], sum(len(str(context.get("content", ""))) for context in contexts)

    selected = []
    skipped_uris = []
    total_chars = 0
    for context in contexts:
        content = str(context.get("content", "") or "")
        content_chars = len(content)
        if total_chars + content_chars > max_chars:
            skipped_uris.append(str(context.get("uri", "")))
            continue
        selected.append(context)
        total_chars += content_chars
    return selected, skipped_uris, total_chars


def build_single_search_reranker() -> Any | None:
    from openviking.models.rerank import RerankClient
    from openviking_cli.utils.config import get_openviking_config

    rerank_config = get_openviking_config().rerank
    if not rerank_config or not rerank_config.is_available():
        return None
    return RerankClient.from_config(rerank_config)


def _single_search_rerank_document(context: dict[str, Any]) -> str:
    content = str(context.get("content", "") or "")
    if content and not content.startswith("[READ ERROR]"):
        return content
    return str(context.get("abstract", "") or "")


def rerank_single_search_contexts(
    question: str,
    contexts: list[dict[str, Any]],
    reranker: Any | None,
    limit: int = DEFAULT_SINGLE_SEARCH_RERANK_LIMIT,
) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
    if not reranker or len(contexts) <= 1:
        return contexts, []

    documents = [_single_search_rerank_document(context) for context in contexts]
    try:
        scores = reranker.rerank_batch(question, documents)
    except Exception:
        return contexts, []

    if not scores or len(scores) != len(contexts):
        return contexts, []

    ranked_contexts = []
    rerank_scores = []
    for context, score in zip(contexts, scores, strict=True):
        if not isinstance(score, (int, float)):
            return contexts, []
        next_context = dict(context)
        next_context["rerank_score"] = float(score)
        ranked_contexts.append(next_context)
        rerank_scores.append(
            {
                "uri": str(context.get("uri", "")),
                "score": float(score),
                "raw_rank": context.get("raw_rank"),
            }
        )

    ranked_contexts.sort(
        key=lambda context: (
            context.get("rerank_score", 0.0),
            -int(context.get("raw_rank", 0) or 0),
        ),
        reverse=True,
    )
    selected_contexts = ranked_contexts[: max(1, limit)]
    return selected_contexts, rerank_scores


def build_single_search_vlm() -> Any:
    from openviking.models.vlm import VLMFactory
    from openviking_cli.utils.config import get_openviking_config

    vlm_config = get_openviking_config().vlm
    return VLMFactory.create(vlm_config._build_vlm_config_dict())


def _response_text(response: Any) -> str:
    if hasattr(response, "content"):
        return str(response.content or "").strip()
    return str(response).strip()


def _token_usage_from_vlm(vlm: Any, response: Any = None) -> dict[str, int]:
    usage = getattr(response, "usage", None)
    if isinstance(usage, dict) and usage:
        return {
            "prompt_tokens": int(usage.get("prompt_tokens", 0) or 0),
            "completion_tokens": int(usage.get("completion_tokens", 0) or 0),
            "total_tokens": int(usage.get("total_tokens", 0) or 0),
        }
    if usage:
        prompt_tokens = int(getattr(usage, "prompt_tokens", 0) or 0)
        completion_tokens = int(getattr(usage, "completion_tokens", 0) or 0)
        return {
            "prompt_tokens": prompt_tokens,
            "completion_tokens": completion_tokens,
            "total_tokens": int(
                getattr(usage, "total_tokens", prompt_tokens + completion_tokens) or 0
            ),
        }

    if hasattr(vlm, "get_token_usage_summary"):
        summary = vlm.get_token_usage_summary()
        return {
            "prompt_tokens": int(summary.get("total_prompt_tokens", 0) or 0),
            "completion_tokens": int(summary.get("total_completion_tokens", 0) or 0),
            "total_tokens": int(summary.get("total_tokens", 0) or 0),
        }
    if hasattr(vlm, "get_token_usage"):
        usage_snapshot = vlm.get_token_usage()
        total_usage = usage_snapshot.get("total_usage", {}) if usage_snapshot else {}
        return {
            "prompt_tokens": int(total_usage.get("prompt_tokens", 0) or 0),
            "completion_tokens": int(total_usage.get("completion_tokens", 0) or 0),
            "total_tokens": int(total_usage.get("total_tokens", 0) or 0),
        }

    return {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0}


def run_single_search_context_answer(
    question: str,
    question_type: str | None = None,
    question_time: str | None = None,
    sender_id: str | None = None,
    session_id: str | None = None,
    openviking_url: str | None = None,
    timeout: int = 300,
    single_search_context_limit: int = DEFAULT_SINGLE_SEARCH_CONTEXT_LIMIT,
    single_search_rerank_limit: int = DEFAULT_SINGLE_SEARCH_RERANK_LIMIT,
    single_search_max_context_chars: int = DEFAULT_SINGLE_SEARCH_MAX_CONTEXT_CHARS,
    debug_print_model_input: bool = False,
) -> tuple[str, dict, float, int, list, list, str]:
    start_time = time.time()
    client = SyncHTTPClient(
        url=openviking_url,
        user=sender_id,
        timeout=timeout,
    )
    try:
        client.initialize()
        target_uri = f"viking://user/{sender_id or 'default'}/memories"
        search_result = client.find(
            question,
            target_uri=target_uri,
            limit=single_search_context_limit,
        )
        contexts = select_single_search_contexts(
            search_result,
            limit=single_search_context_limit,
        )

        retrieved_uris = [context["uri"] for context in contexts]

        for context in contexts:
            uri = context["uri"]
            try:
                context["content"] = client.read(
                    uri,
                    offset=0,
                    limit=-1,
                )
            except Exception as exc:
                context["content"] = f"[READ ERROR] {exc}"

        reranker = build_single_search_reranker()
        rerank_enabled = reranker is not None
        rerank_limit = single_search_rerank_limit
        contexts, rerank_scores = rerank_single_search_contexts(
            question=question,
            contexts=contexts,
            reranker=reranker,
            limit=rerank_limit,
        )
        contexts, skipped_context_uris, context_chars = filter_contexts_by_char_budget(
            contexts,
            max_chars=single_search_max_context_chars,
        )
        context_uris = [context["uri"] for context in contexts]

        prompt = build_single_search_context_prompt(
            question=question,
            question_type=question_type,
            question_time=question_time,
            contexts=contexts,
        )
        vlm = build_single_search_vlm()
        memory_prompt_tokens, memory_chars, memory_tokenizer = (
            count_retrieved_memory_content_tokens(
                contexts,
                model_name=getattr(vlm, "model", None),
            )
        )
        if debug_print_model_input:
            print("\n===== SINGLE SEARCH MODEL INPUT BEGIN =====", flush=True)
            print(prompt, flush=True)
            print("===== SINGLE SEARCH MODEL INPUT END =====", flush=True)
            print("===== SINGLE SEARCH DEBUG =====", flush=True)
            print(f"context_count: {len(context_uris)}", flush=True)
            print(f"memory_content_tokens: {memory_prompt_tokens}", flush=True)
            print(f"memory_content_chars: {memory_chars}", flush=True)
            print(f"memory_tokenizer: {memory_tokenizer}", flush=True)
            print(f"max_context_chars: {single_search_max_context_chars}", flush=True)
            print(f"context_chars: {context_chars}", flush=True)
            print(f"rerank_enabled: {rerank_enabled}", flush=True)
            if rerank_enabled:
                print(f"rerank_limit: {rerank_limit}", flush=True)
            print("retrieved_uris:", flush=True)
            for uri in retrieved_uris:
                print(f"  {uri}", flush=True)
            print("context_uris:", flush=True)
            for uri in context_uris:
                print(f"  {uri}", flush=True)
            if skipped_context_uris:
                print("skipped_context_uris_by_char_limit:", flush=True)
                for uri in skipped_context_uris:
                    print(f"  {uri}", flush=True)
            if rerank_scores:
                print("rerank_scores:", flush=True)
                for item in rerank_scores:
                    print(
                        f"  score={item['score']} raw_rank={item.get('raw_rank')} uri={item['uri']}",
                        flush=True,
                    )
            print("===== SINGLE SEARCH DEBUG END =====", flush=True)
        raw_response = vlm.get_completion(prompt)
        response = _response_text(raw_response)
        token_usage = _token_usage_from_vlm(vlm, raw_response)
        token_usage["memory_prompt_tokens"] = memory_prompt_tokens
        token_usage["memory_chars"] = memory_chars
        token_usage["memory_tokenizer"] = memory_tokenizer
        if debug_print_model_input:
            print("===== SINGLE SEARCH TOKEN USAGE =====", flush=True)
            print(json.dumps(token_usage, ensure_ascii=False, indent=2), flush=True)
            print("===== SINGLE SEARCH TOKEN USAGE END =====", flush=True)
        elapsed = time.time() - start_time
        return (
            response,
            token_usage,
            elapsed,
            1,
            ["single_search", "read", "context_answer"],
            [
                {
                    "iteration": 1,
                    "retrieved_uris": retrieved_uris,
                    "context_uris": context_uris,
                    "rerank_enabled": rerank_enabled,
                    "rerank_limit": rerank_limit if rerank_enabled else 0,
                    "rerank_scores": rerank_scores,
                    "max_context_chars": single_search_max_context_chars,
                    "context_chars": context_chars,
                    "skipped_context_uris_by_char_limit": skipped_context_uris,
                }
            ],
            prompt if debug_print_model_input else "",
        )
    except Exception as exc:
        elapsed = time.time() - start_time
        return (
            f"[SINGLE SEARCH ERROR] {exc}",
            {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0},
            elapsed,
            1,
            ["single_search", "read", "context_answer"],
            [
                {
                    "iteration": 1,
                    "retrieved_uris": [],
                    "context_uris": [],
                    "rerank_enabled": False,
                    "rerank_limit": 0,
                    "rerank_scores": [],
                }
            ],
            "",
        )
    finally:
        try:
            client.close()
        except Exception:
            pass


def main():
    parser = argparse.ArgumentParser(description="VikingBot LongMemEval evaluation script")
    parser.add_argument(
        "input",
        nargs="?",
        default="data/longmemeval_s_cleaned.json",
        help="Path to LongMemEval JSON file",
    )
    parser.add_argument(
        "--output",
        default="./result/longmemeval_qa_result.csv",
        help="Path to output csv file",
    )
    parser.add_argument(
        "--sample",
        type=int,
        default=None,
        help="LongMemEval sample index (0-based), default all samples",
    )
    parser.add_argument(
        "--count", type=int, default=None, help="Number of QA questions to run, default all"
    )
    parser.add_argument(
        "--threads", type=int, default=5, help="Number of concurrent threads, default: 5"
    )
    parser.add_argument(
        "--timeout",
        type=int,
        default=300,
        help="Per-question OpenViking client timeout in seconds, default: 300",
    )
    parser.add_argument(
        "--openviking-url",
        default=None,
        help="OpenViking server URL, e.g. http://127.0.0.1:1934. Defaults to ovcli.conf.",
    )
    parser.add_argument(
        "--answer-mode",
        choices=["single-search-context"],
        default="single-search-context",
        help=argparse.SUPPRESS,
    )
    parser.add_argument(
        "--debug-print-model-input",
        action="store_true",
        help="Print the full model input prompt and retrieved memory token stats.",
    )
    parser.add_argument(
        "--single-search-context-limit",
        type=int,
        default=DEFAULT_SINGLE_SEARCH_CONTEXT_LIMIT,
        help=(
            "Number of memory files to read into the single-search-context prompt, "
            f"default: {DEFAULT_SINGLE_SEARCH_CONTEXT_LIMIT}"
        ),
    )
    parser.add_argument(
        "--single-search-rerank-limit",
        type=int,
        default=DEFAULT_SINGLE_SEARCH_RERANK_LIMIT,
        help=(
            "Number of reranked memory files to keep in the single-search-context prompt, "
            f"default: {DEFAULT_SINGLE_SEARCH_RERANK_LIMIT}"
        ),
    )
    parser.add_argument(
        "--single-search-max-context-chars",
        type=int,
        default=DEFAULT_SINGLE_SEARCH_MAX_CONTEXT_CHARS,
        help=(
            "Maximum total memory content characters to concatenate into the answer prompt. "
            "Files that would exceed the budget are skipped whole; <=0 disables the budget. "
            f"default: {DEFAULT_SINGLE_SEARCH_MAX_CONTEXT_CHARS}"
        ),
    )
    args = parser.parse_args()

    output_path = Path(args.output)
    output_path.parent.mkdir(parents=True, exist_ok=True)
    qa_list = load_longmemeval_qa(args.input, args.sample, args.count)
    total = len(qa_list)
    print(f"Loaded {total} QA questions")

    fieldnames = [
        "sample_id",
        "question",
        "answer",
        "question_type",
        "question_time",
        "response",
        "model_input_prompt",
        "token_usage",
        "time_cost",
        "iteration",
        "tools_used_names",
        "retrieved_uris_by_iteration",
        "result",
    ]
    write_lock = threading.Lock()
    with open(output_path, "w", encoding="utf-8", newline="") as f:
        writer = csv.DictWriter(f, fieldnames=fieldnames)
        writer.writeheader()
        f.flush()

        processed_count = 0
        print(
            f"Starting evaluation with {args.threads} concurrent threads, {total} questions to process"
        )

        def process_qa(qa_item, idx, total_count):
            nonlocal processed_count
            question = qa_item["question"]
            answer = qa_item["answer"]
            sample_id = qa_item["sample_id"]
            question_time = qa_item.get("question_time")
            print(f"Processing {idx}/{total_count}: {question[:60]}...")
            if question_time:
                print(f"  [time context: {question_time}]")

            sender_id = build_sample_user_id(sample_id)
            session_id = build_sample_agent_id(sample_id)
            (
                response,
                token_usage,
                time_cost,
                iteration,
                tools_used_names,
                retrieved_uris_by_iteration,
                model_input_prompt,
            ) = run_single_search_context_answer(
                question,
                qa_item.get("question_type"),
                question_time,
                sender_id=sender_id,
                session_id=session_id,
                openviking_url=args.openviking_url,
                timeout=args.timeout,
                single_search_context_limit=args.single_search_context_limit,
                single_search_rerank_limit=args.single_search_rerank_limit,
                single_search_max_context_chars=args.single_search_max_context_chars,
                debug_print_model_input=args.debug_print_model_input,
            )
            row = {
                "sample_id": sample_id,
                "question": question,
                "answer": answer,
                "question_type": qa_item.get("question_type", ""),
                "question_time": question_time or "",
                "response": response,
                "model_input_prompt": model_input_prompt,
                "token_usage": json.dumps(token_usage, ensure_ascii=False),
                "time_cost": round(time_cost, 2),
                "iteration": iteration,
                "tools_used_names": json.dumps(tools_used_names, ensure_ascii=False),
                "retrieved_uris_by_iteration": json.dumps(
                    retrieved_uris_by_iteration, ensure_ascii=False
                )
                if args.debug_print_model_input
                else "",
                "result": "",
            }

            with write_lock:
                writer.writerow(row)
                f.flush()
                processed_count += 1
                print(f"Completed {processed_count}/{total}, time cost: {round(time_cost, 2)}s")
            return True

        with ThreadPoolExecutor(max_workers=args.threads) as executor:
            futures = []
            for idx, qa_item in enumerate(qa_list, 1):
                futures.append(executor.submit(process_qa, qa_item, idx, total))

            for future in as_completed(futures):
                try:
                    future.result()
                except Exception as e:
                    print(f"Error processing QA item: {str(e)}")

    print(f"Evaluation completed, results saved to {output_path}")


if __name__ == "__main__":
    main()