Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

chore(test_scripts): remove memory policy compare script (#2542)

Qin Haojie455bf1002b1b
diff --git a/test_scripts/memory_policy_compare.py b/test_scripts/memory_policy_compare.py
deleted file mode 100755
index b05271bfb..000000000
--- a/test_scripts/memory_policy_compare.py
+++ /dev/null
@@ -1,593 +0,0 @@
-#!/usr/bin/env python3
-"""
-Compare unset and explicit session memory_policy.memory_types behavior with peer enabled.
-
-Run against an existing local server:
- .venv/bin/python test_scripts/memory_policy_compare.py
-
-Start a local server from ov.conf when needed:
- .venv/bin/python test_scripts/memory_policy_compare.py --start-server
-"""
-
-from __future__ import annotations
-
-import argparse
-import json
-import subprocess
-import sys
-import tempfile
-import time
-from dataclasses import dataclass
-from pathlib import Path
-from typing import Any, Iterable, Optional
-from uuid import uuid4
-
-import httpx
-
-import openviking as ov
-
-DEFAULT_CONFIG_PATH = "ov.conf"
-DEFAULT_URL = "http://127.0.0.1:1934"
-DEFAULT_ROOT_KEY = "test"
-DEFAULT_ACCOUNT_PREFIX = "memory-policy-smoke"
-DEFAULT_POLICY_USER = "policy_default"
-FILTERED_POLICY_USER = "policy_filtered"
-PEER_ID = "policy_peer"
-DEFAULT_POLICY = {"peer": {"enabled": True}}
-FILTERED_POLICY = {
- "peer": {"enabled": True},
- "memory_types": ["profile", "preferences", "trajectories"],
-}
-FILTERED_ALLOWED_TYPES = {"profile", "preferences", "trajectories"}
-TERMINAL_TASK_STATUSES = {"completed", "failed"}
-
-
-@dataclass(frozen=True)
-class CaseSpec:
- name: str
- user_id: str
- session_id: str
- api_key: str
- memory_policy: Optional[dict[str, Any]]
- allowed_types: Optional[set[str]]
- required_any_types: set[str]
-
-
-@dataclass(frozen=True)
-class CaseResult:
- case: CaseSpec
- task: dict[str, Any]
- memory_entries: list[dict[str, Any]]
- memory_types: set[str]
- archive_diff: Optional[dict[str, Any]]
- archive_diff_types: set[str]
- errors: list[str]
-
- @property
- def ok(self) -> bool:
- return not self.errors
-
-
-def load_json_file(path: Path) -> dict[str, Any]:
- try:
- return json.loads(path.read_text(encoding="utf-8-sig"))
- except FileNotFoundError:
- return {}
-
-
-def default_url_from_config(config: dict[str, Any]) -> str:
- server = config.get("server") or {}
- host = server.get("host") or "127.0.0.1"
- port = server.get("port") or 1934
- return f"http://{host}:{port}"
-
-
-def root_key_from_config(config: dict[str, Any]) -> str:
- server = config.get("server") or {}
- return server.get("root_api_key") or DEFAULT_ROOT_KEY
-
-
-def print_json(title: str, payload: Any) -> None:
- print(f"\n== {title} ==")
- print(json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True))
-
-
-def ensure_server_ready(base_url: str) -> bool:
- try:
- response = httpx.get(f"{base_url.rstrip('/')}/health", timeout=5.0)
- response.raise_for_status()
- data = response.json()
- except Exception:
- return False
- return data.get("status") == "ok"
-
-
-def wait_server_ready(base_url: str, timeout: float) -> None:
- deadline = time.time() + timeout
- while time.time() < deadline:
- if ensure_server_ready(base_url):
- return
- time.sleep(1.0)
- raise TimeoutError(f"OpenViking server is not ready: {base_url}")
-
-
-def start_server(config_path: Path, base_url: str, startup_timeout: float) -> subprocess.Popen:
- server_bin = Path(".venv/bin/openviking-server")
- command = [str(server_bin if server_bin.exists() else "openviking-server")]
- command.extend(["--config", str(config_path)])
-
- log_path = Path(tempfile.gettempdir()) / f"openviking-memory-policy-{uuid4().hex}.log"
- log_file = log_path.open("w", encoding="utf-8")
- process = subprocess.Popen(
- command,
- stdout=log_file,
- stderr=subprocess.STDOUT,
- text=True,
- )
-
- try:
- wait_server_ready(base_url, startup_timeout)
- except Exception:
- process.terminate()
- raise RuntimeError(f"Failed to start server. Log: {log_path}") from None
-
- print(f"Started OpenViking server pid={process.pid}, log={log_path}")
- return process
-
-
-def close_client(client: Any) -> None:
- try:
- client.close()
- except Exception:
- pass
-
-
-def make_client(base_url: str, api_key: str, timeout: float):
- client = ov.SyncHTTPClient(
- url=base_url,
- api_key=api_key,
- timeout=timeout,
- profile_enabled=False,
- )
- client.initialize()
- return client
-
-
-def bootstrap_users(
- base_url: str,
- root_key: str,
- account_id: str,
- timeout: float,
-) -> dict[str, str]:
- admin_client = make_client(base_url, root_key, timeout)
- try:
- account = admin_client.admin_create_account(account_id, DEFAULT_POLICY_USER)
- filtered_user = admin_client.admin_register_user(
- account_id,
- FILTERED_POLICY_USER,
- role="user",
- )
- finally:
- close_client(admin_client)
-
- user_keys = {
- DEFAULT_POLICY_USER: account["user_key"],
- FILTERED_POLICY_USER: filtered_user["user_key"],
- }
- print_json(
- "users",
- {
- "account_id": account_id,
- "users": sorted(user_keys),
- },
- )
- return user_keys
-
-
-def build_messages() -> list[dict[str, Any]]:
- return [
- {
- "role": "user",
- "content": (
- "我是林澈,平台工程师,长期维护 OpenViking。"
- "我的稳定偏好是中文沟通、结论先行、代码改动要小而结构清楚,"
- "不要新增没有业务含义的 wrapper。2026-06-09 上午,我们决定把 "
- "PR #2236 的 session memory policy 调整成顶层 memory_types:"
- "默认不限制类型,显式策略只允许白名单里的多个类型。"
- ),
- "created_at": "2026-06-09T10:00:00+08:00",
- },
- {
- "role": "user",
- "peer_id": PEER_ID,
- "content": (
- "我是许知行,PR #2236 的 reviewer,长期负责 session memory policy "
- "和 peer memory 路由审查。我的稳定偏好是测试必须覆盖 self 和 peer,"
- "对比默认 memory_types 不设与显式多个白名单类型。2026-06-09 上午,"
- "我要求 smoke 脚本确认 peer 目录也不会生成白名单之外的 identity 或 soul。"
- ),
- "created_at": "2026-06-09T10:00:30+08:00",
- },
- {
- "role": "assistant",
- "parts": [
- {
- "type": "text",
- "text": (
- "我会先读取 memory_policy 和 session commit 路径,确认策略解析、"
- "类型校验和提取 pipeline 的分发点。"
- ),
- },
- {
- "type": "tool",
- "tool_id": "inspect_memory_policy",
- "tool_name": "rg",
- "skill_uri": "viking://user/skills/codebase_inspection",
- "tool_input": {
- "pattern": "memory_types|extract_execution_memories",
- "path": "openviking/session",
- },
- "tool_output": (
- "Found MemoryPolicy.memory_types as a top-level field. "
- "Session commit validates it against MemoryTypeRegistry, then passes "
- "the same whitelist into each memory extraction entry point. "
- "The default policy enables peer memory and leaves allowed_memory_types "
- "unset, while an explicit policy passes a whitelist into each extraction "
- "pipeline."
- ),
- "tool_status": "completed",
- "duration_ms": 22,
- "prompt_tokens": 128,
- "completion_tokens": 92,
- },
- ],
- "created_at": "2026-06-09T10:01:00+08:00",
- },
- {
- "role": "assistant",
- "peer_id": PEER_ID,
- "content": (
- "我会在同一份对话里保留你的 peer_id,并在提交后同时扫描 self memory "
- "和 peer memory,确认过滤策略对两个目标都生效。"
- ),
- "created_at": "2026-06-09T10:01:30+08:00",
- },
- {
- "role": "user",
- "content": (
- "请写一个 test_scripts 里的脚本,创建两个 user,给他们同一段对话,"
- "但设置不同 memory_policy。一个只开启 peer 但不设置 memory_types,"
- "另一个开启 peer 并设置 profile、preferences、trajectories 这几个类型,"
- "最后对比默认全量和显式白名单在 self 与 peer 两边的产物差异。"
- ),
- "created_at": "2026-06-09T10:02:00+08:00",
- },
- {
- "role": "assistant",
- "parts": [
- {
- "type": "text",
- "text": ("我会用真实 HTTP 服务执行 smoke test,并轮询 commit task 到完成。"),
- },
- {
- "type": "tool",
- "tool_id": "write_compare_script",
- "tool_name": "apply_patch",
- "skill_uri": "viking://user/skills/code_editing",
- "tool_input": {
- "file": "test_scripts/memory_policy_compare.py",
- "operation": "add",
- },
- "tool_output": (
- "Created a script that bootstraps users, creates sessions with "
- "peer-enabled policies, adds identical messages, commits, polls tasks, "
- "and compares self and peer memory files by type."
- ),
- "tool_status": "completed",
- "duration_ms": 31,
- "prompt_tokens": 180,
- "completion_tokens": 120,
- },
- ],
- "created_at": "2026-06-09T10:03:00+08:00",
- },
- {
- "role": "assistant",
- "content": (
- "脚本已完成:未设置 memory_types 的策略应尽量按当前 registry 全量抽取;"
- "显式策略只能在 self 和 peer 两边产生 profile、preferences、"
- "trajectories。"
- ),
- "created_at": "2026-06-09T10:04:00+08:00",
- },
- ]
-
-
-def wait_task(client: Any, task_id: str, timeout: float) -> dict[str, Any]:
- deadline = time.time() + timeout
- latest: dict[str, Any] = {}
- while time.time() < deadline:
- task = client.get_task(task_id) or {}
- latest = task
- status = task.get("status")
- if status in TERMINAL_TASK_STATUSES:
- return task
- time.sleep(2.0)
- raise TimeoutError(f"Task timed out: {task_id}, latest={latest}")
-
-
-def entry_uri(entry: dict[str, Any]) -> str:
- return str(entry.get("uri") or "")
-
-
-def collect_memory_entries(client: Any, user_id: str) -> list[dict[str, Any]]:
- root_uris = [
- f"viking://user/{user_id}/memories",
- f"viking://user/{user_id}/peers/{PEER_ID}/memories",
- ]
- memory_entries: list[dict[str, Any]] = []
- for root_uri in root_uris:
- try:
- entries = client.ls(root_uri, recursive=True, show_all_hidden=True) or []
- except Exception:
- continue
- memory_entries.extend(entry for entry in entries if isinstance(entry, dict))
- return memory_entries
-
-
-def is_peer_memory_uri(uri: str) -> bool:
- return f"/peers/{PEER_ID}/memories/" in uri
-
-
-def memory_type_from_uri(uri: str) -> str:
- if "/memories/trajectories/" in uri:
- return "trajectories"
- if "/memories/experiences/" in uri:
- return "experiences"
- if "/memories/preferences/" in uri:
- return "preferences"
- if "/memories/entities/" in uri:
- return "entities"
- if uri.endswith("/profile.md"):
- return "profile"
- if uri.endswith("/identity.md"):
- return "identity"
- if uri.endswith("/soul.md"):
- return "soul"
- if uri.endswith(".md"):
- return Path(uri).stem
- return ""
-
-
-def collect_memory_types(entries: Iterable[dict[str, Any]]) -> set[str]:
- memory_types = set()
- for entry in entries:
- uri = entry_uri(entry)
- if not uri.endswith(".md"):
- continue
- if Path(uri).name.startswith("."):
- continue
- memory_type = memory_type_from_uri(uri)
- if memory_type:
- memory_types.add(memory_type)
- return memory_types
-
-
-def extracted_memory_types(memory_types: set[str]) -> set[str]:
- return set(memory_types)
-
-
-def read_archive_diff(client: Any, archive_uri: str) -> Optional[dict[str, Any]]:
- try:
- content = client.read(f"{archive_uri.rstrip('/')}/memory_diff.json")
- except Exception:
- return None
- try:
- return json.loads(content)
- except json.JSONDecodeError:
- return None
-
-
-def collect_archive_diff_types(diff: Optional[dict[str, Any]]) -> set[str]:
- if not diff:
- return set()
- operations = diff.get("operations") or {}
- memory_types = set()
- for key in ("adds", "updates", "deletes"):
- for item in operations.get(key) or []:
- memory_type = item.get("memory_type")
- if memory_type:
- memory_types.add(str(memory_type))
- return memory_types
-
-
-def run_case(
- base_url: str,
- case: CaseSpec,
- timeout: float,
-) -> CaseResult:
- client = make_client(base_url, case.api_key, timeout)
- errors: list[str] = []
- try:
- client.create_session(
- session_id=case.session_id,
- memory_policy=case.memory_policy,
- )
- client.batch_add_messages(case.session_id, build_messages())
- commit = client.commit_session(case.session_id)
- task_id = commit.get("task_id")
- archive_uri = commit.get("archive_uri") or (
- f"viking://session/{case.session_id}/history/archive_001"
- )
- if not task_id:
- raise RuntimeError(f"Commit did not return task_id: {commit}")
-
- task = wait_task(client, task_id, timeout)
- if task.get("status") != "completed":
- errors.append(f"commit task failed: {task.get('error') or task}")
-
- entries = collect_memory_entries(client, case.user_id)
- memory_types = collect_memory_types(entries)
- extracted_types = extracted_memory_types(memory_types)
- archive_diff = read_archive_diff(client, archive_uri)
- archive_diff_types = collect_archive_diff_types(archive_diff)
-
- if case.required_any_types and not extracted_types & case.required_any_types:
- errors.append(
- "missing any expected extracted memory type from: "
- f"{sorted(case.required_any_types)}"
- )
- if not any(is_peer_memory_uri(entry_uri(entry)) for entry in entries):
- errors.append(f"peer memory did not produce files for peer_id={PEER_ID}")
-
- if case.allowed_types is not None:
- outside_allowed = extracted_types - case.allowed_types
- if outside_allowed:
- errors.append(
- f"extracted memory types outside explicit whitelist: {sorted(outside_allowed)}"
- )
-
- return CaseResult(
- case=case,
- task=task,
- memory_entries=entries,
- memory_types=memory_types,
- archive_diff=archive_diff,
- archive_diff_types=archive_diff_types,
- errors=errors,
- )
- finally:
- close_client(client)
-
-
-def result_summary(result: CaseResult) -> dict[str, Any]:
- memory_files = sorted(
- entry_uri(entry)
- for entry in result.memory_entries
- if entry_uri(entry).endswith(".md") and not Path(entry_uri(entry)).name.startswith(".")
- )
- peer_memory_files = [uri for uri in memory_files if is_peer_memory_uri(uri)]
- return {
- "case": result.case.name,
- "user_id": result.case.user_id,
- "peer_id": PEER_ID,
- "session_id": result.case.session_id,
- "memory_policy": result.case.memory_policy,
- "task_status": result.task.get("status"),
- "memory_types": sorted(result.memory_types),
- "extracted_memory_types": sorted(extracted_memory_types(result.memory_types)),
- "archive_diff_types": sorted(result.archive_diff_types),
- "memory_files": memory_files,
- "peer_memory_files": peer_memory_files,
- "errors": result.errors,
- }
-
-
-def build_parser() -> argparse.ArgumentParser:
- parser = argparse.ArgumentParser(
- description="Compare session memory_policy.memory_types filtering under two users."
- )
- parser.add_argument("--config", default=DEFAULT_CONFIG_PATH, help="Path to ov.conf.")
- parser.add_argument("--url", default="", help="OpenViking server URL.")
- parser.add_argument("--root-key", default="", help="Root API key.")
- parser.add_argument("--account", default="", help="Account ID to create.")
- parser.add_argument(
- "--timeout",
- type=float,
- default=300.0,
- help="Seconds to wait for server startup and commit tasks.",
- )
- parser.add_argument(
- "--start-server",
- action="store_true",
- help="Start .venv/bin/openviking-server --config ov.conf if health check fails.",
- )
- parser.add_argument(
- "--leave-server-running",
- action="store_true",
- help="Do not terminate the server process started by this script.",
- )
- return parser
-
-
-def main() -> int:
- args = build_parser().parse_args()
- config_path = Path(args.config)
- config = load_json_file(config_path)
- base_url = (args.url or default_url_from_config(config) or DEFAULT_URL).rstrip("/")
- root_key = args.root_key or root_key_from_config(config)
- run_id = time.strftime("%Y%m%d%H%M%S") + "-" + uuid4().hex[:8]
- account_id = args.account or f"{DEFAULT_ACCOUNT_PREFIX}-{run_id}"
-
- server_process: Optional[subprocess.Popen] = None
- if not ensure_server_ready(base_url):
- if not args.start_server:
- print(
- f"OpenViking server is not ready at {base_url}. "
- "Start it first or pass --start-server.",
- file=sys.stderr,
- )
- return 2
- server_process = start_server(config_path, base_url, args.timeout)
-
- try:
- user_keys = bootstrap_users(base_url, root_key, account_id, args.timeout)
- cases = [
- CaseSpec(
- name="policy_memory_types_unset",
- user_id=DEFAULT_POLICY_USER,
- session_id=f"mp_unset_{run_id}",
- api_key=user_keys[DEFAULT_POLICY_USER],
- memory_policy=DEFAULT_POLICY,
- allowed_types=None,
- required_any_types=FILTERED_ALLOWED_TYPES | {"events", "tools", "experiences"},
- ),
- CaseSpec(
- name="policy_filtered",
- user_id=FILTERED_POLICY_USER,
- session_id=f"mp_filtered_{run_id}",
- api_key=user_keys[FILTERED_POLICY_USER],
- memory_policy=FILTERED_POLICY,
- allowed_types=FILTERED_ALLOWED_TYPES,
- required_any_types=FILTERED_ALLOWED_TYPES,
- ),
- ]
-
- results = [run_case(base_url, case, args.timeout) for case in cases]
- default_types = extracted_memory_types(results[0].memory_types)
- filtered_types = extracted_memory_types(results[1].memory_types)
- default_only_types = default_types - filtered_types
- if not default_only_types:
- results[1].errors.append(
- "no observable type difference between unset memory_types and filtered policy"
- )
-
- print_json(
- "comparison",
- {
- "default_only_types": sorted(default_only_types),
- "filtered_allowed_types": sorted(FILTERED_ALLOWED_TYPES),
- "cases": [result_summary(result) for result in results],
- },
- )
-
- if all(result.ok for result in results):
- print(
- "\nPASS: unset memory_types and explicit memory_types whitelist produced "
- "different self/peer memory type sets."
- )
- return 0
-
- print("\nFAIL: memory_policy comparison did not match expectations.", file=sys.stderr)
- return 1
- finally:
- if server_process is not None and not args.leave_server_running:
- server_process.terminate()
- try:
- server_process.wait(timeout=10.0)
- except subprocess.TimeoutExpired:
- server_process.kill()
-
-
-if __name__ == "__main__":
- raise SystemExit(main())