Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
from __future__ import annotations
import json
import os
import re
import shutil
import subprocess
from collections import Counter
from datetime import datetime, timezone
from pathlib import Path
from typing import Any
import yaml
TAU2_DIR = Path(__file__).resolve().parents[1]
REPO_ROOT = TAU2_DIR.parents[2]
CONFIRMATION_AWARE_UPSTREAM_PR = "https://github.com/sierra-research/tau2-bench/pull/297"
CONFIRMATION_AWARE_APPENDIX = """
- If the agent asks you to confirm, authorize, or approve a backend action,
reply with the requested confirmation but do not emit `###STOP###` in the
same turn.
- Emit `###STOP###` only after the agent clearly reports that the requested
backend action has been completed, or when the official transfer /
out-of-scope rules apply.
"""
_ENV_PATTERN = re.compile(r"\$\{([^}:]+)(?::-([^}]*))?\}")
def run_id() -> str:
return datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ")
def normalize_litellm_env() -> dict[str, Any]:
aliases = []
if not os.environ.get("OPENAI_API_KEY") and os.environ.get("ARK_API_KEY"):
os.environ["OPENAI_API_KEY"] = os.environ["ARK_API_KEY"]
aliases.append("OPENAI_API_KEY<-ARK_API_KEY")
ark_base = os.environ.get("ARK_BASE_URL")
openai_base = os.environ.get("OPENAI_API_BASE") or os.environ.get("OPENAI_BASE_URL")
if not openai_base and ark_base:
os.environ["OPENAI_API_BASE"] = ark_base
os.environ["OPENAI_BASE_URL"] = ark_base
aliases.append("OPENAI_API_BASE<-ARK_BASE_URL")
elif os.environ.get("OPENAI_API_BASE") and not os.environ.get("OPENAI_BASE_URL"):
os.environ["OPENAI_BASE_URL"] = os.environ["OPENAI_API_BASE"]
aliases.append("OPENAI_BASE_URL<-OPENAI_API_BASE")
elif os.environ.get("OPENAI_BASE_URL") and not os.environ.get("OPENAI_API_BASE"):
os.environ["OPENAI_API_BASE"] = os.environ["OPENAI_BASE_URL"]
aliases.append("OPENAI_API_BASE<-OPENAI_BASE_URL")
return {
"aliases": aliases,
"has_api_key": bool(os.environ.get("OPENAI_API_KEY") or os.environ.get("ARK_API_KEY")),
"has_base_url": bool(
os.environ.get("OPENAI_API_BASE")
or os.environ.get("OPENAI_BASE_URL")
or os.environ.get("ARK_BASE_URL")
),
}
def render_env(value: Any) -> Any:
if isinstance(value, str):
def replace(match: re.Match[str]) -> str:
name = match.group(1)
default = match.group(2) or ""
return os.environ.get(name, default)
return _ENV_PATTERN.sub(replace, value)
if isinstance(value, list):
return [render_env(item) for item in value]
if isinstance(value, dict):
return {key: render_env(item) for key, item in value.items()}
return value
def deep_merge(base: dict[str, Any], override: dict[str, Any]) -> dict[str, Any]:
merged = dict(base)
for key, value in override.items():
if key in merged and isinstance(merged[key], dict) and isinstance(value, dict):
merged[key] = deep_merge(merged[key], value)
else:
merged[key] = value
return merged
def load_config(path: Path) -> dict[str, Any]:
path = path.expanduser().resolve()
with path.open("r", encoding="utf-8") as handle:
raw = yaml.safe_load(handle) or {}
if not isinstance(raw, dict):
raise ValueError(f"Config must be a mapping: {path}")
parent_name = raw.pop("extends", None)
if parent_name:
parent_path = (path.parent / str(parent_name)).resolve()
parent = load_config(parent_path)
raw = deep_merge(parent, raw)
return render_env(raw)
def resolve_path(path_value: str | Path, *, base: Path | None = None) -> Path:
path = Path(path_value).expanduser()
if path.is_absolute():
return path
return ((base or REPO_ROOT) / path).resolve()
def output_dir(config: dict[str, Any], configured_run_id: str) -> Path:
raw = config.get("paths", {}).get("output_dir", TAU2_DIR / "result")
return resolve_path(raw) / configured_run_id
def write_json(path: Path, payload: Any) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(
json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True) + "\n",
encoding="utf-8",
)
def tau2_result_failures(data: dict[str, Any], *, expected_trials: int = 1) -> list[str]:
tasks = data.get("tasks") or []
simulations = data.get("simulations") or []
failures: list[str] = []
if tasks:
expected_task_ids = {
str(task.get("id", task.get("task_id"))) for task in tasks if isinstance(task, dict)
}
observed_task_ids = {str(sim.get("task_id")) for sim in simulations}
expected = len(tasks) * expected_trials
if len(simulations) != expected:
failures.append(f"expected {expected} simulations, found {len(simulations)}")
if observed_task_ids != expected_task_ids:
missing = sorted(expected_task_ids - observed_task_ids)
extra = sorted(observed_task_ids - expected_task_ids)
failures.append(
f"simulation task ids do not match tasks: missing={missing[:10]} extra={extra[:10]}"
)
expected_pairs = {
(task_id, trial) for task_id in expected_task_ids for trial in range(expected_trials)
}
observed_pairs = [
(str(sim.get("task_id")), int(sim.get("trial", 0))) for sim in simulations
]
duplicate_pairs = sorted(
pair for pair, count in Counter(observed_pairs).items() if count != 1
)
missing_pairs = sorted(expected_pairs - set(observed_pairs))
if duplicate_pairs or missing_pairs:
failures.append(
"simulation task/trial coverage mismatch: "
f"missing={missing_pairs[:10]} duplicate={duplicate_pairs[:10]}"
)
for sim in simulations:
info = sim.get("info") or {}
termination_reason = str(sim.get("termination_reason") or "")
if info.get("failed_after_attempts") or "infrastructure_error" in termination_reason:
failures.append(
"task="
f"{sim.get('task_id')} trial={sim.get('trial', 0)} "
f"termination={termination_reason} error={info.get('error') or info.get('error_type')}"
)
elif not sim.get("messages"):
failures.append(
f"task={sim.get('task_id')} trial={sim.get('trial', 0)} has no messages"
)
return failures
def assert_tau2_results_complete(
data: dict[str, Any], *, context: str, expected_trials: int = 1
) -> None:
failures = tau2_result_failures(data, expected_trials=expected_trials)
if failures:
preview = "; ".join(failures[:5])
more = f"; ... {len(failures) - 5} more" if len(failures) > 5 else ""
raise RuntimeError(f"{context} produced invalid TAU-2 results: {preview}{more}")
def strategy_ids(config: dict[str, Any]) -> list[str]:
strategies = config.get("strategies") or []
if not isinstance(strategies, list):
raise ValueError("strategies must be a list")
ids = []
for item in strategies:
if not isinstance(item, dict) or not item.get("id"):
raise ValueError("each strategy must be a mapping with id")
ids.append(str(item["id"]))
if len(ids) != len(set(ids)):
raise ValueError(f"duplicate strategy ids: {ids}")
return ids
def domains(config: dict[str, Any]) -> list[str]:
values = config.get("benchmark", {}).get("domains") or []
if not isinstance(values, list) or not values:
raise ValueError("benchmark.domains must be a non-empty list")
return [str(item) for item in values]
def tau2_repo(config: dict[str, Any]) -> Path:
raw = config.get("paths", {}).get("tau2_repo")
if not raw:
raise ValueError("paths.tau2_repo is required")
return resolve_path(raw)
def tau2_cli(config: dict[str, Any]) -> str:
return str(config.get("paths", {}).get("tau2_cli") or "tau2")
def _git_commit(path: Path) -> str | None:
if not path.exists():
return None
completed = subprocess.run(
["git", "-C", str(path), "rev-parse", "HEAD"],
text=True,
stdout=subprocess.PIPE,
stderr=subprocess.DEVNULL,
check=False,
)
if completed.returncode != 0:
return None
return completed.stdout.strip() or None
def tau2_context(config: dict[str, Any]) -> dict[str, Any]:
repo = tau2_repo(config)
cli = tau2_cli(config)
return {
"tau2_repo": str(repo),
"tau2_repo_exists": repo.exists(),
"tau2_commit": _git_commit(repo),
"tau2_cli": cli,
"tau2_cli_resolved": shutil.which(cli),
}
def _prompt_paths(repo: Path) -> list[Path]:
return [
repo / "data" / "tau2" / "user_simulator" / "simulation_guidelines.md",
repo / "data" / "tau2" / "user_simulator" / "simulation_guidelines_tools.md",
]
def _has_confirmation_aware_prompt(prompt_text: str) -> bool:
normalized = " ".join(prompt_text.split())
return (
"reply with the requested confirmation" in normalized
and "do not emit `###STOP###` in the same turn" in normalized
)
def _ensure_confirmation_aware_prompt(repo: Path) -> bool:
patched = False
for path in _prompt_paths(repo):
if not path.is_file():
continue
text = path.read_text(encoding="utf-8")
if _has_confirmation_aware_prompt(text):
continue
backup = path.with_suffix(path.suffix + ".openviking.bak")
if not backup.exists():
backup.write_text(text, encoding="utf-8")
path.write_text(text.rstrip() + CONFIRMATION_AWARE_APPENDIX + "\n", encoding="utf-8")
patched = True
return patched
def user_simulator_policy(config: dict[str, Any]) -> str:
policy = config.get("eval", {}).get("user_simulator_policy", "official")
policy = str(policy)
if policy not in {"official", "confirmation_aware"}:
raise ValueError("eval.user_simulator_policy must be 'official' or 'confirmation_aware'")
return policy
def simulator_policy_report(config: dict[str, Any]) -> dict[str, Any]:
policy = user_simulator_policy(config)
repo = tau2_repo(config)
patch_applied = policy == "confirmation_aware" and _ensure_confirmation_aware_prompt(repo)
patch_mode = "direct_prompt_append" if patch_applied else "none"
if policy == "confirmation_aware":
if not patch_applied:
patch_mode = "upstream_or_existing_prompt"
prompt_paths = _prompt_paths(repo)
prompt_text = "\n".join(
path.read_text(encoding="utf-8") for path in prompt_paths if path.is_file()
)
confirmation_aware_prompt = _has_confirmation_aware_prompt(prompt_text)
supported = policy == "official" or confirmation_aware_prompt
claim_boundary = "confirmation_aware_user_simulator_prompt"
if policy == "official":
claim_boundary = (
"official_policy_with_confirmation_aware_checkout"
if confirmation_aware_prompt
else "official_tau2_user_simulator"
)
return {
"user_simulator_policy": policy,
"supported": supported,
"confirmation_aware_prompt_detected": confirmation_aware_prompt,
"confirmation_aware_upstream_pr": CONFIRMATION_AWARE_UPSTREAM_PR,
"patch_applied": patch_applied,
"patch_mode": patch_mode,
"prompt_files": [str(path) for path in prompt_paths],
"backup_files": [
str(path.with_suffix(path.suffix + ".openviking.bak"))
for path in prompt_paths
if path.with_suffix(path.suffix + ".openviking.bak").exists()
],
"claim_boundary": claim_boundary,
}
def split_file(config: dict[str, Any], domain: str) -> Path:
return tau2_repo(config) / "data" / "tau2" / "domains" / domain / "split_tasks.json"