Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
"""Generic remote benchmark batch train/eval orchestration."""
from __future__ import annotations
import inspect
import json
import os
import shutil
from dataclasses import dataclass, field
from datetime import datetime
from hashlib import sha256
from pathlib import Path
from typing import Any
from openviking.server.config import load_server_config
from openviking.server.identity import AuthMode
from openviking.session.train.components.dataset_service import rollout_from_dict, rollout_to_dict
from openviking.session.train.components.event_recorder import (
CompositeEventRecorder,
JsonlEventRecorder,
JsonlPipelineEventHook,
)
from openviking.session.train.components.progress import format_label, label_style
from openviking.session.train.components.remote import RemoteCaseLoader, RemoteRolloutExecutor
from openviking.session.train.components.report_builder import PipelineReportBuilder
from openviking.session.train.components.reporter import (
_accuracy_style,
_style_plain,
emit_run_summary,
fmt_percent,
fmt_percentage_point_abs,
)
from openviking.session.train.components.rollout_artifact_recorder import (
RolloutArtifactEventRecorder,
RolloutArtifactRecorder,
)
from openviking.session.train.components.session_commit import SessionCommitPolicyTrainer
from openviking.session.train.components.snapshotter import ContentHashPolicySnapshotter
from openviking.session.train.context import ExecutionContext, PipelineContext
from openviking.session.train.domain import (
Case,
ExperienceSet,
PolicyUpdatePlan,
Rollout,
RolloutAnalysis,
)
from openviking.session.train.pipeline import OfflinePolicyOptimizationPipeline
from openviking.telemetry import tracer
from openviking_cli.client.http import AsyncHTTPClient
from openviking_cli.utils.config.open_viking_config import OpenVikingConfigSingleton
@dataclass(slots=True)
class BatchTrainEvalConfig:
"""Configuration for one remote benchmark batch train/eval run."""
domain: str
dataset: str
epochs: int = 1
batch_size: int | None = None
concurrency: int = 200
config_path: str | None = None
output_path: str | None = None
keep_default_tools: bool = True
max_iterations: int = 30
server_url: str | None = None
api_key: str | None = None
account_id: str = "default"
user_id: str = "default"
commit_keep_recent_count: int = 0
commit_poll_interval_seconds: float = 2.0
commit_timeout_seconds: float | None = None
commit_concurrency: int = 200
train_index: int | str | list[int] | tuple[int, ...] | None = None
eval_index: int | str | list[int] | tuple[int, ...] | None = None
benchmark_service_url: str | None = None
baseline_force_recompute: bool = False
skip_baseline_eval: bool = False
eval_each_epoch: bool = False
eval_split: str | None = "test"
skip_final_eval: bool = False
trials: int = 8
train_trials: int = 1
reuse_train_rollout_cache: bool = False
clean_result: bool = True
keep_recent_results: int = 5
events_path: str | None = None
result_dir_name: str = "train"
run_timestamp: str = field(default_factory=lambda: datetime.now().strftime("%Y%m%d_%H%M%S"))
def __post_init__(self) -> None:
if not self.dataset:
raise ValueError("dataset is required")
if not self.domain:
raise ValueError("domain is required")
if self.epochs < 0:
raise ValueError("epochs must be >= 0")
if self.batch_size is not None and self.batch_size <= 0:
raise ValueError("batch_size must be > 0")
if self.concurrency <= 0:
raise ValueError("concurrency must be > 0")
if self.max_iterations <= 0:
raise ValueError("max_iterations must be > 0")
if self.commit_poll_interval_seconds <= 0:
raise ValueError("commit_poll_interval_seconds must be > 0")
if self.commit_timeout_seconds is not None and self.commit_timeout_seconds <= 0:
raise ValueError("commit_timeout_seconds must be > 0")
if self.commit_concurrency <= 0:
raise ValueError("commit_concurrency must be > 0")
self.train_index = _normalize_index_filter(self.train_index, label="train_index")
self.eval_index = _normalize_index_filter(self.eval_index, label="eval_index")
if self.eval_split is not None:
normalized_eval_split = str(self.eval_split).strip().lower()
if normalized_eval_split in {"", "none"}:
self.eval_split = None
elif normalized_eval_split not in {"train", "test"}:
raise ValueError("eval_split must be train, test, or none")
else:
self.eval_split = normalized_eval_split
if self.trials <= 0:
raise ValueError("trials must be > 0")
if self.train_trials <= 0:
raise ValueError("train_trials must be > 0")
if self.benchmark_service_url is not None and not self.benchmark_service_url.strip():
raise ValueError("benchmark_service_url must not be empty")
if self.keep_recent_results < 0:
raise ValueError("keep_recent_results must be >= 0")
if not str(self.result_dir_name or "").strip():
raise ValueError("result_dir_name must not be empty")
def _normalize_index_filter(value: Any, *, label: str) -> list[int] | None:
if value is None:
return None
raw_items: list[Any]
if isinstance(value, str):
raw_items = [item.strip() for item in value.split(",") if item.strip()]
elif isinstance(value, int):
raw_items = [value]
else:
raw_items = []
try:
iterable = list(value)
except TypeError as exc:
raise ValueError(f"{label} must be an integer or comma-separated integers") from exc
for item in iterable:
if isinstance(item, str) and "," in item:
raw_items.extend(part.strip() for part in item.split(",") if part.strip())
else:
raw_items.append(item)
result: list[int] = []
for item in raw_items:
try:
index = int(item)
except (TypeError, ValueError) as exc:
raise ValueError(f"{label} must be an integer or comma-separated integers") from exc
if index < 0:
raise ValueError(f"{label} must be >= 0")
if index not in result:
result.append(index)
if not result:
raise ValueError(f"{label} must not be empty")
return result
@dataclass(slots=True)
class BatchTrainEvalReport:
"""Serializable report for remote benchmark batch train/eval."""
dataset: str
domain: str
epochs: int
batch_size: int | None
concurrency: int
commit_concurrency: int
train_index: int | list[int] | None
eval_index: int | list[int] | None
policy_root_uri: str
baseline_eval: dict[str, Any] | None
train_epochs: list[dict[str, Any]] = field(default_factory=list)
epoch_evals: list[dict[str, Any]] = field(default_factory=list)
final_eval: dict[str, Any] | None = None
accuracy_delta: float | None = None
output_path: str | None = None
trace_id: str | None = None
run_id: str = ""
server_url: str = ""
benchmark_service_url: str | None = None
eval_each_epoch: bool = False
eval_split: str | None = "test"
skip_baseline_eval: bool = False
trials: int = 8
train_trials: int = 1
reuse_train_rollout_cache: bool = False
rollouts_root: str | None = None
rollouts_index_path: str | None = None
latest_failed_rollout: str | None = None
clean_result: bool = True
keep_recent_results: int = 5
events_path: str | None = None
result_dir_name: str = "train"
baseline_cache_path: str | None = None
baseline_cache_hit: bool = False
baseline_force_recompute: bool = False
skip_final_eval: bool = False
final_eval_source: str | None = None
def to_dict(self) -> dict[str, Any]:
return {
"dataset": self.dataset,
"domain": self.domain,
"epochs": self.epochs,
"batch_size": self.batch_size,
"concurrency": self.concurrency,
"commit_concurrency": self.commit_concurrency,
"train_index": self.train_index,
"eval_index": self.eval_index,
"policy_root_uri": self.policy_root_uri,
"baseline_eval": self.baseline_eval,
"train_epochs": self.train_epochs,
"epoch_evals": self.epoch_evals,
"final_eval": self.final_eval,
"accuracy_delta": self.accuracy_delta,
"output_path": self.output_path,
"trace_id": self.trace_id,
"run_id": self.run_id,
"server_url": self.server_url,
"benchmark_service_url": self.benchmark_service_url,
"eval_each_epoch": self.eval_each_epoch,
"eval_split": self.eval_split,
"skip_baseline_eval": self.skip_baseline_eval,
"trials": self.trials,
"train_trials": self.train_trials,
"reuse_train_rollout_cache": self.reuse_train_rollout_cache,
"rollouts_root": self.rollouts_root,
"rollouts_index_path": self.rollouts_index_path,
"latest_failed_rollout": self.latest_failed_rollout,
"clean_result": self.clean_result,
"keep_recent_results": self.keep_recent_results,
"events_path": self.events_path,
"result_dir_name": self.result_dir_name,
"baseline_cache_path": self.baseline_cache_path,
"baseline_cache_hit": self.baseline_cache_hit,
"baseline_force_recompute": self.baseline_force_recompute,
"skip_final_eval": self.skip_final_eval,
"final_eval_source": self.final_eval_source,
}
@tracer("train.batch_train_eval.run", ignore_result=True, ignore_args=True)
async def run_batch_train_eval(config: BatchTrainEvalConfig) -> BatchTrainEvalReport:
"""Run baseline eval, commit-based train epochs, and final eval for one dataset/domain."""
_configure_openviking_config(config.config_path)
_clean_result_dir(config)
client = _build_http_client(config)
await client.initialize()
try:
policy_root_uri = f"viking://user/{config.user_id}/memories/experiences"
policy_set = ExperienceSet(
root_uri=policy_root_uri,
policies=[],
metadata=_policy_set_metadata(config, client),
)
run_dir = _run_output_dir(config)
event_recorder = JsonlEventRecorder(
path=_events_path(config),
default_fields={
"dataset": config.dataset,
"domain": config.domain,
"run_timestamp": config.run_timestamp,
},
)
await event_recorder.record(
"run_start",
stage="run_start",
epochs=config.epochs,
concurrency=config.concurrency,
commit_concurrency=config.commit_concurrency,
train_index=_index_payload(config.train_index),
eval_index=_index_payload(config.eval_index),
trials=config.trials,
train_trials=config.train_trials,
reuse_train_rollout_cache=config.reuse_train_rollout_cache,
clean_result=config.clean_result,
keep_recent_results=config.keep_recent_results,
result_dir_name=config.result_dir_name,
baseline_force_recompute=config.baseline_force_recompute,
skip_baseline_eval=config.skip_baseline_eval,
eval_split=config.eval_split,
skip_final_eval=config.skip_final_eval,
baseline_cache_path=(
None
if config.skip_baseline_eval or config.eval_split is None
else str(_baseline_cache_path(config))
),
)
policy_trainer = SessionCommitPolicyTrainer(
client=client,
keep_recent_count=config.commit_keep_recent_count,
poll_interval_seconds=config.commit_poll_interval_seconds,
timeout_seconds=config.commit_timeout_seconds,
commit_concurrency=config.commit_concurrency,
show_progress=True,
progress_label="train",
)
event_recorder.default_fields["run_id"] = policy_trainer.run_id
pipeline = _build_pipeline(config, policy_trainer)
rollout_artifact_recorder = RolloutArtifactRecorder(
run_dir=run_dir,
client=client,
latest_pointer_path=_latest_rollouts_path(config),
)
remote_executor = getattr(pipeline, "rollout_executor", None)
if isinstance(remote_executor, (RemoteRolloutExecutor, CachedEpochZeroTrainRolloutExecutor)):
remote_executor.on_rollout_complete = (
rollout_artifact_recorder.record_rollout_completion
)
policy_trainer.event_recorder = CompositeEventRecorder(
(event_recorder, RolloutArtifactEventRecorder(rollout_artifact_recorder))
)
baseline_eval: dict[str, Any] | None = None
baseline_cache_hit = False
baseline_cache_path = (
None
if config.skip_baseline_eval or config.eval_split is None
else _baseline_cache_path(config)
)
final_eval: dict[str, Any] | None = None
report_builder = PipelineReportBuilder(trial_index_key="eval_trial")
eval_loader = (
None
if config.eval_split is None
else _case_loader(
config,
split=config.eval_split,
sample_index=config.eval_index,
)
)
if (
eval_loader is not None
and not config.skip_baseline_eval
and await eval_loader.split_exists()
):
baseline_result, baseline_cache_hit = await _load_or_run_baseline_eval(
config=config,
pipeline=pipeline,
case_loader=eval_loader,
policy_set=policy_set,
report_builder=report_builder,
event_recorder=event_recorder,
)
if baseline_result is not None:
rollout_artifact_recorder.record_eval(
label=_eval_rollout_stage("baseline", config.eval_split),
epoch=-1,
analyses=baseline_result.analyses,
)
baseline_eval = baseline_result.metadata["report"]
else:
assert baseline_cache_path is not None
baseline_eval = _load_baseline_cache(baseline_cache_path)
if baseline_eval is not None:
_print_baseline_cache_hit(baseline_eval, baseline_cache_path)
train_loader = _case_loader(
config,
split="train",
sample_index=config.train_index,
)
train_context = _pipeline_context(
epoch=0,
training=True,
max_epochs=config.epochs,
eval_each_epoch_case_loader=(
eval_loader
if (
eval_loader is not None
and config.eval_each_epoch
and await eval_loader.split_exists()
)
else None
),
rollout_stage=(
_eval_rollout_stage("epoch", config.eval_split)
if config.eval_split is not None
else None
),
eval_split=config.eval_split,
eval_trials=config.trials,
train_trials=config.train_trials,
trial_index_key="eval_trial",
report_builder=report_builder,
event_recorder=event_recorder,
)
# Register rollout artifact recorder as a lifecycle hook so rollouts
# are written incrementally after each epoch/eval, instead of waiting
# for the full run to finish.
train_context.lifecycle_hooks = list(train_context.lifecycle_hooks) + [
rollout_artifact_recorder
]
train_result = await pipeline.train(
case_loader=train_loader,
policy_set=policy_set,
context=train_context,
)
policy_set = train_result.apply_result.updated_policy_set
# Note: per-epoch rollout artifacts are written incrementally via the
# rollout_artifact_recorder lifecycle hook registered on train_context.
epoch_eval_reports = _epoch_eval_reports(train_result)
final_eval_source: str | None = None
if config.skip_final_eval:
if epoch_eval_reports:
final_eval = dict(epoch_eval_reports[-1])
final_eval_source = "last_epoch_eval"
elif eval_loader is not None and await eval_loader.split_exists():
final_result = await pipeline.eval(
case_loader=eval_loader,
policy_set=policy_set,
context=_pipeline_context(
epoch=config.epochs,
training=False,
max_epochs=1,
rollout_stage=_eval_rollout_stage("final", config.eval_split),
eval_split=config.eval_split,
eval_trials=config.trials,
trial_index_key="eval_trial",
report_builder=report_builder,
event_recorder=event_recorder,
),
)
rollout_artifact_recorder.record_eval(
label=_eval_rollout_stage("final", config.eval_split),
epoch=config.epochs,
analyses=final_result.analyses,
)
final_eval = final_result.metadata["report"]
final_eval_source = f"final_{config.eval_split}"
accuracy_delta = report_builder.accuracy_delta(baseline_eval, final_eval)
rollout_artifact_index = rollout_artifact_recorder.finalize()
report = BatchTrainEvalReport(
dataset=config.dataset,
domain=config.domain,
epochs=config.epochs,
batch_size=config.batch_size,
concurrency=config.concurrency,
commit_concurrency=config.commit_concurrency,
train_index=_index_payload(config.train_index),
eval_index=_index_payload(config.eval_index),
policy_root_uri=policy_root_uri,
baseline_eval=baseline_eval,
train_epochs=list(train_result.metadata.get("train_reports", [])),
epoch_evals=epoch_eval_reports,
final_eval=final_eval,
accuracy_delta=accuracy_delta,
output_path=_default_output_path(config),
trace_id=tracer.get_trace_id() or None,
run_id=policy_trainer.run_id,
server_url=client_url(client),
benchmark_service_url=config.benchmark_service_url,
eval_each_epoch=config.eval_each_epoch,
eval_split=config.eval_split,
skip_baseline_eval=config.skip_baseline_eval,
trials=config.trials,
train_trials=config.train_trials,
reuse_train_rollout_cache=config.reuse_train_rollout_cache,
rollouts_root=rollout_artifact_index.rollouts_root,
rollouts_index_path=str(run_dir / "rollouts_index.json"),
latest_failed_rollout=rollout_artifact_index.latest_failed_rollout,
clean_result=config.clean_result,
keep_recent_results=config.keep_recent_results,
events_path=str(_events_path(config)),
result_dir_name=config.result_dir_name,
baseline_cache_path=(
str(baseline_cache_path) if baseline_cache_path is not None else None
),
baseline_cache_hit=baseline_cache_hit,
baseline_force_recompute=config.baseline_force_recompute,
skip_final_eval=config.skip_final_eval,
final_eval_source=final_eval_source,
)
_write_report(report, config)
await event_recorder.record(
"run_result",
stage="run_result",
trace_id=report.trace_id,
output_path=report.output_path,
rollouts_root=report.rollouts_root,
rollouts_index_path=report.rollouts_index_path,
latest_failed_rollout=report.latest_failed_rollout,
accuracy_delta=report.accuracy_delta,
baseline_cache_path=report.baseline_cache_path,
baseline_cache_hit=report.baseline_cache_hit,
)
await emit_run_summary(
train_context,
title="batch train/eval",
fields={
"dataset": config.dataset,
"domain": config.domain,
"epochs": config.epochs,
"trials": config.trials,
"train_trials": config.train_trials,
"reuse_train_rollout_cache": config.reuse_train_rollout_cache,
"run_id": policy_trainer.run_id,
"trace_id": report.trace_id,
"baseline_cache_hit": report.baseline_cache_hit,
"skip_baseline_eval": report.skip_baseline_eval,
"eval_split": report.eval_split,
"skip_final_eval": report.skip_final_eval,
"final_eval_source": report.final_eval_source,
},
baseline_eval=baseline_eval,
final_eval=final_eval,
accuracy_delta=accuracy_delta,
output_path=report.output_path,
rollouts_root=report.rollouts_root,
rollouts_index_path=report.rollouts_index_path,
latest_failed_rollout=report.latest_failed_rollout,
)
return report
finally:
await client.close()
def _configure_openviking_config(config_path: str | None) -> None:
if config_path:
os.environ["OPENVIKING_CONFIG_FILE"] = str(Path(config_path).expanduser())
OpenVikingConfigSingleton.reset_instance()
def _build_http_client(config: BatchTrainEvalConfig) -> AsyncHTTPClient:
server_url = config.server_url
api_key = config.api_key
auth_mode: AuthMode | None = None
if config.config_path or server_url is None or api_key is None:
server_config = load_server_config(config.config_path)
auth_mode = server_config.get_effective_auth_mode()
server_url = server_url or f"http://{server_config.host}:{server_config.port}"
api_key = api_key or server_config.root_api_key
if auth_mode is None:
auth_mode = AuthMode.API_KEY
# Trusted mode uses X-API-Key as the gateway/root key and takes identity from
# X-OpenViking-Account/User. In api_key/dev modes, user API keys already pin
# identity and account/user assertion headers must not be sent.
account = config.account_id if auth_mode == AuthMode.TRUSTED else None
user = config.user_id if auth_mode == AuthMode.TRUSTED else None
return AsyncHTTPClient(
url=server_url,
api_key=api_key,
account=account,
user=user,
profile_enabled=False,
timeout=max(60.0, (config.commit_timeout_seconds or 600.0) + 30.0),
)
def _epoch_eval_reports(train_result: Any) -> list[dict[str, Any]]:
reports: list[dict[str, Any]] = []
for evaluation in getattr(train_result, "evaluation_passes", []) or []:
report = getattr(evaluation, "metadata", {}).get("report")
if isinstance(report, dict):
reports.append(dict(report))
return reports
def _policy_set_metadata(config: BatchTrainEvalConfig, client: AsyncHTTPClient) -> dict[str, Any]:
return {
"source": "remote_session_commit",
"openviking_url": client_url(client),
"openviking_api_key": getattr(client, "_api_key", None),
"openviking_account": config.account_id,
"openviking_user": config.user_id,
}
def client_url(client: AsyncHTTPClient) -> str:
return str(getattr(client, "_url", ""))
async def _load_or_run_baseline_eval(
*,
config: BatchTrainEvalConfig,
pipeline: OfflinePolicyOptimizationPipeline,
case_loader: RemoteCaseLoader,
policy_set: ExperienceSet,
report_builder: PipelineReportBuilder,
event_recorder: JsonlEventRecorder,
) -> tuple[Any | None, bool]:
cache_path = _baseline_cache_path(config)
if not config.baseline_force_recompute:
cached_report = _load_baseline_cache(cache_path)
if cached_report is not None:
await event_recorder.record(
"baseline_cache_hit",
stage="baseline_cache",
baseline_cache_path=str(cache_path),
eval_split=config.eval_split,
)
return None, True
await event_recorder.record(
"baseline_cache_recompute" if cache_path.exists() else "baseline_cache_miss",
stage="baseline_cache",
baseline_cache_path=str(cache_path),
eval_split=config.eval_split,
)
baseline_result = await pipeline.eval(
case_loader=case_loader,
policy_set=policy_set,
context=_pipeline_context(
epoch=-1,
training=False,
max_epochs=1,
rollout_stage=_eval_rollout_stage("baseline", config.eval_split),
eval_split=config.eval_split,
eval_trials=config.trials,
trial_index_key="eval_trial",
report_builder=report_builder,
event_recorder=event_recorder,
),
)
_write_baseline_cache(cache_path, baseline_result.metadata["report"], config=config)
await event_recorder.record(
"baseline_cache_write",
stage="baseline_cache",
baseline_cache_path=str(cache_path),
eval_split=config.eval_split,
)
return baseline_result, False
def _write_baseline_cache(
path: Path,
report: dict[str, Any],
*,
config: BatchTrainEvalConfig,
) -> None:
payload = {
"cache_version": 1,
"cache_key": _baseline_cache_key(config),
"dataset": config.dataset,
"domain": config.domain,
"split": config.eval_split,
"eval_index": _index_payload(config.eval_index),
"trials": config.trials,
"max_iterations": config.max_iterations,
"keep_default_tools": config.keep_default_tools,
"created_at": datetime.now().isoformat(),
"report": report,
}
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
def _load_baseline_cache(path: Path) -> dict[str, Any] | None:
if not path.exists():
return None
payload = json.loads(path.read_text(encoding="utf-8"))
if payload.get("cache_version") != 1:
raise ValueError(f"unsupported baseline cache version in {path}")
report = payload.get("report")
if not isinstance(report, dict):
raise ValueError(f"baseline cache file has no report: {path}")
return {
**report,
"baseline_cache_hit": True,
"baseline_cache_path": str(path),
}
def _print_baseline_cache_hit(report: dict[str, Any], cache_path: Path) -> None:
"""Print cached baseline info before training starts so users see it immediately."""
label = str(report.get("rollout_stage") or "baseline_test_rollout")
trial_count = int(report.get("trial_count") or 1)
cache_info = f"(from cache: {cache_path.name})"
label_text = _style_plain(format_label(label), label_style(label))
if trial_count > 1:
accuracy_mean = report.get("accuracy_mean")
accuracy_std = report.get("accuracy_std")
cases_per_trial = report.get("case_count_per_trial") or "varies"
print(
f"{label_text} baseline_cache_hit=1 "
f"accuracy={_style_plain(fmt_percent(accuracy_mean), _accuracy_style(accuracy_mean))} "
f"± {_style_plain(fmt_percentage_point_abs(accuracy_std), 'yellow')} "
f"trials={trial_count} cases_per_trial={cases_per_trial} "
f"{cache_info}"
)
return
accuracy = report.get("accuracy")
passed = report.get("passed_count")
total = report.get("case_count")
print(
f"{label_text} baseline_cache_hit=1 "
f"accuracy={_style_plain(fmt_percent(accuracy), _accuracy_style(accuracy))} "
f"passed={passed}/{total} "
f"{cache_info}"
)
def _eval_rollout_stage(kind: str, split: str | None) -> str:
eval_split = str(split or "test")
if kind == "epoch" and eval_split == "train":
return "eval_train_rollout"
return f"{kind}_{eval_split}_rollout"
def _build_pipeline(
config: BatchTrainEvalConfig,
policy_trainer: SessionCommitPolicyTrainer,
) -> OfflinePolicyOptimizationPipeline:
rollout_executor: Any = RemoteRolloutExecutor(
service_url=_require_benchmark_service_url(config),
concurrency=config.concurrency,
show_progress=True,
progress_label="rollout",
options={
"config_path": config.config_path,
"keep_default_tools": config.keep_default_tools,
"max_iterations": config.max_iterations,
},
)
if config.reuse_train_rollout_cache:
rollout_executor = CachedEpochZeroTrainRolloutExecutor(
delegate=rollout_executor,
cache_dir=_train_rollout_cache_dir(config),
cache_key_prefix=_train_rollout_cache_key_prefix(config),
)
return OfflinePolicyOptimizationPipeline(
snapshotter=ContentHashPolicySnapshotter(prefix=f"{config.dataset}-policy-snapshot"),
rollout_executor=rollout_executor,
rollout_analyzer=UnusedRolloutAnalyzer(),
gradient_estimator=UnusedGradientEstimator(),
policy_optimizer=UnusedPolicyOptimizer(),
policy_updater=UnusedPolicyUpdater(),
policy_trainer=policy_trainer,
)
def _pipeline_context(
*,
epoch: int,
training: bool,
max_epochs: int = 1,
rollout_stage: str | None = None,
eval_split: str | None = None,
eval_each_epoch_case_loader: Any = None,
eval_trials: int = 1,
train_trials: int = 1,
trial_index_key: str = "trial",
report_builder: Any = None,
event_recorder: JsonlEventRecorder | None = None,
) -> PipelineContext:
execution_metadata = {"epoch": epoch, "training": training}
if rollout_stage is not None:
execution_metadata["rollout_stage"] = rollout_stage
if eval_split is not None:
execution_metadata["eval_split"] = eval_split
hooks = None
if event_recorder is not None:
from openviking.session.train.components.report_builder import PipelineReportHook
from openviking.session.train.components.reporter import ConsolePipelineReporter
hooks = [
PipelineReportHook(),
JsonlPipelineEventHook(event_recorder),
ConsolePipelineReporter(),
]
return PipelineContext(
analysis_context={"epoch": epoch},
execution_metadata=execution_metadata,
max_epochs=max_epochs,
eval_each_epoch_case_loader=eval_each_epoch_case_loader,
eval_trials=eval_trials,
train_trials=train_trials,
trial_index_key=trial_index_key,
report_builder=report_builder,
**({"lifecycle_hooks": hooks} if hooks is not None else {}),
)
def _case_loader(
config: BatchTrainEvalConfig,
*,
split: str,
sample_index: list[int] | None,
) -> RemoteCaseLoader:
filters: dict[str, Any] = {}
if sample_index is not None:
filters["task_indices"] = list(sample_index)
return RemoteCaseLoader(
service_url=_require_benchmark_service_url(config),
dataset=config.dataset,
domain=config.domain,
split=split,
batch_size=config.batch_size,
filters=filters,
)
@dataclass(slots=True)
class CachedEpochZeroTrainRolloutExecutor:
"""Reuse cached train rollouts only for epoch 0.
The first training epoch is the no-new-memory rollout used to generate the
initial training signal. Later epochs intentionally execute again so they
can observe policy/memory updates from earlier commits.
"""
delegate: Any
cache_dir: Path
cache_key_prefix: str
on_rollout_complete: Any | None = None
async def execute(
self,
cases: list[Case],
policy_set: ExperienceSet,
context: ExecutionContext,
) -> list[Rollout]:
metadata = dict(context.metadata or {})
if not bool(metadata.get("training")) or int(metadata.get("epoch", 0) or 0) != 0:
return await self.delegate.execute(cases, policy_set, context)
case_list = list(cases)
results: list[Rollout | None] = [None] * len(case_list)
misses: list[tuple[int, Case]] = []
for index, case in enumerate(case_list):
cached = self._load(case)
if cached is None:
misses.append((index, case))
else:
cached.policy_snapshot_id = context.policy_snapshot_id
metadata = dict(cached.metadata or {})
metadata["train_rollout_cache_hit"] = True
metadata["train_rollout_cache_path"] = str(self._path(case))
cached.metadata = metadata
results[index] = cached
await self._emit_rollout_complete(
rollout=cached,
index=index,
context=context,
)
if misses:
miss_rollouts = await self.delegate.execute(
[case for _, case in misses],
policy_set,
context,
)
for (index, case), rollout in zip(misses, miss_rollouts, strict=True):
self._write(case, rollout)
results[index] = rollout
await self._emit_rollout_complete(
rollout=rollout,
index=index,
context=context,
)
return [rollout for rollout in results if rollout is not None]
def _path(self, case: Case) -> Path:
payload = {
"prefix": self.cache_key_prefix,
"case": {
"name": case.name,
"task_signature": case.task_signature,
"input": case.input,
"metadata": case.metadata,
},
}
stable = json.dumps(payload, sort_keys=True, separators=(",", ":"), ensure_ascii=False)
digest = sha256(stable.encode("utf-8")).hexdigest()[:24]
return self.cache_dir / f"{_cache_slug(case.name)}_{digest}.json"
def _load(self, case: Case) -> Rollout | None:
path = self._path(case)
if not path.exists():
return None
payload = json.loads(path.read_text(encoding="utf-8"))
if payload.get("cache_version") != 1:
raise ValueError(f"unsupported train rollout cache version in {path}")
rollout_data = payload.get("rollout")
if not isinstance(rollout_data, dict):
raise ValueError(f"train rollout cache file has no rollout: {path}")
return rollout_from_dict(rollout_data)
def _write(self, case: Case, rollout: Rollout) -> None:
path = self._path(case)
payload = {
"cache_version": 1,
"cache_key_prefix": self.cache_key_prefix,
"case_name": case.name,
"task_signature": case.task_signature,
"created_at": datetime.now().isoformat(),
"rollout": rollout_to_dict(rollout),
}
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
async def _emit_rollout_complete(
self,
*,
rollout: Rollout,
index: int,
context: ExecutionContext,
) -> None:
if self.on_rollout_complete is None:
return
result = self.on_rollout_complete(
rollout=rollout,
index=index,
context=context,
)
if inspect.isawaitable(result):
await result
def _require_benchmark_service_url(config: BatchTrainEvalConfig) -> str:
if not config.benchmark_service_url:
raise ValueError(
"benchmark_service_url is required; start benchmark service and pass "
"--benchmark-service-url"
)
return config.benchmark_service_url
class UnusedRolloutAnalyzer:
async def analyze(self, rollout: Rollout, context: Any = None) -> RolloutAnalysis:
raise RuntimeError("eval uses rollout.evaluation; training is handled by policy_trainer")
class UnusedGradientEstimator:
async def estimate(
self, analysis: RolloutAnalysis, experience_set: ExperienceSet, context: Any
):
raise RuntimeError("policy_trainer handles training; gradient estimator must not run")
class UnusedPolicyOptimizer:
async def plan(self, gradients: list[Any], policy_set: ExperienceSet, context: Any):
raise RuntimeError("policy_trainer handles training; policy optimizer must not run")
class UnusedPolicyUpdater:
async def apply(
self,
plan: PolicyUpdatePlan,
policy_set: ExperienceSet,
context: Any,
*,
transaction_handle: Any = None,
):
del transaction_handle
raise RuntimeError("policy_trainer handles training; policy updater must not run")
def _write_report(report: BatchTrainEvalReport, config: BatchTrainEvalConfig) -> None:
output_path = Path(_default_output_path(config)).expanduser()
output_path.parent.mkdir(parents=True, exist_ok=True)
output_path.write_text(
json.dumps(report.to_dict(), ensure_ascii=False, indent=2),
encoding="utf-8",
)
report.output_path = str(output_path)
def _events_path(config: BatchTrainEvalConfig) -> Path:
if config.events_path:
return Path(config.events_path).expanduser()
return _run_output_dir(config) / "events.jsonl"
def _default_output_path(config: BatchTrainEvalConfig) -> str:
if config.output_path:
return str(Path(config.output_path).expanduser())
return str(_run_output_dir(config) / "report.json")
def _baseline_cache_path(config: BatchTrainEvalConfig) -> Path:
return _result_base_dir(config) / "cache" / "baseline" / f"{_baseline_cache_key(config)}.json"
def _train_rollout_cache_dir(config: BatchTrainEvalConfig) -> Path:
return (
_result_base_dir(config)
/ "cache"
/ "train_rollouts"
/ _train_rollout_cache_key_prefix(config)
)
def _train_rollout_cache_key_prefix(config: BatchTrainEvalConfig) -> str:
payload = {
"dataset": config.dataset,
"domain": config.domain,
"split": "train",
"train_index": _index_payload(config.train_index),
"train_trials": config.train_trials,
"max_iterations": config.max_iterations,
"keep_default_tools": config.keep_default_tools,
}
stable = json.dumps(payload, sort_keys=True, separators=(",", ":"), ensure_ascii=False)
digest = sha256(stable.encode("utf-8")).hexdigest()[:16]
index = _index_label(config.train_index)
return f"{_cache_slug(config.domain)}_train_index-{index}_trials-{config.train_trials}_{digest}"
def _baseline_cache_key(config: BatchTrainEvalConfig) -> str:
payload = {
"dataset": config.dataset,
"domain": config.domain,
"split": config.eval_split,
"eval_index": _index_payload(config.eval_index),
"trials": config.trials,
"max_iterations": config.max_iterations,
"keep_default_tools": config.keep_default_tools,
}
stable = json.dumps(payload, sort_keys=True, separators=(",", ":"), ensure_ascii=False)
digest = sha256(stable.encode("utf-8")).hexdigest()[:16]
index = _index_label(config.eval_index)
split = _cache_slug(str(config.eval_split or "none"))
return f"{_cache_slug(config.domain)}_{split}_index-{index}_trials-{config.trials}_{digest}"
def _index_payload(indices: list[int] | None) -> int | list[int] | None:
if indices is None:
return None
return indices[0] if len(indices) == 1 else list(indices)
def _index_label(indices: list[int] | None) -> str:
if indices is None:
return "all"
if len(indices) == 1:
return str(indices[0])
return "multi-" + "-".join(str(item) for item in indices)
def _cache_slug(value: str) -> str:
return (
"".join(ch if ch.isalnum() or ch in ("-", "_") else "-" for ch in value).strip("-")
or "default"
)
def _clean_result_dir(config: BatchTrainEvalConfig) -> None:
if not config.clean_result:
return
if config.output_path:
output_path = Path(config.output_path).expanduser()
if output_path.exists() and output_path.is_file():
output_path.unlink()
return
result_dir = _result_base_dir(config)
result_dir.mkdir(parents=True, exist_ok=True)
protected_names = {_run_dir_name(config)}
run_dirs = []
removed = 0
if result_dir.exists():
for child in result_dir.iterdir():
if child.name in protected_names:
continue
if child.is_dir() and not child.is_symlink():
if _is_default_run_dir(child, config):
run_dirs.append(child)
continue
run_dirs.sort(key=lambda path: (path.stat().st_mtime, path.name), reverse=True)
keep_count = config.keep_recent_results
for stale_dir in run_dirs[keep_count:]:
shutil.rmtree(stale_dir)
removed += 1
print(
f"[batch-train-eval] clean_result=1 path={result_dir} "
f"keep_recent_results={keep_count} removed={removed}",
flush=True,
)
def _is_default_run_dir(path: Path, config: BatchTrainEvalConfig) -> bool:
prefix = f"run_{config.domain}_"
if not path.name.startswith(prefix):
return False
suffix = path.name[len(prefix) :]
return len(suffix) == 15 and suffix[8] == "_" and suffix[:8].isdigit() and suffix[9:].isdigit()
def _run_dir_name(config: BatchTrainEvalConfig) -> str:
return f"run_{config.domain}_{config.run_timestamp}"
def _run_output_dir(config: BatchTrainEvalConfig) -> Path:
if config.output_path:
output_path = Path(config.output_path).expanduser()
return output_path.parent
return _result_base_dir(config) / _run_dir_name(config)
def _result_base_dir(config: BatchTrainEvalConfig) -> Path:
return _repo_root() / "result" / config.dataset / config.result_dir_name
def _latest_rollouts_path(config: BatchTrainEvalConfig) -> Path:
return _repo_root() / "result" / config.dataset / config.result_dir_name / "latest_rollouts"
def _repo_root() -> Path:
return Path(__file__).resolve().parents[3]