Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
# SPDX-License-Identifier: AGPL-3.0
"""PolicyTrainer implementation backed by OpenViking session.commit."""
from __future__ import annotations
import asyncio
import json
import time
from dataclasses import dataclass
from typing import Any
from uuid import uuid4
from openviking.session.train.components.progress import run_with_progress
from openviking.session.train.context import PipelineContext
from openviking.session.train.domain import (
CriterionResult,
ExperienceSet,
PolicyApplyResult,
PolicyUpdatePlan,
Rollout,
RolloutAnalysis,
RolloutTrainingResult,
RubricEvaluation,
)
from openviking.session.train.utils import average_score, validate_rollouts_have_cases
from openviking_cli.client.http import AsyncHTTPClient
_TRAINING_COMMIT_MEMORY_TYPES = ("cases", "trajectories", "experiences")
_TRAINING_CASE_SPEC_PROTOCOL = "openviking.batch_train.case_spec.v1"
_TRAINING_CASE_SPEC_HEADER = "# OpenViking Batch Training CaseSpec v1"
_SESSION_BATCH_ADD_MESSAGE_LIMIT = 100
@dataclass(slots=True)
class SessionCommitPolicyTrainer:
"""Train remotely by writing rollout messages to sessions and committing them."""
client: AsyncHTTPClient
run_id: str = ""
keep_recent_count: int = 0
poll_interval_seconds: float = 2.0
timeout_seconds: float | None = None
commit_concurrency: int = 20
show_progress: bool = False
progress_label: str = "session-commit"
event_recorder: Any | None = None
def __post_init__(self) -> None:
if not self.run_id:
self.run_id = _new_run_id()
if self.poll_interval_seconds <= 0:
raise ValueError("poll_interval_seconds must be > 0")
if self.timeout_seconds is not None and self.timeout_seconds <= 0:
raise ValueError("timeout_seconds must be > 0")
if self.commit_concurrency <= 0:
raise ValueError("commit_concurrency must be > 0")
async def train_rollouts(
self,
rollouts: list[Rollout],
policy_set: ExperienceSet,
context: PipelineContext | Any = None,
analyses: list[RolloutAnalysis] | None = None,
) -> RolloutTrainingResult:
rollout_list = list(rollouts)
validate_rollouts_have_cases(rollout_list)
if analyses is not None and len(analyses) != len(rollout_list):
raise ValueError(
"SessionCommitPolicyTrainer analyses length must match rollouts length when provided"
)
execution_metadata = dict(getattr(context, "execution_metadata", {}) or {})
async def _commit(rollout: Rollout, idx: int) -> dict[str, Any]:
return await self._commit_one(
rollout,
idx,
execution_metadata=execution_metadata,
)
commit_results = await run_with_progress(
rollout_list,
coroutine_factory=_commit,
label="train_start",
enabled=self.show_progress,
description=(
f"Processing {len(rollout_list)} rollouts, concurrency={self.commit_concurrency}"
),
concurrency=self.commit_concurrency,
)
analysis_list = [_analysis_from_rollout(rollout) for rollout in rollout_list]
errors = [item["error"] for item in commit_results if item.get("error")]
apply_result = PolicyApplyResult(
updated_policy_set=policy_set,
errors=errors,
metadata={
"committed_rollout_count": len(commit_results),
"commit_results": commit_results,
"run_id": self.run_id,
},
)
return RolloutTrainingResult(
analyses=analysis_list,
gradients=[],
plan=PolicyUpdatePlan(metadata={"trainer": "session_commit", "run_id": self.run_id}),
apply_result=apply_result,
metadata={
"policy_set_root_uri": policy_set.root_uri,
"rollout_count": len(rollout_list),
"analysis_count": len(analysis_list),
"gradient_count": 0,
"score": average_score(analysis_list),
"source": "session_commit_trainer",
"run_id": self.run_id,
},
)
async def _commit_one(
self,
rollout: Rollout,
index: int,
*,
execution_metadata: dict[str, Any] | None = None,
) -> dict[str, Any]:
session_id = _session_id_for_rollout(rollout, run_id=self.run_id)
stage = "prepare_messages"
try:
messages = _build_commit_messages(rollout)
stage = "create_session"
await self.client.create_session(
session_id=session_id,
options={"memory_policy": _training_commit_memory_policy()},
)
stage = "batch_add_messages"
await self._batch_add_messages(session_id, messages)
stage = "commit_session"
commit_result = await self.client.commit_session(
session_id,
telemetry=True,
keep_recent_count=self.keep_recent_count,
)
task_id = str(commit_result.get("task_id") or "")
archive_uri = str(commit_result.get("archive_uri") or "")
trace_id = _commit_trace_id(commit_result)
telemetry_id = _commit_telemetry_id(commit_result)
await self._record_event(
"train_commit_submitted",
rollout=rollout,
index=index,
session_id=session_id,
stage=stage,
execution_metadata=execution_metadata,
task_id=task_id,
archive_uri=archive_uri,
trace_id=trace_id,
telemetry_id=telemetry_id,
score=_rollout_score(rollout),
)
stage = "wait_task"
task = await self._wait_task(task_id) if task_id else None
task_error = _task_error(task)
if task_error:
print(
f"[session_commit] failed stage={stage} session_id={session_id} "
f"task_id={task_id} trace_id={trace_id or '<none>'} "
f"error={task_error}",
flush=True,
)
await self._record_event(
"train_commit_failed" if task_error else "train_commit_done",
rollout=rollout,
index=index,
session_id=session_id,
stage=stage,
execution_metadata=execution_metadata,
task_id=task_id,
archive_uri=archive_uri,
trace_id=trace_id,
telemetry_id=telemetry_id,
task_status=task.get("status") if isinstance(task, dict) else None,
score=_rollout_score(rollout),
error=task_error,
)
return {
"index": index,
"session_id": session_id,
"stage": stage,
"task_id": task_id,
"archive_uri": archive_uri,
"trace_id": trace_id,
"telemetry_id": telemetry_id,
"task_status": task.get("status") if isinstance(task, dict) else None,
"score": _rollout_score(rollout),
"error": task_error,
}
except Exception as exc:
print(
f"[session_commit] failed stage={stage} session_id={session_id} "
f"task_id=<none> trace_id=<none> error={exc}",
flush=True,
)
await self._record_event(
"train_commit_failed",
rollout=rollout,
index=index,
session_id=session_id,
stage=stage,
execution_metadata=execution_metadata,
task_id="",
archive_uri="",
trace_id=None,
telemetry_id=None,
task_status="failed",
score=_rollout_score(rollout),
error=str(exc),
)
return {
"index": index,
"session_id": session_id,
"stage": stage,
"task_id": "",
"archive_uri": "",
"trace_id": None,
"telemetry_id": None,
"task_status": "failed",
"score": _rollout_score(rollout),
"error": str(exc),
}
async def _record_event(
self,
event: str,
*,
rollout: Rollout,
index: int,
session_id: str,
stage: str,
execution_metadata: dict[str, Any] | None = None,
**fields: Any,
) -> None:
if self.event_recorder is None:
return
record = getattr(self.event_recorder, "record", None)
if record is None:
return
payload = {
"index": index,
"stage": stage,
"session_id": session_id,
**_rollout_event_fields(
rollout,
execution_metadata=execution_metadata,
),
**fields,
}
result = record(event, **payload)
if asyncio.iscoroutine(result):
await result
async def _batch_add_messages(self, session_id: str, messages: list[dict[str, Any]]) -> None:
for start in range(0, len(messages), _SESSION_BATCH_ADD_MESSAGE_LIMIT):
await self.client.batch_add_messages(
session_id, messages[start : start + _SESSION_BATCH_ADD_MESSAGE_LIMIT]
)
async def _wait_task(self, task_id: str) -> dict[str, Any]:
deadline = (
asyncio.get_running_loop().time() + self.timeout_seconds
if self.timeout_seconds is not None
else None
)
while True:
task = await self.client.get_task(task_id)
if task and task.get("status") in {"completed", "failed", "cancelled"}:
return task
if deadline is not None and asyncio.get_running_loop().time() >= deadline:
return {"task_id": task_id, "status": "timeout", "error": "commit task timeout"}
await asyncio.sleep(self.poll_interval_seconds)
def _rollout_event_fields(
rollout: Rollout,
*,
execution_metadata: dict[str, Any] | None = None,
) -> dict[str, Any]:
case = rollout.case
metadata = rollout.metadata or {}
rollout_execution_metadata = metadata.get("execution_metadata", {})
if not isinstance(rollout_execution_metadata, dict):
rollout_execution_metadata = {}
event_execution_metadata = dict(rollout_execution_metadata)
event_execution_metadata.update(execution_metadata or {})
case_input = case.input or {}
return {
"epoch": event_execution_metadata.get("epoch"),
"training": event_execution_metadata.get("training"),
"rollout_stage": event_execution_metadata.get("rollout_stage")
or event_execution_metadata.get("stage"),
"case_name": case.name,
"task_signature": case.task_signature,
"split": (
case_input.get("data_split")
or metadata.get("data_split")
or case_input.get("split")
or metadata.get("split")
),
"task_no": (
case_input.get("task_no")
if case_input.get("task_no") is not None
else metadata.get("task_no")
),
"case_task_id": case_input.get("task_id") or metadata.get("task_id"),
"task_id": case_input.get("task_id") or metadata.get("task_id"),
"policy_snapshot_id": rollout.policy_snapshot_id,
"passed": bool(rollout.evaluation.passed) if rollout.evaluation is not None else None,
}
def _training_commit_memory_policy() -> dict[str, Any]:
return {
"memory_types": list(_TRAINING_COMMIT_MEMORY_TYPES),
"working_memory": {"enabled": False},
}
def _analysis_from_rollout(rollout: Rollout) -> RolloutAnalysis:
return RolloutAnalysis(
evaluation=_rollout_evaluation_or_default(rollout),
trajectories=[],
metadata={
"rollout": rollout,
"rollout_messages": rollout.messages,
"policy_snapshot_id": rollout.policy_snapshot_id,
"evaluation_source": "rollout"
if rollout.evaluation is not None
else "session_commit_default",
},
)
def _rollout_evaluation_or_default(rollout: Rollout) -> RubricEvaluation:
if rollout.evaluation is not None:
return rollout.evaluation
return RubricEvaluation(
passed=False,
score=0.0,
criterion_results=[
CriterionResult(
criterion_name="rollout_evaluation_provided",
passed=False,
score=0.0,
feedback=["Rollout executor did not provide evaluation."],
evidence=[],
metadata={"source": "session_commit_default"},
)
],
feedback=["Rollout executor did not provide evaluation."],
metadata={"source": "session_commit_default"},
)
def _rollout_score(rollout: Rollout) -> float:
if rollout.evaluation is None:
return 0.0
return float(rollout.evaluation.score)
def _task_error(task: dict[str, Any] | None) -> str | None:
if task is None:
return None
if task.get("status") == "failed":
return str(task.get("error") or "task failed")
if task.get("status") == "cancelled":
return str(task.get("error") or "task cancelled")
if task.get("status") == "timeout":
return str(task.get("error") or "task timeout")
return None
def _commit_trace_id(commit_result: dict[str, Any]) -> str | None:
trace_id = commit_result.get("trace_id")
return str(trace_id) if trace_id else None
def _commit_telemetry_id(commit_result: dict[str, Any]) -> str | None:
telemetry = commit_result.get("telemetry")
if not isinstance(telemetry, dict):
return None
telemetry_id = telemetry.get("id")
return str(telemetry_id) if telemetry_id else None
def _session_id_for_rollout(rollout: Rollout, *, run_id: str) -> str:
safe_name = _safe_session_fragment(rollout.case.name)
metadata = rollout.metadata or {}
execution_metadata = metadata.get("execution_metadata", {})
epoch = execution_metadata.get("epoch", "0")
task_no = metadata.get("task_no", "0")
split = metadata.get("data_split", "tau2")
return f"tau2_train_{run_id}_{split}_e{epoch}_t{task_no}_{safe_name}"
def _safe_session_fragment(value: str) -> str:
return "".join(ch if ch.isalnum() or ch in "_.-" else "_" for ch in value)[:80] or "case"
def _new_run_id() -> str:
return f"{int(time.time())}_{uuid4().hex[:8]}"
def _case_spec_message_to_request(rollout: Rollout) -> dict[str, Any]:
text = (
f"{_TRAINING_CASE_SPEC_HEADER}\n\n"
"The following structured case and rubric describe the task that "
"produced this rollout. It is control-plane metadata for the "
"batch training pipeline.\n\n"
f"```json\n{_case_spec_payload_json(rollout)}\n```"
)
return {
"role": "system",
"parts": [{"type": "text", "text": text}],
}
def _case_spec_payload_json(rollout: Rollout) -> str:
import json
return json.dumps(_case_spec_payload(rollout), ensure_ascii=False, indent=2, sort_keys=True)
def _case_spec_payload(rollout: Rollout) -> dict[str, Any]:
case = rollout.case
return {
"protocol": _TRAINING_CASE_SPEC_PROTOCOL,
"case": {
"name": _stable_case_name(rollout),
"task_signature": _stable_task_signature(rollout),
"input": _case_input_payload(case.input),
"metadata": _stable_case_metadata(rollout),
"rubric": {
"name": case.rubric.name,
"description": case.rubric.description,
"criteria": [
{
"name": criterion.name,
"description": criterion.description,
"required": criterion.required,
"weight": criterion.weight,
}
for criterion in case.rubric.criteria
],
},
},
}
def _stable_case_name(rollout: Rollout) -> str:
case = rollout.case
return str(
case.input.get("original_case_name")
or case.metadata.get("original_case_name")
or rollout.metadata.get("original_case_name")
or case.name
)
def _stable_task_signature(rollout: Rollout) -> str:
case = rollout.case
if case.input.get("original_case_name") or case.metadata.get("original_case_name"):
return str(case.task_signature).split(":trial:", 1)[0]
return case.task_signature
def _stable_case_metadata(rollout: Rollout) -> dict[str, Any]:
metadata = dict(rollout.case.metadata or {})
metadata.setdefault("rollout_case_name", rollout.case.name)
metadata.setdefault("rollout_task_signature", rollout.case.task_signature)
return metadata
def _case_input_payload(case_input: dict[str, Any]) -> dict[str, Any]:
allowed_keys = (
"domain",
"split",
"data_split",
"task_id",
"task_no",
"user_query",
)
return {key: case_input[key] for key in allowed_keys if key in case_input}
def _build_commit_messages(rollout: Rollout) -> list[dict[str, Any]]:
"""Build the extraction input without changing the original rollout/evaluation."""
return (
[_case_spec_message_to_request(rollout)]
+ [
_message_to_request(message)
for message in rollout.messages
if not _is_duplicate_tau2_reward_message(message, rollout.evaluation)
]
+ [_evaluation_message_to_request(rollout)]
)
def _is_evaluation_report_copy(text: str, report: Any) -> bool:
"""Match full JSON report copies, not additional feedback or partial evidence."""
if not isinstance(report, (dict, list)):
return False
try:
parsed = json.loads(text)
except (TypeError, ValueError):
return False
# Ignore JSON whitespace/key order, but do not conflate booleans with numbers.
return json.dumps(parsed, sort_keys=True) == json.dumps(report, sort_keys=True)
def _without_evaluation_report_copies(values: list[str], report: Any) -> list[str]:
return [value for value in values if not _is_evaluation_report_copy(value, report)]
def _is_duplicate_tau2_reward_message(message: Any, evaluation: RubricEvaluation | None) -> bool:
"""Drop only the known Tau2 wrapper when every fact is in OutcomeEvaluation."""
if (
evaluation is None
or getattr(message, "id", "") != "tau2-reward"
or message.role != "user"
or len(message.parts) != 1
or getattr(message.parts[0], "type", None) != "text"
):
return False
lines = message.parts[0].text.split("\n", 2)
if (
len(lines) != 3
or lines[0] != f"task_success: {evaluation.passed}"
or not lines[1].startswith("task_reward: ")
or not lines[2].startswith("evaluation report: ")
):
return False
try:
reward = json.loads(lines[1].removeprefix("task_reward: "))
except ValueError:
return False
return (
isinstance(reward, (int, float))
and not isinstance(reward, bool)
and reward == evaluation.score
and _is_evaluation_report_copy(
lines[2].removeprefix("evaluation report: "),
evaluation.metadata.get("evaluation_result"),
)
)
def _evaluation_message_to_request(rollout: Rollout) -> dict[str, Any]:
text = (
"# OpenViking OutcomeEvaluation\n\n"
"The following structured evaluation describes the outcome of the "
"preceding rollout. Use it as the training signal when extracting "
"training memories.\n\n"
f"```json\n{_evaluation_payload_json(rollout)}\n```"
)
return {
"role": "user",
"parts": [{"type": "text", "text": text}],
}
def _evaluation_payload_json(rollout: Rollout) -> str:
return json.dumps(
{"evaluation": _evaluation_payload(rollout.evaluation)},
ensure_ascii=False,
indent=2,
sort_keys=True,
)
def _evaluation_payload(evaluation: RubricEvaluation | None) -> dict[str, Any] | None:
if evaluation is None:
return None
report = evaluation.metadata.get("evaluation_result")
criteria = []
for result in evaluation.criterion_results:
evidence = _without_evaluation_report_copies(result.evidence, report)
if len(evidence) < len(result.evidence):
evidence.append("See evaluation.metadata.evaluation_result.")
criteria.append(
{
"criterion_name": result.criterion_name,
"passed": result.passed,
"score": result.score,
"feedback": _without_evaluation_report_copies(result.feedback, report),
"evidence": evidence,
"metadata": result.metadata,
}
)
return {
"passed": evaluation.passed,
"score": evaluation.score,
"feedback": _without_evaluation_report_copies(evaluation.feedback, report),
"criterion_results": criteria,
"metadata": evaluation.metadata,
}
def _message_to_request(message: Any) -> dict[str, Any]:
data = message.to_dict()
request = {
"role": data["role"],
"parts": data.get("parts", []),
"created_at": data.get("created_at"),
}
if data.get("peer_id") is not None:
request["peer_id"] = data["peer_id"]
return request