Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
# SPDX-License-Identifier: AGPL-3.0
"""``openviking-server ingest`` CLI: replay local agent-harness logs into OpenViking.
Commands:
list-sources show registered harnesses and their config
status show per-session ingest progress (read cursors)
backfill one-shot replay of existing logs
watch incremental, cursor-driven polling of new logs
run honor each harness's configured mode (backfill then watch)
"""
from __future__ import annotations
import asyncio
import signal
from pathlib import Path
from typing import List, Optional, Tuple
import typer
from openviking.ingest.cursor_store import CursorStore, SingleInstanceLock
from openviking.ingest.orchestrator import BackfillStats, IngestOrchestrator, enabled_sources
from openviking.ingest.poller import IngestPoller
from openviking.ingest.registry import SOURCE_REGISTRY
from openviking.ingest.replay import ConversationReplayClient, SessionReplayer
from openviking_cli.utils import get_logger
from openviking_cli.utils.config.consts import DEFAULT_INGEST_STATE_DIR
from openviking_cli.utils.config.ingest_config import IngestConfig
logger = get_logger(__name__)
app = typer.Typer(
add_completion=False,
help="Replay local agent-harness conversation logs into OpenViking.",
)
def _load_ingest_config() -> IngestConfig:
"""Read the ``ingest`` section of ov.conf if present; else a default (env-driven).
Only a *missing* config falls back to defaults; a malformed config surfaces its
validation/JSON error instead of silently looking like "ingest not configured".
"""
from openviking_cli.utils.config.open_viking_config import OpenVikingConfigSingleton
try:
return OpenVikingConfigSingleton.get_instance().ingest
except FileNotFoundError:
return IngestConfig()
def _state_dir(config: IngestConfig) -> Path:
return Path(config.state_dir).expanduser() if config.state_dir else DEFAULT_INGEST_STATE_DIR
def _make_store(config: IngestConfig) -> CursorStore:
return CursorStore(_state_dir(config))
def _acquire_lock(config: IngestConfig) -> SingleInstanceLock:
try:
return SingleInstanceLock(_state_dir(config)).acquire()
except RuntimeError as exc:
typer.echo(f"error: {exc}")
raise typer.Exit(1) from exc
async def _make_replayer(
config: IngestConfig, store: CursorStore
) -> Tuple[ConversationReplayClient, SessionReplayer]:
client = await ConversationReplayClient.create(
url=config.server_url,
api_key=config.api_key,
account=config.account,
user=config.user,
)
replayer = SessionReplayer(
client,
store,
session_id_prefix=config.session_id_prefix,
memory_policy=config.memory_policy or None,
)
return client, replayer
def _print_stats(results: dict[str, BackfillStats], dry_run: bool) -> None:
label = "WOULD replay" if dry_run else "Replayed"
total = BackfillStats()
for name, s in results.items():
typer.echo(
f" {name:12s}: {s.sessions} sessions, {label.lower()} {s.messages} messages, "
f"{s.committed} committed, {s.skipped} skipped"
)
for err in s.errors:
typer.echo(f" ! {err}")
total.merge(s)
typer.echo(
f"{label}: {total.sessions} sessions / {total.messages} messages / "
f"{total.committed} commits" + (f" / {len(total.errors)} errors" if total.errors else "")
)
# --------------------------------------------------------------------------- #
@app.command("list-sources")
def list_sources() -> None:
"""List registered harness adapters and their current config."""
import openviking.ingest.sources # noqa: F401 - populate the registry
config = _load_ingest_config()
typer.echo(f"ingest enabled: {config.enabled} server_url: {config.server_url or '(default)'}")
typer.echo("harnesses:")
for name in sorted(SOURCE_REGISTRY):
hc = config.harnesses.get(name)
if hc is None:
typer.echo(f" {name:12s} (not configured)")
else:
typer.echo(
f" {name:12s} enabled={hc.enabled} mode={hc.mode} paths={hc.paths or '(default)'}"
)
@app.command("status")
def status(
harness: Optional[List[str]] = typer.Option(None, "--harness", "-H", help="Filter by harness"),
) -> None:
"""Show per-session ingest progress recorded in the cursor store."""
config = _load_ingest_config()
store = _make_store(config)
try:
names = harness or None
records = []
for h in names or sorted({r.harness for r in store.all_records()}):
records.extend(store.all_records(h))
if not records:
typer.echo("No sessions ingested yet.")
return
typer.echo(f"{'harness':12s} {'session':24s} {'msgs':>6s} committed_at")
for rec in records:
typer.echo(
f"{rec.harness:12s} {rec.native_session_id[:24]:24s} "
f"{rec.last_appended_count:6d} {rec.last_committed_at or '-'}"
)
finally:
store.close()
@app.command()
def backfill(
harness: Optional[List[str]] = typer.Option(
None, "--harness", "-H", help="Only these harnesses (default: all enabled)"
),
since: Optional[str] = typer.Option(
None, "--since", help="Skip sessions started before this ISO date (e.g. 2026-06-01)"
),
dry_run: bool = typer.Option(False, "--dry-run", help="Count sessions/messages; write nothing"),
reset: bool = typer.Option(
False, "--reset", help="Delete + recreate each OV session before replaying"
),
) -> None:
"""One-shot backfill of existing logs."""
asyncio.run(_run_backfill(harness, since, dry_run, reset))
async def _run_backfill(harness, since, dry_run, reset) -> None:
config = _load_ingest_config()
store = _make_store(config)
client = None
lock = None
try:
only = harness or None
if dry_run:
# No server (and no lock) needed for a read-only dry run.
replayer = SessionReplayer(None, store, session_id_prefix=config.session_id_prefix) # type: ignore[arg-type]
orch = IngestOrchestrator(config, replayer)
results = await orch.backfill(only=only, since=since, dry_run=True)
else:
lock = _acquire_lock(config)
client, replayer = await _make_replayer(config, store)
orch = IngestOrchestrator(config, replayer)
results = await orch.backfill(only=only, since=since, dry_run=False, reset=reset)
_print_stats(results, dry_run)
finally:
if client is not None:
await client.close()
if lock is not None:
lock.release()
store.close()
@app.command()
def watch(
harness: Optional[List[str]] = typer.Option(
None, "--harness", "-H", help="Only these harnesses (default: all enabled)"
),
) -> None:
"""Incrementally watch for new/changed logs and replay them."""
asyncio.run(_run_watch(harness))
@app.command()
def run() -> None:
"""Honor each harness's configured mode: backfill (mode in backfill/both) then watch."""
asyncio.run(_run_all())
async def _run_all() -> None:
config = _load_ingest_config()
store = _make_store(config)
lock = _acquire_lock(config)
client, replayer = await _make_replayer(config, store)
try:
orch = IngestOrchestrator(config, replayer)
results = await orch.backfill(only=None, since=None, dry_run=False)
_print_stats(results, dry_run=False)
watch_sources = [
(n, c, s) for n, c, s in enabled_sources(config) if c.mode in ("watch", "both")
]
if watch_sources:
await _watch_loop(replayer, watch_sources)
finally:
await client.close()
lock.release()
store.close()
async def _run_watch(harness) -> None:
config = _load_ingest_config()
store = _make_store(config)
lock = _acquire_lock(config)
client, replayer = await _make_replayer(config, store)
try:
only = harness or None
sources = [
(n, c, s)
for n, c, s in enabled_sources(config, only)
if c.mode in ("watch", "both") or only is not None
]
if not sources:
typer.echo("No harnesses enabled for watch.")
return
await _watch_loop(replayer, sources)
finally:
await client.close()
lock.release()
store.close()
async def _watch_loop(replayer, sources) -> None:
poller = IngestPoller(sources, replayer)
loop = asyncio.get_running_loop()
for sig in (signal.SIGINT, signal.SIGTERM):
try:
loop.add_signal_handler(sig, poller.stop)
except (NotImplementedError, RuntimeError):
pass # not supported on this platform
await poller.run()
def main() -> None:
app()
if __name__ == "__main__":
main()