BrainAPI is a knowledge graph–powered AI memory layer that transforms unstructured data into structured knowledge, enabling intelligent search, recommendations, and contextual memory for AI agents and applications.
Python ai-agentsai-memoryknowledge-graphknowledge-managementknowledge-representationreccomendation-system
from __future__ import annotations
import json
import threading
from concurrent.futures import ThreadPoolExecutor, as_completed
from dataclasses import asdict, dataclass
from datetime import datetime, timezone
from pathlib import Path
from typing import Any
from rich.console import Console
from rich.progress import Progress
from locomo.client import BrainAPIClient
from locomo.config import Settings
from locomo.dataset import brain_id_for, iter_ingest_units
console = Console()
_write_lock = threading.Lock()
@dataclass
class IngestRecord:
sample_id: str
brain_id: str
unit_id: str
session_key: str
task_id: str | None
status: str
error: str | None
submit_latency_ms: float | None
wait_latency_ms: float | None
granularity: str
dry_run: bool
timestamp: str
cost: dict[str, Any] | None = None
total_llm_tokens: int | None = None
def _utc_now() -> str:
return datetime.now(timezone.utc).isoformat()
def load_jsonl(path: Path) -> list[dict[str, Any]]:
if not path.exists():
return []
rows = []
with path.open(encoding="utf-8") as f:
for line in f:
line = line.strip()
if line:
rows.append(json.loads(line))
return rows
def append_jsonl(path: Path, row: dict[str, Any]) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
with _write_lock:
with path.open("a", encoding="utf-8") as f:
f.write(json.dumps(row, ensure_ascii=False) + "\n")
def completed_unit_ids(ingest_path: Path) -> set[str]:
done = set()
for row in load_jsonl(ingest_path):
status = row.get("status")
if status in {"completed", "partial_failed", "dry_run"}:
key = f"{row.get('sample_id')}::{row.get('unit_id')}"
done.add(key)
return done
def ingest_samples(
settings: Settings,
samples: list[dict[str, Any]],
*,
run_dir: Path,
granularity: str = "session",
concurrency: int = 2,
limit_sessions: int | None = None,
dry_run: bool = False,
resume: bool = True,
task_timeout_s: float = 900.0,
brain_override: str | None = None,
) -> list[IngestRecord]:
run_dir.mkdir(parents=True, exist_ok=True)
ingest_path = run_dir / "ingest.jsonl"
already_done = completed_unit_ids(ingest_path) if resume else set()
if brain_override and len(samples) > 1:
raise SystemExit("--brain can only be used with a single --sample")
jobs: list[dict[str, Any]] = []
for sample in samples:
sample_id = str(sample["sample_id"])
brain_id = brain_override or brain_id_for(sample_id)
for unit in iter_ingest_units(
sample, granularity=granularity, limit_sessions=limit_sessions
):
key = f"{sample_id}::{unit['unit_id']}"
if key in already_done:
continue
jobs.append(
{
"sample_id": sample_id,
"brain_id": brain_id,
"unit": unit,
}
)
if not jobs:
console.print("[green]Nothing to ingest (all units already completed).[/green]")
return []
console.print(
f"Ingesting {len(jobs)} units "
f"(granularity={granularity}, concurrency={concurrency}, dry_run={dry_run})"
)
records: list[IngestRecord] = []
def _one(job: dict[str, Any]) -> IngestRecord:
unit = job["unit"]
if dry_run:
record = IngestRecord(
sample_id=job["sample_id"],
brain_id=job["brain_id"],
unit_id=unit["unit_id"],
session_key=unit["session_key"],
task_id=None,
status="dry_run",
error=None,
submit_latency_ms=None,
wait_latency_ms=None,
granularity=granularity,
dry_run=True,
timestamp=_utc_now(),
)
append_jsonl(ingest_path, asdict(record))
return record
with BrainAPIClient(settings) as client:
try:
submitted = client.ingest_text(
unit["text"],
job["brain_id"],
source_timestamp=unit.get("source_timestamp"),
)
task_id = (submitted.data or {}).get("task_id")
if not task_id:
raise RuntimeError(f"No task_id in ingest response: {submitted.data}")
waited = client.wait_for_task(
task_id, job["brain_id"], timeout_s=task_timeout_s
)
status = (waited.data or {}).get("status", "unknown")
error = (waited.data or {}).get("error")
cost = (waited.data or {}).get("cost")
total_llm_tokens = None
if isinstance(cost, dict):
total_llm_tokens = cost.get("total_llm_tokens")
record = IngestRecord(
sample_id=job["sample_id"],
brain_id=job["brain_id"],
unit_id=unit["unit_id"],
session_key=unit["session_key"],
task_id=task_id,
status=status,
error=error,
submit_latency_ms=submitted.latency_ms,
wait_latency_ms=waited.latency_ms,
granularity=granularity,
dry_run=False,
timestamp=_utc_now(),
cost=cost if isinstance(cost, dict) else None,
total_llm_tokens=(
int(total_llm_tokens) if total_llm_tokens is not None else None
),
)
except Exception as exc:
record = IngestRecord(
sample_id=job["sample_id"],
brain_id=job["brain_id"],
unit_id=unit["unit_id"],
session_key=unit["session_key"],
task_id=None,
status="failed",
error=str(exc),
submit_latency_ms=None,
wait_latency_ms=None,
granularity=granularity,
dry_run=False,
timestamp=_utc_now(),
)
append_jsonl(ingest_path, asdict(record))
return record
with Progress(console=console) as progress:
task = progress.add_task("ingest", total=len(jobs))
with ThreadPoolExecutor(max_workers=max(1, concurrency)) as pool:
futures = [pool.submit(_one, job) for job in jobs]
for future in as_completed(futures):
records.append(future.result())
progress.advance(task)
ok = sum(1 for r in records if r.status in {"completed", "dry_run"})
partial = sum(1 for r in records if r.status == "partial_failed")
failed = sum(1 for r in records if r.status == "failed")
console.print(
f"Ingest done: ok={ok} partial_failed={partial} failed={failed} "
f"-> {ingest_path}"
)
return records
def ensure_run_dir(settings: Settings, run_id: str | None = None) -> tuple[str, Path]:
settings.runs_dir.mkdir(parents=True, exist_ok=True)
if not run_id:
run_id = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ")
run_dir = settings.runs_dir / run_id
run_dir.mkdir(parents=True, exist_ok=True)
return run_id, run_dir
def write_manifest(run_dir: Path, manifest: dict[str, Any]) -> None:
path = run_dir / "manifest.json"
existing: dict[str, Any] = {}
if path.exists():
existing = json.loads(path.read_text(encoding="utf-8"))
existing.update(manifest)
existing["updated_at"] = _utc_now()
if "created_at" not in existing:
existing["created_at"] = _utc_now()
path.write_text(json.dumps(existing, indent=2), encoding="utf-8")