BrainAPI is a knowledge graph–powered AI memory layer that transforms unstructured data into structured knowledge, enabling intelligent search, recommendations, and contextual memory for AI agents and applications.
Python ai-agentsai-memoryknowledge-graphknowledge-managementknowledge-representationreccomendation-system
chore(benchmarks): add BEAM 1M ingest/eval ops scripts
Christianb1af5b1caaa8
diff --git a/benchmarks/scripts/beam_1m_keepalive.sh b/benchmarks/scripts/beam_1m_keepalive.shnew file mode 100755index 0000000..722a922--- /dev/null+++ b/benchmarks/scripts/beam_1m_keepalive.sh@@ -0,0 +1,54 @@+#!/usr/bin/env bash+# Tiny keep-alive: only ensures API LISTEN + celery worker. Never pkill.+# screen -dmS beam1m-keep bash benchmarks/scripts/beam_1m_keepalive.sh+#+# Safe under concurrent ingest: flock around starts; longer /docs probe.+# Does not restart a healthy LISTEN uvicorn (avoids killing live eval).+set +e+SRC=/Users/christiannonis/.brainapi/source+UV_LOG=/tmp/uvicorn-beam1m.log+CEL_LOG=/tmp/brainapi-beam-worker.log+LOCK_DIR=/tmp/beam1m-locks+mkdir -p "$LOCK_DIR"+CELERY_WORKER_CONCURRENCY="${CELERY_WORKER_CONCURRENCY:-4}"++echo "KEEPALIVE_START $(date -u +%Y-%m-%dT%H:%M:%SZ)" >> /tmp/beam-1m-keepalive.log+while [[ ! -f /tmp/beam-1m-1.STOP ]]; do+ if ! lsof -iTCP:8000 -sTCP:LISTEN >/dev/null 2>&1; then+ echo "API_RESTART $(date -u +%H:%M:%S)" >> /tmp/beam-1m-keepalive.log+ (+ flock -n 9 || exit 0+ if lsof -iTCP:8000 -sTCP:LISTEN >/dev/null 2>&1; then+ exit 0+ fi+ nohup bash -c "cd '$SRC' && exec ./.venv/bin/uvicorn src.services.api.app:app --host 127.0.0.1 --port 8000" \+ >> "$UV_LOG" 2>&1 &+ echo $! > /tmp/beam1m-uvicorn.pid+ sleep 5+ ) 9>"$LOCK_DIR/uvicorn.lock"+ else+ # Soft health: log only; do not restart if LISTEN is up (eval-safe).+ code=$(curl -s -o /dev/null --max-time 5 -w '%{http_code}' http://127.0.0.1:8000/docs || echo 000)+ if [[ "$code" != "200" && "$code" != "401" ]]; then+ echo "API_BUSY_OR_SLOW code=$code $(date -u +%H:%M:%S)" >> /tmp/beam-1m-keepalive.log+ fi+ fi+ # Match real worker process only (python .../celery), not bash scripts.+ if ! pgrep -f '/.venv/bin/celery -A src.workers.app worker' >/dev/null 2>&1; then+ echo "CELERY_RESTART $(date -u +%H:%M:%S)" >> /tmp/beam-1m-keepalive.log+ (+ flock -n 9 || exit 0+ if pgrep -f '/.venv/bin/celery -A src.workers.app worker' >/dev/null 2>&1; then+ exit 0+ fi+ nohup bash -c "cd '$SRC' && exec ./.venv/bin/celery -A src.workers.app worker --loglevel=info --pool=threads \+ --concurrency=${CELERY_WORKER_CONCURRENCY} \+ -Q ingest_data,finalize_ingestion,process_architect_relationships,consolidate_graph,chatbot_memory,ingest_file,ingest_structured_data" \+ >> "$CEL_LOG" 2>&1 &+ echo $! > /tmp/beam1m-celery.pid+ sleep 5+ ) 9>"$LOCK_DIR/celery.lock"+ fi+ sleep 20+done+echo "KEEPALIVE_STOP $(date -u +%Y-%m-%dT%H:%M:%SZ)" >> /tmp/beam-1m-keepalive.logdiff --git a/benchmarks/scripts/boot_beam_1m_screen.sh b/benchmarks/scripts/boot_beam_1m_screen.shnew file mode 100755index 0000000..0f43118--- /dev/null+++ b/benchmarks/scripts/boot_beam_1m_screen.sh@@ -0,0 +1,35 @@+#!/usr/bin/env bash+# Boot BEAM 1M ingest inside screen. Keep this file free of long uvicorn/celery+# command strings in the screen argv — use this script path only:+# screen -dmS beam1m /path/to/boot_beam_1m_screen.sh+set +e+SRC=/Users/christiannonis/.brainapi/source+UV_LOG=/tmp/uvicorn-beam1m.log+CEL_LOG=/tmp/brainapi-beam-worker.log+UV_PID=/tmp/beam1m-uvicorn.pid+CEL_PID=/tmp/beam1m-celery.pid++start_api() {+ if lsof -iTCP:8000 -sTCP:LISTEN >/dev/null 2>&1; then+ return 0+ fi+ nohup bash -c "cd '$SRC' && exec ./.venv/bin/uvicorn src.services.api.app:app --host 127.0.0.1 --port 8000" \+ >> "$UV_LOG" 2>&1 &+ echo $! > "$UV_PID"+ sleep 5+}++start_worker() {+ if pgrep -f 'celery -A src.workers.app worker' >/dev/null 2>&1; then+ return 0+ fi+ CELERY_C="${CELERY_WORKER_CONCURRENCY:-4}"+ nohup bash -c "cd '$SRC' && exec ./.venv/bin/celery -A src.workers.app worker --loglevel=info --pool=threads --concurrency=${CELERY_C} -Q ingest_data,finalize_ingestion,process_architect_relationships,consolidate_graph,chatbot_memory,ingest_file,ingest_structured_data" \+ >> "$CEL_LOG" 2>&1 &+ echo $! > "$CEL_PID"+ sleep 5+}++start_api+start_worker+exec /tmp/run_beam_1m_1_ingest_eval.sh >> /tmp/beam-1m-1-run.log 2>&1diff --git a/benchmarks/scripts/eval_beam_1m_1_product.sh b/benchmarks/scripts/eval_beam_1m_1_product.shnew file mode 100644index 0000000..3e62f7a--- /dev/null+++ b/benchmarks/scripts/eval_beam_1m_1_product.sh@@ -0,0 +1,91 @@+#!/usr/bin/env bash+# Evaluate-only BEAM 1M/1 product on existing brain (no ingest / no wipe).+set +e+cd /Users/christiannonis/Documents/Projects/brainapi2/benchmarks+unset BRAINPAT_TOKEN BRAINAPI_TOKEN+set -a; source ./.env; set +a+export TERM=dumb PYTHONUNBUFFERED=1++SRC=/Users/christiannonis/.brainapi/source+UV_LOG=/tmp/uvicorn-beam1m.log+CEL_LOG=/tmp/brainapi-beam-worker.log++ensure_api() {+ for _ in 1 2 3 4 5 6 7 8; do+ if lsof -iTCP:8000 -sTCP:LISTEN >/dev/null 2>&1; then+ code=$(curl -s -o /dev/null --max-time 2 -w '%{http_code}' http://127.0.0.1:8000/docs || echo 000)+ if [[ "$code" == "200" || "$code" == "401" ]]; then+ if ! pgrep -f '/.venv/bin/celery -A src.workers.app worker' >/dev/null 2>&1; then+ nohup bash -c "cd '$SRC' && exec ./.venv/bin/celery -A src.workers.app worker --loglevel=info --pool=threads -Q ingest_data,finalize_ingestion,process_architect_relationships,consolidate_graph,chatbot_memory,ingest_file,ingest_structured_data" \+ >> "$CEL_LOG" 2>&1 &+ sleep 4+ fi+ return 0+ fi+ fi+ if ! lsof -iTCP:8000 -sTCP:LISTEN >/dev/null 2>&1; then+ nohup bash -c "cd '$SRC' && exec ./.venv/bin/uvicorn src.services.api.app:app --host 127.0.0.1 --port 8000" \+ >> "$UV_LOG" 2>&1 &+ sleep 5+ fi+ done+ return 1+}++echo "START_1M_EVAL $(date -u +%Y-%m-%dT%H:%M:%SZ)"+ensure_api || { echo FATAL_API_DOWN; exit 1; }++for ab in \+ abstention \+ contradiction_resolution \+ event_ordering \+ information_extraction \+ instruction_following \+ knowledge_update \+ multi_session_reasoning \+ preference_following \+ summarization \+ temporal_reasoning+do+ echo "==== EVAL $ab $(date -u +%H:%M:%S) ===="+ ensure_api+ ./beam.sh evaluate \+ --size 1M --sample 1 \+ --run beam-1m-1-clean-product \+ --brain beam1m1clean \+ --profile product \+ --abilities "$ab" \+ --concurrency 1 \+ --historical-limit 12 --max-passages 12 --max-facts 40 \+ --no-ppr+done++./beam.sh report --run beam-1m-1-clean-product+.venv/bin/python - <<'PY'+import json, statistics+from pathlib import Path+rows=[json.loads(l) for l in Path('runs/beam-1m-1-clean/ingest.jsonl').read_text().splitlines() if l.strip()]+last={}+for r in rows: last[r.get('unit_id')]=r+comp=[r for r in last.values() if r.get('status')=='completed' and (r.get('cost') or {}).get('llm_source_multiplier') is not None]+mult=[r['cost']['llm_source_multiplier'] for r in comp]+print('INGEST completed', sum(1 for r in last.values() if r.get('status')=='completed'), '/', len(last))+print('PERMANENT_FAILS', sorted(+ uid for uid,r in last.items()+ if r.get('status')=='permanent_failed'+ or (r.get('status')=='failed' and '8192' in (r.get('error') or ''))+))+print('MULTIPLIER n', len(mult),+ 'median', round(statistics.median(mult),1) if mult else None,+ 'mean', round(statistics.mean(mult),1) if mult else None)+rep_path=Path('runs/beam-1m-1-clean-product/report.json')+if rep_path.exists():+ rep=json.loads(rep_path.read_text())+ m=rep.get('metrics') or {}+ hs=m.get('headline_score')+ print('HEADLINE', hs, f"({(hs or 0)*100:.1f}%)" if isinstance(hs,(int,float)) else '')+ print('PER', m.get('per_ability'))+ print('GOAL_B_65', 'MET' if (hs or 0) >= 0.65 else 'MISS')+PY+date -u +%Y-%m-%dT%H:%M:%SZ > /tmp/beam-1m-1.DONE+echo EVAL_DONEdiff --git a/benchmarks/scripts/run_beam_1m_1_ingest_eval.sh b/benchmarks/scripts/run_beam_1m_1_ingest_eval.shnew file mode 100755index 0000000..ccf09c2--- /dev/null+++ b/benchmarks/scripts/run_beam_1m_1_ingest_eval.sh@@ -0,0 +1,220 @@+#!/usr/bin/env bash+# Resilient BEAM 1M/1 ingest + product evaluate.+# Resume-safe. LISTEN-based ensure_api (never pkill healthy uvicorn).+# Uses flock so watchdog/ensure never double-bind :8000.+#+# Parallel ingest (next re-ingest — do NOT wipe live eval brains casually):+# BEAM_INGEST_CONCURRENCY=2|3 (default 2; ≤4; need CELERY_WORKER_CONCURRENCY ≥ N)+# Observed wall ~40h @ concurrency 1 for ~625 turns → ~20–25h @2, ~14–18h @3.+set +e+cd /Users/christiannonis/Documents/Projects/brainapi2/benchmarks+unset BRAINPAT_TOKEN BRAINAPI_TOKEN+set -a; source ./.env; set +a+export TERM=dumb PYTHONUNBUFFERED=1++SRC=/Users/christiannonis/.brainapi/source+UV_LOG=/tmp/uvicorn-beam1m.log+CEL_LOG=/tmp/brainapi-beam-worker.log+LOCK_DIR=/tmp/beam1m-locks+mkdir -p "$LOCK_DIR"++# Harness parallel units. Keep ≤ CELERY_WORKER_CONCURRENCY (product .env default 4).+BEAM_INGEST_CONCURRENCY="${BEAM_INGEST_CONCURRENCY:-2}"+BEAM_INGEST_TIMEOUT="${BEAM_INGEST_TIMEOUT:-2400}"+CELERY_WORKER_CONCURRENCY="${CELERY_WORKER_CONCURRENCY:-4}"++api_ok() {+ lsof -iTCP:8000 -sTCP:LISTEN >/dev/null 2>&1 || return 1+ # Longer timeout under concurrent ingest poll load.+ code=$(curl -s -o /dev/null --max-time 5 -w '%{http_code}' http://127.0.0.1:8000/docs || echo 000)+ [[ "$code" == "200" || "$code" == "401" ]]+}++start_uvicorn() {+ (+ flock -n 9 || return 0+ if lsof -iTCP:8000 -sTCP:LISTEN >/dev/null 2>&1; then+ return 0+ fi+ nohup bash -c "cd '$SRC' && exec ./.venv/bin/uvicorn src.services.api.app:app --host 127.0.0.1 --port 8000" \+ >> "$UV_LOG" 2>&1 &+ echo $! > /tmp/beam1m-uvicorn.pid+ sleep 5+ ) 9>"$LOCK_DIR/uvicorn.lock"+}++start_celery() {+ (+ flock -n 9 || return 0+ # Match the real worker binary, not bash wrappers that mention celery in -c strings.+ if pgrep -f '/.venv/bin/celery -A src.workers.app worker' >/dev/null 2>&1; then+ return 0+ fi+ nohup bash -c "cd '$SRC' && exec ./.venv/bin/celery -A src.workers.app worker --loglevel=info --pool=threads \+ --concurrency=${CELERY_WORKER_CONCURRENCY} \+ -Q ingest_data,finalize_ingestion,process_architect_relationships,consolidate_graph,chatbot_memory,ingest_file,ingest_structured_data" \+ >> "$CEL_LOG" 2>&1 &+ echo $! > /tmp/beam1m-celery.pid+ sleep 5+ ) 9>"$LOCK_DIR/celery.lock"+}++ensure_api() {+ for _ in 1 2 3 4 5 6 7 8 9 10 11 12; do+ if api_ok; then+ start_celery+ return 0+ fi+ start_uvicorn+ start_celery+ sleep 3+ done+ return 1+}++# Terminal units for stall/target: completed + permanent_failed (+ legacy embed-8192 failed).+terminal_count() {+ .venv/bin/python - <<'PY'+import json+import re+from pathlib import Path+from collections import Counter+p = Path("runs/beam-1m-1-clean/ingest.jsonl")+rows = [json.loads(l) for l in p.read_text().splitlines() if l.strip()] if p.exists() else []+last = {}+for r in rows:+ last[r.get("unit_id")] = r+pat = re.compile(r"maximum context length is 8192", re.I)+n = 0+for r in last.values():+ st = r.get("status")+ err = r.get("error") or ""+ if st in {"completed", "partial_failed", "permanent_failed"}:+ n += 1+ elif st == "failed" and pat.search(err):+ n += 1+print(n)+PY+}++completed_count() {+ terminal_count+}++watchdog() {+ while [[ ! -f /tmp/beam-1m-1.STOP ]]; do+ if ! lsof -iTCP:8000 -sTCP:LISTEN >/dev/null 2>&1; then+ echo "WATCHDOG_API_RESTART $(date -u +%H:%M:%S)"+ start_uvicorn+ fi+ if ! pgrep -f '/.venv/bin/celery -A src.workers.app worker' >/dev/null 2>&1; then+ echo "WATCHDOG_CELERY_RESTART $(date -u +%H:%M:%S)"+ start_celery+ fi+ sleep 30+ done+}++# Accept near-complete ingest: a few units permanently fail (embed 8192 context).+TARGET=620+STALL_LIMIT=3+prev_n=-1+stalls=0+rm -f /tmp/beam-1m-1.STOP /tmp/beam-1m-1.DONE+echo "START_1M_INGEST $(date -u +%Y-%m-%dT%H:%M:%SZ) concurrency=$BEAM_INGEST_CONCURRENCY celery_c=$CELERY_WORKER_CONCURRENCY"+ensure_api || { echo FATAL_API_DOWN; exit 1; }+watchdog &+WD_PID=$!+echo "watchdog_pid=$WD_PID"++for try in $(seq 1 300); do+ ensure_api || { echo API_DOWN; sleep 20; continue; }+ done_n=$(completed_count)+ echo "try=$try terminal=$done_n $(date -u +%H:%M:%S)"+ if [[ "$done_n" -ge "$TARGET" ]]; then+ break+ fi+ if [[ "$done_n" -eq "$prev_n" ]]; then+ stalls=$((stalls + 1))+ if [[ "$stalls" -ge "$STALL_LIMIT" ]]; then+ echo "INGEST_STALLED terminal=$done_n (no progress ${stalls} tries) — proceed to eval"+ break+ fi+ else+ stalls=0+ fi+ prev_n=$done_n+ ./beam.sh ingest \+ --size 1M --sample 1 \+ --run beam-1m-1-clean \+ --brain beam1m1clean \+ --concurrency "$BEAM_INGEST_CONCURRENCY" \+ --timeout "$BEAM_INGEST_TIMEOUT"+ sleep 2+done++done_n=$(completed_count)+echo "INGEST_DONE=$done_n"+# Do not STOP keepalive globally; only stop this script's watchdog.+kill "$WD_PID" 2>/dev/null+wait "$WD_PID" 2>/dev/null++if [[ "$done_n" -lt 500 ]]; then+ echo "INGEST_INCOMPLETE abort eval ($done_n/625)"+ date -u +%Y-%m-%dT%H:%M:%SZ > /tmp/beam-1m-1.DONE+ exit 2+fi++# Portable loop (bash array with multiline words crashed as commands in prior run).+for ab in \+ abstention \+ contradiction_resolution \+ event_ordering \+ information_extraction \+ instruction_following \+ knowledge_update \+ multi_session_reasoning \+ preference_following \+ summarization \+ temporal_reasoning+do+ echo "==== EVAL $ab $(date -u +%H:%M:%S) ===="+ ensure_api+ ./beam.sh evaluate \+ --size 1M --sample 1 \+ --run beam-1m-1-clean-product \+ --brain beam1m1clean \+ --profile product \+ --abilities "$ab" \+ --concurrency 1 \+ --historical-limit 12 --max-passages 12 --max-facts 40 \+ --no-ppr+done+./beam.sh report --run beam-1m-1-clean-product+.venv/bin/python - <<'PY'+import json, statistics, re+from pathlib import Path+rows=[json.loads(l) for l in Path('runs/beam-1m-1-clean/ingest.jsonl').read_text().splitlines() if l.strip()]+last={}+for r in rows: last[r.get('unit_id')]=r+pat=re.compile(r'maximum context length is 8192', re.I)+comp=[r for r in last.values() if r.get('status')=='completed' and (r.get('cost') or {}).get('llm_source_multiplier') is not None]+mult=[r['cost']['llm_source_multiplier'] for r in comp]+perm=sorted(+ uid for uid,r in last.items()+ if r.get('status')=='permanent_failed'+ or (r.get('status')=='failed' and pat.search(r.get('error') or ''))+)+print('MULTIPLIER n', len(mult),+ 'median', round(statistics.median(mult),1) if mult else None,+ 'mean', round(statistics.mean(mult),1) if mult else None)+print('PERMANENT_FAILS', perm)+rep_path=Path('runs/beam-1m-1-clean-product/report.json')+if rep_path.exists():+ rep=json.loads(rep_path.read_text())+ m=rep.get('metrics') or {}+ print('HEADLINE', m.get('headline_score'))+ print('PER', m.get('per_ability'))+PY+date -u +%Y-%m-%dT%H:%M:%SZ > /tmp/beam-1m-1.DONE+echo MULTI_DONEdiff --git a/benchmarks/scripts/run_beam_chats_2_3.sh b/benchmarks/scripts/run_beam_chats_2_3.shnew file mode 100755index 0000000..d1ea83d--- /dev/null+++ b/benchmarks/scripts/run_beam_chats_2_3.sh@@ -0,0 +1,109 @@+#!/usr/bin/env bash+set +e+cd /Users/christiannonis/Documents/Projects/brainapi2/benchmarks+unset BRAINPAT_TOKEN BRAINAPI_TOKEN+set -a; source ./.env; set +a+export TERM=dumb PYTHONUNBUFFERED=1++ensure_api() {+ for _ in 1 2 3 4 5 6; do+ if curl -s -o /dev/null --max-time 2 http://127.0.0.1:8000/docs >/dev/null; then+ return 0+ fi+ if ! lsof -iTCP:8000 -sTCP:LISTEN >/dev/null 2>&1; then+ (+ cd /Users/christiannonis/.brainapi/source+ ./.venv/bin/uvicorn src.services.api.app:app --host 127.0.0.1 --port 8000 >> /tmp/uvicorn-stable.log 2>&1 &+ )+ fi+ if ! pgrep -f 'celery -A src.workers.app worker' >/dev/null; then+ (+ cd /Users/christiannonis/.brainapi/source+ ./.venv/bin/celery -A src.workers.app worker --loglevel=info --pool=threads \+ -Q ingest_data,finalize_ingestion,process_architect_relationships,consolidate_graph,chatbot_memory,ingest_file,ingest_structured_data \+ >> /tmp/brainapi-beam-worker.log 2>&1 &+ )+ fi+ sleep 5+ done+}++completed_count() {+ local sample=$1+ .venv/bin/python - "$sample" <<'PY'+import json, sys+from pathlib import Path+from collections import Counter+sample = sys.argv[1]+p = Path(f"runs/beam-100k-{sample}-clean/ingest.jsonl")+rows = [json.loads(l) for l in p.read_text().splitlines() if l.strip()] if p.exists() else []+last = {}+for r in rows:+ last[r.get("unit_id")] = r.get("status")+print(Counter(last.values()).get("completed", 0))+PY+}++for sample in 2 3; do+ echo "==== CHAT $sample ===="+ for try in $(seq 1 30); do+ ensure_api+ done_n=$(completed_count "$sample")+ echo "try=$try completed=$done_n $(date -u +%H:%M:%S)"+ if [[ "$done_n" -ge 70 ]]; then+ break+ fi+ ./beam.sh ingest \+ --size 100K --sample "$sample" \+ --run "beam-100k-${sample}-clean" \+ --brain "beam100k${sample}clean" \+ --concurrency "${BEAM_INGEST_CONCURRENCY:-2}" \+ --timeout 1200+ done+ done_n=$(completed_count "$sample")+ echo "INGEST${sample}_DONE=$done_n"+ if [[ "$done_n" -lt 50 ]]; then+ echo "skip eval for $sample (ingest incomplete)"+ continue+ fi+ abilities=(+ abstention contradiction_resolution event_ordering information_extraction+ instruction_following knowledge_update multi_session_reasoning+ preference_following summarization temporal_reasoning+ )+ for ab in "${abilities[@]}"; do+ ensure_api+ ./beam.sh evaluate \+ --size 100K --sample "$sample" \+ --run "beam-100k-${sample}-clean-product" \+ --brain "beam100k${sample}clean" \+ --profile product \+ --abilities "$ab" \+ --concurrency 1 \+ --historical-limit 12 --max-passages 12 --max-facts 40 \+ --no-ppr+ ensure_api+ ./beam.sh evaluate \+ --size 100K --sample "$sample" \+ --run "beam-100k-${sample}-clean-product" \+ --brain "beam100k${sample}clean" \+ --profile product \+ --abilities "$ab" \+ --concurrency 1 \+ --historical-limit 12 --max-passages 12 --max-facts 40 \+ --no-ppr+ done+ .venv/bin/python - <<PY+from pathlib import Path+from beam.report import write_report+r = write_report(Path("runs/beam-100k-${sample}-clean-product"))+print(+ "CHAT${sample}_HEADLINE",+ (r.get("metrics") or {}).get("headline_score"),+ "n",+ (r.get("metrics") or {}).get("n_questions"),+)+PY+done+echo MULTI_DONE+date -u +%Y-%m-%dT%H:%M:%SZ > /tmp/beam-100k-23-done.DONE