Skip to content

BrainAPI is a knowledge graph–powered AI memory layer that transforms unstructured data into structured knowledge, enabling intelligent search, recommendations, and contextual memory for AI agents and applications.

chore(benchmarks): add BEAM 1M ingest/eval ops scripts

Christianb1af5b1caaa8
diff --git a/benchmarks/scripts/beam_1m_keepalive.sh b/benchmarks/scripts/beam_1m_keepalive.sh
new file mode 100755
index 0000000..722a922
--- /dev/null
+++ b/benchmarks/scripts/beam_1m_keepalive.sh
@@ -0,0 +1,54 @@
+#!/usr/bin/env bash
+# Tiny keep-alive: only ensures API LISTEN + celery worker. Never pkill.
+# screen -dmS beam1m-keep bash benchmarks/scripts/beam_1m_keepalive.sh
+#
+# Safe under concurrent ingest: flock around starts; longer /docs probe.
+# Does not restart a healthy LISTEN uvicorn (avoids killing live eval).
+set +e
+SRC=/Users/christiannonis/.brainapi/source
+UV_LOG=/tmp/uvicorn-beam1m.log
+CEL_LOG=/tmp/brainapi-beam-worker.log
+LOCK_DIR=/tmp/beam1m-locks
+mkdir -p "$LOCK_DIR"
+CELERY_WORKER_CONCURRENCY="${CELERY_WORKER_CONCURRENCY:-4}"
+
+echo "KEEPALIVE_START $(date -u +%Y-%m-%dT%H:%M:%SZ)" >> /tmp/beam-1m-keepalive.log
+while [[ ! -f /tmp/beam-1m-1.STOP ]]; do
+ if ! lsof -iTCP:8000 -sTCP:LISTEN >/dev/null 2>&1; then
+ echo "API_RESTART $(date -u +%H:%M:%S)" >> /tmp/beam-1m-keepalive.log
+ (
+ flock -n 9 || exit 0
+ if lsof -iTCP:8000 -sTCP:LISTEN >/dev/null 2>&1; then
+ exit 0
+ fi
+ nohup bash -c "cd '$SRC' && exec ./.venv/bin/uvicorn src.services.api.app:app --host 127.0.0.1 --port 8000" \
+ >> "$UV_LOG" 2>&1 &
+ echo $! > /tmp/beam1m-uvicorn.pid
+ sleep 5
+ ) 9>"$LOCK_DIR/uvicorn.lock"
+ else
+ # Soft health: log only; do not restart if LISTEN is up (eval-safe).
+ code=$(curl -s -o /dev/null --max-time 5 -w '%{http_code}' http://127.0.0.1:8000/docs || echo 000)
+ if [[ "$code" != "200" && "$code" != "401" ]]; then
+ echo "API_BUSY_OR_SLOW code=$code $(date -u +%H:%M:%S)" >> /tmp/beam-1m-keepalive.log
+ fi
+ fi
+ # Match real worker process only (python .../celery), not bash scripts.
+ if ! pgrep -f '/.venv/bin/celery -A src.workers.app worker' >/dev/null 2>&1; then
+ echo "CELERY_RESTART $(date -u +%H:%M:%S)" >> /tmp/beam-1m-keepalive.log
+ (
+ flock -n 9 || exit 0
+ if pgrep -f '/.venv/bin/celery -A src.workers.app worker' >/dev/null 2>&1; then
+ exit 0
+ fi
+ nohup bash -c "cd '$SRC' && exec ./.venv/bin/celery -A src.workers.app worker --loglevel=info --pool=threads \
+ --concurrency=${CELERY_WORKER_CONCURRENCY} \
+ -Q ingest_data,finalize_ingestion,process_architect_relationships,consolidate_graph,chatbot_memory,ingest_file,ingest_structured_data" \
+ >> "$CEL_LOG" 2>&1 &
+ echo $! > /tmp/beam1m-celery.pid
+ sleep 5
+ ) 9>"$LOCK_DIR/celery.lock"
+ fi
+ sleep 20
+done
+echo "KEEPALIVE_STOP $(date -u +%Y-%m-%dT%H:%M:%SZ)" >> /tmp/beam-1m-keepalive.log
diff --git a/benchmarks/scripts/boot_beam_1m_screen.sh b/benchmarks/scripts/boot_beam_1m_screen.sh
new file mode 100755
index 0000000..0f43118
--- /dev/null
+++ b/benchmarks/scripts/boot_beam_1m_screen.sh
@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+# Boot BEAM 1M ingest inside screen. Keep this file free of long uvicorn/celery
+# command strings in the screen argv — use this script path only:
+# screen -dmS beam1m /path/to/boot_beam_1m_screen.sh
+set +e
+SRC=/Users/christiannonis/.brainapi/source
+UV_LOG=/tmp/uvicorn-beam1m.log
+CEL_LOG=/tmp/brainapi-beam-worker.log
+UV_PID=/tmp/beam1m-uvicorn.pid
+CEL_PID=/tmp/beam1m-celery.pid
+
+start_api() {
+ if lsof -iTCP:8000 -sTCP:LISTEN >/dev/null 2>&1; then
+ return 0
+ fi
+ nohup bash -c "cd '$SRC' && exec ./.venv/bin/uvicorn src.services.api.app:app --host 127.0.0.1 --port 8000" \
+ >> "$UV_LOG" 2>&1 &
+ echo $! > "$UV_PID"
+ sleep 5
+}
+
+start_worker() {
+ if pgrep -f 'celery -A src.workers.app worker' >/dev/null 2>&1; then
+ return 0
+ fi
+ CELERY_C="${CELERY_WORKER_CONCURRENCY:-4}"
+ nohup bash -c "cd '$SRC' && exec ./.venv/bin/celery -A src.workers.app worker --loglevel=info --pool=threads --concurrency=${CELERY_C} -Q ingest_data,finalize_ingestion,process_architect_relationships,consolidate_graph,chatbot_memory,ingest_file,ingest_structured_data" \
+ >> "$CEL_LOG" 2>&1 &
+ echo $! > "$CEL_PID"
+ sleep 5
+}
+
+start_api
+start_worker
+exec /tmp/run_beam_1m_1_ingest_eval.sh >> /tmp/beam-1m-1-run.log 2>&1
diff --git a/benchmarks/scripts/eval_beam_1m_1_product.sh b/benchmarks/scripts/eval_beam_1m_1_product.sh
new file mode 100644
index 0000000..3e62f7a
--- /dev/null
+++ b/benchmarks/scripts/eval_beam_1m_1_product.sh
@@ -0,0 +1,91 @@
+#!/usr/bin/env bash
+# Evaluate-only BEAM 1M/1 product on existing brain (no ingest / no wipe).
+set +e
+cd /Users/christiannonis/Documents/Projects/brainapi2/benchmarks
+unset BRAINPAT_TOKEN BRAINAPI_TOKEN
+set -a; source ./.env; set +a
+export TERM=dumb PYTHONUNBUFFERED=1
+
+SRC=/Users/christiannonis/.brainapi/source
+UV_LOG=/tmp/uvicorn-beam1m.log
+CEL_LOG=/tmp/brainapi-beam-worker.log
+
+ensure_api() {
+ for _ in 1 2 3 4 5 6 7 8; do
+ if lsof -iTCP:8000 -sTCP:LISTEN >/dev/null 2>&1; then
+ code=$(curl -s -o /dev/null --max-time 2 -w '%{http_code}' http://127.0.0.1:8000/docs || echo 000)
+ if [[ "$code" == "200" || "$code" == "401" ]]; then
+ if ! pgrep -f '/.venv/bin/celery -A src.workers.app worker' >/dev/null 2>&1; then
+ nohup bash -c "cd '$SRC' && exec ./.venv/bin/celery -A src.workers.app worker --loglevel=info --pool=threads -Q ingest_data,finalize_ingestion,process_architect_relationships,consolidate_graph,chatbot_memory,ingest_file,ingest_structured_data" \
+ >> "$CEL_LOG" 2>&1 &
+ sleep 4
+ fi
+ return 0
+ fi
+ fi
+ if ! lsof -iTCP:8000 -sTCP:LISTEN >/dev/null 2>&1; then
+ nohup bash -c "cd '$SRC' && exec ./.venv/bin/uvicorn src.services.api.app:app --host 127.0.0.1 --port 8000" \
+ >> "$UV_LOG" 2>&1 &
+ sleep 5
+ fi
+ done
+ return 1
+}
+
+echo "START_1M_EVAL $(date -u +%Y-%m-%dT%H:%M:%SZ)"
+ensure_api || { echo FATAL_API_DOWN; exit 1; }
+
+for ab in \
+ abstention \
+ contradiction_resolution \
+ event_ordering \
+ information_extraction \
+ instruction_following \
+ knowledge_update \
+ multi_session_reasoning \
+ preference_following \
+ summarization \
+ temporal_reasoning
+do
+ echo "==== EVAL $ab $(date -u +%H:%M:%S) ===="
+ ensure_api
+ ./beam.sh evaluate \
+ --size 1M --sample 1 \
+ --run beam-1m-1-clean-product \
+ --brain beam1m1clean \
+ --profile product \
+ --abilities "$ab" \
+ --concurrency 1 \
+ --historical-limit 12 --max-passages 12 --max-facts 40 \
+ --no-ppr
+done
+
+./beam.sh report --run beam-1m-1-clean-product
+.venv/bin/python - <<'PY'
+import json, statistics
+from pathlib import Path
+rows=[json.loads(l) for l in Path('runs/beam-1m-1-clean/ingest.jsonl').read_text().splitlines() if l.strip()]
+last={}
+for r in rows: last[r.get('unit_id')]=r
+comp=[r for r in last.values() if r.get('status')=='completed' and (r.get('cost') or {}).get('llm_source_multiplier') is not None]
+mult=[r['cost']['llm_source_multiplier'] for r in comp]
+print('INGEST completed', sum(1 for r in last.values() if r.get('status')=='completed'), '/', len(last))
+print('PERMANENT_FAILS', sorted(
+ uid for uid,r in last.items()
+ if r.get('status')=='permanent_failed'
+ or (r.get('status')=='failed' and '8192' in (r.get('error') or ''))
+))
+print('MULTIPLIER n', len(mult),
+ 'median', round(statistics.median(mult),1) if mult else None,
+ 'mean', round(statistics.mean(mult),1) if mult else None)
+rep_path=Path('runs/beam-1m-1-clean-product/report.json')
+if rep_path.exists():
+ rep=json.loads(rep_path.read_text())
+ m=rep.get('metrics') or {}
+ hs=m.get('headline_score')
+ print('HEADLINE', hs, f"({(hs or 0)*100:.1f}%)" if isinstance(hs,(int,float)) else '')
+ print('PER', m.get('per_ability'))
+ print('GOAL_B_65', 'MET' if (hs or 0) >= 0.65 else 'MISS')
+PY
+date -u +%Y-%m-%dT%H:%M:%SZ > /tmp/beam-1m-1.DONE
+echo EVAL_DONE
diff --git a/benchmarks/scripts/run_beam_1m_1_ingest_eval.sh b/benchmarks/scripts/run_beam_1m_1_ingest_eval.sh
new file mode 100755
index 0000000..ccf09c2
--- /dev/null
+++ b/benchmarks/scripts/run_beam_1m_1_ingest_eval.sh
@@ -0,0 +1,220 @@
+#!/usr/bin/env bash
+# Resilient BEAM 1M/1 ingest + product evaluate.
+# Resume-safe. LISTEN-based ensure_api (never pkill healthy uvicorn).
+# Uses flock so watchdog/ensure never double-bind :8000.
+#
+# Parallel ingest (next re-ingest — do NOT wipe live eval brains casually):
+# BEAM_INGEST_CONCURRENCY=2|3 (default 2; ≤4; need CELERY_WORKER_CONCURRENCY ≥ N)
+# Observed wall ~40h @ concurrency 1 for ~625 turns → ~20–25h @2, ~14–18h @3.
+set +e
+cd /Users/christiannonis/Documents/Projects/brainapi2/benchmarks
+unset BRAINPAT_TOKEN BRAINAPI_TOKEN
+set -a; source ./.env; set +a
+export TERM=dumb PYTHONUNBUFFERED=1
+
+SRC=/Users/christiannonis/.brainapi/source
+UV_LOG=/tmp/uvicorn-beam1m.log
+CEL_LOG=/tmp/brainapi-beam-worker.log
+LOCK_DIR=/tmp/beam1m-locks
+mkdir -p "$LOCK_DIR"
+
+# Harness parallel units. Keep ≤ CELERY_WORKER_CONCURRENCY (product .env default 4).
+BEAM_INGEST_CONCURRENCY="${BEAM_INGEST_CONCURRENCY:-2}"
+BEAM_INGEST_TIMEOUT="${BEAM_INGEST_TIMEOUT:-2400}"
+CELERY_WORKER_CONCURRENCY="${CELERY_WORKER_CONCURRENCY:-4}"
+
+api_ok() {
+ lsof -iTCP:8000 -sTCP:LISTEN >/dev/null 2>&1 || return 1
+ # Longer timeout under concurrent ingest poll load.
+ code=$(curl -s -o /dev/null --max-time 5 -w '%{http_code}' http://127.0.0.1:8000/docs || echo 000)
+ [[ "$code" == "200" || "$code" == "401" ]]
+}
+
+start_uvicorn() {
+ (
+ flock -n 9 || return 0
+ if lsof -iTCP:8000 -sTCP:LISTEN >/dev/null 2>&1; then
+ return 0
+ fi
+ nohup bash -c "cd '$SRC' && exec ./.venv/bin/uvicorn src.services.api.app:app --host 127.0.0.1 --port 8000" \
+ >> "$UV_LOG" 2>&1 &
+ echo $! > /tmp/beam1m-uvicorn.pid
+ sleep 5
+ ) 9>"$LOCK_DIR/uvicorn.lock"
+}
+
+start_celery() {
+ (
+ flock -n 9 || return 0
+ # Match the real worker binary, not bash wrappers that mention celery in -c strings.
+ if pgrep -f '/.venv/bin/celery -A src.workers.app worker' >/dev/null 2>&1; then
+ return 0
+ fi
+ nohup bash -c "cd '$SRC' && exec ./.venv/bin/celery -A src.workers.app worker --loglevel=info --pool=threads \
+ --concurrency=${CELERY_WORKER_CONCURRENCY} \
+ -Q ingest_data,finalize_ingestion,process_architect_relationships,consolidate_graph,chatbot_memory,ingest_file,ingest_structured_data" \
+ >> "$CEL_LOG" 2>&1 &
+ echo $! > /tmp/beam1m-celery.pid
+ sleep 5
+ ) 9>"$LOCK_DIR/celery.lock"
+}
+
+ensure_api() {
+ for _ in 1 2 3 4 5 6 7 8 9 10 11 12; do
+ if api_ok; then
+ start_celery
+ return 0
+ fi
+ start_uvicorn
+ start_celery
+ sleep 3
+ done
+ return 1
+}
+
+# Terminal units for stall/target: completed + permanent_failed (+ legacy embed-8192 failed).
+terminal_count() {
+ .venv/bin/python - <<'PY'
+import json
+import re
+from pathlib import Path
+from collections import Counter
+p = Path("runs/beam-1m-1-clean/ingest.jsonl")
+rows = [json.loads(l) for l in p.read_text().splitlines() if l.strip()] if p.exists() else []
+last = {}
+for r in rows:
+ last[r.get("unit_id")] = r
+pat = re.compile(r"maximum context length is 8192", re.I)
+n = 0
+for r in last.values():
+ st = r.get("status")
+ err = r.get("error") or ""
+ if st in {"completed", "partial_failed", "permanent_failed"}:
+ n += 1
+ elif st == "failed" and pat.search(err):
+ n += 1
+print(n)
+PY
+}
+
+completed_count() {
+ terminal_count
+}
+
+watchdog() {
+ while [[ ! -f /tmp/beam-1m-1.STOP ]]; do
+ if ! lsof -iTCP:8000 -sTCP:LISTEN >/dev/null 2>&1; then
+ echo "WATCHDOG_API_RESTART $(date -u +%H:%M:%S)"
+ start_uvicorn
+ fi
+ if ! pgrep -f '/.venv/bin/celery -A src.workers.app worker' >/dev/null 2>&1; then
+ echo "WATCHDOG_CELERY_RESTART $(date -u +%H:%M:%S)"
+ start_celery
+ fi
+ sleep 30
+ done
+}
+
+# Accept near-complete ingest: a few units permanently fail (embed 8192 context).
+TARGET=620
+STALL_LIMIT=3
+prev_n=-1
+stalls=0
+rm -f /tmp/beam-1m-1.STOP /tmp/beam-1m-1.DONE
+echo "START_1M_INGEST $(date -u +%Y-%m-%dT%H:%M:%SZ) concurrency=$BEAM_INGEST_CONCURRENCY celery_c=$CELERY_WORKER_CONCURRENCY"
+ensure_api || { echo FATAL_API_DOWN; exit 1; }
+watchdog &
+WD_PID=$!
+echo "watchdog_pid=$WD_PID"
+
+for try in $(seq 1 300); do
+ ensure_api || { echo API_DOWN; sleep 20; continue; }
+ done_n=$(completed_count)
+ echo "try=$try terminal=$done_n $(date -u +%H:%M:%S)"
+ if [[ "$done_n" -ge "$TARGET" ]]; then
+ break
+ fi
+ if [[ "$done_n" -eq "$prev_n" ]]; then
+ stalls=$((stalls + 1))
+ if [[ "$stalls" -ge "$STALL_LIMIT" ]]; then
+ echo "INGEST_STALLED terminal=$done_n (no progress ${stalls} tries) — proceed to eval"
+ break
+ fi
+ else
+ stalls=0
+ fi
+ prev_n=$done_n
+ ./beam.sh ingest \
+ --size 1M --sample 1 \
+ --run beam-1m-1-clean \
+ --brain beam1m1clean \
+ --concurrency "$BEAM_INGEST_CONCURRENCY" \
+ --timeout "$BEAM_INGEST_TIMEOUT"
+ sleep 2
+done
+
+done_n=$(completed_count)
+echo "INGEST_DONE=$done_n"
+# Do not STOP keepalive globally; only stop this script's watchdog.
+kill "$WD_PID" 2>/dev/null
+wait "$WD_PID" 2>/dev/null
+
+if [[ "$done_n" -lt 500 ]]; then
+ echo "INGEST_INCOMPLETE abort eval ($done_n/625)"
+ date -u +%Y-%m-%dT%H:%M:%SZ > /tmp/beam-1m-1.DONE
+ exit 2
+fi
+
+# Portable loop (bash array with multiline words crashed as commands in prior run).
+for ab in \
+ abstention \
+ contradiction_resolution \
+ event_ordering \
+ information_extraction \
+ instruction_following \
+ knowledge_update \
+ multi_session_reasoning \
+ preference_following \
+ summarization \
+ temporal_reasoning
+do
+ echo "==== EVAL $ab $(date -u +%H:%M:%S) ===="
+ ensure_api
+ ./beam.sh evaluate \
+ --size 1M --sample 1 \
+ --run beam-1m-1-clean-product \
+ --brain beam1m1clean \
+ --profile product \
+ --abilities "$ab" \
+ --concurrency 1 \
+ --historical-limit 12 --max-passages 12 --max-facts 40 \
+ --no-ppr
+done
+./beam.sh report --run beam-1m-1-clean-product
+.venv/bin/python - <<'PY'
+import json, statistics, re
+from pathlib import Path
+rows=[json.loads(l) for l in Path('runs/beam-1m-1-clean/ingest.jsonl').read_text().splitlines() if l.strip()]
+last={}
+for r in rows: last[r.get('unit_id')]=r
+pat=re.compile(r'maximum context length is 8192', re.I)
+comp=[r for r in last.values() if r.get('status')=='completed' and (r.get('cost') or {}).get('llm_source_multiplier') is not None]
+mult=[r['cost']['llm_source_multiplier'] for r in comp]
+perm=sorted(
+ uid for uid,r in last.items()
+ if r.get('status')=='permanent_failed'
+ or (r.get('status')=='failed' and pat.search(r.get('error') or ''))
+)
+print('MULTIPLIER n', len(mult),
+ 'median', round(statistics.median(mult),1) if mult else None,
+ 'mean', round(statistics.mean(mult),1) if mult else None)
+print('PERMANENT_FAILS', perm)
+rep_path=Path('runs/beam-1m-1-clean-product/report.json')
+if rep_path.exists():
+ rep=json.loads(rep_path.read_text())
+ m=rep.get('metrics') or {}
+ print('HEADLINE', m.get('headline_score'))
+ print('PER', m.get('per_ability'))
+PY
+date -u +%Y-%m-%dT%H:%M:%SZ > /tmp/beam-1m-1.DONE
+echo MULTI_DONE
diff --git a/benchmarks/scripts/run_beam_chats_2_3.sh b/benchmarks/scripts/run_beam_chats_2_3.sh
new file mode 100755
index 0000000..d1ea83d
--- /dev/null
+++ b/benchmarks/scripts/run_beam_chats_2_3.sh
@@ -0,0 +1,109 @@
+#!/usr/bin/env bash
+set +e
+cd /Users/christiannonis/Documents/Projects/brainapi2/benchmarks
+unset BRAINPAT_TOKEN BRAINAPI_TOKEN
+set -a; source ./.env; set +a
+export TERM=dumb PYTHONUNBUFFERED=1
+
+ensure_api() {
+ for _ in 1 2 3 4 5 6; do
+ if curl -s -o /dev/null --max-time 2 http://127.0.0.1:8000/docs >/dev/null; then
+ return 0
+ fi
+ if ! lsof -iTCP:8000 -sTCP:LISTEN >/dev/null 2>&1; then
+ (
+ cd /Users/christiannonis/.brainapi/source
+ ./.venv/bin/uvicorn src.services.api.app:app --host 127.0.0.1 --port 8000 >> /tmp/uvicorn-stable.log 2>&1 &
+ )
+ fi
+ if ! pgrep -f 'celery -A src.workers.app worker' >/dev/null; then
+ (
+ cd /Users/christiannonis/.brainapi/source
+ ./.venv/bin/celery -A src.workers.app worker --loglevel=info --pool=threads \
+ -Q ingest_data,finalize_ingestion,process_architect_relationships,consolidate_graph,chatbot_memory,ingest_file,ingest_structured_data \
+ >> /tmp/brainapi-beam-worker.log 2>&1 &
+ )
+ fi
+ sleep 5
+ done
+}
+
+completed_count() {
+ local sample=$1
+ .venv/bin/python - "$sample" <<'PY'
+import json, sys
+from pathlib import Path
+from collections import Counter
+sample = sys.argv[1]
+p = Path(f"runs/beam-100k-{sample}-clean/ingest.jsonl")
+rows = [json.loads(l) for l in p.read_text().splitlines() if l.strip()] if p.exists() else []
+last = {}
+for r in rows:
+ last[r.get("unit_id")] = r.get("status")
+print(Counter(last.values()).get("completed", 0))
+PY
+}
+
+for sample in 2 3; do
+ echo "==== CHAT $sample ===="
+ for try in $(seq 1 30); do
+ ensure_api
+ done_n=$(completed_count "$sample")
+ echo "try=$try completed=$done_n $(date -u +%H:%M:%S)"
+ if [[ "$done_n" -ge 70 ]]; then
+ break
+ fi
+ ./beam.sh ingest \
+ --size 100K --sample "$sample" \
+ --run "beam-100k-${sample}-clean" \
+ --brain "beam100k${sample}clean" \
+ --concurrency "${BEAM_INGEST_CONCURRENCY:-2}" \
+ --timeout 1200
+ done
+ done_n=$(completed_count "$sample")
+ echo "INGEST${sample}_DONE=$done_n"
+ if [[ "$done_n" -lt 50 ]]; then
+ echo "skip eval for $sample (ingest incomplete)"
+ continue
+ fi
+ abilities=(
+ abstention contradiction_resolution event_ordering information_extraction
+ instruction_following knowledge_update multi_session_reasoning
+ preference_following summarization temporal_reasoning
+ )
+ for ab in "${abilities[@]}"; do
+ ensure_api
+ ./beam.sh evaluate \
+ --size 100K --sample "$sample" \
+ --run "beam-100k-${sample}-clean-product" \
+ --brain "beam100k${sample}clean" \
+ --profile product \
+ --abilities "$ab" \
+ --concurrency 1 \
+ --historical-limit 12 --max-passages 12 --max-facts 40 \
+ --no-ppr
+ ensure_api
+ ./beam.sh evaluate \
+ --size 100K --sample "$sample" \
+ --run "beam-100k-${sample}-clean-product" \
+ --brain "beam100k${sample}clean" \
+ --profile product \
+ --abilities "$ab" \
+ --concurrency 1 \
+ --historical-limit 12 --max-passages 12 --max-facts 40 \
+ --no-ppr
+ done
+ .venv/bin/python - <<PY
+from pathlib import Path
+from beam.report import write_report
+r = write_report(Path("runs/beam-100k-${sample}-clean-product"))
+print(
+ "CHAT${sample}_HEADLINE",
+ (r.get("metrics") or {}).get("headline_score"),
+ "n",
+ (r.get("metrics") or {}).get("n_questions"),
+)
+PY
+done
+echo MULTI_DONE
+date -u +%Y-%m-%dT%H:%M:%SZ > /tmp/beam-100k-23-done.DONE