A curated collection of practical AI projects implementing OCR systems, RAG, AI agents, and other AI use cases.
#!/usr/bin/env python3
"""Daily data collection pipeline for the AI Research Agent.
Runs Devable Research Agent collectors, filters top items for digest generation, and logs
output to logs/pipeline.log. Designed to be invoked by Windows Task Scheduler
at 7:45am — OpenClaw reads digest_input_today.json at 8am for the Slack digest.
"""
import argparse
import json
import os
import subprocess
import sys
from datetime import datetime, timedelta, timezone
from pathlib import Path
from typing import Optional
PROJECT_ROOT = Path(
os.environ.get("DEVABLE_PROJECT_ROOT", "")
).resolve() if os.environ.get("DEVABLE_PROJECT_ROOT") else Path(__file__).resolve().parent.parent
COLLECT_SCRIPT = PROJECT_ROOT / "skills" / "tracking-list" / "scripts" / "collect.py"
FILTER_SCRIPT = PROJECT_ROOT / "scripts" / "filter_top_items.py"
DATA_FILE = PROJECT_ROOT / "data_today.json"
DIGEST_FILE = PROJECT_ROOT / "digest_input_today.json"
LOG_FILE = PROJECT_ROOT / "logs" / "pipeline.log"
TZ_LAGOS = timezone(timedelta(hours=1))
def _log(message: str) -> None:
LOG_FILE.parent.mkdir(parents=True, exist_ok=True)
timestamp = datetime.now(TZ_LAGOS).strftime("%Y-%m-%d %H:%M:%S")
line = f"[{timestamp}] {message}"
with LOG_FILE.open("a", encoding="utf-8") as handle:
handle.write(line + "\n")
print(line)
def _run_step(name: str, cmd: list) -> bool:
_log(f"START {name}")
_log(f"CMD {' '.join(cmd)}")
try:
result = subprocess.run(
cmd,
cwd=PROJECT_ROOT,
capture_output=True,
text=True,
encoding="utf-8",
errors="replace",
env={**os.environ, "DEVABLE_PROJECT_ROOT": str(PROJECT_ROOT)},
)
except Exception as exc:
_log(f"ERROR {name} failed to launch: {exc}")
return False
if result.stdout:
for line in result.stdout.splitlines():
_log(f" {line}")
if result.stderr:
for line in result.stderr.splitlines():
_log(f" {line}")
if result.returncode != 0:
_log(f"ERROR {name} exited with code {result.returncode}")
return False
_log(f"DONE {name}")
return True
def _parse_generated_at(value: str) -> Optional[datetime]:
if not value:
return None
text = value.strip()
if text.endswith("Z"):
text = text[:-1] + "+00:00"
try:
parsed = datetime.fromisoformat(text)
except ValueError:
return None
if parsed.tzinfo is None:
parsed = parsed.replace(tzinfo=timezone.utc)
return parsed
def _validate_digest_input() -> bool:
"""Ensure digest_input_today.json exists and is fresh (today Lagos or within 24h)."""
if not DIGEST_FILE.exists():
_log(
"ERROR digest_input_today.json missing after filter step — "
"OpenClaw cron will fail. Pipeline aborted."
)
return False
try:
data = json.loads(DIGEST_FILE.read_text(encoding="utf-8"))
except (json.JSONDecodeError, OSError) as exc:
_log(f"ERROR digest_input_today.json unreadable: {exc}")
return False
items = data.get("items") or []
if not items:
_log("ERROR digest_input_today.json has zero items — pipeline aborted.")
return False
generated_at = _parse_generated_at(data.get("generated_at", ""))
now_lagos = datetime.now(TZ_LAGOS)
today_lagos = now_lagos.date()
if generated_at is None:
_log(
"ERROR digest_input_today.json missing or invalid generated_at — "
"run filter_top_items.py before OpenClaw cron."
)
return False
generated_lagos = generated_at.astimezone(TZ_LAGOS)
age = now_lagos - generated_lagos
if generated_lagos.date() != today_lagos and age > timedelta(hours=24):
_log(
f"ERROR digest_input_today.json is stale "
f"(generated_at={generated_lagos.isoformat()}, age={age}) — pipeline aborted."
)
return False
return True
def _write_status_line(data_path: Path, digest_path: Path) -> None:
"""Append one-line pipeline status summary to logs/pipeline.log."""
try:
data = json.loads(data_path.read_text(encoding="utf-8"))
digest = json.loads(digest_path.read_text(encoding="utf-8"))
except (json.JSONDecodeError, OSError) as exc:
_log(f"WARNING could not write status line: {exc}")
return
by_source = data.get("stats", {}).get("by_source", {})
if not by_source:
counts: dict = {}
for item in data.get("items", []):
source = item.get("source", "unknown")
counts[source] = counts.get(source, 0) + 1
source_summary = ", ".join(f"{k}={v}" for k, v in sorted(counts.items()))
else:
source_summary = ", ".join(
f"{name}={info.get('items', 0)}"
for name, info in sorted(by_source.items())
)
digest_count = len(digest.get("items", []))
categories = digest.get("stats", {}).get("by_category", {})
if not categories and digest.get("items"):
if str(FILTER_SCRIPT) not in sys.path:
sys.path.insert(0, str(PROJECT_ROOT / "scripts"))
try:
from filter_top_items import _category_counts
categories = _category_counts(digest.get("items", []))
except Exception:
categories = {}
category_summary = ", ".join(f"{k}={v}" for k, v in sorted(categories.items()))
timestamp = datetime.now(TZ_LAGOS).strftime("%Y-%m-%d %H:%M:%S")
_log(
f"STATUS {timestamp} | collected: {source_summary} | "
f"digest_items={digest_count} | categories: {category_summary}"
)
def run_pipeline(date_str: str, depth: str = "default", top_n: int = 30) -> int:
"""Run collection and filtering. Returns process exit code."""
_log("=" * 60)
_log(f"Pipeline run started (date={date_str}, depth={depth}, top={top_n})")
if not COLLECT_SCRIPT.exists():
_log(f"ERROR collect script not found: {COLLECT_SCRIPT}")
return 1
if not FILTER_SCRIPT.exists():
_log(f"ERROR filter script not found: {FILTER_SCRIPT}")
return 1
collect_cmd = [
sys.executable,
str(COLLECT_SCRIPT),
"--date", date_str,
"--depth", depth,
"--no-cache",
"--output", str(DATA_FILE),
]
if not _run_step("collect", collect_cmd):
_log("Pipeline aborted: collection failed")
return 1
if not DATA_FILE.exists():
_log(f"ERROR expected output missing: {DATA_FILE}")
return 1
filter_cmd = [
sys.executable,
str(FILTER_SCRIPT),
str(DATA_FILE),
str(DIGEST_FILE),
"--top", str(top_n),
]
if not _run_step("filter", filter_cmd):
_log("Pipeline aborted: filtering failed")
return 1
if not _validate_digest_input():
return 1
data_kb = DATA_FILE.stat().st_size / 1024
digest_kb = DIGEST_FILE.stat().st_size / 1024
_log(f"Pipeline complete: {DATA_FILE.name} ({data_kb:.1f}KB), {DIGEST_FILE.name} ({digest_kb:.1f}KB)")
_write_status_line(DATA_FILE, DIGEST_FILE)
return 0
def main() -> None:
parser = argparse.ArgumentParser(description="Run Devable Research Agent daily collection pipeline")
parser.add_argument(
"--date",
default=datetime.now(TZ_LAGOS).strftime("%Y-%m-%d"),
help="Target collection date YYYY-MM-DD (default: today Lagos)",
)
parser.add_argument(
"--depth",
default="default",
choices=["quick", "default", "deep"],
help="Collector depth (default: default)",
)
parser.add_argument(
"--top",
type=int,
default=30,
help="Number of top items for digest input (default: 30)",
)
args = parser.parse_args()
raise SystemExit(run_pipeline(args.date, args.depth, args.top))
if __name__ == "__main__":
main()