Skip to content

AI agent skill that researches any topic across Reddit, X, YouTube, HN, Polymarket, and the web - then synthesizes a grounded summary

9.4 KB
Raw Blame
#!/usr/bin/env python3
"""E2E comparison: run sample queries on both v3 (current branch) and v2.9.5 (main).

Usage:
    python3 tests/e2e_comparison.py [--v2-script PATH]

Outputs a markdown comparison table with per-query metrics.
Passed comparisons exit 0; failed or partial comparisons exit 1, including
otherwise successful commands that report source errors.
Comparison totals sum the metrics of each entity report.
"""

import json
import subprocess
import sys
import time
from pathlib import Path

REPO = Path(__file__).resolve().parents[1]
V3_SCRIPT = str(REPO / "skills" / "last30days" / "scripts" / "last30days.py")

# v2.9.5 from plugin cache (main branch equivalent)
V2_SCRIPT = str(
    Path.home()
    / ".claude/plugins/cache/last30days/last30days/2.9.5/scripts/last30days.py"
)

EVAL_TOPICS_FILE = REPO / "fixtures" / "eval_topics.json"


def _load_queries() -> list[tuple[str, str]]:
    if EVAL_TOPICS_FILE.exists():
        rows = json.loads(EVAL_TOPICS_FILE.read_text())
        return [(row["topic"], row["query_type"]) for row in rows]
    return [
        ("openclaw vs nanoclaw vs ironclaw", "comparison"),
        ("how to deploy on Fly.io", "how_to"),
        ("kanye west", "breaking_news"),
        ("odds of recession", "prediction"),
        ("explain transformer architecture", "concept"),
    ]


QUERIES = _load_queries()


def _v3_metrics(data: dict) -> dict:
    items_by_source = data.get("items_by_source", {})
    errors = list(data.get("errors_by_source", {}))
    for source, outcome in data.get("source_status", {}).items():
        if outcome.get("state") in {
            "partial", "rate-limited", "auth-failed", "payment-required",
            "unreachable", "timeout", "schema-drift", "error",
        } and source not in errors:
            errors.append(source)
    return {
        "sources": sum(1 for items in items_by_source.values() if items),
        "total_items": sum(len(items) for items in items_by_source.values()),
        "candidates": len(data.get("ranked_candidates", [])),
        "clusters": len(data.get("clusters", [])),
        "intent": data["query_plan"].get("intent", "?"),
        "subqueries": len(data["query_plan"].get("subqueries", [])),
        "errors": errors,
    }


def run_query(script: str, topic: str, timeout: int = 180) -> dict:
    """Run a query and return parsed JSON + timing."""
    start = time.time()
    try:
        result = subprocess.run(
            [sys.executable, script, topic, "--emit=json", "--json-profile=raw"],
            capture_output=True,
            text=True,
            timeout=timeout,
        )
        elapsed = time.time() - start
        if result.returncode != 0:
            return {
                "error": result.stderr[:200] or f"engine exited with status {result.returncode}",
                "elapsed": elapsed,
                "sources": 0,
                "candidates": 0,
                "intent": "error",
                "subqueries": 0,
            }
        data = json.loads(result.stdout)

        if data.get("comparison"):
            reports = [(entry["entity"], _v3_metrics(entry["report"])) for entry in data["reports"]]
            if not reports:
                raise ValueError("comparison output contains no reports")
            return {
                "elapsed": elapsed,
                **{key: sum(metrics[key] for _, metrics in reports) for key in (
                    "sources", "total_items", "candidates", "clusters", "subqueries",
                )},
                "intent": "comparison",
                "errors": [f"{entity}: {error}" for entity, metrics in reports for error in metrics["errors"]],
            }

        # v3 shape
        if "query_plan" in data:
            return {"elapsed": elapsed, **_v3_metrics(data)}

        # v2 shape
        sources_with_items = 0
        total_items = 0
        for key in ["reddit", "x", "youtube", "tiktok", "instagram", "hackernews",
                     "bluesky", "truthsocial", "polymarket", "web"]:
            items = data.get(key, [])
            if items:
                sources_with_items += 1
                total_items += len(items)
        return {
            "elapsed": elapsed,
            "sources": sources_with_items,
            "total_items": total_items,
            "candidates": total_items,
            "clusters": 0,
            "intent": data.get("mode", "?"),
            "subqueries": 0,
            "errors": [k for k in ["reddit_error", "x_error", "youtube_error",
                                     "tiktok_error", "instagram_error"]
                        if data.get(k)],
        }
    except subprocess.TimeoutExpired:
        return {
            "error": "timeout",
            "elapsed": timeout,
            "sources": 0,
            "candidates": 0,
            "intent": "timeout",
            "subqueries": 0,
        }
    except Exception as exc:
        return {
            "error": str(exc)[:200],
            "elapsed": time.time() - start,
            "sources": 0,
            "candidates": 0,
            "intent": "error",
            "subqueries": 0,
        }


def main() -> int:
    v2_script = V2_SCRIPT
    if len(sys.argv) > 2 and sys.argv[1] == "--v2-script":
        v2_script = sys.argv[2]

    if not Path(v2_script).exists():
        print(f"v2 script not found at {v2_script}", file=sys.stderr)
        print("Use --v2-script PATH to specify", file=sys.stderr)
        sys.exit(1)

    print("# E2E Comparison: v3.0.0 (branch) vs v2.9.5 (main)")
    print()
    print(f"- v3 script: {V3_SCRIPT}")
    print(f"- v2 script: {v2_script}")
    print(f"- Queries: {len(QUERIES)}")
    print()

    results = []
    for i, (topic, expected_intent) in enumerate(QUERIES, 1):
        print(f"[{i}/{len(QUERIES)}] {topic}", file=sys.stderr)
        sys.stderr.flush()

        print(f"  v3...", end="", file=sys.stderr)
        sys.stderr.flush()
        v3 = run_query(V3_SCRIPT, topic)
        print(f" {v3.get('elapsed', 0):.1f}s", file=sys.stderr)
        sys.stderr.flush()

        print(f"  v2...", end="", file=sys.stderr)
        sys.stderr.flush()
        v2 = run_query(v2_script, topic)
        print(f" {v2.get('elapsed', 0):.1f}s", file=sys.stderr)
        sys.stderr.flush()

        results.append({
            "topic": topic,
            "expected_intent": expected_intent,
            "v3": v3,
            "v2": v2,
        })

    # Print comparison table
    print("| Query | Intent | v3 sources | v2 sources | v3 items | v2 items | v3 time | v2 time | v3 source errors | v2 source errors | v3 command error | v2 command error |")
    print("|-------|--------|-----------|-----------|---------|---------|---------|---------|-----------|-----------|-----------|-----------|")
    for r in results:
        v3, v2 = r["v3"], r["v2"]
        v3_err = ", ".join(v3.get("errors", [])) or "-"
        v2_err = ", ".join(v2.get("errors", [])) or "-"
        v3_command_error = str(v3.get("error", "-")).replace("|", "\\|").replace("\n", " ")
        v2_command_error = str(v2.get("error", "-")).replace("|", "\\|").replace("\n", " ")
        print(
            f"| {r['topic'][:45]} | {v3.get('intent', '?')} | "
            f"{v3.get('sources', 0)} | {v2.get('sources', 0)} | "
            f"{v3.get('total_items', 0)} | {v2.get('total_items', 0)} | "
            f"{v3.get('elapsed', 0):.1f}s | {v2.get('elapsed', 0):.1f}s | "
            f"{v3_err} | {v2_err} | {v3_command_error} | {v2_command_error} |"
        )

    # Summary
    print()
    v3_total_sources = sum(r["v3"].get("sources", 0) for r in results)
    v2_total_sources = sum(r["v2"].get("sources", 0) for r in results)
    v3_total_items = sum(r["v3"].get("total_items", 0) for r in results)
    v2_total_items = sum(r["v2"].get("total_items", 0) for r in results)
    v3_total_time = sum(r["v3"].get("elapsed", 0) for r in results)
    v2_total_time = sum(r["v2"].get("elapsed", 0) for r in results)
    v3_errors = sum(len(r["v3"].get("errors", [])) for r in results)
    v2_errors = sum(len(r["v2"].get("errors", [])) for r in results)
    v3_command_errors = sum("error" in r["v3"] for r in results)
    v2_command_errors = sum("error" in r["v2"] for r in results)
    command_errors = v3_command_errors + v2_command_errors
    outcome = (
        "failed" if command_errors == 2 * len(results)
        else "partial" if command_errors or v3_errors or v2_errors
        else "passed"
    )

    print("## Summary")
    print()
    print(f"| Metric | v3.0.0 | v2.9.5 | Delta |")
    print(f"|--------|--------|--------|-------|")
    print(f"| Total sources with items | {v3_total_sources} | {v2_total_sources} | {v3_total_sources - v2_total_sources:+d} |")
    print(f"| Total items retrieved | {v3_total_items} | {v2_total_items} | {v3_total_items - v2_total_items:+d} |")
    print(f"| Total wall time | {v3_total_time:.1f}s | {v2_total_time:.1f}s | {v3_total_time - v2_total_time:+.1f}s |")
    print(f"| Source errors | {v3_errors} | {v2_errors} | {v3_errors - v2_errors:+d} |")
    print(f"| Command errors | {v3_command_errors} | {v2_command_errors} | {v3_command_errors - v2_command_errors:+d} |")
    print(f"| Avg sources/query | {v3_total_sources/len(results):.1f} | {v2_total_sources/len(results):.1f} | |")
    print(f"| Avg items/query | {v3_total_items/len(results):.1f} | {v2_total_items/len(results):.1f} | |")
    print(f"| Avg time/query | {v3_total_time/len(results):.1f}s | {v2_total_time/len(results):.1f}s | |")
    print(f"\nOutcome: {outcome}")
    return 0 if outcome == "passed" else 1


if __name__ == "__main__":
    sys.exit(main())