AI agent skill that researches any topic across Reddit, X, YouTube, HN, Polymarket, and the web - then synthesizes a grounded summary
Python ai-promptsai-skillblueskyclaudeclaude-codeclawhubdeep-researchhackernewsinstagramopenclawpolymarketrecencyredditresearchsocial-mediatiktoktrendstwitterweb-searchyoutube
#!/usr/bin/env python3
"""E2E comparison: run sample queries on both v3 (current branch) and v2.9.5 (main).
Usage:
python3 tests/e2e_comparison.py [--v2-script PATH]
Outputs a markdown comparison table with per-query metrics.
Passed comparisons exit 0; failed or partial comparisons exit 1, including
otherwise successful commands that report source errors.
Comparison totals sum the metrics of each entity report.
"""
import json
import subprocess
import sys
import time
from pathlib import Path
REPO = Path(__file__).resolve().parents[1]
V3_SCRIPT = str(REPO / "skills" / "last30days" / "scripts" / "last30days.py")
# v2.9.5 from plugin cache (main branch equivalent)
V2_SCRIPT = str(
Path.home()
/ ".claude/plugins/cache/last30days/last30days/2.9.5/scripts/last30days.py"
)
EVAL_TOPICS_FILE = REPO / "fixtures" / "eval_topics.json"
def _load_queries() -> list[tuple[str, str]]:
if EVAL_TOPICS_FILE.exists():
rows = json.loads(EVAL_TOPICS_FILE.read_text())
return [(row["topic"], row["query_type"]) for row in rows]
return [
("openclaw vs nanoclaw vs ironclaw", "comparison"),
("how to deploy on Fly.io", "how_to"),
("kanye west", "breaking_news"),
("odds of recession", "prediction"),
("explain transformer architecture", "concept"),
]
QUERIES = _load_queries()
def _v3_metrics(data: dict) -> dict:
items_by_source = data.get("items_by_source", {})
errors = list(data.get("errors_by_source", {}))
for source, outcome in data.get("source_status", {}).items():
if outcome.get("state") in {
"partial", "rate-limited", "auth-failed", "payment-required",
"unreachable", "timeout", "schema-drift", "error",
} and source not in errors:
errors.append(source)
return {
"sources": sum(1 for items in items_by_source.values() if items),
"total_items": sum(len(items) for items in items_by_source.values()),
"candidates": len(data.get("ranked_candidates", [])),
"clusters": len(data.get("clusters", [])),
"intent": data["query_plan"].get("intent", "?"),
"subqueries": len(data["query_plan"].get("subqueries", [])),
"errors": errors,
}
def run_query(script: str, topic: str, timeout: int = 180) -> dict:
"""Run a query and return parsed JSON + timing."""
start = time.time()
try:
result = subprocess.run(
[sys.executable, script, topic, "--emit=json", "--json-profile=raw"],
capture_output=True,
text=True,
timeout=timeout,
)
elapsed = time.time() - start
if result.returncode != 0:
return {
"error": result.stderr[:200] or f"engine exited with status {result.returncode}",
"elapsed": elapsed,
"sources": 0,
"candidates": 0,
"intent": "error",
"subqueries": 0,
}
data = json.loads(result.stdout)
if data.get("comparison"):
reports = [(entry["entity"], _v3_metrics(entry["report"])) for entry in data["reports"]]
if not reports:
raise ValueError("comparison output contains no reports")
return {
"elapsed": elapsed,
**{key: sum(metrics[key] for _, metrics in reports) for key in (
"sources", "total_items", "candidates", "clusters", "subqueries",
)},
"intent": "comparison",
"errors": [f"{entity}: {error}" for entity, metrics in reports for error in metrics["errors"]],
}
# v3 shape
if "query_plan" in data:
return {"elapsed": elapsed, **_v3_metrics(data)}
# v2 shape
sources_with_items = 0
total_items = 0
for key in ["reddit", "x", "youtube", "tiktok", "instagram", "hackernews",
"bluesky", "truthsocial", "polymarket", "web"]:
items = data.get(key, [])
if items:
sources_with_items += 1
total_items += len(items)
return {
"elapsed": elapsed,
"sources": sources_with_items,
"total_items": total_items,
"candidates": total_items,
"clusters": 0,
"intent": data.get("mode", "?"),
"subqueries": 0,
"errors": [k for k in ["reddit_error", "x_error", "youtube_error",
"tiktok_error", "instagram_error"]
if data.get(k)],
}
except subprocess.TimeoutExpired:
return {
"error": "timeout",
"elapsed": timeout,
"sources": 0,
"candidates": 0,
"intent": "timeout",
"subqueries": 0,
}
except Exception as exc:
return {
"error": str(exc)[:200],
"elapsed": time.time() - start,
"sources": 0,
"candidates": 0,
"intent": "error",
"subqueries": 0,
}
def main() -> int:
v2_script = V2_SCRIPT
if len(sys.argv) > 2 and sys.argv[1] == "--v2-script":
v2_script = sys.argv[2]
if not Path(v2_script).exists():
print(f"v2 script not found at {v2_script}", file=sys.stderr)
print("Use --v2-script PATH to specify", file=sys.stderr)
sys.exit(1)
print("# E2E Comparison: v3.0.0 (branch) vs v2.9.5 (main)")
print()
print(f"- v3 script: {V3_SCRIPT}")
print(f"- v2 script: {v2_script}")
print(f"- Queries: {len(QUERIES)}")
print()
results = []
for i, (topic, expected_intent) in enumerate(QUERIES, 1):
print(f"[{i}/{len(QUERIES)}] {topic}", file=sys.stderr)
sys.stderr.flush()
print(f" v3...", end="", file=sys.stderr)
sys.stderr.flush()
v3 = run_query(V3_SCRIPT, topic)
print(f" {v3.get('elapsed', 0):.1f}s", file=sys.stderr)
sys.stderr.flush()
print(f" v2...", end="", file=sys.stderr)
sys.stderr.flush()
v2 = run_query(v2_script, topic)
print(f" {v2.get('elapsed', 0):.1f}s", file=sys.stderr)
sys.stderr.flush()
results.append({
"topic": topic,
"expected_intent": expected_intent,
"v3": v3,
"v2": v2,
})
# Print comparison table
print("| Query | Intent | v3 sources | v2 sources | v3 items | v2 items | v3 time | v2 time | v3 source errors | v2 source errors | v3 command error | v2 command error |")
print("|-------|--------|-----------|-----------|---------|---------|---------|---------|-----------|-----------|-----------|-----------|")
for r in results:
v3, v2 = r["v3"], r["v2"]
v3_err = ", ".join(v3.get("errors", [])) or "-"
v2_err = ", ".join(v2.get("errors", [])) or "-"
v3_command_error = str(v3.get("error", "-")).replace("|", "\\|").replace("\n", " ")
v2_command_error = str(v2.get("error", "-")).replace("|", "\\|").replace("\n", " ")
print(
f"| {r['topic'][:45]} | {v3.get('intent', '?')} | "
f"{v3.get('sources', 0)} | {v2.get('sources', 0)} | "
f"{v3.get('total_items', 0)} | {v2.get('total_items', 0)} | "
f"{v3.get('elapsed', 0):.1f}s | {v2.get('elapsed', 0):.1f}s | "
f"{v3_err} | {v2_err} | {v3_command_error} | {v2_command_error} |"
)
# Summary
print()
v3_total_sources = sum(r["v3"].get("sources", 0) for r in results)
v2_total_sources = sum(r["v2"].get("sources", 0) for r in results)
v3_total_items = sum(r["v3"].get("total_items", 0) for r in results)
v2_total_items = sum(r["v2"].get("total_items", 0) for r in results)
v3_total_time = sum(r["v3"].get("elapsed", 0) for r in results)
v2_total_time = sum(r["v2"].get("elapsed", 0) for r in results)
v3_errors = sum(len(r["v3"].get("errors", [])) for r in results)
v2_errors = sum(len(r["v2"].get("errors", [])) for r in results)
v3_command_errors = sum("error" in r["v3"] for r in results)
v2_command_errors = sum("error" in r["v2"] for r in results)
command_errors = v3_command_errors + v2_command_errors
outcome = (
"failed" if command_errors == 2 * len(results)
else "partial" if command_errors or v3_errors or v2_errors
else "passed"
)
print("## Summary")
print()
print(f"| Metric | v3.0.0 | v2.9.5 | Delta |")
print(f"|--------|--------|--------|-------|")
print(f"| Total sources with items | {v3_total_sources} | {v2_total_sources} | {v3_total_sources - v2_total_sources:+d} |")
print(f"| Total items retrieved | {v3_total_items} | {v2_total_items} | {v3_total_items - v2_total_items:+d} |")
print(f"| Total wall time | {v3_total_time:.1f}s | {v2_total_time:.1f}s | {v3_total_time - v2_total_time:+.1f}s |")
print(f"| Source errors | {v3_errors} | {v2_errors} | {v3_errors - v2_errors:+d} |")
print(f"| Command errors | {v3_command_errors} | {v2_command_errors} | {v3_command_errors - v2_command_errors:+d} |")
print(f"| Avg sources/query | {v3_total_sources/len(results):.1f} | {v2_total_sources/len(results):.1f} | |")
print(f"| Avg items/query | {v3_total_items/len(results):.1f} | {v2_total_items/len(results):.1f} | |")
print(f"| Avg time/query | {v3_total_time/len(results):.1f}s | {v2_total_time/len(results):.1f}s | |")
print(f"\nOutcome: {outcome}")
return 0 if outcome == "passed" else 1
if __name__ == "__main__":
sys.exit(main())