AI agent skill that researches any topic across Reddit, X, YouTube, HN, Polymarket, and the web - then synthesizes a grounded summary
Python ai-promptsai-skillblueskyclaudeclaude-codeclawhubdeep-researchhackernewsinstagramopenclawpolymarketrecencyredditresearchsocial-mediatiktoktrendstwitterweb-searchyoutube
perf: batch store_findings, dedup source_items in O(1), remove dead code (#206)
Ilia Alshanetsky2acbf8a8698a
diff --git a/skills/last30days/scripts/lib/env.py b/skills/last30days/scripts/lib/env.pyindex f864d7ef..e78f0120 100644--- a/skills/last30days/scripts/lib/env.py+++ b/skills/last30days/scripts/lib/env.py@@ -372,14 +372,6 @@ def config_exists() -> bool:return False-def is_reddit_available(config: dict[str, Any]) -> bool:- """Check if Reddit search is available.-- v3 uses ScrapeCreators only.- """- return bool(config.get('SCRAPECREATORS_API_KEY'))--def get_reddit_source(config: dict[str, Any]) -> str | None:"""Determine which Reddit backend to use.diff --git a/skills/last30days/scripts/lib/fusion.py b/skills/last30days/scripts/lib/fusion.pyindex 7d902f04..80959d5c 100644--- a/skills/last30days/scripts/lib/fusion.py+++ b/skills/last30days/scripts/lib/fusion.py@@ -116,6 +116,8 @@ def weighted_rrf("""Fuse ranked lists into a single candidate pool."""subqueries = {subquery.label: subquery for subquery in plan.subqueries}candidates: dict[str, schema.Candidate] = {}+ # Track (source, item_id) pairs already attached to each candidate for O(1) dedup.+ seen_source_items: dict[str, set[tuple[str, str]]] = {}for (label, source), items in streams.items():subquery = subqueries[label]@@ -154,6 +156,7 @@ def weighted_rrf(]},)+ seen_source_items[key] = {(item.source, item.item_id)}continuecandidate = candidates[key]@@ -179,7 +182,9 @@ def weighted_rrf(candidate.subquery_labels.append(label)if item.source not in candidate.sources:candidate.sources.append(item.source)- if not any(existing.source == item.source and existing.item_id == item.item_id for existing in candidate.source_items):+ source_item_key = (item.source, item.item_id)+ if source_item_key not in seen_source_items[key]:+ seen_source_items[key].add(source_item_key)candidate.source_items.append(item)candidate.metadata.setdefault("provenance", []).append({diff --git a/skills/last30days/scripts/lib/providers.py b/skills/last30days/scripts/lib/providers.pyindex 3516731b..0a5e88cf 100644--- a/skills/last30days/scripts/lib/providers.py+++ b/skills/last30days/scripts/lib/providers.py@@ -93,13 +93,6 @@ class GeminiClient(ReasoningClient):)return extract_gemini_text(payload)- def ground_search(self, model: str, prompt: str) -> dict[str, Any]:- return self._generate_content(model, prompt, tools=[{"google_search": {}}])-- def url_context_json(self, model: str, prompt: str) -> dict[str, Any]:- return self.generate_json(model, prompt, tools=[{"url_context": {}}])--class OpenAIClient(ReasoningClient):name = "openai"diff --git a/skills/last30days/scripts/lib/render.py b/skills/last30days/scripts/lib/render.pyindex c2f336c9..36149790 100644--- a/skills/last30days/scripts/lib/render.py+++ b/skills/last30days/scripts/lib/render.py@@ -1505,16 +1505,6 @@ def _top_comments_list(item: schema.SourceItem | None, limit: int = 3, min_scorereturn [c for c in comments if (c.get("score") or 0) >= min_score][:limit]-def _top_comment_excerpt(item: schema.SourceItem | None) -> str | None:- if not item:- return None- comments = item.metadata.get("top_comments") or []- if not comments or not isinstance(comments[0], dict):- return None- top = comments[0]- return str(top.get("excerpt") or top.get("text") or "").strip() or None--def _comment_insight(item: schema.SourceItem | None) -> str | None:if not item:return Nonediff --git a/skills/last30days/scripts/store.py b/skills/last30days/scripts/store.pyindex bb2687e8..6cf77e65 100644--- a/skills/last30days/scripts/store.py+++ b/skills/last30days/scripts/store.py@@ -346,61 +346,83 @@ def store_findings(findings: List[Dict[str, Any]],) -> Dict[str, int]:"""Store findings with URL-based dedup. Returns counts of new/updated."""- conn = _connect()- new_count = 0- updated_count = 0+ # Collect findings that have a URL, preserving order.+ with_urls: List[tuple[str, Dict[str, Any]]] = []+ for f in findings:+ url = f.get("source_url") or f.get("url")+ if url:+ with_urls.append((url, f))++ if not with_urls:+ conn = _connect()+ try:+ conn.execute(+ "UPDATE research_runs SET findings_new = 0, findings_updated = 0 WHERE id = ?",+ (run_id,),+ )+ conn.commit()+ finally:+ conn.close()+ return {"new": 0, "updated": 0}+ conn = _connect()try:- for f in findings:- url = f.get("source_url") or f.get("url")- if not url:- continue+ # Single batch SELECT to find existing findings by URL.+ urls = [url for url, _ in with_urls]+ placeholders = ",".join("?" for _ in urls)+ rows = conn.execute(+ f"SELECT id, source_url, engagement_score FROM findings WHERE source_url IN ({placeholders})",+ urls,+ ).fetchall()+ existing_by_url = {row["source_url"]: row for row in rows}- existing = conn.execute(- "SELECT id, engagement_score, sighting_count FROM findings WHERE source_url = ?",- (url,),- ).fetchone()+ update_rows: List[tuple] = []+ insert_rows: List[tuple] = []+ for url, f in with_urls:+ existing = existing_by_url.get(url)+ new_engagement = f.get("engagement_score", 0)if existing:- # Update engagement and re-sighting info- new_engagement = f.get("engagement_score", 0)- conn.execute(- """UPDATE findings SET- last_seen = datetime('now'),- sighting_count = sighting_count + 1,- engagement_score = ?,- run_id = ?- WHERE id = ?""",- (- max(new_engagement, existing["engagement_score"] or 0),- run_id,- existing["id"],- ),- )- updated_count += 1+ update_rows.append((+ max(new_engagement, existing["engagement_score"] or 0),+ run_id,+ existing["id"],+ ))else:- # New finding- conn.execute(- """INSERT INTO findings- (run_id, topic_id, source, source_url, source_title,- author, content, summary, engagement_score, relevance_score)- VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""",- (- run_id,- topic_id,- f.get("source", "unknown"),- url,- f.get("source_title") or f.get("title", ""),- f.get("author", ""),- f.get("content") or f.get("text", ""),- f.get("summary", ""),- f.get("engagement_score", 0),- f.get("relevance_score", 0),- ),- )- new_count += 1-- # Update run stats+ insert_rows.append((+ run_id,+ topic_id,+ f.get("source", "unknown"),+ url,+ f.get("source_title") or f.get("title", ""),+ f.get("author", ""),+ f.get("content") or f.get("text", ""),+ f.get("summary", ""),+ new_engagement,+ f.get("relevance_score", 0),+ ))++ if update_rows:+ conn.executemany(+ """UPDATE findings SET+ last_seen = datetime('now'),+ sighting_count = sighting_count + 1,+ engagement_score = ?,+ run_id = ?+ WHERE id = ?""",+ update_rows,+ )+ if insert_rows:+ conn.executemany(+ """INSERT INTO findings+ (run_id, topic_id, source, source_url, source_title,+ author, content, summary, engagement_score, relevance_score)+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""",+ insert_rows,+ )++ new_count = len(insert_rows)+ updated_count = len(update_rows)conn.execute("UPDATE research_runs SET findings_new = ?, findings_updated = ? WHERE id = ?",(new_count, updated_count, run_id),