Skip to content

AI agent skill that researches any topic across Reddit, X, YouTube, HN, Polymarket, and the web - then synthesizes a grounded summary

perf: batch store_findings, dedup source_items in O(1), remove dead code (#206)

Ilia Alshanetsky2acbf8a8698a
diff --git a/skills/last30days/scripts/lib/env.py b/skills/last30days/scripts/lib/env.py
index f864d7ef..e78f0120 100644
--- a/skills/last30days/scripts/lib/env.py
+++ b/skills/last30days/scripts/lib/env.py
@@ -372,14 +372,6 @@ def config_exists() -> bool:
return False
-def is_reddit_available(config: dict[str, Any]) -> bool:
- """Check if Reddit search is available.
-
- v3 uses ScrapeCreators only.
- """
- return bool(config.get('SCRAPECREATORS_API_KEY'))
-
-
def get_reddit_source(config: dict[str, Any]) -> str | None:
"""Determine which Reddit backend to use.
diff --git a/skills/last30days/scripts/lib/fusion.py b/skills/last30days/scripts/lib/fusion.py
index 7d902f04..80959d5c 100644
--- a/skills/last30days/scripts/lib/fusion.py
+++ b/skills/last30days/scripts/lib/fusion.py
@@ -116,6 +116,8 @@ def weighted_rrf(
"""Fuse ranked lists into a single candidate pool."""
subqueries = {subquery.label: subquery for subquery in plan.subqueries}
candidates: dict[str, schema.Candidate] = {}
+ # Track (source, item_id) pairs already attached to each candidate for O(1) dedup.
+ seen_source_items: dict[str, set[tuple[str, str]]] = {}
for (label, source), items in streams.items():
subquery = subqueries[label]
@@ -154,6 +156,7 @@ def weighted_rrf(
]
},
)
+ seen_source_items[key] = {(item.source, item.item_id)}
continue
candidate = candidates[key]
@@ -179,7 +182,9 @@ def weighted_rrf(
candidate.subquery_labels.append(label)
if item.source not in candidate.sources:
candidate.sources.append(item.source)
- if not any(existing.source == item.source and existing.item_id == item.item_id for existing in candidate.source_items):
+ source_item_key = (item.source, item.item_id)
+ if source_item_key not in seen_source_items[key]:
+ seen_source_items[key].add(source_item_key)
candidate.source_items.append(item)
candidate.metadata.setdefault("provenance", []).append(
{
diff --git a/skills/last30days/scripts/lib/providers.py b/skills/last30days/scripts/lib/providers.py
index 3516731b..0a5e88cf 100644
--- a/skills/last30days/scripts/lib/providers.py
+++ b/skills/last30days/scripts/lib/providers.py
@@ -93,13 +93,6 @@ class GeminiClient(ReasoningClient):
)
return extract_gemini_text(payload)
- def ground_search(self, model: str, prompt: str) -> dict[str, Any]:
- return self._generate_content(model, prompt, tools=[{"google_search": {}}])
-
- def url_context_json(self, model: str, prompt: str) -> dict[str, Any]:
- return self.generate_json(model, prompt, tools=[{"url_context": {}}])
-
-
class OpenAIClient(ReasoningClient):
name = "openai"
diff --git a/skills/last30days/scripts/lib/render.py b/skills/last30days/scripts/lib/render.py
index c2f336c9..36149790 100644
--- a/skills/last30days/scripts/lib/render.py
+++ b/skills/last30days/scripts/lib/render.py
@@ -1505,16 +1505,6 @@ def _top_comments_list(item: schema.SourceItem | None, limit: int = 3, min_score
return [c for c in comments if (c.get("score") or 0) >= min_score][:limit]
-def _top_comment_excerpt(item: schema.SourceItem | None) -> str | None:
- if not item:
- return None
- comments = item.metadata.get("top_comments") or []
- if not comments or not isinstance(comments[0], dict):
- return None
- top = comments[0]
- return str(top.get("excerpt") or top.get("text") or "").strip() or None
-
-
def _comment_insight(item: schema.SourceItem | None) -> str | None:
if not item:
return None
diff --git a/skills/last30days/scripts/store.py b/skills/last30days/scripts/store.py
index bb2687e8..6cf77e65 100644
--- a/skills/last30days/scripts/store.py
+++ b/skills/last30days/scripts/store.py
@@ -346,61 +346,83 @@ def store_findings(
findings: List[Dict[str, Any]],
) -> Dict[str, int]:
"""Store findings with URL-based dedup. Returns counts of new/updated."""
- conn = _connect()
- new_count = 0
- updated_count = 0
+ # Collect findings that have a URL, preserving order.
+ with_urls: List[tuple[str, Dict[str, Any]]] = []
+ for f in findings:
+ url = f.get("source_url") or f.get("url")
+ if url:
+ with_urls.append((url, f))
+
+ if not with_urls:
+ conn = _connect()
+ try:
+ conn.execute(
+ "UPDATE research_runs SET findings_new = 0, findings_updated = 0 WHERE id = ?",
+ (run_id,),
+ )
+ conn.commit()
+ finally:
+ conn.close()
+ return {"new": 0, "updated": 0}
+ conn = _connect()
try:
- for f in findings:
- url = f.get("source_url") or f.get("url")
- if not url:
- continue
+ # Single batch SELECT to find existing findings by URL.
+ urls = [url for url, _ in with_urls]
+ placeholders = ",".join("?" for _ in urls)
+ rows = conn.execute(
+ f"SELECT id, source_url, engagement_score FROM findings WHERE source_url IN ({placeholders})",
+ urls,
+ ).fetchall()
+ existing_by_url = {row["source_url"]: row for row in rows}
- existing = conn.execute(
- "SELECT id, engagement_score, sighting_count FROM findings WHERE source_url = ?",
- (url,),
- ).fetchone()
+ update_rows: List[tuple] = []
+ insert_rows: List[tuple] = []
+ for url, f in with_urls:
+ existing = existing_by_url.get(url)
+ new_engagement = f.get("engagement_score", 0)
if existing:
- # Update engagement and re-sighting info
- new_engagement = f.get("engagement_score", 0)
- conn.execute(
- """UPDATE findings SET
- last_seen = datetime('now'),
- sighting_count = sighting_count + 1,
- engagement_score = ?,
- run_id = ?
- WHERE id = ?""",
- (
- max(new_engagement, existing["engagement_score"] or 0),
- run_id,
- existing["id"],
- ),
- )
- updated_count += 1
+ update_rows.append((
+ max(new_engagement, existing["engagement_score"] or 0),
+ run_id,
+ existing["id"],
+ ))
else:
- # New finding
- conn.execute(
- """INSERT INTO findings
- (run_id, topic_id, source, source_url, source_title,
- author, content, summary, engagement_score, relevance_score)
- VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""",
- (
- run_id,
- topic_id,
- f.get("source", "unknown"),
- url,
- f.get("source_title") or f.get("title", ""),
- f.get("author", ""),
- f.get("content") or f.get("text", ""),
- f.get("summary", ""),
- f.get("engagement_score", 0),
- f.get("relevance_score", 0),
- ),
- )
- new_count += 1
-
- # Update run stats
+ insert_rows.append((
+ run_id,
+ topic_id,
+ f.get("source", "unknown"),
+ url,
+ f.get("source_title") or f.get("title", ""),
+ f.get("author", ""),
+ f.get("content") or f.get("text", ""),
+ f.get("summary", ""),
+ new_engagement,
+ f.get("relevance_score", 0),
+ ))
+
+ if update_rows:
+ conn.executemany(
+ """UPDATE findings SET
+ last_seen = datetime('now'),
+ sighting_count = sighting_count + 1,
+ engagement_score = ?,
+ run_id = ?
+ WHERE id = ?""",
+ update_rows,
+ )
+ if insert_rows:
+ conn.executemany(
+ """INSERT INTO findings
+ (run_id, topic_id, source, source_url, source_title,
+ author, content, summary, engagement_score, relevance_score)
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""",
+ insert_rows,
+ )
+
+ new_count = len(insert_rows)
+ updated_count = len(update_rows)
conn.execute(
"UPDATE research_runs SET findings_new = ?, findings_updated = ? WHERE id = ?",
(new_count, updated_count, run_id),