Skip to content

AI agent skill that researches any topic across Reddit, X, YouTube, HN, Polymarket, and the web - then synthesizes a grounded summary

fix: fail fast and rank correctly on valid-looking input across CLI paths

Ilia Alshanetsky7a5ded08ec3d
diff --git a/skills/last30days/scripts/briefing.py b/skills/last30days/scripts/briefing.py
index ac8f6112..1fe1abf3 100644
--- a/skills/last30days/scripts/briefing.py
+++ b/skills/last30days/scripts/briefing.py
@@ -188,7 +188,15 @@ def generate_weekly() -> dict:
"this_week_engagement": this_engagement,
"last_week_engagement": last_engagement,
"engagement_change_pct": round(engagement_change, 1),
- "top_findings": this_week[:5], # Top 5 by engagement (already sorted)
+ # get_new_findings returns first_seen DESC, so sort by engagement
+ # before slicing — otherwise the digest headlines the most recent
+ # items, not the highest-engagement ones (the daily path keys on
+ # engagement too).
+ "top_findings": sorted(
+ this_week,
+ key=lambda f: f.get("engagement_score", 0),
+ reverse=True,
+ )[:5],
})
result = {
diff --git a/skills/last30days/scripts/evaluate_search_quality.py b/skills/last30days/scripts/evaluate_search_quality.py
index c6834c65..5caa9540 100644
--- a/skills/last30days/scripts/evaluate_search_quality.py
+++ b/skills/last30days/scripts/evaluate_search_quality.py
@@ -284,10 +284,17 @@ def get_judgments(
cache_file.parent.mkdir(parents=True, exist_ok=True)
if cache_file.exists():
payload = json.loads(cache_file.read_text())
- return {row["id"]: int(row["grade"]) for row in payload.get("judgments") or []}
+ # The cache key is the topic slug alone, but judgments are model-
+ # specific. Only reuse the cache when it was produced by the same judge
+ # model; otherwise re-judge, so a --judge-model change cannot return
+ # stale grades that silently skew precision@k / nDCG. Caches written
+ # before judge_model was recorded miss here and get refreshed once.
+ if payload.get("judge_model") == judge_model:
+ return {row["id"]: int(row["grade"]) for row in payload.get("judgments") or []}
if not gemini_api_key or not items:
return {}
payload = call_gemini_judge(gemini_api_key, judge_model, build_judge_prompt(topic, query_type, items))
+ payload["judge_model"] = judge_model
cache_file.write_text(json.dumps(payload, indent=2))
return {row["id"]: int(row["grade"]) for row in payload.get("judgments") or []}
diff --git a/skills/last30days/scripts/last30days.py b/skills/last30days/scripts/last30days.py
index 1793642c..b62c437b 100644
--- a/skills/last30days/scripts/last30days.py
+++ b/skills/last30days/scripts/last30days.py
@@ -742,6 +742,10 @@ def main() -> int:
external_plan = _json.loads(plan_str)
except _json.JSONDecodeError as exc:
sys.stderr.write(f"[Planner] Invalid --plan JSON: {exc}\n")
+ # Fail fast instead of silently dropping to the internal planner
+ # and burning a paid run the user did not ask for. Mirrors the
+ # --plan file-read branch above and parse_competitors_plan.
+ raise SystemExit(2)
# Auto-resolve: use web search to discover subreddits/handles before planning.
# This is the engine-side equivalent of SKILL.md Steps 0.55/0.75 for platforms
diff --git a/tests/test_briefing_v3.py b/tests/test_briefing_v3.py
index 1b0a6fd1..05c0c258 100644
--- a/tests/test_briefing_v3.py
+++ b/tests/test_briefing_v3.py
@@ -1,5 +1,6 @@
import tempfile
import unittest
+from datetime import datetime, timedelta, timezone
from pathlib import Path
from unittest import mock
@@ -28,6 +29,63 @@ class BriefingV3Tests(unittest.TestCase):
store._db_override = old_db_override
briefing.BRIEFS_DIR = old_briefs_dir
+ def test_generate_weekly_ranks_top_findings_by_engagement(self):
+ """Weekly digest top_findings must be the highest-engagement items, not
+ the most recent. get_new_findings returns first_seen DESC, so without an
+ explicit engagement sort the digest headlines recent low-engagement noise."""
+ with tempfile.TemporaryDirectory() as tmpdir:
+ db_path = Path(tmpdir) / "research.db"
+ briefs_dir = Path(tmpdir) / "briefs"
+ old_db_override = store._db_override
+ old_briefs_dir = briefing.BRIEFS_DIR
+ try:
+ store._db_override = db_path
+ briefing.BRIEFS_DIR = briefs_dir
+ topic = store.add_topic("test topic")
+ run_id = store.record_run(
+ topic["id"], source_mode="v3", status="completed"
+ )
+
+ now = datetime.now(timezone.utc)
+ # Finding i: i days ago, engagement i. The most recent (i=0) has
+ # the lowest engagement, so a recency sort and an engagement sort
+ # disagree on the top 5.
+ conn = store._connect()
+ try:
+ for i in range(6):
+ first_seen = (now - timedelta(days=i, hours=1)).strftime(
+ "%Y-%m-%d %H:%M:%S"
+ )
+ conn.execute(
+ """INSERT INTO findings
+ (run_id, topic_id, source, source_url,
+ source_title, engagement_score, first_seen)
+ VALUES (?, ?, 'reddit', ?, ?, ?, ?)""",
+ (
+ run_id,
+ topic["id"],
+ f"https://example.com/{i}",
+ f"finding-{i}",
+ float(i),
+ first_seen,
+ ),
+ )
+ conn.commit()
+ finally:
+ conn.close()
+
+ result = briefing.generate_weekly()
+ top = result["topics"][0]["top_findings"]
+ scores = [f["engagement_score"] for f in top]
+
+ self.assertEqual(len(top), 5)
+ self.assertEqual(scores, [5.0, 4.0, 3.0, 2.0, 1.0])
+ # The most-recent, lowest-engagement finding is dropped, not kept.
+ self.assertNotIn(0.0, scores)
+ finally:
+ store._db_override = old_db_override
+ briefing.BRIEFS_DIR = old_briefs_dir
+
def test_save_briefing_uses_utf8_encoding(self):
with tempfile.TemporaryDirectory() as tmpdir:
old_briefs_dir = briefing.BRIEFS_DIR
diff --git a/tests/test_cli_v3.py b/tests/test_cli_v3.py
index b7c2f80b..1e346170 100644
--- a/tests/test_cli_v3.py
+++ b/tests/test_cli_v3.py
@@ -64,6 +64,28 @@ class CliV3Tests(unittest.TestCase):
self.assertIn("ranked_candidates", payload)
self.assertIn("clusters", payload)
+ def test_invalid_plan_json_exits_nonzero(self):
+ """Malformed --plan JSON must fail fast, not silently fall back to the
+ internal planner and burn a paid run the user did not ask for."""
+ result = subprocess.run(
+ [
+ sys.executable,
+ "skills/last30days/scripts/last30days.py",
+ "test topic",
+ "--mock",
+ "--emit=json",
+ "--plan",
+ "{not valid json",
+ ],
+ cwd=REPO_ROOT,
+ capture_output=True,
+ text=True,
+ encoding="utf-8",
+ check=False,
+ )
+ self.assertEqual(2, result.returncode, result.stderr)
+ self.assertIn("Invalid --plan JSON", result.stderr)
+
def test_parse_search_flag_normalizes_aliases_and_dedupes(self):
self.assertEqual(
["grounding", "reddit", "hackernews"],
diff --git a/tests/test_evaluator_v3.py b/tests/test_evaluator_v3.py
index 2dd8612e..a882f12e 100644
--- a/tests/test_evaluator_v3.py
+++ b/tests/test_evaluator_v3.py
@@ -115,7 +115,14 @@ class EvaluatorV3Tests(unittest.TestCase):
output_dir = Path(tmp)
cache_dir = output_dir / "judgments"
cache_dir.mkdir()
- (cache_dir / "topic.json").write_text(json.dumps({"judgments": [{"id": "a", "grade": 3}]}))
+ (cache_dir / "topic.json").write_text(
+ json.dumps(
+ {
+ "judge_model": "gemini-3.1-flash-lite",
+ "judgments": [{"id": "a", "grade": 3}],
+ }
+ )
+ )
cached = evaluator.get_judgments(
output_dir=output_dir,
slug="topic",
@@ -138,6 +145,35 @@ class EvaluatorV3Tests(unittest.TestCase):
)
self.assertEqual({}, skipped)
+ def test_get_judgments_remisses_on_judge_model_change(self):
+ """A cache written by a different judge model must not be reused; a
+ --judge-model change forces a re-judge instead of returning stale grades."""
+ with tempfile.TemporaryDirectory() as tmp:
+ output_dir = Path(tmp)
+ cache_dir = output_dir / "judgments"
+ cache_dir.mkdir()
+ (cache_dir / "topic.json").write_text(
+ json.dumps(
+ {
+ "judge_model": "gemini-3.1-flash-lite",
+ "judgments": [{"id": "a", "grade": 3}],
+ }
+ )
+ )
+ # Same slug, different model, no API key to re-judge: the stale
+ # grades must NOT come back — an empty result signals "re-judge
+ # needed" rather than silently wrong numbers.
+ result = evaluator.get_judgments(
+ output_dir=output_dir,
+ slug="topic",
+ topic="test topic",
+ query_type="general",
+ items=[{"key": "a"}],
+ judge_model="gemini-2.5-pro",
+ gemini_api_key=None,
+ )
+ self.assertEqual({}, result)
+
def test_create_eval_env_and_run_last30days(self):
credential_env = {
key: ""