AI agent skill that researches any topic across Reddit, X, YouTube, HN, Polymarket, and the web - then synthesizes a grounded summary
Python ai-promptsai-skillblueskyclaudeclaude-codeclawhubdeep-researchhackernewsinstagramopenclawpolymarketrecencyredditresearchsocial-mediatiktoktrendstwitterweb-searchyoutube
fix(youtube): spend transcript budget on in-window videos first; gate stale-yt-dlp nudge on actual fetch failures (#535)
Hammad Khan92c104b911ed
diff --git a/skills/last30days/scripts/last30days.py b/skills/last30days/scripts/last30days.pyindex cc0cf66e..0c1a1bb4 100644--- a/skills/last30days/scripts/last30days.py+++ b/skills/last30days/scripts/last30days.py@@ -965,10 +965,12 @@ def main() -> int:if not args.hiring_signals:try:from lib import quality_nudge+ from lib import youtube_yt as _youtube_yt# Populate transcript-fetch ratio so quality_nudge can detect the# degraded-YouTube failure mode (videos returned but transcripts# silently failed - typically a stale yt-dlp binary).youtube_items = report.items_by_source.get("youtube") or []+ _yt_fetch_stats = _youtube_yt.get_transcript_fetch_stats()instagram_items = report.items_by_source.get("instagram") or []research_results = {"youtube_videos_count": len(youtube_items),@@ -985,6 +987,13 @@ def main() -> int:"youtube_captions_disabled_count": sum(1 for it in youtube_items if it.metadata.get("captions_disabled")),+ # Actual yt-dlp fetch outcomes for this run. The counts above are+ # computed from post-pruning items, so they can't tell "fetches+ # failed (stale binary)" from "fetches succeeded but the videos+ # were pruned downstream"; the latter was producing false+ # stale-yt-dlp nudges (#531).+ "youtube_transcript_fetch_attempts": _yt_fetch_stats["attempts"],+ "youtube_transcript_fetch_failures": _yt_fetch_stats["failures"],# Track Instagram returned-zero-items so quality_nudge can detect# the silent-failure case (SC configured but the v2 reels endpoint# 500'd through both the original query and the hashtag retry).diff --git a/skills/last30days/scripts/lib/quality_nudge.py b/skills/last30days/scripts/lib/quality_nudge.pyindex dcd5aa76..e1b6545f 100644--- a/skills/last30days/scripts/lib/quality_nudge.py+++ b/skills/last30days/scripts/lib/quality_nudge.py@@ -63,12 +63,23 @@ def _is_youtube_degraded(research_results: dict, threshold: float) -> bool:who turned off captions can never produce a transcript, so counting thatvideo toward "fetch failures" produces false positives. A singlecaptions-disabled video in a small result set was tripping the nudge.++ When actual fetch outcomes are available, they take precedence over the+ post-pruning ratio: the report counts only see items that survived+ freshness/relevance pruning, so a run where every transcript fetch+ succeeded but the fetched videos were later pruned looks identical to a+ stale-binary run (#531). Zero failures across attempted fetches proves+ the binary works - don't flag."""videos = int(research_results.get("youtube_videos_count") or 0)transcripts = int(research_results.get("youtube_transcripts_count") or 0)captions_disabled = int(research_results.get("youtube_captions_disabled_count") or 0)if videos <= 0:return False+ fetch_attempts = int(research_results.get("youtube_transcript_fetch_attempts") or 0)+ fetch_failures = int(research_results.get("youtube_transcript_fetch_failures") or 0)+ if fetch_attempts > 0 and fetch_failures == 0:+ return Falseeligible = videos - captions_disabledif eligible <= 0:# Every returned video had captions disabled - upstream content fact,diff --git a/skills/last30days/scripts/lib/youtube_yt.py b/skills/last30days/scripts/lib/youtube_yt.pyindex 5e7380c2..0a4380f0 100644--- a/skills/last30days/scripts/lib/youtube_yt.py+++ b/skills/last30days/scripts/lib/youtube_yt.py@@ -33,6 +33,25 @@ TRANSCRIPT_LIMITS = {"deep": 8,}+# Cumulative yt-dlp transcript-fetch stats for the current process. The final+# report only sees post-pruning items, so it can't distinguish "fetches failed+# (stale binary)" from "fetches succeeded but the videos were pruned later".+# quality_nudge reads these via last30days.py to suppress the stale-yt-dlp+# nudge when every attempted fetch actually succeeded. yt-dlp path only: the+# nudge diagnoses the local binary, not the ScrapeCreators API.+_TRANSCRIPT_FETCH_STATS = {"attempts": 0, "failures": 0}+++def get_transcript_fetch_stats() -> Dict[str, int]:+ """Return cumulative transcript-fetch stats for this process."""+ return dict(_TRANSCRIPT_FETCH_STATS)+++def reset_transcript_fetch_stats() -> None:+ """Reset cumulative transcript-fetch stats (used by tests)."""+ _TRANSCRIPT_FETCH_STATS["attempts"] = 0+ _TRANSCRIPT_FETCH_STATS["failures"] = 0+# Max words to keep from each transcriptTRANSCRIPT_MAX_WORDS = 5000@@ -712,6 +731,13 @@ def search_and_transcribe(transcripts = fetch_transcripts_parallel(candidate_ids, out_captions_disabled=captions_disabled_ids,)+ # Record fetch outcomes (captions-disabled videos can never succeed,+ # so they don't count as failures) for the stale-yt-dlp nudge.+ _TRANSCRIPT_FETCH_STATS["attempts"] += len(candidate_ids)+ _TRANSCRIPT_FETCH_STATS["failures"] += sum(+ 1 for vid in candidate_ids+ if not transcripts.get(vid) and vid not in captions_disabled_ids+ )else:_log(f"Transcript limit is 0 for depth={depth}, skipping transcript fetch")@@ -966,8 +992,13 @@ def search_youtube_sc(transcript_limit = TRANSCRIPT_LIMITS.get(depth, TRANSCRIPT_LIMITS["default"])if transcript_limit > 0 and items:attempt_count = min(len(items), transcript_limit * 3)+ # Same in-window-first ordering as search_and_transcribe(): don't let+ # an out-of-window back-catalog (kept by the soft date filter above)+ # consume the transcript budget of videos the freshness scorer keeps.+ in_window = [i for i in items if i.get("date") and i["date"] >= from_date]+ out_of_window = [i for i in items if not (i.get("date") and i["date"] >= from_date)]_log(f"Fetching SC transcripts for up to {attempt_count} videos (target: {transcript_limit})")- for item in items[:attempt_count]:+ for item in (in_window + out_of_window)[:attempt_count]:vid = item["video_id"]if not vid:continuediff --git a/tests/test_quality_nudge.py b/tests/test_quality_nudge.pyindex 9802ba47..8f367b31 100644--- a/tests/test_quality_nudge.py+++ b/tests/test_quality_nudge.py@@ -377,6 +377,72 @@ class TestYouTubeCaptionsDisabledDoesNotFalseFlag:assert "youtube" in q["core_degraded"]+class TestStaleNudgeRequiresActualFetchFailures:+ """Zero failed fetches must suppress the stale-yt-dlp nudge (#531).++ The report counts (youtube_videos_count / youtube_transcripts_count) are+ computed from post-pruning items. A run where every transcript fetch+ succeeded but the fetched videos were later pruned by freshness scoring+ looks identical to a stale-binary run from those counts alone, producing+ a false "stale yt-dlp binary" nudge. When actual fetch outcomes are+ available and show zero failures, the binary demonstrably works.+ """++ def test_zero_failures_does_not_flag(self):+ # The #531 repro: 2 in-report videos, 0 transcripts among them, but+ # all 6 attempted fetches succeeded (on videos pruned later).+ q = _compute(+ ytdlp_installed=True,+ result_overrides={+ "youtube_videos_count": 2,+ "youtube_transcripts_count": 0,+ "youtube_captions_disabled_count": 0,+ "youtube_transcript_fetch_attempts": 6,+ "youtube_transcript_fetch_failures": 0,+ },+ )+ assert "youtube" not in q["core_degraded"]++ def test_actual_failures_still_flag(self):+ q = _compute(+ ytdlp_installed=True,+ result_overrides={+ "youtube_videos_count": 6,+ "youtube_transcripts_count": 0,+ "youtube_captions_disabled_count": 0,+ "youtube_transcript_fetch_attempts": 6,+ "youtube_transcript_fetch_failures": 6,+ },+ )+ assert "youtube" in q["core_degraded"]++ def test_missing_fetch_stats_preserves_existing_behavior(self):+ # Callers that don't pass fetch stats (or the SC path, which doesn't+ # use the local yt-dlp binary) fall back to the ratio heuristic.+ q = _compute(+ ytdlp_installed=True,+ result_overrides={+ "youtube_videos_count": 6,+ "youtube_transcripts_count": 0,+ "youtube_captions_disabled_count": 0,+ },+ )+ assert "youtube" in q["core_degraded"]++ def test_zero_attempts_preserves_existing_behavior(self):+ q = _compute(+ ytdlp_installed=True,+ result_overrides={+ "youtube_videos_count": 6,+ "youtube_transcripts_count": 0,+ "youtube_captions_disabled_count": 0,+ "youtube_transcript_fetch_attempts": 0,+ "youtube_transcript_fetch_failures": 0,+ },+ )+ assert "youtube" in q["core_degraded"]++class TestInstagramSilentFailure:"""Instagram is a `bonus` source via SC. Silent-failure detection: if SCis configured but the source returned zero items, surface a nudge so thediff --git a/tests/test_youtube_yt.py b/tests/test_youtube_yt.pyindex 1414e10f..b1b741a8 100644--- a/tests/test_youtube_yt.py+++ b/tests/test_youtube_yt.py@@ -482,6 +482,79 @@ class TestSearchAndTranscribe(unittest.TestCase):ft_mock.assert_not_called()+class TestTranscriptFetchStats(unittest.TestCase):+ """Track yt-dlp fetch outcomes for quality_nudge (#531 false stale-yt-dlp nudge)."""++ FROM_DATE = "2026-03-01"+ TO_DATE = "2026-03-31"++ def setUp(self):+ youtube_yt.reset_transcript_fetch_stats()++ def _make_item(self, video_id, views, date):+ return {+ "video_id": video_id,+ "title": f"Video {video_id}",+ "url": f"https://www.youtube.com/watch?v={video_id}",+ "channel_name": "TestChannel",+ "date": date,+ "engagement": {"views": views, "likes": 10, "comments": 5},+ "relevance": 0.8,+ "why_relevant": "test",+ "description": "test desc",+ "duration": 600,+ }++ def _run(self, items, fake_parallel=None):+ if fake_parallel is None:+ def fake_parallel(video_ids, max_workers=5, out_captions_disabled=None):+ return {vid: "A detailed transcript about the topic." for vid in video_ids}+ with mock.patch.object(youtube_yt, "search_youtube", return_value={"items": items}), \+ mock.patch.object(youtube_yt, "fetch_transcripts_parallel", side_effect=fake_parallel):+ return youtube_yt.search_and_transcribe(+ "test topic", self.FROM_DATE, self.TO_DATE, depth="default",+ )++ def test_fetch_stats_track_attempts_and_failures(self):+ items = [+ self._make_item("ok1", 3_000, "2026-03-20"),+ self._make_item("fail1", 2_000, "2026-03-15"),+ self._make_item("nocap1", 1_000, "2026-03-10"),+ ]++ def fake_parallel(video_ids, max_workers=5, out_captions_disabled=None):+ result = {}+ for vid in video_ids:+ if vid.startswith("nocap"):+ result[vid] = None+ if out_captions_disabled is not None:+ out_captions_disabled.add(vid)+ elif vid.startswith("fail"):+ result[vid] = None+ else:+ result[vid] = "A detailed transcript about the topic."+ return result++ self._run(items, fake_parallel)++ stats = youtube_yt.get_transcript_fetch_stats()+ self.assertEqual(stats["attempts"], 3)+ # Captions-disabled videos can never succeed; they are not failures.+ self.assertEqual(stats["failures"], 1)++ def test_fetch_stats_zero_failures_when_all_succeed(self):+ # The #531 scenario: every fetch succeeds (on videos later pruned by+ # freshness scoring). failures must be 0 so quality_nudge does not+ # blame a stale yt-dlp binary.+ items = [self._make_item(f"v{i}", 1_000 * (i + 1), "2024-01-15") for i in range(4)]++ self._run(items)++ stats = youtube_yt.get_transcript_fetch_stats()+ self.assertEqual(stats["attempts"], 4)+ self.assertEqual(stats["failures"], 0)++class TestYtdlpSSHRouting(unittest.TestCase):"""LAST30DAYS_YOUTUBE_SSH_HOST routes yt-dlp invocations through SSH for residential IP."""