AI agent skill that researches any topic across Reddit, X, YouTube, HN, Polymarket, and the web - then synthesizes a grounded summary
Python ai-promptsai-skillblueskyclaudeclaude-codeclawhubdeep-researchhackernewsinstagramopenclawpolymarketrecencyredditresearchsocial-mediatiktoktrendstwitterweb-searchyoutube
fix(pipeline): fetch each identical source search once per run (#1260)
Matt Van Horn8c4697b222ae
diff --git a/changelog.d/+dedupe-source-fetches.fixed.md b/changelog.d/+dedupe-source-fetches.fixed.mdnew file mode 100644index 00000000..cb4f5779--- /dev/null+++ b/changelog.d/+dedupe-source-fetches.fixed.md@@ -0,0 +1 @@+A multi-subquery run now fetches each identical TikTok, Instagram, YouTube, and X search once instead of once per subquery, and the thin-source retry reuses a clean result instead of repeating the request. Polymarket and Digg no longer run the same search twice when the thin-source retry's simplified query matches the planner's. Failed fetches are not reused, so the retry still makes a real second attempt.diff --git a/skills/last30days/scripts/last30days.py b/skills/last30days/scripts/last30days.pyindex 6bc00cef..fedaeab9 100644--- a/skills/last30days/scripts/last30days.py+++ b/skills/last30days/scripts/last30days.py@@ -50,7 +50,7 @@ if os.name == "nt":SCRIPT_DIR = Path(__file__).parent.resolve()sys.path.insert(0, str(SCRIPT_DIR))-from lib import competitors as competitors_mod, corpus, dates, discovery_handoff, env, freshness, html_render, http, permission_preflight, pipeline, reddit, registers, render, schema, subproc, ui, x_envelope+from lib import competitors as competitors_mod, corpus, dates, discovery_handoff, env, fetch_memo, freshness, html_render, http, permission_preflight, pipeline, reddit, registers, render, schema, subproc, ui, x_envelopefrom lib.private_files import write_private_textatexit.register(subproc.cleanup_children)@@ -3293,6 +3293,7 @@ def _main(# parallel, so the reset must not live inside the pipeline.http.reset_reddit_keyless_memo()reddit.reset_scrapecreators_memo()+ fetch_memo.reset()resolved_corpus_dirs = corpus.resolve_directories(args.corpus, config.get("LAST30DAYS_CORPUS_DIRS"))diff --git a/skills/last30days/scripts/lib/fetch_memo.py b/skills/last30days/scripts/lib/fetch_memo.pynew file mode 100644index 00000000..c65bd6d8--- /dev/null+++ b/skills/last30days/scripts/lib/fetch_memo.py@@ -0,0 +1,135 @@+"""Run-scoped memo for source fetches.++Subquery streams for TikTok, Instagram, YouTube, and X all search the raw+topic, and the thin-source retry repeats that search (and, for Polymarket and+Digg, a planner subquery that equals the retry's core subject), so one run can+issue the same request several times. :func:`fetch_once` runs a fetch at most once per+key: the first caller fetches, concurrent callers wait for it (bounded by+their own timeout), and later callers get a copy of the cached result.++Only clean results are cached. A failure (a raised exception, a result the+caller's ``cacheable`` predicate rejects, or HTTP failures an adapter+swallowed into the failure sink) reaches only the callers already waiting on+that fetch; the next call fetches again, so the thin-source retry still makes+a real second attempt after a transient failure.++Reset once per command (``last30days.py``) and between tests (``conftest``).+"""++from __future__ import annotations++import copy+import threading+import time+from typing import Any, Callable, Hashable, TypeVar++from . import health, http++T = TypeVar("T")+++class MemoWaitTimeout(TimeoutError):+ """A waiter's budget expired before the in-flight fetch it shares finished."""++ outcome_state = health.TIMEOUT+++class _Slot:+ __slots__ = ("event", "outcome")++ def __init__(self) -> None:+ self.event = threading.Event()+ # (ok, value_or_exception, recorded_failures) once the owner finishes.+ self.outcome: tuple[bool, Any, list] | None = None+++_CACHE: dict[Hashable, Any] = {}+_INFLIGHT: dict[Hashable, _Slot] = {}+_LOCK = threading.Lock()+++def reset() -> None:+ """Forget cached and in-flight fetches. Called once per command, and by tests."""+ with _LOCK:+ _CACHE.clear()+ _INFLIGHT.clear()+++def _always(_value: Any) -> bool:+ return True+++def _deliver_shared(outcome: tuple[bool, Any, list]) -> Any:+ ok, value, failures = outcome+ # The owner's swallowed HTTP failures reach this caller's sink too, or a+ # shared empty result reads as a clean no-results.+ for failure in failures:+ http._record_failure(failure)+ if not ok:+ raise value+ return copy.deepcopy(value)+++def fetch_once(+ key: Hashable,+ fetch: Callable[[], T],+ *,+ timeout: float,+ cacheable: Callable[[T], bool] = _always,+) -> T:+ """Return ``fetch()``, running it at most once per ``key`` while it stays clean.++ ``key`` must include every input that changes the request. ``timeout``+ bounds how long a caller waits on another caller's in-flight fetch; past+ it, :class:`MemoWaitTimeout` is raised. Every caller gets its own copy.+ """+ deadline = time.monotonic() + max(0.0, timeout)+ while True:+ with _LOCK:+ hit = key in _CACHE+ if hit:+ cached = _CACHE[key]+ else:+ slot = _INFLIGHT.get(key)+ owner = slot is None+ if owner:+ slot = _Slot()+ _INFLIGHT[key] = slot+ if hit:+ return copy.deepcopy(cached)+ if owner:+ break+ if not slot.event.wait(max(0.0, deadline - time.monotonic())):+ raise MemoWaitTimeout(+ f"waited {timeout:g}s for an identical in-flight fetch"+ )+ if slot.outcome is not None:+ return _deliver_shared(slot.outcome)+ # The owner was interrupted before publishing (e.g. KeyboardInterrupt):+ # loop and elect a new owner.++ try:+ with http.tee_failures() as recorded:+ try:+ value = fetch()+ ok = True+ except Exception as exc:+ value = exc+ ok = False+ clean = ok and not recorded and cacheable(value)+ # Waiters and the cache copy from this snapshot, never from the+ # owner's own value, which the owner's caller may mutate.+ shared = copy.deepcopy(value) if ok else value+ slot.outcome = (ok, shared, list(recorded))+ with _LOCK:+ # Slot identity guards against a reset() during the fetch.+ if clean and _INFLIGHT.get(key) is slot:+ _CACHE[key] = shared+ finally:+ with _LOCK:+ if _INFLIGHT.get(key) is slot:+ _INFLIGHT.pop(key)+ slot.event.set()+ if not ok:+ raise value+ return valuediff --git a/skills/last30days/scripts/lib/pipeline.py b/skills/last30days/scripts/lib/pipeline.pyindex 2f67f43f..8b75896b 100644--- a/skills/last30days/scripts/lib/pipeline.py+++ b/skills/last30days/scripts/lib/pipeline.py@@ -34,6 +34,7 @@ from . import (dripstack,entity_extract,env,+ fetch_memo,github,grok_x,grounding,@@ -5302,6 +5303,217 @@ def _serve_envelope_lanes()+# Artifact keys that mark a stream result as a failure (or a deferred one).+# The fetch memo never caches such a result, so the thin-source retry and the+# next stream still make a real attempt after a transient failure.+_STREAM_FAILURE_ARTIFACT_KEYS = (+ "_source_outcome",+ "_source_outcome_deferred",+ "_source_outcome_if_empty",+ "_source_outcome_detail_state",+)+++def _clean_stream_result(result: tuple[list[dict], dict]) -> bool:+ _items, artifact = result+ return not (+ isinstance(artifact, dict)+ and any(artifact.get(key) for key in _STREAM_FAILURE_ARTIFACT_KEYS)+ )+++def _clean_search_result(result: Any) -> bool:+ """A legacy ``{"error": ...}`` search result is a failure and is not cached."""+ return not (isinstance(result, dict) and result.get("error"))+++def _memo_wait_seconds(config: dict[str, Any]) -> float:+ """How long a stream may wait on an identical in-flight fetch: its own budget."""+ return max(+ 0.0,+ _research_deadline(config, STREAM_FUTURE_TIMEOUT_SECONDS) - time.monotonic(),+ )+++_X_MEMO_WAIT_GRACE_SECONDS = 5.0+++def _attempt_x_backend_memo(+ backend, query, from_date, to_date, depth, config, warnings=None, deadline=None,+):+ """:func:`_attempt_x_backend`, once per backend and query per run.++ Only the backend call is shared: the judge and prune still run per+ subquery with that subquery's ranking query.+ """+ def _fetch():+ local_warnings: list[str] = []+ items, err, err_state = _attempt_x_backend(+ backend, query, from_date, to_date, depth, config,+ warnings=local_warnings, deadline=deadline,+ )+ return items, err, err_state, local_warnings++ # The owner is bounded by the same chain deadline, so a waiter allows a+ # short grace past it for the owner to publish instead of timing out first.+ wait = (+ max(0.0, deadline - time.monotonic()) + _X_MEMO_WAIT_GRACE_SECONDS+ if deadline is not None+ else _memo_wait_seconds(config)+ )+ try:+ items, err, err_state, backend_warnings = fetch_memo.fetch_once(+ ("x", backend, query, from_date, to_date, depth),+ _fetch,+ timeout=wait,+ cacheable=lambda outcome: not outcome[1],+ )+ except fetch_memo.MemoWaitTimeout as exc:+ # A chain error like any backend timeout, so the chain's own error+ # handling and outcome reporting apply.+ return [], str(exc), health.TIMEOUT+ if warnings is not None:+ warnings.extend(backend_warnings)+ return items, err, err_state+++def _fetch_youtube_stream(+ yt_query: str,+ from_date: str,+ to_date: str,+ depth: str,+ config: dict[str, Any],+ *,+ has_ytdlp: bool,+) -> tuple[list[dict], dict]:+ """Search, backfill, transcribe, and comment-enrich one YouTube query."""+ result = None+ youtube_failure: str | None = None+ recovered_failure: str | None = None+ # ScrapeCreators key (when present) is the default-on backup tier: it+ # powers the per-video transcript fallback, the SC search fallback, and+ # comment enrichment. None when no key, which keeps everything keyless.+ sc_token = (+ config.get("SCRAPECREATORS_API_KEY", "")+ if env.is_youtube_sc_available(config) else None+ )+ # Try yt-dlp first; the SC transcript fallback covers per-video failures.+ if has_ytdlp:+ try:+ result = youtube_yt.search_and_transcribe(+ yt_query, from_date, to_date, depth=depth, token=sc_token,+ )+ if result.get("error"):+ youtube_failure = str(result["error"])+ except Exception as exc:+ youtube_failure = str(exc)+ result = None+ free_items = list((result or {}).get("items") or [])+ min_items = env.youtube_sc_min_items(config)+ backfill_thin = bool(free_items) and len(free_items) < min_items+ thin_detail: str | None = None+ thin_error = False+ thin_retryable = False+ if (not free_items or backfill_thin) and sc_token:+ backfill_scope = (+ http.capture_failures() if free_items else contextlib.nullcontext([])+ )+ try:+ with backfill_scope as sc_failures:+ transcribed_free_ids = {+ item["video_id"] for item in free_items+ if item.get("video_id")+ and str(item.get("transcript_snippet") or "").strip()+ }+ sc_result = youtube_yt.search_youtube_sc(+ yt_query, from_date, to_date, depth=depth, token=sc_token,+ skip_transcript_ids=transcribed_free_ids,+ )+ sc_error = str(sc_result["error"]) if sc_result.get("error") else None+ if backfill_thin:+ merged = _merge_youtube_items(free_items, sc_result.get("items") or [])+ result = {"items": merged}+ thin_detail = (+ f"yt-dlp returned {len(free_items)} videos below the "+ f"{min_items}-video backfill floor; ScrapeCreators added "+ f"{len(merged) - len(free_items)} videos"+ )+ if youtube_failure:+ thin_detail += f"; yt-dlp search: {youtube_failure}"+ if sc_error:+ thin_detail += f"; ScrapeCreators search: {sc_error}"+ if sc_failures:+ thin_detail += f"; ScrapeCreators: {_summarize_lane_failures(sc_failures, 'youtube')}"+ thin_error = bool(youtube_failure or sc_error or sc_failures)+ thin_retryable = thin_error and (+ len(merged) < min_items+ and (not youtube_failure or _retryable_youtube_error(youtube_failure))+ and (not sc_error or _retryable_youtube_error(sc_error))+ and all(+ _retryable_youtube_error(str(f), f.outcome_state)+ for f in sc_failures+ )+ )+ youtube_failure = None+ recovered_failure = None+ else:+ result = sc_result+ if sc_error:+ youtube_failure = sc_error+ elif result.get("items") and youtube_failure:+ recovered_failure = youtube_failure+ youtube_failure = None+ except Exception as exc:+ prior_youtube_failure = youtube_failure+ youtube_failure = str(exc)+ result = {"items": free_items} if free_items else None+ if backfill_thin:+ thin_error = True+ thin_detail = (+ f"yt-dlp returned {len(free_items)} videos below the "+ f"{min_items}-video backfill floor; ScrapeCreators failed: {exc}"+ )+ if prior_youtube_failure:+ thin_detail += f"; yt-dlp search: {prior_youtube_failure}"+ thin_retryable = (+ _retryable_youtube_error(+ str(exc), getattr(exc, "outcome_state", None)+ )+ and (not prior_youtube_failure or _retryable_youtube_error(prior_youtube_failure))+ )+ if result is None:+ result = {"items": []}+ # Enrich top videos with comments (default-on when a key is present).+ items = youtube_yt.parse_youtube_response(result)+ if items and env.is_youtube_comments_available(config):+ youtube_yt.enrich_with_comments(+ items, token=config.get("SCRAPECREATORS_API_KEY", ""),+ )+ if thin_detail:+ if not thin_error:+ return items, {"_source_outcome_detail": thin_detail}+ outcome = _outcome_artifact(health.PARTIAL, thin_detail)+ if thin_retryable:+ return items, {+ "_source_outcome_detail": thin_detail,+ "_source_outcome_deferred": outcome["_source_outcome"],+ }+ return items, outcome+ if youtube_failure:+ state = youtube_yt.classify_run_failure(youtube_failure)+ attempted = state != schema.SKIPPED_UNCONFIGURED+ return items, _outcome_artifact(state, youtube_failure, attempted=attempted)+ if recovered_failure:+ state = youtube_yt.classify_run_failure(recovered_failure)+ attempted = state != schema.SKIPPED_UNCONFIGURED+ return items, {+ "_source_outcome_if_empty": _outcome_artifact(+ state, recovered_failure, attempted=attempted,+ )["_source_outcome"]+ }+ return items, {}++def _retrieve_stream_impl(*,topic: str,@@ -5553,7 +5765,7 @@ def _retrieve_stream_impl(print("[X] chain budget exhausted; skipping remaining backends", file=sys.stderr)breakwith http.tee_failures() as backend_failures:- items, err, err_state = _attempt_x_backend(+ items, err, err_state = _attempt_x_backend_memo(backend, x_query, from_date, to_date, depth, config, warnings=x_warnings,deadline=chain_deadline,)@@ -5666,7 +5878,7 @@ def _retrieve_stream_impl(if _research_stopped(config) or time.monotonic() >= chain_deadline:print("[X] chain budget exhausted; skipping judge retry", file=sys.stderr)else:- retry_items, retry_err, _ = _attempt_x_backend(+ retry_items, retry_err, _ = _attempt_x_backend_memo(used_backend, retry_query, from_date, to_date, depth, config,deadline=chain_deadline,)@@ -5740,167 +5952,87 @@ def _retrieve_stream_impl(# Use raw_topic so expand_youtube_queries() generates diverse variants# from the original user topic, not the planner's narrowed search_query.yt_query = raw_topic or subquery.search_query- result = None- youtube_failure: str | None = None- recovered_failure: str | None = None- # ScrapeCreators key (when present) is the default-on backup tier: it- # powers the per-video transcript fallback, the SC search fallback, and- # comment enrichment. None when no key, which keeps everything keyless.- sc_token = (- config.get("SCRAPECREATORS_API_KEY", "")- if env.is_youtube_sc_available(config) else None+ has_ytdlp = bool(which("yt-dlp"))+ # One memo unit covers yt-dlp search, the ScrapeCreators fallback and+ # merge, transcripts, and comments, so subquery streams and the+ # thin-source retry share one fetch (hosts without yt-dlp included).+ return fetch_memo.fetch_once(+ (+ "youtube", yt_query, from_date, to_date, depth, has_ytdlp,+ env.is_youtube_sc_available(config),+ env.youtube_sc_min_items(config),+ env.is_youtube_comments_available(config),+ ),+ lambda: _fetch_youtube_stream(+ yt_query, from_date, to_date, depth, config, has_ytdlp=has_ytdlp,+ ),+ timeout=_memo_wait_seconds(config),+ cacheable=_clean_stream_result,)- # Try yt-dlp first; the SC transcript fallback covers per-video failures.- if which("yt-dlp"):- try:- result = youtube_yt.search_and_transcribe(- yt_query, from_date, to_date, depth=depth, token=sc_token,- )- if result.get("error"):- youtube_failure = str(result["error"])- except Exception as exc:- youtube_failure = str(exc)- result = None- free_items = list((result or {}).get("items") or [])- min_items = env.youtube_sc_min_items(config)- backfill_thin = bool(free_items) and len(free_items) < min_items- thin_detail: str | None = None- thin_error = False- thin_retryable = False- if (not free_items or backfill_thin) and sc_token:- backfill_scope = (- http.capture_failures() if free_items else contextlib.nullcontext([])- )- try:- with backfill_scope as sc_failures:- transcribed_free_ids = {- item["video_id"] for item in free_items- if item.get("video_id")- and str(item.get("transcript_snippet") or "").strip()- }- sc_result = youtube_yt.search_youtube_sc(- yt_query, from_date, to_date, depth=depth, token=sc_token,- skip_transcript_ids=transcribed_free_ids,- )- sc_error = str(sc_result["error"]) if sc_result.get("error") else None- if backfill_thin:- merged = _merge_youtube_items(free_items, sc_result.get("items") or [])- result = {"items": merged}- thin_detail = (- f"yt-dlp returned {len(free_items)} videos below the "- f"{min_items}-video backfill floor; ScrapeCreators added "- f"{len(merged) - len(free_items)} videos"- )- if youtube_failure:- thin_detail += f"; yt-dlp search: {youtube_failure}"- if sc_error:- thin_detail += f"; ScrapeCreators search: {sc_error}"- if sc_failures:- thin_detail += f"; ScrapeCreators: {_summarize_lane_failures(sc_failures, 'youtube')}"- thin_error = bool(youtube_failure or sc_error or sc_failures)- thin_retryable = thin_error and (- len(merged) < min_items- and (not youtube_failure or _retryable_youtube_error(youtube_failure))- and (not sc_error or _retryable_youtube_error(sc_error))- and all(- _retryable_youtube_error(str(f), f.outcome_state)- for f in sc_failures- )- )- youtube_failure = None- recovered_failure = None- else:- result = sc_result- if sc_error:- youtube_failure = sc_error- elif result.get("items") and youtube_failure:- recovered_failure = youtube_failure- youtube_failure = None- except Exception as exc:- prior_youtube_failure = youtube_failure- youtube_failure = str(exc)- result = {"items": free_items} if free_items else None- if backfill_thin:- thin_error = True- thin_detail = (- f"yt-dlp returned {len(free_items)} videos below the "- f"{min_items}-video backfill floor; ScrapeCreators failed: {exc}"- )- if prior_youtube_failure:- thin_detail += f"; yt-dlp search: {prior_youtube_failure}"- thin_retryable = (- _retryable_youtube_error(- str(exc), getattr(exc, "outcome_state", None)- )- and (not prior_youtube_failure or _retryable_youtube_error(prior_youtube_failure))- )- if result is None:- result = {"items": []}- # Enrich top videos with comments (default-on when a key is present).- items = youtube_yt.parse_youtube_response(result)- if items and env.is_youtube_comments_available(config):- youtube_yt.enrich_with_comments(- items, token=config.get("SCRAPECREATORS_API_KEY", ""),- )- if thin_detail:- if not thin_error:- return items, {"_source_outcome_detail": thin_detail}- outcome = _outcome_artifact(health.PARTIAL, thin_detail)- if thin_retryable:- return items, {- "_source_outcome_detail": thin_detail,- "_source_outcome_deferred": outcome["_source_outcome"],- }- return items, outcome- if youtube_failure:- state = youtube_yt.classify_run_failure(youtube_failure)- attempted = state != schema.SKIPPED_UNCONFIGURED- return items, _outcome_artifact(state, youtube_failure, attempted=attempted)- if recovered_failure:- state = youtube_yt.classify_run_failure(recovered_failure)- attempted = state != schema.SKIPPED_UNCONFIGURED- return items, {- "_source_outcome_if_empty": _outcome_artifact(- state, recovered_failure, attempted=attempted,- )["_source_outcome"]- }- return items, {}if source == "tiktok":# Use raw_topic so expand_tiktok_queries() generates diverse variants# from the original user topic, not the planner's narrowed search_query.tiktok_query = raw_topic or subquery.search_query- result = tiktok.search_and_enrich(- tiktok_query,- from_date,- to_date,- depth=depth,- token=env.get_tiktok_token(config),- hashtags=tiktok_hashtags,- creators=tiktok_creators,++ def _fetch_tiktok() -> tuple[list[dict], dict]:+ result = tiktok.search_and_enrich(+ tiktok_query,+ from_date,+ to_date,+ depth=depth,+ token=env.get_tiktok_token(config),+ hashtags=tiktok_hashtags,+ creators=tiktok_creators,+ )+ items = tiktok.parse_tiktok_response(result)+ if items and env.is_tiktok_comments_available(config):+ sc_token = config.get("SCRAPECREATORS_API_KEY", "")+ tiktok.enrich_with_comments(items, token=sc_token)+ return items, _result_outcome_artifact(source, result)++ return fetch_memo.fetch_once(+ (+ "tiktok", tiktok_query, from_date, to_date, depth,+ bool(env.get_tiktok_token(config)),+ tuple(tiktok_hashtags or ()), tuple(tiktok_creators or ()),+ env.is_tiktok_comments_available(config),+ ),+ _fetch_tiktok,+ timeout=_memo_wait_seconds(config),+ cacheable=_clean_stream_result,)- items = tiktok.parse_tiktok_response(result)- if items and env.is_tiktok_comments_available(config):- sc_token = config.get("SCRAPECREATORS_API_KEY", "")- tiktok.enrich_with_comments(items, token=sc_token)- return items, _result_outcome_artifact(source, result)if source == "instagram":# Use raw_topic so expand_instagram_queries() generates diverse variants# from the original user topic, not the planner's narrowed search_query.ig_query = raw_topic or subquery.search_query- result = instagram.search_and_enrich(- ig_query,- from_date,- to_date,- depth=depth,- token=env.get_instagram_token(config),- ig_creators=ig_creators,- )- items = instagram.parse_instagram_response(result)- if items and env.is_instagram_comments_available(config):- instagram.enrich_with_comments(- items, token=config.get("SCRAPECREATORS_API_KEY", ""),++ def _fetch_instagram() -> tuple[list[dict], dict]:+ result = instagram.search_and_enrich(+ ig_query,+ from_date,+ to_date,+ depth=depth,+ token=env.get_instagram_token(config),+ ig_creators=ig_creators,)- return items, _result_outcome_artifact(source, result)+ items = instagram.parse_instagram_response(result)+ if items and env.is_instagram_comments_available(config):+ instagram.enrich_with_comments(+ items, token=config.get("SCRAPECREATORS_API_KEY", ""),+ )+ return items, _result_outcome_artifact(source, result)++ return fetch_memo.fetch_once(+ (+ "instagram", ig_query, from_date, to_date, depth,+ bool(env.get_instagram_token(config)),+ tuple(ig_creators or ()),+ env.is_instagram_comments_available(config),+ ),+ _fetch_instagram,+ timeout=_memo_wait_seconds(config),+ cacheable=_clean_stream_result,+ )if source == "linkedin":token = config.get("SCRAPECREATORS_API_KEY", "")result = linkedin.search_linkedin(@@ -5955,7 +6087,14 @@ def _retrieve_stream_impl(_result_outcome_artifact(source, result),)if source == "digg":- result = digg.search_digg(subquery.search_query, from_date, to_date, depth=depth)+ # The thin-source retry's core subject often equals a planner+ # subquery's search, so share the identical CLI call within a run.+ result = fetch_memo.fetch_once(+ ("digg", subquery.search_query, from_date, to_date, depth),+ lambda: digg.search_digg(subquery.search_query, from_date, to_date, depth=depth),+ timeout=_memo_wait_seconds(config),+ cacheable=_clean_search_result,+ )items = digg.parse_digg_response(result, query=subquery.search_query)# Enrichment with attached X posts is deferred to# _finalize_items_by_source so it runs on the items that actually@@ -6093,7 +6232,16 @@ def _retrieve_stream_impl(result = truthsocial.search_truthsocial(subquery.search_query, from_date, to_date, depth=depth, config=config)return truthsocial.parse_truthsocial_response(result), _result_outcome_artifact(source, result)if source == "polymarket":- result = polymarket.search_polymarket(subquery.search_query, from_date, to_date, depth=depth)+ # Same sharing as digg: only the search is memoized; parsing still+ # runs per caller with its own relevance topic.+ result = fetch_memo.fetch_once(+ ("polymarket", subquery.search_query, from_date, to_date, depth),+ lambda: polymarket.search_polymarket(+ subquery.search_query, from_date, to_date, depth=depth,+ ),+ timeout=_memo_wait_seconds(config),+ cacheable=_clean_search_result,+ )# Relevance filtering keys off the stable original research topic, not the# per-subquery search_query (which narrows differently on each fanout pass# and would let off-topic markets through on broad subqueries while droppingdiff --git a/tests/conftest.py b/tests/conftest.pyindex 4e2f4636..f9ab7c81 100644--- a/tests/conftest.py+++ b/tests/conftest.py@@ -138,15 +138,18 @@ def _reset_probe_caches():@pytest.fixture(autouse=True)def _reset_reddit_keyless_memo():- """The Reddit run memos (keyless and ScrapeCreators) live for one command; tests are their own commands."""+ """The run memos (Reddit keyless and ScrapeCreators, source fetches) live for one command; tests are their own commands."""+ from lib import fetch_memo as _fetch_memofrom lib import http as _httpfrom lib import reddit as _reddit_http.reset_reddit_keyless_memo()_reddit.reset_scrapecreators_memo()+ _fetch_memo.reset()yield_http.reset_reddit_keyless_memo()_reddit.reset_scrapecreators_memo()+ _fetch_memo.reset()@pytest.fixture(autouse=True)diff --git a/tests/test_fetch_memo.py b/tests/test_fetch_memo.pynew file mode 100644index 00000000..809423e2--- /dev/null+++ b/tests/test_fetch_memo.py@@ -0,0 +1,165 @@+"""Run-scoped fetch memo (U3, R5): one network fetch per distinct key per run,+shared by concurrent callers as independent copies. A failure reaches only the+callers already waiting on it; a later call fetches again."""++import threading+from concurrent.futures import ThreadPoolExecutor++import pytest++from lib import fetch_memo, health, http+++def _fetch_counter(payload):+ calls = []++ def _fetch():+ calls.append(1)+ return payload++ return calls, _fetch+++def test_concurrent_same_key_fetches_once_and_returns_independent_copies():+ release = threading.Event()+ arrived = threading.Barrier(4, timeout=5)+ calls = []++ def _slow():+ calls.append(1)+ release.wait(timeout=5)+ return {"items": [{"id": "a", "tags": []}]}++ def _call():+ arrived.wait()+ return fetch_memo.fetch_once(("tiktok", "q"), _slow, timeout=10)++ with ThreadPoolExecutor(max_workers=4) as pool:+ futures = [pool.submit(_call) for _ in range(4)]+ # Let every caller reach the memo before the owner finishes.+ threading.Timer(0.2, release.set).start()+ results = [f.result(timeout=10) for f in futures]++ assert len(calls) == 1+ assert all(r == {"items": [{"id": "a", "tags": []}]} for r in results)+ results[0]["items"][0]["tags"].append("mutated")+ assert all(r["items"][0]["tags"] == [] for r in results[1:])+++def test_sequential_same_key_serves_an_independent_copy():+ calls, fetch = _fetch_counter({"items": [{"id": "a"}]})+ first = fetch_memo.fetch_once(("youtube", "q"), fetch, timeout=1)+ first["items"].append({"id": "mutated"})+ second = fetch_memo.fetch_once(("youtube", "q"), fetch, timeout=1)+ assert len(calls) == 1+ assert second == {"items": [{"id": "a"}]}+++def test_different_keys_fetch_separately():+ calls, fetch = _fetch_counter({"items": []})+ fetch_memo.fetch_once(("youtube", "q", "2026-09-01", "2026-10-01", "default"), fetch, timeout=1)+ fetch_memo.fetch_once(("youtube", "q", "2026-09-02", "2026-10-01", "default"), fetch, timeout=1)+ fetch_memo.fetch_once(("youtube", "q", "2026-09-01", "2026-10-01", "deep"), fetch, timeout=1)+ assert len(calls) == 3+++def test_owner_failure_reaches_concurrent_waiters_and_a_later_call_refetches():+ release = threading.Event()+ arrived = threading.Barrier(3, timeout=5)+ calls = []+ boom = RuntimeError("HTTP 503: Service Unavailable")++ def _failing():+ calls.append(1)+ release.wait(timeout=5)+ raise boom++ def _call():+ arrived.wait()+ return fetch_memo.fetch_once(("x", "q"), _failing, timeout=10)++ with ThreadPoolExecutor(max_workers=3) as pool:+ futures = [pool.submit(_call) for _ in range(3)]+ threading.Timer(0.2, release.set).start()+ errors = []+ for future in futures:+ with pytest.raises(RuntimeError) as exc_info:+ future.result(timeout=10)+ errors.append(exc_info.value)++ assert len(calls) == 1+ assert all(err is boom for err in errors)++ later_calls, later = _fetch_counter({"items": [{"id": "b"}]})+ assert fetch_memo.fetch_once(("x", "q"), later, timeout=1) == {"items": [{"id": "b"}]}+ assert len(later_calls) == 1+++def test_uncacheable_result_reaches_waiters_but_a_later_call_refetches():+ calls, fetch = _fetch_counter({"items": [], "error": "HTTP 429"})+ clean = lambda value: not value.get("error") # noqa: E731+ fetch_memo.fetch_once(("tiktok", "q"), fetch, timeout=1, cacheable=clean)+ fetch_memo.fetch_once(("tiktok", "q"), fetch, timeout=1, cacheable=clean)+ assert len(calls) == 2+++def test_swallowed_failure_reaches_waiters_and_is_not_cached():+ # Source adapters swallow HTTP errors into the stream's failure sink. A+ # waiter must see the owner's failure, and the result must not be cached.+ release = threading.Event()+ arrived = threading.Barrier(2, timeout=5)+ calls = []++ def _swallow_429():+ calls.append(1)+ release.wait(timeout=5)+ http._record_failure(http.HTTPError("HTTP 429: Too Many Requests", status_code=429))+ return {"items": []}++ def _call():+ arrived.wait()+ with http.capture_failures() as sink:+ result = fetch_memo.fetch_once(("instagram", "q"), _swallow_429, timeout=10)+ return result, [f.status_code for f in sink]++ with ThreadPoolExecutor(max_workers=2) as pool:+ futures = [pool.submit(_call) for _ in range(2)]+ threading.Timer(0.2, release.set).start()+ results = [f.result(timeout=10) for f in futures]++ assert len(calls) == 1+ assert all(sink == [429] for _result, sink in results)+ fetch_memo.fetch_once(("instagram", "q"), _swallow_429, timeout=1)+ assert len(calls) == 2+++def test_waiter_times_out_instead_of_blocking_past_its_budget():+ release = threading.Event()+ owner_started = threading.Event()++ def _hung():+ owner_started.set()+ release.wait(timeout=5)+ return {"items": []}++ owner = threading.Thread(+ target=fetch_memo.fetch_once, args=(("youtube", "q"), _hung), kwargs={"timeout": 10},+ daemon=True,+ )+ owner.start()+ assert owner_started.wait(timeout=5)+ try:+ with pytest.raises(fetch_memo.MemoWaitTimeout) as exc_info:+ fetch_memo.fetch_once(("youtube", "q"), _hung, timeout=0.1)+ assert exc_info.value.outcome_state == health.TIMEOUT+ finally:+ release.set()+ owner.join(timeout=5)+++def test_reset_forgets_cached_results():+ calls, fetch = _fetch_counter({"items": []})+ fetch_memo.fetch_once(("youtube", "q"), fetch, timeout=1)+ fetch_memo.reset()+ fetch_memo.fetch_once(("youtube", "q"), fetch, timeout=1)+ assert len(calls) == 2diff --git a/tests/test_pipeline_cancellation.py b/tests/test_pipeline_cancellation.pyindex bd473180..57fe053e 100644--- a/tests/test_pipeline_cancellation.py+++ b/tests/test_pipeline_cancellation.py@@ -518,7 +518,6 @@ def test_grok_short_remaining_budget_reports_timeout(monkeypatch, partial):def test_rate_limit_stops_queued_streams_before_fanout_finishes(monkeypatch, x_run, failure):topic, _, plan, _ = x_runplan.subqueries = [replace(plan.subqueries[0], label=f"query-{index}") for index in range(6)]- barrier = threading.Barrier(4)calls = []lock = threading.Lock()@@ -526,13 +525,13 @@ def test_rate_limit_stops_queued_streams_before_fanout_finishes(monkeypatch, x_rwith lock:index = len(calls)calls.append(index)- if index < 4:- barrier.wait(2)if index == 0:+ # Hold the 429 until the other in-flight streams are waiting on+ # this same request in the run's fetch memo.+ threading.Event().wait(0.3)if failure == "exception":raise http.HTTPError("Rate limit exceeded", status_code=429)return {"items": [], "error": "xapi: rate limited (429)"}- threading.Event().wait(0.2)return {"items": []}monkeypatch.setattr(pipeline.x_api, "search_x", request)@@ -540,5 +539,7 @@ def test_rate_limit_stops_queued_streams_before_fanout_finishes(monkeypatch, x_rtopic=topic, config={"X_BEARER_TOKEN": "dummy", "_max_source_fetches": 6},depth="quick", requested_sources=["x"], internal_subrun=True,)- assert calls == [0, 1, 2, 3], "queued X streams must see the completed worker's rate-limit signal"+ # Every stream searches the same raw topic: in-flight streams share the+ # one request and its 429, and queued streams see the rate-limit signal.+ assert calls == [0], "X streams must share the rate-limited request, not repeat it"assert report.source_status["x"].state == health.RATE_LIMITEDdiff --git a/tests/test_pipeline_v3.py b/tests/test_pipeline_v3.pyindex 8d771567..a1508224 100644--- a/tests/test_pipeline_v3.py+++ b/tests/test_pipeline_v3.py@@ -1,5 +1,8 @@import threading+import timeimport unittest++import pytestfrom unittest.mock import patchfrom lib import health@@ -2591,5 +2594,357 @@ class TestAmazonSourceGating:assert pipeline.MAX_SOURCE_FETCHES["amazon"] == 1+# ---------------------------------------------------------------------------+# Run-scoped source fetch memo (U4, R5, R6): subquery streams that repeat the+# raw-topic search for TikTok, Instagram, YouTube, and X fetch it once.+# ---------------------------------------------------------------------------++_MEMO_TOPIC = "ghostty terminal"+_MEMO_RANGE = ("2026-09-11", "2026-10-11")+_MEMO_SC_CONFIG = {+ "SCRAPECREATORS_API_KEY": "test-key",+ "INCLUDE_SOURCES": "tiktok,instagram,tiktok_comments,instagram_comments",+}+++def _memo_subqueries(sources, labels=("price", "speed", "themes")):+ return [+ schema.SubQuery(+ label=label,+ search_query=f"ghostty {label}",+ ranking_query=f"What do people say about ghostty {label}?",+ sources=list(sources),+ )+ for label in labels+ ]+++def _raw_topic_subquery(source):+ return schema.SubQuery(+ label="primary",+ search_query=_MEMO_TOPIC,+ ranking_query=f"What do people say about {_MEMO_TOPIC}?",+ sources=[source],+ )+++def _memo_stream(source, subquery, config, **kwargs):+ return pipeline._retrieve_stream(+ topic=_MEMO_TOPIC, subquery=subquery, source=source, config=config,+ depth="default", date_range=_MEMO_RANGE, runtime=_make_runtime(None),+ mock=False, raw_topic=_MEMO_TOPIC, **kwargs,+ )+++def _memo_streams_concurrently(jobs):+ from concurrent.futures import ThreadPoolExecutor++ with ThreadPoolExecutor(max_workers=len(jobs)) as pool:+ futures = [pool.submit(_memo_stream, *job) for job in jobs]+ return [future.result(timeout=10) for future in futures]+++def _tiktok_payload(*_args, **_kwargs):+ return {"items": [{+ "id": "t1", "video_id": "t1", "text": "ghostty terminal setup tour",+ "url": "https://www.tiktok.com/@dev/video/1", "author_name": "dev",+ "date": "2026-10-01", "engagement": {"views": 900, "likes": 90},+ "relevance": 0.9, "why_relevant": "fixture",+ }]}+++def _instagram_payload(*_args, **_kwargs):+ return {"items": [{+ "id": "i1", "video_id": "i1", "text": "ghostty terminal reel",+ "url": "https://www.instagram.com/reel/abc/", "author_name": "dev",+ "date": "2026-10-01", "engagement": {"views": 500, "likes": 50},+ "relevance": 0.9, "why_relevant": "fixture",+ }]}+++def _youtube_payload(*_args, **_kwargs):+ return {"items": [+ {+ "video_id": f"vid{n:08d}", "title": f"ghostty terminal review {n}",+ "url": f"https://www.youtube.com/watch?v=vid{n:08d}", "channel_name": "dev",+ "date": "2026-10-01", "engagement": {"views": 1000 - n, "likes": 10},+ "relevance": 0.9, "why_relevant": "fixture",+ }+ for n in range(6)+ ]}+++class TestSourceFetchMemo:+ def test_three_subquery_streams_fetch_each_source_once(self):+ subqueries = _memo_subqueries(["tiktok", "instagram", "youtube"])+ with patch("lib.tiktok.search_and_enrich", side_effect=_tiktok_payload) as tiktok_search, \+ patch("lib.instagram.search_and_enrich", side_effect=_instagram_payload) as ig_search, \+ patch("lib.youtube_yt.search_and_transcribe", side_effect=_youtube_payload) as yt_search, \+ patch("lib.pipeline.which", return_value="/usr/bin/yt-dlp"), \+ patch("lib.env.is_youtube_comments_available", return_value=True), \+ patch("lib.youtube_yt.enrich_with_comments") as yt_comments, \+ patch("lib.tiktok.enrich_with_comments") as tiktok_comments, \+ patch("lib.instagram.enrich_with_comments") as ig_comments:+ results = _memo_streams_concurrently([+ (source, subquery, dict(_MEMO_SC_CONFIG))+ for subquery in subqueries+ for source in ("tiktok", "instagram", "youtube")+ ])++ assert tiktok_search.call_count == 1+ assert ig_search.call_count == 1+ assert yt_search.call_count == 1+ assert tiktok_comments.call_count == 1+ assert ig_comments.call_count == 1+ assert yt_comments.call_count == 1+ assert all(items for items, _artifact in results)++ def test_scrapecreators_only_youtube_searches_once(self):+ subqueries = _memo_subqueries(["youtube"])+ with patch("lib.pipeline.which", return_value=None), \+ patch("lib.youtube_yt.search_youtube_sc", side_effect=_youtube_payload) as sc_search, \+ patch("lib.env.is_youtube_comments_available", return_value=False):+ results = _memo_streams_concurrently([+ ("youtube", subquery, {"SCRAPECREATORS_API_KEY": "test-key"})+ for subquery in subqueries+ ])+ assert sc_search.call_count == 1+ assert all(len(items) == 6 for items, _artifact in results)++ def test_hashtag_and_keyword_tiktok_calls_are_different_keys(self):+ [subquery] = _memo_subqueries(["tiktok"], labels=("price",))+ with patch("lib.tiktok.search_and_enrich", side_effect=_tiktok_payload) as tiktok_search:+ _memo_stream("tiktok", subquery, {"SCRAPECREATORS_API_KEY": "test-key"})+ _memo_stream("tiktok", subquery, {"SCRAPECREATORS_API_KEY": "test-key"},+ tiktok_hashtags=["ghostty"])+ assert tiktok_search.call_count == 2+ assert tiktok_search.call_args_list[1].kwargs["hashtags"] == ["ghostty"]++ def test_failed_tiktok_search_is_not_reused(self):+ [subquery] = _memo_subqueries(["tiktok"], labels=("price",))+ responses = [{"items": [], "error": "HTTP 503"}, _tiktok_payload()]+ with patch("lib.tiktok.search_and_enrich", side_effect=responses) as tiktok_search:+ first_items, first_artifact = _memo_stream("tiktok", subquery, {"SCRAPECREATORS_API_KEY": "test-key"})+ second_items, _ = _memo_stream("tiktok", subquery, {"SCRAPECREATORS_API_KEY": "test-key"})+ assert tiktok_search.call_count == 2+ assert first_items == [] and first_artifact+ assert second_items++ def test_failed_instagram_search_is_not_reused(self):+ [subquery] = _memo_subqueries(["instagram"], labels=("price",))+ responses = [{"items": [], "error": "HTTP 503"}, _instagram_payload()]+ with patch("lib.instagram.search_and_enrich", side_effect=responses) as ig_search:+ _memo_stream("instagram", subquery, {"SCRAPECREATORS_API_KEY": "test-key"})+ second_items, _ = _memo_stream("instagram", subquery, {"SCRAPECREATORS_API_KEY": "test-key"})+ assert ig_search.call_count == 2+ assert second_items++ @pytest.mark.parametrize("artifact, clean", [+ ({}, True),+ ({"_source_outcome_detail": "yt-dlp thin; backfilled"}, True),+ ({"_source_outcome": {"state": "error"}}, False),+ ({"_source_outcome_deferred": {"state": "partial"}}, False),+ ({"_source_outcome_if_empty": {"state": "error"}}, False),+ ({"_source_outcome_detail_state": "partial"}, False),+ ])+ def test_clean_stream_result_rejects_every_failure_marker(self, artifact, clean):+ assert pipeline._clean_stream_result(([], artifact)) is clean++ def test_x_waiter_gets_owner_result_when_owner_runs_to_chain_deadline(self):+ deadline = time.monotonic() + 0.2++ def slow_backend(*_args, **_kwargs):+ time.sleep(0.35)+ return [{"id": "x1"}], None, None++ from concurrent.futures import ThreadPoolExecutor+ with patch.object(pipeline, "_attempt_x_backend", side_effect=slow_backend) as backend, \+ ThreadPoolExecutor(max_workers=2) as pool:+ futures = [+ pool.submit(+ pipeline._attempt_x_backend_memo,+ "bird", "ghostty", *_MEMO_RANGE, "default", {}, deadline=deadline,+ )+ for _ in range(2)+ ]+ results = [future.result(timeout=10) for future in futures]+ assert backend.call_count == 1+ assert all(items == [{"id": "x1"}] and err is None for items, err, _ in results)++ def test_x_waiter_timeout_is_a_chain_error_not_an_exception(self):+ deadline = time.monotonic() + 0.1++ def hung_backend(*_args, **_kwargs):+ time.sleep(0.6)+ return [{"id": "x1"}], None, None++ from concurrent.futures import ThreadPoolExecutor+ with patch.object(pipeline, "_attempt_x_backend", side_effect=hung_backend), \+ patch.object(pipeline, "_X_MEMO_WAIT_GRACE_SECONDS", 0.1), \+ ThreadPoolExecutor(max_workers=2) as pool:+ owner = pool.submit(+ pipeline._attempt_x_backend_memo,+ "bird", "ghostty", *_MEMO_RANGE, "default", {}, deadline=deadline,+ )+ time.sleep(0.05)+ waiter = pool.submit(+ pipeline._attempt_x_backend_memo,+ "bird", "ghostty", *_MEMO_RANGE, "default", {}, deadline=deadline,+ )+ items, err, err_state = waiter.result(timeout=10)+ owner.result(timeout=10)+ assert items == [] and err+ assert err_state == health.TIMEOUT++ def _thin_youtube_retry(self, config):+ plan = schema.QueryPlan(+ intent="general", freshness_mode="balanced_recent", cluster_mode="topic",+ raw_topic=_MEMO_TOPIC, subqueries=_memo_subqueries(["youtube"]),+ source_weights={"youtube": 1.0},+ )+ pipeline._retry_thin_sources(+ topic=_MEMO_TOPIC, bundle=schema.RetrievalBundle(), plan=plan,+ config=config, depth="default", date_range=_MEMO_RANGE,+ runtime=_make_runtime(None), mock=False, rate_limited_sources=set(),+ rate_limit_lock=threading.Lock(), settings={"per_stream_limit": 20},+ )++ def test_thin_retry_after_clean_youtube_fetch_makes_no_network_call(self):+ [subquery] = _memo_subqueries(["youtube"], labels=("price",))+ with patch("lib.pipeline.which", return_value="/usr/bin/yt-dlp"), \+ patch("lib.youtube_yt.search_and_transcribe", side_effect=_youtube_payload) as yt_search, \+ patch("lib.env.is_youtube_comments_available", return_value=False):+ _memo_stream("youtube", subquery, {})+ self._thin_youtube_retry({})+ assert yt_search.call_count == 1++ def test_thin_retry_after_failed_youtube_fetch_refetches(self):+ [subquery] = _memo_subqueries(["youtube"], labels=("price",))+ responses = [{"items": [], "error": "yt-dlp search timed out"}, _youtube_payload()]+ with patch("lib.pipeline.which", return_value="/usr/bin/yt-dlp"), \+ patch("lib.youtube_yt.search_and_transcribe", side_effect=responses) as yt_search, \+ patch("lib.env.is_youtube_comments_available", return_value=False):+ _items, artifact = _memo_stream("youtube", subquery, {})+ assert artifact["_source_outcome"]["state"] == health.TIMEOUT+ self._thin_youtube_retry({})+ assert yt_search.call_count == 2++ def _thin_retry(self, source):+ plan = schema.QueryPlan(+ intent="general", freshness_mode="balanced_recent", cluster_mode="topic",+ raw_topic=_MEMO_TOPIC,+ subqueries=[_raw_topic_subquery(source)],+ source_weights={source: 1.0},+ )+ pipeline._retry_thin_sources(+ topic=_MEMO_TOPIC, bundle=schema.RetrievalBundle(), plan=plan,+ config={}, depth="default", date_range=_MEMO_RANGE,+ runtime=_make_runtime(None), mock=False, rate_limited_sources=set(),+ rate_limit_lock=threading.Lock(), settings={"per_stream_limit": 20},+ )++ @pytest.mark.parametrize("source, target", [+ ("polymarket", "lib.polymarket.search_polymarket"),+ ("digg", "lib.digg.search_digg"),+ ])+ def test_thin_retry_reuses_clean_raw_topic_fetch(self, source, target):+ # Live runs: the planner's subquery searches the raw topic, the thin+ # retry's core subject is the same string, so the request repeated.+ empty = {"events": [], "results": []}+ with patch(target, return_value=empty) as search:+ _memo_stream(source, _raw_topic_subquery(source), {})+ self._thin_retry(source)+ assert search.call_count == 1++ @pytest.mark.parametrize("source, target", [+ ("polymarket", "lib.polymarket.search_polymarket"),+ ("digg", "lib.digg.search_digg"),+ ])+ def test_thin_retry_refetches_after_failed_fetch(self, source, target):+ failed = {"events": [], "results": [], "error": "HTTP 503"}+ with patch(target, side_effect=[failed, {"events": [], "results": []}]) as search:+ _items, artifact = _memo_stream(source, _raw_topic_subquery(source), {})+ assert artifact.get("_source_outcome")+ self._thin_retry(source)+ assert search.call_count == 2++ @pytest.mark.parametrize("source, target", [+ ("polymarket", "lib.polymarket.search_polymarket"),+ ("digg", "lib.digg.search_digg"),+ ])+ def test_different_queries_fetch_separately(self, source, target):+ with patch(target, return_value={"events": [], "results": []}) as search:+ for subquery in _memo_subqueries([source], labels=("price", "speed")):+ _memo_stream(source, subquery, {})+ assert search.call_count == 2++ def test_x_backend_fetched_once_and_each_subquery_judged_with_its_own_ranking_query(self):+ subqueries = _memo_subqueries(["x"], labels=("price", "speed"))+ posts = [{+ "id": "X1", "text": "ghostty terminal is fast", "author_handle": "dev",+ "url": "https://x.com/dev/status/1", "date": "2026-10-01",+ }]+ judged_with: list[str] = []++ def _judge(items, _topic, *, ranking_query=""):+ judged_with.append(ranking_query)+ return {"on_topic_items": list(items), "on_topic_ratio": 1.0}++ with patch("lib.env.x_backend_chain", return_value=["bird"]), \+ patch("lib.pipeline._fetch_x_backend", return_value=(posts, "")) as fetch, \+ patch("lib.x_judge.should_retry_x_search", return_value=False), \+ patch("lib.x_judge.judge_x_corpus", side_effect=_judge):+ results = [_memo_stream("x", subquery, {}) for subquery in subqueries]++ assert fetch.call_count == 1+ assert sorted(judged_with) == sorted(sq.ranking_query for sq in subqueries)+ assert all(len(items) == 1 for items, _artifact in results)++ def _parity_report(self):+ plan = {+ "intent": "general", "freshness_mode": "balanced_recent", "cluster_mode": "story",+ "subqueries": [+ {"label": sq.label, "search_query": sq.search_query,+ "ranking_query": sq.ranking_query, "sources": list(sq.sources)}+ for sq in _memo_subqueries(["tiktok", "instagram", "youtube"])+ ],+ }+ with patch("lib.tiktok.search_and_enrich", side_effect=_tiktok_payload), \+ patch("lib.instagram.search_and_enrich", side_effect=_instagram_payload), \+ patch("lib.youtube_yt.search_and_transcribe", side_effect=_youtube_payload), \+ patch("lib.youtube_yt.search_youtube_sc", return_value={"items": []}), \+ patch("lib.youtube_yt.backfill_transcripts"), \+ patch("lib.pipeline.which", return_value="/usr/bin/yt-dlp"), \+ patch("lib.env.is_youtube_comments_available", return_value=False), \+ patch("lib.tiktok.enrich_with_comments"), \+ patch("lib.instagram.enrich_with_comments"):+ return pipeline.run(+ topic=_MEMO_TOPIC, config=dict(_MEMO_SC_CONFIG), depth="default",+ requested_sources=["tiktok", "instagram", "youtube"], mock=False,+ external_plan=plan, web_backend="none", save_dir="",+ )++ def test_report_parity_with_memo_bypassed(self):+ from lib import fetch_memo++ def _summary(report):+ return (+ {source: sorted(item.url for item in items)+ for source, items in report.items_by_source.items()},+ {source: outcome.state for source, outcome in report.source_status.items()},+ sorted(report.errors_by_source),+ [candidate.url for candidate in report.ranked_candidates],+ )++ memoized = _summary(self._parity_report())+ fetch_memo.reset()+ with patch("lib.fetch_memo.fetch_once",+ side_effect=lambda _key, fetch, **_kwargs: fetch()):+ bypassed = _summary(self._parity_report())++ assert memoized == bypassed+ assert set(memoized[0]) == {"tiktok", "instagram", "youtube"}++if __name__ == "__main__":unittest.main()diff --git a/tests/test_quality_nudge.py b/tests/test_quality_nudge.pyindex ac84bfc0..5c4c0de3 100644--- a/tests/test_quality_nudge.py+++ b/tests/test_quality_nudge.py@@ -1099,10 +1099,12 @@ class TestXaiErrorRemediation:},)+ # The failed xai call is not memoized, so each subquery tries it; the+ # backup's clean result is fetched once and served to both subqueries.assert xai_search.call_count >= 2- assert fallback_search.call_count >= 2+ assert fallback_search.call_count == 1assert any(- "status/123456789012345671" in item.url+ "status/123456789012345670" in item.urlfor item in second_query_items)assert report.source_status["x"].lane_failure_state == schema.RATE_LIMITEDdiff --git a/tests/test_youtube_backstop.py b/tests/test_youtube_backstop.pyindex 2e79b37a..49b87643 100644--- a/tests/test_youtube_backstop.py+++ b/tests/test_youtube_backstop.py@@ -225,7 +225,7 @@ def test_partial_ytdlp_search_error_stays_visible_after_backfill():assert "yt-dlp bot gate" in artifact["_source_outcome"]["detail"]-def test_full_pipeline_reports_clean_backfill_and_searches_paid_once_per_subquery():+def test_full_pipeline_reports_clean_backfill_and_searches_paid_once_per_run():free = {"video_id": "a", "title": "Vuori review", "date": "2026-06-15","url": "https://www.youtube.com/watch?v=a", "transcript_snippet": "free transcript",@@ -263,7 +263,9 @@ def test_full_pipeline_reports_clean_backfill_and_searches_paid_once_per_subquerexternal_plan=plan, as_of_date="2026-07-01",)- assert search.call_count == 2+ # Both subqueries search the raw topic: the run fetch memo serves the+ # second stream (and the thin retry) from the first clean fetch.+ assert search.call_count == 1assert report.source_status["youtube"].state == health.OKassert "ScrapeCreators added 3" in report.source_status["youtube"].detail@@ -301,8 +303,10 @@ def _run_thin_youtube_report(sc_responses):return report, search.call_count-def test_empty_clean_backfill_can_retry_and_recover():- recovered = [+def test_thin_retry_reuses_a_clean_empty_backfill_instead_of_refetching():+ # A clean (error-free) backfill is the answer for this query in this run:+ # the identical thin-source retry is served from the run fetch memo.+ unused = [{"video_id": str(i), "title": f"Vuori review {i}", "date": "2026-06-15","url": f"https://www.youtube.com/watch?v={i}",@@ -310,18 +314,16 @@ def test_empty_clean_backfill_can_retry_and_recover():for i in range(2)]report, calls = _run_thin_youtube_report([- {"items": []}, {"items": recovered},+ {"items": []}, {"items": unused},])- assert calls == 2+ assert calls == 1assert report.source_status["youtube"].state == health.OK- assert {item.item_id for item in report.items_by_source["youtube"]} >= {- "free", "0",- }+ assert [item.item_id for item in report.items_by_source["youtube"]] == ["free"]def test_clean_sparse_topic_stays_ok_after_empty_backfill_and_retry():report, calls = _run_thin_youtube_report([{"items": []}, {"items": []}])- assert calls == 2+ assert calls == 1assert report.source_status["youtube"].state == health.OKassert [item.item_id for item in report.items_by_source["youtube"]] == ["free"]assert "added 0" in report.source_status["youtube"].detail