Skip to content

AI agent skill that researches any topic across Reddit, X, YouTube, HN, Polymarket, and the web - then synthesizes a grounded summary

fix(pipeline): fetch each identical source search once per run (#1260)

Matt Van Horn8c4697b222ae
diff --git a/changelog.d/+dedupe-source-fetches.fixed.md b/changelog.d/+dedupe-source-fetches.fixed.md
new file mode 100644
index 00000000..cb4f5779
--- /dev/null
+++ b/changelog.d/+dedupe-source-fetches.fixed.md
@@ -0,0 +1 @@
+A multi-subquery run now fetches each identical TikTok, Instagram, YouTube, and X search once instead of once per subquery, and the thin-source retry reuses a clean result instead of repeating the request. Polymarket and Digg no longer run the same search twice when the thin-source retry's simplified query matches the planner's. Failed fetches are not reused, so the retry still makes a real second attempt.
diff --git a/skills/last30days/scripts/last30days.py b/skills/last30days/scripts/last30days.py
index 6bc00cef..fedaeab9 100644
--- a/skills/last30days/scripts/last30days.py
+++ b/skills/last30days/scripts/last30days.py
@@ -50,7 +50,7 @@ if os.name == "nt":
SCRIPT_DIR = Path(__file__).parent.resolve()
sys.path.insert(0, str(SCRIPT_DIR))
-from lib import competitors as competitors_mod, corpus, dates, discovery_handoff, env, freshness, html_render, http, permission_preflight, pipeline, reddit, registers, render, schema, subproc, ui, x_envelope
+from lib import competitors as competitors_mod, corpus, dates, discovery_handoff, env, fetch_memo, freshness, html_render, http, permission_preflight, pipeline, reddit, registers, render, schema, subproc, ui, x_envelope
from lib.private_files import write_private_text
atexit.register(subproc.cleanup_children)
@@ -3293,6 +3293,7 @@ def _main(
# parallel, so the reset must not live inside the pipeline.
http.reset_reddit_keyless_memo()
reddit.reset_scrapecreators_memo()
+ fetch_memo.reset()
resolved_corpus_dirs = corpus.resolve_directories(
args.corpus, config.get("LAST30DAYS_CORPUS_DIRS")
)
diff --git a/skills/last30days/scripts/lib/fetch_memo.py b/skills/last30days/scripts/lib/fetch_memo.py
new file mode 100644
index 00000000..c65bd6d8
--- /dev/null
+++ b/skills/last30days/scripts/lib/fetch_memo.py
@@ -0,0 +1,135 @@
+"""Run-scoped memo for source fetches.
+
+Subquery streams for TikTok, Instagram, YouTube, and X all search the raw
+topic, and the thin-source retry repeats that search (and, for Polymarket and
+Digg, a planner subquery that equals the retry's core subject), so one run can
+issue the same request several times. :func:`fetch_once` runs a fetch at most once per
+key: the first caller fetches, concurrent callers wait for it (bounded by
+their own timeout), and later callers get a copy of the cached result.
+
+Only clean results are cached. A failure (a raised exception, a result the
+caller's ``cacheable`` predicate rejects, or HTTP failures an adapter
+swallowed into the failure sink) reaches only the callers already waiting on
+that fetch; the next call fetches again, so the thin-source retry still makes
+a real second attempt after a transient failure.
+
+Reset once per command (``last30days.py``) and between tests (``conftest``).
+"""
+
+from __future__ import annotations
+
+import copy
+import threading
+import time
+from typing import Any, Callable, Hashable, TypeVar
+
+from . import health, http
+
+T = TypeVar("T")
+
+
+class MemoWaitTimeout(TimeoutError):
+ """A waiter's budget expired before the in-flight fetch it shares finished."""
+
+ outcome_state = health.TIMEOUT
+
+
+class _Slot:
+ __slots__ = ("event", "outcome")
+
+ def __init__(self) -> None:
+ self.event = threading.Event()
+ # (ok, value_or_exception, recorded_failures) once the owner finishes.
+ self.outcome: tuple[bool, Any, list] | None = None
+
+
+_CACHE: dict[Hashable, Any] = {}
+_INFLIGHT: dict[Hashable, _Slot] = {}
+_LOCK = threading.Lock()
+
+
+def reset() -> None:
+ """Forget cached and in-flight fetches. Called once per command, and by tests."""
+ with _LOCK:
+ _CACHE.clear()
+ _INFLIGHT.clear()
+
+
+def _always(_value: Any) -> bool:
+ return True
+
+
+def _deliver_shared(outcome: tuple[bool, Any, list]) -> Any:
+ ok, value, failures = outcome
+ # The owner's swallowed HTTP failures reach this caller's sink too, or a
+ # shared empty result reads as a clean no-results.
+ for failure in failures:
+ http._record_failure(failure)
+ if not ok:
+ raise value
+ return copy.deepcopy(value)
+
+
+def fetch_once(
+ key: Hashable,
+ fetch: Callable[[], T],
+ *,
+ timeout: float,
+ cacheable: Callable[[T], bool] = _always,
+) -> T:
+ """Return ``fetch()``, running it at most once per ``key`` while it stays clean.
+
+ ``key`` must include every input that changes the request. ``timeout``
+ bounds how long a caller waits on another caller's in-flight fetch; past
+ it, :class:`MemoWaitTimeout` is raised. Every caller gets its own copy.
+ """
+ deadline = time.monotonic() + max(0.0, timeout)
+ while True:
+ with _LOCK:
+ hit = key in _CACHE
+ if hit:
+ cached = _CACHE[key]
+ else:
+ slot = _INFLIGHT.get(key)
+ owner = slot is None
+ if owner:
+ slot = _Slot()
+ _INFLIGHT[key] = slot
+ if hit:
+ return copy.deepcopy(cached)
+ if owner:
+ break
+ if not slot.event.wait(max(0.0, deadline - time.monotonic())):
+ raise MemoWaitTimeout(
+ f"waited {timeout:g}s for an identical in-flight fetch"
+ )
+ if slot.outcome is not None:
+ return _deliver_shared(slot.outcome)
+ # The owner was interrupted before publishing (e.g. KeyboardInterrupt):
+ # loop and elect a new owner.
+
+ try:
+ with http.tee_failures() as recorded:
+ try:
+ value = fetch()
+ ok = True
+ except Exception as exc:
+ value = exc
+ ok = False
+ clean = ok and not recorded and cacheable(value)
+ # Waiters and the cache copy from this snapshot, never from the
+ # owner's own value, which the owner's caller may mutate.
+ shared = copy.deepcopy(value) if ok else value
+ slot.outcome = (ok, shared, list(recorded))
+ with _LOCK:
+ # Slot identity guards against a reset() during the fetch.
+ if clean and _INFLIGHT.get(key) is slot:
+ _CACHE[key] = shared
+ finally:
+ with _LOCK:
+ if _INFLIGHT.get(key) is slot:
+ _INFLIGHT.pop(key)
+ slot.event.set()
+ if not ok:
+ raise value
+ return value
diff --git a/skills/last30days/scripts/lib/pipeline.py b/skills/last30days/scripts/lib/pipeline.py
index 2f67f43f..8b75896b 100644
--- a/skills/last30days/scripts/lib/pipeline.py
+++ b/skills/last30days/scripts/lib/pipeline.py
@@ -34,6 +34,7 @@ from . import (
dripstack,
entity_extract,
env,
+ fetch_memo,
github,
grok_x,
grounding,
@@ -5302,6 +5303,217 @@ def _serve_envelope_lanes(
)
+# Artifact keys that mark a stream result as a failure (or a deferred one).
+# The fetch memo never caches such a result, so the thin-source retry and the
+# next stream still make a real attempt after a transient failure.
+_STREAM_FAILURE_ARTIFACT_KEYS = (
+ "_source_outcome",
+ "_source_outcome_deferred",
+ "_source_outcome_if_empty",
+ "_source_outcome_detail_state",
+)
+
+
+def _clean_stream_result(result: tuple[list[dict], dict]) -> bool:
+ _items, artifact = result
+ return not (
+ isinstance(artifact, dict)
+ and any(artifact.get(key) for key in _STREAM_FAILURE_ARTIFACT_KEYS)
+ )
+
+
+def _clean_search_result(result: Any) -> bool:
+ """A legacy ``{"error": ...}`` search result is a failure and is not cached."""
+ return not (isinstance(result, dict) and result.get("error"))
+
+
+def _memo_wait_seconds(config: dict[str, Any]) -> float:
+ """How long a stream may wait on an identical in-flight fetch: its own budget."""
+ return max(
+ 0.0,
+ _research_deadline(config, STREAM_FUTURE_TIMEOUT_SECONDS) - time.monotonic(),
+ )
+
+
+_X_MEMO_WAIT_GRACE_SECONDS = 5.0
+
+
+def _attempt_x_backend_memo(
+ backend, query, from_date, to_date, depth, config, warnings=None, deadline=None,
+):
+ """:func:`_attempt_x_backend`, once per backend and query per run.
+
+ Only the backend call is shared: the judge and prune still run per
+ subquery with that subquery's ranking query.
+ """
+ def _fetch():
+ local_warnings: list[str] = []
+ items, err, err_state = _attempt_x_backend(
+ backend, query, from_date, to_date, depth, config,
+ warnings=local_warnings, deadline=deadline,
+ )
+ return items, err, err_state, local_warnings
+
+ # The owner is bounded by the same chain deadline, so a waiter allows a
+ # short grace past it for the owner to publish instead of timing out first.
+ wait = (
+ max(0.0, deadline - time.monotonic()) + _X_MEMO_WAIT_GRACE_SECONDS
+ if deadline is not None
+ else _memo_wait_seconds(config)
+ )
+ try:
+ items, err, err_state, backend_warnings = fetch_memo.fetch_once(
+ ("x", backend, query, from_date, to_date, depth),
+ _fetch,
+ timeout=wait,
+ cacheable=lambda outcome: not outcome[1],
+ )
+ except fetch_memo.MemoWaitTimeout as exc:
+ # A chain error like any backend timeout, so the chain's own error
+ # handling and outcome reporting apply.
+ return [], str(exc), health.TIMEOUT
+ if warnings is not None:
+ warnings.extend(backend_warnings)
+ return items, err, err_state
+
+
+def _fetch_youtube_stream(
+ yt_query: str,
+ from_date: str,
+ to_date: str,
+ depth: str,
+ config: dict[str, Any],
+ *,
+ has_ytdlp: bool,
+) -> tuple[list[dict], dict]:
+ """Search, backfill, transcribe, and comment-enrich one YouTube query."""
+ result = None
+ youtube_failure: str | None = None
+ recovered_failure: str | None = None
+ # ScrapeCreators key (when present) is the default-on backup tier: it
+ # powers the per-video transcript fallback, the SC search fallback, and
+ # comment enrichment. None when no key, which keeps everything keyless.
+ sc_token = (
+ config.get("SCRAPECREATORS_API_KEY", "")
+ if env.is_youtube_sc_available(config) else None
+ )
+ # Try yt-dlp first; the SC transcript fallback covers per-video failures.
+ if has_ytdlp:
+ try:
+ result = youtube_yt.search_and_transcribe(
+ yt_query, from_date, to_date, depth=depth, token=sc_token,
+ )
+ if result.get("error"):
+ youtube_failure = str(result["error"])
+ except Exception as exc:
+ youtube_failure = str(exc)
+ result = None
+ free_items = list((result or {}).get("items") or [])
+ min_items = env.youtube_sc_min_items(config)
+ backfill_thin = bool(free_items) and len(free_items) < min_items
+ thin_detail: str | None = None
+ thin_error = False
+ thin_retryable = False
+ if (not free_items or backfill_thin) and sc_token:
+ backfill_scope = (
+ http.capture_failures() if free_items else contextlib.nullcontext([])
+ )
+ try:
+ with backfill_scope as sc_failures:
+ transcribed_free_ids = {
+ item["video_id"] for item in free_items
+ if item.get("video_id")
+ and str(item.get("transcript_snippet") or "").strip()
+ }
+ sc_result = youtube_yt.search_youtube_sc(
+ yt_query, from_date, to_date, depth=depth, token=sc_token,
+ skip_transcript_ids=transcribed_free_ids,
+ )
+ sc_error = str(sc_result["error"]) if sc_result.get("error") else None
+ if backfill_thin:
+ merged = _merge_youtube_items(free_items, sc_result.get("items") or [])
+ result = {"items": merged}
+ thin_detail = (
+ f"yt-dlp returned {len(free_items)} videos below the "
+ f"{min_items}-video backfill floor; ScrapeCreators added "
+ f"{len(merged) - len(free_items)} videos"
+ )
+ if youtube_failure:
+ thin_detail += f"; yt-dlp search: {youtube_failure}"
+ if sc_error:
+ thin_detail += f"; ScrapeCreators search: {sc_error}"
+ if sc_failures:
+ thin_detail += f"; ScrapeCreators: {_summarize_lane_failures(sc_failures, 'youtube')}"
+ thin_error = bool(youtube_failure or sc_error or sc_failures)
+ thin_retryable = thin_error and (
+ len(merged) < min_items
+ and (not youtube_failure or _retryable_youtube_error(youtube_failure))
+ and (not sc_error or _retryable_youtube_error(sc_error))
+ and all(
+ _retryable_youtube_error(str(f), f.outcome_state)
+ for f in sc_failures
+ )
+ )
+ youtube_failure = None
+ recovered_failure = None
+ else:
+ result = sc_result
+ if sc_error:
+ youtube_failure = sc_error
+ elif result.get("items") and youtube_failure:
+ recovered_failure = youtube_failure
+ youtube_failure = None
+ except Exception as exc:
+ prior_youtube_failure = youtube_failure
+ youtube_failure = str(exc)
+ result = {"items": free_items} if free_items else None
+ if backfill_thin:
+ thin_error = True
+ thin_detail = (
+ f"yt-dlp returned {len(free_items)} videos below the "
+ f"{min_items}-video backfill floor; ScrapeCreators failed: {exc}"
+ )
+ if prior_youtube_failure:
+ thin_detail += f"; yt-dlp search: {prior_youtube_failure}"
+ thin_retryable = (
+ _retryable_youtube_error(
+ str(exc), getattr(exc, "outcome_state", None)
+ )
+ and (not prior_youtube_failure or _retryable_youtube_error(prior_youtube_failure))
+ )
+ if result is None:
+ result = {"items": []}
+ # Enrich top videos with comments (default-on when a key is present).
+ items = youtube_yt.parse_youtube_response(result)
+ if items and env.is_youtube_comments_available(config):
+ youtube_yt.enrich_with_comments(
+ items, token=config.get("SCRAPECREATORS_API_KEY", ""),
+ )
+ if thin_detail:
+ if not thin_error:
+ return items, {"_source_outcome_detail": thin_detail}
+ outcome = _outcome_artifact(health.PARTIAL, thin_detail)
+ if thin_retryable:
+ return items, {
+ "_source_outcome_detail": thin_detail,
+ "_source_outcome_deferred": outcome["_source_outcome"],
+ }
+ return items, outcome
+ if youtube_failure:
+ state = youtube_yt.classify_run_failure(youtube_failure)
+ attempted = state != schema.SKIPPED_UNCONFIGURED
+ return items, _outcome_artifact(state, youtube_failure, attempted=attempted)
+ if recovered_failure:
+ state = youtube_yt.classify_run_failure(recovered_failure)
+ attempted = state != schema.SKIPPED_UNCONFIGURED
+ return items, {
+ "_source_outcome_if_empty": _outcome_artifact(
+ state, recovered_failure, attempted=attempted,
+ )["_source_outcome"]
+ }
+ return items, {}
+
+
def _retrieve_stream_impl(
*,
topic: str,
@@ -5553,7 +5765,7 @@ def _retrieve_stream_impl(
print("[X] chain budget exhausted; skipping remaining backends", file=sys.stderr)
break
with http.tee_failures() as backend_failures:
- items, err, err_state = _attempt_x_backend(
+ items, err, err_state = _attempt_x_backend_memo(
backend, x_query, from_date, to_date, depth, config, warnings=x_warnings,
deadline=chain_deadline,
)
@@ -5666,7 +5878,7 @@ def _retrieve_stream_impl(
if _research_stopped(config) or time.monotonic() >= chain_deadline:
print("[X] chain budget exhausted; skipping judge retry", file=sys.stderr)
else:
- retry_items, retry_err, _ = _attempt_x_backend(
+ retry_items, retry_err, _ = _attempt_x_backend_memo(
used_backend, retry_query, from_date, to_date, depth, config,
deadline=chain_deadline,
)
@@ -5740,167 +5952,87 @@ def _retrieve_stream_impl(
# Use raw_topic so expand_youtube_queries() generates diverse variants
# from the original user topic, not the planner's narrowed search_query.
yt_query = raw_topic or subquery.search_query
- result = None
- youtube_failure: str | None = None
- recovered_failure: str | None = None
- # ScrapeCreators key (when present) is the default-on backup tier: it
- # powers the per-video transcript fallback, the SC search fallback, and
- # comment enrichment. None when no key, which keeps everything keyless.
- sc_token = (
- config.get("SCRAPECREATORS_API_KEY", "")
- if env.is_youtube_sc_available(config) else None
+ has_ytdlp = bool(which("yt-dlp"))
+ # One memo unit covers yt-dlp search, the ScrapeCreators fallback and
+ # merge, transcripts, and comments, so subquery streams and the
+ # thin-source retry share one fetch (hosts without yt-dlp included).
+ return fetch_memo.fetch_once(
+ (
+ "youtube", yt_query, from_date, to_date, depth, has_ytdlp,
+ env.is_youtube_sc_available(config),
+ env.youtube_sc_min_items(config),
+ env.is_youtube_comments_available(config),
+ ),
+ lambda: _fetch_youtube_stream(
+ yt_query, from_date, to_date, depth, config, has_ytdlp=has_ytdlp,
+ ),
+ timeout=_memo_wait_seconds(config),
+ cacheable=_clean_stream_result,
)
- # Try yt-dlp first; the SC transcript fallback covers per-video failures.
- if which("yt-dlp"):
- try:
- result = youtube_yt.search_and_transcribe(
- yt_query, from_date, to_date, depth=depth, token=sc_token,
- )
- if result.get("error"):
- youtube_failure = str(result["error"])
- except Exception as exc:
- youtube_failure = str(exc)
- result = None
- free_items = list((result or {}).get("items") or [])
- min_items = env.youtube_sc_min_items(config)
- backfill_thin = bool(free_items) and len(free_items) < min_items
- thin_detail: str | None = None
- thin_error = False
- thin_retryable = False
- if (not free_items or backfill_thin) and sc_token:
- backfill_scope = (
- http.capture_failures() if free_items else contextlib.nullcontext([])
- )
- try:
- with backfill_scope as sc_failures:
- transcribed_free_ids = {
- item["video_id"] for item in free_items
- if item.get("video_id")
- and str(item.get("transcript_snippet") or "").strip()
- }
- sc_result = youtube_yt.search_youtube_sc(
- yt_query, from_date, to_date, depth=depth, token=sc_token,
- skip_transcript_ids=transcribed_free_ids,
- )
- sc_error = str(sc_result["error"]) if sc_result.get("error") else None
- if backfill_thin:
- merged = _merge_youtube_items(free_items, sc_result.get("items") or [])
- result = {"items": merged}
- thin_detail = (
- f"yt-dlp returned {len(free_items)} videos below the "
- f"{min_items}-video backfill floor; ScrapeCreators added "
- f"{len(merged) - len(free_items)} videos"
- )
- if youtube_failure:
- thin_detail += f"; yt-dlp search: {youtube_failure}"
- if sc_error:
- thin_detail += f"; ScrapeCreators search: {sc_error}"
- if sc_failures:
- thin_detail += f"; ScrapeCreators: {_summarize_lane_failures(sc_failures, 'youtube')}"
- thin_error = bool(youtube_failure or sc_error or sc_failures)
- thin_retryable = thin_error and (
- len(merged) < min_items
- and (not youtube_failure or _retryable_youtube_error(youtube_failure))
- and (not sc_error or _retryable_youtube_error(sc_error))
- and all(
- _retryable_youtube_error(str(f), f.outcome_state)
- for f in sc_failures
- )
- )
- youtube_failure = None
- recovered_failure = None
- else:
- result = sc_result
- if sc_error:
- youtube_failure = sc_error
- elif result.get("items") and youtube_failure:
- recovered_failure = youtube_failure
- youtube_failure = None
- except Exception as exc:
- prior_youtube_failure = youtube_failure
- youtube_failure = str(exc)
- result = {"items": free_items} if free_items else None
- if backfill_thin:
- thin_error = True
- thin_detail = (
- f"yt-dlp returned {len(free_items)} videos below the "
- f"{min_items}-video backfill floor; ScrapeCreators failed: {exc}"
- )
- if prior_youtube_failure:
- thin_detail += f"; yt-dlp search: {prior_youtube_failure}"
- thin_retryable = (
- _retryable_youtube_error(
- str(exc), getattr(exc, "outcome_state", None)
- )
- and (not prior_youtube_failure or _retryable_youtube_error(prior_youtube_failure))
- )
- if result is None:
- result = {"items": []}
- # Enrich top videos with comments (default-on when a key is present).
- items = youtube_yt.parse_youtube_response(result)
- if items and env.is_youtube_comments_available(config):
- youtube_yt.enrich_with_comments(
- items, token=config.get("SCRAPECREATORS_API_KEY", ""),
- )
- if thin_detail:
- if not thin_error:
- return items, {"_source_outcome_detail": thin_detail}
- outcome = _outcome_artifact(health.PARTIAL, thin_detail)
- if thin_retryable:
- return items, {
- "_source_outcome_detail": thin_detail,
- "_source_outcome_deferred": outcome["_source_outcome"],
- }
- return items, outcome
- if youtube_failure:
- state = youtube_yt.classify_run_failure(youtube_failure)
- attempted = state != schema.SKIPPED_UNCONFIGURED
- return items, _outcome_artifact(state, youtube_failure, attempted=attempted)
- if recovered_failure:
- state = youtube_yt.classify_run_failure(recovered_failure)
- attempted = state != schema.SKIPPED_UNCONFIGURED
- return items, {
- "_source_outcome_if_empty": _outcome_artifact(
- state, recovered_failure, attempted=attempted,
- )["_source_outcome"]
- }
- return items, {}
if source == "tiktok":
# Use raw_topic so expand_tiktok_queries() generates diverse variants
# from the original user topic, not the planner's narrowed search_query.
tiktok_query = raw_topic or subquery.search_query
- result = tiktok.search_and_enrich(
- tiktok_query,
- from_date,
- to_date,
- depth=depth,
- token=env.get_tiktok_token(config),
- hashtags=tiktok_hashtags,
- creators=tiktok_creators,
+
+ def _fetch_tiktok() -> tuple[list[dict], dict]:
+ result = tiktok.search_and_enrich(
+ tiktok_query,
+ from_date,
+ to_date,
+ depth=depth,
+ token=env.get_tiktok_token(config),
+ hashtags=tiktok_hashtags,
+ creators=tiktok_creators,
+ )
+ items = tiktok.parse_tiktok_response(result)
+ if items and env.is_tiktok_comments_available(config):
+ sc_token = config.get("SCRAPECREATORS_API_KEY", "")
+ tiktok.enrich_with_comments(items, token=sc_token)
+ return items, _result_outcome_artifact(source, result)
+
+ return fetch_memo.fetch_once(
+ (
+ "tiktok", tiktok_query, from_date, to_date, depth,
+ bool(env.get_tiktok_token(config)),
+ tuple(tiktok_hashtags or ()), tuple(tiktok_creators or ()),
+ env.is_tiktok_comments_available(config),
+ ),
+ _fetch_tiktok,
+ timeout=_memo_wait_seconds(config),
+ cacheable=_clean_stream_result,
)
- items = tiktok.parse_tiktok_response(result)
- if items and env.is_tiktok_comments_available(config):
- sc_token = config.get("SCRAPECREATORS_API_KEY", "")
- tiktok.enrich_with_comments(items, token=sc_token)
- return items, _result_outcome_artifact(source, result)
if source == "instagram":
# Use raw_topic so expand_instagram_queries() generates diverse variants
# from the original user topic, not the planner's narrowed search_query.
ig_query = raw_topic or subquery.search_query
- result = instagram.search_and_enrich(
- ig_query,
- from_date,
- to_date,
- depth=depth,
- token=env.get_instagram_token(config),
- ig_creators=ig_creators,
- )
- items = instagram.parse_instagram_response(result)
- if items and env.is_instagram_comments_available(config):
- instagram.enrich_with_comments(
- items, token=config.get("SCRAPECREATORS_API_KEY", ""),
+
+ def _fetch_instagram() -> tuple[list[dict], dict]:
+ result = instagram.search_and_enrich(
+ ig_query,
+ from_date,
+ to_date,
+ depth=depth,
+ token=env.get_instagram_token(config),
+ ig_creators=ig_creators,
)
- return items, _result_outcome_artifact(source, result)
+ items = instagram.parse_instagram_response(result)
+ if items and env.is_instagram_comments_available(config):
+ instagram.enrich_with_comments(
+ items, token=config.get("SCRAPECREATORS_API_KEY", ""),
+ )
+ return items, _result_outcome_artifact(source, result)
+
+ return fetch_memo.fetch_once(
+ (
+ "instagram", ig_query, from_date, to_date, depth,
+ bool(env.get_instagram_token(config)),
+ tuple(ig_creators or ()),
+ env.is_instagram_comments_available(config),
+ ),
+ _fetch_instagram,
+ timeout=_memo_wait_seconds(config),
+ cacheable=_clean_stream_result,
+ )
if source == "linkedin":
token = config.get("SCRAPECREATORS_API_KEY", "")
result = linkedin.search_linkedin(
@@ -5955,7 +6087,14 @@ def _retrieve_stream_impl(
_result_outcome_artifact(source, result),
)
if source == "digg":
- result = digg.search_digg(subquery.search_query, from_date, to_date, depth=depth)
+ # The thin-source retry's core subject often equals a planner
+ # subquery's search, so share the identical CLI call within a run.
+ result = fetch_memo.fetch_once(
+ ("digg", subquery.search_query, from_date, to_date, depth),
+ lambda: digg.search_digg(subquery.search_query, from_date, to_date, depth=depth),
+ timeout=_memo_wait_seconds(config),
+ cacheable=_clean_search_result,
+ )
items = digg.parse_digg_response(result, query=subquery.search_query)
# Enrichment with attached X posts is deferred to
# _finalize_items_by_source so it runs on the items that actually
@@ -6093,7 +6232,16 @@ def _retrieve_stream_impl(
result = truthsocial.search_truthsocial(subquery.search_query, from_date, to_date, depth=depth, config=config)
return truthsocial.parse_truthsocial_response(result), _result_outcome_artifact(source, result)
if source == "polymarket":
- result = polymarket.search_polymarket(subquery.search_query, from_date, to_date, depth=depth)
+ # Same sharing as digg: only the search is memoized; parsing still
+ # runs per caller with its own relevance topic.
+ result = fetch_memo.fetch_once(
+ ("polymarket", subquery.search_query, from_date, to_date, depth),
+ lambda: polymarket.search_polymarket(
+ subquery.search_query, from_date, to_date, depth=depth,
+ ),
+ timeout=_memo_wait_seconds(config),
+ cacheable=_clean_search_result,
+ )
# Relevance filtering keys off the stable original research topic, not the
# per-subquery search_query (which narrows differently on each fanout pass
# and would let off-topic markets through on broad subqueries while dropping
diff --git a/tests/conftest.py b/tests/conftest.py
index 4e2f4636..f9ab7c81 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -138,15 +138,18 @@ def _reset_probe_caches():
@pytest.fixture(autouse=True)
def _reset_reddit_keyless_memo():
- """The Reddit run memos (keyless and ScrapeCreators) live for one command; tests are their own commands."""
+ """The run memos (Reddit keyless and ScrapeCreators, source fetches) live for one command; tests are their own commands."""
+ from lib import fetch_memo as _fetch_memo
from lib import http as _http
from lib import reddit as _reddit
_http.reset_reddit_keyless_memo()
_reddit.reset_scrapecreators_memo()
+ _fetch_memo.reset()
yield
_http.reset_reddit_keyless_memo()
_reddit.reset_scrapecreators_memo()
+ _fetch_memo.reset()
@pytest.fixture(autouse=True)
diff --git a/tests/test_fetch_memo.py b/tests/test_fetch_memo.py
new file mode 100644
index 00000000..809423e2
--- /dev/null
+++ b/tests/test_fetch_memo.py
@@ -0,0 +1,165 @@
+"""Run-scoped fetch memo (U3, R5): one network fetch per distinct key per run,
+shared by concurrent callers as independent copies. A failure reaches only the
+callers already waiting on it; a later call fetches again."""
+
+import threading
+from concurrent.futures import ThreadPoolExecutor
+
+import pytest
+
+from lib import fetch_memo, health, http
+
+
+def _fetch_counter(payload):
+ calls = []
+
+ def _fetch():
+ calls.append(1)
+ return payload
+
+ return calls, _fetch
+
+
+def test_concurrent_same_key_fetches_once_and_returns_independent_copies():
+ release = threading.Event()
+ arrived = threading.Barrier(4, timeout=5)
+ calls = []
+
+ def _slow():
+ calls.append(1)
+ release.wait(timeout=5)
+ return {"items": [{"id": "a", "tags": []}]}
+
+ def _call():
+ arrived.wait()
+ return fetch_memo.fetch_once(("tiktok", "q"), _slow, timeout=10)
+
+ with ThreadPoolExecutor(max_workers=4) as pool:
+ futures = [pool.submit(_call) for _ in range(4)]
+ # Let every caller reach the memo before the owner finishes.
+ threading.Timer(0.2, release.set).start()
+ results = [f.result(timeout=10) for f in futures]
+
+ assert len(calls) == 1
+ assert all(r == {"items": [{"id": "a", "tags": []}]} for r in results)
+ results[0]["items"][0]["tags"].append("mutated")
+ assert all(r["items"][0]["tags"] == [] for r in results[1:])
+
+
+def test_sequential_same_key_serves_an_independent_copy():
+ calls, fetch = _fetch_counter({"items": [{"id": "a"}]})
+ first = fetch_memo.fetch_once(("youtube", "q"), fetch, timeout=1)
+ first["items"].append({"id": "mutated"})
+ second = fetch_memo.fetch_once(("youtube", "q"), fetch, timeout=1)
+ assert len(calls) == 1
+ assert second == {"items": [{"id": "a"}]}
+
+
+def test_different_keys_fetch_separately():
+ calls, fetch = _fetch_counter({"items": []})
+ fetch_memo.fetch_once(("youtube", "q", "2026-09-01", "2026-10-01", "default"), fetch, timeout=1)
+ fetch_memo.fetch_once(("youtube", "q", "2026-09-02", "2026-10-01", "default"), fetch, timeout=1)
+ fetch_memo.fetch_once(("youtube", "q", "2026-09-01", "2026-10-01", "deep"), fetch, timeout=1)
+ assert len(calls) == 3
+
+
+def test_owner_failure_reaches_concurrent_waiters_and_a_later_call_refetches():
+ release = threading.Event()
+ arrived = threading.Barrier(3, timeout=5)
+ calls = []
+ boom = RuntimeError("HTTP 503: Service Unavailable")
+
+ def _failing():
+ calls.append(1)
+ release.wait(timeout=5)
+ raise boom
+
+ def _call():
+ arrived.wait()
+ return fetch_memo.fetch_once(("x", "q"), _failing, timeout=10)
+
+ with ThreadPoolExecutor(max_workers=3) as pool:
+ futures = [pool.submit(_call) for _ in range(3)]
+ threading.Timer(0.2, release.set).start()
+ errors = []
+ for future in futures:
+ with pytest.raises(RuntimeError) as exc_info:
+ future.result(timeout=10)
+ errors.append(exc_info.value)
+
+ assert len(calls) == 1
+ assert all(err is boom for err in errors)
+
+ later_calls, later = _fetch_counter({"items": [{"id": "b"}]})
+ assert fetch_memo.fetch_once(("x", "q"), later, timeout=1) == {"items": [{"id": "b"}]}
+ assert len(later_calls) == 1
+
+
+def test_uncacheable_result_reaches_waiters_but_a_later_call_refetches():
+ calls, fetch = _fetch_counter({"items": [], "error": "HTTP 429"})
+ clean = lambda value: not value.get("error") # noqa: E731
+ fetch_memo.fetch_once(("tiktok", "q"), fetch, timeout=1, cacheable=clean)
+ fetch_memo.fetch_once(("tiktok", "q"), fetch, timeout=1, cacheable=clean)
+ assert len(calls) == 2
+
+
+def test_swallowed_failure_reaches_waiters_and_is_not_cached():
+ # Source adapters swallow HTTP errors into the stream's failure sink. A
+ # waiter must see the owner's failure, and the result must not be cached.
+ release = threading.Event()
+ arrived = threading.Barrier(2, timeout=5)
+ calls = []
+
+ def _swallow_429():
+ calls.append(1)
+ release.wait(timeout=5)
+ http._record_failure(http.HTTPError("HTTP 429: Too Many Requests", status_code=429))
+ return {"items": []}
+
+ def _call():
+ arrived.wait()
+ with http.capture_failures() as sink:
+ result = fetch_memo.fetch_once(("instagram", "q"), _swallow_429, timeout=10)
+ return result, [f.status_code for f in sink]
+
+ with ThreadPoolExecutor(max_workers=2) as pool:
+ futures = [pool.submit(_call) for _ in range(2)]
+ threading.Timer(0.2, release.set).start()
+ results = [f.result(timeout=10) for f in futures]
+
+ assert len(calls) == 1
+ assert all(sink == [429] for _result, sink in results)
+ fetch_memo.fetch_once(("instagram", "q"), _swallow_429, timeout=1)
+ assert len(calls) == 2
+
+
+def test_waiter_times_out_instead_of_blocking_past_its_budget():
+ release = threading.Event()
+ owner_started = threading.Event()
+
+ def _hung():
+ owner_started.set()
+ release.wait(timeout=5)
+ return {"items": []}
+
+ owner = threading.Thread(
+ target=fetch_memo.fetch_once, args=(("youtube", "q"), _hung), kwargs={"timeout": 10},
+ daemon=True,
+ )
+ owner.start()
+ assert owner_started.wait(timeout=5)
+ try:
+ with pytest.raises(fetch_memo.MemoWaitTimeout) as exc_info:
+ fetch_memo.fetch_once(("youtube", "q"), _hung, timeout=0.1)
+ assert exc_info.value.outcome_state == health.TIMEOUT
+ finally:
+ release.set()
+ owner.join(timeout=5)
+
+
+def test_reset_forgets_cached_results():
+ calls, fetch = _fetch_counter({"items": []})
+ fetch_memo.fetch_once(("youtube", "q"), fetch, timeout=1)
+ fetch_memo.reset()
+ fetch_memo.fetch_once(("youtube", "q"), fetch, timeout=1)
+ assert len(calls) == 2
diff --git a/tests/test_pipeline_cancellation.py b/tests/test_pipeline_cancellation.py
index bd473180..57fe053e 100644
--- a/tests/test_pipeline_cancellation.py
+++ b/tests/test_pipeline_cancellation.py
@@ -518,7 +518,6 @@ def test_grok_short_remaining_budget_reports_timeout(monkeypatch, partial):
def test_rate_limit_stops_queued_streams_before_fanout_finishes(monkeypatch, x_run, failure):
topic, _, plan, _ = x_run
plan.subqueries = [replace(plan.subqueries[0], label=f"query-{index}") for index in range(6)]
- barrier = threading.Barrier(4)
calls = []
lock = threading.Lock()
@@ -526,13 +525,13 @@ def test_rate_limit_stops_queued_streams_before_fanout_finishes(monkeypatch, x_r
with lock:
index = len(calls)
calls.append(index)
- if index < 4:
- barrier.wait(2)
if index == 0:
+ # Hold the 429 until the other in-flight streams are waiting on
+ # this same request in the run's fetch memo.
+ threading.Event().wait(0.3)
if failure == "exception":
raise http.HTTPError("Rate limit exceeded", status_code=429)
return {"items": [], "error": "xapi: rate limited (429)"}
- threading.Event().wait(0.2)
return {"items": []}
monkeypatch.setattr(pipeline.x_api, "search_x", request)
@@ -540,5 +539,7 @@ def test_rate_limit_stops_queued_streams_before_fanout_finishes(monkeypatch, x_r
topic=topic, config={"X_BEARER_TOKEN": "dummy", "_max_source_fetches": 6},
depth="quick", requested_sources=["x"], internal_subrun=True,
)
- assert calls == [0, 1, 2, 3], "queued X streams must see the completed worker's rate-limit signal"
+ # Every stream searches the same raw topic: in-flight streams share the
+ # one request and its 429, and queued streams see the rate-limit signal.
+ assert calls == [0], "X streams must share the rate-limited request, not repeat it"
assert report.source_status["x"].state == health.RATE_LIMITED
diff --git a/tests/test_pipeline_v3.py b/tests/test_pipeline_v3.py
index 8d771567..a1508224 100644
--- a/tests/test_pipeline_v3.py
+++ b/tests/test_pipeline_v3.py
@@ -1,5 +1,8 @@
import threading
+import time
import unittest
+
+import pytest
from unittest.mock import patch
from lib import health
@@ -2591,5 +2594,357 @@ class TestAmazonSourceGating:
assert pipeline.MAX_SOURCE_FETCHES["amazon"] == 1
+# ---------------------------------------------------------------------------
+# Run-scoped source fetch memo (U4, R5, R6): subquery streams that repeat the
+# raw-topic search for TikTok, Instagram, YouTube, and X fetch it once.
+# ---------------------------------------------------------------------------
+
+_MEMO_TOPIC = "ghostty terminal"
+_MEMO_RANGE = ("2026-09-11", "2026-10-11")
+_MEMO_SC_CONFIG = {
+ "SCRAPECREATORS_API_KEY": "test-key",
+ "INCLUDE_SOURCES": "tiktok,instagram,tiktok_comments,instagram_comments",
+}
+
+
+def _memo_subqueries(sources, labels=("price", "speed", "themes")):
+ return [
+ schema.SubQuery(
+ label=label,
+ search_query=f"ghostty {label}",
+ ranking_query=f"What do people say about ghostty {label}?",
+ sources=list(sources),
+ )
+ for label in labels
+ ]
+
+
+def _raw_topic_subquery(source):
+ return schema.SubQuery(
+ label="primary",
+ search_query=_MEMO_TOPIC,
+ ranking_query=f"What do people say about {_MEMO_TOPIC}?",
+ sources=[source],
+ )
+
+
+def _memo_stream(source, subquery, config, **kwargs):
+ return pipeline._retrieve_stream(
+ topic=_MEMO_TOPIC, subquery=subquery, source=source, config=config,
+ depth="default", date_range=_MEMO_RANGE, runtime=_make_runtime(None),
+ mock=False, raw_topic=_MEMO_TOPIC, **kwargs,
+ )
+
+
+def _memo_streams_concurrently(jobs):
+ from concurrent.futures import ThreadPoolExecutor
+
+ with ThreadPoolExecutor(max_workers=len(jobs)) as pool:
+ futures = [pool.submit(_memo_stream, *job) for job in jobs]
+ return [future.result(timeout=10) for future in futures]
+
+
+def _tiktok_payload(*_args, **_kwargs):
+ return {"items": [{
+ "id": "t1", "video_id": "t1", "text": "ghostty terminal setup tour",
+ "url": "https://www.tiktok.com/@dev/video/1", "author_name": "dev",
+ "date": "2026-10-01", "engagement": {"views": 900, "likes": 90},
+ "relevance": 0.9, "why_relevant": "fixture",
+ }]}
+
+
+def _instagram_payload(*_args, **_kwargs):
+ return {"items": [{
+ "id": "i1", "video_id": "i1", "text": "ghostty terminal reel",
+ "url": "https://www.instagram.com/reel/abc/", "author_name": "dev",
+ "date": "2026-10-01", "engagement": {"views": 500, "likes": 50},
+ "relevance": 0.9, "why_relevant": "fixture",
+ }]}
+
+
+def _youtube_payload(*_args, **_kwargs):
+ return {"items": [
+ {
+ "video_id": f"vid{n:08d}", "title": f"ghostty terminal review {n}",
+ "url": f"https://www.youtube.com/watch?v=vid{n:08d}", "channel_name": "dev",
+ "date": "2026-10-01", "engagement": {"views": 1000 - n, "likes": 10},
+ "relevance": 0.9, "why_relevant": "fixture",
+ }
+ for n in range(6)
+ ]}
+
+
+class TestSourceFetchMemo:
+ def test_three_subquery_streams_fetch_each_source_once(self):
+ subqueries = _memo_subqueries(["tiktok", "instagram", "youtube"])
+ with patch("lib.tiktok.search_and_enrich", side_effect=_tiktok_payload) as tiktok_search, \
+ patch("lib.instagram.search_and_enrich", side_effect=_instagram_payload) as ig_search, \
+ patch("lib.youtube_yt.search_and_transcribe", side_effect=_youtube_payload) as yt_search, \
+ patch("lib.pipeline.which", return_value="/usr/bin/yt-dlp"), \
+ patch("lib.env.is_youtube_comments_available", return_value=True), \
+ patch("lib.youtube_yt.enrich_with_comments") as yt_comments, \
+ patch("lib.tiktok.enrich_with_comments") as tiktok_comments, \
+ patch("lib.instagram.enrich_with_comments") as ig_comments:
+ results = _memo_streams_concurrently([
+ (source, subquery, dict(_MEMO_SC_CONFIG))
+ for subquery in subqueries
+ for source in ("tiktok", "instagram", "youtube")
+ ])
+
+ assert tiktok_search.call_count == 1
+ assert ig_search.call_count == 1
+ assert yt_search.call_count == 1
+ assert tiktok_comments.call_count == 1
+ assert ig_comments.call_count == 1
+ assert yt_comments.call_count == 1
+ assert all(items for items, _artifact in results)
+
+ def test_scrapecreators_only_youtube_searches_once(self):
+ subqueries = _memo_subqueries(["youtube"])
+ with patch("lib.pipeline.which", return_value=None), \
+ patch("lib.youtube_yt.search_youtube_sc", side_effect=_youtube_payload) as sc_search, \
+ patch("lib.env.is_youtube_comments_available", return_value=False):
+ results = _memo_streams_concurrently([
+ ("youtube", subquery, {"SCRAPECREATORS_API_KEY": "test-key"})
+ for subquery in subqueries
+ ])
+ assert sc_search.call_count == 1
+ assert all(len(items) == 6 for items, _artifact in results)
+
+ def test_hashtag_and_keyword_tiktok_calls_are_different_keys(self):
+ [subquery] = _memo_subqueries(["tiktok"], labels=("price",))
+ with patch("lib.tiktok.search_and_enrich", side_effect=_tiktok_payload) as tiktok_search:
+ _memo_stream("tiktok", subquery, {"SCRAPECREATORS_API_KEY": "test-key"})
+ _memo_stream("tiktok", subquery, {"SCRAPECREATORS_API_KEY": "test-key"},
+ tiktok_hashtags=["ghostty"])
+ assert tiktok_search.call_count == 2
+ assert tiktok_search.call_args_list[1].kwargs["hashtags"] == ["ghostty"]
+
+ def test_failed_tiktok_search_is_not_reused(self):
+ [subquery] = _memo_subqueries(["tiktok"], labels=("price",))
+ responses = [{"items": [], "error": "HTTP 503"}, _tiktok_payload()]
+ with patch("lib.tiktok.search_and_enrich", side_effect=responses) as tiktok_search:
+ first_items, first_artifact = _memo_stream("tiktok", subquery, {"SCRAPECREATORS_API_KEY": "test-key"})
+ second_items, _ = _memo_stream("tiktok", subquery, {"SCRAPECREATORS_API_KEY": "test-key"})
+ assert tiktok_search.call_count == 2
+ assert first_items == [] and first_artifact
+ assert second_items
+
+ def test_failed_instagram_search_is_not_reused(self):
+ [subquery] = _memo_subqueries(["instagram"], labels=("price",))
+ responses = [{"items": [], "error": "HTTP 503"}, _instagram_payload()]
+ with patch("lib.instagram.search_and_enrich", side_effect=responses) as ig_search:
+ _memo_stream("instagram", subquery, {"SCRAPECREATORS_API_KEY": "test-key"})
+ second_items, _ = _memo_stream("instagram", subquery, {"SCRAPECREATORS_API_KEY": "test-key"})
+ assert ig_search.call_count == 2
+ assert second_items
+
+ @pytest.mark.parametrize("artifact, clean", [
+ ({}, True),
+ ({"_source_outcome_detail": "yt-dlp thin; backfilled"}, True),
+ ({"_source_outcome": {"state": "error"}}, False),
+ ({"_source_outcome_deferred": {"state": "partial"}}, False),
+ ({"_source_outcome_if_empty": {"state": "error"}}, False),
+ ({"_source_outcome_detail_state": "partial"}, False),
+ ])
+ def test_clean_stream_result_rejects_every_failure_marker(self, artifact, clean):
+ assert pipeline._clean_stream_result(([], artifact)) is clean
+
+ def test_x_waiter_gets_owner_result_when_owner_runs_to_chain_deadline(self):
+ deadline = time.monotonic() + 0.2
+
+ def slow_backend(*_args, **_kwargs):
+ time.sleep(0.35)
+ return [{"id": "x1"}], None, None
+
+ from concurrent.futures import ThreadPoolExecutor
+ with patch.object(pipeline, "_attempt_x_backend", side_effect=slow_backend) as backend, \
+ ThreadPoolExecutor(max_workers=2) as pool:
+ futures = [
+ pool.submit(
+ pipeline._attempt_x_backend_memo,
+ "bird", "ghostty", *_MEMO_RANGE, "default", {}, deadline=deadline,
+ )
+ for _ in range(2)
+ ]
+ results = [future.result(timeout=10) for future in futures]
+ assert backend.call_count == 1
+ assert all(items == [{"id": "x1"}] and err is None for items, err, _ in results)
+
+ def test_x_waiter_timeout_is_a_chain_error_not_an_exception(self):
+ deadline = time.monotonic() + 0.1
+
+ def hung_backend(*_args, **_kwargs):
+ time.sleep(0.6)
+ return [{"id": "x1"}], None, None
+
+ from concurrent.futures import ThreadPoolExecutor
+ with patch.object(pipeline, "_attempt_x_backend", side_effect=hung_backend), \
+ patch.object(pipeline, "_X_MEMO_WAIT_GRACE_SECONDS", 0.1), \
+ ThreadPoolExecutor(max_workers=2) as pool:
+ owner = pool.submit(
+ pipeline._attempt_x_backend_memo,
+ "bird", "ghostty", *_MEMO_RANGE, "default", {}, deadline=deadline,
+ )
+ time.sleep(0.05)
+ waiter = pool.submit(
+ pipeline._attempt_x_backend_memo,
+ "bird", "ghostty", *_MEMO_RANGE, "default", {}, deadline=deadline,
+ )
+ items, err, err_state = waiter.result(timeout=10)
+ owner.result(timeout=10)
+ assert items == [] and err
+ assert err_state == health.TIMEOUT
+
+ def _thin_youtube_retry(self, config):
+ plan = schema.QueryPlan(
+ intent="general", freshness_mode="balanced_recent", cluster_mode="topic",
+ raw_topic=_MEMO_TOPIC, subqueries=_memo_subqueries(["youtube"]),
+ source_weights={"youtube": 1.0},
+ )
+ pipeline._retry_thin_sources(
+ topic=_MEMO_TOPIC, bundle=schema.RetrievalBundle(), plan=plan,
+ config=config, depth="default", date_range=_MEMO_RANGE,
+ runtime=_make_runtime(None), mock=False, rate_limited_sources=set(),
+ rate_limit_lock=threading.Lock(), settings={"per_stream_limit": 20},
+ )
+
+ def test_thin_retry_after_clean_youtube_fetch_makes_no_network_call(self):
+ [subquery] = _memo_subqueries(["youtube"], labels=("price",))
+ with patch("lib.pipeline.which", return_value="/usr/bin/yt-dlp"), \
+ patch("lib.youtube_yt.search_and_transcribe", side_effect=_youtube_payload) as yt_search, \
+ patch("lib.env.is_youtube_comments_available", return_value=False):
+ _memo_stream("youtube", subquery, {})
+ self._thin_youtube_retry({})
+ assert yt_search.call_count == 1
+
+ def test_thin_retry_after_failed_youtube_fetch_refetches(self):
+ [subquery] = _memo_subqueries(["youtube"], labels=("price",))
+ responses = [{"items": [], "error": "yt-dlp search timed out"}, _youtube_payload()]
+ with patch("lib.pipeline.which", return_value="/usr/bin/yt-dlp"), \
+ patch("lib.youtube_yt.search_and_transcribe", side_effect=responses) as yt_search, \
+ patch("lib.env.is_youtube_comments_available", return_value=False):
+ _items, artifact = _memo_stream("youtube", subquery, {})
+ assert artifact["_source_outcome"]["state"] == health.TIMEOUT
+ self._thin_youtube_retry({})
+ assert yt_search.call_count == 2
+
+ def _thin_retry(self, source):
+ plan = schema.QueryPlan(
+ intent="general", freshness_mode="balanced_recent", cluster_mode="topic",
+ raw_topic=_MEMO_TOPIC,
+ subqueries=[_raw_topic_subquery(source)],
+ source_weights={source: 1.0},
+ )
+ pipeline._retry_thin_sources(
+ topic=_MEMO_TOPIC, bundle=schema.RetrievalBundle(), plan=plan,
+ config={}, depth="default", date_range=_MEMO_RANGE,
+ runtime=_make_runtime(None), mock=False, rate_limited_sources=set(),
+ rate_limit_lock=threading.Lock(), settings={"per_stream_limit": 20},
+ )
+
+ @pytest.mark.parametrize("source, target", [
+ ("polymarket", "lib.polymarket.search_polymarket"),
+ ("digg", "lib.digg.search_digg"),
+ ])
+ def test_thin_retry_reuses_clean_raw_topic_fetch(self, source, target):
+ # Live runs: the planner's subquery searches the raw topic, the thin
+ # retry's core subject is the same string, so the request repeated.
+ empty = {"events": [], "results": []}
+ with patch(target, return_value=empty) as search:
+ _memo_stream(source, _raw_topic_subquery(source), {})
+ self._thin_retry(source)
+ assert search.call_count == 1
+
+ @pytest.mark.parametrize("source, target", [
+ ("polymarket", "lib.polymarket.search_polymarket"),
+ ("digg", "lib.digg.search_digg"),
+ ])
+ def test_thin_retry_refetches_after_failed_fetch(self, source, target):
+ failed = {"events": [], "results": [], "error": "HTTP 503"}
+ with patch(target, side_effect=[failed, {"events": [], "results": []}]) as search:
+ _items, artifact = _memo_stream(source, _raw_topic_subquery(source), {})
+ assert artifact.get("_source_outcome")
+ self._thin_retry(source)
+ assert search.call_count == 2
+
+ @pytest.mark.parametrize("source, target", [
+ ("polymarket", "lib.polymarket.search_polymarket"),
+ ("digg", "lib.digg.search_digg"),
+ ])
+ def test_different_queries_fetch_separately(self, source, target):
+ with patch(target, return_value={"events": [], "results": []}) as search:
+ for subquery in _memo_subqueries([source], labels=("price", "speed")):
+ _memo_stream(source, subquery, {})
+ assert search.call_count == 2
+
+ def test_x_backend_fetched_once_and_each_subquery_judged_with_its_own_ranking_query(self):
+ subqueries = _memo_subqueries(["x"], labels=("price", "speed"))
+ posts = [{
+ "id": "X1", "text": "ghostty terminal is fast", "author_handle": "dev",
+ "url": "https://x.com/dev/status/1", "date": "2026-10-01",
+ }]
+ judged_with: list[str] = []
+
+ def _judge(items, _topic, *, ranking_query=""):
+ judged_with.append(ranking_query)
+ return {"on_topic_items": list(items), "on_topic_ratio": 1.0}
+
+ with patch("lib.env.x_backend_chain", return_value=["bird"]), \
+ patch("lib.pipeline._fetch_x_backend", return_value=(posts, "")) as fetch, \
+ patch("lib.x_judge.should_retry_x_search", return_value=False), \
+ patch("lib.x_judge.judge_x_corpus", side_effect=_judge):
+ results = [_memo_stream("x", subquery, {}) for subquery in subqueries]
+
+ assert fetch.call_count == 1
+ assert sorted(judged_with) == sorted(sq.ranking_query for sq in subqueries)
+ assert all(len(items) == 1 for items, _artifact in results)
+
+ def _parity_report(self):
+ plan = {
+ "intent": "general", "freshness_mode": "balanced_recent", "cluster_mode": "story",
+ "subqueries": [
+ {"label": sq.label, "search_query": sq.search_query,
+ "ranking_query": sq.ranking_query, "sources": list(sq.sources)}
+ for sq in _memo_subqueries(["tiktok", "instagram", "youtube"])
+ ],
+ }
+ with patch("lib.tiktok.search_and_enrich", side_effect=_tiktok_payload), \
+ patch("lib.instagram.search_and_enrich", side_effect=_instagram_payload), \
+ patch("lib.youtube_yt.search_and_transcribe", side_effect=_youtube_payload), \
+ patch("lib.youtube_yt.search_youtube_sc", return_value={"items": []}), \
+ patch("lib.youtube_yt.backfill_transcripts"), \
+ patch("lib.pipeline.which", return_value="/usr/bin/yt-dlp"), \
+ patch("lib.env.is_youtube_comments_available", return_value=False), \
+ patch("lib.tiktok.enrich_with_comments"), \
+ patch("lib.instagram.enrich_with_comments"):
+ return pipeline.run(
+ topic=_MEMO_TOPIC, config=dict(_MEMO_SC_CONFIG), depth="default",
+ requested_sources=["tiktok", "instagram", "youtube"], mock=False,
+ external_plan=plan, web_backend="none", save_dir="",
+ )
+
+ def test_report_parity_with_memo_bypassed(self):
+ from lib import fetch_memo
+
+ def _summary(report):
+ return (
+ {source: sorted(item.url for item in items)
+ for source, items in report.items_by_source.items()},
+ {source: outcome.state for source, outcome in report.source_status.items()},
+ sorted(report.errors_by_source),
+ [candidate.url for candidate in report.ranked_candidates],
+ )
+
+ memoized = _summary(self._parity_report())
+ fetch_memo.reset()
+ with patch("lib.fetch_memo.fetch_once",
+ side_effect=lambda _key, fetch, **_kwargs: fetch()):
+ bypassed = _summary(self._parity_report())
+
+ assert memoized == bypassed
+ assert set(memoized[0]) == {"tiktok", "instagram", "youtube"}
+
+
if __name__ == "__main__":
unittest.main()
diff --git a/tests/test_quality_nudge.py b/tests/test_quality_nudge.py
index ac84bfc0..5c4c0de3 100644
--- a/tests/test_quality_nudge.py
+++ b/tests/test_quality_nudge.py
@@ -1099,10 +1099,12 @@ class TestXaiErrorRemediation:
},
)
+ # The failed xai call is not memoized, so each subquery tries it; the
+ # backup's clean result is fetched once and served to both subqueries.
assert xai_search.call_count >= 2
- assert fallback_search.call_count >= 2
+ assert fallback_search.call_count == 1
assert any(
- "status/123456789012345671" in item.url
+ "status/123456789012345670" in item.url
for item in second_query_items
)
assert report.source_status["x"].lane_failure_state == schema.RATE_LIMITED
diff --git a/tests/test_youtube_backstop.py b/tests/test_youtube_backstop.py
index 2e79b37a..49b87643 100644
--- a/tests/test_youtube_backstop.py
+++ b/tests/test_youtube_backstop.py
@@ -225,7 +225,7 @@ def test_partial_ytdlp_search_error_stays_visible_after_backfill():
assert "yt-dlp bot gate" in artifact["_source_outcome"]["detail"]
-def test_full_pipeline_reports_clean_backfill_and_searches_paid_once_per_subquery():
+def test_full_pipeline_reports_clean_backfill_and_searches_paid_once_per_run():
free = {
"video_id": "a", "title": "Vuori review", "date": "2026-06-15",
"url": "https://www.youtube.com/watch?v=a", "transcript_snippet": "free transcript",
@@ -263,7 +263,9 @@ def test_full_pipeline_reports_clean_backfill_and_searches_paid_once_per_subquer
external_plan=plan, as_of_date="2026-07-01",
)
- assert search.call_count == 2
+ # Both subqueries search the raw topic: the run fetch memo serves the
+ # second stream (and the thin retry) from the first clean fetch.
+ assert search.call_count == 1
assert report.source_status["youtube"].state == health.OK
assert "ScrapeCreators added 3" in report.source_status["youtube"].detail
@@ -301,8 +303,10 @@ def _run_thin_youtube_report(sc_responses):
return report, search.call_count
-def test_empty_clean_backfill_can_retry_and_recover():
- recovered = [
+def test_thin_retry_reuses_a_clean_empty_backfill_instead_of_refetching():
+ # A clean (error-free) backfill is the answer for this query in this run:
+ # the identical thin-source retry is served from the run fetch memo.
+ unused = [
{
"video_id": str(i), "title": f"Vuori review {i}", "date": "2026-06-15",
"url": f"https://www.youtube.com/watch?v={i}",
@@ -310,18 +314,16 @@ def test_empty_clean_backfill_can_retry_and_recover():
for i in range(2)
]
report, calls = _run_thin_youtube_report([
- {"items": []}, {"items": recovered},
+ {"items": []}, {"items": unused},
])
- assert calls == 2
+ assert calls == 1
assert report.source_status["youtube"].state == health.OK
- assert {item.item_id for item in report.items_by_source["youtube"]} >= {
- "free", "0",
- }
+ assert [item.item_id for item in report.items_by_source["youtube"]] == ["free"]
def test_clean_sparse_topic_stays_ok_after_empty_backfill_and_retry():
report, calls = _run_thin_youtube_report([{"items": []}, {"items": []}])
- assert calls == 2
+ assert calls == 1
assert report.source_status["youtube"].state == health.OK
assert [item.item_id for item in report.items_by_source["youtube"]] == ["free"]
assert "added 0" in report.source_status["youtube"].detail