Skip to content

DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.

Bound Web Risk lookup batches and scope indexing progress per run

Bingxi Zhao (Frank)4b68b2b452c0
diff --git a/deeptutor/services/rag/pipelines/llamaindex/pipeline.py b/deeptutor/services/rag/pipelines/llamaindex/pipeline.py
index 6ff22251c..2b8467e56 100644
--- a/deeptutor/services/rag/pipelines/llamaindex/pipeline.py
+++ b/deeptutor/services/rag/pipelines/llamaindex/pipeline.py
@@ -38,17 +38,6 @@ DEFAULT_KB_BASE_DIR = str(get_runtime_data_root() / "knowledge_bases")
# is treated as stalled (see _run_with_stall_guard).
_INDEX_STALL_TIMEOUT_SECONDS = 600.0
# How often the stall guard checks the progress heartbeat.
-# Which indexing run currently owns the shared embedding-progress callback
-# slot. ``set_progress_callback`` writes to the process-global LlamaIndex
-# ``Settings.embed_model`` (one slot), and a run that stalls leaves its sync
-# embedding worker running in a thread Python cannot interrupt. That worker
-# keeps calling the stored callback long after the run ended, so without an
-# ownership check the stale worker's batch events land in the *next* job that
-# grabbed the slot — the "infinite Embedding batches / reset to 1/N" leak
-# (#1478). The stall guard stamps each run with a fresh owner and its
-# ``_heartbeat`` refuses to forward progress once ``owner`` no longer matches.
-_ACTIVE_PROGRESS_OWNER: Optional[str] = None
-
_INDEX_STALL_POLL_SECONDS = 5.0
SignatureProvider = Callable[[], EmbeddingSignature | None]
@@ -86,21 +75,15 @@ async def _run_with_stall_guard(
if stall_timeout is None:
stall_timeout = _INDEX_STALL_TIMEOUT_SECONDS
- # Stamp this run as the current owner of the shared progress slot. The
- # ``_heartbeat`` wrapper refuses to forward progress once a newer run has
- # taken ownership (or we have finished), so a worker thread left running
- # by a stall can no longer write its batch events into the next job
- # (#1478 — the "infinite Embedding batches / reset to 1/N" leak).
- owner = f"{time.monotonic_ns()}"
- global _ACTIVE_PROGRESS_OWNER
- _ACTIVE_PROGRESS_OWNER = owner
-
+ # Each run has its own embedding adapter in the operation context. A
+ # process-global owner would silence healthy concurrent runs. Invalidate
+ # only this wrapper when its run ends, including on stall while a sync
+ # worker may still be alive.
+ active = True
last_progress = {"at": time.monotonic()}
def _heartbeat(*args: Any, **kwargs: Any) -> None:
- if _ACTIVE_PROGRESS_OWNER != owner:
- # A concurrent or previous run owns the slot now; this worker is
- # either stale or displaced. Do not feed its progress elsewhere.
+ if not active:
return
last_progress["at"] = time.monotonic()
if progress_callback is not None:
@@ -132,10 +115,7 @@ async def _run_with_stall_guard(
"endpoint and retry."
)
finally:
- # Release ownership of the progress slot exactly when this run still
- # holds it — never clobber a newer run that has already taken over.
- if _ACTIVE_PROGRESS_OWNER == owner:
- _ACTIVE_PROGRESS_OWNER = None
+ active = False
class LlamaIndexPipeline:
diff --git a/deeptutor/services/search/source_filter.py b/deeptutor/services/search/source_filter.py
index 295131d71..5812c43a3 100644
--- a/deeptutor/services/search/source_filter.py
+++ b/deeptutor/services/search/source_filter.py
@@ -2,6 +2,7 @@
from __future__ import annotations
+from concurrent.futures import Future, ThreadPoolExecutor, wait
import ipaddress
import json
import logging
@@ -86,8 +87,12 @@ _WEB_RISK_TIMEOUT_S = 5.0
_WEB_RISK_THREAT_TYPES = ("MALWARE", "SOCIAL_ENGINEERING", "UNWANTED_SOFTWARE")
_WEB_RISK_CACHE_TTL_S = 900.0
_WEB_RISK_CACHE_MAX = 1024
+_WEB_RISK_BATCH_TIMEOUT_S = 5.5
+_WEB_RISK_MAX_INFLIGHT = 32
_web_risk_cache: dict[str, tuple[float, bool]] = {}
_web_risk_cache_lock = threading.Lock()
+_web_risk_inflight: dict[str, Future[bool | None]] = {}
+_web_risk_executor = ThreadPoolExecutor(max_workers=8, thread_name_prefix="web-risk")
def _as_bool(value: Any, default: bool = False) -> bool:
@@ -286,7 +291,8 @@ def _web_risk_rejections(
requester = request_web_risk or _request_web_risk
flagged: set[str] = set()
now = time.monotonic()
- pending: list[str] = []
+ futures: dict[str, Future[bool | None]] = {}
+ created: list[tuple[str, Future[bool | None]]] = []
with _web_risk_cache_lock:
for url in unique:
entry = _web_risk_cache.get(url)
@@ -294,23 +300,45 @@ def _web_risk_rejections(
if entry[1]:
flagged.add(url)
else:
- pending.append(url)
- for url in pending:
- try:
- is_flagged = bool(requester(url, api_key=api_key))
- except Exception as exc: # noqa: BLE001 — network / JSON / quota quirks
- _logger.warning("Web Risk lookup skipped after error: %s", exc)
- continue
- with _web_risk_cache_lock:
- if len(_web_risk_cache) >= _WEB_RISK_CACHE_MAX:
- for stale in list(_web_risk_cache)[: len(_web_risk_cache) // 2]:
- _web_risk_cache.pop(stale, None)
- _web_risk_cache[url] = (time.monotonic(), is_flagged)
- if is_flagged:
- flagged.add(url)
+ future = _web_risk_inflight.get(url)
+ if future is None and len(_web_risk_inflight) < _WEB_RISK_MAX_INFLIGHT:
+ future = _web_risk_executor.submit(_lookup_web_risk, requester, url, api_key)
+ _web_risk_inflight[url] = future
+ created.append((url, future))
+ if future is not None:
+ futures[url] = future
+ for url, future in created:
+ future.add_done_callback(lambda completed, key=url: _cache_web_risk_result(key, completed))
+ if futures:
+ done, _ = wait(set(futures.values()), timeout=_WEB_RISK_BATCH_TIMEOUT_S)
+ for url, future in futures.items():
+ if future in done and future.result():
+ flagged.add(url)
return flagged
+def _lookup_web_risk(requester: Any, url: str, api_key: str) -> bool | None:
+ """A failed lookup is unknown, not a safe result to cache."""
+ try:
+ return bool(requester(url, api_key=api_key))
+ except Exception as exc: # noqa: BLE001 — network / JSON / quota quirks
+ _logger.warning("Web Risk lookup skipped after error: %s", exc)
+ return None
+
+
+def _cache_web_risk_result(url: str, future: Future[bool | None]) -> None:
+ result = future.result()
+ with _web_risk_cache_lock:
+ if _web_risk_inflight.get(url) is future:
+ _web_risk_inflight.pop(url, None)
+ if result is None:
+ return
+ if len(_web_risk_cache) >= _WEB_RISK_CACHE_MAX:
+ for stale in list(_web_risk_cache)[: len(_web_risk_cache) // 2]:
+ _web_risk_cache.pop(stale, None)
+ _web_risk_cache[url] = (time.monotonic(), result)
+
+
def _reference_text(*, title: str = "", snippet: str = "", content: str = "") -> str:
parts = [str(title or "").strip(), str(snippet or "").strip(), str(content or "").strip()]
return "\n".join(part for part in parts if part)
diff --git a/tests/services/rag/test_llamaindex_pipeline_stall.py b/tests/services/rag/test_llamaindex_pipeline_stall.py
index 6783b43fa..7f0704ff9 100644
--- a/tests/services/rag/test_llamaindex_pipeline_stall.py
+++ b/tests/services/rag/test_llamaindex_pipeline_stall.py
@@ -10,6 +10,8 @@ operations with a clear error instead of hanging indefinitely.
from __future__ import annotations
+import asyncio
+from contextvars import ContextVar
import time
from types import SimpleNamespace
from typing import Any
@@ -26,6 +28,12 @@ def _llamaindex_modules() -> tuple[Any, Any, Any]:
return pipeline_module, storage_module, LlamaIndexPipeline
[email protected](autouse=True)
+def _avoid_real_embedding_adapter(monkeypatch: pytest.MonkeyPatch) -> None:
+ pipeline_module, _, _ = _llamaindex_modules()
+ monkeypatch.setattr(pipeline_module, "set_progress_callback", lambda _callback: None)
+
+
async def _async_noop(*args, **kwargs) -> None:
"""Async stand-in for connectivity checks that need no provider call."""
@@ -275,11 +283,42 @@ async def test_stale_wrapper_from_finished_run_is_suppressed_after_ownership_cha
== "done-a"
)
stale_a_cb = captured["cb"]
- # This run released the slot; a newer job takes ownership of it.
- assert pipeline_module._ACTIVE_PROGRESS_OWNER is None
- pipeline_module._ACTIVE_PROGRESS_OWNER = "job-b-owner"
-
stale_a_cb(9, 9)
assert events_a == [(1, 1)]
- pipeline_module._ACTIVE_PROGRESS_OWNER = None
+
+
+async def test_concurrent_indexing_runs_keep_their_own_progress(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """Independent KB adapters must both receive progress during overlap."""
+ pipeline_module, _, _ = _llamaindex_modules()
+ monkeypatch.setattr(pipeline_module, "_INDEX_STALL_POLL_SECONDS", 0.02)
+ adapter = ContextVar("test_embedding_adapter", default="")
+ callbacks: dict[str, Any] = {}
+ monkeypatch.setattr(
+ pipeline_module,
+ "set_progress_callback",
+ lambda cb: callbacks.__setitem__(adapter.get(), cb),
+ )
+ events: dict[str, list[tuple[int, int]]] = {"a": [], "b": []}
+
+ async def run(name: str) -> str:
+ adapter.set(name)
+
+ def work() -> str:
+ for batch in range(1, 5):
+ callbacks[name](batch, 4)
+ time.sleep(0.03)
+ return name
+
+ return await pipeline_module._run_with_stall_guard(
+ work,
+ progress_callback=lambda batch, total: events[name].append((batch, total)),
+ stall_timeout=0.5,
+ )
+
+ assert await asyncio.gather(run("a"), run("b")) == ["a", "b"]
+ assert events["a"] == [(batch, 4) for batch in range(1, 5)]
+ assert events["b"] == [(batch, 4) for batch in range(1, 5)]
diff --git a/tests/services/search/test_web_search_runtime.py b/tests/services/search/test_web_search_runtime.py
index 2cde4a061..a0d47a603 100644
--- a/tests/services/search/test_web_search_runtime.py
+++ b/tests/services/search/test_web_search_runtime.py
@@ -2,6 +2,10 @@
from __future__ import annotations
+from concurrent.futures import ThreadPoolExecutor, wait
+import threading
+import time
+
import pytest
from deeptutor.services.config.provider_runtime import ResolvedSearchConfig
@@ -800,6 +804,7 @@ def test_source_filter_web_risk_caches_lookups() -> None:
from deeptutor.services.search import source_filter
source_filter._web_risk_cache.clear()
+
calls: list[str] = []
def _fake_web_risk(url: str, *, api_key: str) -> bool:
@@ -838,3 +843,48 @@ def test_source_filter_web_risk_caches_lookups() -> None:
assert calls == ["https://school.example/lesson"]
source_filter._web_risk_cache.clear()
+
+
+def test_web_risk_batch_has_one_deadline(monkeypatch: pytest.MonkeyPatch) -> None:
+ from deeptutor.services.search import source_filter
+
+ monkeypatch.setattr(source_filter, "_WEB_RISK_BATCH_TIMEOUT_S", 0.05)
+
+ def slow(_url: str, *, api_key: str) -> bool:
+ time.sleep(0.2)
+ return False
+
+ started = time.monotonic()
+ urls = [f"https://example.org/page-{i}" for i in range(12)]
+ assert source_filter._web_risk_rejections(urls, api_key="test", request_web_risk=slow) == set()
+ assert time.monotonic() - started < 0.15
+ wait(list(source_filter._web_risk_inflight.values()), timeout=2)
+
+
+def test_concurrent_web_risk_calls_share_inflight_lookup() -> None:
+ from deeptutor.services.search import source_filter
+
+ started = threading.Event()
+ release = threading.Event()
+ calls = 0
+
+ def lookup(_url: str, *, api_key: str) -> bool:
+ nonlocal calls
+ calls += 1
+ started.set()
+ assert release.wait(2)
+ return True
+
+ url = "https://example.org/shared-lookup"
+ with ThreadPoolExecutor(max_workers=2) as executor:
+ first = executor.submit(
+ source_filter._web_risk_rejections, [url], api_key="test", request_web_risk=lookup
+ )
+ assert started.wait(2)
+ second = executor.submit(
+ source_filter._web_risk_rejections, [url], api_key="test", request_web_risk=lookup
+ )
+ release.set()
+ assert first.result(timeout=2) == {url}
+ assert second.result(timeout=2) == {url}
+ assert calls == 1