DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.
Python ai-agentsai-tutorclawbotcli-tooldeepresearchinteractive-learninglearningmulti-agent-systemsrag
Bound Web Risk lookup batches and scope indexing progress per run
Bingxi Zhao (Frank)4b68b2b452c0
diff --git a/deeptutor/services/rag/pipelines/llamaindex/pipeline.py b/deeptutor/services/rag/pipelines/llamaindex/pipeline.pyindex 6ff22251c..2b8467e56 100644--- a/deeptutor/services/rag/pipelines/llamaindex/pipeline.py+++ b/deeptutor/services/rag/pipelines/llamaindex/pipeline.py@@ -38,17 +38,6 @@ DEFAULT_KB_BASE_DIR = str(get_runtime_data_root() / "knowledge_bases")# is treated as stalled (see _run_with_stall_guard)._INDEX_STALL_TIMEOUT_SECONDS = 600.0# How often the stall guard checks the progress heartbeat.-# Which indexing run currently owns the shared embedding-progress callback-# slot. ``set_progress_callback`` writes to the process-global LlamaIndex-# ``Settings.embed_model`` (one slot), and a run that stalls leaves its sync-# embedding worker running in a thread Python cannot interrupt. That worker-# keeps calling the stored callback long after the run ended, so without an-# ownership check the stale worker's batch events land in the *next* job that-# grabbed the slot — the "infinite Embedding batches / reset to 1/N" leak-# (#1478). The stall guard stamps each run with a fresh owner and its-# ``_heartbeat`` refuses to forward progress once ``owner`` no longer matches.-_ACTIVE_PROGRESS_OWNER: Optional[str] = None-_INDEX_STALL_POLL_SECONDS = 5.0SignatureProvider = Callable[[], EmbeddingSignature | None]@@ -86,21 +75,15 @@ async def _run_with_stall_guard(if stall_timeout is None:stall_timeout = _INDEX_STALL_TIMEOUT_SECONDS- # Stamp this run as the current owner of the shared progress slot. The- # ``_heartbeat`` wrapper refuses to forward progress once a newer run has- # taken ownership (or we have finished), so a worker thread left running- # by a stall can no longer write its batch events into the next job- # (#1478 — the "infinite Embedding batches / reset to 1/N" leak).- owner = f"{time.monotonic_ns()}"- global _ACTIVE_PROGRESS_OWNER- _ACTIVE_PROGRESS_OWNER = owner-+ # Each run has its own embedding adapter in the operation context. A+ # process-global owner would silence healthy concurrent runs. Invalidate+ # only this wrapper when its run ends, including on stall while a sync+ # worker may still be alive.+ active = Truelast_progress = {"at": time.monotonic()}def _heartbeat(*args: Any, **kwargs: Any) -> None:- if _ACTIVE_PROGRESS_OWNER != owner:- # A concurrent or previous run owns the slot now; this worker is- # either stale or displaced. Do not feed its progress elsewhere.+ if not active:returnlast_progress["at"] = time.monotonic()if progress_callback is not None:@@ -132,10 +115,7 @@ async def _run_with_stall_guard("endpoint and retry.")finally:- # Release ownership of the progress slot exactly when this run still- # holds it — never clobber a newer run that has already taken over.- if _ACTIVE_PROGRESS_OWNER == owner:- _ACTIVE_PROGRESS_OWNER = None+ active = Falseclass LlamaIndexPipeline:diff --git a/deeptutor/services/search/source_filter.py b/deeptutor/services/search/source_filter.pyindex 295131d71..5812c43a3 100644--- a/deeptutor/services/search/source_filter.py+++ b/deeptutor/services/search/source_filter.py@@ -2,6 +2,7 @@from __future__ import annotations+from concurrent.futures import Future, ThreadPoolExecutor, waitimport ipaddressimport jsonimport logging@@ -86,8 +87,12 @@ _WEB_RISK_TIMEOUT_S = 5.0_WEB_RISK_THREAT_TYPES = ("MALWARE", "SOCIAL_ENGINEERING", "UNWANTED_SOFTWARE")_WEB_RISK_CACHE_TTL_S = 900.0_WEB_RISK_CACHE_MAX = 1024+_WEB_RISK_BATCH_TIMEOUT_S = 5.5+_WEB_RISK_MAX_INFLIGHT = 32_web_risk_cache: dict[str, tuple[float, bool]] = {}_web_risk_cache_lock = threading.Lock()+_web_risk_inflight: dict[str, Future[bool | None]] = {}+_web_risk_executor = ThreadPoolExecutor(max_workers=8, thread_name_prefix="web-risk")def _as_bool(value: Any, default: bool = False) -> bool:@@ -286,7 +291,8 @@ def _web_risk_rejections(requester = request_web_risk or _request_web_riskflagged: set[str] = set()now = time.monotonic()- pending: list[str] = []+ futures: dict[str, Future[bool | None]] = {}+ created: list[tuple[str, Future[bool | None]]] = []with _web_risk_cache_lock:for url in unique:entry = _web_risk_cache.get(url)@@ -294,23 +300,45 @@ def _web_risk_rejections(if entry[1]:flagged.add(url)else:- pending.append(url)- for url in pending:- try:- is_flagged = bool(requester(url, api_key=api_key))- except Exception as exc: # noqa: BLE001 — network / JSON / quota quirks- _logger.warning("Web Risk lookup skipped after error: %s", exc)- continue- with _web_risk_cache_lock:- if len(_web_risk_cache) >= _WEB_RISK_CACHE_MAX:- for stale in list(_web_risk_cache)[: len(_web_risk_cache) // 2]:- _web_risk_cache.pop(stale, None)- _web_risk_cache[url] = (time.monotonic(), is_flagged)- if is_flagged:- flagged.add(url)+ future = _web_risk_inflight.get(url)+ if future is None and len(_web_risk_inflight) < _WEB_RISK_MAX_INFLIGHT:+ future = _web_risk_executor.submit(_lookup_web_risk, requester, url, api_key)+ _web_risk_inflight[url] = future+ created.append((url, future))+ if future is not None:+ futures[url] = future+ for url, future in created:+ future.add_done_callback(lambda completed, key=url: _cache_web_risk_result(key, completed))+ if futures:+ done, _ = wait(set(futures.values()), timeout=_WEB_RISK_BATCH_TIMEOUT_S)+ for url, future in futures.items():+ if future in done and future.result():+ flagged.add(url)return flagged+def _lookup_web_risk(requester: Any, url: str, api_key: str) -> bool | None:+ """A failed lookup is unknown, not a safe result to cache."""+ try:+ return bool(requester(url, api_key=api_key))+ except Exception as exc: # noqa: BLE001 — network / JSON / quota quirks+ _logger.warning("Web Risk lookup skipped after error: %s", exc)+ return None+++def _cache_web_risk_result(url: str, future: Future[bool | None]) -> None:+ result = future.result()+ with _web_risk_cache_lock:+ if _web_risk_inflight.get(url) is future:+ _web_risk_inflight.pop(url, None)+ if result is None:+ return+ if len(_web_risk_cache) >= _WEB_RISK_CACHE_MAX:+ for stale in list(_web_risk_cache)[: len(_web_risk_cache) // 2]:+ _web_risk_cache.pop(stale, None)+ _web_risk_cache[url] = (time.monotonic(), result)++def _reference_text(*, title: str = "", snippet: str = "", content: str = "") -> str:parts = [str(title or "").strip(), str(snippet or "").strip(), str(content or "").strip()]return "\n".join(part for part in parts if part)diff --git a/tests/services/rag/test_llamaindex_pipeline_stall.py b/tests/services/rag/test_llamaindex_pipeline_stall.pyindex 6783b43fa..7f0704ff9 100644--- a/tests/services/rag/test_llamaindex_pipeline_stall.py+++ b/tests/services/rag/test_llamaindex_pipeline_stall.py@@ -10,6 +10,8 @@ operations with a clear error instead of hanging indefinitely.from __future__ import annotations+import asyncio+from contextvars import ContextVarimport timefrom types import SimpleNamespacefrom typing import Any@@ -26,6 +28,12 @@ def _llamaindex_modules() -> tuple[Any, Any, Any]:return pipeline_module, storage_module, LlamaIndexPipeline[email protected](autouse=True)+def _avoid_real_embedding_adapter(monkeypatch: pytest.MonkeyPatch) -> None:+ pipeline_module, _, _ = _llamaindex_modules()+ monkeypatch.setattr(pipeline_module, "set_progress_callback", lambda _callback: None)++async def _async_noop(*args, **kwargs) -> None:"""Async stand-in for connectivity checks that need no provider call."""@@ -275,11 +283,42 @@ async def test_stale_wrapper_from_finished_run_is_suppressed_after_ownership_cha== "done-a")stale_a_cb = captured["cb"]- # This run released the slot; a newer job takes ownership of it.- assert pipeline_module._ACTIVE_PROGRESS_OWNER is None- pipeline_module._ACTIVE_PROGRESS_OWNER = "job-b-owner"-stale_a_cb(9, 9)assert events_a == [(1, 1)]- pipeline_module._ACTIVE_PROGRESS_OWNER = None+++async def test_concurrent_indexing_runs_keep_their_own_progress(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """Independent KB adapters must both receive progress during overlap."""+ pipeline_module, _, _ = _llamaindex_modules()+ monkeypatch.setattr(pipeline_module, "_INDEX_STALL_POLL_SECONDS", 0.02)+ adapter = ContextVar("test_embedding_adapter", default="")+ callbacks: dict[str, Any] = {}+ monkeypatch.setattr(+ pipeline_module,+ "set_progress_callback",+ lambda cb: callbacks.__setitem__(adapter.get(), cb),+ )+ events: dict[str, list[tuple[int, int]]] = {"a": [], "b": []}++ async def run(name: str) -> str:+ adapter.set(name)++ def work() -> str:+ for batch in range(1, 5):+ callbacks[name](batch, 4)+ time.sleep(0.03)+ return name++ return await pipeline_module._run_with_stall_guard(+ work,+ progress_callback=lambda batch, total: events[name].append((batch, total)),+ stall_timeout=0.5,+ )++ assert await asyncio.gather(run("a"), run("b")) == ["a", "b"]+ assert events["a"] == [(batch, 4) for batch in range(1, 5)]+ assert events["b"] == [(batch, 4) for batch in range(1, 5)]diff --git a/tests/services/search/test_web_search_runtime.py b/tests/services/search/test_web_search_runtime.pyindex 2cde4a061..a0d47a603 100644--- a/tests/services/search/test_web_search_runtime.py+++ b/tests/services/search/test_web_search_runtime.py@@ -2,6 +2,10 @@from __future__ import annotations+from concurrent.futures import ThreadPoolExecutor, wait+import threading+import time+import pytestfrom deeptutor.services.config.provider_runtime import ResolvedSearchConfig@@ -800,6 +804,7 @@ def test_source_filter_web_risk_caches_lookups() -> None:from deeptutor.services.search import source_filtersource_filter._web_risk_cache.clear()+calls: list[str] = []def _fake_web_risk(url: str, *, api_key: str) -> bool:@@ -838,3 +843,48 @@ def test_source_filter_web_risk_caches_lookups() -> None:assert calls == ["https://school.example/lesson"]source_filter._web_risk_cache.clear()+++def test_web_risk_batch_has_one_deadline(monkeypatch: pytest.MonkeyPatch) -> None:+ from deeptutor.services.search import source_filter++ monkeypatch.setattr(source_filter, "_WEB_RISK_BATCH_TIMEOUT_S", 0.05)++ def slow(_url: str, *, api_key: str) -> bool:+ time.sleep(0.2)+ return False++ started = time.monotonic()+ urls = [f"https://example.org/page-{i}" for i in range(12)]+ assert source_filter._web_risk_rejections(urls, api_key="test", request_web_risk=slow) == set()+ assert time.monotonic() - started < 0.15+ wait(list(source_filter._web_risk_inflight.values()), timeout=2)+++def test_concurrent_web_risk_calls_share_inflight_lookup() -> None:+ from deeptutor.services.search import source_filter++ started = threading.Event()+ release = threading.Event()+ calls = 0++ def lookup(_url: str, *, api_key: str) -> bool:+ nonlocal calls+ calls += 1+ started.set()+ assert release.wait(2)+ return True++ url = "https://example.org/shared-lookup"+ with ThreadPoolExecutor(max_workers=2) as executor:+ first = executor.submit(+ source_filter._web_risk_rejections, [url], api_key="test", request_web_risk=lookup+ )+ assert started.wait(2)+ second = executor.submit(+ source_filter._web_risk_rejections, [url], api_key="test", request_web_risk=lookup+ )+ release.set()+ assert first.result(timeout=2) == {url}+ assert second.result(timeout=2) == {url}+ assert calls == 1