RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
feat: adapt knowledge compilation concurrency (#19156)
buua436447f87d79f06
diff --git a/api/apps/services/dataset_api_service.py b/api/apps/services/dataset_api_service.pyindex ffc6270486..866324fb65 100644--- a/api/apps/services/dataset_api_service.py+++ b/api/apps/services/dataset_api_service.py@@ -2590,6 +2590,17 @@ async def _get_alteration(dataset_id: str, tenant_id: str, kind: str):result = _alteration_result(alteration_current_doc_ids, involved_doc_ids, eligible_doc_ids)if chunk_changes is not None:result.update(chunk_changes)+ if kind == "wiki":+ from rag.advanced_rag.knowlege_compile.wiki_incremental import _wiki_load_refine_failures++ refine_failures = await _wiki_load_refine_failures(kb.tenant_id, dataset_id)+ result.update(+ {+ "retry_required": bool(refine_failures),+ "retry_page_count": len(refine_failures),+ "retry_page_slugs": sorted(refine_failures),+ }+ )return True, resultdiff --git a/rag/advanced_rag/knowlege_compile/runner.py b/rag/advanced_rag/knowlege_compile/runner.pyindex 84aadf5458..692936aaf7 100644--- a/rag/advanced_rag/knowlege_compile/runner.py+++ b/rag/advanced_rag/knowlege_compile/runner.py@@ -197,6 +197,7 @@ async def _upsert_dataset_nav_from_page_index(doc_name: str,progress_cb: Callable[..., None],cancel_check: Callable[[], bool],+ llm_pool: LLMCallPool,) -> None:page_index_templates = [(template_id, parser_cfg) for template_id, parser_cfg in active_templates if _is_page_index_template(parser_cfg)]if not page_index_templates:@@ -255,13 +256,19 @@ async def _upsert_dataset_nav_from_page_index()progress_cb(msg=f"page_index: updating dataset navigation for doc {doc_id} ...")+ pooled_chat_mdl = llm_pool.wrap(+ chat_mdl,+ priority=20,+ label="dataset-nav:page-index",+ context=f"{doc_id}:dataset-nav",+ )await upsert_dataset_nav_doc(tenant_id,kb_id,doc_id,"\n\n".join(summaries),embd_mdl=embedding_model,- chat_mdl=chat_mdl,+ chat_mdl=pooled_chat_mdl,)except TaskCanceledException:raise@@ -286,6 +293,7 @@ async def run_structure_compile_over_batches(progress_cb: Callable[..., None],cancel_check: Callable[[], bool] = lambda: False,record: Callable[[str, dict], None] | None = None,+ llm_pool: LLMCallPool | None = None,) -> dict[str, dict]:"""Extract + merge structures for every non-``tree`` template over anasync stream of chunk batches, then run the optional synthesis phase.@@ -294,7 +302,8 @@ async def run_structure_compile_over_batches(chat model in ``chat_mdl_by_tid``. Chunks arrive as an async iterator ofbatches so callers can stream them from the doc store or hand over anin-memory list; each ``dict`` must expose ``id`` and text- (``content_with_weight`` / ``text``).+ (``content_with_weight`` / ``text``). ``llm_pool`` may be supplied by the+ task orchestrator so tree and non-tree phases share adaptive concurrency.Returns ``{template_id: {"inserted", "updated", "duplicates_dropped"}}``.Raises :class:`TaskCanceledException` when ``cancel_check`` trips.@@ -305,7 +314,7 @@ async def run_structure_compile_over_batches(return {}total = len(active_templates)- llm_pool = LLMCallPool(DOC_STRUCTURE_LLM_POOL_SIZE)+ llm_pool = llm_pool or LLMCallPool(DOC_STRUCTURE_LLM_POOL_SIZE)accumulators: dict[str, list[dict]] = {tid: [] for tid, _ in active_templates}template_kinds: dict[str, str] = {tid: _compilation_template_kind((cfg or {}).get("kind")) for tid, cfg in active_templates}@@ -613,6 +622,7 @@ async def run_structure_compile_over_batches(doc_name=doc_name,progress_cb=progress_cb,cancel_check=cancel_check,+ llm_pool=llm_pool,)# Timeline entity cleanup must happen after every flush has completed;# otherwise an entity can look isolated in one flush and be referenced bydiff --git a/rag/advanced_rag/knowlege_compile/structure.py b/rag/advanced_rag/knowlege_compile/structure.pyindex 663149dfee..e9ed583526 100644--- a/rag/advanced_rag/knowlege_compile/structure.py+++ b/rag/advanced_rag/knowlege_compile/structure.py@@ -19,7 +19,9 @@ import heapqimport jsonimport loggingimport re+import timeimport uuid+from dataclasses import dataclassfrom typing import Awaitable, Callable, Tupleimport xxhash@@ -28,7 +30,6 @@ from common.exceptions import TaskCanceledExceptionfrom common.misc_utils import thread_pool_execfrom common.token_utils import num_tokens_from_stringfrom rag.prompts.generator import gen_json-from ._common import (build_chunk_batches as _build_chunk_batches,encode as _encode,@@ -83,15 +84,67 @@ def _struct_merge_lock_key(kb_id: str, compilation_template_id: str | None) -> sreturn f"struct_merge:{kb_id}:{compilation_template_id or ''}"+@dataclass+class _LLMModelPoolState:+ concurrency: int+ active: int = 0+ successes: int = 0+ last_decrease_at: float = float("-inf")+ last_increase_at: float = float("-inf")++class LLMCallPool:- """Task-scoped priority scheduler for actual chat-model calls."""+ """Task-scoped adaptive priority scheduler for chat-model calls.++ ``max_concurrency`` remains the task-wide hard ceiling. Each model starts+ at that ceiling, halves its own admission limit after an explicit rate+ limit response, retries through the reduced limit, and recovers one slot+ at a time after sustained success.+ """- def __init__(self, max_concurrency: int = 10, max_pending: int | None = None):+ _RATE_LIMIT_MARKERS = (+ "429",+ "rate limit",+ "rate_limit",+ "too many requests",+ "requests per minute",+ "concurrency limit",+ "concurrent request",+ "maximum number of concurrent",+ )++ def __init__(+ self,+ max_concurrency: int = 10,+ max_pending: int | None = None,+ *,+ min_concurrency: int = 1,+ decrease_factor: float = 0.5,+ decrease_cooldown: float = 5.0,+ recovery_successes: int = 20,+ recovery_cooldown: float = 30.0,+ rate_limit_retries: int = 3,+ rate_limit_retry_base_delay: float = 1.0,+ rate_limit_retry_max_delay: float = 30.0,+ clock: Callable[[], float] = time.monotonic,+ on_concurrency_change: Callable[[int, int, str], None] | None = None,+ ):self.max_concurrency = max(1, int(max_concurrency))self.max_pending = max(self.max_concurrency, int(max_pending or self.max_concurrency))+ self.min_concurrency = min(self.max_concurrency, max(1, int(min_concurrency)))+ self.decrease_factor = min(1.0, max(0.01, float(decrease_factor)))+ self.decrease_cooldown = max(0.0, float(decrease_cooldown))+ self.recovery_successes = max(1, int(recovery_successes))+ self.recovery_cooldown = max(0.0, float(recovery_cooldown))+ self.rate_limit_retries = max(0, int(rate_limit_retries))+ self.rate_limit_retry_base_delay = max(0.0, float(rate_limit_retry_base_delay))+ self.rate_limit_retry_max_delay = max(self.rate_limit_retry_base_delay, float(rate_limit_retry_max_delay))+ self._clock = clock+ self._on_concurrency_change = on_concurrency_changeself._active = 0self._ticket = 0- self._waiting: list[tuple[int, int]] = []+ self._waiting: list[tuple[int, int, str]] = []+ self._model_states: dict[str, _LLMModelPoolState] = {}self._condition = asyncio.Condition()@property@@ -103,17 +156,125 @@ class LLMCallPool:return self._active + len(self._waiting)def wrap(self, chat_mdl, *, priority: int, label: str, context: str | None = None):- return PooledChatModel(self, chat_mdl, priority=priority, label=label, context=context)+ return PooledChatModel(+ self,+ chat_mdl,+ model_key=self._model_key(chat_mdl),+ priority=priority,+ label=label,+ context=context,+ )- async def call(self, fn, *, priority: int, label: str, context: str | None = None):+ def concurrency_for(self, chat_mdl) -> int:+ """Return the current adaptive concurrency limit for ``chat_mdl``."""+ return self._state_for(self._model_key(chat_mdl)).concurrency++ @staticmethod+ def _model_key(chat_mdl) -> str:+ config = getattr(chat_mdl, "model_config", None)+ if isinstance(config, dict):+ model_id = str(config.get("id") or config.get("llm_id") or config.get("model_id") or "").strip()+ factory = str(config.get("llm_factory") or "").strip()+ name = str(config.get("llm_name") or "").strip()+ endpoint = str(config.get("api_base") or config.get("base_url") or "").strip()+ if model_id or factory or name or endpoint:+ return ":".join((model_id, factory, name, endpoint))+ name = str(getattr(chat_mdl, "llm_name", "") or "").strip()+ if name:+ return name+ return f"{type(chat_mdl).__module__}.{type(chat_mdl).__qualname__}:{id(chat_mdl)}"++ def _state_for(self, model_key: str) -> _LLMModelPoolState:+ state = self._model_states.get(model_key)+ if state is None:+ state = _LLMModelPoolState(concurrency=self.max_concurrency)+ self._model_states[model_key] = state+ return state++ def _next_admissible_ticket(self) -> tuple[int, int, str] | None:+ if self._active >= self.max_concurrency:+ return None+ # The bounded queue is scanned to avoid one throttled model blocking+ # admissible work for another model at the heap head.+ candidates = [ticket for ticket in self._waiting if self._state_for(ticket[2]).active < self._state_for(ticket[2]).concurrency]+ return min(candidates) if candidates else None++ @classmethod+ def _is_rate_limited(cls, value) -> bool:+ text = str(value).lower()+ return any(marker in text for marker in cls._RATE_LIMIT_MARKERS)++ @staticmethod+ def _is_error_result(result) -> bool:+ return isinstance(result, str) and result.lstrip().lower().startswith("**error**")++ def _record_feedback(self, model_key: str, outcome: str, *, label: str, context: str | None) -> tuple[int, int, str] | None:+ state = self._state_for(model_key)+ now = self._clock()+ if outcome == "rate_limited":+ state.successes = 0+ if now - state.last_decrease_at < self.decrease_cooldown:+ return None+ old_concurrency = state.concurrency+ state.concurrency = max(self.min_concurrency, int(state.concurrency * self.decrease_factor))+ state.last_decrease_at = now+ if state.concurrency != old_concurrency:+ logging.warning(+ "LLM pool concurrency decreased model=%s label=%s context=%s old=%d new=%d active=%d pending=%d",+ model_key,+ label,+ context,+ old_concurrency,+ state.concurrency,+ state.active,+ len(self._waiting),+ )+ return old_concurrency, state.concurrency, "rate limited"+ return None+ if outcome != "success":+ state.successes = 0+ return None+ if state.concurrency >= self.max_concurrency:+ state.successes = 0+ return None+ state.successes += 1+ if state.successes < self.recovery_successes:+ return None+ if now - state.last_decrease_at < self.recovery_cooldown or now - state.last_increase_at < self.recovery_cooldown:+ return None+ old_concurrency = state.concurrency+ state.concurrency = min(self.max_concurrency, state.concurrency + 1)+ state.successes = 0+ state.last_increase_at = now+ logging.info(+ "LLM pool concurrency increased model=%s label=%s context=%s old=%d new=%d active=%d pending=%d",+ model_key,+ label,+ context,+ old_concurrency,+ state.concurrency,+ state.active,+ len(self._waiting),+ )+ return old_concurrency, state.concurrency, "recovered"++ def _notify_concurrency_change(self, old_concurrency: int, new_concurrency: int, reason: str) -> None:+ if self._on_concurrency_change is None:+ return+ try:+ self._on_concurrency_change(old_concurrency, new_concurrency, reason)+ except Exception:+ logging.exception("LLM pool concurrency change callback failed")++ async def _acquire(self, model_key: str, priority: int) -> None:async with self._condition:while self.pending_count >= self.max_pending:await self._condition.wait()- ticket = (int(priority), self._ticket)+ ticket = (int(priority), self._ticket, model_key)self._ticket += 1heapq.heappush(self._waiting, ticket)try:- while self._active >= self.max_concurrency or self._waiting[0] != ticket:+ while self._next_admissible_ticket() != ticket:await self._condition.wait()except BaseException:if ticket in self._waiting:@@ -121,23 +282,69 @@ class LLMCallPool:heapq.heapify(self._waiting)self._condition.notify_all()raise- heapq.heappop(self._waiting)+ self._waiting.remove(ticket)+ heapq.heapify(self._waiting)self._active += 1- try:- result = await fn()- return result- except BaseException:- raise- finally:- async with self._condition:- self._active -= 1- self._condition.notify_all()+ self._state_for(model_key).active += 1++ async def _release(self, model_key: str, outcome: str, *, label: str, context: str | None) -> None:+ async with self._condition:+ self._active -= 1+ self._state_for(model_key).active -= 1+ change = self._record_feedback(model_key, outcome, label=label, context=context)+ self._condition.notify_all()+ if change is not None:+ self._notify_concurrency_change(*change)++ def _rate_limit_retry_delay(self, retry: int) -> float:+ return min(self.rate_limit_retry_max_delay, self.rate_limit_retry_base_delay * (2 ** (retry - 1)))++ async def call(self, fn, *, model_key: str, priority: int, label: str, context: str | None = None):+ for retry in range(self.rate_limit_retries + 1):+ await self._acquire(model_key, priority)+ try:+ result = await fn()+ except asyncio.CancelledError:+ await self._release(model_key, "cancelled", label=label, context=context)+ raise+ except BaseException as exc:+ rate_limited = self._is_rate_limited(exc)+ await self._release(model_key, "rate_limited" if rate_limited else "failed", label=label, context=context)+ if not rate_limited or retry >= self.rate_limit_retries:+ raise+ else:+ error_result = self._is_error_result(result)+ rate_limited = error_result and self._is_rate_limited(result)+ await self._release(+ model_key,+ "rate_limited" if rate_limited else "failed" if error_result else "success",+ label=label,+ context=context,+ )+ if not rate_limited or retry >= self.rate_limit_retries:+ return result++ delay = self._rate_limit_retry_delay(retry + 1)+ logging.warning(+ "LLM pool retrying rate-limited call model=%s label=%s context=%s retry=%d/%d delay=%.2fs concurrency=%d",+ model_key,+ label,+ context,+ retry + 1,+ self.rate_limit_retries,+ delay,+ self._state_for(model_key).concurrency,+ )+ await asyncio.sleep(delay)++ raise AssertionError("LLM pool retry loop exhausted unexpectedly")class PooledChatModel:- def __init__(self, pool: LLMCallPool, chat_mdl, *, priority: int, label: str, context: str | None):+ def __init__(self, pool: LLMCallPool, chat_mdl, *, model_key: str, priority: int, label: str, context: str | None):self._pool = poolself._chat_mdl = chat_mdl+ self._model_key = model_keyself._priority = priorityself._label = labelself._context = context@@ -149,6 +356,7 @@ class PooledChatModel:gen_conf = _knowledge_compile_gen_conf(self._chat_mdl, gen_conf)return await self._pool.call(lambda: self._chat_mdl.async_chat(system, history, gen_conf=gen_conf, **kwargs),+ model_key=self._model_key,priority=self._priority,label=self._label,context=self._context,diff --git a/rag/advanced_rag/knowlege_compile/wiki.py b/rag/advanced_rag/knowlege_compile/wiki.pyindex 3a71d98fbc..cf7b9c182b 100644--- a/rag/advanced_rag/knowlege_compile/wiki.py+++ b/rag/advanced_rag/knowlege_compile/wiki.py@@ -3913,6 +3913,8 @@ async def wiki_refine_from_plan(semaphore = asyncio.Semaphore(max_workers) if max_workers and max_workers > 0 else Nonecompleted = 0completed_lock = asyncio.Lock()+ progress_updates = 20+ report_every = max(1, (total + progress_updates - 1) // progress_updates)async def _write_one(plan_item: dict) -> Optional[dict]:nonlocal completed@@ -3922,7 +3924,6 @@ async def wiki_refine_from_plan(page_type = plan_item.get("page_type") or "concept"async def _run() -> Optional[dict]:- nonlocal completedtry:evidence = _wiki_assemble_evidence(plan_item,@@ -4006,7 +4007,7 @@ async def wiki_refine_from_plan(}except Exception:logging.exception("wiki_refine: writer failed for slug=%s", slug)- return None+ raise# Searchable wiki_page persistence has moved to the task# handler so the doc-storage schema can be controlled in one@@ -4023,21 +4024,27 @@ async def wiki_refine_from_plan(except Exception:logging.exception("wiki_refine: persist_draft failed for slug=%s", slug)- if callback:+ return page++ result: Optional[dict] = None+ try:+ if semaphore is not None:+ async with semaphore:+ result = await _run()+ else:+ result = await _run()+ return result+ finally:+ if callback and result is not None:async with completed_lock:completed += 1done = completed- progress = 0.1 + 0.85 * (done / total)- try:- callback(progress, f"wiki REFINE: {done}/{total} pages written ({slug})")- except Exception:- pass- return page-- if semaphore is not None:- async with semaphore:- return await _run()- return await _run()+ if done % report_every == 0 or done == total:+ progress = 0.1 + 0.85 * (done / total)+ try:+ callback(progress, f"wiki REFINE: {done}/{total} pages completed")+ except Exception:+ logging.exception("wiki REFINE progress callback failed")tasks = [asyncio.create_task(_write_one(p)) for p in pending]if tasks:diff --git a/rag/advanced_rag/knowlege_compile/wiki_incremental.py b/rag/advanced_rag/knowlege_compile/wiki_incremental.pyindex d5c9b4f4d7..02392f68b6 100644--- a/rag/advanced_rag/knowlege_compile/wiki_incremental.py+++ b/rag/advanced_rag/knowlege_compile/wiki_incremental.py@@ -33,9 +33,29 @@ from ._common import ()-# ----- REFINE concurrency control -----+# ----- REFINE progress ------WIKI_REFINE_MAX_CONCURRENT = 20 # shared LLM pool size used by the Wiki runner+WIKI_REFINE_PROGRESS_UPDATES = 20+++async def _wiki_run_refine_tasks(tasks: list, progress: Callable[[str], None]) -> None:+ """Run REFINE page tasks and emit at most roughly 20 progress updates."""+ total = len(tasks)+ if not total:+ return+ report_every = max(1, (total + WIKI_REFINE_PROGRESS_UPDATES - 1) // WIKI_REFINE_PROGRESS_UPDATES)+ scheduled = [asyncio.create_task(task) for task in tasks]+ try:+ for done, task in enumerate(asyncio.as_completed(scheduled), start=1):+ await task+ if done % report_every == 0 or done == total:+ progress(f"{done}/{total} pages completed.")+ except BaseException:+ for task in scheduled:+ if not task.done():+ task.cancel()+ await asyncio.gather(*scheduled, return_exceptions=True)+ raise# ----- constants ----@@ -45,6 +65,7 @@ WIKI_PAGE_COMPILE_KWD = "wiki_page"WIKI_PLAN_GROUP_COMPILE_KWD = "wiki_plan_group"WIKI_DOC_PAGE_SOURCE_COMPILE_KWD = "wiki_doc_page_source"WIKI_CANONICAL_ENTITY_COMPILE_KWD = "wiki_canonical_entity"+WIKI_REFINE_FAILURE_COMPILE_KWD = "wiki_refine_failure"# Entity matching thresholds (not exposed in YAML)ENTITY_MERGE_THRESHOLD = 0.90 # auto-merge@@ -303,6 +324,103 @@ async def _wiki_has_any_pages(tenant_id: str, kb_id: str) -> bool:return False+async def _wiki_load_refine_failures(tenant_id: str, kb_id: str) -> dict[str, dict]:+ """Load persisted REFINE failures keyed by page id."""+ index = search.index_name(tenant_id)+ if not settings.docStoreConn.index_exist(index, kb_id):+ return {}+ fields = ["slug_kwd", "content_with_weight"]+ try:+ res = await thread_pool_exec(+ settings.docStoreConn.search,+ fields,+ [],+ {"compile_kwd": [WIKI_REFINE_FAILURE_COMPILE_KWD]},+ [],+ OrderByExpr(),+ 0,+ 1000,+ index,+ [kb_id],+ )+ rows = settings.docStoreConn.get_fields(res, fields) or {}+ except Exception:+ logging.exception("wiki: failed to load REFINE failures for kb=%s", kb_id)+ return {}++ failures: dict[str, dict] = {}+ for row in rows.values():+ page_id = row.get("slug_kwd")+ payload = row.get("content_with_weight")+ if isinstance(page_id, list):+ page_id = page_id[0] if page_id else ""+ if not isinstance(page_id, str) or not page_id.strip():+ continue+ if isinstance(payload, str):+ try:+ payload = json.loads(payload) if payload else {}+ except (json.JSONDecodeError, TypeError):+ payload = {}+ failures[page_id.strip()] = payload if isinstance(payload, dict) else {}+ return failures+++async def _wiki_record_refine_failure(+ tenant_id: str,+ kb_id: str,+ page_id: str,+ entity_names: list[str] | None = None,+ error: str = "",+) -> None:+ """Persist one failed page so a later Update can retry it."""+ page_id = str(page_id or "").strip()+ if not page_id:+ return+ index = search.index_name(tenant_id)+ payload = json.dumps(+ {+ "page_id": page_id,+ "entity_names": [str(name).strip() for name in entity_names or [] if str(name).strip()],+ "error": str(error or "")[:1000],+ },+ ensure_ascii=False,+ )+ row = {+ "id": _stable_row_id(WIKI_REFINE_FAILURE_COMPILE_KWD, kb_id, page_id),+ "doc_id": str(kb_id),+ "compile_kwd": WIKI_REFINE_FAILURE_COMPILE_KWD,+ "slug_kwd": page_id,+ "content_with_weight": payload,+ "available_int": 0,+ }+ try:+ await thread_pool_exec(+ settings.docStoreConn.delete,+ {"compile_kwd": [WIKI_REFINE_FAILURE_COMPILE_KWD], "slug_kwd": [page_id]},+ index,+ kb_id,+ )+ await thread_pool_exec(settings.docStoreConn.insert, [row], index, kb_id)+ except Exception:+ logging.exception("wiki: failed to persist REFINE failure page=%s", page_id)+++async def _wiki_clear_refine_failure(tenant_id: str, kb_id: str, page_id: str) -> None:+ """Remove the retry marker after a page succeeds or is deleted."""+ page_id = str(page_id or "").strip()+ if not page_id:+ return+ try:+ await thread_pool_exec(+ settings.docStoreConn.delete,+ {"compile_kwd": [WIKI_REFINE_FAILURE_COMPILE_KWD], "slug_kwd": [page_id]},+ search.index_name(tenant_id),+ kb_id,+ )+ except Exception:+ logging.exception("wiki: failed to clear REFINE failure page=%s", page_id)++async def _load_canonical_entities(tenant_id: str,kb_id: str,@@ -1191,16 +1309,26 @@ def _wiki_extract_outlinks_from_content(content: str, kb_id: str = "") -> list[sreturn outlinks-def _inside_wikilink(content: str, pos: int) -> bool:- """Return True iff position ``pos`` falls inside an existing [[...]] span."""- open_pos = content.rfind("[[", 0, pos)- if open_pos < 0:- return False- # Find the FIRST "]]" that closes the "[[...]]" starting at open_pos- close_pos = content.find("]]", open_pos + 2)- if close_pos < 0:- close_pos = len(content)- return open_pos + 2 <= pos < close_pos+_WIKI_PROTECTED_LINK_RE = re.compile(r"\[\[[^\]\n]+\]\]|\[[^\]\n]*\]\([^)\n]+\)")+++def _wiki_find_unlinked_mention(content: str, name: str) -> int:+ """Find the first occurrence of ``name`` outside existing link markup."""+ if not content or not name:+ return -1+ protected_spans = [(match.start(), match.end()) for match in _WIKI_PROTECTED_LINK_RE.finditer(content)]+ raw_open = content.rfind("[[")+ if raw_open >= 0 and content.find("]]", raw_open + 2) < 0:+ protected_spans.append((raw_open, len(content)))+ start = 0+ while True:+ idx = content.find(name, start)+ if idx < 0:+ return -1+ end = idx + len(name)+ if not any(idx < protected_end and end > protected_start for protected_start, protected_end in protected_spans):+ return idx+ start = idx + 1_WIKI_PIPE_LINK_RE = re.compile(r"\[\[([^\[\]\|]+?)\|([^\[\]]+?)\]\]")@@ -3225,12 +3353,9 @@ async def _wiki_finalize(continueif target in existing_links:continue- idx = content.find(name)+ idx = _wiki_find_unlinked_mention(content, name)if idx < 0:continue- # Skip if the occurrence is already inside a [[...]] link span- if _inside_wikilink(content, idx):- continue# Preserve the matched prose as the link label. This matters when# several entities share one page: e.g. the page slug may be# ``entity/五色棒`` while the text mentions@@ -3703,6 +3828,15 @@ async def wiki_compile_incremental(if page_id in existing_pages and names:existing_pages[page_id]["entity_names_kwd"] = names+ # A failed REFINE page must be reconsidered together with the current+ # document delta. The current REDUCE snapshot remains authoritative, so+ # deleted entities can still produce a deletion rather than being blindly+ # retried from stale failure metadata.+ refine_failures = await _wiki_load_refine_failures(tenant_id, kb_id)+ retry_names = {str(name).strip() for failure in refine_failures.values() for name in failure.get("entity_names", []) if str(name).strip()}+ if incremental:+ affected_names.update(retry_names)+# Build canonical claims ON-DEMAND only for affected names, then release# the full claim_index. claim_index is keyed by RAW entity name (from MAP),# so claims must be aggregated through name_resolution onto the canonical@@ -3973,6 +4107,7 @@ async def _wiki_mode_a_run(page_version=existing.get("page_version_int", 0) if existing else 0,)summary["pages_deleted"] += 1+ await _wiki_clear_refine_failure(tenant_id, kb_id, pid)returnnext_version = _as_int(existing.get("page_version_int")) + 1 if existing else 1@@ -4008,6 +4143,9 @@ async def _wiki_mode_a_run(topic_pool=topic_pool,topic_pool_lock=topic_pool_lock,)+ if result is None:+ raise RuntimeError(f"REFINE returned no page content for {pid}")+ await _wiki_clear_refine_failure(tenant_id, kb_id, pid)if refine_mode == "generate":summary["pages_created"] += 1else:@@ -4017,14 +4155,21 @@ async def _wiki_mode_a_run(for did in entry["source_doc_ids"]:doc_updates.setdefault(did, []).append(pid)- except Exception:+ except Exception as exc:logging.exception("wiki A: REFINE failed for %s", pid)+ await _wiki_record_refine_failure(+ tenant_id,+ kb_id,+ pid,+ entry.get("existing_page", {}).get("entity_names_kwd") or [entry.get("page_title", pid)],+ error=str(exc),+ )summary["errors"].append(f"REFINE_FAILED:{pid}")tasks = [_refine_one(pid, entry) for pid, entry in page_deltas.items()]if tasks:- _progress(f"REFINE A: {len(tasks)} pages (LLM pool max {WIKI_REFINE_MAX_CONCURRENT}) ...")- await asyncio.gather(*tasks)+ _progress(f"{len(tasks)} pages ...")+ await _wiki_run_refine_tasks(tasks, _progress)_wiki_log_stats("TOPIC", "selection_summary", mode="A", **topic_selection_stats)for did, pids in doc_updates.items():@@ -4300,6 +4445,7 @@ async def _wiki_mode_b_run(page_version=existing.get("page_version_int", 0) if existing else 0,)if deleted_page is None:+ await _wiki_clear_refine_failure(tenant_id, kb_id, page_key)await _wiki_delete_plan_group(tenant_id, kb_id, page_key)for did in _as_str_list(existing.get("source_doc_ids")) if existing else []:doc_removals.setdefault(did, []).append(page_key)@@ -4341,7 +4487,10 @@ async def _wiki_mode_b_run(topic_pool_lock=topic_pool_lock,member_evidence=member_evidence,)+ if result is None:+ raise RuntimeError(f"REFINE returned no page content for {page_key}")if result:+ await _wiki_clear_refine_failure(tenant_id, kb_id, page_key)if is_new:summary["pages_created"] += 1else:@@ -4367,14 +4516,22 @@ async def _wiki_mode_b_run(for did in old_doc_ids - new_doc_ids:doc_removals.setdefault(did, []).append(page_key)- except Exception:+ except Exception as exc:logging.exception("wiki B: REFINE failed for %s", page_id)+ failed_page_id = page_id[5:] if page_id.startswith("_new_") else page_id+ await _wiki_record_refine_failure(+ tenant_id,+ kb_id,+ failed_page_id,+ [ent.get("entity_name", "") for ent in entities if isinstance(ent, dict)],+ error=str(exc),+ )summary["errors"].append(f"REFINE_FAILED:{page_id}")tasks = [_refine_one(pid, ents) for pid, ents in assignments.items()]if tasks:- _progress(f"REFINE B: {len(tasks)} pages (LLM pool max {WIKI_REFINE_MAX_CONCURRENT}) ...")- await asyncio.gather(*tasks)+ _progress(f"{len(tasks)} pages ...")+ await _wiki_run_refine_tasks(tasks, _progress)_wiki_log_stats("TOPIC", "selection_summary", mode="B", **topic_selection_stats)# Apply doc_page_source updates serially (no race), preserving metadatadiff --git a/rag/flow/compiler/compiler.py b/rag/flow/compiler/compiler.pyindex 6ced075951..d1fda66979 100644--- a/rag/flow/compiler/compiler.py+++ b/rag/flow/compiler/compiler.py@@ -30,6 +30,7 @@ from common.constants import LLMTypefrom common.token_utils import num_tokens_from_stringfrom rag.advanced_rag.knowlege_compile.runner import (DOC_STRUCTURE_COMPILE_BATCH_CHUNKS,+ DOC_STRUCTURE_LLM_POOL_SIZE,load_active_templates,resolve_template_ids_from_groups,run_structure_compile_over_batches,@@ -371,6 +372,7 @@ class Compiler(ProcessBase, LLM):kb_id: str,doc_id: str,doc_name: str,+ llm_pool,) -> None:"""Build and persist tree graphs from the pipeline's in-memory chunks.@@ -418,6 +420,12 @@ class Compiler(ProcessBase, LLM):raptor_service = RaptorService(tree_context)for idx, (template_id, parser_cfg) in enumerate(templates):+ pooled_chat_mdl = llm_pool.wrap(+ chat_mdl_by_tid[template_id],+ priority=30,+ label=f"tree:{template_id}",+ context=f"{doc_id}:{template_id}:tree",+ )raptor_cfg = (parser_cfg or {}).get("raptor") or {}raptor_config = {"prompt": raptor_cfg.get("prompt")@@ -432,7 +440,7 @@ class Compiler(ProcessBase, LLM):tree = await raptor_service.build_doc_tree(chunks=tree_chunks,raptor_config=raptor_config,- chat_mdl=chat_mdl_by_tid[template_id],+ chat_mdl=pooled_chat_mdl,embd_mdl=embedding_model,max_errors=3,)@@ -445,7 +453,7 @@ class Compiler(ProcessBase, LLM):if bool(raptor_cfg.get("rechunk")):self._compile_progress(msg="Compiler: tree rechunking is not supported for in-memory pipeline chunks; keeping original chunks.")- await rewrite_duplicate_tree_names(tree, chat_mdl_by_tid[template_id])+ await rewrite_duplicate_tree_names(tree, pooled_chat_mdl)after_graph = raptor_tree_to_graph(tree)try:await _struct_upsert_tree_graph_rows(@@ -489,7 +497,7 @@ class Compiler(ProcessBase, LLM):doc_id,{"title": tree.get("title"), "graph_text": nav_graph_text},embd_mdl=embedding_model,- chat_mdl=chat_mdl_by_tid[template_id],+ chat_mdl=pooled_chat_mdl,)except Exception:logging.exception("Compiler: tree-template %s dataset navigation upsert failed for doc %s", template_id, doc_id)@@ -615,6 +623,9 @@ class Compiler(ProcessBase, LLM):max_retries=self._param.max_retries,retry_interval=self._param.delay_after_error,)+ from rag.advanced_rag.knowlege_compile.structure import LLMCallPool++ llm_pool = LLMCallPool(DOC_STRUCTURE_LLM_POOL_SIZE)tree_templates, non_tree_templates = split_tree_templates(active_templates)if tree_templates:@@ -627,6 +638,7 @@ class Compiler(ProcessBase, LLM):kb_id,doc_id,doc_name,+ llm_pool,)if non_tree_templates:@@ -663,6 +675,7 @@ class Compiler(ProcessBase, LLM):chunk_batches=_chunk_batches(),progress_cb=self._compile_progress,cancel_check=_cancelled,+ llm_pool=llm_pool,)if first_rechunk_index is None:diff --git a/rag/svr/task_executor_refactor/chunk_post_processor.py b/rag/svr/task_executor_refactor/chunk_post_processor.pyindex d92eb0d13f..1dc17eba60 100644--- a/rag/svr/task_executor_refactor/chunk_post_processor.py+++ b/rag/svr/task_executor_refactor/chunk_post_processor.py@@ -412,6 +412,7 @@ from common.token_utils import num_tokens_from_string# component can share them. Re-exported here for backwards compatibility.from rag.advanced_rag.knowlege_compile.runner import (DOC_STRUCTURE_COMPILE_BATCH_CHUNKS,+ DOC_STRUCTURE_LLM_POOL_SIZE,DOC_STRUCTURE_MERGE_MAX_DOCS, # noqa: F401STRUCTURE_CHAIN_CORRECTION_TIMEOUT_S, # noqa: F401load_active_templates,@@ -928,6 +929,7 @@ async def run_tree_templates(chat_mdl_by_tid: dict[str, "LLMBundle"],embedding_model,doc_name: str,+ llm_pool,) -> None:"""Run the ``tree``-kind compilation templates for the currentdoc. Each pair runs RAPTOR with ``is_tree=True`` via@@ -961,6 +963,12 @@ async def run_tree_templates(raptor_service = RaptorService(ctx)for idx, (template_id, parser_cfg) in enumerate(templates):+ pooled_chat_mdl = llm_pool.wrap(+ chat_mdl_by_tid[template_id],+ priority=30,+ label=f"tree:{template_id}",+ context=f"{doc_id}:{template_id}:tree",+ )raptor_cfg = (parser_cfg or {}).get("raptor") or {}raptor_config = {"prompt": raptor_cfg.get("prompt") or "Please write a concise summary of the following texts:\n{cluster_content}",@@ -977,7 +985,7 @@ async def run_tree_templates(tree = await raptor_service.build_doc_tree(chunks=chunks,raptor_config=raptor_config,- chat_mdl=chat_mdl_by_tid[template_id],+ chat_mdl=pooled_chat_mdl,embd_mdl=embedding_model,max_errors=3,)@@ -1011,7 +1019,7 @@ async def run_tree_templates(doc_id,)- await rewrite_duplicate_tree_names(tree, chat_mdl_by_tid[template_id])+ await rewrite_duplicate_tree_names(tree, pooled_chat_mdl)graph = raptor_tree_to_graph(tree)try:await _struct_upsert_graph_json(@@ -1053,7 +1061,7 @@ async def run_tree_templates(doc_id,{"title": tree.get("title"), "graph_text": nav_graph_text},embd_mdl=embedding_model,- chat_mdl=chat_mdl_by_tid[template_id],+ chat_mdl=pooled_chat_mdl,)except Exception:logging.exception(@@ -1100,6 +1108,9 @@ async def run_document_structure_compile(handler, embedding_model: LLMBundle) ->logging.exception("document_structure_compile: cannot resolve ingestion chat model %s", chat_llm_id)returnchat_mdl_by_tid = {template_id: chat_mdl for template_id, _ in active_templates}+ from rag.advanced_rag.knowlege_compile.structure import LLMCallPool++ llm_pool = LLMCallPool(DOC_STRUCTURE_LLM_POOL_SIZE)tree_templates: list[tuple[str, dict]] = []non_tree_templates: list[tuple[str, dict]] = []@@ -1116,6 +1127,7 @@ async def run_document_structure_compile(handler, embedding_model: LLMBundle) ->chat_mdl_by_tid,embedding_model,doc_name,+ llm_pool,)if not non_tree_templates:@@ -1143,6 +1155,7 @@ async def run_document_structure_compile(handler, embedding_model: LLMBundle) ->progress_cb=ctx.progress_cb,cancel_check=lambda: ctx.has_canceled_func(ctx.id),record=ctx.recording_context.record,+ llm_pool=llm_pool,)diff --git a/rag/svr/task_executor_refactor/dataset_wiki_generator.py b/rag/svr/task_executor_refactor/dataset_wiki_generator.pyindex f96632b147..1330547998 100644--- a/rag/svr/task_executor_refactor/dataset_wiki_generator.py+++ b/rag/svr/task_executor_refactor/dataset_wiki_generator.py@@ -80,9 +80,9 @@ from rag.svr.task_executor_refactor.task_context import TaskContext# room for the function's internal split_chunks packing to do real work.WIKI_MAP_BATCH_CHUNKS = 64-# The pool limits actual MAP LLM calls rather than the surrounding batch-# tasks. This lets the next waiting batch start as soon as a model call-# finishes, without waiting for the previous batch's ES persistence work.+# The pool limits actual LLM calls rather than the surrounding batch tasks and+# adapts each model's admission limit when it reports rate limiting. This lets+# high-capacity and low-concurrency models share one Wiki task safely.WIKI_MAP_LLM_POOL_SIZE = 20# Global MAP admission limit: active calls plus calls waiting in the pool.@@ -96,6 +96,21 @@ WIKI_MAP_QUEUE_SIZE = 5# pool; the pool itself still caps actual LLM requests at 20.WIKI_REFINE_WORKERS = WIKI_MAP_LLM_POOL_SIZE++def _create_wiki_llm_pool(progress: Callable) -> LLMCallPool:+ def _on_concurrency_change(old: int, new: int, reason: str) -> None:+ progress(msg=f"LLM pool concurrency {old} -> {new} ({reason}).")++ pool = LLMCallPool(+ WIKI_MAP_LLM_POOL_SIZE,+ max_pending=WIKI_MAP_MAX_PENDING,+ on_concurrency_change=_on_concurrency_change,+ )+ logging.info("Wiki LLM pool initialized max_concurrency=%d max_pending=%d", pool.max_concurrency, pool.max_pending)+ progress(0.0, f"LLM pool max {pool.max_concurrency}.")+ return pool++# Per-node cap on ``source_chunk_ids`` carried by the canvas graph blob.# Pages can accumulate hundreds of source chunks; the graph response is# meant for fast canvas rendering, not full provenance audit, so we trim@@ -1300,6 +1315,7 @@ async def run_wiki(from api.db.joint_services.tenant_model_service import resolve_model_configprogress = ctx.progress_cb+ map_llm_pool = _create_wiki_llm_pool(progress)progress(0.0, "Loading documents for wiki compilation...")# 1. Resolve KB metadata for PLAN.@@ -1415,7 +1431,6 @@ async def run_wiki(first_example = first_parser_cfg.get("example")kb_writer_instruction: Optional[str] = first_instruction if isinstance(first_instruction, str) and first_instruction.strip() else Nonekb_writer_example: Optional[str] = first_example if isinstance(first_example, str) and first_example.strip() else None- map_llm_pool = LLMCallPool(WIKI_MAP_LLM_POOL_SIZE, max_pending=WIKI_MAP_MAX_PENDING)n_docs = len(resolved_eligible)map_queue: asyncio.Queue = asyncio.Queue(maxsize=WIKI_MAP_QUEUE_SIZE)doc_stats = {@@ -1634,9 +1649,9 @@ async def run_wiki_incremental(from rag.advanced_rag.knowlege_compile.wiki_incremental import (wiki_compile_incremental,)- from rag.advanced_rag.knowlege_compile.structure import LLMCallPoolprogress = ctx.progress_cb+ map_llm_pool = _create_wiki_llm_pool(progress)progress(0.0, "Loading documents for wiki compilation...")# 1. Check if this is incremental (existing MAP rows present)@@ -1765,7 +1780,6 @@ async def run_wiki_incremental(llm_bundle_cache[key] = bundlereturn bundle- map_llm_pool = LLMCallPool(WIKI_MAP_LLM_POOL_SIZE, max_pending=WIKI_MAP_MAX_PENDING)kb_chat_llm_id = Nonefirst_template_found = False@@ -1875,7 +1889,10 @@ async def run_wiki_incremental(# prior run stopped after MAP), fall through so the compiler can rebuild# pages from the stored extracts.has_compiled_pages = await _wiki_has_compiled_pages(ctx.tenant_id, ctx.kb_id) if existing_map_doc_ids else None- if existing_map_doc_ids and has_compiled_pages is True:+ from rag.advanced_rag.knowlege_compile.wiki_incremental import _wiki_load_refine_failures++ has_refine_failures = bool(await _wiki_load_refine_failures(ctx.tenant_id, ctx.kb_id))+ if existing_map_doc_ids and has_compiled_pages is True and not has_refine_failures:from rag.advanced_rag.knowlege_compile.wiki_incremental import (_wiki_finalize,_wiki_load_pages_for_graph,diff --git a/test/unit_test/rag/advanced_rag/knowlege_compile/test_wiki_incremental.py b/test/unit_test/rag/advanced_rag/knowlege_compile/test_wiki_incremental.pyindex 8d1cbf9495..8d81d0b89f 100644--- a/test/unit_test/rag/advanced_rag/knowlege_compile/test_wiki_incremental.py+++ b/test/unit_test/rag/advanced_rag/knowlege_compile/test_wiki_incremental.py@@ -79,6 +79,32 @@ class MockChatModel:pass+async def test_load_refine_failures_decodes_persisted_page_markers(monkeypatch):+ store = MagicMock()+ store.index_exist.return_value = True+ store.get_fields.return_value = {+ "row-1": {+ "slug_kwd": ["entity/alice"],+ "content_with_weight": json.dumps({"entity_names": ["Alice"], "error": "429"}),+ },+ "row-2": {"slug_kwd": "entity/bob", "content_with_weight": "{}"},+ }++ async def fake_thread_pool_exec(*args, **kwargs):+ return object()++ monkeypatch.setattr(_wiki.settings, "docStoreConn", store)+ monkeypatch.setattr(_wiki, "thread_pool_exec", fake_thread_pool_exec)++ failures = await _wiki._wiki_load_refine_failures("tenant", "kb")++ assert failures == {+ "entity/alice": {"entity_names": ["Alice"], "error": "429"},+ "entity/bob": {},+ }++@pytest.mark.asyncio@pytest.mark.parametrize("response",@@ -1143,6 +1169,60 @@ async def test_finalize_auto_links_mentions():assert google_upd["outlinks_int"] == 1+async def test_finalize_auto_link_prefers_longest_name_without_nested_links():+ search_results = [+ {+ "_source": {+ "slug_kwd": "entity/虎牢关",+ "title_kwd": "虎牢关",+ "md_with_weight": "虎牢关是关隘。",+ "page_type_kwd": "entity",+ }+ },+ {+ "_source": {+ "slug_kwd": "entity/关",+ "title_kwd": "关",+ "md_with_weight": "关是称谓。",+ "page_type_kwd": "entity",+ }+ },+ {+ "_source": {+ "slug_kwd": "concept/虎牢关之战",+ "title_kwd": "虎牢关之战",+ "md_with_weight": "虎牢关之战发生在关前。",+ "page_type_kwd": "concept",+ }+ },+ ]+ doc_store = make_doc_store(search_results)++ with (+ patch("common.settings.docStoreConn", doc_store),+ patch(f"{_wiki.__name__}._load_canonical_entities", new_callable=AsyncMock, return_value={}),+ ):+ await _wiki._wiki_finalize(tenant_id="t1", kb_id="kb1", embd_mdl=None)++ update = next(args[0][1] for args in doc_store.update.call_args_list if args[0][0].get("id") == "concept/虎牢关之战")+ body = update["md_with_weight"]+ assert "[虎牢关](artifact/kb1/entity/虎牢关)" in body+ assert "[关](artifact/kb1/entity/关)前" in body+ assert "[虎牢[" not in body+++def test_find_unlinked_mention_skips_raw_and_rendered_links():+ content = "[[entity/虎牢关|虎牢关]]与[虎牢关](artifact/kb1/entity/虎牢关),关前"+ assert _wiki._wiki_find_unlinked_mention(content, "虎牢关") == -1+ assert _wiki._wiki_find_unlinked_mention(content, "关") == content.index("关前")+++def test_find_unlinked_mention_skips_unterminated_raw_link():+ content = "before [[entity/X|Name"+ assert _wiki._wiki_find_unlinked_mention(content, "Name") == -1++@pytest.mark.asyncioasync def test_finalize_preserves_merged_entity_name_as_link_label():"""A merged page must not replace a member mention with its page title."""@@ -1186,13 +1266,6 @@ async def test_finalize_preserves_merged_entity_name_as_link_label():assert "[治世之能臣,乱世之奸雄](artifact/kb1/entity/五色棒)" in update["md_with_weight"]-def test_inside_wikilink():- content = "before [[entity/X]] after"- assert _wiki._inside_wikilink(content, content.index("entity"))- assert not _wiki._inside_wikilink(content, content.index("before"))- assert not _wiki._inside_wikilink(content, content.index("after"))--def test_extract_outlinks_from_content():"""_wiki_extract_outlinks_from_content derives unique ordered outlinks."""content = "See [[concept/B]] and [[entity/apple]] and [[concept/B]] again"diff --git a/test/unit_test/rag/advanced_rag/test_llm_call_pool.py b/test/unit_test/rag/advanced_rag/test_llm_call_pool.pynew file mode 100644index 0000000000..cfd327c510--- /dev/null+++ b/test/unit_test/rag/advanced_rag/test_llm_call_pool.py@@ -0,0 +1,277 @@+import asyncio++import pytest++from rag.advanced_rag.knowlege_compile.structure import LLMCallPool+++class _Clock:+ def __init__(self):+ self.now = 0.0++ def __call__(self):+ return self.now++ def advance(self, seconds: float):+ self.now += seconds+++class _ChatModel:+ def __init__(self, name: str, response: str = "ok"):+ self.llm_name = name+ self.model_config = {"llm_factory": "test", "llm_name": name}+ self.response = response++ async def async_chat(self, system, history, gen_conf=None, **kwargs):+ return self.response+++async def _return(value):+ return value+++def test_llm_call_pool_halves_model_concurrency_once_per_cooldown():+ async def exercise():+ clock = _Clock()+ pool = LLMCallPool(20, decrease_cooldown=5, rate_limit_retries=0, clock=clock)+ model = _ChatModel("limited")++ async def limited_call():+ return "**ERROR**: RATE_LIMIT_EXCEEDED - 429 too many requests"++ await pool.call(limited_call, model_key=pool._model_key(model), priority=10, label="limited")+ assert pool.concurrency_for(model) == 10++ await pool.call(limited_call, model_key=pool._model_key(model), priority=10, label="limited")+ assert pool.concurrency_for(model) == 10++ clock.advance(5)+ await pool.call(limited_call, model_key=pool._model_key(model), priority=10, label="limited")+ assert pool.concurrency_for(model) == 5++ asyncio.run(exercise())+++def test_pooled_chat_model_reports_rate_limit_results_to_the_pool():+ async def exercise():+ pool = LLMCallPool(8, decrease_cooldown=0, rate_limit_retries=0)+ model = _ChatModel("wrapped", "**ERROR**: RATE_LIMIT_EXCEEDED")++ result = await pool.wrap(model, priority=10, label="wrapped").async_chat("system", [])++ assert result == "**ERROR**: RATE_LIMIT_EXCEEDED"+ assert pool.concurrency_for(model) == 4++ asyncio.run(exercise())+++def test_llm_call_pool_retries_after_lowering_concurrency():+ class RetryingChatModel(_ChatModel):+ def __init__(self, name: str):+ super().__init__(name)+ self.calls = 0++ async def async_chat(self, system, history, gen_conf=None, **kwargs):+ self.calls += 1+ if self.calls == 1:+ return "**ERROR**: 429 too many requests"+ return "ok after provider retry"++ async def exercise():+ changes = []+ pool = LLMCallPool(+ 8,+ decrease_cooldown=0,+ rate_limit_retry_base_delay=0,+ on_concurrency_change=lambda old, new, reason: changes.append((old, new, reason)),+ )+ model = RetryingChatModel("internally-retried")++ result = await pool.wrap(model, priority=10, label="wrapped").async_chat("system", [])++ assert result == "ok after provider retry"+ assert model.calls == 2+ assert pool.concurrency_for(model) == 4+ assert changes == [(8, 4, "rate limited")]++ asyncio.run(exercise())+++def test_llm_call_pool_retries_rate_limit_exceptions():+ async def exercise():+ calls = 0+ model = _ChatModel("exception-retry")+ pool = LLMCallPool(8, decrease_cooldown=0, rate_limit_retry_base_delay=0)++ async def rate_limited_once():+ nonlocal calls+ calls += 1+ if calls == 1:+ raise RuntimeError("429 too many requests")+ return "ok"++ result = await pool.call(+ rate_limited_once,+ model_key=pool._model_key(model),+ priority=10,+ label="exception-retry",+ )++ assert result == "ok"+ assert calls == 2+ assert pool.concurrency_for(model) == 4++ asyncio.run(exercise())+++def test_llm_call_pool_recovers_concurrency_gradually_after_success():+ async def exercise():+ clock = _Clock()+ pool = LLMCallPool(+ 8,+ decrease_cooldown=0,+ recovery_successes=2,+ recovery_cooldown=10,+ rate_limit_retries=0,+ clock=clock,+ )+ model = _ChatModel("recovering")+ model_key = pool._model_key(model)++ await pool.call(+ lambda: _return("**ERROR**: RATE_LIMIT_EXCEEDED"),+ model_key=model_key,+ priority=10,+ label="limited",+ )+ assert pool.concurrency_for(model) == 4++ clock.advance(10)+ for _ in range(2):+ await pool.call(lambda: _return("ok"), model_key=model_key, priority=10, label="success")+ assert pool.concurrency_for(model) == 5++ for _ in range(2):+ await pool.call(lambda: _return("ok"), model_key=model_key, priority=10, label="success")+ assert pool.concurrency_for(model) == 5++ clock.advance(10)+ await pool.call(lambda: _return("ok"), model_key=model_key, priority=10, label="success")+ assert pool.concurrency_for(model) == 6++ asyncio.run(exercise())+++def test_llm_call_pool_reports_concurrency_changes():+ async def exercise():+ clock = _Clock()+ changes = []++ def on_change(old, new, reason):+ changes.append((old, new, reason, pool._condition.locked()))++ pool = LLMCallPool(+ 8,+ decrease_cooldown=0,+ recovery_successes=1,+ recovery_cooldown=5,+ rate_limit_retries=0,+ clock=clock,+ on_concurrency_change=on_change,+ )+ model = _ChatModel("reported")+ model_key = pool._model_key(model)++ await pool.call(+ lambda: _return("**ERROR**: 429 too many requests"),+ model_key=model_key,+ priority=10,+ label="limited",+ )+ clock.advance(5)+ await pool.call(lambda: _return("ok"), model_key=model_key, priority=10, label="success")++ assert changes == [(8, 4, "rate limited", False), (4, 5, "recovered", False)]++ asyncio.run(exercise())+++def test_llm_call_pool_isolates_models_and_avoids_head_of_line_blocking():+ async def exercise():+ pool = LLMCallPool(2, max_pending=4, decrease_cooldown=0, rate_limit_retries=0)+ limited_model = _ChatModel("limited")+ healthy_model = _ChatModel("healthy")+ limited_key = pool._model_key(limited_model)+ healthy_key = pool._model_key(healthy_model)++ await pool.call(+ lambda: _return("**ERROR**: 429 too many requests"),+ model_key=limited_key,+ priority=10,+ label="limited",+ )+ assert pool.concurrency_for(limited_model) == 1+ assert pool.concurrency_for(healthy_model) == 2++ release_first = asyncio.Event()+ first_started = asyncio.Event()+ second_started = asyncio.Event()+ healthy_started = asyncio.Event()++ async def first_limited_call():+ first_started.set()+ await release_first.wait()+ return "ok"++ async def second_limited_call():+ second_started.set()+ return "ok"++ async def healthy_call():+ healthy_started.set()+ return "ok"++ first = asyncio.create_task(pool.call(first_limited_call, model_key=limited_key, priority=10, label="limited-1"))+ await first_started.wait()+ second = asyncio.create_task(pool.call(second_limited_call, model_key=limited_key, priority=0, label="limited-2"))+ healthy = asyncio.create_task(pool.call(healthy_call, model_key=healthy_key, priority=20, label="healthy"))++ await asyncio.wait_for(healthy_started.wait(), timeout=1)+ assert not second_started.is_set()++ release_first.set()+ await asyncio.gather(first, second, healthy)+ assert second_started.is_set()++ asyncio.run(exercise())+++def test_llm_call_pool_only_backs_off_for_rate_limit_errors():+ async def exercise():+ pool = LLMCallPool(8, decrease_cooldown=0, rate_limit_retries=0)+ model = _ChatModel("errors")+ model_key = pool._model_key(model)++ with pytest.raises(ValueError, match="invalid request"):+ await pool.call(+ lambda: _raise(ValueError("invalid request")),+ model_key=model_key,+ priority=10,+ label="invalid",+ )+ assert pool.concurrency_for(model) == 8++ with pytest.raises(RuntimeError, match="429"):+ await pool.call(+ lambda: _raise(RuntimeError("429 too many requests")),+ model_key=model_key,+ priority=10,+ label="limited",+ )+ assert pool.concurrency_for(model) == 4++ asyncio.run(exercise())+++async def _raise(exc: Exception):+ raise excdiff --git a/test/unit_test/rag/advanced_rag/test_wiki_refine_progress.py b/test/unit_test/rag/advanced_rag/test_wiki_refine_progress.pynew file mode 100644index 0000000000..64f49d55e4--- /dev/null+++ b/test/unit_test/rag/advanced_rag/test_wiki_refine_progress.py@@ -0,0 +1,19 @@+import asyncio++from rag.advanced_rag.knowlege_compile.wiki_incremental import _wiki_run_refine_tasks+++def test_wiki_refine_progress_is_batched_and_reports_completion():+ async def exercise():+ messages = []++ async def complete():+ return None++ await _wiki_run_refine_tasks([complete() for _ in range(391)], messages.append)++ assert len(messages) == 20+ assert messages[0] == "20/391 pages completed."+ assert messages[-1] == "391/391 pages completed."++ asyncio.run(exercise())diff --git a/web/src/interfaces/database/dataset.ts b/web/src/interfaces/database/dataset.tsindex a05afa4d44..fd4ec45063 100644--- a/web/src/interfaces/database/dataset.ts+++ b/web/src/interfaces/database/dataset.ts@@ -286,6 +286,9 @@ export interface IArtifactAlteration {changed_doc_ids: string[];involved_doc_ids: string[];eligible_doc_ids: string[];+ retry_required?: boolean;+ retry_page_count?: number;+ retry_page_slugs?: string[];}export interface IArtifactGraphRelation {diff --git a/web/src/pages/dataset/compilation/dataset-structure-view.tsx b/web/src/pages/dataset/compilation/dataset-structure-view.tsxindex 2167f75c90..ce4477ccf5 100644--- a/web/src/pages/dataset/compilation/dataset-structure-view.tsx+++ b/web/src/pages/dataset/compilation/dataset-structure-view.tsx@@ -66,7 +66,8 @@ export function DatasetStructureView({ kind }: DatasetStructureViewProps) {const newlyUploaded = alteration?.newly_uploaded ?? 0;const removed = alteration?.removed ?? 0;- const hasChanges = newlyUploaded > 0 || removed > 0;+ const retryPageCount = alteration?.retry_page_count ?? 0;+ const hasChanges = newlyUploaded > 0 || removed > 0 || retryPageCount > 0;const handleRunEnd = useCallback(() => {queryClient.invalidateQueries({@@ -168,6 +169,7 @@ export function DatasetStructureView({ kind }: DatasetStructureViewProps) {hasChanges={hasChanges}newlyUploaded={newlyUploaded}removed={removed}+ retryPageCount={retryPageCount}loading={alterationLoading || runLoading}tooltip={t('knowledgeCompilation.updateStructureTooltip', {newlyUploaded,diff --git a/web/src/pages/dataset/compilation/update-button.tsx b/web/src/pages/dataset/compilation/update-button.tsxindex b064a6c241..d59f6068dd 100644--- a/web/src/pages/dataset/compilation/update-button.tsx+++ b/web/src/pages/dataset/compilation/update-button.tsx@@ -21,6 +21,7 @@ type CompilationUpdateButtonProps = {newlyUploaded: number;removed: number;changed?: number;+ retryPageCount?: number;loading: boolean;tooltip: string;onClick: () => void;@@ -33,6 +34,7 @@ export function CompilationUpdateButton({newlyUploaded,removed,changed = 0,+ retryPageCount = 0,loading,tooltip,onClick,@@ -74,6 +76,11 @@ export function CompilationUpdateButton({{changed}</Badge>)}+ {retryPageCount > 0 && (+ <Badge variant="destructive" className="ml-1">+ {retryPageCount}+ </Badge>+ )}<WandSparkles /></>)}diff --git a/web/src/pages/dataset/compilation/wiki-left-panel/hooks/use-wiki-update.ts b/web/src/pages/dataset/compilation/wiki-left-panel/hooks/use-wiki-update.tsindex 35cd0547e4..64bf000516 100644--- a/web/src/pages/dataset/compilation/wiki-left-panel/hooks/use-wiki-update.ts+++ b/web/src/pages/dataset/compilation/wiki-left-panel/hooks/use-wiki-update.ts@@ -16,7 +16,9 @@ export function useWikiUpdate({ onUpdate }: UseWikiUpdateOptions = {}) {const newlyUploaded = data?.newly_uploaded ?? 0;const removed = data?.removed ?? 0;const changed = data?.changed ?? 0;- const hasChanges = newlyUploaded > 0 || removed > 0 || changed > 0;+ const retryPageCount = data?.retry_page_count ?? 0;+ const hasChanges =+ newlyUploaded > 0 || removed > 0 || changed > 0 || retryPageCount > 0;const handleUpdate = useCallback(async () => {const result = await runArtifactIndex();@@ -30,6 +32,7 @@ export function useWikiUpdate({ onUpdate }: UseWikiUpdateOptions = {}) {newlyUploaded,removed,changed,+ retryPageCount,handleUpdate,loading: queryLoading || mutationLoading,};diff --git a/web/src/pages/dataset/compilation/wiki-left-panel/index.tsx b/web/src/pages/dataset/compilation/wiki-left-panel/index.tsxindex bb965ae02f..63dbcfc411 100644--- a/web/src/pages/dataset/compilation/wiki-left-panel/index.tsx+++ b/web/src/pages/dataset/compilation/wiki-left-panel/index.tsx@@ -50,6 +50,7 @@ export function WikiLeftPanel({newlyUploaded,removed,changed,+ retryPageCount,handleUpdate,loading: updateLoading,} = useWikiUpdate();@@ -74,6 +75,7 @@ export function WikiLeftPanel({newlyUploaded={newlyUploaded}removed={removed}changed={changed}+ retryPageCount={retryPageCount}loading={updateLoading}tooltip={t('knowledgeCompilation.updateTooltip', {newlyUploaded,