Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

feat: adapt knowledge compilation concurrency (#19156)

buua436447f87d79f06
diff --git a/api/apps/services/dataset_api_service.py b/api/apps/services/dataset_api_service.py
index ffc6270486..866324fb65 100644
--- a/api/apps/services/dataset_api_service.py
+++ b/api/apps/services/dataset_api_service.py
@@ -2590,6 +2590,17 @@ async def _get_alteration(dataset_id: str, tenant_id: str, kind: str):
result = _alteration_result(alteration_current_doc_ids, involved_doc_ids, eligible_doc_ids)
if chunk_changes is not None:
result.update(chunk_changes)
+ if kind == "wiki":
+ from rag.advanced_rag.knowlege_compile.wiki_incremental import _wiki_load_refine_failures
+
+ refine_failures = await _wiki_load_refine_failures(kb.tenant_id, dataset_id)
+ result.update(
+ {
+ "retry_required": bool(refine_failures),
+ "retry_page_count": len(refine_failures),
+ "retry_page_slugs": sorted(refine_failures),
+ }
+ )
return True, result
diff --git a/rag/advanced_rag/knowlege_compile/runner.py b/rag/advanced_rag/knowlege_compile/runner.py
index 84aadf5458..692936aaf7 100644
--- a/rag/advanced_rag/knowlege_compile/runner.py
+++ b/rag/advanced_rag/knowlege_compile/runner.py
@@ -197,6 +197,7 @@ async def _upsert_dataset_nav_from_page_index(
doc_name: str,
progress_cb: Callable[..., None],
cancel_check: Callable[[], bool],
+ llm_pool: LLMCallPool,
) -> None:
page_index_templates = [(template_id, parser_cfg) for template_id, parser_cfg in active_templates if _is_page_index_template(parser_cfg)]
if not page_index_templates:
@@ -255,13 +256,19 @@ async def _upsert_dataset_nav_from_page_index(
)
progress_cb(msg=f"page_index: updating dataset navigation for doc {doc_id} ...")
+ pooled_chat_mdl = llm_pool.wrap(
+ chat_mdl,
+ priority=20,
+ label="dataset-nav:page-index",
+ context=f"{doc_id}:dataset-nav",
+ )
await upsert_dataset_nav_doc(
tenant_id,
kb_id,
doc_id,
"\n\n".join(summaries),
embd_mdl=embedding_model,
- chat_mdl=chat_mdl,
+ chat_mdl=pooled_chat_mdl,
)
except TaskCanceledException:
raise
@@ -286,6 +293,7 @@ async def run_structure_compile_over_batches(
progress_cb: Callable[..., None],
cancel_check: Callable[[], bool] = lambda: False,
record: Callable[[str, dict], None] | None = None,
+ llm_pool: LLMCallPool | None = None,
) -> dict[str, dict]:
"""Extract + merge structures for every non-``tree`` template over an
async stream of chunk batches, then run the optional synthesis phase.
@@ -294,7 +302,8 @@ async def run_structure_compile_over_batches(
chat model in ``chat_mdl_by_tid``. Chunks arrive as an async iterator of
batches so callers can stream them from the doc store or hand over an
in-memory list; each ``dict`` must expose ``id`` and text
- (``content_with_weight`` / ``text``).
+ (``content_with_weight`` / ``text``). ``llm_pool`` may be supplied by the
+ task orchestrator so tree and non-tree phases share adaptive concurrency.
Returns ``{template_id: {"inserted", "updated", "duplicates_dropped"}}``.
Raises :class:`TaskCanceledException` when ``cancel_check`` trips.
@@ -305,7 +314,7 @@ async def run_structure_compile_over_batches(
return {}
total = len(active_templates)
- llm_pool = LLMCallPool(DOC_STRUCTURE_LLM_POOL_SIZE)
+ llm_pool = llm_pool or LLMCallPool(DOC_STRUCTURE_LLM_POOL_SIZE)
accumulators: dict[str, list[dict]] = {tid: [] for tid, _ in active_templates}
template_kinds: dict[str, str] = {tid: _compilation_template_kind((cfg or {}).get("kind")) for tid, cfg in active_templates}
@@ -613,6 +622,7 @@ async def run_structure_compile_over_batches(
doc_name=doc_name,
progress_cb=progress_cb,
cancel_check=cancel_check,
+ llm_pool=llm_pool,
)
# Timeline entity cleanup must happen after every flush has completed;
# otherwise an entity can look isolated in one flush and be referenced by
diff --git a/rag/advanced_rag/knowlege_compile/structure.py b/rag/advanced_rag/knowlege_compile/structure.py
index 663149dfee..e9ed583526 100644
--- a/rag/advanced_rag/knowlege_compile/structure.py
+++ b/rag/advanced_rag/knowlege_compile/structure.py
@@ -19,7 +19,9 @@ import heapq
import json
import logging
import re
+import time
import uuid
+from dataclasses import dataclass
from typing import Awaitable, Callable, Tuple
import xxhash
@@ -28,7 +30,6 @@ from common.exceptions import TaskCanceledException
from common.misc_utils import thread_pool_exec
from common.token_utils import num_tokens_from_string
from rag.prompts.generator import gen_json
-
from ._common import (
build_chunk_batches as _build_chunk_batches,
encode as _encode,
@@ -83,15 +84,67 @@ def _struct_merge_lock_key(kb_id: str, compilation_template_id: str | None) -> s
return f"struct_merge:{kb_id}:{compilation_template_id or ''}"
+@dataclass
+class _LLMModelPoolState:
+ concurrency: int
+ active: int = 0
+ successes: int = 0
+ last_decrease_at: float = float("-inf")
+ last_increase_at: float = float("-inf")
+
+
class LLMCallPool:
- """Task-scoped priority scheduler for actual chat-model calls."""
+ """Task-scoped adaptive priority scheduler for chat-model calls.
+
+ ``max_concurrency`` remains the task-wide hard ceiling. Each model starts
+ at that ceiling, halves its own admission limit after an explicit rate
+ limit response, retries through the reduced limit, and recovers one slot
+ at a time after sustained success.
+ """
- def __init__(self, max_concurrency: int = 10, max_pending: int | None = None):
+ _RATE_LIMIT_MARKERS = (
+ "429",
+ "rate limit",
+ "rate_limit",
+ "too many requests",
+ "requests per minute",
+ "concurrency limit",
+ "concurrent request",
+ "maximum number of concurrent",
+ )
+
+ def __init__(
+ self,
+ max_concurrency: int = 10,
+ max_pending: int | None = None,
+ *,
+ min_concurrency: int = 1,
+ decrease_factor: float = 0.5,
+ decrease_cooldown: float = 5.0,
+ recovery_successes: int = 20,
+ recovery_cooldown: float = 30.0,
+ rate_limit_retries: int = 3,
+ rate_limit_retry_base_delay: float = 1.0,
+ rate_limit_retry_max_delay: float = 30.0,
+ clock: Callable[[], float] = time.monotonic,
+ on_concurrency_change: Callable[[int, int, str], None] | None = None,
+ ):
self.max_concurrency = max(1, int(max_concurrency))
self.max_pending = max(self.max_concurrency, int(max_pending or self.max_concurrency))
+ self.min_concurrency = min(self.max_concurrency, max(1, int(min_concurrency)))
+ self.decrease_factor = min(1.0, max(0.01, float(decrease_factor)))
+ self.decrease_cooldown = max(0.0, float(decrease_cooldown))
+ self.recovery_successes = max(1, int(recovery_successes))
+ self.recovery_cooldown = max(0.0, float(recovery_cooldown))
+ self.rate_limit_retries = max(0, int(rate_limit_retries))
+ self.rate_limit_retry_base_delay = max(0.0, float(rate_limit_retry_base_delay))
+ self.rate_limit_retry_max_delay = max(self.rate_limit_retry_base_delay, float(rate_limit_retry_max_delay))
+ self._clock = clock
+ self._on_concurrency_change = on_concurrency_change
self._active = 0
self._ticket = 0
- self._waiting: list[tuple[int, int]] = []
+ self._waiting: list[tuple[int, int, str]] = []
+ self._model_states: dict[str, _LLMModelPoolState] = {}
self._condition = asyncio.Condition()
@property
@@ -103,17 +156,125 @@ class LLMCallPool:
return self._active + len(self._waiting)
def wrap(self, chat_mdl, *, priority: int, label: str, context: str | None = None):
- return PooledChatModel(self, chat_mdl, priority=priority, label=label, context=context)
+ return PooledChatModel(
+ self,
+ chat_mdl,
+ model_key=self._model_key(chat_mdl),
+ priority=priority,
+ label=label,
+ context=context,
+ )
- async def call(self, fn, *, priority: int, label: str, context: str | None = None):
+ def concurrency_for(self, chat_mdl) -> int:
+ """Return the current adaptive concurrency limit for ``chat_mdl``."""
+ return self._state_for(self._model_key(chat_mdl)).concurrency
+
+ @staticmethod
+ def _model_key(chat_mdl) -> str:
+ config = getattr(chat_mdl, "model_config", None)
+ if isinstance(config, dict):
+ model_id = str(config.get("id") or config.get("llm_id") or config.get("model_id") or "").strip()
+ factory = str(config.get("llm_factory") or "").strip()
+ name = str(config.get("llm_name") or "").strip()
+ endpoint = str(config.get("api_base") or config.get("base_url") or "").strip()
+ if model_id or factory or name or endpoint:
+ return ":".join((model_id, factory, name, endpoint))
+ name = str(getattr(chat_mdl, "llm_name", "") or "").strip()
+ if name:
+ return name
+ return f"{type(chat_mdl).__module__}.{type(chat_mdl).__qualname__}:{id(chat_mdl)}"
+
+ def _state_for(self, model_key: str) -> _LLMModelPoolState:
+ state = self._model_states.get(model_key)
+ if state is None:
+ state = _LLMModelPoolState(concurrency=self.max_concurrency)
+ self._model_states[model_key] = state
+ return state
+
+ def _next_admissible_ticket(self) -> tuple[int, int, str] | None:
+ if self._active >= self.max_concurrency:
+ return None
+ # The bounded queue is scanned to avoid one throttled model blocking
+ # admissible work for another model at the heap head.
+ candidates = [ticket for ticket in self._waiting if self._state_for(ticket[2]).active < self._state_for(ticket[2]).concurrency]
+ return min(candidates) if candidates else None
+
+ @classmethod
+ def _is_rate_limited(cls, value) -> bool:
+ text = str(value).lower()
+ return any(marker in text for marker in cls._RATE_LIMIT_MARKERS)
+
+ @staticmethod
+ def _is_error_result(result) -> bool:
+ return isinstance(result, str) and result.lstrip().lower().startswith("**error**")
+
+ def _record_feedback(self, model_key: str, outcome: str, *, label: str, context: str | None) -> tuple[int, int, str] | None:
+ state = self._state_for(model_key)
+ now = self._clock()
+ if outcome == "rate_limited":
+ state.successes = 0
+ if now - state.last_decrease_at < self.decrease_cooldown:
+ return None
+ old_concurrency = state.concurrency
+ state.concurrency = max(self.min_concurrency, int(state.concurrency * self.decrease_factor))
+ state.last_decrease_at = now
+ if state.concurrency != old_concurrency:
+ logging.warning(
+ "LLM pool concurrency decreased model=%s label=%s context=%s old=%d new=%d active=%d pending=%d",
+ model_key,
+ label,
+ context,
+ old_concurrency,
+ state.concurrency,
+ state.active,
+ len(self._waiting),
+ )
+ return old_concurrency, state.concurrency, "rate limited"
+ return None
+ if outcome != "success":
+ state.successes = 0
+ return None
+ if state.concurrency >= self.max_concurrency:
+ state.successes = 0
+ return None
+ state.successes += 1
+ if state.successes < self.recovery_successes:
+ return None
+ if now - state.last_decrease_at < self.recovery_cooldown or now - state.last_increase_at < self.recovery_cooldown:
+ return None
+ old_concurrency = state.concurrency
+ state.concurrency = min(self.max_concurrency, state.concurrency + 1)
+ state.successes = 0
+ state.last_increase_at = now
+ logging.info(
+ "LLM pool concurrency increased model=%s label=%s context=%s old=%d new=%d active=%d pending=%d",
+ model_key,
+ label,
+ context,
+ old_concurrency,
+ state.concurrency,
+ state.active,
+ len(self._waiting),
+ )
+ return old_concurrency, state.concurrency, "recovered"
+
+ def _notify_concurrency_change(self, old_concurrency: int, new_concurrency: int, reason: str) -> None:
+ if self._on_concurrency_change is None:
+ return
+ try:
+ self._on_concurrency_change(old_concurrency, new_concurrency, reason)
+ except Exception:
+ logging.exception("LLM pool concurrency change callback failed")
+
+ async def _acquire(self, model_key: str, priority: int) -> None:
async with self._condition:
while self.pending_count >= self.max_pending:
await self._condition.wait()
- ticket = (int(priority), self._ticket)
+ ticket = (int(priority), self._ticket, model_key)
self._ticket += 1
heapq.heappush(self._waiting, ticket)
try:
- while self._active >= self.max_concurrency or self._waiting[0] != ticket:
+ while self._next_admissible_ticket() != ticket:
await self._condition.wait()
except BaseException:
if ticket in self._waiting:
@@ -121,23 +282,69 @@ class LLMCallPool:
heapq.heapify(self._waiting)
self._condition.notify_all()
raise
- heapq.heappop(self._waiting)
+ self._waiting.remove(ticket)
+ heapq.heapify(self._waiting)
self._active += 1
- try:
- result = await fn()
- return result
- except BaseException:
- raise
- finally:
- async with self._condition:
- self._active -= 1
- self._condition.notify_all()
+ self._state_for(model_key).active += 1
+
+ async def _release(self, model_key: str, outcome: str, *, label: str, context: str | None) -> None:
+ async with self._condition:
+ self._active -= 1
+ self._state_for(model_key).active -= 1
+ change = self._record_feedback(model_key, outcome, label=label, context=context)
+ self._condition.notify_all()
+ if change is not None:
+ self._notify_concurrency_change(*change)
+
+ def _rate_limit_retry_delay(self, retry: int) -> float:
+ return min(self.rate_limit_retry_max_delay, self.rate_limit_retry_base_delay * (2 ** (retry - 1)))
+
+ async def call(self, fn, *, model_key: str, priority: int, label: str, context: str | None = None):
+ for retry in range(self.rate_limit_retries + 1):
+ await self._acquire(model_key, priority)
+ try:
+ result = await fn()
+ except asyncio.CancelledError:
+ await self._release(model_key, "cancelled", label=label, context=context)
+ raise
+ except BaseException as exc:
+ rate_limited = self._is_rate_limited(exc)
+ await self._release(model_key, "rate_limited" if rate_limited else "failed", label=label, context=context)
+ if not rate_limited or retry >= self.rate_limit_retries:
+ raise
+ else:
+ error_result = self._is_error_result(result)
+ rate_limited = error_result and self._is_rate_limited(result)
+ await self._release(
+ model_key,
+ "rate_limited" if rate_limited else "failed" if error_result else "success",
+ label=label,
+ context=context,
+ )
+ if not rate_limited or retry >= self.rate_limit_retries:
+ return result
+
+ delay = self._rate_limit_retry_delay(retry + 1)
+ logging.warning(
+ "LLM pool retrying rate-limited call model=%s label=%s context=%s retry=%d/%d delay=%.2fs concurrency=%d",
+ model_key,
+ label,
+ context,
+ retry + 1,
+ self.rate_limit_retries,
+ delay,
+ self._state_for(model_key).concurrency,
+ )
+ await asyncio.sleep(delay)
+
+ raise AssertionError("LLM pool retry loop exhausted unexpectedly")
class PooledChatModel:
- def __init__(self, pool: LLMCallPool, chat_mdl, *, priority: int, label: str, context: str | None):
+ def __init__(self, pool: LLMCallPool, chat_mdl, *, model_key: str, priority: int, label: str, context: str | None):
self._pool = pool
self._chat_mdl = chat_mdl
+ self._model_key = model_key
self._priority = priority
self._label = label
self._context = context
@@ -149,6 +356,7 @@ class PooledChatModel:
gen_conf = _knowledge_compile_gen_conf(self._chat_mdl, gen_conf)
return await self._pool.call(
lambda: self._chat_mdl.async_chat(system, history, gen_conf=gen_conf, **kwargs),
+ model_key=self._model_key,
priority=self._priority,
label=self._label,
context=self._context,
diff --git a/rag/advanced_rag/knowlege_compile/wiki.py b/rag/advanced_rag/knowlege_compile/wiki.py
index 3a71d98fbc..cf7b9c182b 100644
--- a/rag/advanced_rag/knowlege_compile/wiki.py
+++ b/rag/advanced_rag/knowlege_compile/wiki.py
@@ -3913,6 +3913,8 @@ async def wiki_refine_from_plan(
semaphore = asyncio.Semaphore(max_workers) if max_workers and max_workers > 0 else None
completed = 0
completed_lock = asyncio.Lock()
+ progress_updates = 20
+ report_every = max(1, (total + progress_updates - 1) // progress_updates)
async def _write_one(plan_item: dict) -> Optional[dict]:
nonlocal completed
@@ -3922,7 +3924,6 @@ async def wiki_refine_from_plan(
page_type = plan_item.get("page_type") or "concept"
async def _run() -> Optional[dict]:
- nonlocal completed
try:
evidence = _wiki_assemble_evidence(
plan_item,
@@ -4006,7 +4007,7 @@ async def wiki_refine_from_plan(
}
except Exception:
logging.exception("wiki_refine: writer failed for slug=%s", slug)
- return None
+ raise
# Searchable wiki_page persistence has moved to the task
# handler so the doc-storage schema can be controlled in one
@@ -4023,21 +4024,27 @@ async def wiki_refine_from_plan(
except Exception:
logging.exception("wiki_refine: persist_draft failed for slug=%s", slug)
- if callback:
+ return page
+
+ result: Optional[dict] = None
+ try:
+ if semaphore is not None:
+ async with semaphore:
+ result = await _run()
+ else:
+ result = await _run()
+ return result
+ finally:
+ if callback and result is not None:
async with completed_lock:
completed += 1
done = completed
- progress = 0.1 + 0.85 * (done / total)
- try:
- callback(progress, f"wiki REFINE: {done}/{total} pages written ({slug})")
- except Exception:
- pass
- return page
-
- if semaphore is not None:
- async with semaphore:
- return await _run()
- return await _run()
+ if done % report_every == 0 or done == total:
+ progress = 0.1 + 0.85 * (done / total)
+ try:
+ callback(progress, f"wiki REFINE: {done}/{total} pages completed")
+ except Exception:
+ logging.exception("wiki REFINE progress callback failed")
tasks = [asyncio.create_task(_write_one(p)) for p in pending]
if tasks:
diff --git a/rag/advanced_rag/knowlege_compile/wiki_incremental.py b/rag/advanced_rag/knowlege_compile/wiki_incremental.py
index d5c9b4f4d7..02392f68b6 100644
--- a/rag/advanced_rag/knowlege_compile/wiki_incremental.py
+++ b/rag/advanced_rag/knowlege_compile/wiki_incremental.py
@@ -33,9 +33,29 @@ from ._common import (
)
-# ----- REFINE concurrency control -----
+# ----- REFINE progress -----
-WIKI_REFINE_MAX_CONCURRENT = 20 # shared LLM pool size used by the Wiki runner
+WIKI_REFINE_PROGRESS_UPDATES = 20
+
+
+async def _wiki_run_refine_tasks(tasks: list, progress: Callable[[str], None]) -> None:
+ """Run REFINE page tasks and emit at most roughly 20 progress updates."""
+ total = len(tasks)
+ if not total:
+ return
+ report_every = max(1, (total + WIKI_REFINE_PROGRESS_UPDATES - 1) // WIKI_REFINE_PROGRESS_UPDATES)
+ scheduled = [asyncio.create_task(task) for task in tasks]
+ try:
+ for done, task in enumerate(asyncio.as_completed(scheduled), start=1):
+ await task
+ if done % report_every == 0 or done == total:
+ progress(f"{done}/{total} pages completed.")
+ except BaseException:
+ for task in scheduled:
+ if not task.done():
+ task.cancel()
+ await asyncio.gather(*scheduled, return_exceptions=True)
+ raise
# ----- constants ----
@@ -45,6 +65,7 @@ WIKI_PAGE_COMPILE_KWD = "wiki_page"
WIKI_PLAN_GROUP_COMPILE_KWD = "wiki_plan_group"
WIKI_DOC_PAGE_SOURCE_COMPILE_KWD = "wiki_doc_page_source"
WIKI_CANONICAL_ENTITY_COMPILE_KWD = "wiki_canonical_entity"
+WIKI_REFINE_FAILURE_COMPILE_KWD = "wiki_refine_failure"
# Entity matching thresholds (not exposed in YAML)
ENTITY_MERGE_THRESHOLD = 0.90 # auto-merge
@@ -303,6 +324,103 @@ async def _wiki_has_any_pages(tenant_id: str, kb_id: str) -> bool:
return False
+async def _wiki_load_refine_failures(tenant_id: str, kb_id: str) -> dict[str, dict]:
+ """Load persisted REFINE failures keyed by page id."""
+ index = search.index_name(tenant_id)
+ if not settings.docStoreConn.index_exist(index, kb_id):
+ return {}
+ fields = ["slug_kwd", "content_with_weight"]
+ try:
+ res = await thread_pool_exec(
+ settings.docStoreConn.search,
+ fields,
+ [],
+ {"compile_kwd": [WIKI_REFINE_FAILURE_COMPILE_KWD]},
+ [],
+ OrderByExpr(),
+ 0,
+ 1000,
+ index,
+ [kb_id],
+ )
+ rows = settings.docStoreConn.get_fields(res, fields) or {}
+ except Exception:
+ logging.exception("wiki: failed to load REFINE failures for kb=%s", kb_id)
+ return {}
+
+ failures: dict[str, dict] = {}
+ for row in rows.values():
+ page_id = row.get("slug_kwd")
+ payload = row.get("content_with_weight")
+ if isinstance(page_id, list):
+ page_id = page_id[0] if page_id else ""
+ if not isinstance(page_id, str) or not page_id.strip():
+ continue
+ if isinstance(payload, str):
+ try:
+ payload = json.loads(payload) if payload else {}
+ except (json.JSONDecodeError, TypeError):
+ payload = {}
+ failures[page_id.strip()] = payload if isinstance(payload, dict) else {}
+ return failures
+
+
+async def _wiki_record_refine_failure(
+ tenant_id: str,
+ kb_id: str,
+ page_id: str,
+ entity_names: list[str] | None = None,
+ error: str = "",
+) -> None:
+ """Persist one failed page so a later Update can retry it."""
+ page_id = str(page_id or "").strip()
+ if not page_id:
+ return
+ index = search.index_name(tenant_id)
+ payload = json.dumps(
+ {
+ "page_id": page_id,
+ "entity_names": [str(name).strip() for name in entity_names or [] if str(name).strip()],
+ "error": str(error or "")[:1000],
+ },
+ ensure_ascii=False,
+ )
+ row = {
+ "id": _stable_row_id(WIKI_REFINE_FAILURE_COMPILE_KWD, kb_id, page_id),
+ "doc_id": str(kb_id),
+ "compile_kwd": WIKI_REFINE_FAILURE_COMPILE_KWD,
+ "slug_kwd": page_id,
+ "content_with_weight": payload,
+ "available_int": 0,
+ }
+ try:
+ await thread_pool_exec(
+ settings.docStoreConn.delete,
+ {"compile_kwd": [WIKI_REFINE_FAILURE_COMPILE_KWD], "slug_kwd": [page_id]},
+ index,
+ kb_id,
+ )
+ await thread_pool_exec(settings.docStoreConn.insert, [row], index, kb_id)
+ except Exception:
+ logging.exception("wiki: failed to persist REFINE failure page=%s", page_id)
+
+
+async def _wiki_clear_refine_failure(tenant_id: str, kb_id: str, page_id: str) -> None:
+ """Remove the retry marker after a page succeeds or is deleted."""
+ page_id = str(page_id or "").strip()
+ if not page_id:
+ return
+ try:
+ await thread_pool_exec(
+ settings.docStoreConn.delete,
+ {"compile_kwd": [WIKI_REFINE_FAILURE_COMPILE_KWD], "slug_kwd": [page_id]},
+ search.index_name(tenant_id),
+ kb_id,
+ )
+ except Exception:
+ logging.exception("wiki: failed to clear REFINE failure page=%s", page_id)
+
+
async def _load_canonical_entities(
tenant_id: str,
kb_id: str,
@@ -1191,16 +1309,26 @@ def _wiki_extract_outlinks_from_content(content: str, kb_id: str = "") -> list[s
return outlinks
-def _inside_wikilink(content: str, pos: int) -> bool:
- """Return True iff position ``pos`` falls inside an existing [[...]] span."""
- open_pos = content.rfind("[[", 0, pos)
- if open_pos < 0:
- return False
- # Find the FIRST "]]" that closes the "[[...]]" starting at open_pos
- close_pos = content.find("]]", open_pos + 2)
- if close_pos < 0:
- close_pos = len(content)
- return open_pos + 2 <= pos < close_pos
+_WIKI_PROTECTED_LINK_RE = re.compile(r"\[\[[^\]\n]+\]\]|\[[^\]\n]*\]\([^)\n]+\)")
+
+
+def _wiki_find_unlinked_mention(content: str, name: str) -> int:
+ """Find the first occurrence of ``name`` outside existing link markup."""
+ if not content or not name:
+ return -1
+ protected_spans = [(match.start(), match.end()) for match in _WIKI_PROTECTED_LINK_RE.finditer(content)]
+ raw_open = content.rfind("[[")
+ if raw_open >= 0 and content.find("]]", raw_open + 2) < 0:
+ protected_spans.append((raw_open, len(content)))
+ start = 0
+ while True:
+ idx = content.find(name, start)
+ if idx < 0:
+ return -1
+ end = idx + len(name)
+ if not any(idx < protected_end and end > protected_start for protected_start, protected_end in protected_spans):
+ return idx
+ start = idx + 1
_WIKI_PIPE_LINK_RE = re.compile(r"\[\[([^\[\]\|]+?)\|([^\[\]]+?)\]\]")
@@ -3225,12 +3353,9 @@ async def _wiki_finalize(
continue
if target in existing_links:
continue
- idx = content.find(name)
+ idx = _wiki_find_unlinked_mention(content, name)
if idx < 0:
continue
- # Skip if the occurrence is already inside a [[...]] link span
- if _inside_wikilink(content, idx):
- continue
# Preserve the matched prose as the link label. This matters when
# several entities share one page: e.g. the page slug may be
# ``entity/五色棒`` while the text mentions
@@ -3703,6 +3828,15 @@ async def wiki_compile_incremental(
if page_id in existing_pages and names:
existing_pages[page_id]["entity_names_kwd"] = names
+ # A failed REFINE page must be reconsidered together with the current
+ # document delta. The current REDUCE snapshot remains authoritative, so
+ # deleted entities can still produce a deletion rather than being blindly
+ # retried from stale failure metadata.
+ refine_failures = await _wiki_load_refine_failures(tenant_id, kb_id)
+ retry_names = {str(name).strip() for failure in refine_failures.values() for name in failure.get("entity_names", []) if str(name).strip()}
+ if incremental:
+ affected_names.update(retry_names)
+
# Build canonical claims ON-DEMAND only for affected names, then release
# the full claim_index. claim_index is keyed by RAW entity name (from MAP),
# so claims must be aggregated through name_resolution onto the canonical
@@ -3973,6 +4107,7 @@ async def _wiki_mode_a_run(
page_version=existing.get("page_version_int", 0) if existing else 0,
)
summary["pages_deleted"] += 1
+ await _wiki_clear_refine_failure(tenant_id, kb_id, pid)
return
next_version = _as_int(existing.get("page_version_int")) + 1 if existing else 1
@@ -4008,6 +4143,9 @@ async def _wiki_mode_a_run(
topic_pool=topic_pool,
topic_pool_lock=topic_pool_lock,
)
+ if result is None:
+ raise RuntimeError(f"REFINE returned no page content for {pid}")
+ await _wiki_clear_refine_failure(tenant_id, kb_id, pid)
if refine_mode == "generate":
summary["pages_created"] += 1
else:
@@ -4017,14 +4155,21 @@ async def _wiki_mode_a_run(
for did in entry["source_doc_ids"]:
doc_updates.setdefault(did, []).append(pid)
- except Exception:
+ except Exception as exc:
logging.exception("wiki A: REFINE failed for %s", pid)
+ await _wiki_record_refine_failure(
+ tenant_id,
+ kb_id,
+ pid,
+ entry.get("existing_page", {}).get("entity_names_kwd") or [entry.get("page_title", pid)],
+ error=str(exc),
+ )
summary["errors"].append(f"REFINE_FAILED:{pid}")
tasks = [_refine_one(pid, entry) for pid, entry in page_deltas.items()]
if tasks:
- _progress(f"REFINE A: {len(tasks)} pages (LLM pool max {WIKI_REFINE_MAX_CONCURRENT}) ...")
- await asyncio.gather(*tasks)
+ _progress(f"{len(tasks)} pages ...")
+ await _wiki_run_refine_tasks(tasks, _progress)
_wiki_log_stats("TOPIC", "selection_summary", mode="A", **topic_selection_stats)
for did, pids in doc_updates.items():
@@ -4300,6 +4445,7 @@ async def _wiki_mode_b_run(
page_version=existing.get("page_version_int", 0) if existing else 0,
)
if deleted_page is None:
+ await _wiki_clear_refine_failure(tenant_id, kb_id, page_key)
await _wiki_delete_plan_group(tenant_id, kb_id, page_key)
for did in _as_str_list(existing.get("source_doc_ids")) if existing else []:
doc_removals.setdefault(did, []).append(page_key)
@@ -4341,7 +4487,10 @@ async def _wiki_mode_b_run(
topic_pool_lock=topic_pool_lock,
member_evidence=member_evidence,
)
+ if result is None:
+ raise RuntimeError(f"REFINE returned no page content for {page_key}")
if result:
+ await _wiki_clear_refine_failure(tenant_id, kb_id, page_key)
if is_new:
summary["pages_created"] += 1
else:
@@ -4367,14 +4516,22 @@ async def _wiki_mode_b_run(
for did in old_doc_ids - new_doc_ids:
doc_removals.setdefault(did, []).append(page_key)
- except Exception:
+ except Exception as exc:
logging.exception("wiki B: REFINE failed for %s", page_id)
+ failed_page_id = page_id[5:] if page_id.startswith("_new_") else page_id
+ await _wiki_record_refine_failure(
+ tenant_id,
+ kb_id,
+ failed_page_id,
+ [ent.get("entity_name", "") for ent in entities if isinstance(ent, dict)],
+ error=str(exc),
+ )
summary["errors"].append(f"REFINE_FAILED:{page_id}")
tasks = [_refine_one(pid, ents) for pid, ents in assignments.items()]
if tasks:
- _progress(f"REFINE B: {len(tasks)} pages (LLM pool max {WIKI_REFINE_MAX_CONCURRENT}) ...")
- await asyncio.gather(*tasks)
+ _progress(f"{len(tasks)} pages ...")
+ await _wiki_run_refine_tasks(tasks, _progress)
_wiki_log_stats("TOPIC", "selection_summary", mode="B", **topic_selection_stats)
# Apply doc_page_source updates serially (no race), preserving metadata
diff --git a/rag/flow/compiler/compiler.py b/rag/flow/compiler/compiler.py
index 6ced075951..d1fda66979 100644
--- a/rag/flow/compiler/compiler.py
+++ b/rag/flow/compiler/compiler.py
@@ -30,6 +30,7 @@ from common.constants import LLMType
from common.token_utils import num_tokens_from_string
from rag.advanced_rag.knowlege_compile.runner import (
DOC_STRUCTURE_COMPILE_BATCH_CHUNKS,
+ DOC_STRUCTURE_LLM_POOL_SIZE,
load_active_templates,
resolve_template_ids_from_groups,
run_structure_compile_over_batches,
@@ -371,6 +372,7 @@ class Compiler(ProcessBase, LLM):
kb_id: str,
doc_id: str,
doc_name: str,
+ llm_pool,
) -> None:
"""Build and persist tree graphs from the pipeline's in-memory chunks.
@@ -418,6 +420,12 @@ class Compiler(ProcessBase, LLM):
raptor_service = RaptorService(tree_context)
for idx, (template_id, parser_cfg) in enumerate(templates):
+ pooled_chat_mdl = llm_pool.wrap(
+ chat_mdl_by_tid[template_id],
+ priority=30,
+ label=f"tree:{template_id}",
+ context=f"{doc_id}:{template_id}:tree",
+ )
raptor_cfg = (parser_cfg or {}).get("raptor") or {}
raptor_config = {
"prompt": raptor_cfg.get("prompt")
@@ -432,7 +440,7 @@ class Compiler(ProcessBase, LLM):
tree = await raptor_service.build_doc_tree(
chunks=tree_chunks,
raptor_config=raptor_config,
- chat_mdl=chat_mdl_by_tid[template_id],
+ chat_mdl=pooled_chat_mdl,
embd_mdl=embedding_model,
max_errors=3,
)
@@ -445,7 +453,7 @@ class Compiler(ProcessBase, LLM):
if bool(raptor_cfg.get("rechunk")):
self._compile_progress(msg="Compiler: tree rechunking is not supported for in-memory pipeline chunks; keeping original chunks.")
- await rewrite_duplicate_tree_names(tree, chat_mdl_by_tid[template_id])
+ await rewrite_duplicate_tree_names(tree, pooled_chat_mdl)
after_graph = raptor_tree_to_graph(tree)
try:
await _struct_upsert_tree_graph_rows(
@@ -489,7 +497,7 @@ class Compiler(ProcessBase, LLM):
doc_id,
{"title": tree.get("title"), "graph_text": nav_graph_text},
embd_mdl=embedding_model,
- chat_mdl=chat_mdl_by_tid[template_id],
+ chat_mdl=pooled_chat_mdl,
)
except Exception:
logging.exception("Compiler: tree-template %s dataset navigation upsert failed for doc %s", template_id, doc_id)
@@ -615,6 +623,9 @@ class Compiler(ProcessBase, LLM):
max_retries=self._param.max_retries,
retry_interval=self._param.delay_after_error,
)
+ from rag.advanced_rag.knowlege_compile.structure import LLMCallPool
+
+ llm_pool = LLMCallPool(DOC_STRUCTURE_LLM_POOL_SIZE)
tree_templates, non_tree_templates = split_tree_templates(active_templates)
if tree_templates:
@@ -627,6 +638,7 @@ class Compiler(ProcessBase, LLM):
kb_id,
doc_id,
doc_name,
+ llm_pool,
)
if non_tree_templates:
@@ -663,6 +675,7 @@ class Compiler(ProcessBase, LLM):
chunk_batches=_chunk_batches(),
progress_cb=self._compile_progress,
cancel_check=_cancelled,
+ llm_pool=llm_pool,
)
if first_rechunk_index is None:
diff --git a/rag/svr/task_executor_refactor/chunk_post_processor.py b/rag/svr/task_executor_refactor/chunk_post_processor.py
index d92eb0d13f..1dc17eba60 100644
--- a/rag/svr/task_executor_refactor/chunk_post_processor.py
+++ b/rag/svr/task_executor_refactor/chunk_post_processor.py
@@ -412,6 +412,7 @@ from common.token_utils import num_tokens_from_string
# component can share them. Re-exported here for backwards compatibility.
from rag.advanced_rag.knowlege_compile.runner import (
DOC_STRUCTURE_COMPILE_BATCH_CHUNKS,
+ DOC_STRUCTURE_LLM_POOL_SIZE,
DOC_STRUCTURE_MERGE_MAX_DOCS, # noqa: F401
STRUCTURE_CHAIN_CORRECTION_TIMEOUT_S, # noqa: F401
load_active_templates,
@@ -928,6 +929,7 @@ async def run_tree_templates(
chat_mdl_by_tid: dict[str, "LLMBundle"],
embedding_model,
doc_name: str,
+ llm_pool,
) -> None:
"""Run the ``tree``-kind compilation templates for the current
doc. Each pair runs RAPTOR with ``is_tree=True`` via
@@ -961,6 +963,12 @@ async def run_tree_templates(
raptor_service = RaptorService(ctx)
for idx, (template_id, parser_cfg) in enumerate(templates):
+ pooled_chat_mdl = llm_pool.wrap(
+ chat_mdl_by_tid[template_id],
+ priority=30,
+ label=f"tree:{template_id}",
+ context=f"{doc_id}:{template_id}:tree",
+ )
raptor_cfg = (parser_cfg or {}).get("raptor") or {}
raptor_config = {
"prompt": raptor_cfg.get("prompt") or "Please write a concise summary of the following texts:\n{cluster_content}",
@@ -977,7 +985,7 @@ async def run_tree_templates(
tree = await raptor_service.build_doc_tree(
chunks=chunks,
raptor_config=raptor_config,
- chat_mdl=chat_mdl_by_tid[template_id],
+ chat_mdl=pooled_chat_mdl,
embd_mdl=embedding_model,
max_errors=3,
)
@@ -1011,7 +1019,7 @@ async def run_tree_templates(
doc_id,
)
- await rewrite_duplicate_tree_names(tree, chat_mdl_by_tid[template_id])
+ await rewrite_duplicate_tree_names(tree, pooled_chat_mdl)
graph = raptor_tree_to_graph(tree)
try:
await _struct_upsert_graph_json(
@@ -1053,7 +1061,7 @@ async def run_tree_templates(
doc_id,
{"title": tree.get("title"), "graph_text": nav_graph_text},
embd_mdl=embedding_model,
- chat_mdl=chat_mdl_by_tid[template_id],
+ chat_mdl=pooled_chat_mdl,
)
except Exception:
logging.exception(
@@ -1100,6 +1108,9 @@ async def run_document_structure_compile(handler, embedding_model: LLMBundle) ->
logging.exception("document_structure_compile: cannot resolve ingestion chat model %s", chat_llm_id)
return
chat_mdl_by_tid = {template_id: chat_mdl for template_id, _ in active_templates}
+ from rag.advanced_rag.knowlege_compile.structure import LLMCallPool
+
+ llm_pool = LLMCallPool(DOC_STRUCTURE_LLM_POOL_SIZE)
tree_templates: list[tuple[str, dict]] = []
non_tree_templates: list[tuple[str, dict]] = []
@@ -1116,6 +1127,7 @@ async def run_document_structure_compile(handler, embedding_model: LLMBundle) ->
chat_mdl_by_tid,
embedding_model,
doc_name,
+ llm_pool,
)
if not non_tree_templates:
@@ -1143,6 +1155,7 @@ async def run_document_structure_compile(handler, embedding_model: LLMBundle) ->
progress_cb=ctx.progress_cb,
cancel_check=lambda: ctx.has_canceled_func(ctx.id),
record=ctx.recording_context.record,
+ llm_pool=llm_pool,
)
diff --git a/rag/svr/task_executor_refactor/dataset_wiki_generator.py b/rag/svr/task_executor_refactor/dataset_wiki_generator.py
index f96632b147..1330547998 100644
--- a/rag/svr/task_executor_refactor/dataset_wiki_generator.py
+++ b/rag/svr/task_executor_refactor/dataset_wiki_generator.py
@@ -80,9 +80,9 @@ from rag.svr.task_executor_refactor.task_context import TaskContext
# room for the function's internal split_chunks packing to do real work.
WIKI_MAP_BATCH_CHUNKS = 64
-# The pool limits actual MAP LLM calls rather than the surrounding batch
-# tasks. This lets the next waiting batch start as soon as a model call
-# finishes, without waiting for the previous batch's ES persistence work.
+# The pool limits actual LLM calls rather than the surrounding batch tasks and
+# adapts each model's admission limit when it reports rate limiting. This lets
+# high-capacity and low-concurrency models share one Wiki task safely.
WIKI_MAP_LLM_POOL_SIZE = 20
# Global MAP admission limit: active calls plus calls waiting in the pool.
@@ -96,6 +96,21 @@ WIKI_MAP_QUEUE_SIZE = 5
# pool; the pool itself still caps actual LLM requests at 20.
WIKI_REFINE_WORKERS = WIKI_MAP_LLM_POOL_SIZE
+
+def _create_wiki_llm_pool(progress: Callable) -> LLMCallPool:
+ def _on_concurrency_change(old: int, new: int, reason: str) -> None:
+ progress(msg=f"LLM pool concurrency {old} -> {new} ({reason}).")
+
+ pool = LLMCallPool(
+ WIKI_MAP_LLM_POOL_SIZE,
+ max_pending=WIKI_MAP_MAX_PENDING,
+ on_concurrency_change=_on_concurrency_change,
+ )
+ logging.info("Wiki LLM pool initialized max_concurrency=%d max_pending=%d", pool.max_concurrency, pool.max_pending)
+ progress(0.0, f"LLM pool max {pool.max_concurrency}.")
+ return pool
+
+
# Per-node cap on ``source_chunk_ids`` carried by the canvas graph blob.
# Pages can accumulate hundreds of source chunks; the graph response is
# meant for fast canvas rendering, not full provenance audit, so we trim
@@ -1300,6 +1315,7 @@ async def run_wiki(
from api.db.joint_services.tenant_model_service import resolve_model_config
progress = ctx.progress_cb
+ map_llm_pool = _create_wiki_llm_pool(progress)
progress(0.0, "Loading documents for wiki compilation...")
# 1. Resolve KB metadata for PLAN.
@@ -1415,7 +1431,6 @@ async def run_wiki(
first_example = first_parser_cfg.get("example")
kb_writer_instruction: Optional[str] = first_instruction if isinstance(first_instruction, str) and first_instruction.strip() else None
kb_writer_example: Optional[str] = first_example if isinstance(first_example, str) and first_example.strip() else None
- map_llm_pool = LLMCallPool(WIKI_MAP_LLM_POOL_SIZE, max_pending=WIKI_MAP_MAX_PENDING)
n_docs = len(resolved_eligible)
map_queue: asyncio.Queue = asyncio.Queue(maxsize=WIKI_MAP_QUEUE_SIZE)
doc_stats = {
@@ -1634,9 +1649,9 @@ async def run_wiki_incremental(
from rag.advanced_rag.knowlege_compile.wiki_incremental import (
wiki_compile_incremental,
)
- from rag.advanced_rag.knowlege_compile.structure import LLMCallPool
progress = ctx.progress_cb
+ map_llm_pool = _create_wiki_llm_pool(progress)
progress(0.0, "Loading documents for wiki compilation...")
# 1. Check if this is incremental (existing MAP rows present)
@@ -1765,7 +1780,6 @@ async def run_wiki_incremental(
llm_bundle_cache[key] = bundle
return bundle
- map_llm_pool = LLMCallPool(WIKI_MAP_LLM_POOL_SIZE, max_pending=WIKI_MAP_MAX_PENDING)
kb_chat_llm_id = None
first_template_found = False
@@ -1875,7 +1889,10 @@ async def run_wiki_incremental(
# prior run stopped after MAP), fall through so the compiler can rebuild
# pages from the stored extracts.
has_compiled_pages = await _wiki_has_compiled_pages(ctx.tenant_id, ctx.kb_id) if existing_map_doc_ids else None
- if existing_map_doc_ids and has_compiled_pages is True:
+ from rag.advanced_rag.knowlege_compile.wiki_incremental import _wiki_load_refine_failures
+
+ has_refine_failures = bool(await _wiki_load_refine_failures(ctx.tenant_id, ctx.kb_id))
+ if existing_map_doc_ids and has_compiled_pages is True and not has_refine_failures:
from rag.advanced_rag.knowlege_compile.wiki_incremental import (
_wiki_finalize,
_wiki_load_pages_for_graph,
diff --git a/test/unit_test/rag/advanced_rag/knowlege_compile/test_wiki_incremental.py b/test/unit_test/rag/advanced_rag/knowlege_compile/test_wiki_incremental.py
index 8d1cbf9495..8d81d0b89f 100644
--- a/test/unit_test/rag/advanced_rag/knowlege_compile/test_wiki_incremental.py
+++ b/test/unit_test/rag/advanced_rag/knowlege_compile/test_wiki_incremental.py
@@ -79,6 +79,32 @@ class MockChatModel:
pass
+async def test_load_refine_failures_decodes_persisted_page_markers(monkeypatch):
+ store = MagicMock()
+ store.index_exist.return_value = True
+ store.get_fields.return_value = {
+ "row-1": {
+ "slug_kwd": ["entity/alice"],
+ "content_with_weight": json.dumps({"entity_names": ["Alice"], "error": "429"}),
+ },
+ "row-2": {"slug_kwd": "entity/bob", "content_with_weight": "{}"},
+ }
+
+ async def fake_thread_pool_exec(*args, **kwargs):
+ return object()
+
+ monkeypatch.setattr(_wiki.settings, "docStoreConn", store)
+ monkeypatch.setattr(_wiki, "thread_pool_exec", fake_thread_pool_exec)
+
+ failures = await _wiki._wiki_load_refine_failures("tenant", "kb")
+
+ assert failures == {
+ "entity/alice": {"entity_names": ["Alice"], "error": "429"},
+ "entity/bob": {},
+ }
+
+
@pytest.mark.asyncio
@pytest.mark.parametrize(
"response",
@@ -1143,6 +1169,60 @@ async def test_finalize_auto_links_mentions():
assert google_upd["outlinks_int"] == 1
+async def test_finalize_auto_link_prefers_longest_name_without_nested_links():
+ search_results = [
+ {
+ "_source": {
+ "slug_kwd": "entity/虎牢关",
+ "title_kwd": "虎牢关",
+ "md_with_weight": "虎牢关是关隘。",
+ "page_type_kwd": "entity",
+ }
+ },
+ {
+ "_source": {
+ "slug_kwd": "entity/关",
+ "title_kwd": "关",
+ "md_with_weight": "关是称谓。",
+ "page_type_kwd": "entity",
+ }
+ },
+ {
+ "_source": {
+ "slug_kwd": "concept/虎牢关之战",
+ "title_kwd": "虎牢关之战",
+ "md_with_weight": "虎牢关之战发生在关前。",
+ "page_type_kwd": "concept",
+ }
+ },
+ ]
+ doc_store = make_doc_store(search_results)
+
+ with (
+ patch("common.settings.docStoreConn", doc_store),
+ patch(f"{_wiki.__name__}._load_canonical_entities", new_callable=AsyncMock, return_value={}),
+ ):
+ await _wiki._wiki_finalize(tenant_id="t1", kb_id="kb1", embd_mdl=None)
+
+ update = next(args[0][1] for args in doc_store.update.call_args_list if args[0][0].get("id") == "concept/虎牢关之战")
+ body = update["md_with_weight"]
+ assert "[虎牢关](artifact/kb1/entity/虎牢关)" in body
+ assert "[关](artifact/kb1/entity/关)前" in body
+ assert "[虎牢[" not in body
+
+
+def test_find_unlinked_mention_skips_raw_and_rendered_links():
+ content = "[[entity/虎牢关|虎牢关]]与[虎牢关](artifact/kb1/entity/虎牢关),关前"
+ assert _wiki._wiki_find_unlinked_mention(content, "虎牢关") == -1
+ assert _wiki._wiki_find_unlinked_mention(content, "关") == content.index("关前")
+
+
+def test_find_unlinked_mention_skips_unterminated_raw_link():
+ content = "before [[entity/X|Name"
+ assert _wiki._wiki_find_unlinked_mention(content, "Name") == -1
+
+
@pytest.mark.asyncio
async def test_finalize_preserves_merged_entity_name_as_link_label():
"""A merged page must not replace a member mention with its page title."""
@@ -1186,13 +1266,6 @@ async def test_finalize_preserves_merged_entity_name_as_link_label():
assert "[治世之能臣,乱世之奸雄](artifact/kb1/entity/五色棒)" in update["md_with_weight"]
-def test_inside_wikilink():
- content = "before [[entity/X]] after"
- assert _wiki._inside_wikilink(content, content.index("entity"))
- assert not _wiki._inside_wikilink(content, content.index("before"))
- assert not _wiki._inside_wikilink(content, content.index("after"))
-
-
def test_extract_outlinks_from_content():
"""_wiki_extract_outlinks_from_content derives unique ordered outlinks."""
content = "See [[concept/B]] and [[entity/apple]] and [[concept/B]] again"
diff --git a/test/unit_test/rag/advanced_rag/test_llm_call_pool.py b/test/unit_test/rag/advanced_rag/test_llm_call_pool.py
new file mode 100644
index 0000000000..cfd327c510
--- /dev/null
+++ b/test/unit_test/rag/advanced_rag/test_llm_call_pool.py
@@ -0,0 +1,277 @@
+import asyncio
+
+import pytest
+
+from rag.advanced_rag.knowlege_compile.structure import LLMCallPool
+
+
+class _Clock:
+ def __init__(self):
+ self.now = 0.0
+
+ def __call__(self):
+ return self.now
+
+ def advance(self, seconds: float):
+ self.now += seconds
+
+
+class _ChatModel:
+ def __init__(self, name: str, response: str = "ok"):
+ self.llm_name = name
+ self.model_config = {"llm_factory": "test", "llm_name": name}
+ self.response = response
+
+ async def async_chat(self, system, history, gen_conf=None, **kwargs):
+ return self.response
+
+
+async def _return(value):
+ return value
+
+
+def test_llm_call_pool_halves_model_concurrency_once_per_cooldown():
+ async def exercise():
+ clock = _Clock()
+ pool = LLMCallPool(20, decrease_cooldown=5, rate_limit_retries=0, clock=clock)
+ model = _ChatModel("limited")
+
+ async def limited_call():
+ return "**ERROR**: RATE_LIMIT_EXCEEDED - 429 too many requests"
+
+ await pool.call(limited_call, model_key=pool._model_key(model), priority=10, label="limited")
+ assert pool.concurrency_for(model) == 10
+
+ await pool.call(limited_call, model_key=pool._model_key(model), priority=10, label="limited")
+ assert pool.concurrency_for(model) == 10
+
+ clock.advance(5)
+ await pool.call(limited_call, model_key=pool._model_key(model), priority=10, label="limited")
+ assert pool.concurrency_for(model) == 5
+
+ asyncio.run(exercise())
+
+
+def test_pooled_chat_model_reports_rate_limit_results_to_the_pool():
+ async def exercise():
+ pool = LLMCallPool(8, decrease_cooldown=0, rate_limit_retries=0)
+ model = _ChatModel("wrapped", "**ERROR**: RATE_LIMIT_EXCEEDED")
+
+ result = await pool.wrap(model, priority=10, label="wrapped").async_chat("system", [])
+
+ assert result == "**ERROR**: RATE_LIMIT_EXCEEDED"
+ assert pool.concurrency_for(model) == 4
+
+ asyncio.run(exercise())
+
+
+def test_llm_call_pool_retries_after_lowering_concurrency():
+ class RetryingChatModel(_ChatModel):
+ def __init__(self, name: str):
+ super().__init__(name)
+ self.calls = 0
+
+ async def async_chat(self, system, history, gen_conf=None, **kwargs):
+ self.calls += 1
+ if self.calls == 1:
+ return "**ERROR**: 429 too many requests"
+ return "ok after provider retry"
+
+ async def exercise():
+ changes = []
+ pool = LLMCallPool(
+ 8,
+ decrease_cooldown=0,
+ rate_limit_retry_base_delay=0,
+ on_concurrency_change=lambda old, new, reason: changes.append((old, new, reason)),
+ )
+ model = RetryingChatModel("internally-retried")
+
+ result = await pool.wrap(model, priority=10, label="wrapped").async_chat("system", [])
+
+ assert result == "ok after provider retry"
+ assert model.calls == 2
+ assert pool.concurrency_for(model) == 4
+ assert changes == [(8, 4, "rate limited")]
+
+ asyncio.run(exercise())
+
+
+def test_llm_call_pool_retries_rate_limit_exceptions():
+ async def exercise():
+ calls = 0
+ model = _ChatModel("exception-retry")
+ pool = LLMCallPool(8, decrease_cooldown=0, rate_limit_retry_base_delay=0)
+
+ async def rate_limited_once():
+ nonlocal calls
+ calls += 1
+ if calls == 1:
+ raise RuntimeError("429 too many requests")
+ return "ok"
+
+ result = await pool.call(
+ rate_limited_once,
+ model_key=pool._model_key(model),
+ priority=10,
+ label="exception-retry",
+ )
+
+ assert result == "ok"
+ assert calls == 2
+ assert pool.concurrency_for(model) == 4
+
+ asyncio.run(exercise())
+
+
+def test_llm_call_pool_recovers_concurrency_gradually_after_success():
+ async def exercise():
+ clock = _Clock()
+ pool = LLMCallPool(
+ 8,
+ decrease_cooldown=0,
+ recovery_successes=2,
+ recovery_cooldown=10,
+ rate_limit_retries=0,
+ clock=clock,
+ )
+ model = _ChatModel("recovering")
+ model_key = pool._model_key(model)
+
+ await pool.call(
+ lambda: _return("**ERROR**: RATE_LIMIT_EXCEEDED"),
+ model_key=model_key,
+ priority=10,
+ label="limited",
+ )
+ assert pool.concurrency_for(model) == 4
+
+ clock.advance(10)
+ for _ in range(2):
+ await pool.call(lambda: _return("ok"), model_key=model_key, priority=10, label="success")
+ assert pool.concurrency_for(model) == 5
+
+ for _ in range(2):
+ await pool.call(lambda: _return("ok"), model_key=model_key, priority=10, label="success")
+ assert pool.concurrency_for(model) == 5
+
+ clock.advance(10)
+ await pool.call(lambda: _return("ok"), model_key=model_key, priority=10, label="success")
+ assert pool.concurrency_for(model) == 6
+
+ asyncio.run(exercise())
+
+
+def test_llm_call_pool_reports_concurrency_changes():
+ async def exercise():
+ clock = _Clock()
+ changes = []
+
+ def on_change(old, new, reason):
+ changes.append((old, new, reason, pool._condition.locked()))
+
+ pool = LLMCallPool(
+ 8,
+ decrease_cooldown=0,
+ recovery_successes=1,
+ recovery_cooldown=5,
+ rate_limit_retries=0,
+ clock=clock,
+ on_concurrency_change=on_change,
+ )
+ model = _ChatModel("reported")
+ model_key = pool._model_key(model)
+
+ await pool.call(
+ lambda: _return("**ERROR**: 429 too many requests"),
+ model_key=model_key,
+ priority=10,
+ label="limited",
+ )
+ clock.advance(5)
+ await pool.call(lambda: _return("ok"), model_key=model_key, priority=10, label="success")
+
+ assert changes == [(8, 4, "rate limited", False), (4, 5, "recovered", False)]
+
+ asyncio.run(exercise())
+
+
+def test_llm_call_pool_isolates_models_and_avoids_head_of_line_blocking():
+ async def exercise():
+ pool = LLMCallPool(2, max_pending=4, decrease_cooldown=0, rate_limit_retries=0)
+ limited_model = _ChatModel("limited")
+ healthy_model = _ChatModel("healthy")
+ limited_key = pool._model_key(limited_model)
+ healthy_key = pool._model_key(healthy_model)
+
+ await pool.call(
+ lambda: _return("**ERROR**: 429 too many requests"),
+ model_key=limited_key,
+ priority=10,
+ label="limited",
+ )
+ assert pool.concurrency_for(limited_model) == 1
+ assert pool.concurrency_for(healthy_model) == 2
+
+ release_first = asyncio.Event()
+ first_started = asyncio.Event()
+ second_started = asyncio.Event()
+ healthy_started = asyncio.Event()
+
+ async def first_limited_call():
+ first_started.set()
+ await release_first.wait()
+ return "ok"
+
+ async def second_limited_call():
+ second_started.set()
+ return "ok"
+
+ async def healthy_call():
+ healthy_started.set()
+ return "ok"
+
+ first = asyncio.create_task(pool.call(first_limited_call, model_key=limited_key, priority=10, label="limited-1"))
+ await first_started.wait()
+ second = asyncio.create_task(pool.call(second_limited_call, model_key=limited_key, priority=0, label="limited-2"))
+ healthy = asyncio.create_task(pool.call(healthy_call, model_key=healthy_key, priority=20, label="healthy"))
+
+ await asyncio.wait_for(healthy_started.wait(), timeout=1)
+ assert not second_started.is_set()
+
+ release_first.set()
+ await asyncio.gather(first, second, healthy)
+ assert second_started.is_set()
+
+ asyncio.run(exercise())
+
+
+def test_llm_call_pool_only_backs_off_for_rate_limit_errors():
+ async def exercise():
+ pool = LLMCallPool(8, decrease_cooldown=0, rate_limit_retries=0)
+ model = _ChatModel("errors")
+ model_key = pool._model_key(model)
+
+ with pytest.raises(ValueError, match="invalid request"):
+ await pool.call(
+ lambda: _raise(ValueError("invalid request")),
+ model_key=model_key,
+ priority=10,
+ label="invalid",
+ )
+ assert pool.concurrency_for(model) == 8
+
+ with pytest.raises(RuntimeError, match="429"):
+ await pool.call(
+ lambda: _raise(RuntimeError("429 too many requests")),
+ model_key=model_key,
+ priority=10,
+ label="limited",
+ )
+ assert pool.concurrency_for(model) == 4
+
+ asyncio.run(exercise())
+
+
+async def _raise(exc: Exception):
+ raise exc
diff --git a/test/unit_test/rag/advanced_rag/test_wiki_refine_progress.py b/test/unit_test/rag/advanced_rag/test_wiki_refine_progress.py
new file mode 100644
index 0000000000..64f49d55e4
--- /dev/null
+++ b/test/unit_test/rag/advanced_rag/test_wiki_refine_progress.py
@@ -0,0 +1,19 @@
+import asyncio
+
+from rag.advanced_rag.knowlege_compile.wiki_incremental import _wiki_run_refine_tasks
+
+
+def test_wiki_refine_progress_is_batched_and_reports_completion():
+ async def exercise():
+ messages = []
+
+ async def complete():
+ return None
+
+ await _wiki_run_refine_tasks([complete() for _ in range(391)], messages.append)
+
+ assert len(messages) == 20
+ assert messages[0] == "20/391 pages completed."
+ assert messages[-1] == "391/391 pages completed."
+
+ asyncio.run(exercise())
diff --git a/web/src/interfaces/database/dataset.ts b/web/src/interfaces/database/dataset.ts
index a05afa4d44..fd4ec45063 100644
--- a/web/src/interfaces/database/dataset.ts
+++ b/web/src/interfaces/database/dataset.ts
@@ -286,6 +286,9 @@ export interface IArtifactAlteration {
changed_doc_ids: string[];
involved_doc_ids: string[];
eligible_doc_ids: string[];
+ retry_required?: boolean;
+ retry_page_count?: number;
+ retry_page_slugs?: string[];
}
export interface IArtifactGraphRelation {
diff --git a/web/src/pages/dataset/compilation/dataset-structure-view.tsx b/web/src/pages/dataset/compilation/dataset-structure-view.tsx
index 2167f75c90..ce4477ccf5 100644
--- a/web/src/pages/dataset/compilation/dataset-structure-view.tsx
+++ b/web/src/pages/dataset/compilation/dataset-structure-view.tsx
@@ -66,7 +66,8 @@ export function DatasetStructureView({ kind }: DatasetStructureViewProps) {
const newlyUploaded = alteration?.newly_uploaded ?? 0;
const removed = alteration?.removed ?? 0;
- const hasChanges = newlyUploaded > 0 || removed > 0;
+ const retryPageCount = alteration?.retry_page_count ?? 0;
+ const hasChanges = newlyUploaded > 0 || removed > 0 || retryPageCount > 0;
const handleRunEnd = useCallback(() => {
queryClient.invalidateQueries({
@@ -168,6 +169,7 @@ export function DatasetStructureView({ kind }: DatasetStructureViewProps) {
hasChanges={hasChanges}
newlyUploaded={newlyUploaded}
removed={removed}
+ retryPageCount={retryPageCount}
loading={alterationLoading || runLoading}
tooltip={t('knowledgeCompilation.updateStructureTooltip', {
newlyUploaded,
diff --git a/web/src/pages/dataset/compilation/update-button.tsx b/web/src/pages/dataset/compilation/update-button.tsx
index b064a6c241..d59f6068dd 100644
--- a/web/src/pages/dataset/compilation/update-button.tsx
+++ b/web/src/pages/dataset/compilation/update-button.tsx
@@ -21,6 +21,7 @@ type CompilationUpdateButtonProps = {
newlyUploaded: number;
removed: number;
changed?: number;
+ retryPageCount?: number;
loading: boolean;
tooltip: string;
onClick: () => void;
@@ -33,6 +34,7 @@ export function CompilationUpdateButton({
newlyUploaded,
removed,
changed = 0,
+ retryPageCount = 0,
loading,
tooltip,
onClick,
@@ -74,6 +76,11 @@ export function CompilationUpdateButton({
{changed}
</Badge>
)}
+ {retryPageCount > 0 && (
+ <Badge variant="destructive" className="ml-1">
+ {retryPageCount}
+ </Badge>
+ )}
<WandSparkles />
</>
)}
diff --git a/web/src/pages/dataset/compilation/wiki-left-panel/hooks/use-wiki-update.ts b/web/src/pages/dataset/compilation/wiki-left-panel/hooks/use-wiki-update.ts
index 35cd0547e4..64bf000516 100644
--- a/web/src/pages/dataset/compilation/wiki-left-panel/hooks/use-wiki-update.ts
+++ b/web/src/pages/dataset/compilation/wiki-left-panel/hooks/use-wiki-update.ts
@@ -16,7 +16,9 @@ export function useWikiUpdate({ onUpdate }: UseWikiUpdateOptions = {}) {
const newlyUploaded = data?.newly_uploaded ?? 0;
const removed = data?.removed ?? 0;
const changed = data?.changed ?? 0;
- const hasChanges = newlyUploaded > 0 || removed > 0 || changed > 0;
+ const retryPageCount = data?.retry_page_count ?? 0;
+ const hasChanges =
+ newlyUploaded > 0 || removed > 0 || changed > 0 || retryPageCount > 0;
const handleUpdate = useCallback(async () => {
const result = await runArtifactIndex();
@@ -30,6 +32,7 @@ export function useWikiUpdate({ onUpdate }: UseWikiUpdateOptions = {}) {
newlyUploaded,
removed,
changed,
+ retryPageCount,
handleUpdate,
loading: queryLoading || mutationLoading,
};
diff --git a/web/src/pages/dataset/compilation/wiki-left-panel/index.tsx b/web/src/pages/dataset/compilation/wiki-left-panel/index.tsx
index bb965ae02f..63dbcfc411 100644
--- a/web/src/pages/dataset/compilation/wiki-left-panel/index.tsx
+++ b/web/src/pages/dataset/compilation/wiki-left-panel/index.tsx
@@ -50,6 +50,7 @@ export function WikiLeftPanel({
newlyUploaded,
removed,
changed,
+ retryPageCount,
handleUpdate,
loading: updateLoading,
} = useWikiUpdate();
@@ -74,6 +75,7 @@ export function WikiLeftPanel({
newlyUploaded={newlyUploaded}
removed={removed}
changed={changed}
+ retryPageCount={retryPageCount}
loading={updateLoading}
tooltip={t('knowledgeCompilation.updateTooltip', {
newlyUploaded,