RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
feat(task_executor): add MAX_TASKS_PER_WORKER to recycle long-running workers (#14501)
Yuanhao Lyueb244256690c
diff --git a/docker/.env b/docker/.envindex 4813af9cc9..120f6b6150 100644--- a/docker/.env+++ b/docker/.env@@ -438,6 +438,19 @@ LLM_POOL_RATE_LIMIT_RETRIES=3LLM_POOL_RATE_LIMIT_RETRY_BASE_DELAY=1.0LLM_POOL_RATE_LIMIT_RETRY_MAX_DELAY=30.0+# Recycle the task_executor worker process after this many completed tasks, to+# release the ONNX Runtime / CUDA arena fragmentation that builds up over the+# lifetime of the process. The supervisor loop in entrypoint.sh restarts the+# worker automatically after a clean exit. Set a small value (e.g. 20) on+# low-VRAM GPUs where long-running workers eventually run out of memory.+# The threshold is soft: tasks already in flight are allowed to finish, so up to+# MAX_CONCURRENT_TASKS - 1 extra tasks may complete before the worker exits.+# 0 disables recycling.+# MAX_TASKS_PER_WORKER=0+# How long (seconds) a recycling worker waits for in-flight tasks to finish+# before cancelling them. Only has an effect when MAX_TASKS_PER_WORKER > 0.+# RECYCLE_SHUTDOWN_TIMEOUT=300+#Option to disable login form for SSODISABLE_PASSWORD_LOGIN=falsediff --git a/rag/svr/task_executor.py b/rag/svr/task_executor.pyindex dd7bdada6c..c16c44eef8 100644--- a/rag/svr/task_executor.py+++ b/rag/svr/task_executor.py@@ -173,6 +173,23 @@ FAILED_TASKS = 0CURRENT_TASKS = {}WORKER_HEARTBEAT_TIMEOUT = int(os.environ.get("WORKER_HEARTBEAT_TIMEOUT", "120"))+# Recycle the worker process after this many completed tasks to release memory+# that long-lived libraries cannot reclaim on their own -- notably ONNX+# Runtime's BFCArena, which holds every chunk it allocates until the+# InferenceSession is destroyed. The supervisor loop in `docker/entrypoint.sh`+# (`while true; do ... task_executor.py & wait; sleep 1; done`) restarts the+# process automatically after a clean exit.+# 0 disables recycling and preserves the existing behaviour. A small value such+# as 20 helps on lower-VRAM consumer GPUs where cumulative arena fragmentation+# eventually surfaces as "Available memory of 0" allocation failures.+# The threshold is a soft one: tasks already in flight when it is reached are+# allowed to finish, so a worker may complete up to MAX_CONCURRENT_TASKS - 1+# extra tasks before it exits.+MAX_TASKS_PER_WORKER = int(os.environ.get("MAX_TASKS_PER_WORKER", "0"))+# Only used when recycling is enabled: how long to let in-flight task_managers+# finish before cancelling them, so a single hung task cannot block the exit.+RECYCLE_SHUTDOWN_TIMEOUT = float(os.environ.get("RECYCLE_SHUTDOWN_TIMEOUT", "300"))+_completed_task_count = 0stop_event = threading.Event()@@ -1746,12 +1763,18 @@ async def do_handle_task(task):logging.exception(f"Remove doc({task_doc_id}) from docStore failed when task({task_id}) canceled, exception: {e}")-async def handle_task():+async def handle_task() -> bool:+ """Pull one task off the queue and process it.++ Returns True if a real task was processed (success or failure), False if the+ queue was empty and the call was an idle poll, so callers can tell work apart+ from idling and e.g. avoid advancing the recycle counter during quiet periods.+ """global DONE_TASKS, FAILED_TASKSredis_msg, task = await collect()if not task:await asyncio.sleep(5)- return+ return Falselogging.info(f"handle_task begin for task {json.dumps(task)}")@@ -1814,6 +1837,7 @@ async def handle_task():get_recording_context().save_func_return_value("PipelineOperationLogService.record_pipeline_operation", ret)redis_msg.ack()+ return Trueasync def get_server_ip() -> str:@@ -1909,9 +1933,16 @@ async def report_status():async def task_manager():+ global _completed_task_count+ processed = Falsetry:- await handle_task()+ processed = await handle_task()finally:+ if processed and MAX_TASKS_PER_WORKER > 0:+ _completed_task_count += 1+ if _completed_task_count >= MAX_TASKS_PER_WORKER:+ logging.warning(f"[recycle] reached MAX_TASKS_PER_WORKER={MAX_TASKS_PER_WORKER}; signalling a clean exit so the supervisor can restart the worker.")+ stop_event.set()task_limiter.release()@@ -1960,9 +1991,24 @@ async def main():try:while not stop_event.is_set():await task_limiter.acquire()+ if stop_event.is_set():+ # Another task_manager signalled shutdown while we were blocked+ # on the semaphore; bail out cleanly instead of picking up one+ # more task, so the supervisor can restart the worker.+ logging.info("[recycle] stop_event observed after acquiring task_limiter; releasing the semaphore and exiting the main loop.")+ task_limiter.release()+ breakt = asyncio.create_task(task_manager())tasks.append(t)finally:+ if MAX_TASKS_PER_WORKER > 0 and tasks:+ # Recycling is a planned exit, so let in-flight task_managers finish+ # instead of discarding their work -- but bound the wait so a single+ # hung task cannot keep the worker alive forever.+ _, pending = await asyncio.wait(tasks, timeout=RECYCLE_SHUTDOWN_TIMEOUT)+ if pending:+ logging.warning(f"[recycle] {len(pending)} task(s) still running after {RECYCLE_SHUTDOWN_TIMEOUT}s grace; cancelling them.")+ tasks = list(pending)for t in tasks:t.cancel()await asyncio.gather(*tasks, return_exceptions=True)