Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

Fix task time is incorrect (#18453)

Wang Qi858f882d21b4
diff --git a/rag/svr/task_executor.py b/rag/svr/task_executor.py
index 0ad83449fe..7e1c397bfc 100644
--- a/rag/svr/task_executor.py
+++ b/rag/svr/task_executor.py
@@ -297,7 +297,7 @@ async def get_storage_binary(bucket, name):
@timed_with_recording
@timeout(60 * 80, 1)
-async def build_chunks(task, progress_callback):
+async def build_chunks(task, progress_callback, on_chunking_start=None):
if task["size"] > settings.DOC_MAXIMUM_SIZE:
set_progress(task["id"], prog=-1, msg="File size exceeds( <= %dMb )" % (int(settings.DOC_MAXIMUM_SIZE / 1024 / 1024)))
get_recording_context().record("file_size_exceeded", True)
@@ -352,7 +352,10 @@ async def build_chunks(task, progress_callback):
get_recording_context().record("parser_config_after_merge", parser_config_for_chunk)
try:
+ chunking_wait_started_at = timer()
async with chunk_limiter:
+ if on_chunking_start:
+ on_chunking_start(timer() - chunking_wait_started_at)
task_language = task.get("language") or "Chinese"
cks = await thread_pool_exec(
chunker.chunk,
@@ -1584,8 +1587,13 @@ async def do_handle_task(task):
else:
# Standard chunking methods
task["llm_id"] = doc_task_llm_id
+
+ def on_chunking_start(wait_time):
+ nonlocal task_start_ts
+ task_start_ts += wait_time
+
start_ts = timer()
- chunks = await build_chunks(task, progress_callback)
+ chunks = await build_chunks(task, progress_callback, on_chunking_start)
get_recording_context().record("chunks", chunks)
# Record chunk_ids_count for comparison
chunk_ids = [c.get("id") for c in chunks if isinstance(c, dict) and "id" in c]
diff --git a/rag/svr/task_executor_refactor/chunk_builder.py b/rag/svr/task_executor_refactor/chunk_builder.py
index 51d679df3a..7bcd44a16d 100644
--- a/rag/svr/task_executor_refactor/chunk_builder.py
+++ b/rag/svr/task_executor_refactor/chunk_builder.py
@@ -72,6 +72,7 @@ async def run_chunking(
chunker,
binary: bytes,
ctx: TaskContext,
+ on_chunking_start=None,
) -> List[Dict]:
"""Run document chunking via parser.
@@ -88,7 +89,10 @@ async def run_chunking(
# Merge table parser config
parser_config = merge_table_parser_config_from_kb(ctx.raw_task)
+ chunking_wait_started_at = timer()
async with ctx.chunk_limiter:
+ if on_chunking_start:
+ on_chunking_start(timer() - chunking_wait_started_at)
cks = await thread_pool_exec(
chunker.chunk,
ctx.name,
diff --git a/rag/svr/task_executor_refactor/chunk_service.py b/rag/svr/task_executor_refactor/chunk_service.py
index 67c1679bec..bf5e3d3016 100644
--- a/rag/svr/task_executor_refactor/chunk_service.py
+++ b/rag/svr/task_executor_refactor/chunk_service.py
@@ -91,6 +91,7 @@ class ChunkService:
async def build_chunks(
self,
storage_binary: bytes,
+ on_chunking_start=None,
) -> List[Dict[str, Any]]:
"""Build chunks from document binary.
@@ -133,7 +134,7 @@ class ChunkService:
ctx.recording_context.record("chunk_config", chunk_config)
# Run chunking (delegated)
- cks = await run_chunking(chunker, storage_binary, ctx)
+ cks = await run_chunking(chunker, storage_binary, ctx, on_chunking_start)
# Record raw chunks
self._task_context.recording_context.record("raw_chunks", cks)
diff --git a/rag/svr/task_executor_refactor/task_handler.py b/rag/svr/task_executor_refactor/task_handler.py
index 0b46e16d18..dc92a55de6 100644
--- a/rag/svr/task_executor_refactor/task_handler.py
+++ b/rag/svr/task_executor_refactor/task_handler.py
@@ -577,6 +577,11 @@ class TaskHandler:
task_dataset_id = ctx.kb_id
task_doc_id = ctx.doc_id
task_start_ts = timer()
+
+ def on_chunking_start(wait_time):
+ nonlocal task_start_ts
+ task_start_ts += wait_time
+
doc_task_llm_id = ctx.parser_config.get("llm_id") or ctx.llm_id
ctx.raw_task["llm_id"] = doc_task_llm_id
@@ -590,7 +595,7 @@ class TaskHandler:
if binary is None:
raise FileNotFoundError(f"Can not find file <{ctx.name}> from minio. Could you try it again.")
- chunks = await chunk_service.build_chunks(binary)
+ chunks = await chunk_service.build_chunks(binary, on_chunking_start)
ctx.recording_context.record("chunks", chunks)
chunk_ids = [c.get("id") for c in chunks if isinstance(c, dict) and "id" in c]
ctx.recording_context.record("chunk_ids_count", len(chunk_ids))