Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

Fix: pipeline with compilation + transfomer (extrator) does not work (#19045)

Wang Qib225b8ec6926
diff --git a/rag/flow/extractor/extractor.py b/rag/flow/extractor/extractor.py
index df50d84293..0592eccde9 100644
--- a/rag/flow/extractor/extractor.py
+++ b/rag/flow/extractor/extractor.py
@@ -78,13 +78,23 @@ class Extractor(ProcessBase, LLM):
self.set_output("output_format", "chunks")
self.callback(random.randint(1, 5) / 100.0, "Start to generate.")
inputs = self.get_input_elements()
- chunks = []
+ output_format = kwargs.get("output_format")
+ if output_format == "chunks":
+ chunks = deepcopy(kwargs.get("chunks") or [])
+ elif output_format == "json":
+ chunks = deepcopy(kwargs.get("json") or [])
+ elif output_format in ["markdown", "text", "html"]:
+ payload = kwargs.get(output_format) or ""
+ chunks = [{"text": payload}] if payload else []
+ else:
+ chunks = []
chunks_key = ""
args = {}
for k, v in inputs.items():
args[k] = v["value"]
if isinstance(args[k], list):
- chunks = deepcopy(args[k])
+ if not chunks:
+ chunks = deepcopy(args[k])
chunks_key = k
if chunks:
@@ -99,7 +109,8 @@ class Extractor(ProcessBase, LLM):
prog = 0
for i, ck in enumerate(chunks):
- args[chunks_key] = ck["text"]
+ if chunks_key:
+ args[chunks_key] = ck.get("text") or ck.get("content_with_weight") or ""
msg, sys_prompt = self._sys_prompt_and_msg([], args)
msg.insert(0, {"role": "system", "content": sys_prompt})
ck[self._param.field_name] = await self._generate_async(msg)
diff --git a/rag/svr/task_executor_refactor/dataflow_service.py b/rag/svr/task_executor_refactor/dataflow_service.py
index ef4839019b..f878df91e1 100644
--- a/rag/svr/task_executor_refactor/dataflow_service.py
+++ b/rag/svr/task_executor_refactor/dataflow_service.py
@@ -292,7 +292,7 @@ class DataflowService:
return None, token_consumption
@classmethod
- async def _encode_batch(cls, txts: List[str], embedding_model) -> Tuple[np.ndarray, int]:
+ def _encode_batch(cls, txts: List[str], embedding_model) -> Tuple[np.ndarray, int]:
"""Batch encode texts using the embedding model with truncation."""
truncated = EmbeddingUtils.truncate_texts(txts, embedding_model.max_length)
return embedding_model.encode(truncated)
@@ -307,9 +307,12 @@ class DataflowService:
ck["docnm_kwd"] = ctx.name
ck["create_time"] = str(datetime.now()).replace("T", " ")[:19]
ck["create_timestamp_flt"] = datetime.now().timestamp()
+ text = ck.get("text") or ck.get("content_with_weight") or ck.get("summary") or ck.get("questions") or ""
+ if not isinstance(text, str):
+ text = str(text)
if not ck.get("id"):
- ck["id"] = xxhash.xxh64((ck["text"] + str(ck["doc_id"])).encode("utf-8")).hexdigest()
+ ck["id"] = xxhash.xxh64((text + str(ck["doc_id"])).encode("utf-8")).hexdigest()
if "questions" in ck:
if "question_tks" not in ck:
@@ -334,8 +337,8 @@ class DataflowService:
del ck["metadata"]
if "content_with_weight" not in ck:
- ck["content_with_weight"] = ck["text"] or ""
- del ck["text"]
+ ck["content_with_weight"] = text
+ ck.pop("text", None)
if "positions" in ck:
add_positions(ck, ck["positions"])
diff --git a/rag/svr/task_executor_refactor/embedding_utils.py b/rag/svr/task_executor_refactor/embedding_utils.py
index 70a2c4cf26..0d9b87e60e 100644
--- a/rag/svr/task_executor_refactor/embedding_utils.py
+++ b/rag/svr/task_executor_refactor/embedding_utils.py
@@ -89,7 +89,7 @@ class EmbeddingUtils:
) -> List[str]:
"""Prepare texts for dataflow embedding.
- Extracts content from 'questions', 'summary', or 'text' fields
+ Extracts content from 'questions', 'summary', 'text', or 'content_with_weight' fields
(in priority order).
Args:
@@ -100,7 +100,7 @@ class EmbeddingUtils:
"""
texts = []
for chunk in chunks:
- text = chunk.get("questions", chunk.get("summary", chunk.get("text", "")))
+ text = chunk.get("questions") or chunk.get("summary") or chunk.get("text") or chunk.get("content_with_weight") or ""
texts.append(text)
return texts