RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
Fix: pipeline with compilation + transfomer (extrator) does not work (#19045)
Wang Qib225b8ec6926
diff --git a/rag/flow/extractor/extractor.py b/rag/flow/extractor/extractor.pyindex df50d84293..0592eccde9 100644--- a/rag/flow/extractor/extractor.py+++ b/rag/flow/extractor/extractor.py@@ -78,13 +78,23 @@ class Extractor(ProcessBase, LLM):self.set_output("output_format", "chunks")self.callback(random.randint(1, 5) / 100.0, "Start to generate.")inputs = self.get_input_elements()- chunks = []+ output_format = kwargs.get("output_format")+ if output_format == "chunks":+ chunks = deepcopy(kwargs.get("chunks") or [])+ elif output_format == "json":+ chunks = deepcopy(kwargs.get("json") or [])+ elif output_format in ["markdown", "text", "html"]:+ payload = kwargs.get(output_format) or ""+ chunks = [{"text": payload}] if payload else []+ else:+ chunks = []chunks_key = ""args = {}for k, v in inputs.items():args[k] = v["value"]if isinstance(args[k], list):- chunks = deepcopy(args[k])+ if not chunks:+ chunks = deepcopy(args[k])chunks_key = kif chunks:@@ -99,7 +109,8 @@ class Extractor(ProcessBase, LLM):prog = 0for i, ck in enumerate(chunks):- args[chunks_key] = ck["text"]+ if chunks_key:+ args[chunks_key] = ck.get("text") or ck.get("content_with_weight") or ""msg, sys_prompt = self._sys_prompt_and_msg([], args)msg.insert(0, {"role": "system", "content": sys_prompt})ck[self._param.field_name] = await self._generate_async(msg)diff --git a/rag/svr/task_executor_refactor/dataflow_service.py b/rag/svr/task_executor_refactor/dataflow_service.pyindex ef4839019b..f878df91e1 100644--- a/rag/svr/task_executor_refactor/dataflow_service.py+++ b/rag/svr/task_executor_refactor/dataflow_service.py@@ -292,7 +292,7 @@ class DataflowService:return None, token_consumption@classmethod- async def _encode_batch(cls, txts: List[str], embedding_model) -> Tuple[np.ndarray, int]:+ def _encode_batch(cls, txts: List[str], embedding_model) -> Tuple[np.ndarray, int]:"""Batch encode texts using the embedding model with truncation."""truncated = EmbeddingUtils.truncate_texts(txts, embedding_model.max_length)return embedding_model.encode(truncated)@@ -307,9 +307,12 @@ class DataflowService:ck["docnm_kwd"] = ctx.nameck["create_time"] = str(datetime.now()).replace("T", " ")[:19]ck["create_timestamp_flt"] = datetime.now().timestamp()+ text = ck.get("text") or ck.get("content_with_weight") or ck.get("summary") or ck.get("questions") or ""+ if not isinstance(text, str):+ text = str(text)if not ck.get("id"):- ck["id"] = xxhash.xxh64((ck["text"] + str(ck["doc_id"])).encode("utf-8")).hexdigest()+ ck["id"] = xxhash.xxh64((text + str(ck["doc_id"])).encode("utf-8")).hexdigest()if "questions" in ck:if "question_tks" not in ck:@@ -334,8 +337,8 @@ class DataflowService:del ck["metadata"]if "content_with_weight" not in ck:- ck["content_with_weight"] = ck["text"] or ""- del ck["text"]+ ck["content_with_weight"] = text+ ck.pop("text", None)if "positions" in ck:add_positions(ck, ck["positions"])diff --git a/rag/svr/task_executor_refactor/embedding_utils.py b/rag/svr/task_executor_refactor/embedding_utils.pyindex 70a2c4cf26..0d9b87e60e 100644--- a/rag/svr/task_executor_refactor/embedding_utils.py+++ b/rag/svr/task_executor_refactor/embedding_utils.py@@ -89,7 +89,7 @@ class EmbeddingUtils:) -> List[str]:"""Prepare texts for dataflow embedding.- Extracts content from 'questions', 'summary', or 'text' fields+ Extracts content from 'questions', 'summary', 'text', or 'content_with_weight' fields(in priority order).Args:@@ -100,7 +100,7 @@ class EmbeddingUtils:"""texts = []for chunk in chunks:- text = chunk.get("questions", chunk.get("summary", chunk.get("text", "")))+ text = chunk.get("questions") or chunk.get("summary") or chunk.get("text") or chunk.get("content_with_weight") or ""texts.append(text)return texts