RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(sdk): stop parsing waits on status lookup failures (#19506)
Wenchao An62df716d971c
diff --git a/docs/references/python_api_reference.md b/docs/references/python_api_reference.mdindex a52a947dc8..d7da3bdfad 100644--- a/docs/references/python_api_reference.md+++ b/docs/references/python_api_reference.md@@ -820,9 +820,9 @@ print("Async bulk parsing initiated.")DataSet.parse_documents(document_ids: list[str]) -> list[tuple[str, str, int, int]]```-*Asynchronously* parses documents in the current dataset.+Starts parsing documents in the current dataset and synchronously waits for the results.-This method encapsulates `async_parse_documents()`. It awaits the completion of all parsing tasks before returning detailed results, including the parsing status and statistics for each document. If a keyboard interruption occurs (e.g., `Ctrl+C`), all pending parsing tasks will be canceled gracefully.+This method calls `async_parse_documents()` and blocks while polling until all requested documents reach a terminal state or report complete progress. It then returns the parsing status and statistics for each document. If a keyboard interruption occurs (e.g., `Ctrl+C`), it requests cancellation for the requested documents and continues polling for their final statuses. If a status request fails or a requested document is no longer found, the method raises an exception instead of continuing to poll.#### Parameters@@ -840,7 +840,7 @@ A list of tuples with detailed parsing results:...]```-- `status`: The final parsing state (e.g., `success`, `failed`, `cancelled`).+- `status`: The final parsing state (e.g., `DONE`, `FAIL`, `CANCEL`). If a document has not reached a terminal state but reports `progress >= 1.0`, its status is returned as `DONE`.- `chunk_count`: The number of content chunks created from the document.- `token_count`: The total number of tokens processed.@@ -858,8 +858,6 @@ try:finished = dataset.parse_documents(ids)for doc_id, status, chunk_count, token_count in finished:print(f"Document {doc_id} parsing finished with status: {status}, chunks: {chunk_count}, tokens: {token_count}")-except KeyboardInterrupt:- print("\nParsing interrupted by user. All pending tasks have been cancelled.")except Exception as e:print(f"Parsing failed: {e}")```diff --git a/sdk/python/ragflow_sdk/modules/dataset.py b/sdk/python/ragflow_sdk/modules/dataset.pyindex ee087cd0bb..8674f2bbc1 100644--- a/sdk/python/ragflow_sdk/modules/dataset.py+++ b/sdk/python/ragflow_sdk/modules/dataset.py@@ -119,17 +119,10 @@ class DataSet(Base):finished = []while pending:for doc_id in list(pending):-- def fetch_doc(doc_id: str) -> Document | None:- try:- docs = self.list_documents(id=doc_id)- return docs[0] if docs else None- except Exception:- return None-- doc = fetch_doc(doc_id)- if doc is None:- continue+ docs = self.list_documents(id=doc_id)+ if not docs:+ raise RuntimeError(f"Document {doc_id} not found while waiting for parsing to finish")+ doc = docs[0]if isinstance(doc.run, str) and doc.run.upper() in terminal_states:finished.append((doc_id, doc.run, doc.chunk_count, doc.token_count))pending.discard(doc_id)@@ -149,7 +142,7 @@ class DataSet(Base):def parse_documents(self, document_ids):try:self.async_parse_documents(document_ids)- self._get_documents_status(document_ids)+ return self._get_documents_status(document_ids)except KeyboardInterrupt:self.async_cancel_parse_documents(document_ids)diff --git a/test/testcases/test_sdk_api/test_file_management_within_dataset/test_parse_documents.py b/test/testcases/test_sdk_api/test_file_management_within_dataset/test_parse_documents.pyindex 63ab7ce012..c739b0b38f 100644--- a/test/testcases/test_sdk_api/test_file_management_within_dataset/test_parse_documents.py+++ b/test/testcases/test_sdk_api/test_file_management_within_dataset/test_parse_documents.py@@ -118,48 +118,61 @@ class TestDocumentsParse:@pytest.mark.p2-def test_get_documents_status_handles_retry_terminal_and_progress_paths(add_dataset_func, monkeypatch):+def test_get_documents_status_handles_terminal_and_progress_paths(add_dataset_func, monkeypatch):+ """Collect terminal states and completed progress without another lookup."""dataset = add_dataset_func- call_counts = {"doc-retry": 0, "doc-progress": 0, "doc-exception": 0}-- def _doc(doc_id, run, chunk_count, token_count, progress):- return Document(- dataset.rag,- {- "id": doc_id,- "dataset_id": dataset.id,- "run": run,- "chunk_count": chunk_count,- "token_count": token_count,- "progress": progress,- },- )+ states = {"doc-done": ("DONE", 0.0), "doc-fail": ("FAIL", -1.0), "doc-cancel": ("CANCEL", 0.3), "doc-progress": ("RUNNING", 1.0)}+ calls = []def _list_documents(id=None, **_kwargs):- if id == "doc-retry":- call_counts["doc-retry"] += 1- if call_counts["doc-retry"] == 1:- return []- return [_doc("doc-retry", "DONE", 3, 5, 0.0)]- if id == "doc-progress":- call_counts["doc-progress"] += 1- return [_doc("doc-progress", "RUNNING", 2, 4, 1.0)]- if id == "doc-exception":- call_counts["doc-exception"] += 1- if call_counts["doc-exception"] == 1:- raise Exception("temporary list failure")- return [_doc("doc-exception", "DONE", 7, 11, 0.0)]+ calls.append(id)+ run, progress = states[id]+ return [Document(dataset.rag, {"id": id, "run": run, "progress": progress, "chunk_count": 2, "token_count": 4})]++ monkeypatch.setattr(dataset, "list_documents", _list_documents)+ monkeypatch.setattr("time.sleep", lambda *_args: pytest.fail("terminal documents must not be polled again"))++ finished = dataset._get_documents_status(list(states))+ assert sorted(finished) == sorted((doc_id, "DONE" if run == "RUNNING" else run, 2, 4) for doc_id, (run, _) in states.items())+ assert sorted(calls) == sorted(states)+++def test_get_documents_status_propagates_lookup_error(add_dataset_func, monkeypatch):+ """Surface the original lookup error instead of silently retrying."""+ dataset = add_dataset_func+ error = RuntimeError("temporary list failure")+ calls = []++ def _list_documents(id=None, **_kwargs):+ calls.append(id)+ raise error++ monkeypatch.setattr(dataset, "list_documents", _list_documents)+ monkeypatch.setattr("time.sleep", lambda *_args: pytest.fail("lookup failures must not be retried"))++ with pytest.raises(RuntimeError, match="temporary list failure") as exc_info:+ dataset._get_documents_status(["doc-1"])+ assert exc_info.value is error+ assert calls == ["doc-1"]+++def test_get_documents_status_raises_for_missing_document(add_dataset_func, monkeypatch):+ """Stop waiting when the requested document is no longer returned."""+ dataset = add_dataset_func+ calls = []++ def _list_documents(id=None, **_kwargs):+ calls.append(id)return []monkeypatch.setattr(dataset, "list_documents", _list_documents)- monkeypatch.setattr("time.sleep", lambda *_args, **_kwargs: None)+ monkeypatch.setattr("time.sleep", lambda *_args: pytest.fail("missing documents must not be retried"))- finished = dataset._get_documents_status(["doc-retry", "doc-progress", "doc-exception"])- assert {item[0] for item in finished} == {"doc-retry", "doc-progress", "doc-exception"}- finished_map = {item[0]: item for item in finished}- assert finished_map["doc-retry"][1] == "DONE"- assert finished_map["doc-progress"][1] == "DONE"- assert finished_map["doc-exception"][1] == "DONE"+ with pytest.raises(RuntimeError, match="Document doc-1 not found"):+ dataset._get_documents_status(["doc-1"])+ assert calls == ["doc-1"]@pytest.mark.p2@@ -191,7 +204,8 @@ def test_parse_documents_keyboard_interrupt_triggers_cancel_then_returns_status(@pytest.mark.p2-def test_parse_documents_happy_path_runs_initial_wait_then_returns_status(add_dataset_func, monkeypatch):+def test_parse_documents_returns_first_completed_status(add_dataset_func, monkeypatch):+ """Return the first wait result without polling the documents twice."""dataset = add_dataset_funcstate = {"status_calls": 0}@@ -207,8 +221,8 @@ def test_parse_documents_happy_path_runs_initial_wait_then_returns_status(add_damonkeypatch.setattr(dataset, "_get_documents_status", _status)status = dataset.parse_documents(["doc-1"])- assert state["status_calls"] == 2- assert status == [("doc-1", "DONE-2", 1, 2)]+ assert state["status_calls"] == 1+ assert status == [("doc-1", "DONE-1", 1, 2)]@pytest.mark.p2diff --git a/test/unit_test/sdk/test_parse_polling.py b/test/unit_test/sdk/test_parse_polling.pynew file mode 100644index 0000000000..0876539f76--- /dev/null+++ b/test/unit_test/sdk/test_parse_polling.py@@ -0,0 +1,178 @@+#+# Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+#++"""Synchronous parsing must stop when a status lookup can no longer succeed."""++import json+import time+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer+from threading import Thread+from urllib.parse import parse_qs, urlsplit++import pytest+import requests+from ragflow_sdk import DataSet, RAGFlow++pytestmark = pytest.mark.p2+++def parsing_api(monkeypatch):+ state = {"responses": {}, "requests": [], "sleeps": 0}++ class Handler(BaseHTTPRequestHandler):+ def log_message(self, *args):+ pass++ def reply(self, payload):+ body = json.dumps(payload).encode() if isinstance(payload, dict) else payload+ self.send_response(200)+ self.send_header("Content-Type", "application/json")+ self.send_header("Content-Length", str(len(body)))+ self.end_headers()+ self.wfile.write(body)++ def do_POST(self):+ self.rfile.read(int(self.headers.get("Content-Length", "0")))+ state["requests"].append(("POST", self.path))+ self.reply({"code": 0})++ def do_GET(self):+ doc_id = parse_qs(urlsplit(self.path).query)["id"][0]+ state["requests"].append(("GET", doc_id))+ replies = state["responses"][doc_id]+ self.reply(replies.pop(0) if len(replies) > 1 else replies[0])++ def bounded_sleep(_seconds):+ state["sleeps"] += 1+ if state["sleeps"] > 2:+ pytest.fail("status polling did not stop after a lookup failure")++ monkeypatch.setattr(time, "sleep", bounded_sleep)+ server = ThreadingHTTPServer(("127.0.0.1", 0), Handler)+ thread = Thread(target=server.serve_forever, daemon=True)+ thread.start()+ try:+ rag = RAGFlow("test-key", f"http://127.0.0.1:{server.server_port}")+ yield DataSet(rag, {"id": "kb"}), state+ finally:+ server.shutdown()+ server.server_close()+ thread.join()+++def document_status(doc_id="doc", run="DONE", progress=1.0):+ return {"code": 0, "data": {"docs": [{"id": doc_id, "run": run, "progress": progress, "chunk_count": 2, "token_count": 10}]}}++[email protected]("message", ["Permission denied", "Dataset not found"])+def test_parse_propagates_status_api_error(parsing_api, message):+ dataset, state = parsing_api+ state["responses"]["doc"] = [{"code": 102, "message": message}]+ with pytest.raises(Exception, match=message):+ dataset.parse_documents(["doc"])+ assert state["requests"] == [("POST", "/api/v1/datasets/kb/chunks"), ("GET", "doc")]+ assert state["sleeps"] == 0+++def test_parse_stops_if_document_disappears(parsing_api):+ dataset, state = parsing_api+ state["responses"]["doc"] = [{"code": 0, "data": {"docs": []}}]+ with pytest.raises(RuntimeError, match="doc.*not found"):+ dataset.parse_documents(["doc"])+ assert state["sleeps"] == 0+++def test_parse_propagates_invalid_status_response(parsing_api):+ dataset, state = parsing_api+ state["responses"]["doc"] = [b"invalid json"]+ with pytest.raises(requests.exceptions.JSONDecodeError):+ dataset.parse_documents(["doc"])+ assert state["sleeps"] == 0+++def test_parse_preserves_transport_error(parsing_api, monkeypatch):+ dataset, state = parsing_api+ error = requests.ConnectionError("connection lost")++ def fail_lookup(**_kwargs):+ raise error++ monkeypatch.setattr(dataset, "list_documents", fail_lookup)+ with pytest.raises(requests.ConnectionError) as exc:+ dataset.parse_documents(["doc"])+ assert exc.value is error+ assert state["sleeps"] == 0++[email protected]("run,progress", [("DONE", 1.0), ("FAIL", -1.0), ("CANCEL", 0.3), ("RUNNING", 1.0)])+def test_parse_returns_first_completed_status_without_polling_again(parsing_api, run, progress):+ dataset, state = parsing_api+ # Once terminal status is observed, a later missing document must not erase+ # the result or restart the wait.+ state["responses"]["doc"] = [document_status(run=run, progress=progress), {"code": 0, "data": {"docs": []}}]+ expected_run = "DONE" if run == "RUNNING" else run+ assert dataset.parse_documents(["doc"]) == [("doc", expected_run, 2, 10)]+ assert state["requests"].count(("GET", "doc")) == 1+++def test_parse_still_polls_running_documents(parsing_api):+ dataset, state = parsing_api+ state["responses"]["doc"] = [document_status(run="RUNNING", progress=0.4), document_status()]+ assert dataset.parse_documents(["doc"]) == [("doc", "DONE", 2, 10)]+ assert state["sleeps"] == 1+ assert state["requests"].count(("GET", "doc")) == 2+++def test_keyboard_interrupt_still_cancels_and_collects_status(monkeypatch):+ dataset = DataSet(None, {"id": "kb"})+ calls = []+ monkeypatch.setattr(dataset, "async_parse_documents", lambda ids: calls.append(("start", ids)))+ monkeypatch.setattr(dataset, "async_cancel_parse_documents", lambda ids: calls.append(("cancel", ids)))++ def status(ids):+ calls.append(("status", ids))+ if len(calls) == 2:+ raise KeyboardInterrupt+ return [("doc", "CANCEL", 0, 0)]++ monkeypatch.setattr(dataset, "_get_documents_status", status)+ assert dataset.parse_documents(["doc"]) == [("doc", "CANCEL", 0, 0)]+ assert calls == [("start", ["doc"]), ("status", ["doc"]), ("cancel", ["doc"]), ("status", ["doc"])]+++def test_parse_does_not_requery_finished_documents_in_a_batch(parsing_api):+ dataset, state = parsing_api+ state["responses"]["finished"] = [document_status("finished")]+ state["responses"]["running"] = [document_status("running", "RUNNING", 0.2), document_status("running")]+ assert sorted(dataset.parse_documents(["finished", "running"])) == [("finished", "DONE", 2, 10), ("running", "DONE", 2, 10)]+ assert state["requests"].count(("GET", "finished")) == 1+ assert state["requests"].count(("GET", "running")) == 2+ assert state["sleeps"] == 1+++def test_parse_start_error_is_not_replaced_by_polling(monkeypatch):+ dataset = DataSet(None, {"id": "kb"})+ error = RuntimeError("parse request rejected")++ def reject_start(_ids):+ raise error++ monkeypatch.setattr(dataset, "async_parse_documents", reject_start)+ monkeypatch.setattr(dataset, "_get_documents_status", lambda _ids: pytest.fail("must not poll after a rejected start"))+ with pytest.raises(RuntimeError) as exc:+ dataset.parse_documents(["doc"])+ assert exc.value is error