Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(sdk): stop parsing waits on status lookup failures (#19506)

Wenchao An62df716d971c
diff --git a/docs/references/python_api_reference.md b/docs/references/python_api_reference.md
index a52a947dc8..d7da3bdfad 100644
--- a/docs/references/python_api_reference.md
+++ b/docs/references/python_api_reference.md
@@ -820,9 +820,9 @@ print("Async bulk parsing initiated.")
DataSet.parse_documents(document_ids: list[str]) -> list[tuple[str, str, int, int]]
```
-*Asynchronously* parses documents in the current dataset.
+Starts parsing documents in the current dataset and synchronously waits for the results.
-This method encapsulates `async_parse_documents()`. It awaits the completion of all parsing tasks before returning detailed results, including the parsing status and statistics for each document. If a keyboard interruption occurs (e.g., `Ctrl+C`), all pending parsing tasks will be canceled gracefully.
+This method calls `async_parse_documents()` and blocks while polling until all requested documents reach a terminal state or report complete progress. It then returns the parsing status and statistics for each document. If a keyboard interruption occurs (e.g., `Ctrl+C`), it requests cancellation for the requested documents and continues polling for their final statuses. If a status request fails or a requested document is no longer found, the method raises an exception instead of continuing to poll.
#### Parameters
@@ -840,7 +840,7 @@ A list of tuples with detailed parsing results:
...
]
```
-- `status`: The final parsing state (e.g., `success`, `failed`, `cancelled`).
+- `status`: The final parsing state (e.g., `DONE`, `FAIL`, `CANCEL`). If a document has not reached a terminal state but reports `progress >= 1.0`, its status is returned as `DONE`.
- `chunk_count`: The number of content chunks created from the document.
- `token_count`: The total number of tokens processed.
@@ -858,8 +858,6 @@ try:
finished = dataset.parse_documents(ids)
for doc_id, status, chunk_count, token_count in finished:
print(f"Document {doc_id} parsing finished with status: {status}, chunks: {chunk_count}, tokens: {token_count}")
-except KeyboardInterrupt:
- print("\nParsing interrupted by user. All pending tasks have been cancelled.")
except Exception as e:
print(f"Parsing failed: {e}")
```
diff --git a/sdk/python/ragflow_sdk/modules/dataset.py b/sdk/python/ragflow_sdk/modules/dataset.py
index ee087cd0bb..8674f2bbc1 100644
--- a/sdk/python/ragflow_sdk/modules/dataset.py
+++ b/sdk/python/ragflow_sdk/modules/dataset.py
@@ -119,17 +119,10 @@ class DataSet(Base):
finished = []
while pending:
for doc_id in list(pending):
-
- def fetch_doc(doc_id: str) -> Document | None:
- try:
- docs = self.list_documents(id=doc_id)
- return docs[0] if docs else None
- except Exception:
- return None
-
- doc = fetch_doc(doc_id)
- if doc is None:
- continue
+ docs = self.list_documents(id=doc_id)
+ if not docs:
+ raise RuntimeError(f"Document {doc_id} not found while waiting for parsing to finish")
+ doc = docs[0]
if isinstance(doc.run, str) and doc.run.upper() in terminal_states:
finished.append((doc_id, doc.run, doc.chunk_count, doc.token_count))
pending.discard(doc_id)
@@ -149,7 +142,7 @@ class DataSet(Base):
def parse_documents(self, document_ids):
try:
self.async_parse_documents(document_ids)
- self._get_documents_status(document_ids)
+ return self._get_documents_status(document_ids)
except KeyboardInterrupt:
self.async_cancel_parse_documents(document_ids)
diff --git a/test/testcases/test_sdk_api/test_file_management_within_dataset/test_parse_documents.py b/test/testcases/test_sdk_api/test_file_management_within_dataset/test_parse_documents.py
index 63ab7ce012..c739b0b38f 100644
--- a/test/testcases/test_sdk_api/test_file_management_within_dataset/test_parse_documents.py
+++ b/test/testcases/test_sdk_api/test_file_management_within_dataset/test_parse_documents.py
@@ -118,48 +118,61 @@ class TestDocumentsParse:
@pytest.mark.p2
-def test_get_documents_status_handles_retry_terminal_and_progress_paths(add_dataset_func, monkeypatch):
+def test_get_documents_status_handles_terminal_and_progress_paths(add_dataset_func, monkeypatch):
+ """Collect terminal states and completed progress without another lookup."""
dataset = add_dataset_func
- call_counts = {"doc-retry": 0, "doc-progress": 0, "doc-exception": 0}
-
- def _doc(doc_id, run, chunk_count, token_count, progress):
- return Document(
- dataset.rag,
- {
- "id": doc_id,
- "dataset_id": dataset.id,
- "run": run,
- "chunk_count": chunk_count,
- "token_count": token_count,
- "progress": progress,
- },
- )
+ states = {"doc-done": ("DONE", 0.0), "doc-fail": ("FAIL", -1.0), "doc-cancel": ("CANCEL", 0.3), "doc-progress": ("RUNNING", 1.0)}
+ calls = []
def _list_documents(id=None, **_kwargs):
- if id == "doc-retry":
- call_counts["doc-retry"] += 1
- if call_counts["doc-retry"] == 1:
- return []
- return [_doc("doc-retry", "DONE", 3, 5, 0.0)]
- if id == "doc-progress":
- call_counts["doc-progress"] += 1
- return [_doc("doc-progress", "RUNNING", 2, 4, 1.0)]
- if id == "doc-exception":
- call_counts["doc-exception"] += 1
- if call_counts["doc-exception"] == 1:
- raise Exception("temporary list failure")
- return [_doc("doc-exception", "DONE", 7, 11, 0.0)]
+ calls.append(id)
+ run, progress = states[id]
+ return [Document(dataset.rag, {"id": id, "run": run, "progress": progress, "chunk_count": 2, "token_count": 4})]
+
+ monkeypatch.setattr(dataset, "list_documents", _list_documents)
+ monkeypatch.setattr("time.sleep", lambda *_args: pytest.fail("terminal documents must not be polled again"))
+
+ finished = dataset._get_documents_status(list(states))
+ assert sorted(finished) == sorted((doc_id, "DONE" if run == "RUNNING" else run, 2, 4) for doc_id, (run, _) in states.items())
+ assert sorted(calls) == sorted(states)
+
+
+def test_get_documents_status_propagates_lookup_error(add_dataset_func, monkeypatch):
+ """Surface the original lookup error instead of silently retrying."""
+ dataset = add_dataset_func
+ error = RuntimeError("temporary list failure")
+ calls = []
+
+ def _list_documents(id=None, **_kwargs):
+ calls.append(id)
+ raise error
+
+ monkeypatch.setattr(dataset, "list_documents", _list_documents)
+ monkeypatch.setattr("time.sleep", lambda *_args: pytest.fail("lookup failures must not be retried"))
+
+ with pytest.raises(RuntimeError, match="temporary list failure") as exc_info:
+ dataset._get_documents_status(["doc-1"])
+ assert exc_info.value is error
+ assert calls == ["doc-1"]
+
+
+def test_get_documents_status_raises_for_missing_document(add_dataset_func, monkeypatch):
+ """Stop waiting when the requested document is no longer returned."""
+ dataset = add_dataset_func
+ calls = []
+
+ def _list_documents(id=None, **_kwargs):
+ calls.append(id)
return []
monkeypatch.setattr(dataset, "list_documents", _list_documents)
- monkeypatch.setattr("time.sleep", lambda *_args, **_kwargs: None)
+ monkeypatch.setattr("time.sleep", lambda *_args: pytest.fail("missing documents must not be retried"))
- finished = dataset._get_documents_status(["doc-retry", "doc-progress", "doc-exception"])
- assert {item[0] for item in finished} == {"doc-retry", "doc-progress", "doc-exception"}
- finished_map = {item[0]: item for item in finished}
- assert finished_map["doc-retry"][1] == "DONE"
- assert finished_map["doc-progress"][1] == "DONE"
- assert finished_map["doc-exception"][1] == "DONE"
+ with pytest.raises(RuntimeError, match="Document doc-1 not found"):
+ dataset._get_documents_status(["doc-1"])
+ assert calls == ["doc-1"]
@pytest.mark.p2
@@ -191,7 +204,8 @@ def test_parse_documents_keyboard_interrupt_triggers_cancel_then_returns_status(
@pytest.mark.p2
-def test_parse_documents_happy_path_runs_initial_wait_then_returns_status(add_dataset_func, monkeypatch):
+def test_parse_documents_returns_first_completed_status(add_dataset_func, monkeypatch):
+ """Return the first wait result without polling the documents twice."""
dataset = add_dataset_func
state = {"status_calls": 0}
@@ -207,8 +221,8 @@ def test_parse_documents_happy_path_runs_initial_wait_then_returns_status(add_da
monkeypatch.setattr(dataset, "_get_documents_status", _status)
status = dataset.parse_documents(["doc-1"])
- assert state["status_calls"] == 2
- assert status == [("doc-1", "DONE-2", 1, 2)]
+ assert state["status_calls"] == 1
+ assert status == [("doc-1", "DONE-1", 1, 2)]
@pytest.mark.p2
diff --git a/test/unit_test/sdk/test_parse_polling.py b/test/unit_test/sdk/test_parse_polling.py
new file mode 100644
index 0000000000..0876539f76
--- /dev/null
+++ b/test/unit_test/sdk/test_parse_polling.py
@@ -0,0 +1,178 @@
+#
+# Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""Synchronous parsing must stop when a status lookup can no longer succeed."""
+
+import json
+import time
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+from threading import Thread
+from urllib.parse import parse_qs, urlsplit
+
+import pytest
+import requests
+from ragflow_sdk import DataSet, RAGFlow
+
+pytestmark = pytest.mark.p2
+
+
+def parsing_api(monkeypatch):
+ state = {"responses": {}, "requests": [], "sleeps": 0}
+
+ class Handler(BaseHTTPRequestHandler):
+ def log_message(self, *args):
+ pass
+
+ def reply(self, payload):
+ body = json.dumps(payload).encode() if isinstance(payload, dict) else payload
+ self.send_response(200)
+ self.send_header("Content-Type", "application/json")
+ self.send_header("Content-Length", str(len(body)))
+ self.end_headers()
+ self.wfile.write(body)
+
+ def do_POST(self):
+ self.rfile.read(int(self.headers.get("Content-Length", "0")))
+ state["requests"].append(("POST", self.path))
+ self.reply({"code": 0})
+
+ def do_GET(self):
+ doc_id = parse_qs(urlsplit(self.path).query)["id"][0]
+ state["requests"].append(("GET", doc_id))
+ replies = state["responses"][doc_id]
+ self.reply(replies.pop(0) if len(replies) > 1 else replies[0])
+
+ def bounded_sleep(_seconds):
+ state["sleeps"] += 1
+ if state["sleeps"] > 2:
+ pytest.fail("status polling did not stop after a lookup failure")
+
+ monkeypatch.setattr(time, "sleep", bounded_sleep)
+ server = ThreadingHTTPServer(("127.0.0.1", 0), Handler)
+ thread = Thread(target=server.serve_forever, daemon=True)
+ thread.start()
+ try:
+ rag = RAGFlow("test-key", f"http://127.0.0.1:{server.server_port}")
+ yield DataSet(rag, {"id": "kb"}), state
+ finally:
+ server.shutdown()
+ server.server_close()
+ thread.join()
+
+
+def document_status(doc_id="doc", run="DONE", progress=1.0):
+ return {"code": 0, "data": {"docs": [{"id": doc_id, "run": run, "progress": progress, "chunk_count": 2, "token_count": 10}]}}
+
+
[email protected]("message", ["Permission denied", "Dataset not found"])
+def test_parse_propagates_status_api_error(parsing_api, message):
+ dataset, state = parsing_api
+ state["responses"]["doc"] = [{"code": 102, "message": message}]
+ with pytest.raises(Exception, match=message):
+ dataset.parse_documents(["doc"])
+ assert state["requests"] == [("POST", "/api/v1/datasets/kb/chunks"), ("GET", "doc")]
+ assert state["sleeps"] == 0
+
+
+def test_parse_stops_if_document_disappears(parsing_api):
+ dataset, state = parsing_api
+ state["responses"]["doc"] = [{"code": 0, "data": {"docs": []}}]
+ with pytest.raises(RuntimeError, match="doc.*not found"):
+ dataset.parse_documents(["doc"])
+ assert state["sleeps"] == 0
+
+
+def test_parse_propagates_invalid_status_response(parsing_api):
+ dataset, state = parsing_api
+ state["responses"]["doc"] = [b"invalid json"]
+ with pytest.raises(requests.exceptions.JSONDecodeError):
+ dataset.parse_documents(["doc"])
+ assert state["sleeps"] == 0
+
+
+def test_parse_preserves_transport_error(parsing_api, monkeypatch):
+ dataset, state = parsing_api
+ error = requests.ConnectionError("connection lost")
+
+ def fail_lookup(**_kwargs):
+ raise error
+
+ monkeypatch.setattr(dataset, "list_documents", fail_lookup)
+ with pytest.raises(requests.ConnectionError) as exc:
+ dataset.parse_documents(["doc"])
+ assert exc.value is error
+ assert state["sleeps"] == 0
+
+
[email protected]("run,progress", [("DONE", 1.0), ("FAIL", -1.0), ("CANCEL", 0.3), ("RUNNING", 1.0)])
+def test_parse_returns_first_completed_status_without_polling_again(parsing_api, run, progress):
+ dataset, state = parsing_api
+ # Once terminal status is observed, a later missing document must not erase
+ # the result or restart the wait.
+ state["responses"]["doc"] = [document_status(run=run, progress=progress), {"code": 0, "data": {"docs": []}}]
+ expected_run = "DONE" if run == "RUNNING" else run
+ assert dataset.parse_documents(["doc"]) == [("doc", expected_run, 2, 10)]
+ assert state["requests"].count(("GET", "doc")) == 1
+
+
+def test_parse_still_polls_running_documents(parsing_api):
+ dataset, state = parsing_api
+ state["responses"]["doc"] = [document_status(run="RUNNING", progress=0.4), document_status()]
+ assert dataset.parse_documents(["doc"]) == [("doc", "DONE", 2, 10)]
+ assert state["sleeps"] == 1
+ assert state["requests"].count(("GET", "doc")) == 2
+
+
+def test_keyboard_interrupt_still_cancels_and_collects_status(monkeypatch):
+ dataset = DataSet(None, {"id": "kb"})
+ calls = []
+ monkeypatch.setattr(dataset, "async_parse_documents", lambda ids: calls.append(("start", ids)))
+ monkeypatch.setattr(dataset, "async_cancel_parse_documents", lambda ids: calls.append(("cancel", ids)))
+
+ def status(ids):
+ calls.append(("status", ids))
+ if len(calls) == 2:
+ raise KeyboardInterrupt
+ return [("doc", "CANCEL", 0, 0)]
+
+ monkeypatch.setattr(dataset, "_get_documents_status", status)
+ assert dataset.parse_documents(["doc"]) == [("doc", "CANCEL", 0, 0)]
+ assert calls == [("start", ["doc"]), ("status", ["doc"]), ("cancel", ["doc"]), ("status", ["doc"])]
+
+
+def test_parse_does_not_requery_finished_documents_in_a_batch(parsing_api):
+ dataset, state = parsing_api
+ state["responses"]["finished"] = [document_status("finished")]
+ state["responses"]["running"] = [document_status("running", "RUNNING", 0.2), document_status("running")]
+ assert sorted(dataset.parse_documents(["finished", "running"])) == [("finished", "DONE", 2, 10), ("running", "DONE", 2, 10)]
+ assert state["requests"].count(("GET", "finished")) == 1
+ assert state["requests"].count(("GET", "running")) == 2
+ assert state["sleeps"] == 1
+
+
+def test_parse_start_error_is_not_replaced_by_polling(monkeypatch):
+ dataset = DataSet(None, {"id": "kb"})
+ error = RuntimeError("parse request rejected")
+
+ def reject_start(_ids):
+ raise error
+
+ monkeypatch.setattr(dataset, "async_parse_documents", reject_start)
+ monkeypatch.setattr(dataset, "_get_documents_status", lambda _ids: pytest.fail("must not poll after a rejected start"))
+ with pytest.raises(RuntimeError) as exc:
+ dataset.parse_documents(["doc"])
+ assert exc.value is error