Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(api): return parsing-status counts at the top level of each dataset (#17595) (#17611)

Tai Anb149a392f4a3
diff --git a/api/apps/services/dataset_api_service.py b/api/apps/services/dataset_api_service.py
index 57fa627362..1c8b2749f4 100644
--- a/api/apps/services/dataset_api_service.py
+++ b/api/apps/services/dataset_api_service.py
@@ -502,7 +502,10 @@ def list_datasets(tenant_id: str, args: dict):
user_dict = user_map.get(kb["tenant_id"], {})
kb.update({"nickname": user_dict.get("nickname", ""), "tenant_avatar": user_dict.get("avatar", "")})
if status_by_kb:
- kb["parsing_status"] = status_by_kb.get(kb["id"], {})
+ # The documented contract (HTTP API + Python SDK references) places the
+ # counts at the top level of each dataset record, not under a nested
+ # "parsing_status" object.
+ kb.update(status_by_kb.get(kb["id"], {}))
response_data_list.append(remap_dictionary_keys(kb))
embed_model_names = get_composite_model_name_by_ids([m["embedding_model"] for m in response_data_list])
diff --git a/test/unit_test/api/apps/services/test_dataset_api_service_list_datasets.py b/test/unit_test/api/apps/services/test_dataset_api_service_list_datasets.py
index 52c5d4d197..5c665f6ef7 100644
--- a/test/unit_test/api/apps/services/test_dataset_api_service_list_datasets.py
+++ b/test/unit_test/api/apps/services/test_dataset_api_service_list_datasets.py
@@ -13,14 +13,18 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
-"""Regression tests for list_datasets() honoring include_parsing_status (#16855).
+"""Regression tests for list_datasets() honoring include_parsing_status (#16855, #17595).
The ``ListDatasetReq`` model declares ``include_parsing_status: bool = False``,
but ``dataset_api_service.list_datasets`` historically ignored the flag and
returned no parsing-status counts. These tests lock in the contract that
``include_parsing_status`` controls whether
``DocumentService.get_parsing_status_by_kb_ids`` is invoked and whether the
-result is attached to each kb record.
+counts are merged into each kb record.
+
+Per the HTTP API and Python SDK references, the counts belong at the *top level*
+of each dataset record (``done_count``, ``fail_count``, ...), not under a nested
+``parsing_status`` object (#17595).
"""
import importlib.util
@@ -62,8 +66,8 @@ def _identity_remap(source_data, key_aliases=None):
"""Identity stand-in for ``remap_dictionary_keys``.
The real helper only renames a few keys (e.g. ``chunk_num`` -> ``chunk_count``)
- and otherwise copies through. For these tests we only care that
- ``parsing_status`` is preserved on the output record, so identity is enough.
+ and otherwise copies through. For these tests we only care that the parsing
+ status counts are preserved on the output record, so identity is enough.
"""
if key_aliases is None:
return dict(source_data)
@@ -198,6 +202,23 @@ def _load_list_datasets_module(monkeypatch, *, kbs, parsing_status_by_kb):
return module, parsing_status_mock, get_list_mock
+#: Fields ``DocumentService.get_parsing_status_by_kb_ids`` reports, as documented
+#: for the List Datasets endpoint.
+_PARSING_STATUS_FIELDS = (
+ "unstart_count",
+ "running_count",
+ "cancel_count",
+ "done_count",
+ "fail_count",
+)
+
+
+def _assert_no_counts(record, context=""):
+ assert "parsing_status" not in record, context
+ for field in _PARSING_STATUS_FIELDS:
+ assert field not in record, f"{field} {context}"
+
+
def _stub_kbs():
return [
{"id": "kb-a", "tenant_id": "tenant-1", "name": "Alpha", "embedding_model": "emb-a"},
@@ -206,7 +227,7 @@ def _stub_kbs():
def test_list_datasets_without_include_parsing_status_does_not_call_helper(monkeypatch):
- """No flag → no helper call, no parsing_status on response."""
+ """No flag → no helper call, no counts on response."""
module, parsing_status_mock, get_list_mock = _load_list_datasets_module(
monkeypatch,
kbs=_stub_kbs(),
@@ -219,13 +240,13 @@ def test_list_datasets_without_include_parsing_status_does_not_call_helper(monke
assert payload["total"] == 2
assert len(payload["data"]) == 2
for record in payload["data"]:
- assert "parsing_status" not in record
+ _assert_no_counts(record)
parsing_status_mock.assert_not_called()
get_list_mock.assert_called_once()
def test_list_datasets_with_include_parsing_status_true_attaches_counts(monkeypatch):
- """Flag True → helper called once with the kb ids, counts attached."""
+ """Flag True → helper called once with the kb ids, counts merged at top level."""
status_by_kb = {
"kb-a": {
"unstart_count": 3,
@@ -255,10 +276,11 @@ def test_list_datasets_with_include_parsing_status_true_attaches_counts(monkeypa
assert ok is True
parsing_status_mock.assert_called_once_with(["kb-a", "kb-b"])
- assert "parsing_status" in payload["data"][0]
by_id = {r["id"]: r for r in payload["data"]}
- assert by_id["kb-a"]["parsing_status"] == status_by_kb["kb-a"]
- assert by_id["kb-b"]["parsing_status"] == status_by_kb["kb-b"]
+ for kb_id, counts in status_by_kb.items():
+ assert "parsing_status" not in by_id[kb_id]
+ for field, value in counts.items():
+ assert by_id[kb_id][field] == value
def test_list_datasets_with_include_parsing_status_string_true(monkeypatch):
@@ -279,6 +301,7 @@ def test_list_datasets_with_include_parsing_status_string_true(monkeypatch):
assert ok is True
parsing_status_mock.assert_called_once_with(["kb-a", "kb-b"])
+ assert payload["data"][0]["done_count"] == 1
def test_list_datasets_with_ids_filters_query_once(monkeypatch):
@@ -315,7 +338,7 @@ def test_list_datasets_with_include_parsing_status_false_skips_helper(monkeypatc
assert ok is True
for record in payload["data"]:
- assert "parsing_status" not in record
+ _assert_no_counts(record)
parsing_status_mock.assert_not_called()
@@ -335,7 +358,7 @@ def test_list_datasets_with_include_parsing_status_string_false_skips_helper(mon
)
assert ok is True, falsy
for record in payload["data"]:
- assert "parsing_status" not in record, falsy
+ _assert_no_counts(record, falsy)
parsing_status_mock.assert_not_called()
@@ -357,8 +380,8 @@ def test_list_datasets_with_empty_kb_list_skips_helper_even_when_flag_true(monke
parsing_status_mock.assert_not_called()
-def test_list_datasets_with_include_parsing_status_missing_kb_gets_empty_dict(monkeypatch):
- """If the helper omits a kb_id, the response record gets an empty dict."""
+def test_list_datasets_with_include_parsing_status_missing_kb_gets_no_counts(monkeypatch):
+ """If the helper omits a kb_id, the response record simply carries no counts."""
module, parsing_status_mock, _ = _load_list_datasets_module(
monkeypatch,
kbs=_stub_kbs(),
@@ -374,8 +397,8 @@ def test_list_datasets_with_include_parsing_status_missing_kb_gets_empty_dict(mo
assert ok is True
by_id = {r["id"]: r for r in payload["data"]}
- assert by_id["kb-a"]["parsing_status"]["unstart_count"] == 1
- assert by_id["kb-b"]["parsing_status"] == {}
+ assert by_id["kb-a"]["unstart_count"] == 1
+ _assert_no_counts(by_id["kb-b"])
parsing_status_mock.assert_called_once()