RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(api): return parsing-status counts at the top level of each dataset (#17595) (#17611)
Tai Anb149a392f4a3
diff --git a/api/apps/services/dataset_api_service.py b/api/apps/services/dataset_api_service.pyindex 57fa627362..1c8b2749f4 100644--- a/api/apps/services/dataset_api_service.py+++ b/api/apps/services/dataset_api_service.py@@ -502,7 +502,10 @@ def list_datasets(tenant_id: str, args: dict):user_dict = user_map.get(kb["tenant_id"], {})kb.update({"nickname": user_dict.get("nickname", ""), "tenant_avatar": user_dict.get("avatar", "")})if status_by_kb:- kb["parsing_status"] = status_by_kb.get(kb["id"], {})+ # The documented contract (HTTP API + Python SDK references) places the+ # counts at the top level of each dataset record, not under a nested+ # "parsing_status" object.+ kb.update(status_by_kb.get(kb["id"], {}))response_data_list.append(remap_dictionary_keys(kb))embed_model_names = get_composite_model_name_by_ids([m["embedding_model"] for m in response_data_list])diff --git a/test/unit_test/api/apps/services/test_dataset_api_service_list_datasets.py b/test/unit_test/api/apps/services/test_dataset_api_service_list_datasets.pyindex 52c5d4d197..5c665f6ef7 100644--- a/test/unit_test/api/apps/services/test_dataset_api_service_list_datasets.py+++ b/test/unit_test/api/apps/services/test_dataset_api_service_list_datasets.py@@ -13,14 +13,18 @@# See the License for the specific language governing permissions and# limitations under the License.#-"""Regression tests for list_datasets() honoring include_parsing_status (#16855).+"""Regression tests for list_datasets() honoring include_parsing_status (#16855, #17595).The ``ListDatasetReq`` model declares ``include_parsing_status: bool = False``,but ``dataset_api_service.list_datasets`` historically ignored the flag andreturned no parsing-status counts. These tests lock in the contract that``include_parsing_status`` controls whether``DocumentService.get_parsing_status_by_kb_ids`` is invoked and whether the-result is attached to each kb record.+counts are merged into each kb record.++Per the HTTP API and Python SDK references, the counts belong at the *top level*+of each dataset record (``done_count``, ``fail_count``, ...), not under a nested+``parsing_status`` object (#17595)."""import importlib.util@@ -62,8 +66,8 @@ def _identity_remap(source_data, key_aliases=None):"""Identity stand-in for ``remap_dictionary_keys``.The real helper only renames a few keys (e.g. ``chunk_num`` -> ``chunk_count``)- and otherwise copies through. For these tests we only care that- ``parsing_status`` is preserved on the output record, so identity is enough.+ and otherwise copies through. For these tests we only care that the parsing+ status counts are preserved on the output record, so identity is enough."""if key_aliases is None:return dict(source_data)@@ -198,6 +202,23 @@ def _load_list_datasets_module(monkeypatch, *, kbs, parsing_status_by_kb):return module, parsing_status_mock, get_list_mock+#: Fields ``DocumentService.get_parsing_status_by_kb_ids`` reports, as documented+#: for the List Datasets endpoint.+_PARSING_STATUS_FIELDS = (+ "unstart_count",+ "running_count",+ "cancel_count",+ "done_count",+ "fail_count",+)+++def _assert_no_counts(record, context=""):+ assert "parsing_status" not in record, context+ for field in _PARSING_STATUS_FIELDS:+ assert field not in record, f"{field} {context}"++def _stub_kbs():return [{"id": "kb-a", "tenant_id": "tenant-1", "name": "Alpha", "embedding_model": "emb-a"},@@ -206,7 +227,7 @@ def _stub_kbs():def test_list_datasets_without_include_parsing_status_does_not_call_helper(monkeypatch):- """No flag → no helper call, no parsing_status on response."""+ """No flag → no helper call, no counts on response."""module, parsing_status_mock, get_list_mock = _load_list_datasets_module(monkeypatch,kbs=_stub_kbs(),@@ -219,13 +240,13 @@ def test_list_datasets_without_include_parsing_status_does_not_call_helper(monkeassert payload["total"] == 2assert len(payload["data"]) == 2for record in payload["data"]:- assert "parsing_status" not in record+ _assert_no_counts(record)parsing_status_mock.assert_not_called()get_list_mock.assert_called_once()def test_list_datasets_with_include_parsing_status_true_attaches_counts(monkeypatch):- """Flag True → helper called once with the kb ids, counts attached."""+ """Flag True → helper called once with the kb ids, counts merged at top level."""status_by_kb = {"kb-a": {"unstart_count": 3,@@ -255,10 +276,11 @@ def test_list_datasets_with_include_parsing_status_true_attaches_counts(monkeypaassert ok is Trueparsing_status_mock.assert_called_once_with(["kb-a", "kb-b"])- assert "parsing_status" in payload["data"][0]by_id = {r["id"]: r for r in payload["data"]}- assert by_id["kb-a"]["parsing_status"] == status_by_kb["kb-a"]- assert by_id["kb-b"]["parsing_status"] == status_by_kb["kb-b"]+ for kb_id, counts in status_by_kb.items():+ assert "parsing_status" not in by_id[kb_id]+ for field, value in counts.items():+ assert by_id[kb_id][field] == valuedef test_list_datasets_with_include_parsing_status_string_true(monkeypatch):@@ -279,6 +301,7 @@ def test_list_datasets_with_include_parsing_status_string_true(monkeypatch):assert ok is Trueparsing_status_mock.assert_called_once_with(["kb-a", "kb-b"])+ assert payload["data"][0]["done_count"] == 1def test_list_datasets_with_ids_filters_query_once(monkeypatch):@@ -315,7 +338,7 @@ def test_list_datasets_with_include_parsing_status_false_skips_helper(monkeypatcassert ok is Truefor record in payload["data"]:- assert "parsing_status" not in record+ _assert_no_counts(record)parsing_status_mock.assert_not_called()@@ -335,7 +358,7 @@ def test_list_datasets_with_include_parsing_status_string_false_skips_helper(mon)assert ok is True, falsyfor record in payload["data"]:- assert "parsing_status" not in record, falsy+ _assert_no_counts(record, falsy)parsing_status_mock.assert_not_called()@@ -357,8 +380,8 @@ def test_list_datasets_with_empty_kb_list_skips_helper_even_when_flag_true(monkeparsing_status_mock.assert_not_called()-def test_list_datasets_with_include_parsing_status_missing_kb_gets_empty_dict(monkeypatch):- """If the helper omits a kb_id, the response record gets an empty dict."""+def test_list_datasets_with_include_parsing_status_missing_kb_gets_no_counts(monkeypatch):+ """If the helper omits a kb_id, the response record simply carries no counts."""module, parsing_status_mock, _ = _load_list_datasets_module(monkeypatch,kbs=_stub_kbs(),@@ -374,8 +397,8 @@ def test_list_datasets_with_include_parsing_status_missing_kb_gets_empty_dict(moassert ok is Trueby_id = {r["id"]: r for r in payload["data"]}- assert by_id["kb-a"]["parsing_status"]["unstart_count"] == 1- assert by_id["kb-b"]["parsing_status"] == {}+ assert by_id["kb-a"]["unstart_count"] == 1+ _assert_no_counts(by_id["kb-b"])parsing_status_mock.assert_called_once()