Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix: avoid quadratic dedup in RAG merge paths (#18135)

I Kartik Reddyab40c90118f2
diff --git a/common/misc_utils.py b/common/misc_utils.py
index d9f6b4dbec..9decef3c83 100644
--- a/common/misc_utils.py
+++ b/common/misc_utils.py
@@ -288,3 +288,58 @@ async def thread_pool_exec_long_time(func, *args, **kwargs):
inner = functools.partial(func, *args, **kwargs)
return await loop.run_in_executor(_LONG_TIME_THREAD_POOL_EXECUTOR, ctx.run, inner)
return await loop.run_in_executor(_LONG_TIME_THREAD_POOL_EXECUTOR, ctx.run, func, *args)
+
+
+class _CanonKey:
+ """Wraps a canonicalized structure so it can never collide with an
+ unrelated plain hashable value (e.g. a string equal to another value's
+ repr())."""
+
+ __slots__ = ("_key",)
+
+ def __init__(self, key):
+ self._key = key
+
+ def __eq__(self, other):
+ return isinstance(other, _CanonKey) and self._key == other._key
+
+ def __hash__(self):
+ return hash(self._key)
+
+
+def _canonicalize(value):
+ """Recursively convert JSON-like unhashable values (dict/list/set) into an
+ equality-preserving hashable form: dict equality ignores key order, list
+ equality doesn't. Distinct types that are never equal to each other in
+ Python (list vs. tuple) get distinct tags; types that compare equal by
+ value (set vs. frozenset) share one."""
+ if isinstance(value, dict):
+ return ("__dict__", frozenset((k, _canonicalize(v)) for k, v in value.items()))
+ if isinstance(value, list):
+ return ("__list__", tuple(_canonicalize(v) for v in value))
+ if isinstance(value, tuple):
+ return ("__tuple__", tuple(_canonicalize(v) for v in value))
+ if isinstance(value, (set, frozenset)):
+ return ("__set__", frozenset(_canonicalize(v) for v in value))
+ try:
+ hash(value)
+ return value
+ except TypeError:
+ return ("__repr__", repr(value))
+
+
+def hashable_key(value):
+ """Return a value usable as a set/dict key, falling back to a recursive
+ canonicalization for unhashable (malformed) values instead of raising
+ TypeError.
+
+ Already-hashable values are returned as-is, so canonicalization only ever
+ applies inside an unhashable value. A top-level ``frozenset`` therefore
+ keeps its own key rather than sharing one with an equal ``set``; provenance
+ values (ids, descriptions) are never sets, so this costs nothing on the hot
+ path."""
+ try:
+ hash(value)
+ return value
+ except TypeError:
+ return _CanonKey(_canonicalize(value))
diff --git a/rag/advanced_rag/tree_structured_query_decomposition_retrieval.py b/rag/advanced_rag/tree_structured_query_decomposition_retrieval.py
index 3b481b713a..4f2bf6461b 100644
--- a/rag/advanced_rag/tree_structured_query_decomposition_retrieval.py
+++ b/rag/advanced_rag/tree_structured_query_decomposition_retrieval.py
@@ -17,6 +17,7 @@ import asyncio
import logging
from functools import partial
from api.db.services.llm_service import LLMBundle
+from common.misc_utils import hashable_key
from rag.prompts import kb_prompt
from rag.prompts.generator import sufficiency_check, multi_queries_gen
from rag.utils.web_search_conn import create_web_search_provider
@@ -79,15 +80,19 @@ class TreeStructuredQueryDecompositionRetrieval:
chunk_info[k] = kbinfos[k]
else:
# Merge newly retrieved information, avoiding duplicates
- cids = [c["chunk_id"] for c in chunk_info["chunks"]]
+ cids = {hashable_key(c["chunk_id"]) for c in chunk_info["chunks"]}
for c in kbinfos["chunks"]:
- if c["chunk_id"] not in cids:
+ key = hashable_key(c["chunk_id"])
+ if key not in cids:
chunk_info["chunks"].append(c)
+ cids.add(key)
- dids = [d["doc_id"] for d in chunk_info["doc_aggs"]]
+ dids = {hashable_key(d["doc_id"]) for d in chunk_info["doc_aggs"]}
for d in kbinfos["doc_aggs"]:
- if d["doc_id"] not in dids:
+ key = hashable_key(d["doc_id"])
+ if key not in dids:
chunk_info["doc_aggs"].append(d)
+ dids.add(key)
chunk_info["total"] = chunk_info.get("total", 0) + kbinfos.get("total", 0)
diff --git a/rag/svr/task_executor_refactor/dataset_structure_merger.py b/rag/svr/task_executor_refactor/dataset_structure_merger.py
index ab1ee77812..5fa5e7f67c 100644
--- a/rag/svr/task_executor_refactor/dataset_structure_merger.py
+++ b/rag/svr/task_executor_refactor/dataset_structure_merger.py
@@ -41,7 +41,7 @@ from typing import Optional
import xxhash
from common import settings
-from common.misc_utils import thread_pool_exec
+from common.misc_utils import hashable_key, thread_pool_exec
from rag.advanced_rag.knowlege_compile._common import (
encode as _encode,
)
@@ -350,23 +350,34 @@ async def _merge_bucket(
all_docs: list[str] = []
all_descriptions: list[str] = []
all_original_names: list[str] = []
+ seen_chunks: set[str] = set()
+ seen_docs: set[str] = set()
+ seen_descriptions: set[str] = set()
+ seen_original_names: set[str] = set()
mention_count = 0
for r in rows:
payload = json.loads(r.get("content_with_weight") or "{}")
chunks = r.get("source_chunk_ids") or []
for c in chunks:
- if c not in all_chunks:
+ c_key = hashable_key(c)
+ if c_key not in seen_chunks:
all_chunks.append(c)
+ seen_chunks.add(c_key)
doc = r.get("doc_id") or ""
- if doc and doc not in all_docs:
+ doc_key = hashable_key(doc)
+ if doc and doc_key not in seen_docs:
all_docs.append(doc)
+ seen_docs.add(doc_key)
mention_count += int(r.get("mention_count_int") or payload.get("mention_count", 1))
desc = payload.get("description", "")
- if desc and desc not in all_descriptions:
+ desc_key = hashable_key(desc)
+ if desc and desc_key not in seen_descriptions:
all_descriptions.append(desc)
+ seen_descriptions.add(desc_key)
orig = _struct_entity_name(payload) or ""
- if orig and orig not in all_original_names:
+ if orig and orig not in seen_original_names:
all_original_names.append(orig)
+ seen_original_names.add(orig)
# Use the longest description (most complete) and best original-cased name
best_desc = max(all_descriptions, key=len) if all_descriptions else name_lower
@@ -435,18 +446,27 @@ async def _merge_relations(
all_docs: list[str] = []
# Use the longest description for tokenization
all_desc: list[str] = []
+ seen_chunks: set[str] = set()
+ seen_docs: set[str] = set()
+ seen_desc: set[str] = set()
for r in rows:
payload = json.loads(r.get("content_with_weight") or "{}")
chunks = r.get("source_chunk_ids") or []
for c in chunks:
- if c not in all_chunks:
+ c_key = hashable_key(c)
+ if c_key not in seen_chunks:
all_chunks.append(c)
+ seen_chunks.add(c_key)
doc = r.get("doc_id") or ""
- if doc and doc not in all_docs:
+ doc_key = hashable_key(doc)
+ if doc and doc_key not in seen_docs:
all_docs.append(doc)
+ seen_docs.add(doc_key)
desc = payload.get("description") or f"{src} {rel_type} {tgt}"
- if desc not in all_desc:
+ desc_key = hashable_key(desc)
+ if desc_key not in seen_desc:
all_desc.append(desc)
+ seen_desc.add(desc_key)
best_desc = max(all_desc, key=len)
ltks, sm_ltks = _tokenize_for_search(best_desc)
@@ -547,7 +567,11 @@ async def _cleanup_deleted_docs(
if not cursor:
return True # no markers — nothing to do
- deleted_doc_ids = list(set(cursor.values()))
+ deleted_doc_ids_by_key: dict = {}
+ for v in cursor.values():
+ deleted_doc_ids_by_key.setdefault(hashable_key(v), v)
+ deleted_doc_ids_set = set(deleted_doc_ids_by_key.keys())
+ deleted_doc_ids = list(deleted_doc_ids_by_key.values())
# ── Pass 2: find affected dataset rows and remove ghosts ─────────
dataset_del_cond: dict = {
@@ -587,7 +611,7 @@ async def _cleanup_deleted_docs(
current: list = row.get("doc_ids_kwd") or []
if not isinstance(current, list):
current = []
- remaining = [d for d in current if d not in deleted_doc_ids]
+ remaining = [d for d in current if hashable_key(d) not in deleted_doc_ids_set]
if not remaining:
ids_to_delete.append(row["id"])
elif len(remaining) < len(current):
diff --git a/test/unit_test/common/test_hashable_key.py b/test/unit_test/common/test_hashable_key.py
new file mode 100644
index 0000000000..851629ba2d
--- /dev/null
+++ b/test/unit_test/common/test_hashable_key.py
@@ -0,0 +1,67 @@
+#
+# Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+"""Tests for ``hashable_key``, the set-backed dedup key used by the RAG merge
+paths (dataset_structure_merger, tree_structured_query_decomposition_retrieval).
+Malformed provenance values (lists/dicts where a string was expected) must be
+usable as set members without raising, and without merging values that aren't
+actually equal."""
+
+import pytest
+
+from common.misc_utils import hashable_key
+
+
+def test_hashable_values_pass_through_unchanged():
+ for value in ("c1", 7, None, ("a", "b"), frozenset({"x"})):
+ assert hashable_key(value) == value
+
+
+def test_unhashable_values_are_usable_as_set_members():
+ seen = {hashable_key(["bad", "id"]), hashable_key({"k": "v"})}
+ assert hashable_key(["bad", "id"]) in seen
+ assert hashable_key(["other", "id"]) not in seen
+
+
+def test_dict_key_order_is_irrelevant():
+ assert hashable_key({"a": 1, "b": [2]}) == hashable_key({"b": [2], "a": 1})
+
+
+def test_nested_unequal_dicts_stay_distinct():
+ assert hashable_key({"a": {"b": 1}}) != hashable_key({"a": {"b": 2}})
+
+
+def test_list_and_tuple_stay_distinct():
+ # ["a"] != ("a",) in Python, so their keys must differ too
+ assert hashable_key([{"a": 1}]) != hashable_key(({"a": 1},))
+
+
+def test_nested_set_and_frozenset_share_one_key():
+ # {"a"} == frozenset({"a"}) in Python, so once canonicalized their keys must match
+ assert hashable_key([{"a"}]) == hashable_key([frozenset({"a"})])
+
+
+def test_canonical_key_never_equals_a_plain_string():
+ # the old repr()-based fallback merged an unhashable value with a genuine
+ # string that happened to equal its repr()
+ assert hashable_key(["bad", "id"]) != repr(["bad", "id"])
+ assert hashable_key(["bad", "id"]) != str(["bad", "id"])
diff --git a/test/unit_test/rag/advanced_rag/test_tree_structured_query_decomposition_retrieval.py b/test/unit_test/rag/advanced_rag/test_tree_structured_query_decomposition_retrieval.py
new file mode 100644
index 0000000000..82612188a2
--- /dev/null
+++ b/test/unit_test/rag/advanced_rag/test_tree_structured_query_decomposition_retrieval.py
@@ -0,0 +1,195 @@
+#
+# Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+"""Regression tests for TreeStructuredQueryDecompositionRetrieval._async_update_chunk_info.
+
+The historical implementation rebuilt ``cids``/``dids`` as a plain list on
+every merge round and checked membership with ``in`` — O(n) per item, O(n^2)
+per round. The fix tracks a set alongside the ordered list. These tests pin
+the observable behavior (dedup + relative order across rounds) so the
+quadratic pattern cannot silently come back.
+"""
+
+import pytest
+
+from rag.advanced_rag.tree_structured_query_decomposition_retrieval import (
+ TreeStructuredQueryDecompositionRetrieval,
+)
+
+
+def _retrieval() -> TreeStructuredQueryDecompositionRetrieval:
+ return TreeStructuredQueryDecompositionRetrieval(chat_mdl=None, prompt_config={})
+
+
+def _chunk(cid: str) -> dict:
+ return {"chunk_id": cid, "content_with_weight": f"content-{cid}"}
+
+
+def _doc(did: str) -> dict:
+ return {"doc_id": did, "doc_name": f"name-{did}"}
+
+
+async def test_first_round_assigns_results_directly():
+ retrieval = _retrieval()
+ chunk_info = {"total": 0, "chunks": [], "doc_aggs": []}
+ kbinfos = {"total": 2, "chunks": [_chunk("c1"), _chunk("c2")], "doc_aggs": [_doc("d1")]}
+
+ await retrieval._async_update_chunk_info(chunk_info, kbinfos)
+
+ assert chunk_info["chunks"] == kbinfos["chunks"]
+ assert chunk_info["doc_aggs"] == kbinfos["doc_aggs"]
+ assert chunk_info["total"] == 2
+
+
+async def test_merge_round_dedups_against_existing_ids_and_preserves_order():
+ retrieval = _retrieval()
+ chunk_info = {
+ "total": 2,
+ "chunks": [_chunk("c1"), _chunk("c2")],
+ "doc_aggs": [_doc("d1")],
+ }
+ # c2 already exists (existing-ID overlap), c2 also repeats within this
+ # same incoming batch (duplicate-input overlap) — only c3/c4 and d2 are new.
+ kbinfos = {
+ "total": 3,
+ "chunks": [_chunk("c2"), _chunk("c3"), _chunk("c2"), _chunk("c4")],
+ "doc_aggs": [_doc("d1"), _doc("d2")],
+ }
+
+ await retrieval._async_update_chunk_info(chunk_info, kbinfos)
+
+ assert [c["chunk_id"] for c in chunk_info["chunks"]] == ["c1", "c2", "c3", "c4"]
+ assert [d["doc_id"] for d in chunk_info["doc_aggs"]] == ["d1", "d2"]
+ assert chunk_info["total"] == 5
+
+
+async def test_merge_round_tolerates_unhashable_chunk_and_doc_ids():
+ """Malformed provenance (e.g. a chunk_id/doc_id that's a list/dict instead
+ of a string) must not raise TypeError just because dedup is set-backed."""
+ retrieval = _retrieval()
+ chunk_info = {
+ "total": 1,
+ "chunks": [{"chunk_id": ["nested", "id"], "content_with_weight": "c1"}],
+ "doc_aggs": [{"doc_id": {"bad": "id"}, "doc_name": "d1"}],
+ }
+ kbinfos = {
+ "total": 1,
+ "chunks": [{"chunk_id": ["nested", "id"], "content_with_weight": "dup"}, _chunk("c2")],
+ "doc_aggs": [{"doc_id": {"bad": "id"}, "doc_name": "dup"}, _doc("d2")],
+ }
+
+ await retrieval._async_update_chunk_info(chunk_info, kbinfos)
+
+ # the malformed existing entries are deduped (canonical-key based), new valid ones are appended
+ assert len(chunk_info["chunks"]) == 2
+ assert chunk_info["chunks"][1]["chunk_id"] == "c2"
+ assert len(chunk_info["doc_aggs"]) == 2
+ assert chunk_info["doc_aggs"][1]["doc_id"] == "d2"
+
+
+async def test_merge_round_dedups_dicts_regardless_of_key_order():
+ """dict equality ignores key order, so a dict doc_id/chunk_id with the same
+ keys/values in a different order must still be treated as a duplicate."""
+ retrieval = _retrieval()
+ chunk_info = {
+ "total": 1,
+ "chunks": [{"chunk_id": {"a": 1, "b": 2}, "content_with_weight": "c1"}],
+ "doc_aggs": [],
+ }
+ kbinfos = {
+ "total": 1,
+ # same dict, keys reordered -> must dedup, not be treated as new
+ "chunks": [{"chunk_id": {"b": 2, "a": 1}, "content_with_weight": "dup"}, _chunk("c2")],
+ "doc_aggs": [],
+ }
+
+ await retrieval._async_update_chunk_info(chunk_info, kbinfos)
+
+ assert len(chunk_info["chunks"]) == 2
+ assert chunk_info["chunks"][1]["chunk_id"] == "c2"
+
+
+async def test_merge_round_does_not_collide_string_with_malformed_repr():
+ """A real string chunk_id that happens to equal repr(<malformed value>)
+ must not be treated as a duplicate of that malformed value."""
+ retrieval = _retrieval()
+ malformed = ["nested", "id"]
+ lookalike = repr(malformed) # e.g. "['nested', 'id']"
+ chunk_info = {
+ "total": 1,
+ "chunks": [{"chunk_id": malformed, "content_with_weight": "c1"}],
+ "doc_aggs": [],
+ }
+ kbinfos = {
+ "total": 1,
+ "chunks": [{"chunk_id": lookalike, "content_with_weight": "c2"}],
+ "doc_aggs": [],
+ }
+
+ await retrieval._async_update_chunk_info(chunk_info, kbinfos)
+
+ # both must be kept -> they are not the same chunk despite the repr() collision
+ assert len(chunk_info["chunks"]) == 2
+ assert chunk_info["chunks"][1]["chunk_id"] == lookalike
+
+
+async def test_merge_round_treats_nested_list_and_tuple_as_distinct():
+ """A list and a tuple with equal elements are never equal in Python, so a
+ chunk_id nesting one of each must not be deduped against the other."""
+ retrieval = _retrieval()
+ chunk_info = {
+ "total": 1,
+ "chunks": [{"chunk_id": {"ids": ["a", "b"]}, "content_with_weight": "c1"}],
+ "doc_aggs": [],
+ }
+ kbinfos = {
+ "total": 1,
+ "chunks": [{"chunk_id": {"ids": ("a", "b")}, "content_with_weight": "c2"}],
+ "doc_aggs": [],
+ }
+
+ await retrieval._async_update_chunk_info(chunk_info, kbinfos)
+
+ assert [chunk["chunk_id"]["ids"] for chunk in chunk_info["chunks"]] == [
+ ["a", "b"],
+ ("a", "b"),
+ ]
+
+
+async def test_multiple_merge_rounds_keep_accumulating_without_duplicates():
+ retrieval = _retrieval()
+ chunk_info = {"total": 0, "chunks": [], "doc_aggs": []}
+
+ await retrieval._async_update_chunk_info(chunk_info, {"total": 1, "chunks": [_chunk("c1")], "doc_aggs": [_doc("d1")]})
+ await retrieval._async_update_chunk_info(chunk_info, {"total": 1, "chunks": [_chunk("c1"), _chunk("c2")], "doc_aggs": [_doc("d1"), _doc("d2")]})
+ await retrieval._async_update_chunk_info(chunk_info, {"total": 1, "chunks": [_chunk("c2"), _chunk("c3")], "doc_aggs": [_doc("d2"), _doc("d3")]})
+
+ assert [c["chunk_id"] for c in chunk_info["chunks"]] == ["c1", "c2", "c3"]
+ assert [d["doc_id"] for d in chunk_info["doc_aggs"]] == ["d1", "d2", "d3"]
+ assert chunk_info["total"] == 3
diff --git a/test/unit_test/rag/svr/task_executor_refactor/test_dataset_structure_merger_dedup.py b/test/unit_test/rag/svr/task_executor_refactor/test_dataset_structure_merger_dedup.py
new file mode 100644
index 0000000000..6ab36e6d63
--- /dev/null
+++ b/test/unit_test/rag/svr/task_executor_refactor/test_dataset_structure_merger_dedup.py
@@ -0,0 +1,324 @@
+#
+# Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+"""Regression tests for the quadratic dedup fix in dataset_structure_merger.py.
+
+``_merge_bucket``, ``_merge_relations`` and ``_cleanup_deleted_docs`` used to
+dedup by scanning a plain list (``item not in some_list``) on every row —
+O(n^2) over a bucket/page. The fix tracks a ``seen`` set alongside each
+ordered list. These tests pin dedup + relative-order behavior across
+duplicate/interleaved/malformed inputs so the quadratic pattern (and any
+accidental order regression from a bare ``set()`` swap) cannot come back
+silently.
+"""
+
+import json
+from unittest.mock import AsyncMock, MagicMock, patch
+
+import pytest
+
+from rag.svr.task_executor_refactor.dataset_structure_merger import (
+ _cleanup_deleted_docs,
+ _merge_bucket,
+ _merge_relations,
+)
+
+
+def _entity_row(name: str, *, type_: str = "org", description: str | None = None, chunks=None, doc_id=None, mention_count=None) -> dict:
+ payload = {"name": name, "type": type_}
+ if description is not None:
+ payload["description"] = description
+ row: dict = {"content_with_weight": json.dumps(payload)}
+ if chunks is not None:
+ row["source_chunk_ids"] = chunks
+ if doc_id is not None:
+ row["doc_id"] = doc_id
+ if mention_count is not None:
+ row["mention_count_int"] = mention_count
+ return row
+
+
+async def test_merge_bucket_dedups_chunks_docs_descriptions_and_names_per_group():
+ rows = [
+ _entity_row("Apple Inc", description="A tech company.", chunks=["c1", "c2"], doc_id="docA"),
+ # same group (grouping is by lowercased name + type, no alias merging); duplicate doc_id (docA) and duplicate chunk (c2); longest description should win as "best"
+ _entity_row("APPLE INC", description="Maker of iPhone and iPad devices worldwide.", chunks=["c2", "c3"], doc_id="docA"),
+ # exact duplicate name + description of row 1 (existing-ID overlap); duplicate chunk c1
+ _entity_row("Apple Inc", description="A tech company.", chunks=["c1", "c4"], doc_id="docB"),
+ # malformed row: no chunks/doc_id/description at all, and a name duplicate of row 1 & 3
+ _entity_row("Apple Inc"),
+ # unrelated entity — must not leak into the "apple" group's dedup sets
+ _entity_row("Widget Co", type_="product", chunks=["c5"], doc_id="docC"),
+ ]
+
+ rows_out = await _merge_bucket("tenant1", "kb1", "compile1", None, rows, None)
+
+ by_name_kwd = {r["name_kwd"]: r for r in rows_out}
+ assert set(by_name_kwd) == {"apple inc", "widget co"}
+
+ apple = by_name_kwd["apple inc"]
+ assert apple["source_chunk_ids"] == ["c1", "c2", "c3", "c4"]
+ assert apple["doc_ids_kwd"] == ["docA", "docB"]
+ assert apple["mention_count_int"] == 4 # one implicit mention per row, including the malformed one
+
+ payload_out = json.loads(apple["content_with_weight"])
+ assert payload_out["description"] == "Maker of iPhone and iPad devices worldwide."
+ assert payload_out["name"].lower() == "apple inc" # original casing kept; which casing wins is not pinned
+
+ widget = by_name_kwd["widget co"]
+ assert widget["source_chunk_ids"] == ["c5"]
+ assert widget["doc_ids_kwd"] == ["docC"]
+
+
+async def test_merge_bucket_tolerates_unhashable_chunk_ids_and_descriptions():
+ """A malformed source_chunk_ids entry or description (list/dict instead of
+ a string) must not raise TypeError just because dedup is set-backed."""
+ rows = [
+ _entity_row("Gadget Co", description=["not", "a", "string"], chunks=[["bad", "id"], "c1"], doc_id="docA"),
+ # exact duplicate malformed chunk id + malformed description -> both deduped via repr()
+ _entity_row("Gadget Co", description=["not", "a", "string"], chunks=[["bad", "id"], "c2"], doc_id="docA"),
+ ]
+
+ rows_out = await _merge_bucket("tenant1", "kb1", "compile1", None, rows, None)
+
+ assert len(rows_out) == 1
+ row = rows_out[0]
+ assert row["source_chunk_ids"] == [["bad", "id"], "c1", "c2"]
+
+
+async def test_merge_bucket_dedups_dict_chunks_regardless_of_key_order():
+ """Dict equality ignores key order, so a source_chunk_ids entry that's the
+ same dict with keys reordered must still be treated as a duplicate."""
+ rows = [
+ _entity_row("Widget Co", chunks=[{"a": 1, "b": 2}], doc_id="docA"),
+ _entity_row("Widget Co", chunks=[{"b": 2, "a": 1}, "c2"], doc_id="docA"),
+ ]
+
+ rows_out = await _merge_bucket("tenant1", "kb1", "compile1", None, rows, None)
+
+ assert len(rows_out) == 1
+ assert rows_out[0]["source_chunk_ids"] == [{"a": 1, "b": 2}, "c2"]
+
+
+async def test_merge_bucket_does_not_collide_description_string_with_malformed_repr():
+ """A genuine string description that happens to equal repr(<malformed
+ description>) must not be discarded as a false duplicate."""
+ malformed = ["not", "a", "string"]
+ lookalike = repr(malformed)
+ rows = [
+ _entity_row("Widget Co", description=malformed, chunks=["c1"], doc_id="docA"),
+ _entity_row("Widget Co", description=lookalike, chunks=["c2"], doc_id="docA"),
+ ]
+
+ rows_out = await _merge_bucket("tenant1", "kb1", "compile1", None, rows, None)
+
+ assert len(rows_out) == 1
+ payload_out = json.loads(rows_out[0]["content_with_weight"])
+ # the longer, genuine description must win — it must not have been
+ # dropped because its repr() collided with the malformed value's key
+ assert payload_out["description"] == lookalike
+
+
+async def test_merge_bucket_treats_nested_list_and_tuple_chunk_ids_as_distinct():
+ """A list and a tuple with equal elements are never equal in Python, so a
+ source_chunk_ids entry nesting one of each must not be deduped together."""
+ rows = [
+ _entity_row("Widget Co", chunks=[{"ids": ["a", "b"]}], doc_id="docA"),
+ _entity_row("Widget Co", chunks=[{"ids": ("a", "b")}], doc_id="docA"),
+ ]
+
+ rows_out = await _merge_bucket("tenant1", "kb1", "compile1", None, rows, None)
+
+ assert len(rows_out) == 1
+ assert rows_out[0]["source_chunk_ids"] == [{"ids": ["a", "b"]}, {"ids": ("a", "b")}]
+
+
+def _relation_row(*, from_="apple", to="iphone", type_="produces", chunks=None, doc_id=None, use_row_level_fallback=False) -> dict:
+ if use_row_level_fallback:
+ # "type" stays in the payload (no row-level fallback exists for it);
+ # only "from"/"to" are exercised via from_entity_kwd/to_entity_kwd.
+ row: dict = {"content_with_weight": json.dumps({"type": type_}), "from_entity_kwd": from_, "to_entity_kwd": to}
+ else:
+ row = {"content_with_weight": json.dumps({"from": from_, "to": to, "type": type_})}
+ if chunks is not None:
+ row["source_chunk_ids"] = chunks
+ if doc_id is not None:
+ row["doc_id"] = doc_id
+ return row
+
+
+async def test_merge_relations_dedups_chunks_and_docs_per_group():
+ rows = [
+ _relation_row(chunks=["c1", "c2"], doc_id="docA"),
+ _relation_row(chunks=["c2", "c3"], doc_id="docA"), # duplicate doc + chunk
+ _relation_row(chunks=["c1", "c4"], doc_id="docB"), # duplicate chunk
+ _relation_row(use_row_level_fallback=True), # malformed: no chunks/doc_id, falls back to row-level entity kwds
+ _relation_row(to="ipad", chunks=["c5"], doc_id="docC"), # different group — must stay isolated
+ ]
+
+ rows_out = await _merge_relations("tenant1", "kb1", "compile1", None, rows, None)
+
+ by_target = {r["to_entity_kwd"]: r for r in rows_out}
+ assert set(by_target) == {"iphone", "ipad"}
+
+ iphone = by_target["iphone"]
+ assert iphone["source_chunk_ids"] == ["c1", "c2", "c3", "c4"]
+ assert iphone["doc_ids_kwd"] == ["docA", "docB"]
+
+ ipad = by_target["ipad"]
+ assert ipad["source_chunk_ids"] == ["c5"]
+ assert ipad["doc_ids_kwd"] == ["docC"]
+
+
+async def test_merge_relations_dedups_dict_chunks_regardless_of_key_order():
+ """Malformed source_chunk_ids/doc_id entries that are the same dict with
+ keys reordered must still be treated as duplicates in relation merging."""
+ rows = [
+ _relation_row(chunks=[{"a": 1, "b": 2}], doc_id={"x": 1, "y": 2}),
+ _relation_row(chunks=[{"b": 2, "a": 1}, "c2"], doc_id={"y": 2, "x": 1}),
+ ]
+
+ rows_out = await _merge_relations("tenant1", "kb1", "compile1", None, rows, None)
+
+ assert len(rows_out) == 1
+ row = rows_out[0]
+ assert row["source_chunk_ids"] == [{"a": 1, "b": 2}, "c2"]
+ assert row["doc_ids_kwd"] == [{"x": 1, "y": 2}]
+
+
+def _fake_get_fields(res, fields):
+ return {row["id"]: {f: row.get(f) for f in fields} for row in res}
+
+
+async def test_cleanup_deleted_docs_dedups_deleted_ids_and_preserves_live_docs():
+ # Pass 1: two deletion markers point at the same doc (e.g. a retried
+ # record_doc_deletion call) — the dedup must collapse them.
+ pass1_rows = [
+ {"id": "meta1", "deleted_doc_id": "docA"},
+ {"id": "meta2", "deleted_doc_id": "docA"},
+ {"id": "meta3", "deleted_doc_id": "docB"},
+ ]
+ pass2_rows = [
+ # duplicate + interleaved deleted ids, one live doc left over -> update, order preserved
+ {"id": "entity1", "doc_ids_kwd": ["docA", "docA", "docB", "docC"]},
+ # only deleted docs left -> ghost, delete
+ {"id": "entity2", "doc_ids_kwd": ["docA"]},
+ # malformed provenance (not a list) must not crash the pass
+ {"id": "entity3", "doc_ids_kwd": "docA"},
+ ]
+
+ mock_conn = MagicMock()
+ mock_conn.index_exist.return_value = True
+ mock_conn.search.side_effect = [pass1_rows, pass2_rows]
+ mock_conn.get_fields.side_effect = _fake_get_fields
+
+ mock_settings = MagicMock()
+ mock_settings.docStoreConn = mock_conn
+
+ with (
+ patch("rag.svr.task_executor_refactor.dataset_structure_merger.settings", mock_settings),
+ patch("rag.svr.task_executor_refactor.dataset_structure_merger._index_delete", new_callable=AsyncMock) as mock_index_delete,
+ ):
+ ok = await _cleanup_deleted_docs("tenant1", "kb1", "compile1", None)
+
+ assert ok is True
+ mock_index_delete.assert_awaited_once_with("tenant1", "kb1", {"id": ["entity2", "entity3"]})
+
+ update_calls = [c for c in mock_conn.update.call_args_list]
+ assert len(update_calls) == 1
+ args = update_calls[0].args
+ assert args[0] == {"id": ["entity1"]}
+ assert args[1] == {"doc_ids_kwd": ["docC"]}
+
+
+async def test_cleanup_deleted_docs_tolerates_unhashable_doc_ids_kwd_entries():
+ """A doc_ids_kwd entry that's a list/dict (malformed provenance) must not
+ raise TypeError just because the membership filter is set-backed."""
+ pass1_rows = [{"id": "meta1", "deleted_doc_id": "docA"}]
+ pass2_rows = [
+ {"id": "entity1", "doc_ids_kwd": ["docA", ["bad", "entry"], "docC"]},
+ ]
+
+ mock_conn = MagicMock()
+ mock_conn.index_exist.return_value = True
+ mock_conn.search.side_effect = [pass1_rows, pass2_rows]
+ mock_conn.get_fields.side_effect = _fake_get_fields
+
+ mock_settings = MagicMock()
+ mock_settings.docStoreConn = mock_conn
+
+ with (
+ patch("rag.svr.task_executor_refactor.dataset_structure_merger.settings", mock_settings),
+ patch("rag.svr.task_executor_refactor.dataset_structure_merger._index_delete", new_callable=AsyncMock),
+ ):
+ ok = await _cleanup_deleted_docs("tenant1", "kb1", "compile1", None)
+
+ assert ok is True
+ update_calls = mock_conn.update.call_args_list
+ assert len(update_calls) == 1
+ args = update_calls[0].args
+ assert args[0] == {"id": ["entity1"]}
+ assert args[1] == {"doc_ids_kwd": [["bad", "entry"], "docC"]}
+
+
+async def test_cleanup_deleted_docs_matches_reordered_malformed_doc_id():
+ """A deleted marker doc_id that's a dict must still match a doc_ids_kwd
+ entry holding the same dict with keys in a different order."""
+ pass1_rows = [{"id": "meta1", "deleted_doc_id": {"a": 1, "b": 2}}]
+ pass2_rows = [
+ {"id": "entity1", "doc_ids_kwd": [{"b": 2, "a": 1}, "docC"]},
+ ]
+
+ mock_conn = MagicMock()
+ mock_conn.index_exist.return_value = True
+ mock_conn.search.side_effect = [pass1_rows, pass2_rows]
+ mock_conn.get_fields.side_effect = _fake_get_fields
+
+ mock_settings = MagicMock()
+ mock_settings.docStoreConn = mock_conn
+
+ with (
+ patch("rag.svr.task_executor_refactor.dataset_structure_merger.settings", mock_settings),
+ patch("rag.svr.task_executor_refactor.dataset_structure_merger._index_delete", new_callable=AsyncMock),
+ ):
+ ok = await _cleanup_deleted_docs("tenant1", "kb1", "compile1", None)
+
+ assert ok is True
+ update_calls = mock_conn.update.call_args_list
+ assert len(update_calls) == 1
+ args = update_calls[0].args
+ assert args[0] == {"id": ["entity1"]}
+ # the reordered dict must be recognized as the deleted doc and removed
+ assert args[1] == {"doc_ids_kwd": ["docC"]}