RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix: avoid quadratic dedup in RAG merge paths (#18135)
I Kartik Reddyab40c90118f2
diff --git a/common/misc_utils.py b/common/misc_utils.pyindex d9f6b4dbec..9decef3c83 100644--- a/common/misc_utils.py+++ b/common/misc_utils.py@@ -288,3 +288,58 @@ async def thread_pool_exec_long_time(func, *args, **kwargs):inner = functools.partial(func, *args, **kwargs)return await loop.run_in_executor(_LONG_TIME_THREAD_POOL_EXECUTOR, ctx.run, inner)return await loop.run_in_executor(_LONG_TIME_THREAD_POOL_EXECUTOR, ctx.run, func, *args)+++class _CanonKey:+ """Wraps a canonicalized structure so it can never collide with an+ unrelated plain hashable value (e.g. a string equal to another value's+ repr())."""++ __slots__ = ("_key",)++ def __init__(self, key):+ self._key = key++ def __eq__(self, other):+ return isinstance(other, _CanonKey) and self._key == other._key++ def __hash__(self):+ return hash(self._key)+++def _canonicalize(value):+ """Recursively convert JSON-like unhashable values (dict/list/set) into an+ equality-preserving hashable form: dict equality ignores key order, list+ equality doesn't. Distinct types that are never equal to each other in+ Python (list vs. tuple) get distinct tags; types that compare equal by+ value (set vs. frozenset) share one."""+ if isinstance(value, dict):+ return ("__dict__", frozenset((k, _canonicalize(v)) for k, v in value.items()))+ if isinstance(value, list):+ return ("__list__", tuple(_canonicalize(v) for v in value))+ if isinstance(value, tuple):+ return ("__tuple__", tuple(_canonicalize(v) for v in value))+ if isinstance(value, (set, frozenset)):+ return ("__set__", frozenset(_canonicalize(v) for v in value))+ try:+ hash(value)+ return value+ except TypeError:+ return ("__repr__", repr(value))+++def hashable_key(value):+ """Return a value usable as a set/dict key, falling back to a recursive+ canonicalization for unhashable (malformed) values instead of raising+ TypeError.++ Already-hashable values are returned as-is, so canonicalization only ever+ applies inside an unhashable value. A top-level ``frozenset`` therefore+ keeps its own key rather than sharing one with an equal ``set``; provenance+ values (ids, descriptions) are never sets, so this costs nothing on the hot+ path."""+ try:+ hash(value)+ return value+ except TypeError:+ return _CanonKey(_canonicalize(value))diff --git a/rag/advanced_rag/tree_structured_query_decomposition_retrieval.py b/rag/advanced_rag/tree_structured_query_decomposition_retrieval.pyindex 3b481b713a..4f2bf6461b 100644--- a/rag/advanced_rag/tree_structured_query_decomposition_retrieval.py+++ b/rag/advanced_rag/tree_structured_query_decomposition_retrieval.py@@ -17,6 +17,7 @@ import asyncioimport loggingfrom functools import partialfrom api.db.services.llm_service import LLMBundle+from common.misc_utils import hashable_keyfrom rag.prompts import kb_promptfrom rag.prompts.generator import sufficiency_check, multi_queries_genfrom rag.utils.web_search_conn import create_web_search_provider@@ -79,15 +80,19 @@ class TreeStructuredQueryDecompositionRetrieval:chunk_info[k] = kbinfos[k]else:# Merge newly retrieved information, avoiding duplicates- cids = [c["chunk_id"] for c in chunk_info["chunks"]]+ cids = {hashable_key(c["chunk_id"]) for c in chunk_info["chunks"]}for c in kbinfos["chunks"]:- if c["chunk_id"] not in cids:+ key = hashable_key(c["chunk_id"])+ if key not in cids:chunk_info["chunks"].append(c)+ cids.add(key)- dids = [d["doc_id"] for d in chunk_info["doc_aggs"]]+ dids = {hashable_key(d["doc_id"]) for d in chunk_info["doc_aggs"]}for d in kbinfos["doc_aggs"]:- if d["doc_id"] not in dids:+ key = hashable_key(d["doc_id"])+ if key not in dids:chunk_info["doc_aggs"].append(d)+ dids.add(key)chunk_info["total"] = chunk_info.get("total", 0) + kbinfos.get("total", 0)diff --git a/rag/svr/task_executor_refactor/dataset_structure_merger.py b/rag/svr/task_executor_refactor/dataset_structure_merger.pyindex ab1ee77812..5fa5e7f67c 100644--- a/rag/svr/task_executor_refactor/dataset_structure_merger.py+++ b/rag/svr/task_executor_refactor/dataset_structure_merger.py@@ -41,7 +41,7 @@ from typing import Optionalimport xxhashfrom common import settings-from common.misc_utils import thread_pool_exec+from common.misc_utils import hashable_key, thread_pool_execfrom rag.advanced_rag.knowlege_compile._common import (encode as _encode,)@@ -350,23 +350,34 @@ async def _merge_bucket(all_docs: list[str] = []all_descriptions: list[str] = []all_original_names: list[str] = []+ seen_chunks: set[str] = set()+ seen_docs: set[str] = set()+ seen_descriptions: set[str] = set()+ seen_original_names: set[str] = set()mention_count = 0for r in rows:payload = json.loads(r.get("content_with_weight") or "{}")chunks = r.get("source_chunk_ids") or []for c in chunks:- if c not in all_chunks:+ c_key = hashable_key(c)+ if c_key not in seen_chunks:all_chunks.append(c)+ seen_chunks.add(c_key)doc = r.get("doc_id") or ""- if doc and doc not in all_docs:+ doc_key = hashable_key(doc)+ if doc and doc_key not in seen_docs:all_docs.append(doc)+ seen_docs.add(doc_key)mention_count += int(r.get("mention_count_int") or payload.get("mention_count", 1))desc = payload.get("description", "")- if desc and desc not in all_descriptions:+ desc_key = hashable_key(desc)+ if desc and desc_key not in seen_descriptions:all_descriptions.append(desc)+ seen_descriptions.add(desc_key)orig = _struct_entity_name(payload) or ""- if orig and orig not in all_original_names:+ if orig and orig not in seen_original_names:all_original_names.append(orig)+ seen_original_names.add(orig)# Use the longest description (most complete) and best original-cased namebest_desc = max(all_descriptions, key=len) if all_descriptions else name_lower@@ -435,18 +446,27 @@ async def _merge_relations(all_docs: list[str] = []# Use the longest description for tokenizationall_desc: list[str] = []+ seen_chunks: set[str] = set()+ seen_docs: set[str] = set()+ seen_desc: set[str] = set()for r in rows:payload = json.loads(r.get("content_with_weight") or "{}")chunks = r.get("source_chunk_ids") or []for c in chunks:- if c not in all_chunks:+ c_key = hashable_key(c)+ if c_key not in seen_chunks:all_chunks.append(c)+ seen_chunks.add(c_key)doc = r.get("doc_id") or ""- if doc and doc not in all_docs:+ doc_key = hashable_key(doc)+ if doc and doc_key not in seen_docs:all_docs.append(doc)+ seen_docs.add(doc_key)desc = payload.get("description") or f"{src} {rel_type} {tgt}"- if desc not in all_desc:+ desc_key = hashable_key(desc)+ if desc_key not in seen_desc:all_desc.append(desc)+ seen_desc.add(desc_key)best_desc = max(all_desc, key=len)ltks, sm_ltks = _tokenize_for_search(best_desc)@@ -547,7 +567,11 @@ async def _cleanup_deleted_docs(if not cursor:return True # no markers — nothing to do- deleted_doc_ids = list(set(cursor.values()))+ deleted_doc_ids_by_key: dict = {}+ for v in cursor.values():+ deleted_doc_ids_by_key.setdefault(hashable_key(v), v)+ deleted_doc_ids_set = set(deleted_doc_ids_by_key.keys())+ deleted_doc_ids = list(deleted_doc_ids_by_key.values())# ── Pass 2: find affected dataset rows and remove ghosts ─────────dataset_del_cond: dict = {@@ -587,7 +611,7 @@ async def _cleanup_deleted_docs(current: list = row.get("doc_ids_kwd") or []if not isinstance(current, list):current = []- remaining = [d for d in current if d not in deleted_doc_ids]+ remaining = [d for d in current if hashable_key(d) not in deleted_doc_ids_set]if not remaining:ids_to_delete.append(row["id"])elif len(remaining) < len(current):diff --git a/test/unit_test/common/test_hashable_key.py b/test/unit_test/common/test_hashable_key.pynew file mode 100644index 0000000000..851629ba2d--- /dev/null+++ b/test/unit_test/common/test_hashable_key.py@@ -0,0 +1,67 @@+#+# Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+#+"""Tests for ``hashable_key``, the set-backed dedup key used by the RAG merge+paths (dataset_structure_merger, tree_structured_query_decomposition_retrieval).+Malformed provenance values (lists/dicts where a string was expected) must be+usable as set members without raising, and without merging values that aren't+actually equal."""++import pytest++from common.misc_utils import hashable_key+++def test_hashable_values_pass_through_unchanged():+ for value in ("c1", 7, None, ("a", "b"), frozenset({"x"})):+ assert hashable_key(value) == value+++def test_unhashable_values_are_usable_as_set_members():+ seen = {hashable_key(["bad", "id"]), hashable_key({"k": "v"})}+ assert hashable_key(["bad", "id"]) in seen+ assert hashable_key(["other", "id"]) not in seen+++def test_dict_key_order_is_irrelevant():+ assert hashable_key({"a": 1, "b": [2]}) == hashable_key({"b": [2], "a": 1})+++def test_nested_unequal_dicts_stay_distinct():+ assert hashable_key({"a": {"b": 1}}) != hashable_key({"a": {"b": 2}})+++def test_list_and_tuple_stay_distinct():+ # ["a"] != ("a",) in Python, so their keys must differ too+ assert hashable_key([{"a": 1}]) != hashable_key(({"a": 1},))+++def test_nested_set_and_frozenset_share_one_key():+ # {"a"} == frozenset({"a"}) in Python, so once canonicalized their keys must match+ assert hashable_key([{"a"}]) == hashable_key([frozenset({"a"})])+++def test_canonical_key_never_equals_a_plain_string():+ # the old repr()-based fallback merged an unhashable value with a genuine+ # string that happened to equal its repr()+ assert hashable_key(["bad", "id"]) != repr(["bad", "id"])+ assert hashable_key(["bad", "id"]) != str(["bad", "id"])diff --git a/test/unit_test/rag/advanced_rag/test_tree_structured_query_decomposition_retrieval.py b/test/unit_test/rag/advanced_rag/test_tree_structured_query_decomposition_retrieval.pynew file mode 100644index 0000000000..82612188a2--- /dev/null+++ b/test/unit_test/rag/advanced_rag/test_tree_structured_query_decomposition_retrieval.py@@ -0,0 +1,195 @@+#+# Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+#+"""Regression tests for TreeStructuredQueryDecompositionRetrieval._async_update_chunk_info.++The historical implementation rebuilt ``cids``/``dids`` as a plain list on+every merge round and checked membership with ``in`` — O(n) per item, O(n^2)+per round. The fix tracks a set alongside the ordered list. These tests pin+the observable behavior (dedup + relative order across rounds) so the+quadratic pattern cannot silently come back.+"""++import pytest++from rag.advanced_rag.tree_structured_query_decomposition_retrieval import (+ TreeStructuredQueryDecompositionRetrieval,+)+++def _retrieval() -> TreeStructuredQueryDecompositionRetrieval:+ return TreeStructuredQueryDecompositionRetrieval(chat_mdl=None, prompt_config={})+++def _chunk(cid: str) -> dict:+ return {"chunk_id": cid, "content_with_weight": f"content-{cid}"}+++def _doc(did: str) -> dict:+ return {"doc_id": did, "doc_name": f"name-{did}"}+++async def test_first_round_assigns_results_directly():+ retrieval = _retrieval()+ chunk_info = {"total": 0, "chunks": [], "doc_aggs": []}+ kbinfos = {"total": 2, "chunks": [_chunk("c1"), _chunk("c2")], "doc_aggs": [_doc("d1")]}++ await retrieval._async_update_chunk_info(chunk_info, kbinfos)++ assert chunk_info["chunks"] == kbinfos["chunks"]+ assert chunk_info["doc_aggs"] == kbinfos["doc_aggs"]+ assert chunk_info["total"] == 2+++async def test_merge_round_dedups_against_existing_ids_and_preserves_order():+ retrieval = _retrieval()+ chunk_info = {+ "total": 2,+ "chunks": [_chunk("c1"), _chunk("c2")],+ "doc_aggs": [_doc("d1")],+ }+ # c2 already exists (existing-ID overlap), c2 also repeats within this+ # same incoming batch (duplicate-input overlap) — only c3/c4 and d2 are new.+ kbinfos = {+ "total": 3,+ "chunks": [_chunk("c2"), _chunk("c3"), _chunk("c2"), _chunk("c4")],+ "doc_aggs": [_doc("d1"), _doc("d2")],+ }++ await retrieval._async_update_chunk_info(chunk_info, kbinfos)++ assert [c["chunk_id"] for c in chunk_info["chunks"]] == ["c1", "c2", "c3", "c4"]+ assert [d["doc_id"] for d in chunk_info["doc_aggs"]] == ["d1", "d2"]+ assert chunk_info["total"] == 5+++async def test_merge_round_tolerates_unhashable_chunk_and_doc_ids():+ """Malformed provenance (e.g. a chunk_id/doc_id that's a list/dict instead+ of a string) must not raise TypeError just because dedup is set-backed."""+ retrieval = _retrieval()+ chunk_info = {+ "total": 1,+ "chunks": [{"chunk_id": ["nested", "id"], "content_with_weight": "c1"}],+ "doc_aggs": [{"doc_id": {"bad": "id"}, "doc_name": "d1"}],+ }+ kbinfos = {+ "total": 1,+ "chunks": [{"chunk_id": ["nested", "id"], "content_with_weight": "dup"}, _chunk("c2")],+ "doc_aggs": [{"doc_id": {"bad": "id"}, "doc_name": "dup"}, _doc("d2")],+ }++ await retrieval._async_update_chunk_info(chunk_info, kbinfos)++ # the malformed existing entries are deduped (canonical-key based), new valid ones are appended+ assert len(chunk_info["chunks"]) == 2+ assert chunk_info["chunks"][1]["chunk_id"] == "c2"+ assert len(chunk_info["doc_aggs"]) == 2+ assert chunk_info["doc_aggs"][1]["doc_id"] == "d2"+++async def test_merge_round_dedups_dicts_regardless_of_key_order():+ """dict equality ignores key order, so a dict doc_id/chunk_id with the same+ keys/values in a different order must still be treated as a duplicate."""+ retrieval = _retrieval()+ chunk_info = {+ "total": 1,+ "chunks": [{"chunk_id": {"a": 1, "b": 2}, "content_with_weight": "c1"}],+ "doc_aggs": [],+ }+ kbinfos = {+ "total": 1,+ # same dict, keys reordered -> must dedup, not be treated as new+ "chunks": [{"chunk_id": {"b": 2, "a": 1}, "content_with_weight": "dup"}, _chunk("c2")],+ "doc_aggs": [],+ }++ await retrieval._async_update_chunk_info(chunk_info, kbinfos)++ assert len(chunk_info["chunks"]) == 2+ assert chunk_info["chunks"][1]["chunk_id"] == "c2"+++async def test_merge_round_does_not_collide_string_with_malformed_repr():+ """A real string chunk_id that happens to equal repr(<malformed value>)+ must not be treated as a duplicate of that malformed value."""+ retrieval = _retrieval()+ malformed = ["nested", "id"]+ lookalike = repr(malformed) # e.g. "['nested', 'id']"+ chunk_info = {+ "total": 1,+ "chunks": [{"chunk_id": malformed, "content_with_weight": "c1"}],+ "doc_aggs": [],+ }+ kbinfos = {+ "total": 1,+ "chunks": [{"chunk_id": lookalike, "content_with_weight": "c2"}],+ "doc_aggs": [],+ }++ await retrieval._async_update_chunk_info(chunk_info, kbinfos)++ # both must be kept -> they are not the same chunk despite the repr() collision+ assert len(chunk_info["chunks"]) == 2+ assert chunk_info["chunks"][1]["chunk_id"] == lookalike+++async def test_merge_round_treats_nested_list_and_tuple_as_distinct():+ """A list and a tuple with equal elements are never equal in Python, so a+ chunk_id nesting one of each must not be deduped against the other."""+ retrieval = _retrieval()+ chunk_info = {+ "total": 1,+ "chunks": [{"chunk_id": {"ids": ["a", "b"]}, "content_with_weight": "c1"}],+ "doc_aggs": [],+ }+ kbinfos = {+ "total": 1,+ "chunks": [{"chunk_id": {"ids": ("a", "b")}, "content_with_weight": "c2"}],+ "doc_aggs": [],+ }++ await retrieval._async_update_chunk_info(chunk_info, kbinfos)++ assert [chunk["chunk_id"]["ids"] for chunk in chunk_info["chunks"]] == [+ ["a", "b"],+ ("a", "b"),+ ]+++async def test_multiple_merge_rounds_keep_accumulating_without_duplicates():+ retrieval = _retrieval()+ chunk_info = {"total": 0, "chunks": [], "doc_aggs": []}++ await retrieval._async_update_chunk_info(chunk_info, {"total": 1, "chunks": [_chunk("c1")], "doc_aggs": [_doc("d1")]})+ await retrieval._async_update_chunk_info(chunk_info, {"total": 1, "chunks": [_chunk("c1"), _chunk("c2")], "doc_aggs": [_doc("d1"), _doc("d2")]})+ await retrieval._async_update_chunk_info(chunk_info, {"total": 1, "chunks": [_chunk("c2"), _chunk("c3")], "doc_aggs": [_doc("d2"), _doc("d3")]})++ assert [c["chunk_id"] for c in chunk_info["chunks"]] == ["c1", "c2", "c3"]+ assert [d["doc_id"] for d in chunk_info["doc_aggs"]] == ["d1", "d2", "d3"]+ assert chunk_info["total"] == 3diff --git a/test/unit_test/rag/svr/task_executor_refactor/test_dataset_structure_merger_dedup.py b/test/unit_test/rag/svr/task_executor_refactor/test_dataset_structure_merger_dedup.pynew file mode 100644index 0000000000..6ab36e6d63--- /dev/null+++ b/test/unit_test/rag/svr/task_executor_refactor/test_dataset_structure_merger_dedup.py@@ -0,0 +1,324 @@+#+# Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+#+"""Regression tests for the quadratic dedup fix in dataset_structure_merger.py.++``_merge_bucket``, ``_merge_relations`` and ``_cleanup_deleted_docs`` used to+dedup by scanning a plain list (``item not in some_list``) on every row —+O(n^2) over a bucket/page. The fix tracks a ``seen`` set alongside each+ordered list. These tests pin dedup + relative-order behavior across+duplicate/interleaved/malformed inputs so the quadratic pattern (and any+accidental order regression from a bare ``set()`` swap) cannot come back+silently.+"""++import json+from unittest.mock import AsyncMock, MagicMock, patch++import pytest++from rag.svr.task_executor_refactor.dataset_structure_merger import (+ _cleanup_deleted_docs,+ _merge_bucket,+ _merge_relations,+)+++def _entity_row(name: str, *, type_: str = "org", description: str | None = None, chunks=None, doc_id=None, mention_count=None) -> dict:+ payload = {"name": name, "type": type_}+ if description is not None:+ payload["description"] = description+ row: dict = {"content_with_weight": json.dumps(payload)}+ if chunks is not None:+ row["source_chunk_ids"] = chunks+ if doc_id is not None:+ row["doc_id"] = doc_id+ if mention_count is not None:+ row["mention_count_int"] = mention_count+ return row+++async def test_merge_bucket_dedups_chunks_docs_descriptions_and_names_per_group():+ rows = [+ _entity_row("Apple Inc", description="A tech company.", chunks=["c1", "c2"], doc_id="docA"),+ # same group (grouping is by lowercased name + type, no alias merging); duplicate doc_id (docA) and duplicate chunk (c2); longest description should win as "best"+ _entity_row("APPLE INC", description="Maker of iPhone and iPad devices worldwide.", chunks=["c2", "c3"], doc_id="docA"),+ # exact duplicate name + description of row 1 (existing-ID overlap); duplicate chunk c1+ _entity_row("Apple Inc", description="A tech company.", chunks=["c1", "c4"], doc_id="docB"),+ # malformed row: no chunks/doc_id/description at all, and a name duplicate of row 1 & 3+ _entity_row("Apple Inc"),+ # unrelated entity — must not leak into the "apple" group's dedup sets+ _entity_row("Widget Co", type_="product", chunks=["c5"], doc_id="docC"),+ ]++ rows_out = await _merge_bucket("tenant1", "kb1", "compile1", None, rows, None)++ by_name_kwd = {r["name_kwd"]: r for r in rows_out}+ assert set(by_name_kwd) == {"apple inc", "widget co"}++ apple = by_name_kwd["apple inc"]+ assert apple["source_chunk_ids"] == ["c1", "c2", "c3", "c4"]+ assert apple["doc_ids_kwd"] == ["docA", "docB"]+ assert apple["mention_count_int"] == 4 # one implicit mention per row, including the malformed one++ payload_out = json.loads(apple["content_with_weight"])+ assert payload_out["description"] == "Maker of iPhone and iPad devices worldwide."+ assert payload_out["name"].lower() == "apple inc" # original casing kept; which casing wins is not pinned++ widget = by_name_kwd["widget co"]+ assert widget["source_chunk_ids"] == ["c5"]+ assert widget["doc_ids_kwd"] == ["docC"]+++async def test_merge_bucket_tolerates_unhashable_chunk_ids_and_descriptions():+ """A malformed source_chunk_ids entry or description (list/dict instead of+ a string) must not raise TypeError just because dedup is set-backed."""+ rows = [+ _entity_row("Gadget Co", description=["not", "a", "string"], chunks=[["bad", "id"], "c1"], doc_id="docA"),+ # exact duplicate malformed chunk id + malformed description -> both deduped via repr()+ _entity_row("Gadget Co", description=["not", "a", "string"], chunks=[["bad", "id"], "c2"], doc_id="docA"),+ ]++ rows_out = await _merge_bucket("tenant1", "kb1", "compile1", None, rows, None)++ assert len(rows_out) == 1+ row = rows_out[0]+ assert row["source_chunk_ids"] == [["bad", "id"], "c1", "c2"]+++async def test_merge_bucket_dedups_dict_chunks_regardless_of_key_order():+ """Dict equality ignores key order, so a source_chunk_ids entry that's the+ same dict with keys reordered must still be treated as a duplicate."""+ rows = [+ _entity_row("Widget Co", chunks=[{"a": 1, "b": 2}], doc_id="docA"),+ _entity_row("Widget Co", chunks=[{"b": 2, "a": 1}, "c2"], doc_id="docA"),+ ]++ rows_out = await _merge_bucket("tenant1", "kb1", "compile1", None, rows, None)++ assert len(rows_out) == 1+ assert rows_out[0]["source_chunk_ids"] == [{"a": 1, "b": 2}, "c2"]+++async def test_merge_bucket_does_not_collide_description_string_with_malformed_repr():+ """A genuine string description that happens to equal repr(<malformed+ description>) must not be discarded as a false duplicate."""+ malformed = ["not", "a", "string"]+ lookalike = repr(malformed)+ rows = [+ _entity_row("Widget Co", description=malformed, chunks=["c1"], doc_id="docA"),+ _entity_row("Widget Co", description=lookalike, chunks=["c2"], doc_id="docA"),+ ]++ rows_out = await _merge_bucket("tenant1", "kb1", "compile1", None, rows, None)++ assert len(rows_out) == 1+ payload_out = json.loads(rows_out[0]["content_with_weight"])+ # the longer, genuine description must win — it must not have been+ # dropped because its repr() collided with the malformed value's key+ assert payload_out["description"] == lookalike+++async def test_merge_bucket_treats_nested_list_and_tuple_chunk_ids_as_distinct():+ """A list and a tuple with equal elements are never equal in Python, so a+ source_chunk_ids entry nesting one of each must not be deduped together."""+ rows = [+ _entity_row("Widget Co", chunks=[{"ids": ["a", "b"]}], doc_id="docA"),+ _entity_row("Widget Co", chunks=[{"ids": ("a", "b")}], doc_id="docA"),+ ]++ rows_out = await _merge_bucket("tenant1", "kb1", "compile1", None, rows, None)++ assert len(rows_out) == 1+ assert rows_out[0]["source_chunk_ids"] == [{"ids": ["a", "b"]}, {"ids": ("a", "b")}]+++def _relation_row(*, from_="apple", to="iphone", type_="produces", chunks=None, doc_id=None, use_row_level_fallback=False) -> dict:+ if use_row_level_fallback:+ # "type" stays in the payload (no row-level fallback exists for it);+ # only "from"/"to" are exercised via from_entity_kwd/to_entity_kwd.+ row: dict = {"content_with_weight": json.dumps({"type": type_}), "from_entity_kwd": from_, "to_entity_kwd": to}+ else:+ row = {"content_with_weight": json.dumps({"from": from_, "to": to, "type": type_})}+ if chunks is not None:+ row["source_chunk_ids"] = chunks+ if doc_id is not None:+ row["doc_id"] = doc_id+ return row+++async def test_merge_relations_dedups_chunks_and_docs_per_group():+ rows = [+ _relation_row(chunks=["c1", "c2"], doc_id="docA"),+ _relation_row(chunks=["c2", "c3"], doc_id="docA"), # duplicate doc + chunk+ _relation_row(chunks=["c1", "c4"], doc_id="docB"), # duplicate chunk+ _relation_row(use_row_level_fallback=True), # malformed: no chunks/doc_id, falls back to row-level entity kwds+ _relation_row(to="ipad", chunks=["c5"], doc_id="docC"), # different group — must stay isolated+ ]++ rows_out = await _merge_relations("tenant1", "kb1", "compile1", None, rows, None)++ by_target = {r["to_entity_kwd"]: r for r in rows_out}+ assert set(by_target) == {"iphone", "ipad"}++ iphone = by_target["iphone"]+ assert iphone["source_chunk_ids"] == ["c1", "c2", "c3", "c4"]+ assert iphone["doc_ids_kwd"] == ["docA", "docB"]++ ipad = by_target["ipad"]+ assert ipad["source_chunk_ids"] == ["c5"]+ assert ipad["doc_ids_kwd"] == ["docC"]+++async def test_merge_relations_dedups_dict_chunks_regardless_of_key_order():+ """Malformed source_chunk_ids/doc_id entries that are the same dict with+ keys reordered must still be treated as duplicates in relation merging."""+ rows = [+ _relation_row(chunks=[{"a": 1, "b": 2}], doc_id={"x": 1, "y": 2}),+ _relation_row(chunks=[{"b": 2, "a": 1}, "c2"], doc_id={"y": 2, "x": 1}),+ ]++ rows_out = await _merge_relations("tenant1", "kb1", "compile1", None, rows, None)++ assert len(rows_out) == 1+ row = rows_out[0]+ assert row["source_chunk_ids"] == [{"a": 1, "b": 2}, "c2"]+ assert row["doc_ids_kwd"] == [{"x": 1, "y": 2}]+++def _fake_get_fields(res, fields):+ return {row["id"]: {f: row.get(f) for f in fields} for row in res}+++async def test_cleanup_deleted_docs_dedups_deleted_ids_and_preserves_live_docs():+ # Pass 1: two deletion markers point at the same doc (e.g. a retried+ # record_doc_deletion call) — the dedup must collapse them.+ pass1_rows = [+ {"id": "meta1", "deleted_doc_id": "docA"},+ {"id": "meta2", "deleted_doc_id": "docA"},+ {"id": "meta3", "deleted_doc_id": "docB"},+ ]+ pass2_rows = [+ # duplicate + interleaved deleted ids, one live doc left over -> update, order preserved+ {"id": "entity1", "doc_ids_kwd": ["docA", "docA", "docB", "docC"]},+ # only deleted docs left -> ghost, delete+ {"id": "entity2", "doc_ids_kwd": ["docA"]},+ # malformed provenance (not a list) must not crash the pass+ {"id": "entity3", "doc_ids_kwd": "docA"},+ ]++ mock_conn = MagicMock()+ mock_conn.index_exist.return_value = True+ mock_conn.search.side_effect = [pass1_rows, pass2_rows]+ mock_conn.get_fields.side_effect = _fake_get_fields++ mock_settings = MagicMock()+ mock_settings.docStoreConn = mock_conn++ with (+ patch("rag.svr.task_executor_refactor.dataset_structure_merger.settings", mock_settings),+ patch("rag.svr.task_executor_refactor.dataset_structure_merger._index_delete", new_callable=AsyncMock) as mock_index_delete,+ ):+ ok = await _cleanup_deleted_docs("tenant1", "kb1", "compile1", None)++ assert ok is True+ mock_index_delete.assert_awaited_once_with("tenant1", "kb1", {"id": ["entity2", "entity3"]})++ update_calls = [c for c in mock_conn.update.call_args_list]+ assert len(update_calls) == 1+ args = update_calls[0].args+ assert args[0] == {"id": ["entity1"]}+ assert args[1] == {"doc_ids_kwd": ["docC"]}+++async def test_cleanup_deleted_docs_tolerates_unhashable_doc_ids_kwd_entries():+ """A doc_ids_kwd entry that's a list/dict (malformed provenance) must not+ raise TypeError just because the membership filter is set-backed."""+ pass1_rows = [{"id": "meta1", "deleted_doc_id": "docA"}]+ pass2_rows = [+ {"id": "entity1", "doc_ids_kwd": ["docA", ["bad", "entry"], "docC"]},+ ]++ mock_conn = MagicMock()+ mock_conn.index_exist.return_value = True+ mock_conn.search.side_effect = [pass1_rows, pass2_rows]+ mock_conn.get_fields.side_effect = _fake_get_fields++ mock_settings = MagicMock()+ mock_settings.docStoreConn = mock_conn++ with (+ patch("rag.svr.task_executor_refactor.dataset_structure_merger.settings", mock_settings),+ patch("rag.svr.task_executor_refactor.dataset_structure_merger._index_delete", new_callable=AsyncMock),+ ):+ ok = await _cleanup_deleted_docs("tenant1", "kb1", "compile1", None)++ assert ok is True+ update_calls = mock_conn.update.call_args_list+ assert len(update_calls) == 1+ args = update_calls[0].args+ assert args[0] == {"id": ["entity1"]}+ assert args[1] == {"doc_ids_kwd": [["bad", "entry"], "docC"]}+++async def test_cleanup_deleted_docs_matches_reordered_malformed_doc_id():+ """A deleted marker doc_id that's a dict must still match a doc_ids_kwd+ entry holding the same dict with keys in a different order."""+ pass1_rows = [{"id": "meta1", "deleted_doc_id": {"a": 1, "b": 2}}]+ pass2_rows = [+ {"id": "entity1", "doc_ids_kwd": [{"b": 2, "a": 1}, "docC"]},+ ]++ mock_conn = MagicMock()+ mock_conn.index_exist.return_value = True+ mock_conn.search.side_effect = [pass1_rows, pass2_rows]+ mock_conn.get_fields.side_effect = _fake_get_fields++ mock_settings = MagicMock()+ mock_settings.docStoreConn = mock_conn++ with (+ patch("rag.svr.task_executor_refactor.dataset_structure_merger.settings", mock_settings),+ patch("rag.svr.task_executor_refactor.dataset_structure_merger._index_delete", new_callable=AsyncMock),+ ):+ ok = await _cleanup_deleted_docs("tenant1", "kb1", "compile1", None)++ assert ok is True+ update_calls = mock_conn.update.call_args_list+ assert len(update_calls) == 1+ args = update_calls[0].args+ assert args[0] == {"id": ["entity1"]}+ # the reordered dict must be recognized as the deleted doc and removed+ assert args[1] == {"doc_ids_kwd": ["docC"]}