Skip to content

armgit / LEANN

Public Imported from GitHub
0

[MLsys2026 Best Paper]: https://arxiv.org/abs/2506.08276. RAG on Everything with LEANN. Enjoy 97% storage savings while running a fast, accurate, and 100% private RAG application on your personal device.

fix(sync): hash multi-document files for change detection (#290)

Aakash Suresh7b2c7341ffc8
diff --git a/packages/leann-core/src/leann/sync.py b/packages/leann-core/src/leann/sync.py
index d0a69e5..d944477 100644
--- a/packages/leann-core/src/leann/sync.py
+++ b/packages/leann-core/src/leann/sync.py
@@ -107,15 +107,19 @@ class FileSynchronizer:
# print('reader.iter_data() length', len(list(reader.iter_data())))
for file in reader.iter_data():
- if len(file) > 1:
- # print('file length is greater than 1', file)
- continue # SimpleDirectoryReader can load more than 1 documents for weird file types e.g. PDFs
- file = file[0]
+ if not file:
+ continue
+ file_path = file[0].metadata.get("file_path", "")
+ if not file_path:
+ continue
try:
- file_hash = hash_data(file.text)
- file_hashes[file.metadata["file_path"]] = file_hash
+ # Combine text from all documents for this file (e.g. multi-page PDFs).
+ # Previously we skipped len(file) > 1, which dropped every such file from
+ # file_hashes — first build then saw "no changes" with an empty index (#290).
+ combined_text = "".join(doc.text for doc in file)
+ file_hashes[file_path] = hash_data(combined_text)
except Exception:
- logger.error(f"Cannot hash file {file.metadata['file_path']}")
+ logger.error(f"Cannot hash file {file_path}")
continue
return file_hashes
diff --git a/tests/test_sync.py b/tests/test_sync.py
index 38ef332..57f9bcb 100644
--- a/tests/test_sync.py
+++ b/tests/test_sync.py
@@ -77,6 +77,29 @@ class TestFileSynchronizer(unittest.TestCase):
assert result == {os.path.join(temp_dir, "file.txt"): hash_data("hello world")}
+ def test_generate_file_hashes_multi_document_same_path(self):
+ """Multi-page PDFs etc. yield multiple docs per path; combine text for hashing (#290)."""
+ temp_dir = tempfile.gettempdir()
+ path = os.path.join(temp_dir, "doc.pdf")
+ fs = FileSynchronizer(temp_dir, auto_load=False)
+
+ p1 = Mock()
+ p1.text = "aa"
+ p1.metadata = {"file_path": path}
+ p2 = Mock()
+ p2.text = "bb"
+ p2.metadata = {"file_path": path}
+
+ mock_reader_instance = Mock()
+ mock_reader_instance.iter_data.return_value = [[p1, p2]]
+
+ with patch("leann.sync.SimpleDirectoryReader") as mock_reader:
+ mock_reader.return_value = mock_reader_instance
+
+ result = fs.generate_file_hashes()
+
+ assert result == {path: hash_data("aabb")}
+
def test_build_merkle_tree(self):
fs = FileSynchronizer(".", auto_load=False)