[MLsys2026 Best Paper]: https://arxiv.org/abs/2506.08276. RAG on Everything with LEANN. Enjoy 97% storage savings while running a fast, accurate, and 100% private RAG application on your personal device.
Python aifaissgpt-osslangchainllama-indexllmlocalstorageoffline-firstollamaprivacypythonragretrieval-augmented-generationvector-databasevector-searchvectors
fix(sync): hash multi-document files for change detection (#290)
Aakash Suresh7b2c7341ffc8
diff --git a/packages/leann-core/src/leann/sync.py b/packages/leann-core/src/leann/sync.pyindex d0a69e5..d944477 100644--- a/packages/leann-core/src/leann/sync.py+++ b/packages/leann-core/src/leann/sync.py@@ -107,15 +107,19 @@ class FileSynchronizer:# print('reader.iter_data() length', len(list(reader.iter_data())))for file in reader.iter_data():- if len(file) > 1:- # print('file length is greater than 1', file)- continue # SimpleDirectoryReader can load more than 1 documents for weird file types e.g. PDFs- file = file[0]+ if not file:+ continue+ file_path = file[0].metadata.get("file_path", "")+ if not file_path:+ continuetry:- file_hash = hash_data(file.text)- file_hashes[file.metadata["file_path"]] = file_hash+ # Combine text from all documents for this file (e.g. multi-page PDFs).+ # Previously we skipped len(file) > 1, which dropped every such file from+ # file_hashes — first build then saw "no changes" with an empty index (#290).+ combined_text = "".join(doc.text for doc in file)+ file_hashes[file_path] = hash_data(combined_text)except Exception:- logger.error(f"Cannot hash file {file.metadata['file_path']}")+ logger.error(f"Cannot hash file {file_path}")continuereturn file_hashesdiff --git a/tests/test_sync.py b/tests/test_sync.pyindex 38ef332..57f9bcb 100644--- a/tests/test_sync.py+++ b/tests/test_sync.py@@ -77,6 +77,29 @@ class TestFileSynchronizer(unittest.TestCase):assert result == {os.path.join(temp_dir, "file.txt"): hash_data("hello world")}+ def test_generate_file_hashes_multi_document_same_path(self):+ """Multi-page PDFs etc. yield multiple docs per path; combine text for hashing (#290)."""+ temp_dir = tempfile.gettempdir()+ path = os.path.join(temp_dir, "doc.pdf")+ fs = FileSynchronizer(temp_dir, auto_load=False)++ p1 = Mock()+ p1.text = "aa"+ p1.metadata = {"file_path": path}+ p2 = Mock()+ p2.text = "bb"+ p2.metadata = {"file_path": path}++ mock_reader_instance = Mock()+ mock_reader_instance.iter_data.return_value = [[p1, p2]]++ with patch("leann.sync.SimpleDirectoryReader") as mock_reader:+ mock_reader.return_value = mock_reader_instance++ result = fs.generate_file_hashes()++ assert result == {path: hash_data("aabb")}+def test_build_merkle_tree(self):fs = FileSynchronizer(".", auto_load=False)