[MLsys2026 Best Paper]: https://arxiv.org/abs/2506.08276. RAG on Everything with LEANN. Enjoy 97% storage savings while running a fast, accurate, and 100% private RAG application on your personal device.
Python aifaissgpt-osslangchainllama-indexllmlocalstorageoffline-firstollamaprivacypythonragretrieval-augmented-generationvector-databasevector-searchvectors
feat: add file syncing logic using a merkle tree (#219)
Antonee3e334c1f90
diff --git a/packages/leann-core/src/leann/sync.py b/packages/leann-core/src/leann/sync.pynew file mode 100644index 0000000..1d5a0e0--- /dev/null+++ b/packages/leann-core/src/leann/sync.py@@ -0,0 +1,158 @@+import logging+import os+import pickle+from dataclasses import dataclass, field+from hashlib import sha256+from typing import Optional++from llama_index.core import SimpleDirectoryReader++logger = logging.getLogger(__name__)+++def hash_data(data: str | bytes):+ if isinstance(data, str):+ data = data.encode()+ return sha256(data).hexdigest()+++@dataclass+class MerkleTreeNode:+ hash: str+ data: str+ children: dict[str, "MerkleTreeNode"] = field(default_factory=dict)+ parent_id: str | None = None+++class MerkleTree:+ def __init__(self):+ self.nodes: dict[str, MerkleTreeNode] = {}+ self.root: MerkleTreeNode | None = None++ def add_node(self, data: str, parent_id=None, hash: Optional[str] = None):+ hash = hash_data(data) if hash is None else hash++ node = MerkleTreeNode(hash=hash, data=data, parent_id=parent_id)+ self.nodes[hash] = node++ if parent_id is None:+ self.root = node+ else:+ self.nodes[parent_id].children[hash] = node++ return hash++ def compare_with(self, other: "MerkleTree"):+ """+ Simple comparison of two flat trees. Check the individual file hashes+ only if the root has changed, otherwise return no changes.+ """+ assert self.root is not None and other.root is not None++ if self.root.hash == other.root.hash:+ return [], [], []++ old_files = self.root.children+ new_files = other.root.children++ all_nodes = new_files.keys() | old_files.keys()++ added, removed, modified = [], [], []+ for path in all_nodes:+ if path in new_files and path in old_files:+ if new_files[path].data != old_files[path].data:+ modified.append(path)+ elif path in new_files and path not in old_files:+ added.append(path)+ else:+ removed.append(path)++ return added, removed, modified+++class FileSynchronizer:+ def __init__(+ self,+ root_dir: str,+ ignore_patterns: Optional[list] = None,+ include_extensions: Optional[list] = None,+ auto_load=True,+ ):+ if not os.path.isdir(root_dir):+ raise ValueError("This is not a valid directory")+ self.root_dir = root_dir+ self.ignore_patterns = ignore_patterns+ self.include_extensions = include_extensions+ if auto_load:+ self.load_snapshot()++ def generate_file_hashes(self):+ file_hashes = {}+ reader = SimpleDirectoryReader(+ self.root_dir,+ recursive=True,+ exclude=self.ignore_patterns,+ required_exts=self.include_extensions,+ exclude_empty=True,+ )++ for file in reader.iter_data():+ if len(file) > 1:+ continue # SimpleDirectoryReader can load more than 1 documents for weird file types e.g. PDFs+ file = file[0]+ try:+ file_hash = hash_data(file.text)+ file_hashes[file.metadata["file_path"]] = file_hash+ except Exception:+ logger.error(f"Cannot hash file {file.metadata['file_path']}")+ continue++ return file_hashes++ def build_merkle_tree(self, file_hashes):+ """+ Build a flat merkle tree suitable for quick checking of file changes.+ """+ tree = MerkleTree()++ sorted_paths = sorted(file_hashes)+ root_data = "".join(path + file_hashes[path] for path in sorted_paths)++ root_id = tree.add_node(root_data)++ for path in sorted_paths:+ tree.add_node(file_hashes[path], parent_id=root_id, hash=path)++ return tree++ def check_for_changes(self):+ file_hashes = self.generate_file_hashes()+ new_tree = self.build_merkle_tree(file_hashes)++ changes = self.tree.compare_with(new_tree)++ if changes:+ self.tree = new_tree+ self.save_snapshot()++ return changes++ @property+ def snapshot_path(self):+ return f"{self.root_dir}.sync_context.pickle"++ def save_snapshot(self):+ assert self.tree is not None++ with open(self.snapshot_path, "wb") as f:+ pickle.dump(self.tree, f)++ def load_snapshot(self):+ try:+ with open(self.snapshot_path, "rb") as f:+ self.tree = pickle.load(f)+ except FileNotFoundError:+ file_hashes = self.generate_file_hashes()+ self.tree = self.build_merkle_tree(file_hashes)+ self.save_snapshot()+ # yooooodiff --git a/tests/test_sync.py b/tests/test_sync.pynew file mode 100644index 0000000..2094233--- /dev/null+++ b/tests/test_sync.py@@ -0,0 +1,122 @@+import unittest+from unittest.mock import Mock, patch++from leann.sync import FileSynchronizer, MerkleTree, hash_data+++class TestMerkleTreeCompare(unittest.TestCase):+ def test_no_changes_if_root_hash_same(self):+ tree1 = Mock()+ tree2 = Mock()++ tree1.root = Mock(hash="root_hash")+ tree2.root = Mock(hash="root_hash")++ added, removed, modified = MerkleTree.compare_with(tree1, tree2)++ self.assertEqual(added, [])+ self.assertEqual(removed, [])+ self.assertEqual(modified, [])++ def test_added_removed_modified(self):+ tree1 = Mock()+ tree2 = Mock()++ # Mock file nodes+ file_a_new = Mock()+ file_b_new = Mock()+ file_a_old = Mock()+ file_c_old = Mock()++ # Equality behavior+ file_a_new.__eq__ = Mock(return_value=False)++ tree1.root = Mock(+ hash="new_root",+ children={+ "a.txt": file_a_new,+ "b.txt": file_b_new,+ },+ )++ tree2.root = Mock(+ hash="old_root",+ children={+ "a.txt": file_a_old,+ "c.txt": file_c_old,+ },+ )++ added, removed, modified = MerkleTree.compare_with(tree1, tree2)++ self.assertEqual(added, ["c.txt"])+ self.assertEqual(removed, ["b.txt"])+ self.assertEqual(modified, ["a.txt"])+++class TestFileSynchronizer(unittest.TestCase):+ def test_generate_file_hashes(self):+ fs = FileSynchronizer("/tmp", auto_load=False)++ mock_file = Mock()+ mock_file.text = "hello world"+ mock_file.metadata = {"file_path": "/tmp/file.txt"}++ mock_reader_instance = Mock()+ mock_reader_instance.iter_data.return_value = [+ [mock_file],+ ]++ with patch("leann.sync.SimpleDirectoryReader") as mock_reader:+ mock_reader.return_value = mock_reader_instance++ result = fs.generate_file_hashes()++ assert result == {"/tmp/file.txt": hash_data("hello world")}++ def test_build_merkle_tree(self):+ fs = FileSynchronizer(".", auto_load=False)++ file_hashes = {+ "a.txt": "hashA",+ "b.txt": "hashB",+ }++ tree = fs.build_merkle_tree(file_hashes)++ # Root exists+ assert tree.root is not None++ # Children added correctly+ assert set(tree.root.children.keys()) == {"a.txt", "b.txt"}++ # Child nodes have correct data+ assert tree.root.children["a.txt"].data == "hashA"+ assert tree.root.children["b.txt"].data == "hashB"++ expected_root_data = "a.txt" + "hashA" + "b.txt" + "hashB"+ assert tree.root.hash == hash_data(expected_root_data)++ def test_check_for_changes_detected(self):+ fs = FileSynchronizer.__new__(FileSynchronizer)++ fs.generate_file_hashes = Mock(return_value={"a.txt": "hash"})+ fs.build_merkle_tree = Mock(return_value=Mock())++ old_tree = Mock()+ new_tree = fs.build_merkle_tree.return_value++ old_tree.compare_with.return_value = (["a.txt"], [], [])+ fs.tree = old_tree++ fs.save_snapshot = Mock()++ changes = fs.check_for_changes()++ assert changes == (["a.txt"], [], [])++ fs.build_merkle_tree.assert_called_once_with({"a.txt": "hash"})+ old_tree.compare_with.assert_called_once_with(new_tree)++ fs.save_snapshot.assert_called_once()+ assert fs.tree is new_tree