Personal AI, On Personal Devices
Python
fix(memory): replace chunks when re-indexing sources (#675)
Haotian Zhengf2df968aa552
diff --git a/rust/crates/openjarvis-python/src/storage.rs b/rust/crates/openjarvis-python/src/storage.rsindex 332aff2a..b8a98cbb 100644--- a/rust/crates/openjarvis-python/src/storage.rs+++ b/rust/crates/openjarvis-python/src/storage.rs@@ -33,6 +33,33 @@ impl PySQLiteMemory {.map_err(|e| PyErr::new::<pyo3::exceptions::PyRuntimeError, _>(e.to_string()))}+ fn replace_source(+ &self,+ source: &str,+ documents: Vec<(String, Option<String>)>,+ ) -> PyResult<Vec<String>> {+ let parsed_documents = documents+ .into_iter()+ .map(|(content, metadata)| {+ let metadata = metadata+ .map(|value| serde_json::from_str(&value))+ .transpose()+ .map_err(|e| {+ PyErr::new::<pyo3::exceptions::PyValueError, _>(e.to_string())+ })?;+ Ok((content, metadata))+ })+ .collect::<PyResult<Vec<_>>>()?;+ let document_refs = parsed_documents+ .iter()+ .map(|(content, metadata)| (content.as_str(), metadata.as_ref()))+ .collect::<Vec<_>>();++ self.inner+ .replace_source(source, &document_refs)+ .map_err(|e| PyErr::new::<pyo3::exceptions::PyRuntimeError, _>(e.to_string()))+ }+#[pyo3(signature = (query, top_k=5))]fn retrieve(&self, query: &str, top_k: usize) -> PyResult<String> {let results = selfdiff --git a/rust/crates/openjarvis-tools/src/storage/sqlite.rs b/rust/crates/openjarvis-tools/src/storage/sqlite.rsindex 20dc39cf..fbec628c 100644--- a/rust/crates/openjarvis-tools/src/storage/sqlite.rs+++ b/rust/crates/openjarvis-tools/src/storage/sqlite.rs@@ -94,6 +94,57 @@ impl SQLiteMemory {pub fn in_memory() -> Result<Self, OpenJarvisError> {Self::new(Path::new(":memory:"))}++ /// Atomically replace every document for *source* with *documents*.+ pub fn replace_source(+ &self,+ source: &str,+ documents: &[(&str, Option<&Value>)],+ ) -> Result<Vec<String>, OpenJarvisError> {+ let mut conn = self.conn.lock();+ let tx = conn.transaction().map_err(|e| {+ OpenJarvisError::Io(std::io::Error::other(e.to_string()))+ })?;++ tx.execute(+ "DELETE FROM documents_fts+ WHERE rowid IN (SELECT rowid FROM documents WHERE source = ?1)",+ rusqlite::params![source],+ )+ .map_err(|e| OpenJarvisError::Io(std::io::Error::other(e.to_string())))?;+ tx.execute(+ "DELETE FROM documents WHERE source = ?1",+ rusqlite::params![source],+ )+ .map_err(|e| OpenJarvisError::Io(std::io::Error::other(e.to_string())))?;++ let mut doc_ids = Vec::with_capacity(documents.len());+ for (content, metadata) in documents {+ let doc_id = Uuid::new_v4().to_string();+ let meta_str = metadata+ .map(|m| serde_json::to_string(m).unwrap_or_default())+ .unwrap_or_else(|| "{}".to_string());++ tx.execute(+ "INSERT INTO documents (id, content, source, metadata)+ VALUES (?1, ?2, ?3, ?4)",+ rusqlite::params![doc_id, content, source, meta_str],+ )+ .map_err(|e| OpenJarvisError::Io(std::io::Error::other(e.to_string())))?;++ let rowid = tx.last_insert_rowid();+ tx.execute(+ "INSERT INTO documents_fts (rowid, content, source) VALUES (?1, ?2, ?3)",+ rusqlite::params![rowid, content, source],+ )+ .map_err(|e| OpenJarvisError::Io(std::io::Error::other(e.to_string())))?;+ doc_ids.push(doc_id);+ }++ tx.commit()+ .map_err(|e| OpenJarvisError::Io(std::io::Error::other(e.to_string())))?;+ Ok(doc_ids)+ }}impl MemoryBackend for SQLiteMemory {@@ -306,6 +357,40 @@ mod tests {assert_eq!(mem.count().unwrap(), 0);}+ #[test]+ fn test_sqlite_replace_source_is_idempotent() {+ let mem = SQLiteMemory::in_memory().unwrap();++ mem.replace_source("notes.txt", &[("old project notes", None)])+ .unwrap();+ assert_eq!(mem.count().unwrap(), 1);++ mem.replace_source("notes.txt", &[("updated project notes", None)])+ .unwrap();+ assert_eq!(mem.count().unwrap(), 1);++ assert!(mem.retrieve("old", 5).unwrap().is_empty());+ let updated = mem.retrieve("updated", 5).unwrap();+ assert_eq!(updated.len(), 1);+ assert_eq!(updated[0].source, "notes.txt");+ }++ #[test]+ fn test_sqlite_replace_source_preserves_other_sources() {+ let mem = SQLiteMemory::in_memory().unwrap();+ mem.store("keep this manual", "manual.txt", None).unwrap();+ mem.replace_source("notes.txt", &[("old project notes", None)])+ .unwrap();++ mem.replace_source("notes.txt", &[("updated project notes", None)])+ .unwrap();++ assert_eq!(mem.count().unwrap(), 2);+ let manual = mem.retrieve("manual", 5).unwrap();+ assert_eq!(manual.len(), 1);+ assert_eq!(manual[0].source, "manual.txt");+ }+#[test]fn test_sqlite_case_insensitive_search() {let mem = SQLiteMemory::in_memory().unwrap();diff --git a/src/openjarvis/cli/memory_cmd.py b/src/openjarvis/cli/memory_cmd.pyindex 263b1086..92bb55a7 100644--- a/src/openjarvis/cli/memory_cmd.py+++ b/src/openjarvis/cli/memory_cmd.py@@ -89,15 +89,39 @@ def index(mem = _get_backend(backend)try:- for chunk in track(chunks, description="Storing chunks...", console=console):- mem.store(- chunk.content,- source=chunk.source,- metadata={- "offset": chunk.offset,- "index": chunk.index,- },- )+ replace_source = getattr(mem, "replace_source", None)+ if callable(replace_source):+ documents_by_source = {}+ for chunk in chunks:+ documents_by_source.setdefault(chunk.source, []).append(+ (+ chunk.content,+ {+ "offset": chunk.offset,+ "index": chunk.index,+ },+ )+ )+ for source, documents in track(+ documents_by_source.items(),+ description="Replacing sources...",+ console=console,+ ):+ replace_source(source, documents)+ else:+ for chunk in track(+ chunks,+ description="Storing chunks...",+ console=console,+ ):+ mem.store(+ chunk.content,+ source=chunk.source,+ metadata={+ "offset": chunk.offset,+ "index": chunk.index,+ },+ )finally:if hasattr(mem, "close"):mem.close()diff --git a/src/openjarvis/tools/storage/sqlite.py b/src/openjarvis/tools/storage/sqlite.pyindex 707093c3..c468fde2 100644--- a/src/openjarvis/tools/storage/sqlite.py+++ b/src/openjarvis/tools/storage/sqlite.py@@ -95,6 +95,29 @@ class SQLiteMemory(MemoryBackend):)return doc_id+ def replace_source(+ self,+ source: str,+ documents: List[tuple[str, Optional[Dict[str, Any]]]],+ ) -> List[str]:+ """Atomically replace all documents associated with *source*."""+ payload = [+ (content, json.dumps(metadata) if metadata else None)+ for content, metadata in documents+ ]+ doc_ids = self._rust_impl.replace_source(source, payload)+ bus = get_event_bus()+ for doc_id in doc_ids:+ bus.publish(+ EventType.MEMORY_STORE,+ {+ "backend": self.backend_id,+ "doc_id": doc_id,+ "source": source,+ },+ )+ return doc_ids+def retrieve(self,query: str,diff --git a/tests/cli/test_memory_cmd.py b/tests/cli/test_memory_cmd.pyindex 64201c16..6c3a9dc3 100644--- a/tests/cli/test_memory_cmd.py+++ b/tests/cli/test_memory_cmd.py@@ -40,6 +40,35 @@ def test_memory_index_file(tmp_path: Path, monkeypatch):assert "Indexed" in result.output or "chunk" in result.output+def test_memory_index_replaces_existing_source(tmp_path: Path, monkeypatch):+ """Re-indexing a file replaces its previous chunks."""+ _register_sqlite()+ db_path = str(tmp_path / "mem.db")+ doc = tmp_path / "doc.txt"+ doc.write_text(" ".join(["legacy"] * 100), encoding="utf-8")++ mod = importlib.import_module("openjarvis.cli.memory_cmd")+ monkeypatch.setattr(+ mod,+ "_get_backend",+ lambda b=None: SQLiteMemory(db_path=db_path),+ )++ first = CliRunner().invoke(cli, ["memory", "index", str(doc)])+ assert first.exit_code == 0++ doc.write_text(" ".join(["updated"] * 100), encoding="utf-8")+ second = CliRunner().invoke(cli, ["memory", "index", str(doc)])+ assert second.exit_code == 0++ backend = SQLiteMemory(db_path=db_path)+ assert backend.count() == 1+ assert backend.retrieve("legacy") == []+ updated = backend.retrieve("updated")+ assert len(updated) == 1+ assert updated[0].source == str(doc)++def test_memory_index_nonexistent(tmp_path: Path):"""Indexing a nonexistent path should fail."""_register_sqlite()