Hypergraph is more powerful. Transform unstructured text into structured knowledge with LLMs. Graphs, hypergraphs, and spatio-temporal extractions — with one command.
feat: patch the FAISS index in place on removal/edit instead of full rebuild
yifanfeng9738c310ca3d26
diff --git a/docs/en/cli/commands/remove.md b/docs/en/cli/commands/remove.mdindex 9adc7d8..7d12e40 100644--- a/docs/en/cli/commands/remove.md+++ b/docs/en/cli/commands/remove.md@@ -79,7 +79,9 @@ he remove ./ka/ --edit-edge "Apple-acquired-Beats" \## After removal-Removal invalidates the search index (the stale `index/` directory is deleted). Rebuild it before searching or chatting again:+When the knowledge abstract already has a search index, `he remove` **patches it in place** — only the affected vectors are deleted/re-embedded, and `he build-index` is *not* required afterwards. Search stays usable immediately.++If no index had been built (or in-place patching was not possible), any stale on-disk index is removed instead; rebuild it before searching or chatting again:```bashhe build-index ./ka/diff --git a/docs/zh/cli/commands/remove.md b/docs/zh/cli/commands/remove.mdindex f264435..d8011d2 100644--- a/docs/zh/cli/commands/remove.md+++ b/docs/zh/cli/commands/remove.md@@ -79,7 +79,9 @@ he remove ./ka/ --edit-edge "Apple-acquired-Beats" \## 删除之后-删除会使搜索索引失效(过期的 `index/` 目录会被删除)。再次搜索或对话前请先重建:+如果知识摘要已有搜索索引,`he remove` 会**原地修补索引**——只删除/重嵌受影响的向量,之后**无需**再执行 `he build-index`,搜索立即可用。++如果此前没有构建过索引(或原地修补不可用),过期的磁盘索引会被删除;再次搜索或对话前请先重建:```bashhe build-index ./ka/diff --git a/hyperextract/cli/cli.py b/hyperextract/cli/cli.pyindex 57bb5ee..db023aa 100644--- a/hyperextract/cli/cli.py+++ b/hyperextract/cli/cli.py@@ -1223,6 +1223,8 @@ def remove_items(table.add_column("Field")table.add_column("Items")for field, items in report.items():+ if field == "index_patched":+ continuetable.add_row(field, "\n".join(items) if items else "—")console.print(table)removed_any = any(@@ -1262,11 +1264,18 @@ def remove_items(ka.dump(path)- # The in-memory index was invalidated by the mutation; drop the stale- # on-disk index so search never serves deleted knowledge.- index_dir = path / "index"- if index_dir.exists():- shutil.rmtree(index_dir)+ if report.get("index_patched", False):+ # The vector index was patched in place and persisted by dump() —+ # search stays usable without a rebuild.+ console.print("[dim]Search index patched in place — no rebuild needed.[/dim]")+ else:+ # No index was built (or patching fell back): drop any stale on-disk+ # index so search never serves deleted knowledge.+ index_dir = path / "index"+ if index_dir.exists():+ shutil.rmtree(index_dir)+ console.print(+ f"[dim]Rebuild the search index with: he build-index {ka_path}[/dim]"+ )console.print(f"[bold green]Done![/bold green] Knowledge Abstract updated: {path}")- console.print(f"[dim]Rebuild the search index with: he build-index {ka_path}[/dim]")diff --git a/hyperextract/types/graph.py b/hyperextract/types/graph.pyindex ae839e2..22ddf82 100644--- a/hyperextract/types/graph.py+++ b/hyperextract/types/graph.py@@ -13,6 +13,7 @@ from typing import (TypeVar,)+from langchain_core.documents import Documentfrom langchain_core.embeddings import Embeddingsfrom langchain_core.language_models.chat_models import BaseChatModelfrom langchain_core.messages import AIMessage@@ -89,6 +90,72 @@ DEFAULT_EDIT_PROMPT = ()+def _patch_vector_index(+ memory: OMem,+ vs,+ removed_keys: set,+ upserted_keys: set,+) -> bool:+ """Partially update a built FAISS index for the affected keys.++ Deletes the vectors of ``removed_keys``/``upserted_keys`` and re-embeds+ only the upserted items — instead of OMem's default behavior of dropping+ the whole index (which forces a full re-embed on next use).++ The ``key → vector_id`` map is derived by scanning the docstore, which+ already stores the key per vector, so no storage-format change is needed.++ Args:+ memory: The OMem instance (used for key-extraction serialization).+ vs: The langchain FAISS vectorstore held by ``memory._index``.+ removed_keys: Keys whose vectors must be deleted.+ upserted_keys: Keys present in ``memory`` whose vectors must be+ re-embedded (delete old + insert new).++ Returns:+ True if the index was patched in place; False on any failure, in+ which case the caller must drop the index and let OMem rebuild it+ fully (the pre-partial-patch behavior).++ Note:+ Uses ``OMem._index`` / ``_serialize_for_embedding`` — protected+ surface of the same-author ontomem package until a public+ partial-update API exists there.+ """+ try:+ id_by_key = {}+ for doc_id, doc in vs.docstore._dict.items():+ doc_key = doc.metadata.get("key")+ if doc_key is not None:+ id_by_key[doc_key] = doc_id++ stale_ids = [+ id_by_key[k] for k in removed_keys | upserted_keys if k in id_by_key+ ]+ if stale_ids:+ vs.delete(stale_ids)++ documents = []+ for key in upserted_keys:+ item = memory.get(key)+ if item is None:+ continue+ documents.append(+ Document(+ page_content=memory._serialize_for_embedding(item),+ metadata={"key": key, "raw": item.model_dump()},+ )+ )+ if documents:+ vs.add_documents(documents)+ return True+ except Exception as e:+ logger.warning(+ "stage=index_patch_failed error=%s falling_back_to_full_rebuild", e+ )+ return False++class GraphEditMixin:"""Hard-delete and LLM-assisted soft-delete for graph-family AutoTypes.@@ -104,16 +171,23 @@ class GraphEditMixin:"""Hard-delete nodes by key, together with any edges they anchor.Edges touching a removed node would dangle, so every such edge is- removed as well. The search index is invalidated; the next search or- `build_index` rebuilds it.+ removed as well. When the search index is built, it is patched in+ place (only the affected vectors are deleted) instead of discarded.Args:keys: Node keys (as produced by `node_key_extractor`).Returns:- Report dict with `removed_nodes`, `not_found_nodes`, and- `removed_orphan_edges`.+ Report dict with `removed_nodes`, `not_found_nodes`,+ `removed_orphan_edges`, and `index_patched`."""+ node_vs = self._node_memory._index+ edge_vs = self._edge_memory._index+ # Detach the vectorstores so OMem's remove() can't discard them; they+ # are patched and reattached after the storage mutations below.+ self._node_memory._index = None+ self._edge_memory._index = None+removed: list[str] = []not_found: list[str] = []removed_node_keys = set()@@ -133,39 +207,77 @@ class GraphEditMixin:if self._edge_memory.remove(edge_key):orphaned.append(edge_key)+ node_ok = True+ if node_vs is not None:+ node_ok = _patch_vector_index(+ self._node_memory, node_vs, removed_node_keys, set()+ )+ self._node_memory._index = node_vs if node_ok else None+ edge_ok = True+ if edge_vs is not None:+ edge_ok = _patch_vector_index(+ self._edge_memory, edge_vs, set(orphaned), set()+ )+ self._edge_memory._index = edge_vs if edge_ok else None+ index_patched = (node_vs is None or node_ok) and (edge_vs is None or edge_ok)+logger.info(- "stage=remove_nodes removed=%d not_found=%d orphan_edges=%d",+ "stage=remove_nodes removed=%d not_found=%d orphan_edges=%d index_patched=%s",len(removed),len(not_found),len(orphaned),+ index_patched and (node_vs is not None or edge_vs is not None),)return {"removed_nodes": removed,"not_found_nodes": not_found,"removed_orphan_edges": orphaned,+ "index_patched": index_patched+ and (node_vs is not None or edge_vs is not None),}def remove_edges(self, *keys: str) -> dict:"""Hard-delete edges by key.+ When the search index is built, it is patched in place (only the+ affected vectors are deleted) instead of discarded.+Args:keys: Edge keys (as produced by `edge_key_extractor`).Returns:- Report dict with `removed_edges` and `not_found_edges`.+ Report dict with `removed_edges`, `not_found_edges`, and+ `index_patched`."""+ edge_vs = self._edge_memory._index+ self._edge_memory._index = None+removed: list[str] = []not_found: list[str] = []+ removed_edge_keys = set()for key in keys:if self._edge_memory.remove(key):removed.append(key)+ removed_edge_keys.add(key)else:not_found.append(key)+ edge_ok = True+ if edge_vs is not None:+ edge_ok = _patch_vector_index(+ self._edge_memory, edge_vs, removed_edge_keys, set()+ )+ self._edge_memory._index = edge_vs if edge_ok else None+ index_patched = edge_vs is None or edge_ok+logger.info("stage=remove_edges removed=%d not_found=%d", len(removed), len(not_found))- return {"removed_edges": removed, "not_found_edges": not_found}+ return {+ "removed_edges": removed,+ "not_found_edges": not_found,+ "index_patched": index_patched,+ }# ==================== Soft Delete (LLM-assisted) ====================@@ -179,6 +291,9 @@ class GraphEditMixin:) -> dict:"""Soft-delete a fact from a node via LLM-assisted rewrite.+ When the search index is built, its vector for this node is updated+ in place (re-embedded once) instead of discarding the index.+Args:key: Node key to edit.remove_fact: Fact to remove from the node (exclusive with@@ -188,18 +303,37 @@ class GraphEditMixin:dry_run: Return the proposed rewrite without applying it.Returns:- Report dict with `changed`, `applied`, `old`, and `new`.+ Report dict with `changed`, `applied`, `old`, `new`, and+ `index_patched`."""- return self._edit_item(- self._node_memory,- self.node_editor,- self.node_key_extractor,- "node",- key,- remove_fact,- instruction,- dry_run,- )+ node_vs = self._node_memory._index+ self._node_memory._index = None+ try:+ report = self._edit_item(+ self._node_memory,+ self.node_editor,+ self.node_key_extractor,+ "node",+ key,+ remove_fact,+ instruction,+ dry_run,+ )+ except Exception:+ self._node_memory._index = node_vs+ raise++ if node_vs is not None:+ if report["changed"]:+ index_patched = _patch_vector_index(+ self._node_memory, node_vs, set(), {key}+ )+ self._node_memory._index = node_vs if index_patched else None+ else:+ self._node_memory._index = node_vs+ index_patched = True+ report["index_patched"] = index_patched+ return reportdef edit_edge(self,@@ -211,6 +345,9 @@ class GraphEditMixin:) -> dict:"""Soft-delete a fact from an edge via LLM-assisted rewrite.+ When the search index is built, its vector for this edge is updated+ in place (re-embedded once) instead of discarding the index.+Args:key: Edge key to edit.remove_fact: Fact to remove from the edge (exclusive with@@ -220,18 +357,37 @@ class GraphEditMixin:dry_run: Return the proposed rewrite without applying it.Returns:- Report dict with `changed`, `applied`, `old`, and `new`.+ Report dict with `changed`, `applied`, `old`, `new`, and+ `index_patched`."""- return self._edit_item(- self._edge_memory,- self.edge_editor,- self.edge_key_extractor,- "edge",- key,- remove_fact,- instruction,- dry_run,- )+ edge_vs = self._edge_memory._index+ self._edge_memory._index = None+ try:+ report = self._edit_item(+ self._edge_memory,+ self.edge_editor,+ self.edge_key_extractor,+ "edge",+ key,+ remove_fact,+ instruction,+ dry_run,+ )+ except Exception:+ self._edge_memory._index = edge_vs+ raise++ if edge_vs is not None:+ if report["changed"]:+ index_patched = _patch_vector_index(+ self._edge_memory, edge_vs, set(), {key}+ )+ self._edge_memory._index = edge_vs if index_patched else None+ else:+ self._edge_memory._index = edge_vs+ index_patched = True+ report["index_patched"] = index_patched+ return reportclass AutoGraphSchema(BaseModel, Generic[NodeSchema, EdgeSchema]):diff --git a/tests/cli/test_remove.py b/tests/cli/test_remove.pyindex b91291c..a6efd9c 100644--- a/tests/cli/test_remove.py+++ b/tests/cli/test_remove.py@@ -108,6 +108,28 @@ class TestHardDeleteCli:assert [n["name"] for n in _data(ka)["nodes"]] == ["Apple", "Google"]assert not list(ka.glob("data.json.bak.*"))+ def test_built_index_is_patched_not_discarded(self, tmp_path, monkeypatch):+ """With a real built index, he remove patches it in place."""+ g = _real_ka()+ g.metadata["template"] = "general/graph"+ g.metadata["lang"] = "en"+ g.build_index()+ ka = tmp_path / "ka"+ g.dump(ka)+ assert (ka / "index" / "node_index" / "index.faiss").exists()+ monkeypatch.setattr(climod.Template, "create", staticmethod(lambda *a, **k: g))++ result = runner.invoke(app, ["remove", str(ka), "--node", "Apple", "-y"])++ assert result.exit_code == 0, result.output+ assert "patched in place" in result.output+ # The index survives, and its docstore no longer holds the removed node.+ assert (ka / "index" / "node_index" / "index.faiss").exists()+ docstore = (ka / "index" / "node_index" / "index.pkl").read_bytes()+ assert b"Apple" not in docstore+ assert b"Google" in docstore+ assert [n["name"] for n in _data(ka)["nodes"]] == ["Google"]+def test_no_match_reports_and_writes_nothing(self, ka_env):ka, _ = ka_envresult = runner.invoke(app, ["remove", str(ka), "--node", "Ghost", "-y"])diff --git a/tests/types/test_graph_removal.py b/tests/types/test_graph_removal.pyindex 18c6cb3..6f78f71 100644--- a/tests/types/test_graph_removal.py+++ b/tests/types/test_graph_removal.py@@ -1,7 +1,6 @@"""Hard delete (remove_nodes / remove_edges) and soft delete (edit_*) forgraph-family AutoTypes (issue #84)."""-import pytestfrom pydantic import BaseModel, Field@@ -161,6 +160,126 @@ class FakeEditor:return self.rewritten+class TestIndexPatching:+ """Phase A (#84): removals/edits patch the FAISS index in place.++ Assertions inspect the FAISS docstore directly (keys + re-embedded+ page_content) — MockEmbeddings is hash-based, so semantic-search+ result ordering is not meaningful here.+ """++ @staticmethod+ def _docstore(memory):+ return memory._index.docstore._dict++ def test_remove_node_keeps_index_without_stale_vectors(self):+ g = _graph()+ g._node_memory.add(+ [Entity(name="Apple"), Entity(name="Google"), Entity(name="Meta")]+ )+ g._edge_memory.add(+ [Relation(source="Apple", target="Google", relation_type="partner")]+ )+ g.build_index()+ assert g._node_memory.has_index()++ report = g.remove_nodes("Apple")++ assert report["index_patched"] is True+ # Index must survive the removal (no full-rebuild fallback).+ assert g._node_memory.has_index()+ assert g._edge_memory.has_index()+ # The removed node's vector and its edge's vector are gone.+ node_keys = {d.metadata["key"] for d in self._docstore(g._node_memory).values()}+ edge_keys = {d.metadata["key"] for d in self._docstore(g._edge_memory).values()}+ assert node_keys == {"Google", "Meta"}+ assert edge_keys == set()++ def test_remove_node_docstore_shrinks(self):+ g = _graph()+ g._node_memory.add([Entity(name="A"), Entity(name="B")])+ g.build_index()+ size_before = len(g._node_memory._index.docstore._dict)++ g.remove_nodes("A")++ size_after = len(g._node_memory._index.docstore._dict)+ assert size_after == size_before - 1++ def test_edit_node_refreshes_vector(self):+ g = _graph()+ g._node_memory.add(+ [Entity(name="A", description="A was founded by X. A is in Y.")]+ )+ g.build_index()+ g.node_editor = FakeEditor(Entity(name="A", description="A is in Y."))++ report = g.edit_node("A", remove_fact="founded by X")++ assert report["index_patched"] is True+ assert g._node_memory.has_index()+ # Exactly one vector for the key, re-embedded with the new text.+ docs = [+ d+ for d in self._docstore(g._node_memory).values()+ if d.metadata["key"] == "A"+ ]+ assert len(docs) == 1+ assert "founded by X" not in docs[0].page_content+ assert "A is in Y" in docs[0].page_content++ def test_no_index_built_falls_back_to_lazy_rebuild(self):+ g = _graph()+ g._node_memory.add([Entity(name="A"), Entity(name="B")])+ # No build_index() call — nothing to patch.++ report = g.remove_nodes("A")++ assert report["index_patched"] is False+ assert not g._node_memory.has_index()+ # OMem's lazy rebuild still reflects the removal.+ assert "A" not in {n.name for n in g.nodes}++ def test_patched_index_survives_dump_load(self, tmp_path):+ g = _graph()+ g._node_memory.add([Entity(name="A"), Entity(name="B")])+ g._edge_memory.add([Relation(source="A", target="B", relation_type="r")])+ g.build_index()+ g.remove_nodes("A")+ g.dump(tmp_path)++ reloaded = _graph()+ reloaded.load(tmp_path)+ assert reloaded._node_memory.has_index()++ node_keys = {+ d.metadata["key"] for d in self._docstore(reloaded._node_memory).values()+ }+ assert node_keys == {"B"}+ edge_keys = {+ d.metadata["key"] for d in self._docstore(reloaded._edge_memory).values()+ }+ assert edge_keys == set()++ def test_hypergraph_removal_patches_index(self):+ h = _hypergraph()+ h._node_memory.add([Entity(name="A"), Entity(name="B"), Entity(name="C")])+ h._edge_memory.add(+ [HyperRelation(participants=["A", "B"], relation_type="group")]+ )+ h.build_index()++ report = h.remove_nodes("A")++ assert report["index_patched"] is True+ assert h._node_memory.has_index()+ node_keys = {d.metadata["key"] for d in self._docstore(h._node_memory).values()}+ assert node_keys == {"B", "C"}+ # The hyperedge containing the removed node is gone from the index.+ edge_keys = {d.metadata["key"] for d in self._docstore(h._edge_memory).values()}+ assert edge_keys == set()++class TestSoftDelete:"""edit_node / edit_edge: LLM-assisted fact removal with guardrails."""@@ -259,9 +378,7 @@ class TestSoftDelete:))- report = g.edit_edge(- "A-acquired-B", remove_fact="first announced in March"- )+ report = g.edit_edge("A-acquired-B", remove_fact="first announced in March")assert report["applied"] is Trueassert "announced in March" not in g.edges[0].description