Skip to content

Hypergraph is more powerful. Transform unstructured text into structured knowledge with LLMs. Graphs, hypergraphs, and spatio-temporal extractions — with one command.

feat: patch the FAISS index in place on removal/edit instead of full rebuild

yifanfeng9738c310ca3d26
diff --git a/docs/en/cli/commands/remove.md b/docs/en/cli/commands/remove.md
index 9adc7d8..7d12e40 100644
--- a/docs/en/cli/commands/remove.md
+++ b/docs/en/cli/commands/remove.md
@@ -79,7 +79,9 @@ he remove ./ka/ --edit-edge "Apple-acquired-Beats" \
## After removal
-Removal invalidates the search index (the stale `index/` directory is deleted). Rebuild it before searching or chatting again:
+When the knowledge abstract already has a search index, `he remove` **patches it in place** — only the affected vectors are deleted/re-embedded, and `he build-index` is *not* required afterwards. Search stays usable immediately.
+
+If no index had been built (or in-place patching was not possible), any stale on-disk index is removed instead; rebuild it before searching or chatting again:
```bash
he build-index ./ka/
diff --git a/docs/zh/cli/commands/remove.md b/docs/zh/cli/commands/remove.md
index f264435..d8011d2 100644
--- a/docs/zh/cli/commands/remove.md
+++ b/docs/zh/cli/commands/remove.md
@@ -79,7 +79,9 @@ he remove ./ka/ --edit-edge "Apple-acquired-Beats" \
## 删除之后
-删除会使搜索索引失效(过期的 `index/` 目录会被删除)。再次搜索或对话前请先重建:
+如果知识摘要已有搜索索引,`he remove` 会**原地修补索引**——只删除/重嵌受影响的向量,之后**无需**再执行 `he build-index`,搜索立即可用。
+
+如果此前没有构建过索引(或原地修补不可用),过期的磁盘索引会被删除;再次搜索或对话前请先重建:
```bash
he build-index ./ka/
diff --git a/hyperextract/cli/cli.py b/hyperextract/cli/cli.py
index 57bb5ee..db023aa 100644
--- a/hyperextract/cli/cli.py
+++ b/hyperextract/cli/cli.py
@@ -1223,6 +1223,8 @@ def remove_items(
table.add_column("Field")
table.add_column("Items")
for field, items in report.items():
+ if field == "index_patched":
+ continue
table.add_row(field, "\n".join(items) if items else "—")
console.print(table)
removed_any = any(
@@ -1262,11 +1264,18 @@ def remove_items(
ka.dump(path)
- # The in-memory index was invalidated by the mutation; drop the stale
- # on-disk index so search never serves deleted knowledge.
- index_dir = path / "index"
- if index_dir.exists():
- shutil.rmtree(index_dir)
+ if report.get("index_patched", False):
+ # The vector index was patched in place and persisted by dump() —
+ # search stays usable without a rebuild.
+ console.print("[dim]Search index patched in place — no rebuild needed.[/dim]")
+ else:
+ # No index was built (or patching fell back): drop any stale on-disk
+ # index so search never serves deleted knowledge.
+ index_dir = path / "index"
+ if index_dir.exists():
+ shutil.rmtree(index_dir)
+ console.print(
+ f"[dim]Rebuild the search index with: he build-index {ka_path}[/dim]"
+ )
console.print(f"[bold green]Done![/bold green] Knowledge Abstract updated: {path}")
- console.print(f"[dim]Rebuild the search index with: he build-index {ka_path}[/dim]")
diff --git a/hyperextract/types/graph.py b/hyperextract/types/graph.py
index ae839e2..22ddf82 100644
--- a/hyperextract/types/graph.py
+++ b/hyperextract/types/graph.py
@@ -13,6 +13,7 @@ from typing import (
TypeVar,
)
+from langchain_core.documents import Document
from langchain_core.embeddings import Embeddings
from langchain_core.language_models.chat_models import BaseChatModel
from langchain_core.messages import AIMessage
@@ -89,6 +90,72 @@ DEFAULT_EDIT_PROMPT = (
)
+def _patch_vector_index(
+ memory: OMem,
+ vs,
+ removed_keys: set,
+ upserted_keys: set,
+) -> bool:
+ """Partially update a built FAISS index for the affected keys.
+
+ Deletes the vectors of ``removed_keys``/``upserted_keys`` and re-embeds
+ only the upserted items — instead of OMem's default behavior of dropping
+ the whole index (which forces a full re-embed on next use).
+
+ The ``key → vector_id`` map is derived by scanning the docstore, which
+ already stores the key per vector, so no storage-format change is needed.
+
+ Args:
+ memory: The OMem instance (used for key-extraction serialization).
+ vs: The langchain FAISS vectorstore held by ``memory._index``.
+ removed_keys: Keys whose vectors must be deleted.
+ upserted_keys: Keys present in ``memory`` whose vectors must be
+ re-embedded (delete old + insert new).
+
+ Returns:
+ True if the index was patched in place; False on any failure, in
+ which case the caller must drop the index and let OMem rebuild it
+ fully (the pre-partial-patch behavior).
+
+ Note:
+ Uses ``OMem._index`` / ``_serialize_for_embedding`` — protected
+ surface of the same-author ontomem package until a public
+ partial-update API exists there.
+ """
+ try:
+ id_by_key = {}
+ for doc_id, doc in vs.docstore._dict.items():
+ doc_key = doc.metadata.get("key")
+ if doc_key is not None:
+ id_by_key[doc_key] = doc_id
+
+ stale_ids = [
+ id_by_key[k] for k in removed_keys | upserted_keys if k in id_by_key
+ ]
+ if stale_ids:
+ vs.delete(stale_ids)
+
+ documents = []
+ for key in upserted_keys:
+ item = memory.get(key)
+ if item is None:
+ continue
+ documents.append(
+ Document(
+ page_content=memory._serialize_for_embedding(item),
+ metadata={"key": key, "raw": item.model_dump()},
+ )
+ )
+ if documents:
+ vs.add_documents(documents)
+ return True
+ except Exception as e:
+ logger.warning(
+ "stage=index_patch_failed error=%s falling_back_to_full_rebuild", e
+ )
+ return False
+
+
class GraphEditMixin:
"""Hard-delete and LLM-assisted soft-delete for graph-family AutoTypes.
@@ -104,16 +171,23 @@ class GraphEditMixin:
"""Hard-delete nodes by key, together with any edges they anchor.
Edges touching a removed node would dangle, so every such edge is
- removed as well. The search index is invalidated; the next search or
- `build_index` rebuilds it.
+ removed as well. When the search index is built, it is patched in
+ place (only the affected vectors are deleted) instead of discarded.
Args:
keys: Node keys (as produced by `node_key_extractor`).
Returns:
- Report dict with `removed_nodes`, `not_found_nodes`, and
- `removed_orphan_edges`.
+ Report dict with `removed_nodes`, `not_found_nodes`,
+ `removed_orphan_edges`, and `index_patched`.
"""
+ node_vs = self._node_memory._index
+ edge_vs = self._edge_memory._index
+ # Detach the vectorstores so OMem's remove() can't discard them; they
+ # are patched and reattached after the storage mutations below.
+ self._node_memory._index = None
+ self._edge_memory._index = None
+
removed: list[str] = []
not_found: list[str] = []
removed_node_keys = set()
@@ -133,39 +207,77 @@ class GraphEditMixin:
if self._edge_memory.remove(edge_key):
orphaned.append(edge_key)
+ node_ok = True
+ if node_vs is not None:
+ node_ok = _patch_vector_index(
+ self._node_memory, node_vs, removed_node_keys, set()
+ )
+ self._node_memory._index = node_vs if node_ok else None
+ edge_ok = True
+ if edge_vs is not None:
+ edge_ok = _patch_vector_index(
+ self._edge_memory, edge_vs, set(orphaned), set()
+ )
+ self._edge_memory._index = edge_vs if edge_ok else None
+ index_patched = (node_vs is None or node_ok) and (edge_vs is None or edge_ok)
+
logger.info(
- "stage=remove_nodes removed=%d not_found=%d orphan_edges=%d",
+ "stage=remove_nodes removed=%d not_found=%d orphan_edges=%d index_patched=%s",
len(removed),
len(not_found),
len(orphaned),
+ index_patched and (node_vs is not None or edge_vs is not None),
)
return {
"removed_nodes": removed,
"not_found_nodes": not_found,
"removed_orphan_edges": orphaned,
+ "index_patched": index_patched
+ and (node_vs is not None or edge_vs is not None),
}
def remove_edges(self, *keys: str) -> dict:
"""Hard-delete edges by key.
+ When the search index is built, it is patched in place (only the
+ affected vectors are deleted) instead of discarded.
+
Args:
keys: Edge keys (as produced by `edge_key_extractor`).
Returns:
- Report dict with `removed_edges` and `not_found_edges`.
+ Report dict with `removed_edges`, `not_found_edges`, and
+ `index_patched`.
"""
+ edge_vs = self._edge_memory._index
+ self._edge_memory._index = None
+
removed: list[str] = []
not_found: list[str] = []
+ removed_edge_keys = set()
for key in keys:
if self._edge_memory.remove(key):
removed.append(key)
+ removed_edge_keys.add(key)
else:
not_found.append(key)
+ edge_ok = True
+ if edge_vs is not None:
+ edge_ok = _patch_vector_index(
+ self._edge_memory, edge_vs, removed_edge_keys, set()
+ )
+ self._edge_memory._index = edge_vs if edge_ok else None
+ index_patched = edge_vs is None or edge_ok
+
logger.info(
"stage=remove_edges removed=%d not_found=%d", len(removed), len(not_found)
)
- return {"removed_edges": removed, "not_found_edges": not_found}
+ return {
+ "removed_edges": removed,
+ "not_found_edges": not_found,
+ "index_patched": index_patched,
+ }
# ==================== Soft Delete (LLM-assisted) ====================
@@ -179,6 +291,9 @@ class GraphEditMixin:
) -> dict:
"""Soft-delete a fact from a node via LLM-assisted rewrite.
+ When the search index is built, its vector for this node is updated
+ in place (re-embedded once) instead of discarding the index.
+
Args:
key: Node key to edit.
remove_fact: Fact to remove from the node (exclusive with
@@ -188,18 +303,37 @@ class GraphEditMixin:
dry_run: Return the proposed rewrite without applying it.
Returns:
- Report dict with `changed`, `applied`, `old`, and `new`.
+ Report dict with `changed`, `applied`, `old`, `new`, and
+ `index_patched`.
"""
- return self._edit_item(
- self._node_memory,
- self.node_editor,
- self.node_key_extractor,
- "node",
- key,
- remove_fact,
- instruction,
- dry_run,
- )
+ node_vs = self._node_memory._index
+ self._node_memory._index = None
+ try:
+ report = self._edit_item(
+ self._node_memory,
+ self.node_editor,
+ self.node_key_extractor,
+ "node",
+ key,
+ remove_fact,
+ instruction,
+ dry_run,
+ )
+ except Exception:
+ self._node_memory._index = node_vs
+ raise
+
+ if node_vs is not None:
+ if report["changed"]:
+ index_patched = _patch_vector_index(
+ self._node_memory, node_vs, set(), {key}
+ )
+ self._node_memory._index = node_vs if index_patched else None
+ else:
+ self._node_memory._index = node_vs
+ index_patched = True
+ report["index_patched"] = index_patched
+ return report
def edit_edge(
self,
@@ -211,6 +345,9 @@ class GraphEditMixin:
) -> dict:
"""Soft-delete a fact from an edge via LLM-assisted rewrite.
+ When the search index is built, its vector for this edge is updated
+ in place (re-embedded once) instead of discarding the index.
+
Args:
key: Edge key to edit.
remove_fact: Fact to remove from the edge (exclusive with
@@ -220,18 +357,37 @@ class GraphEditMixin:
dry_run: Return the proposed rewrite without applying it.
Returns:
- Report dict with `changed`, `applied`, `old`, and `new`.
+ Report dict with `changed`, `applied`, `old`, `new`, and
+ `index_patched`.
"""
- return self._edit_item(
- self._edge_memory,
- self.edge_editor,
- self.edge_key_extractor,
- "edge",
- key,
- remove_fact,
- instruction,
- dry_run,
- )
+ edge_vs = self._edge_memory._index
+ self._edge_memory._index = None
+ try:
+ report = self._edit_item(
+ self._edge_memory,
+ self.edge_editor,
+ self.edge_key_extractor,
+ "edge",
+ key,
+ remove_fact,
+ instruction,
+ dry_run,
+ )
+ except Exception:
+ self._edge_memory._index = edge_vs
+ raise
+
+ if edge_vs is not None:
+ if report["changed"]:
+ index_patched = _patch_vector_index(
+ self._edge_memory, edge_vs, set(), {key}
+ )
+ self._edge_memory._index = edge_vs if index_patched else None
+ else:
+ self._edge_memory._index = edge_vs
+ index_patched = True
+ report["index_patched"] = index_patched
+ return report
class AutoGraphSchema(BaseModel, Generic[NodeSchema, EdgeSchema]):
diff --git a/tests/cli/test_remove.py b/tests/cli/test_remove.py
index b91291c..a6efd9c 100644
--- a/tests/cli/test_remove.py
+++ b/tests/cli/test_remove.py
@@ -108,6 +108,28 @@ class TestHardDeleteCli:
assert [n["name"] for n in _data(ka)["nodes"]] == ["Apple", "Google"]
assert not list(ka.glob("data.json.bak.*"))
+ def test_built_index_is_patched_not_discarded(self, tmp_path, monkeypatch):
+ """With a real built index, he remove patches it in place."""
+ g = _real_ka()
+ g.metadata["template"] = "general/graph"
+ g.metadata["lang"] = "en"
+ g.build_index()
+ ka = tmp_path / "ka"
+ g.dump(ka)
+ assert (ka / "index" / "node_index" / "index.faiss").exists()
+ monkeypatch.setattr(climod.Template, "create", staticmethod(lambda *a, **k: g))
+
+ result = runner.invoke(app, ["remove", str(ka), "--node", "Apple", "-y"])
+
+ assert result.exit_code == 0, result.output
+ assert "patched in place" in result.output
+ # The index survives, and its docstore no longer holds the removed node.
+ assert (ka / "index" / "node_index" / "index.faiss").exists()
+ docstore = (ka / "index" / "node_index" / "index.pkl").read_bytes()
+ assert b"Apple" not in docstore
+ assert b"Google" in docstore
+ assert [n["name"] for n in _data(ka)["nodes"]] == ["Google"]
+
def test_no_match_reports_and_writes_nothing(self, ka_env):
ka, _ = ka_env
result = runner.invoke(app, ["remove", str(ka), "--node", "Ghost", "-y"])
diff --git a/tests/types/test_graph_removal.py b/tests/types/test_graph_removal.py
index 18c6cb3..6f78f71 100644
--- a/tests/types/test_graph_removal.py
+++ b/tests/types/test_graph_removal.py
@@ -1,7 +1,6 @@
"""Hard delete (remove_nodes / remove_edges) and soft delete (edit_*) for
graph-family AutoTypes (issue #84)."""
-
import pytest
from pydantic import BaseModel, Field
@@ -161,6 +160,126 @@ class FakeEditor:
return self.rewritten
+class TestIndexPatching:
+ """Phase A (#84): removals/edits patch the FAISS index in place.
+
+ Assertions inspect the FAISS docstore directly (keys + re-embedded
+ page_content) — MockEmbeddings is hash-based, so semantic-search
+ result ordering is not meaningful here.
+ """
+
+ @staticmethod
+ def _docstore(memory):
+ return memory._index.docstore._dict
+
+ def test_remove_node_keeps_index_without_stale_vectors(self):
+ g = _graph()
+ g._node_memory.add(
+ [Entity(name="Apple"), Entity(name="Google"), Entity(name="Meta")]
+ )
+ g._edge_memory.add(
+ [Relation(source="Apple", target="Google", relation_type="partner")]
+ )
+ g.build_index()
+ assert g._node_memory.has_index()
+
+ report = g.remove_nodes("Apple")
+
+ assert report["index_patched"] is True
+ # Index must survive the removal (no full-rebuild fallback).
+ assert g._node_memory.has_index()
+ assert g._edge_memory.has_index()
+ # The removed node's vector and its edge's vector are gone.
+ node_keys = {d.metadata["key"] for d in self._docstore(g._node_memory).values()}
+ edge_keys = {d.metadata["key"] for d in self._docstore(g._edge_memory).values()}
+ assert node_keys == {"Google", "Meta"}
+ assert edge_keys == set()
+
+ def test_remove_node_docstore_shrinks(self):
+ g = _graph()
+ g._node_memory.add([Entity(name="A"), Entity(name="B")])
+ g.build_index()
+ size_before = len(g._node_memory._index.docstore._dict)
+
+ g.remove_nodes("A")
+
+ size_after = len(g._node_memory._index.docstore._dict)
+ assert size_after == size_before - 1
+
+ def test_edit_node_refreshes_vector(self):
+ g = _graph()
+ g._node_memory.add(
+ [Entity(name="A", description="A was founded by X. A is in Y.")]
+ )
+ g.build_index()
+ g.node_editor = FakeEditor(Entity(name="A", description="A is in Y."))
+
+ report = g.edit_node("A", remove_fact="founded by X")
+
+ assert report["index_patched"] is True
+ assert g._node_memory.has_index()
+ # Exactly one vector for the key, re-embedded with the new text.
+ docs = [
+ d
+ for d in self._docstore(g._node_memory).values()
+ if d.metadata["key"] == "A"
+ ]
+ assert len(docs) == 1
+ assert "founded by X" not in docs[0].page_content
+ assert "A is in Y" in docs[0].page_content
+
+ def test_no_index_built_falls_back_to_lazy_rebuild(self):
+ g = _graph()
+ g._node_memory.add([Entity(name="A"), Entity(name="B")])
+ # No build_index() call — nothing to patch.
+
+ report = g.remove_nodes("A")
+
+ assert report["index_patched"] is False
+ assert not g._node_memory.has_index()
+ # OMem's lazy rebuild still reflects the removal.
+ assert "A" not in {n.name for n in g.nodes}
+
+ def test_patched_index_survives_dump_load(self, tmp_path):
+ g = _graph()
+ g._node_memory.add([Entity(name="A"), Entity(name="B")])
+ g._edge_memory.add([Relation(source="A", target="B", relation_type="r")])
+ g.build_index()
+ g.remove_nodes("A")
+ g.dump(tmp_path)
+
+ reloaded = _graph()
+ reloaded.load(tmp_path)
+ assert reloaded._node_memory.has_index()
+
+ node_keys = {
+ d.metadata["key"] for d in self._docstore(reloaded._node_memory).values()
+ }
+ assert node_keys == {"B"}
+ edge_keys = {
+ d.metadata["key"] for d in self._docstore(reloaded._edge_memory).values()
+ }
+ assert edge_keys == set()
+
+ def test_hypergraph_removal_patches_index(self):
+ h = _hypergraph()
+ h._node_memory.add([Entity(name="A"), Entity(name="B"), Entity(name="C")])
+ h._edge_memory.add(
+ [HyperRelation(participants=["A", "B"], relation_type="group")]
+ )
+ h.build_index()
+
+ report = h.remove_nodes("A")
+
+ assert report["index_patched"] is True
+ assert h._node_memory.has_index()
+ node_keys = {d.metadata["key"] for d in self._docstore(h._node_memory).values()}
+ assert node_keys == {"B", "C"}
+ # The hyperedge containing the removed node is gone from the index.
+ edge_keys = {d.metadata["key"] for d in self._docstore(h._edge_memory).values()}
+ assert edge_keys == set()
+
+
class TestSoftDelete:
"""edit_node / edit_edge: LLM-assisted fact removal with guardrails."""
@@ -259,9 +378,7 @@ class TestSoftDelete:
)
)
- report = g.edit_edge(
- "A-acquired-B", remove_fact="first announced in March"
- )
+ report = g.edit_edge("A-acquired-B", remove_fact="first announced in March")
assert report["applied"] is True
assert "announced in March" not in g.edges[0].description