Skip to content

Hypergraph is more powerful. Transform unstructured text into structured knowledge with LLMs. Graphs, hypergraphs, and spatio-temporal extractions — with one command.

feat: document upsert semantics + per-file attribution + spatiotemporal provenance tests

yifanfeng97cbb4acb09a39
diff --git a/hyperextract/cli/cli.py b/hyperextract/cli/cli.py
index 3f626ff..74c4af3 100644
--- a/hyperextract/cli/cli.py
+++ b/hyperextract/cli/cli.py
@@ -333,19 +333,24 @@ def parse(
progress.update(task, description="Processing directory...")
text_files = collect_directory_text_inputs(input_path)
+ # Per-file provenance: each file is attributed by its stem
+ # unless a single --source id was given explicitly.
+ file_sources = [source or file_path.stem for file_path in text_files]
+
all_text = []
- for file_path in text_files:
+ for file_path, file_source in zip(text_files, file_sources):
text = read_input(str(file_path))
all_text.append(text)
- console.print(f"[dim]Loaded {file_path.name}: {len(text)} chars[/dim]")
-
- combined_text = "\n\n".join(all_text)
- console.print(f"[dim]Total input: {len(combined_text)} characters[/dim]")
+ console.print(
+ f"[dim]Loaded {file_path.name}: {len(text)} chars "
+ f"(source: {file_source})[/dim]"
+ )
progress.update(task, description="Extracting knowledge...")
logger.debug("stage=feed_text_invoked")
- ka.feed_text(combined_text, source_id=source)
- logger.info("stage=knowledge_extracted chars=%d", len(combined_text))
+ for file_path, file_source, text in zip(text_files, file_sources, all_text):
+ ka.feed_text(text, source_id=file_source)
+ logger.info("stage=knowledge_extracted files=%d", len(text_files))
else:
progress.update(task, description="Reading input...")
require_supported_text_input(input)
diff --git a/hyperextract/types/graph.py b/hyperextract/types/graph.py
index f9daa86..2ef4519 100644
--- a/hyperextract/types/graph.py
+++ b/hyperextract/types/graph.py
@@ -194,6 +194,34 @@ class GraphEditMixin:
) or self._edge_memory._sources.get(source_id)
return record.content_hash if record else None
+ # ==================== Document Upsert ====================
+
+ def feed_text(
+ self,
+ text: str,
+ *,
+ source_id: str | None = None,
+ content_hash: str | None = None,
+ ) -> "AutoGraph":
+ """Feed text with document-level upsert semantics.
+
+ When ``source_id`` was attributed before, the previous version's
+ contributions are rolled back first (exact re-merge from surviving
+ sources), so facts removed from the updated document do not linger.
+ First-time attribution behaves like a normal feed.
+ """
+ if (
+ source_id
+ and self._node_memory.track_sources
+ and source_id in self.sources()
+ ):
+ logger.info("stage=source_upsert rollback source=%s", source_id)
+ if source_id in self._node_memory.sources():
+ self._node_memory.remove_source(source_id)
+ if source_id in self._edge_memory.sources():
+ self._edge_memory.remove_source(source_id)
+ return super().feed_text(text, source_id=source_id, content_hash=content_hash)
+
# ==================== Removal ====================
def remove_nodes(self, *keys: str) -> dict:
diff --git a/tests/cli/test_parse_input_types.py b/tests/cli/test_parse_input_types.py
index 46b1af0..94a9886 100644
--- a/tests/cli/test_parse_input_types.py
+++ b/tests/cli/test_parse_input_types.py
@@ -140,7 +140,8 @@ class TestParseInputTypes:
result2 = _invoke_parse(folder, out2)
assert result2.exit_code == 0
- ka2.feed_text.assert_called_once_with("upper md", source_id=None)
+ # Per-file provenance: auto-attributed by file stem.
+ ka2.feed_text.assert_called_once_with("upper md", source_id="README")
def test_stdin_is_not_suffix_checked(self, tmp_path):
out = tmp_path / "out"
diff --git a/tests/types/test_graph_removal.py b/tests/types/test_graph_removal.py
index 3654e42..07d651d 100644
--- a/tests/types/test_graph_removal.py
+++ b/tests/types/test_graph_removal.py
@@ -2,9 +2,14 @@
graph-family AutoTypes (issue #84)."""
import pytest
+from ontomem.merger import MergeStrategy
from pydantic import BaseModel, Field
-from hyperextract.types import AutoGraph, AutoHypergraph
+from hyperextract.types import (
+ AutoGraph,
+ AutoHypergraph,
+ AutoTemporalGraph,
+)
from tests.mocks import MockChatModel, MockEmbeddings
@@ -35,6 +40,8 @@ def _graph():
nodes_in_edge_extractor=lambda x: (x.source, x.target),
llm_client=MockChatModel(),
embedder=MockEmbeddings(),
+ node_strategy_or_merger=MergeStrategy.MERGE_FIELD,
+ edge_strategy_or_merger=MergeStrategy.MERGE_FIELD,
)
@@ -173,6 +180,115 @@ class FakeChatModel:
return _Runnable()
+class _FakeGraphExtractor:
+ """One-stage extractor stub returning a canned graph."""
+
+ def __init__(self, result):
+ self.result = result
+
+ def invoke(self, input, config=None):
+ return self.result
+
+ def batch(self, inputs, config=None, return_exceptions=False, **kwargs):
+ return [self.result for _ in inputs]
+
+
+class TestAutoUpsert:
+ """Re-feeding the same source_id replaces that document's version
+ (facts removed from the new version do not linger)."""
+
+ def test_second_feed_rolls_back_v1_contributions(self):
+ g = _graph()
+ v1 = g.graph_schema(
+ nodes=[
+ Entity(name="A", description="fact v1"),
+ Entity(name="B", description="only in v1"),
+ ],
+ edges=[],
+ )
+ g.data_extractor = _FakeGraphExtractor(v1)
+ g.feed_text("v1 text", source_id="doc-1")
+ assert {n.name for n in g.nodes} == {"A", "B"}
+
+ v2 = g.graph_schema(nodes=[Entity(name="A", description="fact v2")], edges=[])
+ g.data_extractor = _FakeGraphExtractor(v2)
+ g.feed_text("v2 text", source_id="doc-1")
+
+ # B existed only in v1 → rolled back; A carries the v2 fact.
+ assert {n.name for n in g.nodes} == {"A"}
+ a = next(n for n in g.nodes if n.name == "A")
+ assert a.description == "fact v2"
+
+ def test_ledger_keeps_only_current_version(self):
+ g = _graph()
+ v1 = g.graph_schema(nodes=[Entity(name="A", description="v1")], edges=[])
+ g.data_extractor = _FakeGraphExtractor(v1)
+ g.feed_text("v1 text", source_id="doc-1")
+ assert len(g._node_memory._sources["doc-1"].raw_items) == 1
+
+ v2 = g.graph_schema(nodes=[Entity(name="A", description="v2")], edges=[])
+ g.data_extractor = _FakeGraphExtractor(v2)
+ g.feed_text("v2 text", source_id="doc-1")
+
+ raws = g._node_memory._sources["doc-1"].raw_items
+ assert len(raws) == 1
+ assert raws[0]["description"] == "v2"
+
+ def test_feed_without_source_does_not_rollback(self):
+ g = _graph()
+ v1 = g.graph_schema(
+ nodes=[
+ Entity(name="A", description="v1"),
+ Entity(name="B", description="B"),
+ ],
+ edges=[],
+ )
+ g.data_extractor = _FakeGraphExtractor(v1)
+ g.feed_text("v1 text", source_id="doc-1")
+
+ # v2 fed WITHOUT a source: no rollback — B stays.
+ v2 = g.graph_schema(nodes=[Entity(name="A", description="v2")], edges=[])
+ g.data_extractor = _FakeGraphExtractor(v2)
+ g.feed_text("v2 text")
+
+ assert {n.name for n in g.nodes} == {"A", "B"}
+
+
+class TestSpatiotemporalProvenance:
+ """Temporal/spatial subclasses participate in source provenance."""
+
+ def _temporal(self):
+ return AutoTemporalGraph(
+ node_schema=Entity,
+ edge_schema=Relation,
+ node_key_extractor=lambda x: x.name,
+ edge_key_extractor=lambda x: f"{x.source}-{x.relation_type}-{x.target}",
+ time_in_edge_extractor=lambda x: "",
+ nodes_in_edge_extractor=lambda x: (x.source, x.target),
+ llm_client=MockChatModel(),
+ embedder=MockEmbeddings(),
+ observation_time="2026-09-05",
+ extraction_mode="one_stage",
+ chunk_size=100_000,
+ )
+
+ def test_temporal_feed_records_and_rolls_back(self):
+ g = self._temporal()
+ g.data_extractor = _FakeGraphExtractor(
+ g.graph_schema(
+ nodes=[Entity(name="T1")],
+ edges=[Relation(source="T1", target="T1", relation_type="self")],
+ )
+ )
+ g.feed_text("temporal doc", source_id="doc-1")
+ assert "doc-1" in g._node_memory._sources
+ assert "doc-1" in g._edge_memory._sources
+
+ report = g.remove_source("doc-1")
+ assert set(report["removed_nodes"]) == {"T1"}
+ assert g.empty()
+
+
class TestIndexPatching:
"""Phase A (#84): removals/edits patch the FAISS index in place.