Skip to content

Hypergraph is more powerful. Transform unstructured text into structured knowledge with LLMs. Graphs, hypergraphs, and spatio-temporal extractions — with one command.

feat: archive source documents under ka documents/ (per-document provenance evidence)

yifanfeng973581962760b3
diff --git a/hyperextract/cli/cli.py b/hyperextract/cli/cli.py
index 74c4af3..013b7ea 100644
--- a/hyperextract/cli/cli.py
+++ b/hyperextract/cli/cli.py
@@ -359,6 +359,10 @@ def parse(
progress.update(task, description="Extracting knowledge...")
logger.debug("stage=feed_text_invoked")
+ if source:
+ from hyperextract.utils.document_store import SourceDocumentStore
+
+ SourceDocumentStore(output_path).store_file(source, input)
ka.feed_text(text, source_id=source)
logger.info("stage=knowledge_extracted chars=%d", len(text))
@@ -751,6 +755,7 @@ def info(
sources_table.add_column("Source ID", style="cyan")
sources_table.add_column("Raw Items", justify="right")
sources_table.add_column("Content Hash")
+ sources_table.add_column("Archived Document")
ledger_files = [
(path / "sources_nodes.json", "nodes"),
@@ -767,6 +772,9 @@ def info(
"sources_ledger_read_failed path=%s error=%s", ledger_path, e
)
continue
+ from hyperextract.utils.document_store import SourceDocumentStore
+
+ doc_store = SourceDocumentStore(path)
for entry in entries:
sid = entry.get("source_id")
if sid is None:
@@ -778,8 +786,13 @@ def info(
if combined:
for sid, info_row in sorted(combined.items()):
+ archived = doc_store.find(sid)
+ archived_name = archived[0].name if archived else "—"
sources_table.add_row(
- sid, str(info_row["raw_items"]), info_row["content_hash"] or "—"
+ sid,
+ str(info_row["raw_items"]),
+ info_row["content_hash"] or "—",
+ archived_name,
)
console.print(sources_table)
else:
@@ -998,6 +1011,11 @@ def feed(
"--refeed",
help="Re-ingest even if this source's content hash is unchanged",
),
+ store_doc: bool = typer.Option(
+ True,
+ "--store-doc/--no-store-doc",
+ help="Archive the source document under documents/ (requires --source)",
+ ),
):
"""Append knowledge to an existing Knowledge Abstract."""
logger.info("command=feed ka_path=%s input=%s", ka_path, input)
@@ -1059,6 +1077,20 @@ def feed(
)
raise typer.Exit(0)
+ # Archive the source document (provenance evidence, kept across
+ # rollbacks; purge with he remove --document ... --purge-documents).
+ stored_doc = None
+ if store_doc and source:
+ from hyperextract.utils.document_store import SourceDocumentStore
+
+ original_name = "stdin.txt" if input == "-" else Path(input).name
+ store = SourceDocumentStore(output_path)
+ if input == "-":
+ stored_doc = store.store_text(source, text, original_name)
+ else:
+ stored_doc = store.store_file(source, input)
+ console.print(f"[dim]Document archived: {stored_doc}[/dim]")
+
progress.update(task, description="Appending knowledge...")
logger.debug("stage=feed_text_invoked")
ka.feed_text(text, source_id=source, content_hash=text_hash_to_record)
@@ -1231,6 +1263,11 @@ def remove_items(
help="Rollback strategy for --document: exact (re-merge surviving "
"sources) or touched (delete all affected keys)",
),
+ purge_documents: bool = typer.Option(
+ False,
+ "--purge-documents",
+ help="With --document: also delete the archived source document file",
+ ),
):
"""Delete nodes/edges by key, remove a fact (LLM-assisted), or roll back a whole document.
@@ -1343,6 +1380,13 @@ def remove_items(
console.print()
if document:
+ if purge_documents:
+ from hyperextract.utils.document_store import SourceDocumentStore
+
+ removed_files = SourceDocumentStore(path).purge(document)
+ for f in removed_files:
+ console.print(f"[dim]Archived document deleted: {f}[/dim]")
+
table = Table(title=f"Document Rollback Report — {document}")
table.add_column("Field")
table.add_column("Items")
diff --git a/hyperextract/utils/document_store.py b/hyperextract/utils/document_store.py
new file mode 100644
index 0000000..98e2147
--- /dev/null
+++ b/hyperextract/utils/document_store.py
@@ -0,0 +1,66 @@
+"""Archive source documents inside the KA directory (``documents/``).
+
+Files are named deterministically from the ``source_id`` so they can be
+located later without a metadata lookup:
+
+ documents/<sha256(source_id)[:12]>-<original basename>
+
+This keeps one *current* copy per source: re-feeding the same source_id
+overwrites the archived file, mirroring the ledger's bounded
+"current version per source" policy.
+"""
+
+import hashlib
+import shutil
+from pathlib import Path
+
+DOCUMENTS_DIR = "documents"
+_HASH_LEN = 12
+
+
+def source_hash(source_id: str) -> str:
+ """Stable short hash used as the archived filename prefix."""
+ return hashlib.sha256(source_id.encode("utf-8")).hexdigest()[:_HASH_LEN]
+
+
+class SourceDocumentStore:
+ """Filesystem archive of raw source documents for one KA."""
+
+ def __init__(self, ka_root: Path):
+ self.root = Path(ka_root) / DOCUMENTS_DIR
+
+ def path_for(self, source_id: str, original_name: str) -> Path:
+ """Deterministic archive path for a source (original name kept for humans)."""
+ safe_name = Path(original_name).name or "document.txt"
+ return self.root / f"{source_hash(source_id)}-{safe_name}"
+
+ def store_text(
+ self, source_id: str, text: str, original_name: str = "document.txt"
+ ) -> Path:
+ """Archive text content (used for stdin feeds)."""
+ self.root.mkdir(parents=True, exist_ok=True)
+ path = self.path_for(source_id, original_name)
+ path.write_text(text, encoding="utf-8")
+ return path
+
+ def store_file(self, source_id: str, input_path: str | Path) -> Path:
+ """Archive an original file (raw bytes preserved)."""
+ src = Path(input_path)
+ self.root.mkdir(parents=True, exist_ok=True)
+ dst = self.path_for(source_id, src.name)
+ shutil.copy2(src, dst)
+ return dst
+
+ def find(self, source_id: str) -> list[Path]:
+ """All archived files for a source (usually one)."""
+ if not self.root.exists():
+ return []
+ return sorted(self.root.glob(f"{source_hash(source_id)}-*"))
+
+ def purge(self, source_id: str) -> list[Path]:
+ """Delete archived files for a source. Returns the removed paths."""
+ removed = []
+ for path in self.find(source_id):
+ path.unlink()
+ removed.append(path)
+ return removed