Hypergraph is more powerful. Transform unstructured text into structured knowledge with LLMs. Graphs, hypergraphs, and spatio-temporal extractions — with one command.
feat: archive source documents under ka documents/ (per-document provenance evidence)
yifanfeng973581962760b3
diff --git a/hyperextract/cli/cli.py b/hyperextract/cli/cli.pyindex 74c4af3..013b7ea 100644--- a/hyperextract/cli/cli.py+++ b/hyperextract/cli/cli.py@@ -359,6 +359,10 @@ def parse(progress.update(task, description="Extracting knowledge...")logger.debug("stage=feed_text_invoked")+ if source:+ from hyperextract.utils.document_store import SourceDocumentStore++ SourceDocumentStore(output_path).store_file(source, input)ka.feed_text(text, source_id=source)logger.info("stage=knowledge_extracted chars=%d", len(text))@@ -751,6 +755,7 @@ def info(sources_table.add_column("Source ID", style="cyan")sources_table.add_column("Raw Items", justify="right")sources_table.add_column("Content Hash")+ sources_table.add_column("Archived Document")ledger_files = [(path / "sources_nodes.json", "nodes"),@@ -767,6 +772,9 @@ def info("sources_ledger_read_failed path=%s error=%s", ledger_path, e)continue+ from hyperextract.utils.document_store import SourceDocumentStore++ doc_store = SourceDocumentStore(path)for entry in entries:sid = entry.get("source_id")if sid is None:@@ -778,8 +786,13 @@ def info(if combined:for sid, info_row in sorted(combined.items()):+ archived = doc_store.find(sid)+ archived_name = archived[0].name if archived else "—"sources_table.add_row(- sid, str(info_row["raw_items"]), info_row["content_hash"] or "—"+ sid,+ str(info_row["raw_items"]),+ info_row["content_hash"] or "—",+ archived_name,)console.print(sources_table)else:@@ -998,6 +1011,11 @@ def feed("--refeed",help="Re-ingest even if this source's content hash is unchanged",),+ store_doc: bool = typer.Option(+ True,+ "--store-doc/--no-store-doc",+ help="Archive the source document under documents/ (requires --source)",+ ),):"""Append knowledge to an existing Knowledge Abstract."""logger.info("command=feed ka_path=%s input=%s", ka_path, input)@@ -1059,6 +1077,20 @@ def feed()raise typer.Exit(0)+ # Archive the source document (provenance evidence, kept across+ # rollbacks; purge with he remove --document ... --purge-documents).+ stored_doc = None+ if store_doc and source:+ from hyperextract.utils.document_store import SourceDocumentStore++ original_name = "stdin.txt" if input == "-" else Path(input).name+ store = SourceDocumentStore(output_path)+ if input == "-":+ stored_doc = store.store_text(source, text, original_name)+ else:+ stored_doc = store.store_file(source, input)+ console.print(f"[dim]Document archived: {stored_doc}[/dim]")+progress.update(task, description="Appending knowledge...")logger.debug("stage=feed_text_invoked")ka.feed_text(text, source_id=source, content_hash=text_hash_to_record)@@ -1231,6 +1263,11 @@ def remove_items(help="Rollback strategy for --document: exact (re-merge surviving ""sources) or touched (delete all affected keys)",),+ purge_documents: bool = typer.Option(+ False,+ "--purge-documents",+ help="With --document: also delete the archived source document file",+ ),):"""Delete nodes/edges by key, remove a fact (LLM-assisted), or roll back a whole document.@@ -1343,6 +1380,13 @@ def remove_items(console.print()if document:+ if purge_documents:+ from hyperextract.utils.document_store import SourceDocumentStore++ removed_files = SourceDocumentStore(path).purge(document)+ for f in removed_files:+ console.print(f"[dim]Archived document deleted: {f}[/dim]")+table = Table(title=f"Document Rollback Report — {document}")table.add_column("Field")table.add_column("Items")diff --git a/hyperextract/utils/document_store.py b/hyperextract/utils/document_store.pynew file mode 100644index 0000000..98e2147--- /dev/null+++ b/hyperextract/utils/document_store.py@@ -0,0 +1,66 @@+"""Archive source documents inside the KA directory (``documents/``).++Files are named deterministically from the ``source_id`` so they can be+located later without a metadata lookup:++ documents/<sha256(source_id)[:12]>-<original basename>++This keeps one *current* copy per source: re-feeding the same source_id+overwrites the archived file, mirroring the ledger's bounded+"current version per source" policy.+"""++import hashlib+import shutil+from pathlib import Path++DOCUMENTS_DIR = "documents"+_HASH_LEN = 12+++def source_hash(source_id: str) -> str:+ """Stable short hash used as the archived filename prefix."""+ return hashlib.sha256(source_id.encode("utf-8")).hexdigest()[:_HASH_LEN]+++class SourceDocumentStore:+ """Filesystem archive of raw source documents for one KA."""++ def __init__(self, ka_root: Path):+ self.root = Path(ka_root) / DOCUMENTS_DIR++ def path_for(self, source_id: str, original_name: str) -> Path:+ """Deterministic archive path for a source (original name kept for humans)."""+ safe_name = Path(original_name).name or "document.txt"+ return self.root / f"{source_hash(source_id)}-{safe_name}"++ def store_text(+ self, source_id: str, text: str, original_name: str = "document.txt"+ ) -> Path:+ """Archive text content (used for stdin feeds)."""+ self.root.mkdir(parents=True, exist_ok=True)+ path = self.path_for(source_id, original_name)+ path.write_text(text, encoding="utf-8")+ return path++ def store_file(self, source_id: str, input_path: str | Path) -> Path:+ """Archive an original file (raw bytes preserved)."""+ src = Path(input_path)+ self.root.mkdir(parents=True, exist_ok=True)+ dst = self.path_for(source_id, src.name)+ shutil.copy2(src, dst)+ return dst++ def find(self, source_id: str) -> list[Path]:+ """All archived files for a source (usually one)."""+ if not self.root.exists():+ return []+ return sorted(self.root.glob(f"{source_hash(source_id)}-*"))++ def purge(self, source_id: str) -> list[Path]:+ """Delete archived files for a source. Returns the removed paths."""+ removed = []+ for path in self.find(source_id):+ path.unlink()+ removed.append(path)+ return removed