Hypergraph is more powerful. Transform unstructured text into structured knowledge with LLMs. Graphs, hypergraphs, and spatio-temporal extractions — with one command.
feat: let he feed ingest directories like he parse
Frankie-Xud5e4c7a88a98
diff --git a/docs/en/cli/commands/feed.md b/docs/en/cli/commands/feed.mdindex ceec9ae..4ec9cc9 100644--- a/docs/en/cli/commands/feed.md+++ b/docs/en/cli/commands/feed.md@@ -15,7 +15,7 @@ he feed KA_PATH INPUT [OPTIONS]| Argument | Description ||----------|-------------|| `KA_PATH` | Path to existing knowledge abstract directory |-| `INPUT` | Input file or `-` for stdin |+| `INPUT` | Input file, directory, or `-` for stdin |Supported suffixes: `.txt`/`.md` always; PDF, Word, PowerPoint, Excel, HTML,CSV, JSON, XML, EPUB and more via the optional ingest extra —@@ -69,6 +69,12 @@ he feed ./ka/ doc2.mdhe feed ./ka/ doc3.md```+Or feed a directory (same as `he parse`): each supported file is ingested, attributed by file stem unless `--source` is set. Unsupported extensions are skipped with a warning; if the directory has no readable files, the command exits non-zero.++```bash+he feed ./ka/ ./updates/+```+Or use a loop:```bashdiff --git a/docs/zh/cli/commands/feed.md b/docs/zh/cli/commands/feed.mdindex 2aad521..cfb5885 100644--- a/docs/zh/cli/commands/feed.md+++ b/docs/zh/cli/commands/feed.md@@ -15,7 +15,7 @@ he feed KA_PATH INPUT [OPTIONS]| 参数 | 描述 ||----------|-------------|| `KA_PATH` | 现有知识库目录的路径 |-| `INPUT` | 输入文件或 `-` 表示标准输入 |+| `INPUT` | 输入文件、目录,或 `-` 表示标准输入 |支持的后缀:`.txt`/`.md` 始终支持;PDF、Word、PowerPoint、Excel、HTML、CSV、JSON、XML、EPUB 等需要可选的 ingest extra——`pip install "hyperextract[ingest]"`(完整表格见 [he parse](parse.md))。扫描版(纯图片)PDF 没有文字层——请先做 OCR。标准输入(`-`)不检查后缀。@@ -65,6 +65,12 @@ he feed ./ka/ doc2.mdhe feed ./ka/ doc3.md```+也可以喂入目录(与 `he parse` 一致):每个支持的文件单独入库,来源 id 默认为文件 stem,除非指定了 `--source`。不支持的扩展名会跳过并警告;目录里没有可读文件则非 0 退出。++```bash+he feed ./ka/ ./updates/+```+或使用循环:```bashdiff --git a/hyperextract/cli/cli.py b/hyperextract/cli/cli.pyindex 4f6d33d..8209f3d 100644--- a/hyperextract/cli/cli.py+++ b/hyperextract/cli/cli.py@@ -1089,10 +1089,60 @@ def talk(console.print(f"[dim] he show {ka_path} # Visualize[/dim]")+def _feed_one_document(+ ka,+ output_path: Path,+ input: str,+ source: str | None,+ refeed: bool,+ store_doc: bool,+) -> bool:+ """Ingest one file or stdin into ``ka``. Returns False if skipped unchanged."""+ require_supported_text_input(input)+ text = read_input(input)+ console.print(+ f"[dim]Input {Path(input).name if input != '-' else 'stdin'}: "+ f"{len(text)} characters"+ f"{f' (source: {source})' if source else ''}[/dim]"+ )++ text_hash_to_record = None+ if source:+ text_hash_to_record = hashlib.sha256(text.encode("utf-8")).hexdigest()+ if not refeed:+ recorded = ka.source_content_hash(source)+ if recorded == text_hash_to_record:+ logger.info("stage=source_unchanged source=%s", source)+ console.print(+ f"[yellow]Source '{source}' is unchanged (content hash "+ "matches) — nothing to do.[/yellow] "+ "Use --refeed to re-ingest anyway."+ )+ return False++ if store_doc and source:+ from hyperextract.utils.document_store import SourceDocumentStore++ original_name = "stdin.txt" if input == "-" else Path(input).name+ store = SourceDocumentStore(output_path)+ if input == "-":+ stored_doc = store.store_text(source, text, original_name)+ else:+ stored_doc = store.store_file(source, input)+ console.print(f"[dim]Document archived: {stored_doc}[/dim]")++ logger.debug("stage=feed_text_invoked")+ ka.feed_text(text, source_id=source, content_hash=text_hash_to_record)+ logger.info("stage=knowledge_appended chars=%d source=%s", len(text), source)+ return True++@app.command(name="feed")def feed(ka_path: str = typer.Argument(..., help="Knowledge Abstract directory"),- input: str = typer.Argument(..., help="Input file path or '-' for stdin"),+ input: str = typer.Argument(+ ..., help="Input file path, directory, or '-' for stdin"+ ),template: str | None = typer.Option(None, "--template", "-t", help="Template"),lang: str | None = typer.Option(None, "--lang", "-l", help="Language"),source: str | None = typer.Option(@@ -1151,50 +1201,49 @@ def feed(ka.load(output_path)- progress.update(task, description="Reading input...")- require_supported_text_input(input)- text = read_input(input)- console.print(f"[dim]Input text: {len(text)} characters[/dim]")-- # Change detection: skip unchanged sources without any LLM calls.- text_hash_to_record = None- if source:- text_hash_to_record = hashlib.sha256(text.encode("utf-8")).hexdigest()- if not refeed:- recorded = ka.source_content_hash(source)- if recorded == text_hash_to_record:- logger.info("stage=source_unchanged source=%s", source)- console.print(- f"[yellow]Source '{source}' is unchanged (content hash "- "matches) — nothing to do.[/yellow] "- "Use --refeed to re-ingest anyway."- )- raise typer.Exit(0)-- # Archive the source document (provenance evidence, kept across- # rollbacks; purge with he remove --document ... --purge-documents).- stored_doc = None- if store_doc and source:- from hyperextract.utils.document_store import SourceDocumentStore-- original_name = "stdin.txt" if input == "-" else Path(input).name- store = SourceDocumentStore(output_path)- if input == "-":- stored_doc = store.store_text(source, text, original_name)- else:- stored_doc = store.store_file(source, input)- console.print(f"[dim]Document archived: {stored_doc}[/dim]")-- progress.update(task, description="Appending knowledge...")- logger.debug("stage=feed_text_invoked")- ka.feed_text(text, source_id=source, content_hash=text_hash_to_record)- logger.info("stage=knowledge_appended chars=%d", len(text))+ input_path = Path(input) if input != "-" else None+ ingested = 0+ if input_path is not None and input_path.is_dir():+ progress.update(task, description="Processing directory...")+ text_files = collect_directory_text_inputs(input_path)+ file_sources = [source or file_path.stem for file_path in text_files]+ progress.update(task, description="Appending knowledge...")+ for file_path, file_source in zip(text_files, file_sources):+ if _feed_one_document(+ ka,+ output_path,+ str(file_path),+ file_source,+ refeed,+ store_doc,+ ):+ ingested += 1+ logger.info(+ "stage=directory_feed_complete files=%d ingested=%d",+ len(text_files),+ ingested,+ )+ else:+ progress.update(task, description="Reading input...")+ if not _feed_one_document(+ ka, output_path, input, source, refeed, store_doc+ ):+ raise typer.Exit(0)+ ingested = 1- progress.update(task, description="Saving data...")- ka.dump(output_path)- logger.info("stage=data_saved")+ if ingested:+ progress.update(task, description="Saving data...")+ ka.dump(output_path)+ logger.info("stage=data_saved")console.print()+ if ingested == 0:+ console.print(+ "[yellow]No documents were ingested.[/yellow] "+ "Supported files were unchanged or skipped."+ )+ return+console.print(f"[bold green]Success![/bold green] Knowledge appended to {output_path}")diff --git a/tests/cli/test_parse_input_types.py b/tests/cli/test_parse_input_types.pyindex 9459a8a..31b56c8 100644--- a/tests/cli/test_parse_input_types.py+++ b/tests/cli/test_parse_input_types.py@@ -221,6 +221,34 @@ class TestFeedInputTypes:assert result.exit_code == 0, result.outputassert "Hello Hyper Extract from DOCX" in ka.feed_text.call_args[0][0]+ def test_feed_directory_two_txt_sources(self, tmp_path):+ ka_dir = _make_ka(tmp_path)+ folder = tmp_path / "docs"+ folder.mkdir()+ (folder / "alpha.txt").write_text("first document", encoding="utf-8")+ (folder / "beta.txt").write_text("second document", encoding="utf-8")+ with _mock_feed() as ka:+ result = runner.invoke(app, ["feed", str(ka_dir), str(folder)])++ assert result.exit_code == 0, result.output+ ids = [call.kwargs.get("source_id") for call in ka.feed_text.call_args_list]+ texts = [call.args[0] for call in ka.feed_text.call_args_list]+ assert ids == ["alpha", "beta"]+ assert texts == ["first document", "second document"]+ ka.dump.assert_called()++ def test_feed_directory_unsupported_only_exits(self, tmp_path):+ ka_dir = _make_ka(tmp_path)+ folder = tmp_path / "docs"+ folder.mkdir()+ (folder / "data.bin").write_bytes(b"\x00\x01")+ with _mock_feed() as ka:+ result = runner.invoke(app, ["feed", str(ka_dir), str(folder)])++ assert result.exit_code == 1+ ka.feed_text.assert_not_called()+ ka.dump.assert_not_called()+def test_feed_document_without_backend_rejected_with_hint(self, tmp_path, monkeypatch):@@ -239,6 +267,37 @@ class TestFeedInputTypes:ka.dump.assert_not_called()+def test_feed_directory_writes_two_sources_to_ledger(tmp_path):+ """Two txt files become two AutoDocument source-ledger entries after feed."""+ from hyperextract.types import AutoDocument+ from tests.mocks import MockChatModel, MockEmbeddings++ ka_dir = tmp_path / "ka"+ seed = AutoDocument(llm_client=MockChatModel(), embedder=MockEmbeddings())+ seed.metadata["template"] = "general/graph"+ seed.metadata["lang"] = "en"+ seed.dump(ka_dir)++ folder = tmp_path / "docs"+ folder.mkdir()+ (folder / "alpha.txt").write_text("first document", encoding="utf-8")+ (folder / "beta.txt").write_text("second document", encoding="utf-8")++ live = AutoDocument(llm_client=MockChatModel(), embedder=MockEmbeddings())+ with (+ patch("hyperextract.cli.cli.validate_config"),+ patch("hyperextract.cli.cli.Template.create", return_value=live),+ ):+ result = runner.invoke(app, ["feed", str(ka_dir), str(folder)])++ assert result.exit_code == 0, result.output+ assert set(live.sources()) == {"alpha", "beta"}+ data = json.loads((ka_dir / "data.json").read_text(encoding="utf-8"))+ assert len(data.get("chunks", [])) == 2+ ledger = json.loads((ka_dir / "sources_chunks.json").read_text(encoding="utf-8"))+ assert {entry["source_id"] for entry in ledger} == {"alpha", "beta"}++def test_backend_probe_function_exists():# Direct sanity check so the import surface stays stable.assert callable(markitdown_available)