Skip to content

Hypergraph is more powerful. Transform unstructured text into structured knowledge with LLMs. Graphs, hypergraphs, and spatio-temporal extractions — with one command.

feat: let he feed ingest directories like he parse

Frankie-Xud5e4c7a88a98
diff --git a/docs/en/cli/commands/feed.md b/docs/en/cli/commands/feed.md
index ceec9ae..4ec9cc9 100644
--- a/docs/en/cli/commands/feed.md
+++ b/docs/en/cli/commands/feed.md
@@ -15,7 +15,7 @@ he feed KA_PATH INPUT [OPTIONS]
| Argument | Description |
|----------|-------------|
| `KA_PATH` | Path to existing knowledge abstract directory |
-| `INPUT` | Input file or `-` for stdin |
+| `INPUT` | Input file, directory, or `-` for stdin |
Supported suffixes: `.txt`/`.md` always; PDF, Word, PowerPoint, Excel, HTML,
CSV, JSON, XML, EPUB and more via the optional ingest extra —
@@ -69,6 +69,12 @@ he feed ./ka/ doc2.md
he feed ./ka/ doc3.md
```
+Or feed a directory (same as `he parse`): each supported file is ingested, attributed by file stem unless `--source` is set. Unsupported extensions are skipped with a warning; if the directory has no readable files, the command exits non-zero.
+
+```bash
+he feed ./ka/ ./updates/
+```
+
Or use a loop:
```bash
diff --git a/docs/zh/cli/commands/feed.md b/docs/zh/cli/commands/feed.md
index 2aad521..cfb5885 100644
--- a/docs/zh/cli/commands/feed.md
+++ b/docs/zh/cli/commands/feed.md
@@ -15,7 +15,7 @@ he feed KA_PATH INPUT [OPTIONS]
| 参数 | 描述 |
|----------|-------------|
| `KA_PATH` | 现有知识库目录的路径 |
-| `INPUT` | 输入文件或 `-` 表示标准输入 |
+| `INPUT` | 输入文件、目录,或 `-` 表示标准输入 |
支持的后缀:`.txt`/`.md` 始终支持;PDF、Word、PowerPoint、Excel、HTML、CSV、JSON、XML、EPUB 等需要可选的 ingest extra——`pip install "hyperextract[ingest]"`(完整表格见 [he parse](parse.md))。扫描版(纯图片)PDF 没有文字层——请先做 OCR。标准输入(`-`)不检查后缀。
@@ -65,6 +65,12 @@ he feed ./ka/ doc2.md
he feed ./ka/ doc3.md
```
+也可以喂入目录(与 `he parse` 一致):每个支持的文件单独入库,来源 id 默认为文件 stem,除非指定了 `--source`。不支持的扩展名会跳过并警告;目录里没有可读文件则非 0 退出。
+
+```bash
+he feed ./ka/ ./updates/
+```
+
或使用循环:
```bash
diff --git a/hyperextract/cli/cli.py b/hyperextract/cli/cli.py
index 4f6d33d..8209f3d 100644
--- a/hyperextract/cli/cli.py
+++ b/hyperextract/cli/cli.py
@@ -1089,10 +1089,60 @@ def talk(
console.print(f"[dim] he show {ka_path} # Visualize[/dim]")
+def _feed_one_document(
+ ka,
+ output_path: Path,
+ input: str,
+ source: str | None,
+ refeed: bool,
+ store_doc: bool,
+) -> bool:
+ """Ingest one file or stdin into ``ka``. Returns False if skipped unchanged."""
+ require_supported_text_input(input)
+ text = read_input(input)
+ console.print(
+ f"[dim]Input {Path(input).name if input != '-' else 'stdin'}: "
+ f"{len(text)} characters"
+ f"{f' (source: {source})' if source else ''}[/dim]"
+ )
+
+ text_hash_to_record = None
+ if source:
+ text_hash_to_record = hashlib.sha256(text.encode("utf-8")).hexdigest()
+ if not refeed:
+ recorded = ka.source_content_hash(source)
+ if recorded == text_hash_to_record:
+ logger.info("stage=source_unchanged source=%s", source)
+ console.print(
+ f"[yellow]Source '{source}' is unchanged (content hash "
+ "matches) — nothing to do.[/yellow] "
+ "Use --refeed to re-ingest anyway."
+ )
+ return False
+
+ if store_doc and source:
+ from hyperextract.utils.document_store import SourceDocumentStore
+
+ original_name = "stdin.txt" if input == "-" else Path(input).name
+ store = SourceDocumentStore(output_path)
+ if input == "-":
+ stored_doc = store.store_text(source, text, original_name)
+ else:
+ stored_doc = store.store_file(source, input)
+ console.print(f"[dim]Document archived: {stored_doc}[/dim]")
+
+ logger.debug("stage=feed_text_invoked")
+ ka.feed_text(text, source_id=source, content_hash=text_hash_to_record)
+ logger.info("stage=knowledge_appended chars=%d source=%s", len(text), source)
+ return True
+
+
@app.command(name="feed")
def feed(
ka_path: str = typer.Argument(..., help="Knowledge Abstract directory"),
- input: str = typer.Argument(..., help="Input file path or '-' for stdin"),
+ input: str = typer.Argument(
+ ..., help="Input file path, directory, or '-' for stdin"
+ ),
template: str | None = typer.Option(None, "--template", "-t", help="Template"),
lang: str | None = typer.Option(None, "--lang", "-l", help="Language"),
source: str | None = typer.Option(
@@ -1151,50 +1201,49 @@ def feed(
ka.load(output_path)
- progress.update(task, description="Reading input...")
- require_supported_text_input(input)
- text = read_input(input)
- console.print(f"[dim]Input text: {len(text)} characters[/dim]")
-
- # Change detection: skip unchanged sources without any LLM calls.
- text_hash_to_record = None
- if source:
- text_hash_to_record = hashlib.sha256(text.encode("utf-8")).hexdigest()
- if not refeed:
- recorded = ka.source_content_hash(source)
- if recorded == text_hash_to_record:
- logger.info("stage=source_unchanged source=%s", source)
- console.print(
- f"[yellow]Source '{source}' is unchanged (content hash "
- "matches) — nothing to do.[/yellow] "
- "Use --refeed to re-ingest anyway."
- )
- raise typer.Exit(0)
-
- # Archive the source document (provenance evidence, kept across
- # rollbacks; purge with he remove --document ... --purge-documents).
- stored_doc = None
- if store_doc and source:
- from hyperextract.utils.document_store import SourceDocumentStore
-
- original_name = "stdin.txt" if input == "-" else Path(input).name
- store = SourceDocumentStore(output_path)
- if input == "-":
- stored_doc = store.store_text(source, text, original_name)
- else:
- stored_doc = store.store_file(source, input)
- console.print(f"[dim]Document archived: {stored_doc}[/dim]")
-
- progress.update(task, description="Appending knowledge...")
- logger.debug("stage=feed_text_invoked")
- ka.feed_text(text, source_id=source, content_hash=text_hash_to_record)
- logger.info("stage=knowledge_appended chars=%d", len(text))
+ input_path = Path(input) if input != "-" else None
+ ingested = 0
+ if input_path is not None and input_path.is_dir():
+ progress.update(task, description="Processing directory...")
+ text_files = collect_directory_text_inputs(input_path)
+ file_sources = [source or file_path.stem for file_path in text_files]
+ progress.update(task, description="Appending knowledge...")
+ for file_path, file_source in zip(text_files, file_sources):
+ if _feed_one_document(
+ ka,
+ output_path,
+ str(file_path),
+ file_source,
+ refeed,
+ store_doc,
+ ):
+ ingested += 1
+ logger.info(
+ "stage=directory_feed_complete files=%d ingested=%d",
+ len(text_files),
+ ingested,
+ )
+ else:
+ progress.update(task, description="Reading input...")
+ if not _feed_one_document(
+ ka, output_path, input, source, refeed, store_doc
+ ):
+ raise typer.Exit(0)
+ ingested = 1
- progress.update(task, description="Saving data...")
- ka.dump(output_path)
- logger.info("stage=data_saved")
+ if ingested:
+ progress.update(task, description="Saving data...")
+ ka.dump(output_path)
+ logger.info("stage=data_saved")
console.print()
+ if ingested == 0:
+ console.print(
+ "[yellow]No documents were ingested.[/yellow] "
+ "Supported files were unchanged or skipped."
+ )
+ return
+
console.print(
f"[bold green]Success![/bold green] Knowledge appended to {output_path}"
)
diff --git a/tests/cli/test_parse_input_types.py b/tests/cli/test_parse_input_types.py
index 9459a8a..31b56c8 100644
--- a/tests/cli/test_parse_input_types.py
+++ b/tests/cli/test_parse_input_types.py
@@ -221,6 +221,34 @@ class TestFeedInputTypes:
assert result.exit_code == 0, result.output
assert "Hello Hyper Extract from DOCX" in ka.feed_text.call_args[0][0]
+ def test_feed_directory_two_txt_sources(self, tmp_path):
+ ka_dir = _make_ka(tmp_path)
+ folder = tmp_path / "docs"
+ folder.mkdir()
+ (folder / "alpha.txt").write_text("first document", encoding="utf-8")
+ (folder / "beta.txt").write_text("second document", encoding="utf-8")
+ with _mock_feed() as ka:
+ result = runner.invoke(app, ["feed", str(ka_dir), str(folder)])
+
+ assert result.exit_code == 0, result.output
+ ids = [call.kwargs.get("source_id") for call in ka.feed_text.call_args_list]
+ texts = [call.args[0] for call in ka.feed_text.call_args_list]
+ assert ids == ["alpha", "beta"]
+ assert texts == ["first document", "second document"]
+ ka.dump.assert_called()
+
+ def test_feed_directory_unsupported_only_exits(self, tmp_path):
+ ka_dir = _make_ka(tmp_path)
+ folder = tmp_path / "docs"
+ folder.mkdir()
+ (folder / "data.bin").write_bytes(b"\x00\x01")
+ with _mock_feed() as ka:
+ result = runner.invoke(app, ["feed", str(ka_dir), str(folder)])
+
+ assert result.exit_code == 1
+ ka.feed_text.assert_not_called()
+ ka.dump.assert_not_called()
+
def test_feed_document_without_backend_rejected_with_hint(
self, tmp_path, monkeypatch
):
@@ -239,6 +267,37 @@ class TestFeedInputTypes:
ka.dump.assert_not_called()
+def test_feed_directory_writes_two_sources_to_ledger(tmp_path):
+ """Two txt files become two AutoDocument source-ledger entries after feed."""
+ from hyperextract.types import AutoDocument
+ from tests.mocks import MockChatModel, MockEmbeddings
+
+ ka_dir = tmp_path / "ka"
+ seed = AutoDocument(llm_client=MockChatModel(), embedder=MockEmbeddings())
+ seed.metadata["template"] = "general/graph"
+ seed.metadata["lang"] = "en"
+ seed.dump(ka_dir)
+
+ folder = tmp_path / "docs"
+ folder.mkdir()
+ (folder / "alpha.txt").write_text("first document", encoding="utf-8")
+ (folder / "beta.txt").write_text("second document", encoding="utf-8")
+
+ live = AutoDocument(llm_client=MockChatModel(), embedder=MockEmbeddings())
+ with (
+ patch("hyperextract.cli.cli.validate_config"),
+ patch("hyperextract.cli.cli.Template.create", return_value=live),
+ ):
+ result = runner.invoke(app, ["feed", str(ka_dir), str(folder)])
+
+ assert result.exit_code == 0, result.output
+ assert set(live.sources()) == {"alpha", "beta"}
+ data = json.loads((ka_dir / "data.json").read_text(encoding="utf-8"))
+ assert len(data.get("chunks", [])) == 2
+ ledger = json.loads((ka_dir / "sources_chunks.json").read_text(encoding="utf-8"))
+ assert {entry["source_id"] for entry in ledger} == {"alpha", "beta"}
+
+
def test_backend_probe_function_exists():
# Direct sanity check so the import surface stays stable.
assert callable(markitdown_available)