Skip to content

Hypergraph is more powerful. Transform unstructured text into structured knowledge with LLMs. Graphs, hypergraphs, and spatio-temporal extractions — with one command.

feat: add general/base_document Gallery preset

Frankie-Xub1a942ed6296
diff --git a/examples/README.md b/examples/README.md
index 2374ee0..072f7de 100644
--- a/examples/README.md
+++ b/examples/README.md
@@ -48,6 +48,7 @@ examples/
│ ├── tesla.md # Tesla biography data
│ ├── tesla_question.md # Query questions
│ ├── autotypes/ # AutoType demos
+│ │ ├── document_demo.py # Raw document chunks
│ │ ├── graph_demo.py # Knowledge graph
│ │ ├── list_demo.py # List extraction
│ │ ├── set_demo.py # Set extraction
@@ -79,6 +80,7 @@ examples/
├── sushi.md # Su Shi biography data
├── sushi_question.md # Query questions
├── autotypes/ # AutoType demos
+ │ ├── document_demo.py
│ ├── graph_demo.py
│ ├── list_demo.py
│ ├── set_demo.py
@@ -113,8 +115,9 @@ Each demo showcases a specific extraction type:
| Demo | Description |
|------|-------------|
+| `document_demo.py` | Store retrievable raw chunks (no LLM extraction) |
| `graph_demo.py` | Extract entities & relationships (knowledge graph) |
-| `list_demo.py` | Extract list of items |
+| `list_demo.py` | Extract list of items
| `set_demo.py` | Extract deduplicated set |
| `model_demo.py` | Extract structured summary |
| `temporal_graph_demo.py` | Extract temporal relationships |
diff --git a/examples/README_ZH.md b/examples/README_ZH.md
index 77abe9d..f2f2b78 100644
--- a/examples/README_ZH.md
+++ b/examples/README_ZH.md
@@ -49,6 +49,7 @@ examples/
│ ├── tesla.md # 特斯拉传记数据
│ ├── tesla_question.md # 查询问题
│ ├── autotypes/ # 自动类型演示
+│ │ ├── document_demo.py # 原文分块
│ │ ├── graph_demo.py # 知识图谱
│ │ ├── list_demo.py # 列表提取
│ │ ├── set_demo.py # 集合提取
@@ -80,6 +81,7 @@ examples/
├── sushi.md # 苏轼传记数据
├── sushi_question.md # 查询问题
├── autotypes/ # 自动类型演示
+ │ ├── document_demo.py
│ ├── graph_demo.py
│ ├── list_demo.py
│ ├── set_demo.py
@@ -114,6 +116,7 @@ examples/
| 演示 | 描述 |
|------|------|
+| `document_demo.py` | 存储可检索原文块(不跑 LLM 抽取) |
| `graph_demo.py` | 提取实体和关系(知识图谱) |
| `list_demo.py` | 提取项目列表 |
| `set_demo.py` | 提取去重集合 |
diff --git a/examples/en/autotypes/document_demo.py b/examples/en/autotypes/document_demo.py
new file mode 100644
index 0000000..0366a5e
--- /dev/null
+++ b/examples/en/autotypes/document_demo.py
@@ -0,0 +1,60 @@
+"""
+AutoDocument Demo - Tesla corpus chunks
+
+Load the Gallery `general/base_document` preset and store raw text chunks.
+AutoDocument does not run LLM extraction; the LLM client is only used by
+optional chat after chunks are indexed.
+
+Usage:
+ python examples/en/autotypes/document_demo.py
+"""
+
+from pathlib import Path
+
+from dotenv import load_dotenv
+from langchain_openai import ChatOpenAI, OpenAIEmbeddings
+
+from hyperextract.utils.template_engine import Template
+
+project_root = Path(__file__).resolve().parent.parent.parent.parent
+
+load_dotenv()
+
+INPUT_FILE = project_root / "examples" / "en" / "tesla.md"
+QUESTION_FILE = project_root / "examples" / "en" / "tesla_question.md"
+
+
+if __name__ == "__main__":
+ with open(INPUT_FILE, encoding="utf-8") as f:
+ text = f.read()
+ with open(QUESTION_FILE, encoding="utf-8") as f:
+ questions = [line.strip() for line in f if line.strip()]
+
+ llm = ChatOpenAI(model="gpt-4o-mini", temperature=0)
+ embedder = OpenAIEmbeddings(model="text-embedding-3-small")
+
+ print("\n" + "=" * 60)
+ print(" AutoDocument Demo - Tesla corpus")
+ print("=" * 60)
+ print("Chunking text via general/base_document (no LLM extraction)...")
+
+ corpus = Template.create("general/base_document", "en", llm, embedder)
+ corpus.feed_text(text, source_id="tesla")
+
+ print(f"\nStored {len(corpus.data.chunks)} chunks")
+ for chunk in corpus.data.chunks[:3]:
+ preview = chunk.content[:80].replace("\n", " ")
+ print(f" - {preview}...")
+
+ corpus.build_index()
+
+ print("-" * 60)
+ print("Q&A")
+ print("-" * 60)
+ for q in questions[:2]:
+ print(f"\nQ: {q}")
+ try:
+ result = corpus.chat(q)
+ print(f"A: {result.content}")
+ except Exception as e:
+ print(f"Error: {e}")
diff --git a/examples/zh/autotypes/document_demo.py b/examples/zh/autotypes/document_demo.py
new file mode 100644
index 0000000..77c89f7
--- /dev/null
+++ b/examples/zh/autotypes/document_demo.py
@@ -0,0 +1,59 @@
+"""
+AutoDocument 示例 - 苏轼语料分块
+
+加载 Gallery 预设 `general/base_document`,将原文切成可检索块。
+AutoDocument 不跑 LLM 抽取;LLM 客户端仅用于可选的分块问答。
+
+使用方法:
+ python examples/zh/autotypes/document_demo.py
+"""
+
+from pathlib import Path
+
+from dotenv import load_dotenv
+from langchain_openai import ChatOpenAI, OpenAIEmbeddings
+
+from hyperextract.utils.template_engine import Template
+
+project_root = Path(__file__).resolve().parent.parent.parent.parent
+
+load_dotenv()
+
+INPUT_FILE = project_root / "examples" / "zh" / "sushi.md"
+QUESTION_FILE = project_root / "examples" / "zh" / "sushi_question.md"
+
+
+if __name__ == "__main__":
+ with open(INPUT_FILE, encoding="utf-8") as f:
+ text = f.read()
+ with open(QUESTION_FILE, encoding="utf-8") as f:
+ questions = [line.strip() for line in f if line.strip()]
+
+ llm = ChatOpenAI(model="gpt-4o-mini", temperature=0)
+ embedder = OpenAIEmbeddings(model="text-embedding-3-small")
+
+ print("\n" + "=" * 60)
+ print(" AutoDocument 示例 - 苏轼语料")
+ print("=" * 60)
+ print("通过 general/base_document 分块(不跑 LLM 抽取)...")
+
+ corpus = Template.create("general/base_document", "zh", llm, embedder)
+ corpus.feed_text(text, source_id="sushi")
+
+ print(f"\n已存储 {len(corpus.data.chunks)} 个块")
+ for chunk in corpus.data.chunks[:3]:
+ preview = chunk.content[:80].replace("\n", " ")
+ print(f" - {preview}...")
+
+ corpus.build_index()
+
+ print("-" * 60)
+ print("问答")
+ print("-" * 60)
+ for q in questions[:2]:
+ print(f"\nQ: {q}")
+ try:
+ result = corpus.chat(q)
+ print(f"A: {result.content}")
+ except Exception as e:
+ print(f"Error: {e}")
diff --git a/hyperextract/templates/presets/general/base_document.yaml b/hyperextract/templates/presets/general/base_document.yaml
new file mode 100644
index 0000000..99dd97e
--- /dev/null
+++ b/hyperextract/templates/presets/general/base_document.yaml
@@ -0,0 +1,39 @@
+language: [zh, en]
+
+name: base_document
+type: document
+tags: [general, document]
+
+description:
+ zh: '通用文档语料模板 - 将文本切成可检索的原文块。AutoDocument 不跑 LLM 抽取,output.fields 仅为满足 TemplateCfg,不参与抽取。'
+ en: 'General-purpose document corpus template - splits text into retrievable raw chunks. AutoDocument does not run LLM extraction; output.fields exist only to satisfy TemplateCfg and are not extracted.'
+
+output:
+ description:
+ zh: '占位输出。AutoDocument 存储原文块,不按这些字段抽取。'
+ en: 'Placeholder output. AutoDocument stores raw chunks and does not extract these fields.'
+ fields:
+ - name: content
+ type: str
+ description:
+ zh: '占位字段,满足 TemplateCfg;不参与抽取。'
+ en: 'Placeholder field required by TemplateCfg; not used for extraction.'
+
+guideline:
+ target:
+ zh: 'AutoDocument 不跑 LLM 抽取。将文本按块存储以便检索。'
+ en: 'AutoDocument does not run LLM extraction. Store text as chunks for retrieval.'
+ rules:
+ zh:
+ - '只切分并存储原文,不要发明图谱 schema。'
+ - 'output.fields 是占位,不参与抽取。'
+ en:
+ - 'Chunk and store the source text only; do not invent a graph schema.'
+ - 'output.fields are placeholders and are not extracted.'
+
+display:
+ label: '{content}'
+
+options:
+ chunk_size: 2048
+ chunk_overlap: 256
diff --git a/tests/template_engine/test_factory.py b/tests/template_engine/test_factory.py
index f8f7f87..2d20cc7 100644
--- a/tests/template_engine/test_factory.py
+++ b/tests/template_engine/test_factory.py
@@ -207,3 +207,17 @@ class TestTemplateFactoryDocumentType:
from hyperextract.types import AutoDocument as Canonical
assert Exported is Canonical
+
+ def test_create_gallery_base_document_preset(self, llm_client, embedder):
+ from hyperextract import AutoDocument
+ from hyperextract.utils.template_engine import Gallery, Template
+
+ assert Gallery.get("general/base_document") is not None
+ result = Template.create(
+ "general/base_document",
+ "en",
+ llm_client,
+ embedder,
+ )
+ assert isinstance(result, AutoDocument)
+ assert result.metadata.get("type") == "document"