Skip to content

Hypergraph is more powerful. Transform unstructured text into structured knowledge with LLMs. Graphs, hypergraphs, and spatio-temporal extractions — with one command.

Refactor code for improved readability and consistency

yifanfeng970428ff1b9649
diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml
index 34ec41c..149832e 100644
--- a/.github/workflows/lint.yml
+++ b/.github/workflows/lint.yml
@@ -39,30 +39,3 @@ jobs:
- name: Run ruff formatter check
run: ruff format --check hyperextract
-
- mypy:
- name: Type Check (mypy)
- runs-on: ubuntu-latest
-
- steps:
- - name: Checkout repository
- uses: actions/checkout@v4
-
- - name: Set up Python
- uses: actions/setup-python@v5
- with:
- python-version: "3.12"
-
- - name: Install uv
- uses: astral-sh/setup-uv@v5
- with:
- enable-cache: true
-
- - name: Install dependencies
- run: |
- uv venv
- uv pip install -e ".[all]"
- uv pip install mypy
-
- - name: Run mypy
- run: uv run mypy hyperextract --ignore-missing-imports
diff --git a/README.md b/README.md
index 79eca7a..f838b2f 100644
--- a/README.md
+++ b/README.md
@@ -170,13 +170,13 @@ ka.show(result)
Our framework embraces complexity without making you write boilerplate code.
-<img src="docs/assets/autotypes.jpg" alt="Knowledge Structures Matrix" width="700" style="max-width: 100%;">
+<img src="docs/assets/autotypes.jpg" alt="Knowledge Structures Matrix" width="750" style="max-width: 100%;">
### Example: AutoGraph Visualization
Here is the knowledge graph visualization after `AutoGraph` extraction:
-<img src="docs/assets/en_show.jpg" alt="AutoGraph Visualization" width="600" style="max-width: 100%; border-radius: 8px; box-shadow: 0 4px 12px rgba(0,0,0,0.1);">
+<img src="docs/assets/en_show.jpg" alt="AutoGraph Visualization" width="750" style="max-width: 100%; border-radius: 8px; box-shadow: 0 4px 12px rgba(0,0,0,0.1);">
## 🛠️ Architecture Overview
diff --git a/README_ZH.md b/README_ZH.md
index 631f357..007d3d6 100644
--- a/README_ZH.md
+++ b/README_ZH.md
@@ -170,13 +170,13 @@ ka.show(result)
拒绝样板代码,纯干货聚焦数据本身。
-<img src="docs/assets/autotypes.jpg" alt="知识结构矩阵" width="700" style="max-width: 100%;">
+<img src="docs/assets/autotypes.jpg" alt="知识结构矩阵" width="750" style="max-width: 100%;">
### 示例:AutoGraph 知识图谱可视化
以下是 `AutoGraph` 类型提取后的知识图谱可视化效果:
-<img src="docs/assets/zh_show.jpg" alt="AutoGraph 可视化" width="600" style="max-width: 100%; border-radius: 8px; box-shadow: 0 4px 12px rgba(0,0,0,0.1);">
+<img src="docs/assets/zh_show.jpg" alt="AutoGraph 可视化" width="750" style="max-width: 100%; border-radius: 8px; box-shadow: 0 4px 12px rgba(0,0,0,0.1);">
## 🛠️ 系统架构
diff --git a/hyperextract/cli/cli.py b/hyperextract/cli/cli.py
index d45f638..401f267 100644
--- a/hyperextract/cli/cli.py
+++ b/hyperextract/cli/cli.py
@@ -54,35 +54,38 @@ def main(
"""Hyper-Extract CLI - Transform document into knowledge-abstract."""
if version:
from . import __version__
+
console.print(f"[bold]Hyper-Extract CLI[/bold] version {__version__}")
raise typer.Exit()
if ctx.invoked_subcommand is None:
from . import __version__
-
+
console.print()
console.print(Text(LOGO, style="bold cyan"))
-
+
title_text = Text("HYPER-EXTRACT", style="bold cyan")
version_text = Text(f"v{__version__}", style="dim white")
- desc_text = Text("Transform document into knowledge-abstract", style="dim", no_wrap=True)
-
+ desc_text = Text(
+ "Transform document into knowledge-abstract", style="dim", no_wrap=True
+ )
+
header = Table(box=None, show_header=False, pad_edge=False)
header.add_column(no_wrap=True)
header.add_column(style="dim white", no_wrap=True)
header.add_row(title_text, version_text)
-
+
console.print(header)
console.print(desc_text)
console.print()
-
+
from rich.rule import Rule
-
+
console.print(Rule(style="cyan dim"))
console.print()
-
+
from rich.panel import Panel
-
+
def make_section(title: str, commands: list[tuple[str, str]]) -> Panel:
table = Table(box=None, show_header=False, pad_edge=False)
table.add_column(style="green bold", no_wrap=True)
@@ -97,32 +100,47 @@ def main(
title_align="center",
width=80,
)
-
+
sections = [
- make_section("🚀 Getting Started", [
- ("he list template", "List available templates"),
- ("he list method", "List extraction methods"),
- ("he config --help", "Manage LLM/Embedder config"),
- ]),
- make_section("✨ Create Knowledge Abstract (KA)", [
- ("he parse <input_document> -o <ka_path>", "Extract KA from document"),
- ("he feed <ka_path> <input_document>", "Add document to existing KA"),
- ("he build-index <ka_path>", "Build semantic search index"),
- ]),
- make_section("🔍 Explore Knowledge Abstract (KA)", [
- ("he info <ka_path>", "View KA info & stats"),
- ("he talk <ka_path> [-i]", "Chat with KA"),
- ("he search <ka_path> <query>", "Semantic search"),
- ("he show <ka_path>", "Visualize KA"),
- ]),
+ make_section(
+ "🚀 Getting Started",
+ [
+ ("he list template", "List available templates"),
+ ("he list method", "List extraction methods"),
+ ("he config --help", "Manage LLM/Embedder config"),
+ ],
+ ),
+ make_section(
+ "✨ Create Knowledge Abstract (KA)",
+ [
+ (
+ "he parse <input_document> -o <ka_path>",
+ "Extract KA from document",
+ ),
+ (
+ "he feed <ka_path> <input_document>",
+ "Add document to existing KA",
+ ),
+ ("he build-index <ka_path>", "Build semantic search index"),
+ ],
+ ),
+ make_section(
+ "🔍 Explore Knowledge Abstract (KA)",
+ [
+ ("he info <ka_path>", "View KA info & stats"),
+ ("he talk <ka_path> [-i]", "Chat with KA"),
+ ("he search <ka_path> <query>", "Semantic search"),
+ ("he show <ka_path>", "Visualize KA"),
+ ],
+ ),
]
-
+
for section in sections:
console.print(section)
console.print()
console.print(Rule(style="cyan dim"))
console.print()
-
+
hint_text = Text("💡 Tip: Run ", style="dim")
hint_text.append("he --help", style="bold cyan")
hint_text.append(" for detailed documentation", style="dim")
@@ -167,19 +185,24 @@ def select_template_interactive() -> Optional[str]:
if 0 <= idx < len(template_list):
return template_list[idx][0]
else:
- console.print(f"[red]Invalid number. Please choose 1-{len(template_list)}[/red]")
+ console.print(
+ f"[red]Invalid number. Please choose 1-{len(template_list)}[/red]"
+ )
else:
query_lower = choice.lower()
- matches = [(i, p, c) for i, (p, c) in enumerate(template_list)
- if query_lower in p.lower() or
- (c.description and query_lower in str(c.description).lower())]
+ matches = [
+ (i, p, c)
+ for i, (p, c) in enumerate(template_list)
+ if query_lower in p.lower()
+ or (c.description and query_lower in str(c.description).lower())
+ ]
if len(matches) == 1:
return matches[0][1]
elif len(matches) > 1:
console.print(f"[yellow]Found {len(matches)} matches:[/yellow]")
for i, path, cfg in matches:
- console.print(f" [{i+1}] {path}")
+ console.print(f" [{i + 1}] {path}")
continue
else:
console.print("[yellow]No matches found. Try another keyword.[/yellow]")
@@ -187,13 +210,26 @@ def select_template_interactive() -> Optional[str]:
@app.command(name="parse")
def parse(
- input: str = typer.Argument(..., help="Input file path, directory, or '-' for stdin"),
+ input: str = typer.Argument(
+ ..., help="Input file path, directory, or '-' for stdin"
+ ),
output: str = typer.Option(..., "--output", "-o", help="Output directory"),
- template: Optional[str] = typer.Option(None, "--template", "-t", help="Template (omit for interactive selection)"),
- method: Optional[str] = typer.Option(None, "--method", "-m", help="Method template (e.g., light_rag, hyper_rag)"),
- lang: Optional[str] = typer.Option(None, "--lang", "-l", help="Language (zh/en). Required for knowledge templates, optional for methods (default: en)"),
+ template: Optional[str] = typer.Option(
+ None, "--template", "-t", help="Template (omit for interactive selection)"
+ ),
+ method: Optional[str] = typer.Option(
+ None, "--method", "-m", help="Method template (e.g., light_rag, hyper_rag)"
+ ),
+ lang: Optional[str] = typer.Option(
+ None,
+ "--lang",
+ "-l",
+ help="Language (zh/en). Required for knowledge templates, optional for methods (default: en)",
+ ),
force: bool = typer.Option(False, "--force", "-f", help="Force overwrite"),
- no_index: bool = typer.Option(False, "--no-index", help="Skip building search index"),
+ no_index: bool = typer.Option(
+ False, "--no-index", help="Skip building search index"
+ ),
):
"""Extract knowledge from text to a new directory."""
validate_config()
@@ -210,17 +246,23 @@ def parse(
if is_method_template:
if lang is not None:
- console.print("[dim]Note: Method templates use English prompts. --lang parameter is ignored.[/dim]")
+ console.print(
+ "[dim]Note: Method templates use English prompts. --lang parameter is ignored.[/dim]"
+ )
lang = "en"
elif lang is None:
- console.print("[red]Error:[/red] --lang is required for knowledge templates. Use --lang en or --lang zh.")
+ console.print(
+ "[red]Error:[/red] --lang is required for knowledge templates. Use --lang en or --lang zh."
+ )
raise typer.Exit(1)
output_path = Path(output)
if output_path.exists() and not force:
if any(output_path.iterdir()):
- console.print("[red]Error:[/red] Output directory already exists and is not empty. Use --force to overwrite.")
+ console.print(
+ "[red]Error:[/red] Output directory already exists and is not empty. Use --force to overwrite."
+ )
raise typer.Exit(1)
output_path.mkdir(parents=True, exist_ok=True)
@@ -243,7 +285,11 @@ def parse(
input_path = Path(input)
- with Progress(SpinnerColumn(), TextColumn("[progress.description]{task.description}"), console=console) as progress:
+ with Progress(
+ SpinnerColumn(),
+ TextColumn("[progress.description]{task.description}"),
+ console=console,
+ ) as progress:
task = progress.add_task("Creating template instance...", total=None)
ka = Template.create(template, lang)
@@ -252,7 +298,9 @@ def parse(
progress.update(task, description="Processing directory...")
text_files = list(input_path.glob("*.txt")) + list(input_path.glob("*.md"))
if not text_files:
- console.print(f"[red]Error:[/red] No .txt or .md files found in {input}")
+ console.print(
+ f"[red]Error:[/red] No .txt or .md files found in {input}"
+ )
raise typer.Exit(1)
all_text = []
@@ -280,9 +328,12 @@ def parse(
if template_config is None:
if template.endswith(".yaml"):
import shutil
+
filename = Path(template).name
shutil.copy(template, output_path / filename)
- console.print(f"[dim]Custom template '{filename}' saved to KA directory[/dim]")
+ console.print(
+ f"[dim]Custom template '{filename}' saved to KA directory[/dim]"
+ )
ka.dump(output_path)
@@ -294,19 +345,35 @@ def parse(
ka.dump(output_path)
console.print()
- console.print(f"[bold green]Success![/bold green] Knowledge extracted to {output_path}")
+ console.print(
+ f"[bold green]Success![/bold green] Knowledge extracted to {output_path}"
+ )
console.print()
if no_index:
console.print("[dim]Note: Index was not built.[/dim]")
- console.print(f"[dim] he build-index {output} # Build index to enable search/talk[/dim]")
- console.print(f"[dim] he feed {output} <new_document> # Append more documents[/dim]")
+ console.print(
+ f"[dim] he build-index {output} # Build index to enable search/talk[/dim]"
+ )
+ console.print(
+ f"[dim] he feed {output} <new_document> # Append more documents[/dim]"
+ )
else:
console.print("[dim]What's next?[/dim]")
- console.print(f"[dim] he show {output} # Visualize knowledge graph[/dim]")
- console.print(f"[dim] he feed {output} <new_document> # Append more documents[/dim]")
- console.print(f"[dim] he search {output} \"keyword\" # Semantic search[/dim]")
- console.print(f"[dim] he talk {output} -i # Interactive chat[/dim]")
- console.print(f"[dim] he talk {output} -q \"your question\" # Single query[/dim]")
+ console.print(
+ f"[dim] he show {output} # Visualize knowledge graph[/dim]"
+ )
+ console.print(
+ f"[dim] he feed {output} <new_document> # Append more documents[/dim]"
+ )
+ console.print(
+ f'[dim] he search {output} "keyword" # Semantic search[/dim]'
+ )
+ console.print(
+ f"[dim] he talk {output} -i # Interactive chat[/dim]"
+ )
+ console.print(
+ f'[dim] he talk {output} -q "your question" # Single query[/dim]'
+ )
@app.command(name="show")
@@ -341,7 +408,9 @@ def show(ka_path: str = typer.Argument(..., help="Knowledge Abstract directory")
console.print()
console.print("[dim]Continue exploring:[/dim]")
- console.print(f"[dim] he search {ka_path} \"keyword\" # Search specific content[/dim]")
+ console.print(
+ f'[dim] he search {ka_path} "keyword" # Search specific content[/dim]'
+ )
console.print(f"[dim] he talk {ka_path} -i # Interactive chat[/dim]")
@@ -387,7 +456,9 @@ def info(ka_path: str = typer.Argument(..., help="Knowledge Abstract directory")
table.add_row("Nodes", str(node_count))
table.add_row("Edges", str(edge_count))
- table.add_row("Index", "[green]Built[/green]" if index_exists else "[red]Not Built[/red]")
+ table.add_row(
+ "Index", "[green]Built[/green]" if index_exists else "[red]Not Built[/red]"
+ )
console.print(table)
@@ -411,7 +482,11 @@ def search(
console.print(f"[blue]Top K:[/blue] {top_k}")
console.print()
- with Progress(SpinnerColumn(), TextColumn("[progress.description]{task.description}"), console=console) as progress:
+ with Progress(
+ SpinnerColumn(),
+ TextColumn("[progress.description]{task.description}"),
+ console=console,
+ ) as progress:
task = progress.add_task("Searching...", total=None)
try:
@@ -437,22 +512,34 @@ def search(
for i, result in enumerate(results, 1):
console.print(f"[bold cyan]Result {i}:[/bold cyan]")
if hasattr(result, "model_dump"):
- console.print_json(json.dumps(result.model_dump(), indent=2, ensure_ascii=False))
+ console.print_json(
+ json.dumps(result.model_dump(), indent=2, ensure_ascii=False)
+ )
elif hasattr(result, "dict"):
- console.print_json(json.dumps(result.dict(), indent=2, ensure_ascii=False))
+ console.print_json(
+ json.dumps(result.dict(), indent=2, ensure_ascii=False)
+ )
else:
console.print(str(result))
console.print()
console.print("[dim]Continue:[/dim]")
- console.print(f"[dim] he talk {ka_path} -q \"question about results\" # Deep dive[/dim]")
- console.print(f"[dim] he talk {ka_path} -i # Interactive mode[/dim]")
- console.print(f"[dim] he show {ka_path} # Visualize[/dim]")
+ console.print(
+ f'[dim] he talk {ka_path} -q "question about results" # Deep dive[/dim]'
+ )
+ console.print(
+ f"[dim] he talk {ka_path} -i # Interactive mode[/dim]"
+ )
+ console.print(
+ f"[dim] he show {ka_path} # Visualize[/dim]"
+ )
def chat_loop(ka, ka_path: str):
"""Interactive chat loop."""
- console.print("\n[bold green]Entering interactive mode. Type 'exit' or 'quit' to stop.[/bold green]\n")
+ console.print(
+ "\n[bold green]Entering interactive mode. Type 'exit' or 'quit' to stop.[/bold green]\n"
+ )
while True:
try:
query = console.input("[bold cyan]>[/bold cyan] ")
@@ -460,9 +547,13 @@ def chat_loop(ka, ka_path: str):
console.print("\n[dim]Goodbye![/dim]")
console.print()
console.print("[dim]Other useful commands:[/dim]")
- console.print(f"[dim] he show {ka_path} # Visualize[/dim]")
- console.print(f"[dim] he search {ka_path} \"keyword\" # Search[/dim]")
- console.print(f"[dim] he info {ka_path} # View info[/dim]")
+ console.print(
+ f"[dim] he show {ka_path} # Visualize[/dim]"
+ )
+ console.print(f'[dim] he search {ka_path} "keyword" # Search[/dim]')
+ console.print(
+ f"[dim] he info {ka_path} # View info[/dim]"
+ )
break
if not query.strip():
continue
@@ -475,7 +566,7 @@ def chat_loop(ka, ka_path: str):
console.print()
console.print("[dim]Other useful commands:[/dim]")
console.print(f"[dim] he show {ka_path} # Visualize[/dim]")
- console.print(f"[dim] he search {ka_path} \"keyword\" # Search[/dim]")
+ console.print(f'[dim] he search {ka_path} "keyword" # Search[/dim]')
console.print(f"[dim] he info {ka_path} # View info[/dim]")
break
except Exception as e:
@@ -487,7 +578,9 @@ def talk(
ka_path: str = typer.Argument(..., help="Knowledge Abstract directory"),
query: Optional[str] = typer.Option(None, "--query", "-q", help="Question to ask"),
top_k: int = typer.Option(3, "--top-k", "-n", help="Number of context items"),
- interactive: bool = typer.Option(False, "--interactive", "-i", help="Interactive mode"),
+ interactive: bool = typer.Option(
+ False, "--interactive", "-i", help="Interactive mode"
+ ),
):
"""Chat with Knowledge Abstract."""
validate_config()
@@ -501,7 +594,9 @@ def talk(
console.print(f"[blue]Top K:[/blue] {top_k}")
console.print()
elif query is None:
- console.print("[red]Error:[/red] Please provide a query or use --interactive mode")
+ console.print(
+ "[red]Error:[/red] Please provide a query or use --interactive mode"
+ )
raise typer.Exit(1)
else:
console.print(f"[blue]Query:[/blue] {query}")
@@ -509,7 +604,11 @@ def talk(
console.print(f"[blue]Top K:[/blue] {top_k}")
console.print()
- with Progress(SpinnerColumn(), TextColumn("[progress.description]{task.description}"), console=console) as progress:
+ with Progress(
+ SpinnerColumn(),
+ TextColumn("[progress.description]{task.description}"),
+ console=console,
+ ) as progress:
task = progress.add_task("Loading...", total=None)
try:
@@ -542,8 +641,10 @@ def talk(
console.print()
console.print("[dim]Continue:[/dim]")
- console.print(f"[dim] he talk {ka_path} -i # Enter interactive mode[/dim]")
- console.print(f"[dim] he search {ka_path} \"keyword\" # Search more[/dim]")
+ console.print(
+ f"[dim] he talk {ka_path} -i # Enter interactive mode[/dim]"
+ )
+ console.print(f'[dim] he search {ka_path} "keyword" # Search more[/dim]')
console.print(f"[dim] he show {ka_path} # Visualize[/dim]")
@@ -561,7 +662,9 @@ def feed(
metadata = load_ka_metadata(output_path)
if not metadata:
- console.print(f"[red]Error:[/red] Not a valid Knowledge Abstract directory: {ka_path}")
+ console.print(
+ f"[red]Error:[/red] Not a valid Knowledge Abstract directory: {ka_path}"
+ )
raise typer.Exit(1)
if template is None:
@@ -582,7 +685,11 @@ def feed(
console.print(f"[red]Error:[/red] {e}")
raise typer.Exit(1)
- with Progress(SpinnerColumn(), TextColumn("[progress.description]{task.description}"), console=console) as progress:
+ with Progress(
+ SpinnerColumn(),
+ TextColumn("[progress.description]{task.description}"),
+ console=console,
+ ) as progress:
task = progress.add_task("Loading existing knowledge...", total=None)
ka.load(output_path)
@@ -598,11 +705,15 @@ def feed(
ka.dump(output_path)
console.print()
- console.print(f"[bold green]Success![/bold green] Knowledge appended to {output_path}")
+ console.print(
+ f"[bold green]Success![/bold green] Knowledge appended to {output_path}"
+ )
console.print()
console.print("[dim]Next steps:[/dim]")
console.print(f"[dim] he show {ka_path} # Visualize[/dim]")
- console.print(f"[dim] he build-index {ka_path} # Rebuild index (if needed)[/dim]")
+ console.print(
+ f"[dim] he build-index {ka_path} # Rebuild index (if needed)[/dim]"
+ )
@app.command(name="build-index")
@@ -617,7 +728,9 @@ def build_index(
index_dir = path / "index"
if index_dir.exists() and any(index_dir.iterdir()) and not force:
- console.print("[yellow]Warning:[/yellow] Index already exists. Use --force to rebuild.")
+ console.print(
+ "[yellow]Warning:[/yellow] Index already exists. Use --force to rebuild."
+ )
console.print(f"[dim]Index location: {index_dir}[/dim]")
raise typer.Exit(0)
@@ -627,7 +740,11 @@ def build_index(
console.print(f"[blue]Language:[/blue] {lang}")
console.print()
- with Progress(SpinnerColumn(), TextColumn("[progress.description]{task.description}"), console=console) as progress:
+ with Progress(
+ SpinnerColumn(),
+ TextColumn("[progress.description]{task.description}"),
+ console=console,
+ ) as progress:
task = progress.add_task("Initializing...", total=None)
try:
@@ -654,5 +771,5 @@ def build_index(
console.print(f"[bold green]Success![/bold green] Index built for {ka_path}")
console.print()
console.print("[dim]Now you can:[/dim]")
- console.print(f"[dim] he search {ka_path} \"keyword\" # Semantic search[/dim]")
+ console.print(f'[dim] he search {ka_path} "keyword" # Semantic search[/dim]')
console.print(f"[dim] he talk {ka_path} -i # Interactive chat[/dim]")
diff --git a/hyperextract/cli/commands/config.py b/hyperextract/cli/commands/config.py
index c4efc6e..5b65f9c 100644
--- a/hyperextract/cli/commands/config.py
+++ b/hyperextract/cli/commands/config.py
@@ -24,70 +24,77 @@ def config_callback(
"""Show configuration help when running 'he config' without subcommand."""
if ctx.invoked_subcommand is not None:
return
-
+
from ..utils import LOGO
-
+
console.print()
console.print(Text(LOGO, style="bold cyan"))
-
+
from rich.rule import Rule
-
+
console.print(Rule(style="cyan dim"))
console.print()
-
+
title_text = Text("CONFIGURATION", style="bold cyan")
desc_text = Text("Manage LLM and Embedder settings", style="dim")
-
+
header = Table(box=None, show_header=False, pad_edge=False)
header.add_column(no_wrap=True)
header.add_column(style="dim white", no_wrap=True)
header.add_row(title_text, desc_text)
-
+
console.print(header)
console.print()
console.print(Rule(style="cyan dim"))
console.print()
-
+
console.print("[bold cyan]Available Commands:[/bold cyan]")
console.print()
-
+
commands_info = [
- ("he config init", "Interactive configuration setup (recommended for first-time users)"),
+ (
+ "he config init",
+ "Interactive configuration setup (recommended for first-time users)",
+ ),
("he config show", "Display current configuration"),
("he config llm", "Configure LLM settings"),
("he config embedder", "Configure Embedder settings"),
]
-
+
for cmd, desc in commands_info:
console.print(f" [green]{cmd:<30}[/green] {desc}")
-
+
console.print()
console.print(Rule(style="cyan dim"))
console.print()
-
+
console.print("[bold cyan]Quick Start:[/bold cyan]")
console.print()
- console.print(" [yellow]1.[/yellow] Run [green]he config init[/green] for interactive setup")
+ console.print(
+ " [yellow]1.[/yellow] Run [green]he config init[/green] for interactive setup"
+ )
console.print(" [yellow]2.[/yellow] Or configure individually:")
console.print(" [green]he config llm --api-key YOUR_KEY[/green]")
console.print(" [green]he config embedder --api-key YOUR_KEY[/green]")
console.print()
-
+
console.print("[bold cyan]Environment Variables (alternative):[/bold cyan]")
console.print()
- console.print(" [green]OPENAI_API_KEY[/green] - OpenAI API key (used if not set in config)")
+ console.print(
+ " [green]OPENAI_API_KEY[/green] - OpenAI API key (used if not set in config)"
+ )
console.print(" [green]OPENAI_BASE_URL[/green] - Custom API base URL (optional)")
console.print()
-
+
console.print(Rule(style="cyan dim"))
console.print()
-
+
hint_text = Text("💡 Tip: Run ", style="dim")
hint_text.append("he config <command> --help", style="bold cyan")
hint_text.append(" for detailed command usage", style="dim")
console.print(hint_text)
console.print()
-
+
raise typer.Exit()
@@ -95,7 +102,7 @@ def _show_config():
"""Show current configuration."""
config = ConfigManager()
cfg = config.show()
-
+
table = Table(title="Hyper-Extract Configuration")
table.add_column("Service", style="cyan", width=15)
table.add_column("Model", style="yellow", width=30)
@@ -161,7 +168,9 @@ def llm(
table.add_column("Key", style="cyan")
table.add_column("Value", style="green")
table.add_row("Model", cfg.model)
- table.add_row("API Key", cfg.api_key[:10] + "..." if cfg.api_key else "(not set)")
+ table.add_row(
+ "API Key", cfg.api_key[:10] + "..." if cfg.api_key else "(not set)"
+ )
table.add_row("Base URL", cfg.base_url or "(default)")
console.print(table)
return
@@ -199,7 +208,9 @@ def embedder(
"-u",
help="Custom API base URL",
),
- show: bool = typer.Option(False, "--show", help="Show current Embedder configuration"),
+ show: bool = typer.Option(
+ False, "--show", help="Show current Embedder configuration"
+ ),
unset: bool = typer.Option(False, "--unset", help="Unset Embedder configuration"),
):
"""Configure Embedder settings."""
@@ -211,7 +222,9 @@ def embedder(
table.add_column("Key", style="cyan")
table.add_column("Value", style="green")
table.add_row("Model", cfg.model)
- table.add_row("API Key", cfg.api_key[:10] + "..." if cfg.api_key else "(not set)")
+ table.add_row(
+ "API Key", cfg.api_key[:10] + "..." if cfg.api_key else "(not set)"
+ )
table.add_row("Base URL", cfg.base_url or "(default)")
console.print(table)
return
@@ -273,13 +286,15 @@ def init(
console.print("[bold]Step 1: LLM Configuration[/bold]")
model = console.input(" Model (default: gpt-4o-mini): ") or "gpt-4o-mini"
-
+
llm_api_key = None
while not llm_api_key:
llm_api_key = console.input(" API Key: ")
if not llm_api_key:
- console.print(" [red]API Key is required. Please enter your API key.[/red]")
-
+ console.print(
+ " [red]API Key is required. Please enter your API key.[/red]"
+ )
+
llm_base_url = console.input(" Base URL (optional, press Enter to skip): ") or None
config.set_llm(
@@ -291,14 +306,19 @@ def init(
console.print()
console.print("[bold]Step 2: Embedder Configuration[/bold]")
- emb_model = console.input(" Model (default: text-embedding-3-small): ") or "text-embedding-3-small"
-
+ emb_model = (
+ console.input(" Model (default: text-embedding-3-small): ")
+ or "text-embedding-3-small"
+ )
+
emb_api_key = None
while not emb_api_key:
emb_api_key = console.input(" API Key: ")
if not emb_api_key:
- console.print(" [red]API Key is required. Please enter your API key.[/red]")
-
+ console.print(
+ " [red]API Key is required. Please enter your API key.[/red]"
+ )
+
emb_base_url = console.input(" Base URL (optional, press Enter to skip): ") or None
config.set_embedder(
diff --git a/hyperextract/cli/commands/list.py b/hyperextract/cli/commands/list.py
index ea517dd..823f070 100644
--- a/hyperextract/cli/commands/list.py
+++ b/hyperextract/cli/commands/list.py
@@ -26,10 +26,18 @@ def _get_description(cfg, lang: str = "en") -> str:
@app.command(name="template")
def template(
- query: Optional[str] = typer.Option(None, "--query", "-q", help="Query to search templates"),
- autotype: Optional[str] = typer.Option(None, "--autotype", "-a", help="Filter by autotype"),
- language: Optional[str] = typer.Option(None, "--lang", "-l", help="Language filter (en/zh/all, default: en)"),
- include_methods: bool = typer.Option(True, "--include-methods/--no-methods", help="Include method templates"),
+ query: Optional[str] = typer.Option(
+ None, "--query", "-q", help="Query to search templates"
+ ),
+ autotype: Optional[str] = typer.Option(
+ None, "--autotype", "-a", help="Filter by autotype"
+ ),
+ language: Optional[str] = typer.Option(
+ None, "--lang", "-l", help="Language filter (en/zh/all, default: en)"
+ ),
+ include_methods: bool = typer.Option(
+ True, "--include-methods/--no-methods", help="Include method templates"
+ ),
):
"""List all available templates with detailed information."""
target_lang = language if language else "en"
@@ -50,11 +58,16 @@ def template(
for lang in languages:
templates.append((path, cfg.type, _get_description(cfg, lang)))
else:
- lang = target_lang if target_lang in languages else (languages[0] if languages else "en")
+ lang = (
+ target_lang
+ if target_lang in languages
+ else (languages[0] if languages else "en")
+ )
templates.append((path, cfg.type, _get_description(cfg, lang)))
if include_methods and target_lang != "zh":
from hyperextract.methods import list_method_cfgs
+
method_templates = list_method_cfgs()
for name, cfg in method_templates.items():
templates.append((name, cfg.type, cfg.description))
@@ -88,12 +101,16 @@ def template(
console.print(" [cyan]model[/cyan] - Structured data model")
console.print(" [cyan]set[/cyan] - Entity collection")
console.print()
- console.print("[dim]Tip: Use [bold]he parse <input> -t <template_id> -l <lang>[/bold] to extract with a template[/dim]")
+ console.print(
+ "[dim]Tip: Use [bold]he parse <input> -t <template_id> -l <lang>[/bold] to extract with a template[/dim]"
+ )
@app.command(name="method")
def method(
- query: Optional[str] = typer.Option(None, "--query", "-q", help="Query to search methods"),
+ query: Optional[str] = typer.Option(
+ None, "--query", "-q", help="Query to search methods"
+ ),
):
"""List all available extraction methods with detailed information."""
from hyperextract.methods import list_methods
@@ -103,8 +120,10 @@ def method(
if query:
query_lower = query.lower()
items = {
- k: v for k, v in items.items()
- if query_lower in k.lower() or query_lower in v.get("description", "").lower()
+ k: v
+ for k, v in items.items()
+ if query_lower in k.lower()
+ or query_lower in v.get("description", "").lower()
}
if not items:
@@ -125,17 +144,17 @@ def method(
for name, info in items.items():
method_type = info["type"]
description = info["description"]
- table.add_row(
- Text(f"method/\n{name}", style="cyan"),
- method_type,
- description
- )
+ table.add_row(Text(f"method/\n{name}", style="cyan"), method_type, description)
console.print(table)
console.print(f"\n[dim]Total: {len(items)} methods[/dim]")
console.print()
console.print("[dim]Method explanations:[/dim]")
console.print(" [cyan]graph[/cyan] - Binary relation graph (entity-relationship)")
- console.print(" [cyan]hypergraph[/cyan] - N-ary hypergraph (multi-entity relations)")
+ console.print(
+ " [cyan]hypergraph[/cyan] - N-ary hypergraph (multi-entity relations)"
+ )
console.print()
- console.print("[dim]Tip: Use [bold]he parse <input> -m <method_name>[/bold] to extract with a method[/dim]")
+ console.print(
+ "[dim]Tip: Use [bold]he parse <input> -m <method_name>[/bold] to extract with a method[/dim]"
+ )
diff --git a/hyperextract/cli/config.py b/hyperextract/cli/config.py
index 13c001a..c8f3438 100644
--- a/hyperextract/cli/config.py
+++ b/hyperextract/cli/config.py
@@ -162,10 +162,16 @@ class ConfigManager:
embedder_config = self.get_embedder_config()
if not llm_config.api_key:
- return False, "LLM API key is not configured. Run 'he config llm --api-key YOUR_KEY'"
+ return (
+ False,
+ "LLM API key is not configured. Run 'he config llm --api-key YOUR_KEY'",
+ )
if not embedder_config.api_key:
- return False, "Embedder API key is not configured. Run 'he config embedder --api-key YOUR_KEY'"
+ return (
+ False,
+ "Embedder API key is not configured. Run 'he config embedder --api-key YOUR_KEY'",
+ )
return True, "Configuration is valid"
diff --git a/hyperextract/methods/__init__.py b/hyperextract/methods/__init__.py
index 2b25c7f..0290331 100644
--- a/hyperextract/methods/__init__.py
+++ b/hyperextract/methods/__init__.py
@@ -27,4 +27,3 @@ __all__ = [
"list_method_cfgs",
"MethodCfg",
]
-
diff --git a/hyperextract/methods/rag/__init__.py b/hyperextract/methods/rag/__init__.py
index ad0a501..435303d 100644
--- a/hyperextract/methods/rag/__init__.py
+++ b/hyperextract/methods/rag/__init__.py
@@ -17,4 +17,3 @@ __all__ = [
"Light_RAG",
"Graph_RAG",
]
-
diff --git a/hyperextract/methods/rag/cog_rag.py b/hyperextract/methods/rag/cog_rag.py
index a74a137..84ebe17 100644
--- a/hyperextract/methods/rag/cog_rag.py
+++ b/hyperextract/methods/rag/cog_rag.py
@@ -459,7 +459,7 @@ class Cog_RAG:
# 导出根目录 data.json 以供 CLI 合法性校验和 info 命令统计
root_data = {
"nodes": [n.model_dump() for n in self.nodes],
- "edges": [e.model_dump() for e in self.edges]
+ "edges": [e.model_dump() for e in self.edges],
}
with open(os.path.join(folder_path, "data.json"), "w", encoding="utf-8") as f:
json.dump(root_data, f, ensure_ascii=False, indent=2)
@@ -479,11 +479,15 @@ class Cog_RAG:
self.metadata = params
theme_path = os.path.join(folder_path, "theme_layer")
- if os.path.exists(theme_path) and os.path.exists(os.path.join(theme_path, "data.json")):
+ if os.path.exists(theme_path) and os.path.exists(
+ os.path.join(theme_path, "data.json")
+ ):
self.theme_layer.load(theme_path)
detail_path = os.path.join(folder_path, "detail_layer")
- if os.path.exists(detail_path) and os.path.exists(os.path.join(detail_path, "data.json")):
+ if os.path.exists(detail_path) and os.path.exists(
+ os.path.join(detail_path, "data.json")
+ ):
self.detail_layer.load(detail_path)
def show(
@@ -495,17 +499,17 @@ class Cog_RAG:
print("\n[Cog_RAG] 这是一个双层图谱系统。请选择你想可视化的层级:")
print("1. Detail Layer (细节层:具体实体及其关系) - [默认]")
print("2. Theme Layer (主题层:宏观主题及其参与者)")
-
+
try:
choice = input("请输入选项 [1/2]: ").strip()
except EOFError:
choice = "1"
-
+
layer_map = {
"2": ("Theme Layer", self.theme_layer),
"1": ("Detail Layer", self.detail_layer),
}
-
+
layer_name, layer_obj = layer_map.get(choice, layer_map["1"])
print(f"Visualizing Cog_RAG ({layer_name})...")
diff --git a/hyperextract/methods/rag/graph_rag.py b/hyperextract/methods/rag/graph_rag.py
index c945ee2..f91a8e8 100644
--- a/hyperextract/methods/rag/graph_rag.py
+++ b/hyperextract/methods/rag/graph_rag.py
@@ -15,6 +15,7 @@ from ontomem.merger import CustomRuleMerger
try:
from graspologic.partition import hierarchical_leiden
+
HAS_GRASPOLOGIC = True
except ImportError:
HAS_GRASPOLOGIC = False
@@ -24,6 +25,7 @@ def _ensure_networkx():
"""Lazy import networkx, raise helpful error if not installed."""
try:
import networkx as nx
+
return nx
except ImportError:
raise ImportError(
@@ -31,6 +33,7 @@ def _ensure_networkx():
"Install with: pip install networkx"
)
+
# ============================================================================
# Node Schema
# ============================================================================
@@ -72,20 +75,26 @@ class EdgeSchema(BaseModel):
# Community Report Schema
# ============================================================================
+
class CommunityFinding(BaseModel):
summary: str = Field(description="Summary of the insight")
explanation: str = Field(description="Detailed explanation of the insight")
+
class CommunityReport(BaseModel):
"""Represents a summary of a graph community."""
-
+
title: str = Field(description="Short title for the community")
- summary: str = Field(description="Detailed executive summary of the community's structure and key entities")
+ summary: str = Field(
+ description="Detailed executive summary of the community's structure and key entities"
+ )
rating: float = Field(description="Impact severity rating (0-10)")
findings: List[CommunityFinding] = Field(description="List of key findings")
# Optional fields (filled programmatically)
id: Optional[str] = Field(default=None, description="Community ID")
- key_entities: Optional[List[str]] = Field(default_factory=list, description="List of key entities in this community")
+ key_entities: Optional[List[str]] = Field(
+ default_factory=list, description="List of key entities in this community"
+ )
# ============================================================================
@@ -278,26 +287,32 @@ class Graph_RAG(AutoGraph[NodeSchema, EdgeSchema]):
# Community Management
self.community_reports: Dict[str, CommunityReport] = {}
self._community_graph: Optional[Any] = None
- self._community_hierarchy: Dict[int, Dict[str, List[str]]] = {} # level -> {community_id: [node_names]}
- self._node_to_community: Dict[str, Dict[str, Any]] = {} # node_name -> {level: community_id}
+ self._community_hierarchy: Dict[
+ int, Dict[str, List[str]]
+ ] = {} # level -> {community_id: [node_names]}
+ self._node_to_community: Dict[
+ str, Dict[str, Any]
+ ] = {} # node_name -> {level: community_id}
def dump(self, folder_path: str | Path) -> None:
"""Saves graph state: internal data, community reports, and GraphML for visualization."""
root = Path(folder_path)
-
+
if self.verbose:
print("Saving Results...")
print("=" * 80)
-
+
# 1. Save standard graph data (nodes, edges, index)
super().dump(folder_path)
# 2. Save Community Data (only if exists)
if self.community_reports:
community_data = {
- "reports": {k: v.model_dump() for k, v in self.community_reports.items()},
+ "reports": {
+ k: v.model_dump() for k, v in self.community_reports.items()
+ },
"hierarchy": {str(k): v for k, v in self._community_hierarchy.items()},
- "node_map": self._node_to_community
+ "node_map": self._node_to_community,
}
comm_path = root / "community_data.json"
@@ -321,10 +336,12 @@ class Graph_RAG(AutoGraph[NodeSchema, EdgeSchema]):
data = json.load(f)
self.community_reports = {
- k: CommunityReport.model_validate(v) for k, v in data.get("reports", {}).items()
+ k: CommunityReport.model_validate(v)
+ for k, v in data.get("reports", {}).items()
}
self._community_hierarchy = {
- int(k) if k.isdigit() else k: v for k, v in data.get("hierarchy", {}).items()
+ int(k) if k.isdigit() else k: v
+ for k, v in data.get("hierarchy", {}).items()
}
self._node_to_community = data.get("node_map", {})
@@ -343,14 +360,16 @@ class Graph_RAG(AutoGraph[NodeSchema, EdgeSchema]):
for n in self.nodes:
self._community_graph.add_node(n.name, **n.model_dump())
for e in self.edges:
- self._community_graph.add_edge(e.source, e.target, weight=float(e.strength), **e.model_dump())
+ self._community_graph.add_edge(
+ e.source, e.target, weight=float(e.strength), **e.model_dump()
+ )
return self._community_graph
def build_communities(self, level: int = 0):
"""
Detects communities in the graph and generates reports for them.
Uses Leiden algorithm (via graspologic).
-
+
Args:
level: The hierarchical level for Leiden algorithm (default 0).
"""
@@ -363,21 +382,28 @@ class Graph_RAG(AutoGraph[NodeSchema, EdgeSchema]):
if self.verbose:
print("Building graph topology...")
-
+
# Construct NetworkX graph
G = nx.Graph()
for node in self.nodes:
G.add_node(node.name, **node.model_dump())
-
+
for edge in self.edges:
# Use 'weight' if available, mapped from 'strength'
- G.add_edge(edge.source, edge.target, weight=float(edge.strength), **edge.model_dump())
-
+ G.add_edge(
+ edge.source,
+ edge.target,
+ weight=float(edge.strength),
+ **edge.model_dump(),
+ )
+
self._community_graph = G
if not HAS_GRASPOLOGIC:
if self.verbose:
- print("Error: `graspologic` is not installed. Please install it to use Leiden community detection.")
+ print(
+ "Error: `graspologic` is not installed. Please install it to use Leiden community detection."
+ )
return
# Community Detection Strategy
@@ -386,13 +412,13 @@ class Graph_RAG(AutoGraph[NodeSchema, EdgeSchema]):
if self.verbose:
print("Using Hierarchical Leiden algorithm (graspologic)...")
-
+
try:
# Create a subgraph without isolated nodes for Leiden to avoid warnings
G_leiden = G.copy()
isolates = list(nx.isolates(G_leiden))
G_leiden.remove_nodes_from(isolates)
-
+
if G_leiden.number_of_nodes() == 0:
if self.verbose:
print("No connected nodes found. Skipping community detection.")
@@ -401,32 +427,32 @@ class Graph_RAG(AutoGraph[NodeSchema, EdgeSchema]):
# Hierarchical Leiden returns a list of CommunityAssignment objects
# Each assignment has (node, cluster, level)
hierarchical_clusters = hierarchical_leiden(G_leiden, max_cluster_size=100)
-
+
# Transform to our internal structure: level -> community_id -> [nodes]
self._community_hierarchy = {}
self._node_to_community = {}
-
+
# Group by level first
clusters_by_level = {}
for assignment in hierarchical_clusters:
lvl = assignment.level
cluster_id = f"LEVEL_{lvl}_CID_{assignment.cluster}"
node = assignment.node
-
+
if lvl not in clusters_by_level:
clusters_by_level[lvl] = {}
if cluster_id not in clusters_by_level[lvl]:
clusters_by_level[lvl][cluster_id] = []
-
+
clusters_by_level[lvl][cluster_id].append(node)
-
+
# Track node mapping
if node not in self._node_to_community:
self._node_to_community[node] = {}
self._node_to_community[node][f"level_{lvl}"] = cluster_id
self._community_hierarchy = clusters_by_level
-
+
# Select the requested level for reports
# In graspologic, level 0 is usually the top-most (coarsest)
if level in self._community_hierarchy:
@@ -442,7 +468,7 @@ class Graph_RAG(AutoGraph[NodeSchema, EdgeSchema]):
if self.verbose:
print(f"Leiden algorithm failed: {e}.")
return
-
+
if self.verbose:
print(f"Detected {len(communities)} communities using {community_algo}.")
@@ -450,10 +476,11 @@ class Graph_RAG(AutoGraph[NodeSchema, EdgeSchema]):
if self.verbose:
print("Generating community reports...")
self.community_reports = {}
-
- chain = (
- ChatPromptTemplate.from_template(COMMUNITY_REPORT_PROMPT)
- | self.llm_client.with_structured_output(CommunityReport, method="function_calling")
+
+ chain = ChatPromptTemplate.from_template(
+ COMMUNITY_REPORT_PROMPT
+ ) | self.llm_client.with_structured_output(
+ CommunityReport, method="function_calling"
)
for i, community_nodes in enumerate(communities):
@@ -462,25 +489,32 @@ class Graph_RAG(AutoGraph[NodeSchema, EdgeSchema]):
subgraph_nodes = [n for n in self.nodes if n.name in community_nodes]
# Edges where *both* source and target are in the community (strict)
# Or edges where *at least one* is in the community (relaxed) -> tailored for coverage
- subgraph_edges = [e for e in self.edges if e.source in community_nodes and e.target in community_nodes]
-
+ subgraph_edges = [
+ e
+ for e in self.edges
+ if e.source in community_nodes and e.target in community_nodes
+ ]
+
if not subgraph_nodes:
continue
# Format minimal context for LLM to save tokens
- entities_txt = "\\n".join([f"- {n.name} ({n.type}): {n.description}" for n in subgraph_nodes])
- relationships_txt = "\\n".join([f"- {e.source}->{e.target}: {e.description}" for e in subgraph_edges])
-
+ entities_txt = "\\n".join(
+ [f"- {n.name} ({n.type}): {n.description}" for n in subgraph_nodes]
+ )
+ relationships_txt = "\\n".join(
+ [f"- {e.source}->{e.target}: {e.description}" for e in subgraph_edges]
+ )
+
try:
- report = chain.invoke({
- "entities": entities_txt,
- "relationships": relationships_txt
- })
+ report = chain.invoke(
+ {"entities": entities_txt, "relationships": relationships_txt}
+ )
# Post-processing
report.id = cid
- report.key_entities = list(community_nodes)[:10] # Track top entities
+ report.key_entities = list(community_nodes)[:10] # Track top entities
self.community_reports[cid] = report
-
+
# Check mapping for manual fallback methods
for node_name in community_nodes:
if node_name not in self._node_to_community:
@@ -492,7 +526,9 @@ class Graph_RAG(AutoGraph[NodeSchema, EdgeSchema]):
print(f"Failed to generate report for community {cid}: {e}")
if self.verbose:
- print(f"Successfully generated {len(self.community_reports)} community reports.")
+ print(
+ f"Successfully generated {len(self.community_reports)} community reports."
+ )
# ============================================================================
# Search Override with Community Support
@@ -547,9 +583,7 @@ class Graph_RAG(AutoGraph[NodeSchema, EdgeSchema]):
return {}
sorted_reports = sorted(
- self.community_reports.values(),
- key=lambda r: r.rating,
- reverse=True
+ self.community_reports.values(), key=lambda r: r.rating, reverse=True
)[:3]
return {
diff --git a/hyperextract/methods/rag/hyper_rag.py b/hyperextract/methods/rag/hyper_rag.py
index fa11dc0..41bf8c7 100644
--- a/hyperextract/methods/rag/hyper_rag.py
+++ b/hyperextract/methods/rag/hyper_rag.py
@@ -206,4 +206,3 @@ class Hyper_RAG(AutoHypergraph[NodeSchema, EdgeSchema]):
max_workers=max_workers,
verbose=verbose,
)
-
diff --git a/hyperextract/methods/rag/light_rag.py b/hyperextract/methods/rag/light_rag.py
index 5034d3f..17370d9 100644
--- a/hyperextract/methods/rag/light_rag.py
+++ b/hyperextract/methods/rag/light_rag.py
@@ -152,7 +152,7 @@ class Light_RAG(AutoGraph[NodeSchema, EdgeSchema]):
node_key_fn = lambda x: x.name
# Edge key: (source, target) tuple
- # Note: LightRAG typically treats relations as directed.
+ # Note: LightRAG typically treats relations as directed.
# For undirected behavior, one might sort specific relation types, but here we enforce Source->Target directionality uniqueness.
edge_key_fn = lambda x: f"{x.source}->{x.target}"
diff --git a/hyperextract/methods/registry.py b/hyperextract/methods/registry.py
index 514e7f2..8e3a4fa 100644
--- a/hyperextract/methods/registry.py
+++ b/hyperextract/methods/registry.py
@@ -98,7 +98,13 @@ def list_method_cfgs() -> Dict[str, MethodCfg]:
def _init_registry():
"""Initialize the registry with built-in methods."""
- from hyperextract.methods.rag import Light_RAG, Hyper_RAG, HyperGraph_RAG, Cog_RAG, Graph_RAG
+ from hyperextract.methods.rag import (
+ Light_RAG,
+ Hyper_RAG,
+ HyperGraph_RAG,
+ Cog_RAG,
+ Graph_RAG,
+ )
from hyperextract.methods.typical import iText2KG, iText2KG_Star, KG_Gen, Atom
register_method(
diff --git a/hyperextract/methods/typical/__init__.py b/hyperextract/methods/typical/__init__.py
index d6336b6..2923aa2 100644
--- a/hyperextract/methods/typical/__init__.py
+++ b/hyperextract/methods/typical/__init__.py
@@ -15,4 +15,3 @@ __all__ = [
"KG_Gen",
"Atom",
]
-
diff --git a/hyperextract/methods/typical/atom.py b/hyperextract/methods/typical/atom.py
index 6763f73..1611683 100644
--- a/hyperextract/methods/typical/atom.py
+++ b/hyperextract/methods/typical/atom.py
@@ -500,7 +500,10 @@ class Atom(AutoGraph[NodeSchema, EdgeSchema]):
fact_chunks.append(formatted_chunk)
# 6. Batch Extract Edges directly from Fact Chunks
- edge_inputs = [{"source_text": chunk, "observation_time": obs_date_str} for chunk in fact_chunks]
+ edge_inputs = [
+ {"source_text": chunk, "observation_time": obs_date_str}
+ for chunk in fact_chunks
+ ]
chunk_edge_lists = self.edge_extractor.batch(
edge_inputs, config={"max_concurrency": self.max_workers}
)
diff --git a/hyperextract/methods/typical/itext2kg.py b/hyperextract/methods/typical/itext2kg.py
index 6877e75..a478936 100644
--- a/hyperextract/methods/typical/itext2kg.py
+++ b/hyperextract/methods/typical/itext2kg.py
@@ -1,7 +1,7 @@
"""iText2KG implementation using AutoGraph as the core engine.
-This module provides a specialized implementation of the iText2KG algorithm
-using the AutoGraph framework, designed for extracting high-quality,
+This module provides a specialized implementation of the iText2KG algorithm
+using the AutoGraph framework, designed for extracting high-quality,
standardized triple-based knowledge graphs.
Prompts and schemas are adapted from the original iText2KG* implementation.
@@ -184,4 +184,3 @@ class iText2KG(AutoGraph[NodeSchema, EdgeSchema]):
max_workers=max_workers,
verbose=verbose,
)
-
diff --git a/hyperextract/methods/typical/itext2kg_star.py b/hyperextract/methods/typical/itext2kg_star.py
index 7ed3359..70af0a8 100644
--- a/hyperextract/methods/typical/itext2kg_star.py
+++ b/hyperextract/methods/typical/itext2kg_star.py
@@ -337,4 +337,3 @@ class iText2KG_Star(AutoGraph[NodeSchema, EdgeSchema]):
f"✅ Node matching complete: Nodes {len(nodes)} -> {len(self.nodes)}, Edges: {len(edges)}"
)
return self
-
diff --git a/hyperextract/methods/typical/kg_gen.py b/hyperextract/methods/typical/kg_gen.py
index 1f3b28a..94c8205 100644
--- a/hyperextract/methods/typical/kg_gen.py
+++ b/hyperextract/methods/typical/kg_gen.py
@@ -291,4 +291,3 @@ class KG_Gen(AutoGraph[NodeSchema, EdgeSchema]):
f"Edges {original_edge_count} -> {len(self.edges)}"
)
return self
-
diff --git a/hyperextract/types/__init__.py b/hyperextract/types/__init__.py
index 21f5eb2..36abe1c 100644
--- a/hyperextract/types/__init__.py
+++ b/hyperextract/types/__init__.py
@@ -47,8 +47,9 @@ These types inject "Observation Context" (Current Time/Location) into the LLM pr
"""
import os
+
# Suppress Transformers warnings
-os.environ['TRANSFORMERS_NO_ADVISORY_WARNINGS'] = '1'
+os.environ["TRANSFORMERS_NO_ADVISORY_WARNINGS"] = "1"
from .base import BaseAutoType
from .model import AutoModel
diff --git a/hyperextract/types/hypergraph.py b/hyperextract/types/hypergraph.py
index d758800..73bc40a 100644
--- a/hyperextract/types/hypergraph.py
+++ b/hyperextract/types/hypergraph.py
@@ -296,7 +296,7 @@ class AutoHypergraph(
verbose=verbose,
fields_for_index=edge_fields_for_index, # Pass edge field selection to OMem
)
-
+
# Store label extractors for visualization
self._node_label_extractor = node_label_extractor
self._edge_label_extractor = edge_label_extractor
diff --git a/hyperextract/types/list.py b/hyperextract/types/list.py
index 51fbcec..66cd8e2 100644
--- a/hyperextract/types/list.py
+++ b/hyperextract/types/list.py
@@ -377,8 +377,8 @@ class AutoList(BaseAutoType[AutoListSchema[ItemSchema]], Generic[ItemSchema]):
on_chat=chat_callback,
context={
"title": f"{self.item_schema.__name__} List",
- "description": f"Visualizing {len(self.items)} items in AutoList"
- }
+ "description": f"Visualizing {len(self.items)} items in AutoList",
+ },
)
# ==================== Pythonic Sequence Operations ====================
diff --git a/hyperextract/types/set.py b/hyperextract/types/set.py
index c1e89c4..ea32048 100644
--- a/hyperextract/types/set.py
+++ b/hyperextract/types/set.py
@@ -421,8 +421,8 @@ class AutoSet(BaseAutoType[AutoSetSchema[ItemSchema]], Generic[ItemSchema]):
on_chat=chat_callback,
context={
"title": f"{self.item_schema.__name__} Set",
- "description": f"Visualizing {len(self.items)} unique items in AutoSet"
- }
+ "description": f"Visualizing {len(self.items)} unique items in AutoSet",
+ },
)
# ==================== Set Interface Methods ====================
diff --git a/hyperextract/types/spatial_graph.py b/hyperextract/types/spatial_graph.py
index 2d0424a..854fb15 100644
--- a/hyperextract/types/spatial_graph.py
+++ b/hyperextract/types/spatial_graph.py
@@ -269,7 +269,9 @@ class AutoSpatialGraph(AutoGraph[NodeSchema, EdgeSchema]):
}
)
- return self.edge_extractor.batch(inputs, config={"max_concurrency": self.max_workers})
+ return self.edge_extractor.batch(
+ inputs, config={"max_concurrency": self.max_workers}
+ )
def _extract_data_by_one_stage(self, text: str) -> Any:
"""Override: Inject observation_location into one-stage extraction."""
diff --git a/hyperextract/types/spatio_temporal_graph.py b/hyperextract/types/spatio_temporal_graph.py
index 3f45692..78c1f37 100644
--- a/hyperextract/types/spatio_temporal_graph.py
+++ b/hyperextract/types/spatio_temporal_graph.py
@@ -286,7 +286,9 @@ class AutoSpatioTemporalGraph(AutoGraph[NodeSchema, EdgeSchema]):
}
)
- return self.edge_extractor.batch(inputs, config={"max_concurrency": self.max_workers})
+ return self.edge_extractor.batch(
+ inputs, config={"max_concurrency": self.max_workers}
+ )
def _extract_data_by_one_stage(self, text: str) -> Any:
"""Inject observation_time and observation_location into one-stage extraction."""
diff --git a/hyperextract/types/temporal_graph.py b/hyperextract/types/temporal_graph.py
index 0571c83..3155254 100644
--- a/hyperextract/types/temporal_graph.py
+++ b/hyperextract/types/temporal_graph.py
@@ -268,8 +268,7 @@ class AutoTemporalGraph(AutoGraph[NodeSchema, EdgeSchema]):
def _extract_edges_batch(
self, chunks: List[str], node_lists: List[NodeListSchema[NodeSchema]]
) -> List[EdgeListSchema[EdgeSchema]]:
- """Override: Inject observation_time into edge extraction during two-stage extraction.
- """
+ """Override: Inject observation_time into edge extraction during two-stage extraction."""
inputs = []
for chunk, node_list in zip(chunks, node_lists):
nodes = node_list.items if node_list else []
@@ -287,7 +286,9 @@ class AutoTemporalGraph(AutoGraph[NodeSchema, EdgeSchema]):
}
)
- return self.edge_extractor.batch(inputs, config={"max_concurrency": self.max_workers})
+ return self.edge_extractor.batch(
+ inputs, config={"max_concurrency": self.max_workers}
+ )
def _extract_data_by_one_stage(self, text: str) -> Any:
"""Override: Inject observation_time into one-stage extraction."""
diff --git a/hyperextract/utils/client.py b/hyperextract/utils/client.py
index efe0cc9..e6a02ea 100644
--- a/hyperextract/utils/client.py
+++ b/hyperextract/utils/client.py
@@ -11,7 +11,7 @@ DEFAULT_CONFIG_DIR = Path.home() / ".he"
DEFAULT_CONFIG_FILE = DEFAULT_CONFIG_DIR / "config.toml"
-def get_client(config_path: str|Path = None) -> Tuple[BaseChatModel, Embeddings]:
+def get_client(config_path: str | Path = None) -> Tuple[BaseChatModel, Embeddings]:
"""Get OpenAI LLM client and Embedder from config.
Args:
diff --git a/hyperextract/utils/logging.py b/hyperextract/utils/logging.py
index b174693..855e44d 100644
--- a/hyperextract/utils/logging.py
+++ b/hyperextract/utils/logging.py
@@ -23,6 +23,7 @@ def configure_logging(
output_file: Optional file path to write logs.
"""
import os
+
level = os.getenv(ENV_LOG_LEVEL, level).upper()
level_value = getattr(logging, level, logging.WARNING)
@@ -34,7 +35,8 @@ def configure_logging(
structlog.processors.StackInfoRenderer(),
structlog.processors.format_exc_info,
structlog.processors.UnicodeDecoder(),
- structlog.dev.ConsoleRenderer(colors=True) if not json_output
+ structlog.dev.ConsoleRenderer(colors=True)
+ if not json_output
else structlog.processors.JSONRenderer(),
],
wrapper_class=structlog.stdlib.BoundLogger,
@@ -54,6 +56,7 @@ def configure_logging(
if final_output_file:
from pathlib import Path
+
log_path = Path(final_output_file)
log_path.parent.mkdir(parents=True, exist_ok=True)
file_handler = logging.FileHandler(str(log_path), encoding="utf-8")
diff --git a/hyperextract/utils/template_engine/factory.py b/hyperextract/utils/template_engine/factory.py
index 11e4251..63c12a4 100644
--- a/hyperextract/utils/template_engine/factory.py
+++ b/hyperextract/utils/template_engine/factory.py
@@ -465,13 +465,17 @@ class TemplateFactory:
match template_cfg.type:
case "model":
- template = cls.create_model(template_cfg, llm_client, embedder, **kwargs)
+ template = cls.create_model(
+ template_cfg, llm_client, embedder, **kwargs
+ )
case "list":
template = cls.create_list(template_cfg, llm_client, embedder, **kwargs)
case "set":
template = cls.create_set(template_cfg, llm_client, embedder, **kwargs)
case "graph":
- template = cls.create_graph(template_cfg, llm_client, embedder, **kwargs)
+ template = cls.create_graph(
+ template_cfg, llm_client, embedder, **kwargs
+ )
case "hypergraph":
template = cls.create_hypergraph(
template_cfg, llm_client, embedder, **kwargs
diff --git a/hyperextract/utils/template_engine/parsers/guideline.py b/hyperextract/utils/template_engine/parsers/guideline.py
index c5a3008..00e1bf6 100644
--- a/hyperextract/utils/template_engine/parsers/guideline.py
+++ b/hyperextract/utils/template_engine/parsers/guideline.py
@@ -99,8 +99,6 @@ def parse_guideline(guideline, autotype: str) -> str | Tuple[str, str, str]:
return "\n\n".join(main_parts), "\n\n".join(node_parts), "\n\n".join(edge_parts)
-
-
__all__ = [
"parse_guideline",
]
diff --git a/hyperextract/utils/template_engine/template.py b/hyperextract/utils/template_engine/template.py
index 1ecfa33..291b322 100644
--- a/hyperextract/utils/template_engine/template.py
+++ b/hyperextract/utils/template_engine/template.py
@@ -132,9 +132,8 @@ class Template:
if include_methods:
from hyperextract.methods import list_method_cfgs
+
method_templates = list_method_cfgs()
templates.update(method_templates)
return templates
-
-
diff --git a/mkdocs.yml b/mkdocs.yml
index ed60649..0e725fc 100644
--- a/mkdocs.yml
+++ b/mkdocs.yml
@@ -3,7 +3,7 @@
site_name: Hyper-Extract
site_description: LLM-powered knowledge extraction framework
site_author: Yifan Feng
-site_url: https://yifanfeng97.github.io/hyper-extract/
+site_url: https://yifanfeng97.github.io/Hyper-Extract/
repo_name: yifanfeng97/hyper-extract
repo_url: https://github.com/yifanfeng97/hyper-extract