Skip to content

Hypergraph is more powerful. Transform unstructured text into structured knowledge with LLMs. Graphs, hypergraphs, and spatio-temporal extractions — with one command.

refactor(template_engine): 重构模板配置和解析逻辑

yifanfeng97bba1496e7381
diff --git a/hyperextract/cli/utils.py b/hyperextract/cli/utils.py
index 470fa3b..b9875e6 100644
--- a/hyperextract/cli/utils.py
+++ b/hyperextract/cli/utils.py
@@ -6,7 +6,7 @@ from pathlib import Path
from rich.console import Console
from typing import Optional, Tuple
-from hyperextract.utils.template_engine import Template, TemplateConfig
+from hyperextract.utils.template_engine import Template, TemplateCfg
from .config import ConfigManager
console = Console()
@@ -192,7 +192,7 @@ def create_template(template: str, lang: str = "zh"):
return Template.create(template, lang)
-def get_template_config(template: str, lang: str = "zh") -> Optional[TemplateConfig]:
+def get_template_config(template: str, lang: str = "zh") -> Optional[TemplateCfg]:
"""Get template config.
Args:
@@ -200,16 +200,16 @@ def get_template_config(template: str, lang: str = "zh") -> Optional[TemplateCon
lang: Language (zh/en)
Returns:
- TemplateConfig if found
+ TemplateCfg if found
"""
return Template.get(template, lang)
-def get_auto_type_from_config(config: TemplateConfig) -> str:
+def get_auto_type_from_config(config: TemplateCfg) -> str:
"""Infer AutoType from template config.
Args:
- config: TemplateConfig instance
+ config: TemplateCfg instance
Returns:
AutoType string (e.g., "AutoGraph")
diff --git a/hyperextract/utils/template_engine/__init__.py b/hyperextract/utils/template_engine/__init__.py
index 505a9fd..5be7eb9 100644
--- a/hyperextract/utils/template_engine/__init__.py
+++ b/hyperextract/utils/template_engine/__init__.py
@@ -4,10 +4,11 @@ Main Components:
- Template: Unified API for template operations
- Gallery: Template library management
- TemplateFactory: Creates template instances from configuration
-- TemplateConfig: Configuration model
+- RawTemplateCfg: Raw YAML config (multilingual) for validation
+- TemplateCfg: Single-language config for actual use
- ConfigLoader: Loads YAML configuration
-- SchemaBuilder: Dynamically generates Pydantic Schema
-- PromptParser: Parses prompts based on template type
+- SchemaParser: Parses output schemas based on template type
+- GuidelineParser: Parses prompts based on template type
- IdentifierResolver: Resolves identifier extractors
"""
@@ -15,15 +16,26 @@ from .template import Template
from .gallery import Gallery
from .factory import TemplateFactory, TemplateWrapper
from .builder import (
- TemplateConfig,
+ RawTemplateCfg,
+ TemplateCfg,
ConfigLoader,
SchemaBuilder,
+ SchemaParser,
+ GuidelineParser,
PromptParser,
IdentifierResolver,
OptionsBuilder,
+ parse_multi_lang,
+ GuidelineSchema,
+ GuidelineSchemaMono,
+ OutputSchema,
+ OutputSchemaMono,
+ DisplaySchema,
+ DisplaySchemaMono,
+ IdentifiersSchema,
+ FieldSchema,
+ FieldsDefinition,
)
-from .builder.prompt import Guideline
-from .builder.schema import OutputDefinition, FieldsDefinition
__all__ = [
@@ -31,13 +43,23 @@ __all__ = [
"Gallery",
"TemplateFactory",
"TemplateWrapper",
- "TemplateConfig",
+ "RawTemplateCfg",
+ "TemplateCfg",
"ConfigLoader",
"SchemaBuilder",
+ "SchemaParser",
+ "GuidelineParser",
"PromptParser",
"IdentifierResolver",
"OptionsBuilder",
- "Guideline",
- "OutputDefinition",
+ "parse_multi_lang",
+ "GuidelineSchema",
+ "GuidelineSchemaMono",
+ "OutputSchema",
+ "OutputSchemaMono",
+ "DisplaySchema",
+ "DisplaySchemaMono",
+ "IdentifiersSchema",
+ "FieldSchema",
"FieldsDefinition",
]
diff --git a/hyperextract/utils/template_engine/builder/__init__.py b/hyperextract/utils/template_engine/builder/__init__.py
index 84861a6..15b1598 100644
--- a/hyperextract/utils/template_engine/builder/__init__.py
+++ b/hyperextract/utils/template_engine/builder/__init__.py
@@ -1,9 +1,25 @@
"""Builder module - Configuration models, loaders and builders."""
-from .schema import SchemaBuilder
-from .prompt import PromptParser
+from .loader import (
+ RawTemplateCfg,
+ TemplateCfg,
+ ConfigLoader,
+ parse_multi_lang,
+ GuidelineSchema,
+ GuidelineSchemaMono,
+ FieldSchema,
+ FieldsDefinition,
+ OutputSchema,
+ OutputSchemaMono,
+ OptionsSchema,
+ DisplaySchema,
+ DisplaySchemaMono,
+ IdentifiersSchema,
+ TemplateSchema,
+)
+from .output import SchemaParser, SchemaBuilder
+from .guideline import GuidelineParser, PromptParser
from .identifiers import IdentifierResolver
-from .loader import TemplateConfig, ConfigLoader
from .options import (
Options,
OptionsBuilder,
@@ -11,11 +27,26 @@ from .options import (
__all__ = [
- "TemplateConfig",
+ "RawTemplateCfg",
+ "TemplateCfg",
"ConfigLoader",
- "Options",
+ "parse_multi_lang",
+ "GuidelineSchema",
+ "GuidelineSchemaMono",
+ "FieldSchema",
+ "FieldsDefinition",
+ "OutputSchema",
+ "OutputSchemaMono",
+ "OptionsSchema",
+ "DisplaySchema",
+ "DisplaySchemaMono",
+ "IdentifiersSchema",
+ "TemplateSchema",
+ "SchemaParser",
"SchemaBuilder",
+ "GuidelineParser",
"PromptParser",
"IdentifierResolver",
+ "Options",
"OptionsBuilder",
]
diff --git a/hyperextract/utils/template_engine/builder/prompt.py b/hyperextract/utils/template_engine/builder/guideline.py
similarity index 51%
rename from hyperextract/utils/template_engine/builder/prompt.py
rename to hyperextract/utils/template_engine/builder/guideline.py
index 7d92fcc..b6badd1 100644
--- a/hyperextract/utils/template_engine/builder/prompt.py
+++ b/hyperextract/utils/template_engine/builder/guideline.py
@@ -1,45 +1,40 @@
-"""Extraction models and prompt builder."""
+"""Extraction prompt parser from guideline."""
from typing import Tuple
-from .tools import get_text, TemplateSchema, GuidelineSchema
-
-
-__all__ = [
- "PromptParser",
-]
+from .loader import parse_multi_lang, GuidelineSchemaMono
LABEL_MAPPING = {
"zh": {
"role_and_task": "角色与任务",
- "source_text": "源文本",
"known_entities": "已知实体",
- "extraction_rules": "提取规则",
+ "rules": "提取规则",
"entity_rules": "实体提取规则",
"relation_rules": "关系提取规则",
"time_rules": "时间规则",
"location_rules": "位置规则",
+ "source_text": "源文本",
},
"en": {
"role_and_task": "Role and Task",
- "source_text": "Source Text",
"known_entities": "Known Entities",
- "extraction_rules": "Extraction Rules",
+ "rules": "Extraction Rules",
"entity_rules": "Entity Extraction Rules",
"relation_rules": "Relation Extraction Rules",
"time_rules": "Time Rules",
"location_rules": "Location Rules",
+ "source_text": "Source Text",
},
}
-def PromptParser(
- template: TemplateSchema, language: str = "en"
+def GuidelineParser(
+ template: GuidelineSchemaMono, language: str = "en"
) -> Tuple[str, str, str]:
- """Parse template and return prompts based on autotype.
+ """Parse guideline and return prompts based on autotype.
Args:
- template: Template Configuration instance
+ template: Guideline Configuration instance (mono-language version)
language: Language code (default: "en")
Returns:
@@ -47,61 +42,77 @@ def PromptParser(
- For non-graph types: returns main_prompt
- For graph types in two_stage mode: returns (main_prompt, node_prompt, edge_prompt)
"""
- labels = LABEL_MAPPING.get(language, LABEL_MAPPING["zh"])
- autotype = template.type
- guideline: GuidelineSchema = template.guideline
+ labels = LABEL_MAPPING.get(language, LABEL_MAPPING["en"])
+ guideline: GuidelineSchemaMono = template
prefix_prompt = (
- f"# {labels.get('role_and_task')}:\n{get_text(guideline.target, language)}"
+ f"# {labels.get('role_and_task')}:\n{guideline.target}"
)
+ autotype = getattr(template, 'type', None) or "model"
+
if autotype in ("model", "list", "set"):
parts = [prefix_prompt]
- parts.append(
- f"## {labels.get('extraction_rules')}:\n{get_text(guideline.rules, language)}"
- )
+ if guideline.rules:
+ parts.append(
+ f"## {labels.get('rules')}:\n{guideline.rules}"
+ )
parts.append(f"## {labels.get('source_text')}:\n{{source_text}}")
return "\n\n".join(parts)
else:
- # main prompt
main_parts = [prefix_prompt]
- main_parts.append(
- f"## {labels.get('entity_rules')}:\n{get_text(guideline.rules, language)}"
- )
- main_parts.append(
- f"## {labels.get('relation_rules')}:\n{get_text(guideline.rules_for_relations, language)}"
- )
- if autotype in ("temporal_graph", "spatio_temporal_graph"):
+ if guideline.rules_for_entities:
main_parts.append(
- f"## {labels.get('time_rules')}:\n{get_text(guideline.rules_for_time, language)}"
+ f"## {labels.get('entity_rules')}:\n{guideline.rules_for_entities}"
)
- if autotype in ("spatial_graph", "spatio_temporal_graph"):
+ if guideline.rules_for_relations:
main_parts.append(
- f"## {labels.get('location_rules')}:\n{get_text(guideline.rules_for_location, language)}"
+ f"## {labels.get('relation_rules')}:\n{guideline.rules_for_relations}"
)
+ if autotype in ("temporal_graph", "spatio_temporal_graph"):
+ if guideline.rules_for_time:
+ main_parts.append(
+ f"## {labels.get('time_rules')}:\n{guideline.rules_for_time}"
+ )
+ if autotype in ("spatial_graph", "spatio_temporal_graph"):
+ if guideline.rules_for_location:
+ main_parts.append(
+ f"## {labels.get('location_rules')}:\n{guideline.rules_for_location}"
+ )
main_parts.append(f"## {labels.get('source_text')}:\n{{source_text}}")
- # node prompt
node_parts = [prefix_prompt]
- node_parts.append(
- f"## {labels.get('entity_rules')}:\n{get_text(guideline.rules, language)}"
- )
+ if guideline.rules_for_entities:
+ node_parts.append(
+ f"## {labels.get('entity_rules')}:\n{guideline.rules_for_entities}"
+ )
node_parts.append(f"## {labels.get('source_text')}:\n{{source_text}}")
- # edge prompt
edge_parts = [prefix_prompt]
- edge_parts.append(
- f"## {labels.get('relation_rules')}:\n{get_text(guideline.rules_for_relations, language)}"
- )
- if autotype in ("temporal_graph", "spatio_temporal_graph"):
+ if guideline.rules_for_relations:
edge_parts.append(
- f"## {labels.get('time_rules')}:\n{get_text(guideline.rules_for_time, language)}"
+ f"## {labels.get('relation_rules')}:\n{guideline.rules_for_relations}"
)
+ if autotype in ("temporal_graph", "spatio_temporal_graph"):
+ if guideline.rules_for_time:
+ edge_parts.append(
+ f"## {labels.get('time_rules')}:\n{guideline.rules_for_time}"
+ )
if autotype in ("spatial_graph", "spatio_temporal_graph"):
- edge_parts.append(
- f"## {labels.get('location_rules')}:\n{get_text(guideline.rules_for_location, language)}"
- )
+ if guideline.rules_for_location:
+ edge_parts.append(
+ f"## {labels.get('location_rules')}:\n{guideline.rules_for_location}"
+ )
edge_parts.append(f"## {labels.get('known_entities')}:\n{{known_nodes}}")
edge_parts.append(f"## {labels.get('source_text')}:\n{{source_text}}")
return "\n\n".join(main_parts), "\n\n".join(node_parts), "\n\n".join(edge_parts)
+
+
+PromptParser = GuidelineParser
+
+
+__all__ = [
+ "GuidelineParser",
+ "PromptParser",
+]
diff --git a/hyperextract/utils/template_engine/builder/loader.py b/hyperextract/utils/template_engine/builder/loader.py
index 6f0e1a6..b002414 100644
--- a/hyperextract/utils/template_engine/builder/loader.py
+++ b/hyperextract/utils/template_engine/builder/loader.py
@@ -2,58 +2,187 @@
import yaml
from pathlib import Path
-from typing import Any, Dict, List, Literal, Optional, Union
-from pydantic import BaseModel, Field, field_validator, ConfigDict
+from typing import Any, Dict, List, Literal, Union
+from pydantic import BaseModel, Field
-from .prompt import Guideline
-from .identifiers import Identifiers
-from .schema import ModelSchema, ItemSchema, NodeSchema, EdgeSchema, OutputDefinition
-from .options import Options
+
+VALID_MERGE_STRATEGIES = Literal[
+ "merge_field",
+ "keep_incoming",
+ "keep_existing",
+ "llm_balanced",
+ "llm_prefer_incoming",
+ "llm_prefer_existing",
+]
+
+
+def parse_multi_lang(
+ value: str | List[str] | Dict[str, str | List[str]],
+ language: str,
+) -> str:
+ """Get multilingual text value, supports list format."""
+ if isinstance(value, str):
+ return value
+ elif isinstance(value, list):
+ return "\n".join(f"{i + 1}. {item}" for i, item in enumerate(value))
+ else:
+ dict_value = value.get(language)
+ if isinstance(dict_value, str):
+ return dict_value
+ if isinstance(dict_value, list):
+ return "\n".join(f"{i + 1}. {item}" for i, item in enumerate(dict_value))
-class Display(BaseModel):
- """Display configuration."""
+class GuidelineSchema(BaseModel):
+ """Extraction guideline configuration"""
- label: Optional[str] = None
- entity_label: Optional[str] = None
- relation_label: Optional[str] = None
+ target: str | List[str] | Dict[str, str | List[str]]
+ rules: str | List[str] | Dict[str, str | List[str]] | None = None
+ rules_for_entities: str | List[str] | Dict[str, str | List[str]] | None = None
+ rules_for_relations: str | List[str] | Dict[str, str | List[str]] | None = None
+ rules_for_time: str | List[str] | Dict[str, str | List[str]] | None = None
+ rules_for_location: str | List[str] | Dict[str, str | List[str]] | None = None
-class TemplateConfig(BaseModel):
- """Template configuration top-level model."""
+class FieldSchema(BaseModel):
+ """Field definition"""
- language: Optional[Union[str, List[str]]] = "zh"
+ name: str
+ type: str
+ description: str | Dict[str, str]
+ required: bool = False
+ default: Any | None = None
+
+
+class OutputUnitSchema(BaseModel):
+ """Output unit schema"""
+
+ description: str | Dict[str, str]
+ fields: List[FieldSchema] = Field(default_factory=list)
+
+class NaiveOutputSchema(BaseModel):
+ """Naive output schema"""
+
+ description: str | Dict[str, str]
+ fields: List[FieldSchema] = Field(default_factory=list)
+
+class GraphOutputSchema(BaseModel):
+ """Graph output schema"""
+
+ description: str | Dict[str, str]
+ entities: OutputUnitSchema
+ relations: OutputUnitSchema
+
+class OptionsSchema(BaseModel):
+ """Runtime options"""
+
+ chunk_size: int = 2048
+ chunk_overlap: int = 256
+ max_workers: int = 10
+ verbose: bool = False
+ # [non-graph] AutoModel, AutoList, AutoSet
+ merge_strategy: VALID_MERGE_STRATEGIES = "llm_balanced"
+ fields_for_search: List[str] | None = None
+ # [graph] AutoGraph, AutoHypergraph, AutoTemporalGraph, AutoSpatialGraph, AutoSpatioTemporalGraph
+ entity_merge_strategy: VALID_MERGE_STRATEGIES = "llm_balanced"
+ relation_merge_strategy: VALID_MERGE_STRATEGIES = "llm_balanced"
+ extraction_mode: str | None = None
+ entity_fields_for_search: List[str] | None = None
+ relation_fields_for_search: List[str] | None = None
+ # [graph] AutoTemporalGraph, AutoSpatioTemporalGraph
+ observation_time: str | None = None
+ # [graph] AutoSpatialGraph, AutoSpatioTemporalGraph
+ observation_location: str | None = None
+
+
+class DisplaySchema(BaseModel):
+ """Display configuration"""
+
+ # [non-graph] AutoModel, AutoList, AutoSet
+ label: str | None = None
+ # [graph] AutoGraph, AutoHypergraph, AutoTemporalGraph, AutoSpatialGraph, AutoSpatioTemporalGraph
+ entity_label: Union[str, Dict[str, str | None]] = None
+ relation_label: Union[str, Dict[str, str | None]] = None
+
+
+class IdentifiersSchema(BaseModel):
+ """Identifier configuration"""
+
+ # [non-graph] AutoSet
+ item_id: str | None = None
+ # [graph] AutoGraph, AutoHypergraph, AutoTemporalGraph, AutoSpatialGraph, AutoSpatioTemporalGraph
+ entity_id: str | None = None
+ relation_id: str | None = None
+ relation_members: str | Dict[str, str | None, List[str]] | None = None
+ # [graph] AutoTemporalGraph, AutoSpatioTemporalGraph
+ time_field: str | None = None
+ # [graph] AutoSpatialGraph, AutoSpatioTemporalGraph
+ location_field: str | None = None
+
+
+class RawTemplateCfg(BaseModel):
+ """Template configuration - raw config loaded from YAML (multilingual)."""
+
+ language: str | List[str] = "en"
name: str
type: Literal[
- "model", "list", "set", "graph", "hypergraph",
- "temporal_graph", "spatial_graph", "spatio_temporal_graph"
+ "model",
+ "list",
+ "set",
+ "graph",
+ "hypergraph",
+ "temporal_graph",
+ "spatial_graph",
+ "spatio_temporal_graph",
]
- tags: Optional[List[str]] = None
- description: Optional[Union[str, Dict[str, str]]] = None
- output: Optional[OutputDefinition] = None
- model_schema: Optional[ModelSchema] = Field(None, alias="schema")
- item_schema: Optional[ItemSchema] = None
- node_schema: Optional[NodeSchema] = None
- edge_schema: Optional[EdgeSchema] = None
- guideline: Optional[Guideline] = None
- identifiers: Optional[Identifiers] = None
- options: Optional[Options] = None
- display: Optional[Display] = None
-
- model_config = ConfigDict(populate_by_name=True)
-
- @field_validator("description", mode="before")
- @classmethod
- def parse_description(cls, v):
- """Parse description field."""
- return v
-
- def model_dump(self, **kwargs):
- """Override model_dump to handle alias."""
- data = super().model_dump(**kwargs)
- if "schema" in data and data["schema"] is None:
- data["schema"] = data.pop("model_schema", None)
- return data
+ tags: List[str]
+ description: str | Dict[str, str]
+ output: NaiveOutputSchema | GraphOutputSchema
+ guideline: GuidelineSchema
+ identifiers: IdentifiersSchema | None = None
+ options: OptionsSchema | None = None
+ display: DisplaySchema
+
+
+class TemplateCfg(BaseModel):
+ """Single-language version of Template configuration."""
+
+ language: str | List[str] = "en"
+ name: str
+ type: Literal[
+ "model",
+ "list",
+ "set",
+ "graph",
+ "hypergraph",
+ "temporal_graph",
+ "spatial_graph",
+ "spatio_temporal_graph",
+ ]
+ tags: List[str]
+ description: str | Dict[str, str]
+ output: NaiveOutputSchema | GraphOutputSchema
+ guideline: GuidelineSchema
+ identifiers: IdentifiersSchema | None = None
+ options: OptionsSchema | None = None
+ display: DisplaySchema
+
+def extract_specific_language_config(
+ multi: RawTemplateCfg, language: str
+) -> TemplateCfg:
+ """Extract specific language config from multilingual config."""
+ lang = multi.language
+ if isinstance(lang, list):
+ lang = lang[0] if lang else language
+ elif lang is None:
+ lang = language
+
+ return TemplateCfg(
+ language=lang,
+ name=multi.name,
+ type=multi.type,
+ tags=multi.tags or [],
+ )
class ConfigLoader:
@@ -70,18 +199,20 @@ class ConfigLoader:
return yaml.safe_load(f) or {}
@staticmethod
- def load_config(file_path: Union[str, Path]) -> TemplateConfig:
+ def load_config(file_path: Union[str, Path]) -> RawTemplateCfg:
"""Load and validate configuration."""
data = ConfigLoader.load_yaml(file_path)
- return TemplateConfig(**data)
+ return RawTemplateCfg(**data)
@staticmethod
- def load_config_from_dict(data: Dict[str, Any]) -> TemplateConfig:
+ def load_config_from_dict(data: Dict[str, Any]) -> RawTemplateCfg:
"""Load configuration from dict."""
- return TemplateConfig(**data)
+ return RawTemplateCfg(**data)
@staticmethod
- def get_text_value(value: Union[str, Dict[str, str], None], language: str = "zh") -> Optional[str]:
+ def get_text_value(
+ value: Union[str, Dict[str, str], None], language: str = "zh"
+ ) -> str | None:
"""Get multilingual text value."""
if value is None:
return None
@@ -95,14 +226,32 @@ class ConfigLoader:
def validate_type(type_value: str) -> bool:
"""Validate if type is valid."""
valid_types = {
- "model", "list", "set", "graph", "hypergraph",
- "temporal_graph", "spatial_graph", "spatio_temporal_graph"
+ "model",
+ "list",
+ "set",
+ "graph",
+ "hypergraph",
+ "temporal_graph",
+ "spatial_graph",
+ "spatio_temporal_graph",
}
return type_value in valid_types
__all__ = [
- "Display",
- "TemplateConfig",
+ "parse_multi_lang",
+ "GuidelineSchema",
+ "GuidelineSchemaMono",
+ "FieldSchema",
+ "FieldsDefinition",
+ "OutputSchema",
+ "OutputSchemaMono",
+ "OptionsSchema",
+ "DisplaySchema",
+ "DisplaySchemaMono",
+ "IdentifiersSchema",
+ "TemplateSchema",
+ "RawTemplateCfg",
+ "TemplateCfg",
"ConfigLoader",
]
diff --git a/hyperextract/utils/template_engine/builder/options.py b/hyperextract/utils/template_engine/builder/options.py
index f5f802b..830a195 100644
--- a/hyperextract/utils/template_engine/builder/options.py
+++ b/hyperextract/utils/template_engine/builder/options.py
@@ -1,11 +1,11 @@
"""Options models and builder."""
-from typing import Any, Callable, List, Optional, Type
-from pydantic import BaseModel, Field
+from typing import Callable, List, Optional, Type
+from pydantic import BaseModel
from ontomem.merger import MergeStrategy
-from .schema import SchemaBuilder
+from .output import SchemaBuilder
from .identifiers import IdentifierResolver
diff --git a/hyperextract/utils/template_engine/builder/output.py b/hyperextract/utils/template_engine/builder/output.py
new file mode 100644
index 0000000..0c448ad
--- /dev/null
+++ b/hyperextract/utils/template_engine/builder/output.py
@@ -0,0 +1,233 @@
+"""Output schema parser."""
+
+from typing import Any, List, Type, Union, Tuple
+from pydantic import Field, create_model
+
+
+TYPE_MAPPING = {
+ "string": str,
+ "int": int,
+ "float": float,
+ "bool": bool,
+ "array": list,
+}
+
+
+def _get_text(value, language: str) -> str:
+ """Get multilingual text value."""
+ if value is None:
+ return ""
+ elif isinstance(value, str):
+ return value
+ elif isinstance(value, list):
+ return "\n".join(f"{i + 1}. {item}" for i, item in enumerate(value))
+ else:
+ dict_value = value.get(language) if isinstance(value, dict) else None
+ if isinstance(dict_value, str):
+ return dict_value
+ if isinstance(dict_value, list):
+ return "\n".join(f"{i + 1}. {item}" for i, item in enumerate(dict_value))
+ return ""
+
+
+def build_field_type(field) -> Type:
+ """Build field type from field (supports dict or object)."""
+ if isinstance(field, dict):
+ field_type_str = field.get("type", "string")
+ else:
+ field_type_str = field.type
+
+ if field_type_str == "array":
+ if isinstance(field, dict):
+ item_type_str = field.get("item_type")
+ else:
+ item_type_str = field.item_type
+ if item_type_str:
+ item_type = TYPE_MAPPING.get(item_type_str, str)
+ return List[item_type]
+ return List
+ return TYPE_MAPPING.get(field_type_str, str)
+
+
+def build_field_default(field) -> Any:
+ """Build field default value (supports dict or object)."""
+ if isinstance(field, dict):
+ is_required = field.get("required", False)
+ default_val = field.get("default")
+ field_type = field.get("type", "string")
+ else:
+ is_required = field.required
+ default_val = field.default
+ field_type = field.type
+
+ if is_required:
+ return ...
+ if default_val is not None:
+ return default_val
+ if field_type == "array":
+ return []
+ return None
+
+
+def build_field_description(field, language: str) -> str:
+ """Build field description with multilingual support (supports dict or object)."""
+ if isinstance(field, dict):
+ description = field.get("description")
+ else:
+ description = field.description
+
+ if description is None:
+ return ""
+ return _get_text(description, language)
+
+
+def _build_schema_from_fields(
+ fields: List,
+ schema_name: str,
+ language: str
+) -> Type:
+ """Build Pydantic schema from field list."""
+ schema_fields = {}
+ for field in fields:
+ if isinstance(field, dict):
+ field_name = field.get("name")
+ else:
+ field_name = field.name
+
+ if not field_name:
+ continue
+
+ field_type = build_field_type(field)
+ default = build_field_default(field)
+ description = build_field_description(field, language)
+ schema_fields[field_name] = (field_type, Field(default=default, description=description))
+ return create_model(schema_name, **schema_fields)
+
+
+def _get_schema_fields(schema):
+ """Get fields from schema, handling both dict and object."""
+ if schema is None:
+ return None
+ if isinstance(schema, dict):
+ return schema.get("fields")
+ if hasattr(schema, "fields"):
+ return schema.fields
+ return None
+
+
+def _get_output_value(output, attr: str):
+ """Get value from output, handling both dict and object."""
+ if output is None:
+ return None
+ if isinstance(output, dict):
+ return output.get(attr)
+ return getattr(output, attr, None)
+
+
+def _build_non_graph_schema(config, language: str) -> Type:
+ """Build schema for non-graph types (model, list, set)."""
+ autotype = config.type
+
+ output = config.output
+ output_fields = _get_output_value(output, "fields") if output else None
+ if output_fields:
+ schema_name = f"{config.name}Schema"
+ return _build_schema_from_fields(output_fields, schema_name, language)
+
+ if autotype == "model":
+ model_schema = getattr(config, "model_schema", None)
+ if model_schema:
+ fields = _get_schema_fields(model_schema)
+ if fields:
+ schema_name = f"{getattr(model_schema, '__name__', config.name)}Schema"
+ return _build_schema_from_fields(fields, schema_name, language)
+
+ if autotype in ("list", "set"):
+ item_schema = getattr(config, "item_schema", None)
+ if item_schema:
+ fields = _get_schema_fields(item_schema)
+ if fields:
+ schema_name = f"{getattr(item_schema, '__name__', config.name)}Schema"
+ return _build_schema_from_fields(fields, schema_name, language)
+
+ raise ValueError(f"No schema definition found for {autotype} type: {config.name}")
+
+
+def _build_graph_schemas(config, language: str) -> Tuple[Type, Type]:
+ """Build schemas for graph types."""
+ output = config.output
+ entities = _get_output_value(output, "entities")
+ relations = _get_output_value(output, "relations")
+
+ if entities and relations:
+ entity_fields = _get_output_value(entities, "fields")
+ relation_fields = _get_output_value(relations, "fields")
+ if entity_fields and relation_fields:
+ entity_schema = _build_schema_from_fields(
+ entity_fields,
+ f"{config.name}Entity",
+ language
+ )
+ relation_schema = _build_schema_from_fields(
+ relation_fields,
+ f"{config.name}Relation",
+ language
+ )
+ return entity_schema, relation_schema
+
+ node_schema = getattr(config, "node_schema", None)
+ edge_schema = getattr(config, "edge_schema", None)
+ if node_schema and edge_schema:
+ entity_schema = _build_schema_from_fields(
+ node_schema.fields if hasattr(node_schema, 'fields') else node_schema.get('fields'),
+ f"{getattr(node_schema, '__name__', node_schema.get('__name__', 'Node'))}Entity",
+ language
+ )
+ relation_schema = _build_schema_from_fields(
+ edge_schema.fields if hasattr(edge_schema, 'fields') else edge_schema.get('fields'),
+ f"{getattr(edge_schema, '__name__', edge_schema.get('__name__', 'Edge'))}Relation",
+ language
+ )
+ return entity_schema, relation_schema
+
+ raise ValueError(f"No schema definition found for graph type: {config.name}")
+
+
+def SchemaParser(
+ config,
+ language: str = "zh"
+) -> Union[Type, Tuple[Type, Type]]:
+ """Parse template and return schemas based on autotype.
+
+ Args:
+ config: Template Configuration instance (single-language)
+ language: Language code (default: "zh")
+
+ Returns:
+ - For non-graph types (model, list, set): returns data_schema
+ - For graph types: returns (entity_schema, relation_schema)
+ """
+ autotype = config.type
+
+ if autotype in ("model", "list", "set"):
+ return _build_non_graph_schema(config, language)
+ else:
+ return _build_graph_schemas(config, language)
+
+
+class SchemaBuilder:
+ """Schema Builder - kept for backward compatibility."""
+
+ @staticmethod
+ def build_schema(config):
+ """Deprecated: Use SchemaParser instead."""
+ return SchemaParser(config, "zh")
+
+
+__all__ = [
+ "SchemaParser",
+ "SchemaBuilder",
+ "build_field_type",
+ "build_field_default",
+ "build_field_description",
+]
diff --git a/hyperextract/utils/template_engine/builder/schema.py b/hyperextract/utils/template_engine/builder/schema.py
deleted file mode 100644
index 1f7cbea..0000000
--- a/hyperextract/utils/template_engine/builder/schema.py
+++ /dev/null
@@ -1,235 +0,0 @@
-"""Schema models and builder."""
-
-from typing import Any, Dict, List, Literal, Optional, Union, Type
-from pydantic import BaseModel, Field, field_validator, create_model
-
-
-class FieldDescription(BaseModel):
- """Field description with multilingual support."""
-
- zh: Optional[str] = None
- en: Optional[str] = None
-
- def get(self, language: str = "zh") -> str:
- """Get description for specified language."""
- if isinstance(self, str):
- return self
- return getattr(self, language) or self.zh or ""
-
-
-class SchemaField(BaseModel):
- """Schema field definition."""
-
- name: str
- type: Literal["string", "int", "float", "bool", "array"]
- description: Optional[Union[str, Dict[str, str]]] = None
- required: bool = False
- default: Optional[Any] = None
- item_type: Optional[str] = None
-
- @field_validator("description", mode="before")
- @classmethod
- def parse_description(cls, v):
- """Parse description field, supports plain string or multilingual dict."""
- return v
-
-
-class FieldsDefinition(BaseModel):
- """Fields definition container with description."""
-
- description: Optional[Union[str, Dict[str, str]]] = None
- fields: List[SchemaField] = Field(default_factory=list)
-
-
-class OutputDefinition(BaseModel):
- """Output definition - unified data structure definition."""
-
- description: Optional[Union[str, Dict[str, str]]] = None
- fields: Optional[List[SchemaField]] = None
- entities: Optional[FieldsDefinition] = None
- relations: Optional[FieldsDefinition] = None
-
-
-class BaseSchema(BaseModel):
- """Base schema configuration."""
-
- fields: List[SchemaField] = Field(default_factory=list)
-
-
-class ModelSchema(BaseSchema):
- """Model type schema."""
- pass
-
-
-class ItemSchema(BaseSchema):
- """List/Set type item schema."""
- pass
-
-
-class NodeSchema(BaseSchema):
- """Graph type node schema."""
- pass
-
-
-class EdgeSchema(BaseSchema):
- """Graph type edge schema."""
- pass
-
-
-TYPE_MAPPING = {
- "string": str,
- "int": int,
- "float": float,
- "bool": bool,
- "array": List,
-}
-
-
-class SchemaBuilder:
- """Schema Builder - Dynamically generates Pydantic Schema."""
-
- @staticmethod
- def build_field_type(field: SchemaField) -> Type:
- """Build field type."""
- if field.type == "array":
- if field.item_type:
- item_type = TYPE_MAPPING.get(field.item_type, str)
- return List[item_type]
- return List
- return TYPE_MAPPING.get(field.type, str)
-
- @staticmethod
- def build_field_default(field: SchemaField) -> Any:
- """Build field default value."""
- if field.required:
- return ...
- if field.default is not None:
- return field.default
- if field.type == "array":
- return []
- return None
-
- @staticmethod
- def build_field_description(field: SchemaField) -> str:
- """Build field description."""
- if field.description is None:
- return ""
- if isinstance(field.description, str):
- return field.description
- if isinstance(field.description, dict):
- return field.description.get("zh") or field.description.get("en") or ""
- return str(field.description)
-
- @staticmethod
- def build_schema_description(description: Optional[Union[str, Dict[str, str]]]) -> str:
- """Build schema-level description."""
- if description is None:
- return ""
- if isinstance(description, str):
- return description
- if isinstance(description, dict):
- return description.get("zh") or description.get("en") or ""
- return str(description)
-
- @classmethod
- def _build_schema_from_fields(cls, fields: List[SchemaField]) -> Type[BaseModel]:
- """Build Pydantic schema from field list."""
- schema_fields = {}
- for field in fields:
- field_type = cls.build_field_type(field)
- default = cls.build_field_default(field)
- description = cls.build_field_description(field)
- schema_fields[field.name] = (field_type, Field(default=default, description=description))
- return create_model("GeneratedSchema", **schema_fields)
-
- @classmethod
- def build_model_schema(cls, schema: ModelSchema) -> Type[BaseModel]:
- """Build Model type Schema."""
- return cls._build_schema_from_fields(schema.fields)
-
- @classmethod
- def build_item_schema(cls, schema: ItemSchema) -> Type[BaseModel]:
- """Build List/Set type Item Schema."""
- return cls._build_schema_from_fields(schema.fields)
-
- @classmethod
- def build_node_schema(cls, schema: NodeSchema) -> Type[BaseModel]:
- """Build Graph type Node Schema."""
- return cls._build_schema_from_fields(schema.fields)
-
- @classmethod
- def build_edge_schema(cls, schema: EdgeSchema) -> Type[BaseModel]:
- """Build Graph type Edge Schema."""
- return cls._build_schema_from_fields(schema.fields)
-
- @classmethod
- def build_output_schema(cls, output: OutputDefinition) -> Type[BaseModel]:
- """Build schema from OutputDefinition (for non-graph types)."""
- if output.fields is None:
- raise ValueError("OutputDefinition must have fields for non-graph types")
- model = cls._build_schema_from_fields(output.fields)
- if output.description:
- model.__doc__ = cls.build_schema_description(output.description)
- return model
-
- @classmethod
- def build_entity_schema(cls, entities: FieldsDefinition) -> Type[BaseModel]:
- """Build entity schema from FieldsDefinition."""
- model = cls._build_schema_from_fields(entities.fields)
- if entities.description:
- model.__doc__ = cls.build_schema_description(entities.description)
- return model
-
- @classmethod
- def build_relation_schema(cls, relations: FieldsDefinition) -> Type[BaseModel]:
- """Build relation schema from FieldsDefinition."""
- model = cls._build_schema_from_fields(relations.fields)
- if relations.description:
- model.__doc__ = cls.build_schema_description(relations.description)
- return model
-
- @classmethod
- def build_schema(cls, config) -> Dict[str, Type[BaseModel]]:
- """Build corresponding Schema based on configuration."""
- result = {}
-
- if config.output is not None:
- autotype = config.type
-
- if autotype in ("model", "list", "set"):
- if config.output.fields:
- result["schema"] = cls.build_output_schema(config.output)
-
- elif autotype in ("graph", "hypergraph", "temporal_graph", "spatial_graph", "spatio_temporal_graph"):
- if config.output.entities:
- result["entity_schema"] = cls.build_entity_schema(config.output.entities)
- if config.output.relations:
- result["relation_schema"] = cls.build_relation_schema(config.output.relations)
-
- if config.model_schema is not None:
- result["schema"] = cls.build_model_schema(config.model_schema)
-
- if config.item_schema is not None:
- result["item_schema"] = cls.build_item_schema(config.item_schema)
-
- if config.node_schema is not None:
- result["entity_schema"] = cls.build_node_schema(config.node_schema)
-
- if config.edge_schema is not None:
- result["relation_schema"] = cls.build_edge_schema(config.edge_schema)
-
- return result
-
-
-__all__ = [
- "FieldDescription",
- "SchemaField",
- "FieldsDefinition",
- "OutputDefinition",
- "BaseSchema",
- "ModelSchema",
- "ItemSchema",
- "NodeSchema",
- "EdgeSchema",
- "SchemaBuilder",
-]
diff --git a/hyperextract/utils/template_engine/builder/tools.py b/hyperextract/utils/template_engine/builder/tools.py
deleted file mode 100644
index 37a225a..0000000
--- a/hyperextract/utils/template_engine/builder/tools.py
+++ /dev/null
@@ -1,172 +0,0 @@
-"""Data model summary - convenient for code hints and intuitive viewing"""
-
-from typing import List, Optional, Union, Dict, Any
-from pydantic import BaseModel
-
-
-class GuidelineSchema(BaseModel):
- """Extraction guideline configuration"""
-
- # [all] AutoModel, AutoList, AutoSet, AutoGraph, AutoHypergraph, AutoTemporalGraph, AutoSpatialGraph, AutoSpatioTemporalGraph
- target: Optional[Union[str, List[str], Dict[str, Union[str, List[str]]]]] = None
-
- # [non-graph] AutoModel, AutoList, AutoSet
- rules: Optional[Union[str, List[str], Dict[str, Union[str, List[str]]]]] = None
-
- # [graph] AutoGraph, AutoHypergraph, AutoTemporalGraph, AutoSpatialGraph, AutoSpatioTemporalGraph
- rules_for_entities: Optional[
- Union[str, List[str], Dict[str, Union[str, List[str]]]]
- ] = None
-
- # [graph] AutoGraph, AutoHypergraph, AutoTemporalGraph, AutoSpatialGraph, AutoSpatioTemporalGraph
- rules_for_relations: Optional[
- Union[str, List[str], Dict[str, Union[str, List[str]]]]
- ] = None
-
- # [temporal] AutoTemporalGraph, AutoSpatioTemporalGraph
- rules_for_time: Optional[
- Union[str, List[str], Dict[str, Union[str, List[str]]]]
- ] = None
-
- # [spatial] AutoSpatialGraph, AutoSpatioTemporalGraph
- rules_for_location: Optional[
- Union[str, List[str], Dict[str, Union[str, List[str]]]]
- ] = None
-
-
-class FieldSchema(BaseModel):
- """Field definition"""
-
- # [all]
- name: str
- type: str
- description: Optional[str] = None
- required: bool = False
- default: Optional[Any] = None
-
-
-class OutputSchema(BaseModel):
- """Output definition"""
-
- description: Optional[str] = None
-
- # [non-graph] AutoModel, AutoList, AutoSet
- fields: Optional[List[FieldSchema]] = None
-
- # [graph] AutoGraph, AutoHypergraph, AutoTemporalGraph, AutoSpatialGraph, AutoSpatioTemporalGraph
- entities: Optional[List[FieldSchema]] = None
-
- # [graph] AutoGraph, AutoHypergraph, AutoTemporalGraph, AutoSpatialGraph, AutoSpatioTemporalGraph
- relations: Optional[List[FieldSchema]] = None
-
-
-class OptionsSchema(BaseModel):
- """Runtime options"""
-
- # [non-graph] AutoModel, AutoSet
- merge_strategy: Optional[str] = None
-
- # [graph] AutoGraph, AutoHypergraph, AutoTemporalGraph, AutoSpatialGraph, AutoSpatioTemporalGraph
- entity_merge_strategy: Optional[str] = None
-
- # [graph] AutoGraph, AutoHypergraph, AutoTemporalGraph, AutoSpatialGraph, AutoSpatioTemporalGraph
- relation_merge_strategy: Optional[str] = None
-
- extraction_mode: Optional[str] = None
-
- # [temporal] AutoTemporalGraph, AutoSpatioTemporalGraph
- observation_time: Optional[str] = None
-
- # [spatial] AutoSpatialGraph, AutoSpatioTemporalGraph
- observation_location: Optional[str] = None
-
- # [all]
- chunk_size: int = 2048
- chunk_overlap: int = 256
- max_workers: int = 10
- verbose: bool = False
- fields_for_search: Optional[List[str]] = None
- entity_fields_for_search: Optional[List[str]] = None
- relation_fields_for_search: Optional[List[str]] = None
-
-
-class DisplaySchema(BaseModel):
- """Display configuration"""
-
- # [non-graph] AutoModel, AutoList, AutoSet
- label: Optional[str] = None
-
- # [graph] AutoGraph, AutoHypergraph, AutoTemporalGraph, AutoSpatialGraph, AutoSpatioTemporalGraph
- entity_label: Optional[str] = None
-
- # [graph] AutoGraph, AutoHypergraph, AutoTemporalGraph, AutoSpatialGraph, AutoSpatioTemporalGraph
- relation_label: Optional[str] = None
-
-
-class IdentifiersSchema(BaseModel):
- """Identifier configuration"""
-
- # [non-graph] AutoSet
- item_id: Optional[str] = None
-
- # [graph] AutoGraph, AutoHypergraph, AutoTemporalGraph, AutoSpatialGraph, AutoSpatioTemporalGraph
- entity_id: Optional[str] = None
-
- # [graph] AutoGraph, AutoHypergraph, AutoTemporalGraph, AutoSpatialGraph, AutoSpatioTemporalGraph
- relation_id: Optional[str] = None
-
- # [graph] AutoGraph, AutoHypergraph, AutoTemporalGraph, AutoSpatialGraph, AutoSpatioTemporalGraph
- relation_members: Optional[Dict[str, str]] = None
-
- # [temporal] AutoTemporalGraph, AutoSpatioTemporalGraph
- time_field: Optional[str] = None
-
- # [spatial] AutoSpatialGraph, AutoSpatioTemporalGraph
- location_field: Optional[str] = None
-
-
-class TemplateSchema(BaseModel):
- """Template configuration - corresponds to YAML configuration file"""
-
- language: str | List[str] = "en"
- name: str
- type: str
- tags: List[str]
- description: Optional[str] = None
- output: Optional[OutputSchema] = None
- guideline: Optional[GuidelineSchema] = None
- identifiers: Optional[IdentifiersSchema] = None
- options: Optional[OptionsSchema] = None
- display: Optional[DisplaySchema] = None
-
-
-def get_text(
- value: Optional[Union[str, List[str], Dict[str, Union[str, List[str]]]]],
- language: Optional[str] = None,
-) -> Optional[str]:
- """Get multilingual text value, supports list format."""
- if value is None:
- return None
- elif isinstance(value, str):
- return value
- elif isinstance(value, list):
- return "\n".join(f"{i + 1}. {item}" for i, item in enumerate(value))
- else:
- # multi-language dict
- dict_value = value.get(language)
- if isinstance(dict_value, str):
- return dict_value
- if isinstance(dict_value, list):
- return "\n".join(f"{i + 1}. {item}" for i, item in enumerate(dict_value))
-
-
-__all__ = [
- "get_text",
- "GuidelineSchema",
- "FieldSchema",
- "OutputSchema",
- "OptionsSchema",
- "DisplaySchema",
- "IdentifiersSchema",
- "TemplateSchema",
-]
diff --git a/hyperextract/utils/template_engine/factory.py b/hyperextract/utils/template_engine/factory.py
index 6dba7e9..052a394 100644
--- a/hyperextract/utils/template_engine/factory.py
+++ b/hyperextract/utils/template_engine/factory.py
@@ -9,11 +9,13 @@ from langchain_core.language_models import BaseChatModel
from langchain_core.embeddings import Embeddings
from .builder import (
- TemplateConfig,
+ RawTemplateCfg,
OptionsBuilder,
Options,
+ SchemaParser,
+ IdentifierResolver,
)
-from .builder.prompt import PromptParser
+from .builder.guideline import PromptParser
if TYPE_CHECKING:
@@ -31,12 +33,12 @@ if TYPE_CHECKING:
class TemplateFactory:
@staticmethod
- def get_language(config: TemplateConfig) -> str:
+ def get_language(config: RawTemplateCfg) -> str:
"""Get language setting."""
return OptionsBuilder.get_language(config)
@staticmethod
- def resolve_display_extractors(config: TemplateConfig, schema_dict: dict):
+ def resolve_display_extractors(config: RawTemplateCfg, schema_dict: dict = None):
"""Generate label_extractor function based on display configuration."""
display = config.display
if not display:
@@ -76,29 +78,29 @@ class TemplateFactory:
@classmethod
def create_model(
- cls, config: TemplateConfig, llm_client: BaseChatModel, embedder: Embeddings
+ cls, config: RawTemplateCfg, llm_client: BaseChatModel, embedder: Embeddings
) -> "AutoModel":
"""Create AutoModel template."""
from hyperextract.types import AutoModel
language = cls.get_language(config)
- schema_dict = OptionsBuilder.validate_model_config(config)
- schema_class = schema_dict.get("schema")
+ data_schema = SchemaParser(config, language)
- prompt, _, _ = PromptParser(config, language)
+ config_mono = config.for_language(language)
+ prompt, _, _ = PromptParser(config_mono, language)
options = OptionsBuilder.build_model_options(config, language)
return AutoModel(
- data_schema=schema_class,
+ data_schema=data_schema,
llm_client=llm_client,
embedder=embedder,
strategy_or_merger=OptionsBuilder.resolve_merge_strategy(
options.merge_strategy,
lambda x: str(x),
llm_client,
- schema_class,
+ data_schema,
),
prompt=prompt,
chunk_size=options.chunk_size,
@@ -109,22 +111,22 @@ class TemplateFactory:
@classmethod
def create_list(
- cls, config: TemplateConfig, llm_client: BaseChatModel, embedder: Embeddings
+ cls, config: RawTemplateCfg, llm_client: BaseChatModel, embedder: Embeddings
) -> "AutoList":
"""Create AutoList template."""
from hyperextract.types import AutoList
language = cls.get_language(config)
- schema_dict = OptionsBuilder.validate_list_config(config)
- item_schema_class = schema_dict.get("item_schema")
+ data_schema = SchemaParser(config, language)
- prompt, _, _ = PromptParser(config, language)
+ config_mono = config.for_language(language)
+ prompt, _, _ = PromptParser(config_mono, language)
options = OptionsBuilder.build_list_options(config)
return AutoList(
- item_schema=item_schema_class,
+ item_schema=data_schema,
llm_client=llm_client,
embedder=embedder,
prompt=prompt,
@@ -136,23 +138,24 @@ class TemplateFactory:
@classmethod
def create_set(
- cls, config: TemplateConfig, llm_client: BaseChatModel, embedder: Embeddings
+ cls, config: RawTemplateCfg, llm_client: BaseChatModel, embedder: Embeddings
) -> "AutoSet":
"""Create AutoSet template."""
from hyperextract.types import AutoSet
language = cls.get_language(config)
- schema_dict, identifiers = OptionsBuilder.validate_set_config(config)
- item_schema_class = schema_dict.get("item_schema")
+ data_schema = SchemaParser(config, language)
+ identifiers = IdentifierResolver.resolve_all(config)
item_id_extractor = identifiers.get("item_id_extractor")
- prompt, _, _ = PromptParser(config, language)
+ config_mono = config.for_language(language)
+ prompt, _, _ = PromptParser(config_mono, language)
options = OptionsBuilder.build_set_options(config, language)
return AutoSet(
- item_schema=item_schema_class,
+ item_schema=data_schema,
item_key_extractor=item_id_extractor,
llm_client=llm_client,
embedder=embedder,
@@ -160,7 +163,7 @@ class TemplateFactory:
options.merge_strategy,
item_id_extractor,
llm_client,
- item_schema_class,
+ data_schema,
),
prompt=prompt,
chunk_size=options.chunk_size,
@@ -171,27 +174,27 @@ class TemplateFactory:
@classmethod
def create_graph(
- cls, config: TemplateConfig, llm_client: BaseChatModel, embedder: Embeddings
+ cls, config: RawTemplateCfg, llm_client: BaseChatModel, embedder: Embeddings
) -> "AutoGraph":
"""Create AutoGraph template."""
from hyperextract.types import AutoGraph
language = cls.get_language(config)
- schema_dict, identifiers = OptionsBuilder.validate_graph_config(config)
- entity_schema_class = schema_dict.get("entity_schema")
- relation_schema_class = schema_dict.get("relation_schema")
+ entity_schema, relation_schema = SchemaParser(config, language)
+ identifiers = IdentifierResolver.resolve_all(config)
entity_key_extractor = identifiers.get("entity_key_extractor")
relation_key_extractor = identifiers.get("relation_key_extractor")
entities_in_relation_extractor = identifiers.get("entities_in_relation_extractor")
options = OptionsBuilder.build_graph_options(config, language)
- prompt, prompt_for_entity_extraction, prompt_for_relation_extraction = PromptParser(config, language)
+ config_mono = config.for_language(language)
+ prompt, prompt_for_entity_extraction, prompt_for_relation_extraction = PromptParser(config_mono, language)
return AutoGraph(
- entity_schema=entity_schema_class,
- relation_schema=relation_schema_class,
+ entity_schema=entity_schema,
+ relation_schema=relation_schema,
entity_key_extractor=entity_key_extractor,
relation_key_extractor=relation_key_extractor,
entities_in_relation_extractor=entities_in_relation_extractor,
@@ -202,13 +205,13 @@ class TemplateFactory:
options.entity_merge_strategy,
entity_key_extractor,
llm_client,
- entity_schema_class,
+ entity_schema,
),
relation_strategy_or_merger=OptionsBuilder.resolve_merge_strategy(
options.relation_merge_strategy,
relation_key_extractor,
llm_client,
- relation_schema_class,
+ relation_schema,
),
prompt=prompt,
prompt_for_entity_extraction=prompt_for_entity_extraction,
@@ -223,23 +226,20 @@ class TemplateFactory:
@classmethod
def create_hypergraph(
- cls, config: TemplateConfig, llm_client: BaseChatModel, embedder: Embeddings
+ cls, config: RawTemplateCfg, llm_client: BaseChatModel, embedder: Embeddings
) -> "AutoHypergraph":
return cls.create_graph(config, llm_client, embedder)
@classmethod
def create_temporal_graph(
- cls, config: TemplateConfig, llm_client: BaseChatModel, embedder: Embeddings
+ cls, config: RawTemplateCfg, llm_client: BaseChatModel, embedder: Embeddings
) -> "AutoTemporalGraph":
from hyperextract.types import AutoTemporalGraph
language = cls.get_language(config)
- schema_dict, identifiers = OptionsBuilder.validate_temporal_graph_config(
- config
- )
- entity_schema_class = schema_dict.get("entity_schema")
- relation_schema_class = schema_dict.get("relation_schema")
+ entity_schema, relation_schema = SchemaParser(config, language)
+ identifiers = IdentifierResolver.resolve_all(config)
entity_key_extractor = identifiers.get("entity_key_extractor")
relation_key_extractor = identifiers.get("relation_key_extractor")
entities_in_relation_extractor = identifiers.get("entities_in_relation_extractor")
@@ -247,11 +247,12 @@ class TemplateFactory:
options = OptionsBuilder.build_temporal_graph_options(config, language)
- prompt, prompt_for_entity_extraction, prompt_for_relation_extraction = PromptParser(config, language)
+ config_mono = config.for_language(language)
+ prompt, prompt_for_entity_extraction, prompt_for_relation_extraction = PromptParser(config_mono, language)
return AutoTemporalGraph(
- node_schema=entity_schema_class,
- edge_schema=relation_schema_class,
+ node_schema=entity_schema,
+ edge_schema=relation_schema,
node_key_extractor=entity_key_extractor,
edge_key_extractor=relation_key_extractor,
nodes_in_edge_extractor=entities_in_relation_extractor,
@@ -264,13 +265,13 @@ class TemplateFactory:
options.entity_merge_strategy,
entity_key_extractor,
llm_client,
- entity_schema_class,
+ entity_schema,
),
edge_strategy_or_merger=OptionsBuilder.resolve_merge_strategy(
options.relation_merge_strategy,
relation_key_extractor,
llm_client,
- relation_schema_class,
+ relation_schema,
),
prompt=prompt,
prompt_for_node_extraction=prompt_for_entity_extraction,
@@ -285,17 +286,14 @@ class TemplateFactory:
@classmethod
def create_spatial_graph(
- cls, config: TemplateConfig, llm_client: BaseChatModel, embedder: Embeddings
+ cls, config: RawTemplateCfg, llm_client: BaseChatModel, embedder: Embeddings
) -> "AutoSpatialGraph":
from hyperextract.types import AutoSpatialGraph
language = cls.get_language(config)
- schema_dict, identifiers = OptionsBuilder.validate_spatial_graph_config(
- config
- )
- entity_schema_class = schema_dict.get("entity_schema")
- relation_schema_class = schema_dict.get("relation_schema")
+ entity_schema, relation_schema = SchemaParser(config, language)
+ identifiers = IdentifierResolver.resolve_all(config)
entity_key_extractor = identifiers.get("entity_key_extractor")
relation_key_extractor = identifiers.get("relation_key_extractor")
entities_in_relation_extractor = identifiers.get("entities_in_relation_extractor")
@@ -303,11 +301,12 @@ class TemplateFactory:
options = OptionsBuilder.build_spatial_graph_options(config, language)
- prompt, prompt_for_entity_extraction, prompt_for_relation_extraction = PromptParser(config, language)
+ config_mono = config.for_language(language)
+ prompt, prompt_for_entity_extraction, prompt_for_relation_extraction = PromptParser(config_mono, language)
return AutoSpatialGraph(
- entity_schema=entity_schema_class,
- relation_schema=relation_schema_class,
+ entity_schema=entity_schema,
+ relation_schema=relation_schema,
entity_key_extractor=entity_key_extractor,
relation_key_extractor=relation_key_extractor,
entities_in_relation_extractor=entities_in_relation_extractor,
@@ -320,13 +319,13 @@ class TemplateFactory:
options.entity_merge_strategy,
entity_key_extractor,
llm_client,
- entity_schema_class,
+ entity_schema,
),
relation_strategy_or_merger=OptionsBuilder.resolve_merge_strategy(
options.relation_merge_strategy,
relation_key_extractor,
llm_client,
- relation_schema_class,
+ relation_schema,
),
prompt=prompt,
prompt_for_entity_extraction=prompt_for_entity_extraction,
@@ -341,17 +340,14 @@ class TemplateFactory:
@classmethod
def create_spatio_temporal_graph(
- cls, config: TemplateConfig, llm_client: BaseChatModel, embedder: Embeddings
+ cls, config: RawTemplateCfg, llm_client: BaseChatModel, embedder: Embeddings
) -> "AutoSpatioTemporalGraph":
from hyperextract.types import AutoSpatioTemporalGraph
language = cls.get_language(config)
- schema_dict, identifiers = (
- OptionsBuilder.validate_spatio_temporal_graph_config(config)
- )
- entity_schema_class = schema_dict.get("entity_schema")
- relation_schema_class = schema_dict.get("relation_schema")
+ entity_schema, relation_schema = SchemaParser(config, language)
+ identifiers = IdentifierResolver.resolve_all(config)
entity_key_extractor = identifiers.get("entity_key_extractor")
relation_key_extractor = identifiers.get("relation_key_extractor")
entities_in_relation_extractor = identifiers.get("entities_in_relation_extractor")
@@ -362,11 +358,12 @@ class TemplateFactory:
config, language
)
- prompt, prompt_for_entity_extraction, prompt_for_relation_extraction = PromptParser(config, language)
+ config_mono = config.for_language(language)
+ prompt, prompt_for_entity_extraction, prompt_for_relation_extraction = PromptParser(config_mono, language)
return AutoSpatioTemporalGraph(
- entity_schema=entity_schema_class,
- relation_schema=relation_schema_class,
+ entity_schema=entity_schema,
+ relation_schema=relation_schema,
entity_key_extractor=entity_key_extractor,
relation_key_extractor=relation_key_extractor,
entities_in_relation_extractor=entities_in_relation_extractor,
@@ -381,13 +378,13 @@ class TemplateFactory:
options.entity_merge_strategy,
entity_key_extractor,
llm_client,
- entity_schema_class,
+ entity_schema,
),
relation_strategy_or_merger=OptionsBuilder.resolve_merge_strategy(
options.relation_merge_strategy,
relation_key_extractor,
llm_client,
- relation_schema_class,
+ relation_schema,
),
prompt=prompt,
prompt_for_entity_extraction=prompt_for_entity_extraction,
@@ -403,7 +400,7 @@ class TemplateFactory:
@classmethod
def create(
cls,
- config: TemplateConfig,
+ config: RawTemplateCfg,
llm_client: BaseChatModel,
embedder: Embeddings,
**kwargs,
diff --git a/hyperextract/utils/template_engine/gallery.py b/hyperextract/utils/template_engine/gallery.py
index 80141ea..d5e0bca 100644
--- a/hyperextract/utils/template_engine/gallery.py
+++ b/hyperextract/utils/template_engine/gallery.py
@@ -7,7 +7,7 @@ import os
from pathlib import Path
from typing import Dict, List, Optional
-from .builder import TemplateConfig, ConfigLoader
+from .builder import RawTemplateCfg, ConfigLoader
class Gallery:
@@ -32,12 +32,12 @@ class Gallery:
_instance: Optional["Gallery"] = None
def __init__(self):
- self._configs: Dict[str, TemplateConfig] = {}
+ self._configs: Dict[str, RawTemplateCfg] = {}
self._configs_by_type: Dict[str, List[str]] = {}
self._configs_by_tag: Dict[str, List[str]] = {}
@classmethod
- def get(cls, name: str) -> Optional[TemplateConfig]:
+ def get(cls, name: str) -> Optional[RawTemplateCfg]:
return cls._instance._configs.get(name) if cls._instance else None
@classmethod
@@ -65,21 +65,21 @@ class Gallery:
cls._instance._load_config(path)
@classmethod
- def get_by_type(cls, type_value: str) -> List[TemplateConfig]:
+ def get_by_type(cls, type_value: str) -> List[RawTemplateCfg]:
if not cls._instance:
return []
names = cls._instance._configs_by_type.get(type_value, [])
return [cls._instance._configs[name] for name in names if name in cls._instance._configs]
@classmethod
- def get_by_tag(cls, tag: str) -> List[TemplateConfig]:
+ def get_by_tag(cls, tag: str) -> List[RawTemplateCfg]:
if not cls._instance:
return []
names = cls._instance._configs_by_tag.get(tag, [])
return [cls._instance._configs[name] for name in names if name in cls._instance._configs]
@classmethod
- def search(cls, query: str = None, type_value: str = None, tag: str = None, language: str = None) -> List[TemplateConfig]:
+ def search(cls, query: str = None, type_value: str = None, tag: str = None, language: str = None) -> List[RawTemplateCfg]:
"""Search templates by query, type, tag, or language.
Args:
@@ -89,7 +89,7 @@ class Gallery:
language: Filter by language (e.g., "zh", "en")
Returns:
- List of matching TemplateConfig
+ List of matching RawTemplateCfg
"""
if not cls._instance:
return []
@@ -215,7 +215,7 @@ class Gallery:
return TemplateFactory.create(config, llm_client, embedder, **kwargs)
@classmethod
- def _resolve_config(cls, name: str, lang: str = "zh") -> TemplateConfig:
+ def _resolve_config(cls, name: str, lang: str = "zh") -> RawTemplateCfg:
"""Resolve template name to config object."""
config = cls.get(name)
if config is not None:
@@ -247,7 +247,7 @@ class Gallery:
except Exception as e:
print(f"Failed to load config {file_path}: {e}")
- def _register(self, config: TemplateConfig) -> None:
+ def _register(self, config: RawTemplateCfg) -> None:
self._configs[config.name] = config
if config.type not in self._configs_by_type:
diff --git a/hyperextract/utils/template_engine/template.py b/hyperextract/utils/template_engine/template.py
index 7d15bb0..0e02dcd 100644
--- a/hyperextract/utils/template_engine/template.py
+++ b/hyperextract/utils/template_engine/template.py
@@ -9,7 +9,7 @@ from langchain_core.language_models import BaseChatModel
from langchain_core.embeddings import Embeddings
from .gallery import Gallery
-from .builder import TemplateConfig
+from .builder import RawTemplateCfg
class Template:
@@ -50,7 +50,7 @@ class Template:
type_value: Optional[str] = None,
tag: Optional[str] = None,
language: Optional[str] = None,
- ) -> List[TemplateConfig]:
+ ) -> List[RawTemplateCfg]:
"""Search templates by various criteria.
Args:
@@ -60,7 +60,7 @@ class Template:
language: Filter by language (e.g., "zh", "en")
Returns:
- List of matching TemplateConfig objects
+ List of matching RawTemplateCfg objects
"""
return Gallery.search(
query=query,
@@ -70,7 +70,7 @@ class Template:
)
@staticmethod
- def get(name: str, lang: str = "zh") -> Optional[TemplateConfig]:
+ def get(name: str, lang: str = "zh") -> Optional[RawTemplateCfg]:
"""Get template configuration by name.
Args:
@@ -78,7 +78,7 @@ class Template:
lang: Language, default "zh"
Returns:
- TemplateConfig or None if not found
+ RawTemplateCfg or None if not found
"""
return Gallery.get(name)