Skip to content

Hypergraph is more powerful. Transform unstructured text into structured knowledge with LLMs. Graphs, hypergraphs, and spatio-temporal extractions — with one command.

refactor(template_engine): 简化模板引擎的多语言处理逻辑

yifanfeng972657ada7adc3
diff --git a/hyperextract/utils/template_engine/__init__.py b/hyperextract/utils/template_engine/__init__.py
index 62f1e85..a8f765a 100644
--- a/hyperextract/utils/template_engine/__init__.py
+++ b/hyperextract/utils/template_engine/__init__.py
@@ -14,15 +14,10 @@ Main Components:
from .template import Template
from .gallery import Gallery
-from .factory import TemplateFactory, TemplateWrapper
+from .factory import TemplateFactory
from .parsers import (
TemplateCfg,
- OutputParser,
- GuidelineParser,
- PromptParser,
- IdentifierResolver,
- OptionsBuilder,
- _localize_data,
+ load_template,
localize_template,
)
@@ -31,13 +26,7 @@ __all__ = [
"Template",
"Gallery",
"TemplateFactory",
- "TemplateWrapper",
"TemplateCfg",
- "OutputParser",
- "GuidelineParser",
- "PromptParser",
- "IdentifierResolver",
- "OptionsBuilder",
- "_localize_data",
+ "load_template",
"localize_template",
]
diff --git a/hyperextract/utils/template_engine/factory.py b/hyperextract/utils/template_engine/factory.py
index 12c9ac9..a253fd6 100644
--- a/hyperextract/utils/template_engine/factory.py
+++ b/hyperextract/utils/template_engine/factory.py
@@ -14,7 +14,8 @@ from .parsers import (
Options,
OutputParser,
IdentifierResolver,
- PromptParser,
+ GuidelineParser,
+ localize_template,
)
@@ -32,11 +33,6 @@ if TYPE_CHECKING:
class TemplateFactory:
- @staticmethod
- def get_language(config: TemplateCfg) -> str:
- """Get language setting."""
- return OptionsBuilder.get_language(config)
-
@staticmethod
def resolve_display_extractors(config: TemplateCfg, schema_dict: dict = None):
"""Generate label_extractor function based on display configuration."""
@@ -83,14 +79,11 @@ class TemplateFactory:
"""Create AutoModel template."""
from hyperextract.types import AutoModel
- language = cls.get_language(config)
-
- data_schema = OutputParser(config, language)
+ data_schema = OutputParser(config)
- config_mono = config.for_language(language)
- prompt, _, _ = PromptParser(config_mono, language)
+ prompt, _, _ = GuidelineParser(config.guideline, config.type)
- options = OptionsBuilder.build_model_options(config, language)
+ options = OptionsBuilder.build_model_options(config)
return AutoModel(
data_schema=data_schema,
@@ -116,12 +109,9 @@ class TemplateFactory:
"""Create AutoList template."""
from hyperextract.types import AutoList
- language = cls.get_language(config)
+ data_schema = OutputParser(config)
- data_schema = OutputParser(config, language)
-
- config_mono = config.for_language(language)
- prompt, _, _ = PromptParser(config_mono, language)
+ prompt, _, _ = GuidelineParser(config.guideline, config.type)
options = OptionsBuilder.build_list_options(config)
@@ -143,16 +133,13 @@ class TemplateFactory:
"""Create AutoSet template."""
from hyperextract.types import AutoSet
- language = cls.get_language(config)
-
- data_schema = OutputParser(config, language)
+ data_schema = OutputParser(config)
identifiers = IdentifierResolver.resolve_all(config)
item_id_extractor = identifiers.get("item_id_extractor")
- config_mono = config.for_language(language)
- prompt, _, _ = PromptParser(config_mono, language)
+ prompt, _, _ = GuidelineParser(config.guideline, config.type)
- options = OptionsBuilder.build_set_options(config, language)
+ options = OptionsBuilder.build_set_options(config)
return AutoSet(
item_schema=data_schema,
@@ -179,9 +166,7 @@ class TemplateFactory:
"""Create AutoGraph template."""
from hyperextract.types import AutoGraph
- language = cls.get_language(config)
-
- entity_schema, relation_schema = OutputParser(config, language)
+ entity_schema, relation_schema = OutputParser(config)
identifiers = IdentifierResolver.resolve_all(config)
entity_key_extractor = identifiers.get("entity_key_extractor")
relation_key_extractor = identifiers.get("relation_key_extractor")
@@ -189,11 +174,10 @@ class TemplateFactory:
"entities_in_relation_extractor"
)
- options = OptionsBuilder.build_graph_options(config, language)
+ options = OptionsBuilder.build_graph_options(config)
- config_mono = config.for_language(language)
prompt, prompt_for_entity_extraction, prompt_for_relation_extraction = (
- PromptParser(config_mono, language)
+ GuidelineParser(config.guideline, config.type)
)
return AutoGraph(
@@ -240,9 +224,7 @@ class TemplateFactory:
) -> "AutoTemporalGraph":
from hyperextract.types import AutoTemporalGraph
- language = cls.get_language(config)
-
- entity_schema, relation_schema = OutputParser(config, language)
+ entity_schema, relation_schema = OutputParser(config)
identifiers = IdentifierResolver.resolve_all(config)
entity_key_extractor = identifiers.get("entity_key_extractor")
relation_key_extractor = identifiers.get("relation_key_extractor")
@@ -251,11 +233,10 @@ class TemplateFactory:
)
time_in_relation_extractor = identifiers.get("time_in_relation_extractor")
- options = OptionsBuilder.build_temporal_graph_options(config, language)
+ options = OptionsBuilder.build_temporal_graph_options(config)
- config_mono = config.for_language(language)
prompt, prompt_for_entity_extraction, prompt_for_relation_extraction = (
- PromptParser(config_mono, language)
+ GuidelineParser(config.guideline, config.type)
)
return AutoTemporalGraph(
@@ -298,9 +279,7 @@ class TemplateFactory:
) -> "AutoSpatialGraph":
from hyperextract.types import AutoSpatialGraph
- language = cls.get_language(config)
-
- entity_schema, relation_schema = OutputParser(config, language)
+ entity_schema, relation_schema = OutputParser(config)
identifiers = IdentifierResolver.resolve_all(config)
entity_key_extractor = identifiers.get("entity_key_extractor")
relation_key_extractor = identifiers.get("relation_key_extractor")
@@ -311,11 +290,10 @@ class TemplateFactory:
"location_in_relation_extractor"
)
- options = OptionsBuilder.build_spatial_graph_options(config, language)
+ options = OptionsBuilder.build_spatial_graph_options(config)
- config_mono = config.for_language(language)
prompt, prompt_for_entity_extraction, prompt_for_relation_extraction = (
- PromptParser(config_mono, language)
+ GuidelineParser(config.guideline, config.type)
)
return AutoSpatialGraph(
@@ -358,9 +336,7 @@ class TemplateFactory:
) -> "AutoSpatioTemporalGraph":
from hyperextract.types import AutoSpatioTemporalGraph
- language = cls.get_language(config)
-
- entity_schema, relation_schema = OutputParser(config, language)
+ entity_schema, relation_schema = OutputParser(config)
identifiers = IdentifierResolver.resolve_all(config)
entity_key_extractor = identifiers.get("entity_key_extractor")
relation_key_extractor = identifiers.get("relation_key_extractor")
@@ -372,11 +348,10 @@ class TemplateFactory:
"location_in_relation_extractor"
)
- options = OptionsBuilder.build_spatio_temporal_graph_options(config, language)
+ options = OptionsBuilder.build_spatio_temporal_graph_options(config)
- config_mono = config.for_language(language)
prompt, prompt_for_entity_extraction, prompt_for_relation_extraction = (
- PromptParser(config_mono, language)
+ GuidelineParser(config.guideline, config.type)
)
return AutoSpatioTemporalGraph(
@@ -421,6 +396,7 @@ class TemplateFactory:
config: TemplateCfg,
llm_client: BaseChatModel,
embedder: Embeddings,
+ language: str = "zh",
**kwargs,
):
"""Create template instance based on configuration.
@@ -429,6 +405,7 @@ class TemplateFactory:
config: Template configuration
llm_client: LLM client
embedder: Embedding model
+ language: Language code for localization (e.g., 'zh', 'en')
**kwargs: Additional parameters to override config parameters
e.g., observation_time="2024-06-15", observation_location="Beijing"
@@ -439,11 +416,15 @@ class TemplateFactory:
# Basic usage
template = TemplateFactory.create(config, llm, embedder)
+ # For specific language
+ template = TemplateFactory.create(config, llm, embedder, language="en")
+
# For spatio-temporal templates, pass observation_time etc.
template = TemplateFactory.create(
config,
llm,
embedder,
+ language="zh",
observation_time="2024-06-15",
observation_location="Beijing"
)
@@ -453,15 +434,18 @@ class TemplateFactory:
config,
llm,
embedder,
+ language="zh",
chunk_size=4096,
max_workers=20
)
"""
+ config_mono = localize_template(config, language)
+
if kwargs:
- base_options = config.options or Options()
+ base_options = config_mono.options or Options()
options_dict = base_options.model_dump()
options_dict.update(kwargs)
- config.options = Options(**options_dict)
+ config_mono.options = Options(**options_dict)
autotype_map = {
"model": cls.create_model,
@@ -474,14 +458,14 @@ class TemplateFactory:
"spatio_temporal_graph": cls.create_spatio_temporal_graph,
}
- creator = autotype_map.get(config.type)
+ creator = autotype_map.get(config_mono.type)
if creator is None:
- raise ValueError(f"Unsupported type: {config.type}")
+ raise ValueError(f"Unsupported type: {config_mono.type}")
- template = creator(config, llm_client, embedder)
- display_extractors = cls.resolve_display_extractors(config, {})
+ template = creator(config_mono, llm_client, embedder)
+ display_extractors = cls.resolve_display_extractors(config_mono, {})
- return TemplateWrapper(template, display_extractors, config.type)
+ return TemplateWrapper(template, display_extractors, config_mono.type)
def safe_getattr(obj, name, default):
diff --git a/hyperextract/utils/template_engine/parsers/__init__.py b/hyperextract/utils/template_engine/parsers/__init__.py
index 20f1bc1..1482c79 100644
--- a/hyperextract/utils/template_engine/parsers/__init__.py
+++ b/hyperextract/utils/template_engine/parsers/__init__.py
@@ -2,12 +2,11 @@
from .loader import (
TemplateCfg,
- ConfigLoader,
- _localize_data,
+ load_template,
localize_template,
)
from .output import OutputParser
-from .guideline import GuidelineParser, PromptParser
+from .guideline import GuidelineParser
from .identifiers import IdentifierResolver
from .options import (
Options,
@@ -17,12 +16,10 @@ from .options import (
__all__ = [
"TemplateCfg",
- "ConfigLoader",
- "_localize_data",
+ "load_template",
"localize_template",
"OutputParser",
"GuidelineParser",
- "PromptParser",
"IdentifierResolver",
"Options",
"OptionsBuilder",
diff --git a/hyperextract/utils/template_engine/parsers/guideline.py b/hyperextract/utils/template_engine/parsers/guideline.py
index d6ff501..f10e5f4 100644
--- a/hyperextract/utils/template_engine/parsers/guideline.py
+++ b/hyperextract/utils/template_engine/parsers/guideline.py
@@ -1,7 +1,6 @@
"""Extraction prompt parser from guideline."""
from typing import Tuple
-from .loader import NaiveGuidelineSchema, GraphGuidelineSchema
LABEL_MAPPING = {
@@ -28,37 +27,28 @@ LABEL_MAPPING = {
}
-def GuidelineParser(
- template: NaiveGuidelineSchema | GraphGuidelineSchema, language: str = "en"
-) -> Tuple[str, str, str]:
- """Parse guideline and return prompts based on autotype.
+def GuidelineParser(guideline, autotype: str = "model") -> Tuple[str, str, str]:
+ """Parse guideline and return prompts based on autotype (config is already localized).
Args:
- template: Guideline Configuration instance (mono-language version)
- language: Language code (default: "en")
+ guideline: Guideline instance (single-language, all fields are strings)
+ autotype: Template type (default: "model")
Returns:
tuple: main_prompt or (main_prompt, node_prompt, edge_prompt)
- For non-graph types: returns main_prompt
- For graph types in two_stage mode: returns (main_prompt, node_prompt, edge_prompt)
"""
- labels = LABEL_MAPPING.get(language, LABEL_MAPPING["en"])
- guideline: NaiveGuidelineSchema | GraphGuidelineSchema = template
+ labels = LABEL_MAPPING.get("zh", LABEL_MAPPING["en"])
- prefix_prompt = (
- f"# {labels.get('role_and_task')}:\n{guideline.target}"
- )
+ prefix_prompt = f"# {labels.get('role_and_task')}:\n{guideline.target}"
- autotype = getattr(template, 'type', None) or "model"
-
if autotype in ("model", "list", "set"):
parts = [prefix_prompt]
if guideline.rules:
- parts.append(
- f"## {labels.get('rules')}:\n{guideline.rules}"
- )
+ parts.append(f"## {labels.get('rules')}:\n{guideline.rules}")
parts.append(f"## {labels.get('source_text')}:\n{{source_text}}")
- return "\n\n".join(parts)
+ return "\n\n".join(parts), "", ""
else:
main_parts = [prefix_prompt]
if guideline.rules_for_entities:
@@ -109,10 +99,8 @@ def GuidelineParser(
return "\n\n".join(main_parts), "\n\n".join(node_parts), "\n\n".join(edge_parts)
-PromptParser = GuidelineParser
__all__ = [
"GuidelineParser",
- "PromptParser",
]
diff --git a/hyperextract/utils/template_engine/parsers/options.py b/hyperextract/utils/template_engine/parsers/options.py
index 1b0fefa..165cb45 100644
--- a/hyperextract/utils/template_engine/parsers/options.py
+++ b/hyperextract/utils/template_engine/parsers/options.py
@@ -254,7 +254,7 @@ class OptionsBuilder:
return MergeStrategy.LLM.BALANCED
@classmethod
- def build_model_options(cls, config, language: str = "zh"):
+ def build_model_options(cls, config):
"""Build options for AutoModel."""
options = config.options or Options()
return ModelOptions(
@@ -277,7 +277,7 @@ class OptionsBuilder:
)
@classmethod
- def build_set_options(cls, config, language: str = "zh"):
+ def build_set_options(cls, config):
"""Build options for AutoSet."""
options = config.options or Options()
return SetOptions(
@@ -289,7 +289,7 @@ class OptionsBuilder:
)
@classmethod
- def build_graph_options(cls, config, language: str = "zh"):
+ def build_graph_options(cls, config):
"""Build options for AutoGraph/AutoHypergraph."""
options = config.options or Options()
extraction_mode = cls.get_extraction_mode(config)
@@ -306,7 +306,7 @@ class OptionsBuilder:
)
@classmethod
- def build_temporal_graph_options(cls, config, language: str = "zh"):
+ def build_temporal_graph_options(cls, config):
"""Build options for AutoTemporalGraph."""
options = config.options or Options()
extraction_mode = cls.get_extraction_mode(config)
@@ -324,7 +324,7 @@ class OptionsBuilder:
)
@classmethod
- def build_spatial_graph_options(cls, config, language: str = "zh"):
+ def build_spatial_graph_options(cls, config):
"""Build options for AutoSpatialGraph."""
options = config.options or Options()
extraction_mode = cls.get_extraction_mode(config)
@@ -342,7 +342,7 @@ class OptionsBuilder:
)
@classmethod
- def build_spatio_temporal_graph_options(cls, config, language: str = "zh"):
+ def build_spatio_temporal_graph_options(cls, config):
"""Build options for AutoSpatioTemporalGraph."""
options = config.options or Options()
extraction_mode = cls.get_extraction_mode(config)
diff --git a/hyperextract/utils/template_engine/parsers/output.py b/hyperextract/utils/template_engine/parsers/output.py
index 51f5779..90048b1 100644
--- a/hyperextract/utils/template_engine/parsers/output.py
+++ b/hyperextract/utils/template_engine/parsers/output.py
@@ -13,25 +13,8 @@ TYPE_MAPPING = {
}
-def _get_text(value, language: str) -> str:
- """Get multilingual text value."""
- if value is None:
- return ""
- elif isinstance(value, str):
- return value
- elif isinstance(value, list):
- return "\n".join(f"{i + 1}. {item}" for i, item in enumerate(value))
- else:
- dict_value = value.get(language) if isinstance(value, dict) else None
- if isinstance(dict_value, str):
- return dict_value
- if isinstance(dict_value, list):
- return "\n".join(f"{i + 1}. {item}" for i, item in enumerate(dict_value))
- return ""
-
-
def build_field_type(field) -> Type:
- """Build field type from field (supports dict or object)."""
+ """Build field type from field."""
if isinstance(field, dict):
field_type_str = field.get("type", "string")
else:
@@ -50,41 +33,34 @@ def build_field_type(field) -> Type:
def build_field_default(field) -> Any:
- """Build field default value (supports dict or object)."""
+ """Build field default value."""
if isinstance(field, dict):
is_required = field.get("required", False)
default_val = field.get("default")
- field_type = field.get("type", "string")
else:
is_required = field.required
default_val = field.default
- field_type = field.type
if is_required:
return ...
if default_val is not None:
return default_val
+ field_type = field.get("type", "string") if isinstance(field, dict) else field.type
if field_type == "list":
return []
return None
-def build_field_description(field, language: str) -> str:
- """Build field description with multilingual support (supports dict or object)."""
+def build_field_description(field) -> str:
+ """Get field description (config is already localized, just return string)."""
if isinstance(field, dict):
- description = field.get("description")
- else:
- description = field.description
-
- if description is None:
- return ""
- return _get_text(description, language)
+ return field.get("description", "") or ""
+ return field.description or ""
def _build_schema_from_fields(
fields: List,
schema_name: str,
- language: str
) -> Type:
"""Build Pydantic schema from field list."""
schema_fields = {}
@@ -99,13 +75,13 @@ def _build_schema_from_fields(
field_type = build_field_type(field)
default = build_field_default(field)
- description = build_field_description(field, language)
+ description = build_field_description(field)
schema_fields[field_name] = (field_type, Field(default=default, description=description))
return create_model(schema_name, **schema_fields)
def _get_schema_fields(schema):
- """Get fields from schema, handling both dict and object."""
+ """Get fields from schema."""
if schema is None:
return None
if isinstance(schema, dict):
@@ -116,7 +92,7 @@ def _get_schema_fields(schema):
def _get_output_value(output, attr: str):
- """Get value from output, handling both dict and object."""
+ """Get value from output."""
if output is None:
return None
if isinstance(output, dict):
@@ -124,7 +100,7 @@ def _get_output_value(output, attr: str):
return getattr(output, attr, None)
-def _build_non_graph_schema(config, language: str) -> Type:
+def _build_non_graph_schema(config) -> Type:
"""Build schema for non-graph types (model, list, set)."""
autotype = config.type
@@ -132,7 +108,7 @@ def _build_non_graph_schema(config, language: str) -> Type:
output_fields = _get_output_value(output, "fields") if output else None
if output_fields:
schema_name = f"{config.name}Schema"
- return _build_schema_from_fields(output_fields, schema_name, language)
+ return _build_schema_from_fields(output_fields, schema_name)
if autotype == "model":
model_schema = getattr(config, "model_schema", None)
@@ -140,7 +116,7 @@ def _build_non_graph_schema(config, language: str) -> Type:
fields = _get_schema_fields(model_schema)
if fields:
schema_name = f"{getattr(model_schema, '__name__', config.name)}Schema"
- return _build_schema_from_fields(fields, schema_name, language)
+ return _build_schema_from_fields(fields, schema_name)
if autotype in ("list", "set"):
item_schema = getattr(config, "item_schema", None)
@@ -148,12 +124,12 @@ def _build_non_graph_schema(config, language: str) -> Type:
fields = _get_schema_fields(item_schema)
if fields:
schema_name = f"{getattr(item_schema, '__name__', config.name)}Schema"
- return _build_schema_from_fields(fields, schema_name, language)
+ return _build_schema_from_fields(fields, schema_name)
raise ValueError(f"No schema definition found for {autotype} type: {config.name}")
-def _build_graph_schemas(config, language: str) -> Tuple[Type, Type]:
+def _build_graph_schemas(config) -> Tuple[Type, Type]:
"""Build schemas for graph types."""
output = config.output
entities = _get_output_value(output, "entities")
@@ -166,12 +142,10 @@ def _build_graph_schemas(config, language: str) -> Tuple[Type, Type]:
entity_schema = _build_schema_from_fields(
entity_fields,
f"{config.name}Entity",
- language
)
relation_schema = _build_schema_from_fields(
relation_fields,
f"{config.name}Relation",
- language
)
return entity_schema, relation_schema
@@ -181,27 +155,21 @@ def _build_graph_schemas(config, language: str) -> Tuple[Type, Type]:
entity_schema = _build_schema_from_fields(
node_schema.fields if hasattr(node_schema, 'fields') else node_schema.get('fields'),
f"{getattr(node_schema, '__name__', node_schema.get('__name__', 'Node'))}Entity",
- language
)
relation_schema = _build_schema_from_fields(
edge_schema.fields if hasattr(edge_schema, 'fields') else edge_schema.get('fields'),
f"{getattr(edge_schema, '__name__', edge_schema.get('__name__', 'Edge'))}Relation",
- language
)
return entity_schema, relation_schema
raise ValueError(f"No schema definition found for graph type: {config.name}")
-def OutputParser(
- config,
- language: str = "zh"
-) -> Union[Type, Tuple[Type, Type]]:
- """Parse template and return schemas based on autotype.
+def OutputParser(config) -> Union[Type, Tuple[Type, Type]]:
+ """Parse template and return schemas based on autotype (config is already localized).
Args:
config: Template Configuration instance (single-language)
- language: Language code (default: "zh")
Returns:
- For non-graph types (model, list, set): returns data_schema
@@ -210,10 +178,9 @@ def OutputParser(
autotype = config.type
if autotype in ("model", "list", "set"):
- return _build_non_graph_schema(config, language)
+ return _build_non_graph_schema(config)
else:
- return _build_graph_schemas(config, language)
-
+ return _build_graph_schemas(config)
__all__ = [