Hypergraph is more powerful. Transform unstructured text into structured knowledge with LLMs. Graphs, hypergraphs, and spatio-temporal extractions — with one command.
refactor(template_engine): 简化模板引擎的多语言处理逻辑
yifanfeng972657ada7adc3
diff --git a/hyperextract/utils/template_engine/__init__.py b/hyperextract/utils/template_engine/__init__.pyindex 62f1e85..a8f765a 100644--- a/hyperextract/utils/template_engine/__init__.py+++ b/hyperextract/utils/template_engine/__init__.py@@ -14,15 +14,10 @@ Main Components:from .template import Templatefrom .gallery import Gallery-from .factory import TemplateFactory, TemplateWrapper+from .factory import TemplateFactoryfrom .parsers import (TemplateCfg,- OutputParser,- GuidelineParser,- PromptParser,- IdentifierResolver,- OptionsBuilder,- _localize_data,+ load_template,localize_template,)@@ -31,13 +26,7 @@ __all__ = ["Template","Gallery","TemplateFactory",- "TemplateWrapper","TemplateCfg",- "OutputParser",- "GuidelineParser",- "PromptParser",- "IdentifierResolver",- "OptionsBuilder",- "_localize_data",+ "load_template","localize_template",]diff --git a/hyperextract/utils/template_engine/factory.py b/hyperextract/utils/template_engine/factory.pyindex 12c9ac9..a253fd6 100644--- a/hyperextract/utils/template_engine/factory.py+++ b/hyperextract/utils/template_engine/factory.py@@ -14,7 +14,8 @@ from .parsers import (Options,OutputParser,IdentifierResolver,- PromptParser,+ GuidelineParser,+ localize_template,)@@ -32,11 +33,6 @@ if TYPE_CHECKING:class TemplateFactory:- @staticmethod- def get_language(config: TemplateCfg) -> str:- """Get language setting."""- return OptionsBuilder.get_language(config)-@staticmethoddef resolve_display_extractors(config: TemplateCfg, schema_dict: dict = None):"""Generate label_extractor function based on display configuration."""@@ -83,14 +79,11 @@ class TemplateFactory:"""Create AutoModel template."""from hyperextract.types import AutoModel- language = cls.get_language(config)-- data_schema = OutputParser(config, language)+ data_schema = OutputParser(config)- config_mono = config.for_language(language)- prompt, _, _ = PromptParser(config_mono, language)+ prompt, _, _ = GuidelineParser(config.guideline, config.type)- options = OptionsBuilder.build_model_options(config, language)+ options = OptionsBuilder.build_model_options(config)return AutoModel(data_schema=data_schema,@@ -116,12 +109,9 @@ class TemplateFactory:"""Create AutoList template."""from hyperextract.types import AutoList- language = cls.get_language(config)+ data_schema = OutputParser(config)- data_schema = OutputParser(config, language)-- config_mono = config.for_language(language)- prompt, _, _ = PromptParser(config_mono, language)+ prompt, _, _ = GuidelineParser(config.guideline, config.type)options = OptionsBuilder.build_list_options(config)@@ -143,16 +133,13 @@ class TemplateFactory:"""Create AutoSet template."""from hyperextract.types import AutoSet- language = cls.get_language(config)-- data_schema = OutputParser(config, language)+ data_schema = OutputParser(config)identifiers = IdentifierResolver.resolve_all(config)item_id_extractor = identifiers.get("item_id_extractor")- config_mono = config.for_language(language)- prompt, _, _ = PromptParser(config_mono, language)+ prompt, _, _ = GuidelineParser(config.guideline, config.type)- options = OptionsBuilder.build_set_options(config, language)+ options = OptionsBuilder.build_set_options(config)return AutoSet(item_schema=data_schema,@@ -179,9 +166,7 @@ class TemplateFactory:"""Create AutoGraph template."""from hyperextract.types import AutoGraph- language = cls.get_language(config)-- entity_schema, relation_schema = OutputParser(config, language)+ entity_schema, relation_schema = OutputParser(config)identifiers = IdentifierResolver.resolve_all(config)entity_key_extractor = identifiers.get("entity_key_extractor")relation_key_extractor = identifiers.get("relation_key_extractor")@@ -189,11 +174,10 @@ class TemplateFactory:"entities_in_relation_extractor")- options = OptionsBuilder.build_graph_options(config, language)+ options = OptionsBuilder.build_graph_options(config)- config_mono = config.for_language(language)prompt, prompt_for_entity_extraction, prompt_for_relation_extraction = (- PromptParser(config_mono, language)+ GuidelineParser(config.guideline, config.type))return AutoGraph(@@ -240,9 +224,7 @@ class TemplateFactory:) -> "AutoTemporalGraph":from hyperextract.types import AutoTemporalGraph- language = cls.get_language(config)-- entity_schema, relation_schema = OutputParser(config, language)+ entity_schema, relation_schema = OutputParser(config)identifiers = IdentifierResolver.resolve_all(config)entity_key_extractor = identifiers.get("entity_key_extractor")relation_key_extractor = identifiers.get("relation_key_extractor")@@ -251,11 +233,10 @@ class TemplateFactory:)time_in_relation_extractor = identifiers.get("time_in_relation_extractor")- options = OptionsBuilder.build_temporal_graph_options(config, language)+ options = OptionsBuilder.build_temporal_graph_options(config)- config_mono = config.for_language(language)prompt, prompt_for_entity_extraction, prompt_for_relation_extraction = (- PromptParser(config_mono, language)+ GuidelineParser(config.guideline, config.type))return AutoTemporalGraph(@@ -298,9 +279,7 @@ class TemplateFactory:) -> "AutoSpatialGraph":from hyperextract.types import AutoSpatialGraph- language = cls.get_language(config)-- entity_schema, relation_schema = OutputParser(config, language)+ entity_schema, relation_schema = OutputParser(config)identifiers = IdentifierResolver.resolve_all(config)entity_key_extractor = identifiers.get("entity_key_extractor")relation_key_extractor = identifiers.get("relation_key_extractor")@@ -311,11 +290,10 @@ class TemplateFactory:"location_in_relation_extractor")- options = OptionsBuilder.build_spatial_graph_options(config, language)+ options = OptionsBuilder.build_spatial_graph_options(config)- config_mono = config.for_language(language)prompt, prompt_for_entity_extraction, prompt_for_relation_extraction = (- PromptParser(config_mono, language)+ GuidelineParser(config.guideline, config.type))return AutoSpatialGraph(@@ -358,9 +336,7 @@ class TemplateFactory:) -> "AutoSpatioTemporalGraph":from hyperextract.types import AutoSpatioTemporalGraph- language = cls.get_language(config)-- entity_schema, relation_schema = OutputParser(config, language)+ entity_schema, relation_schema = OutputParser(config)identifiers = IdentifierResolver.resolve_all(config)entity_key_extractor = identifiers.get("entity_key_extractor")relation_key_extractor = identifiers.get("relation_key_extractor")@@ -372,11 +348,10 @@ class TemplateFactory:"location_in_relation_extractor")- options = OptionsBuilder.build_spatio_temporal_graph_options(config, language)+ options = OptionsBuilder.build_spatio_temporal_graph_options(config)- config_mono = config.for_language(language)prompt, prompt_for_entity_extraction, prompt_for_relation_extraction = (- PromptParser(config_mono, language)+ GuidelineParser(config.guideline, config.type))return AutoSpatioTemporalGraph(@@ -421,6 +396,7 @@ class TemplateFactory:config: TemplateCfg,llm_client: BaseChatModel,embedder: Embeddings,+ language: str = "zh",**kwargs,):"""Create template instance based on configuration.@@ -429,6 +405,7 @@ class TemplateFactory:config: Template configurationllm_client: LLM clientembedder: Embedding model+ language: Language code for localization (e.g., 'zh', 'en')**kwargs: Additional parameters to override config parameterse.g., observation_time="2024-06-15", observation_location="Beijing"@@ -439,11 +416,15 @@ class TemplateFactory:# Basic usagetemplate = TemplateFactory.create(config, llm, embedder)+ # For specific language+ template = TemplateFactory.create(config, llm, embedder, language="en")+# For spatio-temporal templates, pass observation_time etc.template = TemplateFactory.create(config,llm,embedder,+ language="zh",observation_time="2024-06-15",observation_location="Beijing")@@ -453,15 +434,18 @@ class TemplateFactory:config,llm,embedder,+ language="zh",chunk_size=4096,max_workers=20)"""+ config_mono = localize_template(config, language)+if kwargs:- base_options = config.options or Options()+ base_options = config_mono.options or Options()options_dict = base_options.model_dump()options_dict.update(kwargs)- config.options = Options(**options_dict)+ config_mono.options = Options(**options_dict)autotype_map = {"model": cls.create_model,@@ -474,14 +458,14 @@ class TemplateFactory:"spatio_temporal_graph": cls.create_spatio_temporal_graph,}- creator = autotype_map.get(config.type)+ creator = autotype_map.get(config_mono.type)if creator is None:- raise ValueError(f"Unsupported type: {config.type}")+ raise ValueError(f"Unsupported type: {config_mono.type}")- template = creator(config, llm_client, embedder)- display_extractors = cls.resolve_display_extractors(config, {})+ template = creator(config_mono, llm_client, embedder)+ display_extractors = cls.resolve_display_extractors(config_mono, {})- return TemplateWrapper(template, display_extractors, config.type)+ return TemplateWrapper(template, display_extractors, config_mono.type)def safe_getattr(obj, name, default):diff --git a/hyperextract/utils/template_engine/parsers/__init__.py b/hyperextract/utils/template_engine/parsers/__init__.pyindex 20f1bc1..1482c79 100644--- a/hyperextract/utils/template_engine/parsers/__init__.py+++ b/hyperextract/utils/template_engine/parsers/__init__.py@@ -2,12 +2,11 @@from .loader import (TemplateCfg,- ConfigLoader,- _localize_data,+ load_template,localize_template,)from .output import OutputParser-from .guideline import GuidelineParser, PromptParser+from .guideline import GuidelineParserfrom .identifiers import IdentifierResolverfrom .options import (Options,@@ -17,12 +16,10 @@ from .options import (__all__ = ["TemplateCfg",- "ConfigLoader",- "_localize_data",+ "load_template","localize_template","OutputParser","GuidelineParser",- "PromptParser","IdentifierResolver","Options","OptionsBuilder",diff --git a/hyperextract/utils/template_engine/parsers/guideline.py b/hyperextract/utils/template_engine/parsers/guideline.pyindex d6ff501..f10e5f4 100644--- a/hyperextract/utils/template_engine/parsers/guideline.py+++ b/hyperextract/utils/template_engine/parsers/guideline.py@@ -1,7 +1,6 @@"""Extraction prompt parser from guideline."""from typing import Tuple-from .loader import NaiveGuidelineSchema, GraphGuidelineSchemaLABEL_MAPPING = {@@ -28,37 +27,28 @@ LABEL_MAPPING = {}-def GuidelineParser(- template: NaiveGuidelineSchema | GraphGuidelineSchema, language: str = "en"-) -> Tuple[str, str, str]:- """Parse guideline and return prompts based on autotype.+def GuidelineParser(guideline, autotype: str = "model") -> Tuple[str, str, str]:+ """Parse guideline and return prompts based on autotype (config is already localized).Args:- template: Guideline Configuration instance (mono-language version)- language: Language code (default: "en")+ guideline: Guideline instance (single-language, all fields are strings)+ autotype: Template type (default: "model")Returns:tuple: main_prompt or (main_prompt, node_prompt, edge_prompt)- For non-graph types: returns main_prompt- For graph types in two_stage mode: returns (main_prompt, node_prompt, edge_prompt)"""- labels = LABEL_MAPPING.get(language, LABEL_MAPPING["en"])- guideline: NaiveGuidelineSchema | GraphGuidelineSchema = template+ labels = LABEL_MAPPING.get("zh", LABEL_MAPPING["en"])- prefix_prompt = (- f"# {labels.get('role_and_task')}:\n{guideline.target}"- )+ prefix_prompt = f"# {labels.get('role_and_task')}:\n{guideline.target}"- autotype = getattr(template, 'type', None) or "model"-if autotype in ("model", "list", "set"):parts = [prefix_prompt]if guideline.rules:- parts.append(- f"## {labels.get('rules')}:\n{guideline.rules}"- )+ parts.append(f"## {labels.get('rules')}:\n{guideline.rules}")parts.append(f"## {labels.get('source_text')}:\n{{source_text}}")- return "\n\n".join(parts)+ return "\n\n".join(parts), "", ""else:main_parts = [prefix_prompt]if guideline.rules_for_entities:@@ -109,10 +99,8 @@ def GuidelineParser(return "\n\n".join(main_parts), "\n\n".join(node_parts), "\n\n".join(edge_parts)-PromptParser = GuidelineParser__all__ = ["GuidelineParser",- "PromptParser",]diff --git a/hyperextract/utils/template_engine/parsers/options.py b/hyperextract/utils/template_engine/parsers/options.pyindex 1b0fefa..165cb45 100644--- a/hyperextract/utils/template_engine/parsers/options.py+++ b/hyperextract/utils/template_engine/parsers/options.py@@ -254,7 +254,7 @@ class OptionsBuilder:return MergeStrategy.LLM.BALANCED@classmethod- def build_model_options(cls, config, language: str = "zh"):+ def build_model_options(cls, config):"""Build options for AutoModel."""options = config.options or Options()return ModelOptions(@@ -277,7 +277,7 @@ class OptionsBuilder:)@classmethod- def build_set_options(cls, config, language: str = "zh"):+ def build_set_options(cls, config):"""Build options for AutoSet."""options = config.options or Options()return SetOptions(@@ -289,7 +289,7 @@ class OptionsBuilder:)@classmethod- def build_graph_options(cls, config, language: str = "zh"):+ def build_graph_options(cls, config):"""Build options for AutoGraph/AutoHypergraph."""options = config.options or Options()extraction_mode = cls.get_extraction_mode(config)@@ -306,7 +306,7 @@ class OptionsBuilder:)@classmethod- def build_temporal_graph_options(cls, config, language: str = "zh"):+ def build_temporal_graph_options(cls, config):"""Build options for AutoTemporalGraph."""options = config.options or Options()extraction_mode = cls.get_extraction_mode(config)@@ -324,7 +324,7 @@ class OptionsBuilder:)@classmethod- def build_spatial_graph_options(cls, config, language: str = "zh"):+ def build_spatial_graph_options(cls, config):"""Build options for AutoSpatialGraph."""options = config.options or Options()extraction_mode = cls.get_extraction_mode(config)@@ -342,7 +342,7 @@ class OptionsBuilder:)@classmethod- def build_spatio_temporal_graph_options(cls, config, language: str = "zh"):+ def build_spatio_temporal_graph_options(cls, config):"""Build options for AutoSpatioTemporalGraph."""options = config.options or Options()extraction_mode = cls.get_extraction_mode(config)diff --git a/hyperextract/utils/template_engine/parsers/output.py b/hyperextract/utils/template_engine/parsers/output.pyindex 51f5779..90048b1 100644--- a/hyperextract/utils/template_engine/parsers/output.py+++ b/hyperextract/utils/template_engine/parsers/output.py@@ -13,25 +13,8 @@ TYPE_MAPPING = {}-def _get_text(value, language: str) -> str:- """Get multilingual text value."""- if value is None:- return ""- elif isinstance(value, str):- return value- elif isinstance(value, list):- return "\n".join(f"{i + 1}. {item}" for i, item in enumerate(value))- else:- dict_value = value.get(language) if isinstance(value, dict) else None- if isinstance(dict_value, str):- return dict_value- if isinstance(dict_value, list):- return "\n".join(f"{i + 1}. {item}" for i, item in enumerate(dict_value))- return ""--def build_field_type(field) -> Type:- """Build field type from field (supports dict or object)."""+ """Build field type from field."""if isinstance(field, dict):field_type_str = field.get("type", "string")else:@@ -50,41 +33,34 @@ def build_field_type(field) -> Type:def build_field_default(field) -> Any:- """Build field default value (supports dict or object)."""+ """Build field default value."""if isinstance(field, dict):is_required = field.get("required", False)default_val = field.get("default")- field_type = field.get("type", "string")else:is_required = field.requireddefault_val = field.default- field_type = field.typeif is_required:return ...if default_val is not None:return default_val+ field_type = field.get("type", "string") if isinstance(field, dict) else field.typeif field_type == "list":return []return None-def build_field_description(field, language: str) -> str:- """Build field description with multilingual support (supports dict or object)."""+def build_field_description(field) -> str:+ """Get field description (config is already localized, just return string)."""if isinstance(field, dict):- description = field.get("description")- else:- description = field.description-- if description is None:- return ""- return _get_text(description, language)+ return field.get("description", "") or ""+ return field.description or ""def _build_schema_from_fields(fields: List,schema_name: str,- language: str) -> Type:"""Build Pydantic schema from field list."""schema_fields = {}@@ -99,13 +75,13 @@ def _build_schema_from_fields(field_type = build_field_type(field)default = build_field_default(field)- description = build_field_description(field, language)+ description = build_field_description(field)schema_fields[field_name] = (field_type, Field(default=default, description=description))return create_model(schema_name, **schema_fields)def _get_schema_fields(schema):- """Get fields from schema, handling both dict and object."""+ """Get fields from schema."""if schema is None:return Noneif isinstance(schema, dict):@@ -116,7 +92,7 @@ def _get_schema_fields(schema):def _get_output_value(output, attr: str):- """Get value from output, handling both dict and object."""+ """Get value from output."""if output is None:return Noneif isinstance(output, dict):@@ -124,7 +100,7 @@ def _get_output_value(output, attr: str):return getattr(output, attr, None)-def _build_non_graph_schema(config, language: str) -> Type:+def _build_non_graph_schema(config) -> Type:"""Build schema for non-graph types (model, list, set)."""autotype = config.type@@ -132,7 +108,7 @@ def _build_non_graph_schema(config, language: str) -> Type:output_fields = _get_output_value(output, "fields") if output else Noneif output_fields:schema_name = f"{config.name}Schema"- return _build_schema_from_fields(output_fields, schema_name, language)+ return _build_schema_from_fields(output_fields, schema_name)if autotype == "model":model_schema = getattr(config, "model_schema", None)@@ -140,7 +116,7 @@ def _build_non_graph_schema(config, language: str) -> Type:fields = _get_schema_fields(model_schema)if fields:schema_name = f"{getattr(model_schema, '__name__', config.name)}Schema"- return _build_schema_from_fields(fields, schema_name, language)+ return _build_schema_from_fields(fields, schema_name)if autotype in ("list", "set"):item_schema = getattr(config, "item_schema", None)@@ -148,12 +124,12 @@ def _build_non_graph_schema(config, language: str) -> Type:fields = _get_schema_fields(item_schema)if fields:schema_name = f"{getattr(item_schema, '__name__', config.name)}Schema"- return _build_schema_from_fields(fields, schema_name, language)+ return _build_schema_from_fields(fields, schema_name)raise ValueError(f"No schema definition found for {autotype} type: {config.name}")-def _build_graph_schemas(config, language: str) -> Tuple[Type, Type]:+def _build_graph_schemas(config) -> Tuple[Type, Type]:"""Build schemas for graph types."""output = config.outputentities = _get_output_value(output, "entities")@@ -166,12 +142,10 @@ def _build_graph_schemas(config, language: str) -> Tuple[Type, Type]:entity_schema = _build_schema_from_fields(entity_fields,f"{config.name}Entity",- language)relation_schema = _build_schema_from_fields(relation_fields,f"{config.name}Relation",- language)return entity_schema, relation_schema@@ -181,27 +155,21 @@ def _build_graph_schemas(config, language: str) -> Tuple[Type, Type]:entity_schema = _build_schema_from_fields(node_schema.fields if hasattr(node_schema, 'fields') else node_schema.get('fields'),f"{getattr(node_schema, '__name__', node_schema.get('__name__', 'Node'))}Entity",- language)relation_schema = _build_schema_from_fields(edge_schema.fields if hasattr(edge_schema, 'fields') else edge_schema.get('fields'),f"{getattr(edge_schema, '__name__', edge_schema.get('__name__', 'Edge'))}Relation",- language)return entity_schema, relation_schemaraise ValueError(f"No schema definition found for graph type: {config.name}")-def OutputParser(- config,- language: str = "zh"-) -> Union[Type, Tuple[Type, Type]]:- """Parse template and return schemas based on autotype.+def OutputParser(config) -> Union[Type, Tuple[Type, Type]]:+ """Parse template and return schemas based on autotype (config is already localized).Args:config: Template Configuration instance (single-language)- language: Language code (default: "zh")Returns:- For non-graph types (model, list, set): returns data_schema@@ -210,10 +178,9 @@ def OutputParser(autotype = config.typeif autotype in ("model", "list", "set"):- return _build_non_graph_schema(config, language)+ return _build_non_graph_schema(config)else:- return _build_graph_schemas(config, language)-+ return _build_graph_schemas(config)__all__ = [