Hypergraph is more powerful. Transform unstructured text into structured knowledge with LLMs. Graphs, hypergraphs, and spatio-temporal extractions — with one command.
refactor(template_engine): 重构解析器函数命名和输出模式处理
yifanfeng974732381907c8
diff --git a/hyperextract/utils/template_engine/factory.py b/hyperextract/utils/template_engine/factory.pyindex a253fd6..b636d6c 100644--- a/hyperextract/utils/template_engine/factory.py+++ b/hyperextract/utils/template_engine/factory.py@@ -12,9 +12,9 @@ from .parsers import (TemplateCfg,OptionsBuilder,Options,- OutputParser,+ parse_output,IdentifierResolver,- GuidelineParser,+ parse_guideline,localize_template,)@@ -79,9 +79,9 @@ class TemplateFactory:"""Create AutoModel template."""from hyperextract.types import AutoModel- data_schema = OutputParser(config)+ data_schema = parse_output(config)- prompt, _, _ = GuidelineParser(config.guideline, config.type)+ prompt, _, _ = parse_guideline(config.guideline, config.type)options = OptionsBuilder.build_model_options(config)@@ -109,9 +109,9 @@ class TemplateFactory:"""Create AutoList template."""from hyperextract.types import AutoList- data_schema = OutputParser(config)+ data_schema = parse_output(config)- prompt, _, _ = GuidelineParser(config.guideline, config.type)+ prompt, _, _ = parse_guideline(config.guideline, config.type)options = OptionsBuilder.build_list_options(config)@@ -133,11 +133,11 @@ class TemplateFactory:"""Create AutoSet template."""from hyperextract.types import AutoSet- data_schema = OutputParser(config)+ data_schema = parse_output(config)identifiers = IdentifierResolver.resolve_all(config)item_id_extractor = identifiers.get("item_id_extractor")- prompt, _, _ = GuidelineParser(config.guideline, config.type)+ prompt, _, _ = parse_guideline(config.guideline, config.type)options = OptionsBuilder.build_set_options(config)@@ -166,7 +166,7 @@ class TemplateFactory:"""Create AutoGraph template."""from hyperextract.types import AutoGraph- entity_schema, relation_schema = OutputParser(config)+ entity_schema, relation_schema = parse_output(config)identifiers = IdentifierResolver.resolve_all(config)entity_key_extractor = identifiers.get("entity_key_extractor")relation_key_extractor = identifiers.get("relation_key_extractor")@@ -177,7 +177,7 @@ class TemplateFactory:options = OptionsBuilder.build_graph_options(config)prompt, prompt_for_entity_extraction, prompt_for_relation_extraction = (- GuidelineParser(config.guideline, config.type)+ parse_guideline(config.guideline, config.type))return AutoGraph(@@ -224,7 +224,7 @@ class TemplateFactory:) -> "AutoTemporalGraph":from hyperextract.types import AutoTemporalGraph- entity_schema, relation_schema = OutputParser(config)+ entity_schema, relation_schema = parse_output(config)identifiers = IdentifierResolver.resolve_all(config)entity_key_extractor = identifiers.get("entity_key_extractor")relation_key_extractor = identifiers.get("relation_key_extractor")@@ -236,7 +236,7 @@ class TemplateFactory:options = OptionsBuilder.build_temporal_graph_options(config)prompt, prompt_for_entity_extraction, prompt_for_relation_extraction = (- GuidelineParser(config.guideline, config.type)+ parse_guideline(config.guideline, config.type))return AutoTemporalGraph(@@ -279,7 +279,7 @@ class TemplateFactory:) -> "AutoSpatialGraph":from hyperextract.types import AutoSpatialGraph- entity_schema, relation_schema = OutputParser(config)+ entity_schema, relation_schema = parse_output(config)identifiers = IdentifierResolver.resolve_all(config)entity_key_extractor = identifiers.get("entity_key_extractor")relation_key_extractor = identifiers.get("relation_key_extractor")@@ -293,7 +293,7 @@ class TemplateFactory:options = OptionsBuilder.build_spatial_graph_options(config)prompt, prompt_for_entity_extraction, prompt_for_relation_extraction = (- GuidelineParser(config.guideline, config.type)+ parse_guideline(config.guideline, config.type))return AutoSpatialGraph(@@ -336,7 +336,7 @@ class TemplateFactory:) -> "AutoSpatioTemporalGraph":from hyperextract.types import AutoSpatioTemporalGraph- entity_schema, relation_schema = OutputParser(config)+ entity_schema, relation_schema = parse_output(config)identifiers = IdentifierResolver.resolve_all(config)entity_key_extractor = identifiers.get("entity_key_extractor")relation_key_extractor = identifiers.get("relation_key_extractor")@@ -351,7 +351,7 @@ class TemplateFactory:options = OptionsBuilder.build_spatio_temporal_graph_options(config)prompt, prompt_for_entity_extraction, prompt_for_relation_extraction = (- GuidelineParser(config.guideline, config.type)+ parse_guideline(config.guideline, config.type))return AutoSpatioTemporalGraph(diff --git a/hyperextract/utils/template_engine/parsers/__init__.py b/hyperextract/utils/template_engine/parsers/__init__.pyindex 1482c79..c615f90 100644--- a/hyperextract/utils/template_engine/parsers/__init__.py+++ b/hyperextract/utils/template_engine/parsers/__init__.py@@ -5,8 +5,8 @@ from .loader import (load_template,localize_template,)-from .output import OutputParser-from .guideline import GuidelineParser+from .output import parse_output+from .guideline import parse_guidelinefrom .identifiers import IdentifierResolverfrom .options import (Options,@@ -18,8 +18,8 @@ __all__ = ["TemplateCfg","load_template","localize_template",- "OutputParser",- "GuidelineParser",+ "parse_output",+ "parse_guideline","IdentifierResolver","Options","OptionsBuilder",diff --git a/hyperextract/utils/template_engine/parsers/guideline.py b/hyperextract/utils/template_engine/parsers/guideline.pyindex f10e5f4..c5a3008 100644--- a/hyperextract/utils/template_engine/parsers/guideline.py+++ b/hyperextract/utils/template_engine/parsers/guideline.py@@ -27,7 +27,7 @@ LABEL_MAPPING = {}-def GuidelineParser(guideline, autotype: str = "model") -> Tuple[str, str, str]:+def parse_guideline(guideline, autotype: str) -> str | Tuple[str, str, str]:"""Parse guideline and return prompts based on autotype (config is already localized).Args:@@ -48,7 +48,7 @@ def GuidelineParser(guideline, autotype: str = "model") -> Tuple[str, str, str]:if guideline.rules:parts.append(f"## {labels.get('rules')}:\n{guideline.rules}")parts.append(f"## {labels.get('source_text')}:\n{{source_text}}")- return "\n\n".join(parts), "", ""+ return "\n\n".join(parts)else:main_parts = [prefix_prompt]if guideline.rules_for_entities:@@ -102,5 +102,5 @@ def GuidelineParser(guideline, autotype: str = "model") -> Tuple[str, str, str]:__all__ = [- "GuidelineParser",+ "parse_guideline",]diff --git a/hyperextract/utils/template_engine/parsers/options.py b/hyperextract/utils/template_engine/parsers/options.pyindex 165cb45..3ce4066 100644--- a/hyperextract/utils/template_engine/parsers/options.py+++ b/hyperextract/utils/template_engine/parsers/options.py@@ -5,7 +5,7 @@ from pydantic import BaseModelfrom ontomem.merger import MergeStrategy-from .output import OutputParser+from .output import parse_outputfrom .identifiers import IdentifierResolver@@ -363,7 +363,7 @@ class OptionsBuilder:@classmethoddef validate_model_config(cls, config):"""Validate AutoModel configuration."""- schema_dict = OutputParser(config)+ schema_dict = parse_output(config)if schema_dict.get("schema") is None:raise ValueError(f"AutoModel requires schema definition: {config.name}")return schema_dict@@ -371,7 +371,7 @@ class OptionsBuilder:@classmethoddef validate_list_config(cls, config):"""Validate AutoList configuration."""- schema_dict = OutputParser(config)+ schema_dict = parse_output(config)if schema_dict.get("item_schema") is None:raise ValueError(f"AutoList requires item_schema definition: {config.name}")return schema_dict@@ -379,7 +379,7 @@ class OptionsBuilder:@classmethoddef validate_set_config(cls, config):"""Validate AutoSet configuration."""- schema_dict = OutputParser(config)+ schema_dict = parse_output(config)if schema_dict.get("item_schema") is None:raise ValueError(f"AutoSet requires item_schema definition: {config.name}")@@ -392,7 +392,7 @@ class OptionsBuilder:@classmethoddef validate_graph_config(cls, config):"""Validate AutoGraph/AutoHypergraph configuration."""- schema_dict = OutputParser(config)+ schema_dict = parse_output(config)if schema_dict.get("entity_schema") is None or schema_dict.get("relation_schema") is None:raise ValueError(f"AutoGraph requires entity_schema and relation_schema definition: {config.name}")diff --git a/hyperextract/utils/template_engine/parsers/output.py b/hyperextract/utils/template_engine/parsers/output.pyindex 117b31e..6047d91 100644--- a/hyperextract/utils/template_engine/parsers/output.py+++ b/hyperextract/utils/template_engine/parsers/output.py@@ -1,7 +1,13 @@"""Output schema parser."""-from typing import Any, List, Type, Union, Tuple-from pydantic import Field, create_model+from typing import List+from .schemas import (+ FieldSchema,+ NaiveOutputSchema,+ GraphOutputSchema,+ NaiveOutputSchema,+)+from pydantic import Field, create_model, BaseModelTYPE_MAPPING = {@@ -9,139 +15,76 @@ TYPE_MAPPING = {"int": int,"float": float,"bool": bool,- "list": list,+ "list[str]": List[str],}-def build_field_type(field) -> Type:- """Build field type from field definition."""- if isinstance(field, dict):- field_type_str = field.get("type", "str")- else:- field_type_str = field.type-- if field_type_str == "list":- if isinstance(field, dict):- item_type_str = field.get("item_type")- else:- item_type_str = getattr(field, "item_type", None)- if item_type_str:- item_type = TYPE_MAPPING.get(item_type_str, str)- return List[item_type]- return List- return TYPE_MAPPING.get(field_type_str, str)---def build_field_default(field) -> Any:- """Build field default value."""- if isinstance(field, dict):- is_required = field.get("required", False)- default_val = field.get("default")- else:- is_required = getattr(field, "required", False)- default_val = getattr(field, "default", None)-- if is_required:- return ...- if default_val is not None:- return default_val- field_type = field.get("type", "str") if isinstance(field, dict) else getattr(field, "type", "str")- if field_type == "list":- return []- return None---def build_field_description(field) -> str:- """Get field description (config is already localized)."""- if isinstance(field, dict):- return field.get("description", "") or ""- return getattr(field, "description", "") or ""---def _build_schema_from_fields(- fields: List,- schema_name: str,-) -> Type:- """Build Pydantic schema from field list."""+def build_naive_schema(+ name: str,+ fields: List[FieldSchema],+ description: str,+) -> BaseModel:+ """Build Pydantic schema for naive types (model, list, set)."""schema_fields = {}for field in fields:- if isinstance(field, dict):- field_name = field.get("name")- else:- field_name = getattr(field, "name", None)-- if not field_name:- continue-- field_type = build_field_type(field)- default = build_field_default(field)- description = build_field_description(field)- schema_fields[field_name] = (field_type, Field(default=default, description=description))- return create_model(schema_name, **schema_fields)---def _get_output_value(output, attr: str):- """Get value from output object."""- if output is None:- return None- if isinstance(output, dict):- return output.get(attr)- return getattr(output, attr, None)---def _build_naive_schema(config) -> Type:- """Build schema for non-graph types (model, list, set)."""- output = config.output- output_fields = _get_output_value(output, "fields") if output else None-- if output_fields:- schema_name = f"{config.name}Schema"- return _build_schema_from_fields(output_fields, schema_name)-- raise ValueError(f"No fields definition found for {config.type} type: {config.name}")---def _build_graph_schemas(config) -> Tuple[Type, Type]:- """Build schemas for graph types (graph, hypergraph, temporal_graph, spatial_graph, spatio_temporal_graph)."""- output = config.output- entities = _get_output_value(output, "entities")- relations = _get_output_value(output, "relations")-- if entities and relations:- entity_fields = _get_output_value(entities, "fields")- relation_fields = _get_output_value(relations, "fields")- if entity_fields and relation_fields:- entity_schema = _build_schema_from_fields(- entity_fields,- f"{config.name}Entity",- )- relation_schema = _build_schema_from_fields(- relation_fields,- f"{config.name}Relation",- )- return entity_schema, relation_schema-- raise ValueError(f"No schema definition found for graph type: {config.name}")---def OutputParser(config) -> Union[Type, Tuple[Type, Type]]:+ kwargs = {+ "description": field.description,+ }+ if field.required is not None:+ kwargs["required"] = field.required+ if field.default is not None:+ kwargs["default"] = field.default+ schema_fields[field.name] = (TYPE_MAPPING[field.type], Field(**kwargs))+ return create_model(name, __doc__=description, **schema_fields)+++def build_graph_schema(+ name: str,+ entity: NaiveOutputSchema,+ relation: NaiveOutputSchema,+ description: str,+) -> BaseModel:+ """Build Pydantic schema for graph types (graph, hypergraph, temporal_graph, spatial_graph, spatio_temporal_graph)."""+ schema_fields = {+ "entities": build_naive_schema(+ entity.name,+ entity.fields,+ entity.description,+ ),+ "relations": build_naive_schema(+ relation.name,+ relation.fields,+ relation.description,+ ),+ }+ return create_model(name, __doc__=description, **schema_fields)+++def parse_output(+ output: NaiveOutputSchema | GraphOutputSchema, autotype: str+) -> BaseModel:"""Parse template and return schemas based on autotype.Args:- config: TemplateCfg instance (single-language, all fields are strings)+ config: TemplateCfg instance- Returns:- - For non-graph types (model, list, set): returns data_schema- - For graph types: returns (entity_schema, relation_schema)"""- autotype = config.typeif autotype in ("model", "list", "set"):- return _build_naive_schema(config)+ return build_naive_schema(+ output.name,+ output.fields,+ output.description,+ )else:- return _build_graph_schemas(config)+ return build_graph_schema(+ output.name,+ output.entities,+ output.relations,+ output.description,+ )__all__ = [- "OutputParser",+ "parse_output",]diff --git a/hyperextract/utils/template_engine/parsers/schemas/base.py b/hyperextract/utils/template_engine/parsers/schemas/base.pyindex c6256ec..c2c04e4 100644--- a/hyperextract/utils/template_engine/parsers/schemas/base.py+++ b/hyperextract/utils/template_engine/parsers/schemas/base.py@@ -21,10 +21,12 @@ VALID_AUTOTYPES = Literal["spatio_temporal_graph",]+VALID_FIELD_TYPES = Literal["str", "int", "float", "bool", "list[str]"]+class FieldSchema(BaseModel):name: str- type: Literal["str", "int", "float", "bool", "list", "dict"]+ type: VALID_FIELD_TYPESdescription: str | Dict[str, str]- required: bool = False+ required: bool | None = Nonedefault: Any | None = None