Skip to content

Hypergraph is more powerful. Transform unstructured text into structured knowledge with LLMs. Graphs, hypergraphs, and spatio-temporal extractions — with one command.

refactor(template_engine): 重构解析器函数命名和输出模式处理

yifanfeng974732381907c8
diff --git a/hyperextract/utils/template_engine/factory.py b/hyperextract/utils/template_engine/factory.py
index a253fd6..b636d6c 100644
--- a/hyperextract/utils/template_engine/factory.py
+++ b/hyperextract/utils/template_engine/factory.py
@@ -12,9 +12,9 @@ from .parsers import (
TemplateCfg,
OptionsBuilder,
Options,
- OutputParser,
+ parse_output,
IdentifierResolver,
- GuidelineParser,
+ parse_guideline,
localize_template,
)
@@ -79,9 +79,9 @@ class TemplateFactory:
"""Create AutoModel template."""
from hyperextract.types import AutoModel
- data_schema = OutputParser(config)
+ data_schema = parse_output(config)
- prompt, _, _ = GuidelineParser(config.guideline, config.type)
+ prompt, _, _ = parse_guideline(config.guideline, config.type)
options = OptionsBuilder.build_model_options(config)
@@ -109,9 +109,9 @@ class TemplateFactory:
"""Create AutoList template."""
from hyperextract.types import AutoList
- data_schema = OutputParser(config)
+ data_schema = parse_output(config)
- prompt, _, _ = GuidelineParser(config.guideline, config.type)
+ prompt, _, _ = parse_guideline(config.guideline, config.type)
options = OptionsBuilder.build_list_options(config)
@@ -133,11 +133,11 @@ class TemplateFactory:
"""Create AutoSet template."""
from hyperextract.types import AutoSet
- data_schema = OutputParser(config)
+ data_schema = parse_output(config)
identifiers = IdentifierResolver.resolve_all(config)
item_id_extractor = identifiers.get("item_id_extractor")
- prompt, _, _ = GuidelineParser(config.guideline, config.type)
+ prompt, _, _ = parse_guideline(config.guideline, config.type)
options = OptionsBuilder.build_set_options(config)
@@ -166,7 +166,7 @@ class TemplateFactory:
"""Create AutoGraph template."""
from hyperextract.types import AutoGraph
- entity_schema, relation_schema = OutputParser(config)
+ entity_schema, relation_schema = parse_output(config)
identifiers = IdentifierResolver.resolve_all(config)
entity_key_extractor = identifiers.get("entity_key_extractor")
relation_key_extractor = identifiers.get("relation_key_extractor")
@@ -177,7 +177,7 @@ class TemplateFactory:
options = OptionsBuilder.build_graph_options(config)
prompt, prompt_for_entity_extraction, prompt_for_relation_extraction = (
- GuidelineParser(config.guideline, config.type)
+ parse_guideline(config.guideline, config.type)
)
return AutoGraph(
@@ -224,7 +224,7 @@ class TemplateFactory:
) -> "AutoTemporalGraph":
from hyperextract.types import AutoTemporalGraph
- entity_schema, relation_schema = OutputParser(config)
+ entity_schema, relation_schema = parse_output(config)
identifiers = IdentifierResolver.resolve_all(config)
entity_key_extractor = identifiers.get("entity_key_extractor")
relation_key_extractor = identifiers.get("relation_key_extractor")
@@ -236,7 +236,7 @@ class TemplateFactory:
options = OptionsBuilder.build_temporal_graph_options(config)
prompt, prompt_for_entity_extraction, prompt_for_relation_extraction = (
- GuidelineParser(config.guideline, config.type)
+ parse_guideline(config.guideline, config.type)
)
return AutoTemporalGraph(
@@ -279,7 +279,7 @@ class TemplateFactory:
) -> "AutoSpatialGraph":
from hyperextract.types import AutoSpatialGraph
- entity_schema, relation_schema = OutputParser(config)
+ entity_schema, relation_schema = parse_output(config)
identifiers = IdentifierResolver.resolve_all(config)
entity_key_extractor = identifiers.get("entity_key_extractor")
relation_key_extractor = identifiers.get("relation_key_extractor")
@@ -293,7 +293,7 @@ class TemplateFactory:
options = OptionsBuilder.build_spatial_graph_options(config)
prompt, prompt_for_entity_extraction, prompt_for_relation_extraction = (
- GuidelineParser(config.guideline, config.type)
+ parse_guideline(config.guideline, config.type)
)
return AutoSpatialGraph(
@@ -336,7 +336,7 @@ class TemplateFactory:
) -> "AutoSpatioTemporalGraph":
from hyperextract.types import AutoSpatioTemporalGraph
- entity_schema, relation_schema = OutputParser(config)
+ entity_schema, relation_schema = parse_output(config)
identifiers = IdentifierResolver.resolve_all(config)
entity_key_extractor = identifiers.get("entity_key_extractor")
relation_key_extractor = identifiers.get("relation_key_extractor")
@@ -351,7 +351,7 @@ class TemplateFactory:
options = OptionsBuilder.build_spatio_temporal_graph_options(config)
prompt, prompt_for_entity_extraction, prompt_for_relation_extraction = (
- GuidelineParser(config.guideline, config.type)
+ parse_guideline(config.guideline, config.type)
)
return AutoSpatioTemporalGraph(
diff --git a/hyperextract/utils/template_engine/parsers/__init__.py b/hyperextract/utils/template_engine/parsers/__init__.py
index 1482c79..c615f90 100644
--- a/hyperextract/utils/template_engine/parsers/__init__.py
+++ b/hyperextract/utils/template_engine/parsers/__init__.py
@@ -5,8 +5,8 @@ from .loader import (
load_template,
localize_template,
)
-from .output import OutputParser
-from .guideline import GuidelineParser
+from .output import parse_output
+from .guideline import parse_guideline
from .identifiers import IdentifierResolver
from .options import (
Options,
@@ -18,8 +18,8 @@ __all__ = [
"TemplateCfg",
"load_template",
"localize_template",
- "OutputParser",
- "GuidelineParser",
+ "parse_output",
+ "parse_guideline",
"IdentifierResolver",
"Options",
"OptionsBuilder",
diff --git a/hyperextract/utils/template_engine/parsers/guideline.py b/hyperextract/utils/template_engine/parsers/guideline.py
index f10e5f4..c5a3008 100644
--- a/hyperextract/utils/template_engine/parsers/guideline.py
+++ b/hyperextract/utils/template_engine/parsers/guideline.py
@@ -27,7 +27,7 @@ LABEL_MAPPING = {
}
-def GuidelineParser(guideline, autotype: str = "model") -> Tuple[str, str, str]:
+def parse_guideline(guideline, autotype: str) -> str | Tuple[str, str, str]:
"""Parse guideline and return prompts based on autotype (config is already localized).
Args:
@@ -48,7 +48,7 @@ def GuidelineParser(guideline, autotype: str = "model") -> Tuple[str, str, str]:
if guideline.rules:
parts.append(f"## {labels.get('rules')}:\n{guideline.rules}")
parts.append(f"## {labels.get('source_text')}:\n{{source_text}}")
- return "\n\n".join(parts), "", ""
+ return "\n\n".join(parts)
else:
main_parts = [prefix_prompt]
if guideline.rules_for_entities:
@@ -102,5 +102,5 @@ def GuidelineParser(guideline, autotype: str = "model") -> Tuple[str, str, str]:
__all__ = [
- "GuidelineParser",
+ "parse_guideline",
]
diff --git a/hyperextract/utils/template_engine/parsers/options.py b/hyperextract/utils/template_engine/parsers/options.py
index 165cb45..3ce4066 100644
--- a/hyperextract/utils/template_engine/parsers/options.py
+++ b/hyperextract/utils/template_engine/parsers/options.py
@@ -5,7 +5,7 @@ from pydantic import BaseModel
from ontomem.merger import MergeStrategy
-from .output import OutputParser
+from .output import parse_output
from .identifiers import IdentifierResolver
@@ -363,7 +363,7 @@ class OptionsBuilder:
@classmethod
def validate_model_config(cls, config):
"""Validate AutoModel configuration."""
- schema_dict = OutputParser(config)
+ schema_dict = parse_output(config)
if schema_dict.get("schema") is None:
raise ValueError(f"AutoModel requires schema definition: {config.name}")
return schema_dict
@@ -371,7 +371,7 @@ class OptionsBuilder:
@classmethod
def validate_list_config(cls, config):
"""Validate AutoList configuration."""
- schema_dict = OutputParser(config)
+ schema_dict = parse_output(config)
if schema_dict.get("item_schema") is None:
raise ValueError(f"AutoList requires item_schema definition: {config.name}")
return schema_dict
@@ -379,7 +379,7 @@ class OptionsBuilder:
@classmethod
def validate_set_config(cls, config):
"""Validate AutoSet configuration."""
- schema_dict = OutputParser(config)
+ schema_dict = parse_output(config)
if schema_dict.get("item_schema") is None:
raise ValueError(f"AutoSet requires item_schema definition: {config.name}")
@@ -392,7 +392,7 @@ class OptionsBuilder:
@classmethod
def validate_graph_config(cls, config):
"""Validate AutoGraph/AutoHypergraph configuration."""
- schema_dict = OutputParser(config)
+ schema_dict = parse_output(config)
if schema_dict.get("entity_schema") is None or schema_dict.get("relation_schema") is None:
raise ValueError(f"AutoGraph requires entity_schema and relation_schema definition: {config.name}")
diff --git a/hyperextract/utils/template_engine/parsers/output.py b/hyperextract/utils/template_engine/parsers/output.py
index 117b31e..6047d91 100644
--- a/hyperextract/utils/template_engine/parsers/output.py
+++ b/hyperextract/utils/template_engine/parsers/output.py
@@ -1,7 +1,13 @@
"""Output schema parser."""
-from typing import Any, List, Type, Union, Tuple
-from pydantic import Field, create_model
+from typing import List
+from .schemas import (
+ FieldSchema,
+ NaiveOutputSchema,
+ GraphOutputSchema,
+ NaiveOutputSchema,
+)
+from pydantic import Field, create_model, BaseModel
TYPE_MAPPING = {
@@ -9,139 +15,76 @@ TYPE_MAPPING = {
"int": int,
"float": float,
"bool": bool,
- "list": list,
+ "list[str]": List[str],
}
-def build_field_type(field) -> Type:
- """Build field type from field definition."""
- if isinstance(field, dict):
- field_type_str = field.get("type", "str")
- else:
- field_type_str = field.type
-
- if field_type_str == "list":
- if isinstance(field, dict):
- item_type_str = field.get("item_type")
- else:
- item_type_str = getattr(field, "item_type", None)
- if item_type_str:
- item_type = TYPE_MAPPING.get(item_type_str, str)
- return List[item_type]
- return List
- return TYPE_MAPPING.get(field_type_str, str)
-
-
-def build_field_default(field) -> Any:
- """Build field default value."""
- if isinstance(field, dict):
- is_required = field.get("required", False)
- default_val = field.get("default")
- else:
- is_required = getattr(field, "required", False)
- default_val = getattr(field, "default", None)
-
- if is_required:
- return ...
- if default_val is not None:
- return default_val
- field_type = field.get("type", "str") if isinstance(field, dict) else getattr(field, "type", "str")
- if field_type == "list":
- return []
- return None
-
-
-def build_field_description(field) -> str:
- """Get field description (config is already localized)."""
- if isinstance(field, dict):
- return field.get("description", "") or ""
- return getattr(field, "description", "") or ""
-
-
-def _build_schema_from_fields(
- fields: List,
- schema_name: str,
-) -> Type:
- """Build Pydantic schema from field list."""
+def build_naive_schema(
+ name: str,
+ fields: List[FieldSchema],
+ description: str,
+) -> BaseModel:
+ """Build Pydantic schema for naive types (model, list, set)."""
schema_fields = {}
for field in fields:
- if isinstance(field, dict):
- field_name = field.get("name")
- else:
- field_name = getattr(field, "name", None)
-
- if not field_name:
- continue
-
- field_type = build_field_type(field)
- default = build_field_default(field)
- description = build_field_description(field)
- schema_fields[field_name] = (field_type, Field(default=default, description=description))
- return create_model(schema_name, **schema_fields)
-
-
-def _get_output_value(output, attr: str):
- """Get value from output object."""
- if output is None:
- return None
- if isinstance(output, dict):
- return output.get(attr)
- return getattr(output, attr, None)
-
-
-def _build_naive_schema(config) -> Type:
- """Build schema for non-graph types (model, list, set)."""
- output = config.output
- output_fields = _get_output_value(output, "fields") if output else None
-
- if output_fields:
- schema_name = f"{config.name}Schema"
- return _build_schema_from_fields(output_fields, schema_name)
-
- raise ValueError(f"No fields definition found for {config.type} type: {config.name}")
-
-
-def _build_graph_schemas(config) -> Tuple[Type, Type]:
- """Build schemas for graph types (graph, hypergraph, temporal_graph, spatial_graph, spatio_temporal_graph)."""
- output = config.output
- entities = _get_output_value(output, "entities")
- relations = _get_output_value(output, "relations")
-
- if entities and relations:
- entity_fields = _get_output_value(entities, "fields")
- relation_fields = _get_output_value(relations, "fields")
- if entity_fields and relation_fields:
- entity_schema = _build_schema_from_fields(
- entity_fields,
- f"{config.name}Entity",
- )
- relation_schema = _build_schema_from_fields(
- relation_fields,
- f"{config.name}Relation",
- )
- return entity_schema, relation_schema
-
- raise ValueError(f"No schema definition found for graph type: {config.name}")
-
-
-def OutputParser(config) -> Union[Type, Tuple[Type, Type]]:
+ kwargs = {
+ "description": field.description,
+ }
+ if field.required is not None:
+ kwargs["required"] = field.required
+ if field.default is not None:
+ kwargs["default"] = field.default
+ schema_fields[field.name] = (TYPE_MAPPING[field.type], Field(**kwargs))
+ return create_model(name, __doc__=description, **schema_fields)
+
+
+def build_graph_schema(
+ name: str,
+ entity: NaiveOutputSchema,
+ relation: NaiveOutputSchema,
+ description: str,
+) -> BaseModel:
+ """Build Pydantic schema for graph types (graph, hypergraph, temporal_graph, spatial_graph, spatio_temporal_graph)."""
+ schema_fields = {
+ "entities": build_naive_schema(
+ entity.name,
+ entity.fields,
+ entity.description,
+ ),
+ "relations": build_naive_schema(
+ relation.name,
+ relation.fields,
+ relation.description,
+ ),
+ }
+ return create_model(name, __doc__=description, **schema_fields)
+
+
+def parse_output(
+ output: NaiveOutputSchema | GraphOutputSchema, autotype: str
+) -> BaseModel:
"""Parse template and return schemas based on autotype.
Args:
- config: TemplateCfg instance (single-language, all fields are strings)
+ config: TemplateCfg instance
- Returns:
- - For non-graph types (model, list, set): returns data_schema
- - For graph types: returns (entity_schema, relation_schema)
"""
- autotype = config.type
if autotype in ("model", "list", "set"):
- return _build_naive_schema(config)
+ return build_naive_schema(
+ output.name,
+ output.fields,
+ output.description,
+ )
else:
- return _build_graph_schemas(config)
+ return build_graph_schema(
+ output.name,
+ output.entities,
+ output.relations,
+ output.description,
+ )
__all__ = [
- "OutputParser",
+ "parse_output",
]
diff --git a/hyperextract/utils/template_engine/parsers/schemas/base.py b/hyperextract/utils/template_engine/parsers/schemas/base.py
index c6256ec..c2c04e4 100644
--- a/hyperextract/utils/template_engine/parsers/schemas/base.py
+++ b/hyperextract/utils/template_engine/parsers/schemas/base.py
@@ -21,10 +21,12 @@ VALID_AUTOTYPES = Literal[
"spatio_temporal_graph",
]
+VALID_FIELD_TYPES = Literal["str", "int", "float", "bool", "list[str]"]
+
class FieldSchema(BaseModel):
name: str
- type: Literal["str", "int", "float", "bool", "list", "dict"]
+ type: VALID_FIELD_TYPES
description: str | Dict[str, str]
- required: bool = False
+ required: bool | None = None
default: Any | None = None