Hypergraph is more powerful. Transform unstructured text into structured knowledge with LLMs. Graphs, hypergraphs, and spatio-temporal extractions — with one command.
Revert "Merge pull request #28 from li5435945-ship-it/fix/batch-none-handling"
SLca2888579c2d
diff --git a/hyperextract/types/base.py b/hyperextract/types/base.pyindex 69a94d6..d43bc1f 100644--- a/hyperextract/types/base.py+++ b/hyperextract/types/base.py@@ -290,9 +290,6 @@ class BaseAutoType(ABC, Generic[T]):self._summarize_extracted(result),)- # Filter out None results from failed LLM extractions- extracted_data_list = self._filter_none_results(extracted_data_list)-logger.debug("stage=merge_start num_items=%d", len(extracted_data_list))merged_data = self.merge_batch_data(extracted_data_list)logger.debug(@@ -301,36 +298,6 @@ class BaseAutoType(ABC, Generic[T]):)return merged_data- def _filter_none_results(self, results: list, default_factory=None) -> list:- """Replace None results from batch LLM extractions with empty defaults.-- When the LLM fails to parse output (e.g., context length exceeded,- json_schema validation failure), batch() returns None for that item.- This method replaces None with empty objects to maintain index alignment- with the original chunks list.-- Args:- results: List of extraction results, may contain None values.- default_factory: Callable that creates a default empty object.- If None, None values are simply removed (legacy behavior).-- Returns:- List with None values replaced or removed.- """- if not results:- return results- none_count = sum(1 for r in results if r is None)- if none_count > 0:- logger.warning(- "stage=batch_filter none_results_detected none_count=%d total=%d",- none_count,- len(results),- )- if default_factory is not None:- return [r if r is not None else default_factory() for r in results]- return [r for r in results if r is not None]- return results-def _summarize_extracted(self, data: T) -> str:"""Return a concise summary of extracted data for debug logging."""try:diff --git a/hyperextract/types/graph.py b/hyperextract/types/graph.pyindex aab2345..afecdeb 100644--- a/hyperextract/types/graph.py+++ b/hyperextract/types/graph.py@@ -492,10 +492,6 @@ class AutoGraph(graph_list = self.data_extractor.batch(inputs, config={"max_concurrency": self.max_workers})- graph_list = self._filter_none_results(- graph_list,- default_factory=lambda: self.graph_schema(nodes=[], edges=[]),- )logger.debug("stage=one_stage_batch_complete graphs=%d", len(graph_list))logger.debug("stage=one_stage_merge_start")@@ -582,13 +578,9 @@ class AutoGraph(List of NodeListSchema objects with extracted nodes."""inputs = [{"source_text": chunk} for chunk in chunks]- results = self.node_extractor.batch(+ return self.node_extractor.batch(inputs, config={"max_concurrency": self.max_workers})- return self._filter_none_results(- results,- default_factory=lambda: self.node_list_schema(items=[]),- )def _extract_edges_batch(self, chunks: List[str], node_lists: List[NodeListSchema[NodeSchema]]@@ -613,13 +605,9 @@ class AutoGraph(inputs.append({"source_text": chunk, "known_nodes": known_nodes})- results = self.edge_extractor.batch(+ return self.edge_extractor.batch(inputs, config={"max_concurrency": self.max_workers})- return self._filter_none_results(- results,- default_factory=lambda: self.edge_list_schema(items=[]),- )def _prune_dangling_edges(self, graph: AutoGraphSchema[NodeSchema, EdgeSchema]@@ -684,11 +672,6 @@ class AutoGraph(Returns:Merged graph."""- # Handle empty input (all batch results were None/filtered out)- if not data_list_or_tuple:- logger.warning("stage=merge_batch_empty input_is_empty")- return self.graph_schema(nodes=[], edges=[])-logger.debug("stage=merge_batch_start input_type=%s","tuple"@@ -708,20 +691,12 @@ class AutoGraph("Invalid input format for batch merging")nodes_lists, edges_lists = data_list_or_tuple[0], data_list_or_tuple[1]-- # Handle empty nodes/edges lists- if not nodes_lists and not edges_lists:- logger.warning("stage=merge_batch_empty_tuple nodes_and_edges_empty")- return self.graph_schema(nodes=[], edges=[])-- if nodes_lists:- assert isinstance(nodes_lists[0][0], self.node_schema), (- "Invalid node list format for batch merging"- )- if edges_lists:- assert isinstance(edges_lists[0][0], self.edge_schema), (- "Invalid edge list format for batch merging"- )+ assert isinstance(nodes_lists[0][0], self.node_schema), (+ "Invalid node list format for batch merging"+ )+ assert isinstance(edges_lists[0][0], self.edge_schema), (+ "Invalid edge list format for batch merging"+ )all_nodes, all_edges = [], []for node_list, edge_list in zip(nodes_lists, edges_lists):diff --git a/hyperextract/types/hypergraph.py b/hyperextract/types/hypergraph.pyindex 53dea78..73bc40a 100644--- a/hyperextract/types/hypergraph.py+++ b/hyperextract/types/hypergraph.py@@ -460,10 +460,6 @@ class AutoHypergraph(graph_list = self.data_extractor.batch(inputs, config={"max_concurrency": self.max_workers})- graph_list = self._filter_none_results(- graph_list,- default_factory=lambda: self.graph_schema(nodes=[], edges=[]),- )# Merge multiple hypergraphsreturn self.merge_batch_data(graph_list)@@ -506,13 +502,9 @@ class AutoHypergraph() -> List[NodeListSchema[NodeSchema]]:"""Batch extract nodes from multiple text chunks."""inputs = [{"source_text": chunk} for chunk in chunks]- results = self.node_extractor.batch(+ return self.node_extractor.batch(inputs, config={"max_concurrency": self.max_workers})- return self._filter_none_results(- results,- default_factory=lambda: self.node_list_schema(items=[]),- )def _extract_edges_batch(self, chunks: List[str], node_lists: List[NodeListSchema[NodeSchema]]@@ -529,13 +521,9 @@ class AutoHypergraph(inputs.append({"source_text": chunk, "known_nodes": known_nodes})- results = self.edge_extractor.batch(+ return self.edge_extractor.batch(inputs, config={"max_concurrency": self.max_workers})- return self._filter_none_results(- results,- default_factory=lambda: self.edge_list_schema(items=[]),- )def _prune_dangling_edges(self, graph: AutoHypergraphSchema@@ -605,11 +593,6 @@ class AutoHypergraph(Returns:Merged hypergraph."""- # Handle empty input (all batch results were None/filtered out)- if not data_list_or_tuple:- logger.warning("stage=merge_batch_empty input_is_empty")- return self.graph_schema(nodes=[], edges=[])-if isinstance(data_list_or_tuple, list) and isinstance(data_list_or_tuple[0], self.graph_schema):@@ -625,20 +608,12 @@ class AutoHypergraph("Invalid input format for batch merging")nodes_lists, edges_lists = data_list_or_tuple[0], data_list_or_tuple[1]-- # Handle empty nodes/edges lists- if not nodes_lists and not edges_lists:- logger.warning("stage=merge_batch_empty_tuple nodes_and_edges_empty")- return self.graph_schema(nodes=[], edges=[])-- if nodes_lists:- assert isinstance(nodes_lists[0][0], self.node_schema), (- "Invalid node list format for batch merging"- )- if edges_lists:- assert isinstance(edges_lists[0][0], self.edge_schema), (- "Invalid edge list format for batch merging"- )+ assert isinstance(nodes_lists[0][0], self.node_schema), (+ "Invalid node list format for batch merging"+ )+ assert isinstance(edges_lists[0][0], self.edge_schema), (+ "Invalid edge list format for batch merging"+ )all_nodes, all_edges = [], []for node_list, edge_list in zip(nodes_lists, edges_lists):diff --git a/hyperextract/types/spatial_graph.py b/hyperextract/types/spatial_graph.pyindex b2f8a82..854fb15 100644--- a/hyperextract/types/spatial_graph.py+++ b/hyperextract/types/spatial_graph.py@@ -269,13 +269,9 @@ class AutoSpatialGraph(AutoGraph[NodeSchema, EdgeSchema]):})- results = self.edge_extractor.batch(+ return self.edge_extractor.batch(inputs, config={"max_concurrency": self.max_workers})- return self._filter_none_results(- results,- default_factory=lambda: self.edge_list_schema(items=[]),- )def _extract_data_by_one_stage(self, text: str) -> Any:"""Override: Inject observation_location into one-stage extraction."""@@ -299,10 +295,6 @@ class AutoSpatialGraph(AutoGraph[NodeSchema, EdgeSchema]):graph_list = self.data_extractor.batch(inputs, config={"max_concurrency": self.max_workers})- graph_list = self._filter_none_results(- graph_list,- default_factory=lambda: self.graph_schema(nodes=[], edges=[]),- )return self.merge_batch_data(graph_list)diff --git a/hyperextract/types/spatio_temporal_graph.py b/hyperextract/types/spatio_temporal_graph.pyindex fc7ac91..78c1f37 100644--- a/hyperextract/types/spatio_temporal_graph.py+++ b/hyperextract/types/spatio_temporal_graph.py@@ -286,13 +286,9 @@ class AutoSpatioTemporalGraph(AutoGraph[NodeSchema, EdgeSchema]):})- results = self.edge_extractor.batch(+ return self.edge_extractor.batch(inputs, config={"max_concurrency": self.max_workers})- return self._filter_none_results(- results,- default_factory=lambda: self.edge_list_schema(items=[]),- )def _extract_data_by_one_stage(self, text: str) -> Any:"""Inject observation_time and observation_location into one-stage extraction."""@@ -318,10 +314,6 @@ class AutoSpatioTemporalGraph(AutoGraph[NodeSchema, EdgeSchema]):graph_list = self.data_extractor.batch(inputs, config={"max_concurrency": self.max_workers})- graph_list = self._filter_none_results(- graph_list,- default_factory=lambda: self.graph_schema(nodes=[], edges=[]),- )return self.merge_batch_data(graph_list)diff --git a/hyperextract/types/temporal_graph.py b/hyperextract/types/temporal_graph.pyindex 41adac9..3155254 100644--- a/hyperextract/types/temporal_graph.py+++ b/hyperextract/types/temporal_graph.py@@ -286,13 +286,9 @@ class AutoTemporalGraph(AutoGraph[NodeSchema, EdgeSchema]):})- results = self.edge_extractor.batch(+ return self.edge_extractor.batch(inputs, config={"max_concurrency": self.max_workers})- return self._filter_none_results(- results,- default_factory=lambda: self.edge_list_schema(items=[]),- )def _extract_data_by_one_stage(self, text: str) -> Any:"""Override: Inject observation_time into one-stage extraction."""@@ -310,10 +306,6 @@ class AutoTemporalGraph(AutoGraph[NodeSchema, EdgeSchema]):graph_list = self.data_extractor.batch(inputs, config={"max_concurrency": self.max_workers})- graph_list = self._filter_none_results(- graph_list,- default_factory=lambda: self.graph_schema(nodes=[], edges=[]),- )return self.merge_batch_data(graph_list)