Hypergraph is more powerful. Transform unstructured text into structured knowledge with LLMs. Graphs, hypergraphs, and spatio-temporal extractions — with one command.
fix: handle None results from batch() when LLM extraction fails
SL7fb7b4f3ab91
diff --git a/hyperextract/types/base.py b/hyperextract/types/base.pyindex d43bc1f..69a94d6 100644--- a/hyperextract/types/base.py+++ b/hyperextract/types/base.py@@ -290,6 +290,9 @@ class BaseAutoType(ABC, Generic[T]):self._summarize_extracted(result),)+ # Filter out None results from failed LLM extractions+ extracted_data_list = self._filter_none_results(extracted_data_list)+logger.debug("stage=merge_start num_items=%d", len(extracted_data_list))merged_data = self.merge_batch_data(extracted_data_list)logger.debug(@@ -298,6 +301,36 @@ class BaseAutoType(ABC, Generic[T]):)return merged_data+ def _filter_none_results(self, results: list, default_factory=None) -> list:+ """Replace None results from batch LLM extractions with empty defaults.++ When the LLM fails to parse output (e.g., context length exceeded,+ json_schema validation failure), batch() returns None for that item.+ This method replaces None with empty objects to maintain index alignment+ with the original chunks list.++ Args:+ results: List of extraction results, may contain None values.+ default_factory: Callable that creates a default empty object.+ If None, None values are simply removed (legacy behavior).++ Returns:+ List with None values replaced or removed.+ """+ if not results:+ return results+ none_count = sum(1 for r in results if r is None)+ if none_count > 0:+ logger.warning(+ "stage=batch_filter none_results_detected none_count=%d total=%d",+ none_count,+ len(results),+ )+ if default_factory is not None:+ return [r if r is not None else default_factory() for r in results]+ return [r for r in results if r is not None]+ return results+def _summarize_extracted(self, data: T) -> str:"""Return a concise summary of extracted data for debug logging."""try:diff --git a/hyperextract/types/graph.py b/hyperextract/types/graph.pyindex afecdeb..aab2345 100644--- a/hyperextract/types/graph.py+++ b/hyperextract/types/graph.py@@ -492,6 +492,10 @@ class AutoGraph(graph_list = self.data_extractor.batch(inputs, config={"max_concurrency": self.max_workers})+ graph_list = self._filter_none_results(+ graph_list,+ default_factory=lambda: self.graph_schema(nodes=[], edges=[]),+ )logger.debug("stage=one_stage_batch_complete graphs=%d", len(graph_list))logger.debug("stage=one_stage_merge_start")@@ -578,9 +582,13 @@ class AutoGraph(List of NodeListSchema objects with extracted nodes."""inputs = [{"source_text": chunk} for chunk in chunks]- return self.node_extractor.batch(+ results = self.node_extractor.batch(inputs, config={"max_concurrency": self.max_workers})+ return self._filter_none_results(+ results,+ default_factory=lambda: self.node_list_schema(items=[]),+ )def _extract_edges_batch(self, chunks: List[str], node_lists: List[NodeListSchema[NodeSchema]]@@ -605,9 +613,13 @@ class AutoGraph(inputs.append({"source_text": chunk, "known_nodes": known_nodes})- return self.edge_extractor.batch(+ results = self.edge_extractor.batch(inputs, config={"max_concurrency": self.max_workers})+ return self._filter_none_results(+ results,+ default_factory=lambda: self.edge_list_schema(items=[]),+ )def _prune_dangling_edges(self, graph: AutoGraphSchema[NodeSchema, EdgeSchema]@@ -672,6 +684,11 @@ class AutoGraph(Returns:Merged graph."""+ # Handle empty input (all batch results were None/filtered out)+ if not data_list_or_tuple:+ logger.warning("stage=merge_batch_empty input_is_empty")+ return self.graph_schema(nodes=[], edges=[])+logger.debug("stage=merge_batch_start input_type=%s","tuple"@@ -691,12 +708,20 @@ class AutoGraph("Invalid input format for batch merging")nodes_lists, edges_lists = data_list_or_tuple[0], data_list_or_tuple[1]- assert isinstance(nodes_lists[0][0], self.node_schema), (- "Invalid node list format for batch merging"- )- assert isinstance(edges_lists[0][0], self.edge_schema), (- "Invalid edge list format for batch merging"- )++ # Handle empty nodes/edges lists+ if not nodes_lists and not edges_lists:+ logger.warning("stage=merge_batch_empty_tuple nodes_and_edges_empty")+ return self.graph_schema(nodes=[], edges=[])++ if nodes_lists:+ assert isinstance(nodes_lists[0][0], self.node_schema), (+ "Invalid node list format for batch merging"+ )+ if edges_lists:+ assert isinstance(edges_lists[0][0], self.edge_schema), (+ "Invalid edge list format for batch merging"+ )all_nodes, all_edges = [], []for node_list, edge_list in zip(nodes_lists, edges_lists):diff --git a/hyperextract/types/hypergraph.py b/hyperextract/types/hypergraph.pyindex 73bc40a..53dea78 100644--- a/hyperextract/types/hypergraph.py+++ b/hyperextract/types/hypergraph.py@@ -460,6 +460,10 @@ class AutoHypergraph(graph_list = self.data_extractor.batch(inputs, config={"max_concurrency": self.max_workers})+ graph_list = self._filter_none_results(+ graph_list,+ default_factory=lambda: self.graph_schema(nodes=[], edges=[]),+ )# Merge multiple hypergraphsreturn self.merge_batch_data(graph_list)@@ -502,9 +506,13 @@ class AutoHypergraph() -> List[NodeListSchema[NodeSchema]]:"""Batch extract nodes from multiple text chunks."""inputs = [{"source_text": chunk} for chunk in chunks]- return self.node_extractor.batch(+ results = self.node_extractor.batch(inputs, config={"max_concurrency": self.max_workers})+ return self._filter_none_results(+ results,+ default_factory=lambda: self.node_list_schema(items=[]),+ )def _extract_edges_batch(self, chunks: List[str], node_lists: List[NodeListSchema[NodeSchema]]@@ -521,9 +529,13 @@ class AutoHypergraph(inputs.append({"source_text": chunk, "known_nodes": known_nodes})- return self.edge_extractor.batch(+ results = self.edge_extractor.batch(inputs, config={"max_concurrency": self.max_workers})+ return self._filter_none_results(+ results,+ default_factory=lambda: self.edge_list_schema(items=[]),+ )def _prune_dangling_edges(self, graph: AutoHypergraphSchema@@ -593,6 +605,11 @@ class AutoHypergraph(Returns:Merged hypergraph."""+ # Handle empty input (all batch results were None/filtered out)+ if not data_list_or_tuple:+ logger.warning("stage=merge_batch_empty input_is_empty")+ return self.graph_schema(nodes=[], edges=[])+if isinstance(data_list_or_tuple, list) and isinstance(data_list_or_tuple[0], self.graph_schema):@@ -608,12 +625,20 @@ class AutoHypergraph("Invalid input format for batch merging")nodes_lists, edges_lists = data_list_or_tuple[0], data_list_or_tuple[1]- assert isinstance(nodes_lists[0][0], self.node_schema), (- "Invalid node list format for batch merging"- )- assert isinstance(edges_lists[0][0], self.edge_schema), (- "Invalid edge list format for batch merging"- )++ # Handle empty nodes/edges lists+ if not nodes_lists and not edges_lists:+ logger.warning("stage=merge_batch_empty_tuple nodes_and_edges_empty")+ return self.graph_schema(nodes=[], edges=[])++ if nodes_lists:+ assert isinstance(nodes_lists[0][0], self.node_schema), (+ "Invalid node list format for batch merging"+ )+ if edges_lists:+ assert isinstance(edges_lists[0][0], self.edge_schema), (+ "Invalid edge list format for batch merging"+ )all_nodes, all_edges = [], []for node_list, edge_list in zip(nodes_lists, edges_lists):diff --git a/hyperextract/types/spatial_graph.py b/hyperextract/types/spatial_graph.pyindex 854fb15..b2f8a82 100644--- a/hyperextract/types/spatial_graph.py+++ b/hyperextract/types/spatial_graph.py@@ -269,9 +269,13 @@ class AutoSpatialGraph(AutoGraph[NodeSchema, EdgeSchema]):})- return self.edge_extractor.batch(+ results = self.edge_extractor.batch(inputs, config={"max_concurrency": self.max_workers})+ return self._filter_none_results(+ results,+ default_factory=lambda: self.edge_list_schema(items=[]),+ )def _extract_data_by_one_stage(self, text: str) -> Any:"""Override: Inject observation_location into one-stage extraction."""@@ -295,6 +299,10 @@ class AutoSpatialGraph(AutoGraph[NodeSchema, EdgeSchema]):graph_list = self.data_extractor.batch(inputs, config={"max_concurrency": self.max_workers})+ graph_list = self._filter_none_results(+ graph_list,+ default_factory=lambda: self.graph_schema(nodes=[], edges=[]),+ )return self.merge_batch_data(graph_list)diff --git a/hyperextract/types/spatio_temporal_graph.py b/hyperextract/types/spatio_temporal_graph.pyindex 78c1f37..fc7ac91 100644--- a/hyperextract/types/spatio_temporal_graph.py+++ b/hyperextract/types/spatio_temporal_graph.py@@ -286,9 +286,13 @@ class AutoSpatioTemporalGraph(AutoGraph[NodeSchema, EdgeSchema]):})- return self.edge_extractor.batch(+ results = self.edge_extractor.batch(inputs, config={"max_concurrency": self.max_workers})+ return self._filter_none_results(+ results,+ default_factory=lambda: self.edge_list_schema(items=[]),+ )def _extract_data_by_one_stage(self, text: str) -> Any:"""Inject observation_time and observation_location into one-stage extraction."""@@ -314,6 +318,10 @@ class AutoSpatioTemporalGraph(AutoGraph[NodeSchema, EdgeSchema]):graph_list = self.data_extractor.batch(inputs, config={"max_concurrency": self.max_workers})+ graph_list = self._filter_none_results(+ graph_list,+ default_factory=lambda: self.graph_schema(nodes=[], edges=[]),+ )return self.merge_batch_data(graph_list)diff --git a/hyperextract/types/temporal_graph.py b/hyperextract/types/temporal_graph.pyindex 3155254..41adac9 100644--- a/hyperextract/types/temporal_graph.py+++ b/hyperextract/types/temporal_graph.py@@ -286,9 +286,13 @@ class AutoTemporalGraph(AutoGraph[NodeSchema, EdgeSchema]):})- return self.edge_extractor.batch(+ results = self.edge_extractor.batch(inputs, config={"max_concurrency": self.max_workers})+ return self._filter_none_results(+ results,+ default_factory=lambda: self.edge_list_schema(items=[]),+ )def _extract_data_by_one_stage(self, text: str) -> Any:"""Override: Inject observation_time into one-stage extraction."""@@ -306,6 +310,10 @@ class AutoTemporalGraph(AutoGraph[NodeSchema, EdgeSchema]):graph_list = self.data_extractor.batch(inputs, config={"max_concurrency": self.max_workers})+ graph_list = self._filter_none_results(+ graph_list,+ default_factory=lambda: self.graph_schema(nodes=[], edges=[]),+ )return self.merge_batch_data(graph_list)