Skip to content

Hypergraph is more powerful. Transform unstructured text into structured knowledge with LLMs. Graphs, hypergraphs, and spatio-temporal extractions — with one command.

Revert "Merge pull request #28 from li5435945-ship-it/fix/batch-none-handling"

SLca2888579c2d
diff --git a/hyperextract/types/base.py b/hyperextract/types/base.py
index 69a94d6..d43bc1f 100644
--- a/hyperextract/types/base.py
+++ b/hyperextract/types/base.py
@@ -290,9 +290,6 @@ class BaseAutoType(ABC, Generic[T]):
self._summarize_extracted(result),
)
- # Filter out None results from failed LLM extractions
- extracted_data_list = self._filter_none_results(extracted_data_list)
-
logger.debug("stage=merge_start num_items=%d", len(extracted_data_list))
merged_data = self.merge_batch_data(extracted_data_list)
logger.debug(
@@ -301,36 +298,6 @@ class BaseAutoType(ABC, Generic[T]):
)
return merged_data
- def _filter_none_results(self, results: list, default_factory=None) -> list:
- """Replace None results from batch LLM extractions with empty defaults.
-
- When the LLM fails to parse output (e.g., context length exceeded,
- json_schema validation failure), batch() returns None for that item.
- This method replaces None with empty objects to maintain index alignment
- with the original chunks list.
-
- Args:
- results: List of extraction results, may contain None values.
- default_factory: Callable that creates a default empty object.
- If None, None values are simply removed (legacy behavior).
-
- Returns:
- List with None values replaced or removed.
- """
- if not results:
- return results
- none_count = sum(1 for r in results if r is None)
- if none_count > 0:
- logger.warning(
- "stage=batch_filter none_results_detected none_count=%d total=%d",
- none_count,
- len(results),
- )
- if default_factory is not None:
- return [r if r is not None else default_factory() for r in results]
- return [r for r in results if r is not None]
- return results
-
def _summarize_extracted(self, data: T) -> str:
"""Return a concise summary of extracted data for debug logging."""
try:
diff --git a/hyperextract/types/graph.py b/hyperextract/types/graph.py
index aab2345..afecdeb 100644
--- a/hyperextract/types/graph.py
+++ b/hyperextract/types/graph.py
@@ -492,10 +492,6 @@ class AutoGraph(
graph_list = self.data_extractor.batch(
inputs, config={"max_concurrency": self.max_workers}
)
- graph_list = self._filter_none_results(
- graph_list,
- default_factory=lambda: self.graph_schema(nodes=[], edges=[]),
- )
logger.debug("stage=one_stage_batch_complete graphs=%d", len(graph_list))
logger.debug("stage=one_stage_merge_start")
@@ -582,13 +578,9 @@ class AutoGraph(
List of NodeListSchema objects with extracted nodes.
"""
inputs = [{"source_text": chunk} for chunk in chunks]
- results = self.node_extractor.batch(
+ return self.node_extractor.batch(
inputs, config={"max_concurrency": self.max_workers}
)
- return self._filter_none_results(
- results,
- default_factory=lambda: self.node_list_schema(items=[]),
- )
def _extract_edges_batch(
self, chunks: List[str], node_lists: List[NodeListSchema[NodeSchema]]
@@ -613,13 +605,9 @@ class AutoGraph(
inputs.append({"source_text": chunk, "known_nodes": known_nodes})
- results = self.edge_extractor.batch(
+ return self.edge_extractor.batch(
inputs, config={"max_concurrency": self.max_workers}
)
- return self._filter_none_results(
- results,
- default_factory=lambda: self.edge_list_schema(items=[]),
- )
def _prune_dangling_edges(
self, graph: AutoGraphSchema[NodeSchema, EdgeSchema]
@@ -684,11 +672,6 @@ class AutoGraph(
Returns:
Merged graph.
"""
- # Handle empty input (all batch results were None/filtered out)
- if not data_list_or_tuple:
- logger.warning("stage=merge_batch_empty input_is_empty")
- return self.graph_schema(nodes=[], edges=[])
-
logger.debug(
"stage=merge_batch_start input_type=%s",
"tuple"
@@ -708,20 +691,12 @@ class AutoGraph(
"Invalid input format for batch merging"
)
nodes_lists, edges_lists = data_list_or_tuple[0], data_list_or_tuple[1]
-
- # Handle empty nodes/edges lists
- if not nodes_lists and not edges_lists:
- logger.warning("stage=merge_batch_empty_tuple nodes_and_edges_empty")
- return self.graph_schema(nodes=[], edges=[])
-
- if nodes_lists:
- assert isinstance(nodes_lists[0][0], self.node_schema), (
- "Invalid node list format for batch merging"
- )
- if edges_lists:
- assert isinstance(edges_lists[0][0], self.edge_schema), (
- "Invalid edge list format for batch merging"
- )
+ assert isinstance(nodes_lists[0][0], self.node_schema), (
+ "Invalid node list format for batch merging"
+ )
+ assert isinstance(edges_lists[0][0], self.edge_schema), (
+ "Invalid edge list format for batch merging"
+ )
all_nodes, all_edges = [], []
for node_list, edge_list in zip(nodes_lists, edges_lists):
diff --git a/hyperextract/types/hypergraph.py b/hyperextract/types/hypergraph.py
index 53dea78..73bc40a 100644
--- a/hyperextract/types/hypergraph.py
+++ b/hyperextract/types/hypergraph.py
@@ -460,10 +460,6 @@ class AutoHypergraph(
graph_list = self.data_extractor.batch(
inputs, config={"max_concurrency": self.max_workers}
)
- graph_list = self._filter_none_results(
- graph_list,
- default_factory=lambda: self.graph_schema(nodes=[], edges=[]),
- )
# Merge multiple hypergraphs
return self.merge_batch_data(graph_list)
@@ -506,13 +502,9 @@ class AutoHypergraph(
) -> List[NodeListSchema[NodeSchema]]:
"""Batch extract nodes from multiple text chunks."""
inputs = [{"source_text": chunk} for chunk in chunks]
- results = self.node_extractor.batch(
+ return self.node_extractor.batch(
inputs, config={"max_concurrency": self.max_workers}
)
- return self._filter_none_results(
- results,
- default_factory=lambda: self.node_list_schema(items=[]),
- )
def _extract_edges_batch(
self, chunks: List[str], node_lists: List[NodeListSchema[NodeSchema]]
@@ -529,13 +521,9 @@ class AutoHypergraph(
inputs.append({"source_text": chunk, "known_nodes": known_nodes})
- results = self.edge_extractor.batch(
+ return self.edge_extractor.batch(
inputs, config={"max_concurrency": self.max_workers}
)
- return self._filter_none_results(
- results,
- default_factory=lambda: self.edge_list_schema(items=[]),
- )
def _prune_dangling_edges(
self, graph: AutoHypergraphSchema
@@ -605,11 +593,6 @@ class AutoHypergraph(
Returns:
Merged hypergraph.
"""
- # Handle empty input (all batch results were None/filtered out)
- if not data_list_or_tuple:
- logger.warning("stage=merge_batch_empty input_is_empty")
- return self.graph_schema(nodes=[], edges=[])
-
if isinstance(data_list_or_tuple, list) and isinstance(
data_list_or_tuple[0], self.graph_schema
):
@@ -625,20 +608,12 @@ class AutoHypergraph(
"Invalid input format for batch merging"
)
nodes_lists, edges_lists = data_list_or_tuple[0], data_list_or_tuple[1]
-
- # Handle empty nodes/edges lists
- if not nodes_lists and not edges_lists:
- logger.warning("stage=merge_batch_empty_tuple nodes_and_edges_empty")
- return self.graph_schema(nodes=[], edges=[])
-
- if nodes_lists:
- assert isinstance(nodes_lists[0][0], self.node_schema), (
- "Invalid node list format for batch merging"
- )
- if edges_lists:
- assert isinstance(edges_lists[0][0], self.edge_schema), (
- "Invalid edge list format for batch merging"
- )
+ assert isinstance(nodes_lists[0][0], self.node_schema), (
+ "Invalid node list format for batch merging"
+ )
+ assert isinstance(edges_lists[0][0], self.edge_schema), (
+ "Invalid edge list format for batch merging"
+ )
all_nodes, all_edges = [], []
for node_list, edge_list in zip(nodes_lists, edges_lists):
diff --git a/hyperextract/types/spatial_graph.py b/hyperextract/types/spatial_graph.py
index b2f8a82..854fb15 100644
--- a/hyperextract/types/spatial_graph.py
+++ b/hyperextract/types/spatial_graph.py
@@ -269,13 +269,9 @@ class AutoSpatialGraph(AutoGraph[NodeSchema, EdgeSchema]):
}
)
- results = self.edge_extractor.batch(
+ return self.edge_extractor.batch(
inputs, config={"max_concurrency": self.max_workers}
)
- return self._filter_none_results(
- results,
- default_factory=lambda: self.edge_list_schema(items=[]),
- )
def _extract_data_by_one_stage(self, text: str) -> Any:
"""Override: Inject observation_location into one-stage extraction."""
@@ -299,10 +295,6 @@ class AutoSpatialGraph(AutoGraph[NodeSchema, EdgeSchema]):
graph_list = self.data_extractor.batch(
inputs, config={"max_concurrency": self.max_workers}
)
- graph_list = self._filter_none_results(
- graph_list,
- default_factory=lambda: self.graph_schema(nodes=[], edges=[]),
- )
return self.merge_batch_data(graph_list)
diff --git a/hyperextract/types/spatio_temporal_graph.py b/hyperextract/types/spatio_temporal_graph.py
index fc7ac91..78c1f37 100644
--- a/hyperextract/types/spatio_temporal_graph.py
+++ b/hyperextract/types/spatio_temporal_graph.py
@@ -286,13 +286,9 @@ class AutoSpatioTemporalGraph(AutoGraph[NodeSchema, EdgeSchema]):
}
)
- results = self.edge_extractor.batch(
+ return self.edge_extractor.batch(
inputs, config={"max_concurrency": self.max_workers}
)
- return self._filter_none_results(
- results,
- default_factory=lambda: self.edge_list_schema(items=[]),
- )
def _extract_data_by_one_stage(self, text: str) -> Any:
"""Inject observation_time and observation_location into one-stage extraction."""
@@ -318,10 +314,6 @@ class AutoSpatioTemporalGraph(AutoGraph[NodeSchema, EdgeSchema]):
graph_list = self.data_extractor.batch(
inputs, config={"max_concurrency": self.max_workers}
)
- graph_list = self._filter_none_results(
- graph_list,
- default_factory=lambda: self.graph_schema(nodes=[], edges=[]),
- )
return self.merge_batch_data(graph_list)
diff --git a/hyperextract/types/temporal_graph.py b/hyperextract/types/temporal_graph.py
index 41adac9..3155254 100644
--- a/hyperextract/types/temporal_graph.py
+++ b/hyperextract/types/temporal_graph.py
@@ -286,13 +286,9 @@ class AutoTemporalGraph(AutoGraph[NodeSchema, EdgeSchema]):
}
)
- results = self.edge_extractor.batch(
+ return self.edge_extractor.batch(
inputs, config={"max_concurrency": self.max_workers}
)
- return self._filter_none_results(
- results,
- default_factory=lambda: self.edge_list_schema(items=[]),
- )
def _extract_data_by_one_stage(self, text: str) -> Any:
"""Override: Inject observation_time into one-stage extraction."""
@@ -310,10 +306,6 @@ class AutoTemporalGraph(AutoGraph[NodeSchema, EdgeSchema]):
graph_list = self.data_extractor.batch(
inputs, config={"max_concurrency": self.max_workers}
)
- graph_list = self._filter_none_results(
- graph_list,
- default_factory=lambda: self.graph_schema(nodes=[], edges=[]),
- )
return self.merge_batch_data(graph_list)