Skip to content

Hypergraph is more powerful. Transform unstructured text into structured knowledge with LLMs. Graphs, hypergraphs, and spatio-temporal extractions — with one command.

feat(base): add debug logging for LLM chunk inputs and results

David Li6622ec928f70
diff --git a/hyperextract/types/base.py b/hyperextract/types/base.py
index 84ae85b..2493c67 100644
--- a/hyperextract/types/base.py
+++ b/hyperextract/types/base.py
@@ -250,24 +250,58 @@ class BaseAutoType(ABC, Generic[T]):
logger.debug("stage=extract_start input_chars=%d chunk_size=%d", len(text), self.chunk_size)
if len(text) <= self.chunk_size:
- logger.debug("stage=extract_single_chunk")
+ logger.debug("stage=extract_single_chunk chunk_text_preview=%s", text[:200])
extracted_data = self.data_extractor.invoke({"source_text": text})
+ logger.debug(
+ "stage=extract_single_chunk_result chunk=0 result_summary=%s",
+ self._summarize_extracted(extracted_data),
+ )
extracted_data_list = [extracted_data]
else:
chunks = self.text_splitter.split_text(text)
logger.debug("stage=text_split num_chunks=%d", len(chunks))
+ for i, chunk in enumerate(chunks):
+ logger.debug(
+ "stage=chunk_before_llm chunk_index=%d chunk_chars=%d chunk_text_preview=%s",
+ i, len(chunk), chunk[:200],
+ )
inputs = [{"source_text": chunk} for chunk in chunks]
- logger.debug("stage=llm_batch_start max_concurrency=%d", self.max_workers)
+ logger.debug("stage=llm_batch_start max_concurrency=%d num_inputs=%d", self.max_workers, len(inputs))
extracted_data_list = self.data_extractor.batch(
inputs, config={"max_concurrency": self.max_workers}
)
logger.debug("stage=llm_batch_complete results=%d", len(extracted_data_list))
+ for i, result in enumerate(extracted_data_list):
+ logger.debug(
+ "stage=chunk_llm_result chunk_index=%d result_summary=%s",
+ i, self._summarize_extracted(result),
+ )
logger.debug("stage=merge_start num_items=%d", len(extracted_data_list))
merged_data = self.merge_batch_data(extracted_data_list)
- logger.debug("stage=extract_complete")
+ logger.debug("stage=extract_complete merged_summary=%s", self._summarize_extracted(merged_data))
return merged_data
+ def _summarize_extracted(self, data: T) -> str:
+ """Return a concise summary of extracted data for debug logging."""
+ try:
+ dump = data.model_dump()
+ # Count entities and relations for graph-type schemas
+ entities = len(dump.get("entities", []))
+ relations = len(dump.get("relations", []))
+ if entities or relations:
+ return f"entities={entities} relations={relations}"
+ # Generic fallback: list top-level keys with their lengths
+ parts = []
+ for key, val in dump.items():
+ if isinstance(val, (list, tuple)):
+ parts.append(f"{key}={len(val)}")
+ elif isinstance(val, str):
+ parts.append(f"{key}={val[:50]!r}")
+ return ", ".join(parts) if parts else str(dump)[:100]
+ except Exception:
+ return repr(data)[:100]
+
def parse(self, text: str) -> "BaseAutoType[T]":
"""
Parses knowledge into a NEW instance without modifying the current one.