Hypergraph is more powerful. Transform unstructured text into structured knowledge with LLMs. Graphs, hypergraphs, and spatio-temporal extractions — with one command.
feat(base): add debug logging for LLM chunk inputs and results
David Li6622ec928f70
diff --git a/hyperextract/types/base.py b/hyperextract/types/base.pyindex 84ae85b..2493c67 100644--- a/hyperextract/types/base.py+++ b/hyperextract/types/base.py@@ -250,24 +250,58 @@ class BaseAutoType(ABC, Generic[T]):logger.debug("stage=extract_start input_chars=%d chunk_size=%d", len(text), self.chunk_size)if len(text) <= self.chunk_size:- logger.debug("stage=extract_single_chunk")+ logger.debug("stage=extract_single_chunk chunk_text_preview=%s", text[:200])extracted_data = self.data_extractor.invoke({"source_text": text})+ logger.debug(+ "stage=extract_single_chunk_result chunk=0 result_summary=%s",+ self._summarize_extracted(extracted_data),+ )extracted_data_list = [extracted_data]else:chunks = self.text_splitter.split_text(text)logger.debug("stage=text_split num_chunks=%d", len(chunks))+ for i, chunk in enumerate(chunks):+ logger.debug(+ "stage=chunk_before_llm chunk_index=%d chunk_chars=%d chunk_text_preview=%s",+ i, len(chunk), chunk[:200],+ )inputs = [{"source_text": chunk} for chunk in chunks]- logger.debug("stage=llm_batch_start max_concurrency=%d", self.max_workers)+ logger.debug("stage=llm_batch_start max_concurrency=%d num_inputs=%d", self.max_workers, len(inputs))extracted_data_list = self.data_extractor.batch(inputs, config={"max_concurrency": self.max_workers})logger.debug("stage=llm_batch_complete results=%d", len(extracted_data_list))+ for i, result in enumerate(extracted_data_list):+ logger.debug(+ "stage=chunk_llm_result chunk_index=%d result_summary=%s",+ i, self._summarize_extracted(result),+ )logger.debug("stage=merge_start num_items=%d", len(extracted_data_list))merged_data = self.merge_batch_data(extracted_data_list)- logger.debug("stage=extract_complete")+ logger.debug("stage=extract_complete merged_summary=%s", self._summarize_extracted(merged_data))return merged_data+ def _summarize_extracted(self, data: T) -> str:+ """Return a concise summary of extracted data for debug logging."""+ try:+ dump = data.model_dump()+ # Count entities and relations for graph-type schemas+ entities = len(dump.get("entities", []))+ relations = len(dump.get("relations", []))+ if entities or relations:+ return f"entities={entities} relations={relations}"+ # Generic fallback: list top-level keys with their lengths+ parts = []+ for key, val in dump.items():+ if isinstance(val, (list, tuple)):+ parts.append(f"{key}={len(val)}")+ elif isinstance(val, str):+ parts.append(f"{key}={val[:50]!r}")+ return ", ".join(parts) if parts else str(dump)[:100]+ except Exception:+ return repr(data)[:100]+def parse(self, text: str) -> "BaseAutoType[T]":"""Parses knowledge into a NEW instance without modifying the current one.