LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
UN-2946 [MISC] Flush embedding usage records on indexing path (#1962)
Chandrasekharan M993f6761134f
diff --git a/unstract/sdk1/src/unstract/sdk1/embedding.py b/unstract/sdk1/src/unstract/sdk1/embedding.pyindex 4e30c6201..109f2cd6a 100644--- a/unstract/sdk1/src/unstract/sdk1/embedding.py+++ b/unstract/sdk1/src/unstract/sdk1/embedding.py@@ -281,3 +281,22 @@ class EmbeddingCompat(BaseEmbedding):def test_connection(self) -> bool:return self._embedding_instance.test_connection()++ def flush_pending_usage(self) -> list[dict]:+ """Drain pending usage rows from registered callback handlers."""+ if not self.callback_manager:+ return []+ records: list[dict] = []+ for handler in self.callback_manager.handlers:+ if not hasattr(handler, "flush_pending_usage"):+ continue+ # Per-handler guard so one bad handler doesn't drop the rest.+ try:+ records.extend(handler.flush_pending_usage())+ except Exception:+ logger.warning(+ "Failed to flush usage from embedding handler %s",+ type(handler).__name__,+ exc_info=True,+ )+ return recordsdiff --git a/workers/executor/executors/legacy_executor.py b/workers/executor/executors/legacy_executor.pyindex f319e1d24..cfd98ccec 100644--- a/workers/executor/executors/legacy_executor.py+++ b/workers/executor/executors/legacy_executor.py@@ -659,13 +659,15 @@ class LegacyExecutor(BaseExecutor):elif not is_single_pass:# ---- Step 3: Index per output with dedup ----step += 1- index_metrics = self._run_pipeline_index(+ index_metrics, index_records = self._run_pipeline_index(context=context,index_template=index_template,answer_params=answer_params,extracted_text=extracted_text,usage_kwargs=extract_params.get("usage_kwargs", {}),)+ if index_records:+ pipeline_records.extend(index_records)# ---- Step 4: Table settings injection ----if not is_single_pass:@@ -891,99 +893,132 @@ class LegacyExecutor(BaseExecutor):answer_params: dict,extracted_text: str,usage_kwargs: dict | None = None,- ) -> dict:+ ) -> tuple[dict, list[dict]]:"""Run per-output indexing with dedup for the structure pipeline.Args:usage_kwargs: Audit-tracking kwargs (``run_id``,``execution_id``, ``file_name``) propagated to theembedding adapter so its callback can record usage- rows against the correct file_execution_id. Without- this, embedding usage is missing from the API- deployment response metadata.+ rows against the correct file_execution_id.Returns:- Dict of index metrics keyed by output name.+ (index_metrics, usage_records) — metrics keyed by output+ name and the flat list of usage rows collected across all+ child ``_handle_index`` calls."""- import datetime-tool_settings = answer_params.get("tool_settings", {})outputs = answer_params.get("outputs", [])- tool_id = index_template.get("tool_id", "")- file_hash = index_template.get("file_hash", "")- is_highlight = index_template.get("is_highlight_enabled", False)- platform_api_key = index_template.get("platform_api_key", "")- extracted_file_path = index_template.get("extracted_file_path", "")- usage_kwargs = usage_kwargs or {}-index_metrics: dict = {}+ index_records: list[dict] = []seen_params: set = set()for output in outputs:- chunk_size = output.get("chunk-size", 0)- chunk_overlap = output.get("chunk-overlap", 0)- vector_db = tool_settings.get("vector-db", "")- embedding = tool_settings.get("embedding", "")- x2text = tool_settings.get("x2text_adapter", "")-- param_key = (- f"chunk_size={chunk_size}_"- f"chunk_overlap={chunk_overlap}_"- f"vector_db={vector_db}_"- f"embedding={embedding}_"- f"x2text={x2text}"+ self._index_pipeline_output(+ context=context,+ output=output,+ index_template=index_template,+ tool_settings=tool_settings,+ extracted_text=extracted_text,+ usage_kwargs=usage_kwargs or {},+ seen_params=seen_params,+ index_metrics=index_metrics,+ index_records=index_records,)- if chunk_size != 0 and param_key not in seen_params:- seen_params.add(param_key)+ return index_metrics, index_records- indexing_start = datetime.datetime.now()- logger.info(- "Pipeline indexing: chunk_size=%s chunk_overlap=%s vector_db=%s",- chunk_size,- chunk_overlap,- vector_db,- )+ def _index_pipeline_output(+ self,+ *,+ context: ExecutionContext,+ output: dict,+ index_template: dict,+ tool_settings: dict,+ extracted_text: str,+ usage_kwargs: dict,+ seen_params: set,+ index_metrics: dict,+ index_records: list[dict],+ ) -> None:+ """Index a single structure-pipeline output entry in-place."""+ import datetime- index_ctx = ExecutionContext(- executor_name=context.executor_name,- operation=Operation.INDEX.value,- run_id=context.run_id,- execution_source=context.execution_source,- organization_id=context.organization_id,- request_id=context.request_id,- log_events_id=context.log_events_id,- execution_id=context.execution_id,- file_execution_id=context.file_execution_id,- executor_params={- "embedding_instance_id": embedding,- "vector_db_instance_id": vector_db,- "x2text_instance_id": x2text,- "chunk_size": chunk_size,- "chunk_overlap": chunk_overlap,- "file_path": extracted_file_path,- "reindex": True,- "tool_id": tool_id,- "file_hash": file_hash,- "enable_highlight": is_highlight,- "extracted_text": extracted_text,- "platform_api_key": platform_api_key,- "usage_kwargs": usage_kwargs,- },- )- index_result = self._handle_index(index_ctx)- if not index_result.success:- logger.warning(- "Pipeline indexing failed for %s: %s",- param_key,- index_result.error,- )+ chunk_size = output.get("chunk-size", 0)+ if chunk_size == 0:+ return- elapsed = (datetime.datetime.now() - indexing_start).total_seconds()- output_name = output.get("name", "")- index_metrics[output_name] = {"indexing": {"time_taken(s)": elapsed}}+ chunk_overlap = output.get("chunk-overlap", 0)+ vector_db = tool_settings.get("vector-db", "")+ embedding = tool_settings.get("embedding", "")+ x2text = tool_settings.get("x2text_adapter", "")++ param_key = (+ f"chunk_size={chunk_size}_"+ f"chunk_overlap={chunk_overlap}_"+ f"vector_db={vector_db}_"+ f"embedding={embedding}_"+ f"x2text={x2text}"+ )+ if param_key in seen_params:+ return+ seen_params.add(param_key)- return index_metrics+ indexing_start = datetime.datetime.now()+ logger.info(+ "Pipeline indexing: chunk_size=%s chunk_overlap=%s vector_db=%s",+ chunk_size,+ chunk_overlap,+ vector_db,+ )++ index_ctx = ExecutionContext(+ executor_name=context.executor_name,+ operation=Operation.INDEX.value,+ run_id=context.run_id,+ execution_source=context.execution_source,+ organization_id=context.organization_id,+ request_id=context.request_id,+ log_events_id=context.log_events_id,+ execution_id=context.execution_id,+ file_execution_id=context.file_execution_id,+ executor_params={+ "embedding_instance_id": embedding,+ "vector_db_instance_id": vector_db,+ "x2text_instance_id": x2text,+ "chunk_size": chunk_size,+ "chunk_overlap": chunk_overlap,+ "file_path": index_template.get("extracted_file_path", ""),+ "reindex": True,+ "tool_id": index_template.get("tool_id", ""),+ "file_hash": index_template.get("file_hash", ""),+ "enable_highlight": index_template.get("is_highlight_enabled", False),+ "extracted_text": extracted_text,+ "platform_api_key": index_template.get("platform_api_key", ""),+ "usage_kwargs": usage_kwargs,+ },+ )+ try:+ index_result = self._handle_index(index_ctx)+ except LegacyExecutorError as e:+ # Preserve usage rows accrued from prior iterations.+ e.partial_usage_records = index_records + e.partial_usage_records+ raise+ if not index_result.success:+ # Abort on returned-failure so downstream steps don't run+ # against an incomplete vector store.+ raise LegacyExecutorError(+ message=f"Pipeline indexing failed for {param_key}: {index_result.error}",+ code=500,+ partial_usage_records=list(index_records),+ )+ child_records = (index_result.metadata or {}).get("usage_records") or []+ if child_records:+ index_records.extend(child_records)++ elapsed = (datetime.datetime.now() - indexing_start).total_seconds()+ output_name = output.get("name", "")+ index_metrics[output_name] = {"indexing": {"time_taken(s)": elapsed}}@staticmethoddef _merge_pipeline_metrics(metrics1: dict, metrics2: dict) -> dict:@@ -1008,6 +1043,23 @@ class LegacyExecutor(BaseExecutor):# Phase 2C — Index handler# ------------------------------------------------------------------+ @staticmethod+ def _missing_index_params(+ *,+ embedding_instance_id: str,+ vector_db_instance_id: str,+ x2text_instance_id: str,+ file_path: str,+ ) -> list[str]:+ """Return required-param keys that are unset for an INDEX op."""+ checks = (+ (embedding_instance_id, IKeys.EMBEDDING_INSTANCE_ID),+ (vector_db_instance_id, IKeys.VECTOR_DB_INSTANCE_ID),+ (x2text_instance_id, IKeys.X2TEXT_INSTANCE_ID),+ (file_path, IKeys.FILE_PATH),+ )+ return [key for value, key in checks if not value]+def _handle_index(self, context: ExecutionContext) -> ExecutionResult:"""Handle ``Operation.INDEX`` — vector DB indexing.@@ -1027,15 +1079,12 @@ class LegacyExecutor(BaseExecutor):extracted_text: str = params.get(IKeys.EXTRACTED_TEXT, "")platform_api_key: str = params.get("platform_api_key", "")- missing = []- if not embedding_instance_id:- missing.append(IKeys.EMBEDDING_INSTANCE_ID)- if not vector_db_instance_id:- missing.append(IKeys.VECTOR_DB_INSTANCE_ID)- if not x2text_instance_id:- missing.append(IKeys.X2TEXT_INSTANCE_ID)- if not file_path:- missing.append(IKeys.FILE_PATH)+ missing = self._missing_index_params(+ embedding_instance_id=embedding_instance_id,+ vector_db_instance_id=vector_db_instance_id,+ x2text_instance_id=x2text_instance_id,+ file_path=file_path,+ )if missing:return ExecutionResult.failure(error=f"Missing required params: {', '.join(missing)}"@@ -1110,6 +1159,7 @@ class LegacyExecutor(BaseExecutor):index_cls, embedding_compat, vector_db_cls = self._get_indexing_deps()vector_db = None+ embedding = Nonetry:index = index_cls(tool=shim,@@ -1152,7 +1202,11 @@ class LegacyExecutor(BaseExecutor):"Skipping re-index: doc_id=%s already in vector DB and reindex=False",doc_id,)- return ExecutionResult(success=True, data={IKeys.DOC_ID: doc_id})+ return ExecutionResult(+ success=True,+ data={IKeys.DOC_ID: doc_id},+ metadata={"usage_records": embedding.flush_pending_usage()},+ )shim.stream_log("Re-indexing document" if doc_id_found else "Indexing document"@@ -1169,7 +1223,11 @@ class LegacyExecutor(BaseExecutor):Path(file_path).name,)shim.stream_log("Document indexing completed")- return ExecutionResult(success=True, data={IKeys.DOC_ID: doc_id})+ return ExecutionResult(+ success=True,+ data={IKeys.DOC_ID: doc_id},+ metadata={"usage_records": embedding.flush_pending_usage()},+ )except Exception as e:logger.error("Indexing failed: file=%s error=%s",@@ -1177,8 +1235,19 @@ class LegacyExecutor(BaseExecutor):str(e),)status_code = getattr(e, "status_code", 500)+ partial = []+ if embedding is not None:+ try:+ partial = list(embedding.flush_pending_usage())+ except Exception:+ logger.warning(+ "Failed to flush embedding usage during indexing error path",+ exc_info=True,+ )raise LegacyExecutorError(- message=f"Error while indexing: {e}", code=status_code+ message=f"Error while indexing: {e}",+ code=status_code,+ partial_usage_records=partial,) from efinally:if vector_db is not None:diff --git a/workers/tests/test_phase5d.py b/workers/tests/test_phase5d.pyindex 0a0489b5d..8b3252c9e 100644--- a/workers/tests/test_phase5d.py+++ b/workers/tests/test_phase5d.py@@ -283,9 +283,10 @@ class TestNormalPipeline:)# Simulate index metrics by patching _run_pipeline_indexexecutor._run_pipeline_index = MagicMock(- return_value={- "field_a": {"indexing": {"time_taken(s)": 0.5}},- }+ return_value=(+ {"field_a": {"indexing": {"time_taken(s)": 0.5}}},+ [],+ ))ctx = _make_pipeline_context({@@ -303,6 +304,89 @@ class TestNormalPipeline:assert "indexing" in metrics["field_a"]+# ---------------------------------------------------------------------------+# Tests — Embedding usage record propagation+# ---------------------------------------------------------------------------+++class TestPipelineUsageRecordsPropagation:+ """Embedding usage rows surface in the pipeline result.++ Exercises the real _run_pipeline_index → _index_pipeline_output →+ _handle_index chain so the usage-record carrier wiring is verified+ end-to-end rather than stubbed at the top of the chain.+ """++ def test_index_usage_records_propagate(self, executor):+ executor._handle_extract = MagicMock(+ return_value=ExecutionResult(+ success=True, data={"extracted_text": "text"}+ )+ )+ embedding_row = {+ "model_name": "emb-1",+ "embedding_tokens": 42,+ "cost_in_dollars": 0.001,+ }+ executor._handle_index = MagicMock(+ return_value=ExecutionResult(+ success=True,+ data={"doc_id": "doc-1"},+ metadata={"usage_records": [embedding_row]},+ )+ )+ executor._handle_answer_prompt = MagicMock(+ return_value=ExecutionResult(+ success=True,+ data={+ "output": {"field_a": "v"},+ "metadata": {},+ "metrics": {"field_a": {}},+ },+ )+ )++ ctx = _make_pipeline_context({+ "extract_params": _base_extract_params(),+ "index_template": _base_index_template(),+ "answer_params": _base_answer_params(),+ "pipeline_options": _base_pipeline_options(),+ })+ result = executor._handle_structure_pipeline(ctx)++ assert result.success+ usage_records = (result.metadata or {}).get("usage_records", [])+ assert embedding_row in usage_records++ def test_index_returned_failure_aborts_pipeline(self, executor):+ from executor.executors.exceptions import LegacyExecutorError++ executor._handle_extract = MagicMock(+ return_value=ExecutionResult(+ success=True, data={"extracted_text": "text"}+ )+ )+ executor._handle_index = MagicMock(+ return_value=ExecutionResult.failure(+ error="Missing required params: embedding_instance_id"+ )+ )+ executor._handle_answer_prompt = MagicMock()++ ctx = _make_pipeline_context({+ "extract_params": _base_extract_params(),+ "index_template": _base_index_template(),+ "answer_params": _base_answer_params(),+ "pipeline_options": _base_pipeline_options(),+ })+ with pytest.raises(LegacyExecutorError) as exc_info:+ executor._handle_structure_pipeline(ctx)++ assert "Pipeline indexing failed" in exc_info.value.message+ # Pipeline must short-circuit before answering.+ assert executor._handle_answer_prompt.call_count == 0++# ---------------------------------------------------------------------------# Tests — Extract failure propagation# ---------------------------------------------------------------------------