LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
[FIX] Executor Queue for Agentic extraction (#1893)
harini-venkataraman7983c98fc1b8
diff --git a/backend/prompt_studio/prompt_studio_core_v2/static/select_choices.json b/backend/prompt_studio/prompt_studio_core_v2/static/select_choices.jsonindex ea4fdfca2..1ba7fa8e4 100644--- a/backend/prompt_studio/prompt_studio_core_v2/static/select_choices.json+++ b/backend/prompt_studio/prompt_studio_core_v2/static/select_choices.json@@ -14,7 +14,6 @@"date":"date","boolean":"boolean","json":"json",- "line_item":"line-item","table":"table"},"output_processing":{diff --git a/backend/workflow_manager/workflow_v2/workflow_helper.py b/backend/workflow_manager/workflow_v2/workflow_helper.pyindex 74662a65f..abe20703e 100644--- a/backend/workflow_manager/workflow_v2/workflow_helper.py+++ b/backend/workflow_manager/workflow_v2/workflow_helper.py@@ -598,11 +598,7 @@ class WorkflowHelper:@staticmethod@celery_app.task(name="async_execute_bin",- autoretry_for=(Exception,),max_retries=0,- retry_backoff=True,- retry_backoff_max=500,- retry_jitter=True,)def execute_bin(schema_name: str,diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yamlindex 3d7777fe6..4dd8c3866 100644--- a/docker/docker-compose.yaml+++ b/docker/docker-compose.yaml@@ -529,7 +529,7 @@ services:- EXECUTOR_METRICS_PORT=8088- HEALTH_PORT=8088# Configurable Celery options- - CELERY_QUEUES_EXECUTOR=${CELERY_QUEUES_EXECUTOR:-celery_executor_legacy}+ - CELERY_QUEUES_EXECUTOR=${CELERY_QUEUES_EXECUTOR:-celery_executor_legacy,celery_executor_agentic}- CELERY_POOL=${WORKER_EXECUTOR_POOL:-prefork}- CELERY_PREFETCH_MULTIPLIER=${WORKER_EXECUTOR_PREFETCH_MULTIPLIER:-1}- CELERY_CONCURRENCY=${WORKER_EXECUTOR_CONCURRENCY:-2}diff --git a/workers/executor/executors/legacy_executor.py b/workers/executor/executors/legacy_executor.pyindex 6a2c1be67..cf33c4321 100644--- a/workers/executor/executors/legacy_executor.py+++ b/workers/executor/executors/legacy_executor.py@@ -1059,9 +1059,7 @@ class LegacyExecutor(BaseExecutor):) -> dict[str, Any]:"""Replace 'NA' strings with None in structured output."""for k, v in structured_output.items():- if isinstance(v, str) and v.lower() == "na":- structured_output[k] = None- elif isinstance(v, list):+ if isinstance(v, list):for i, item in enumerate(v):if isinstance(item, str) and item.lower() == "na":v[i] = None@@ -1698,16 +1696,19 @@ class LegacyExecutor(BaseExecutor):)elif output_type == PSKeys.EMAIL:- email_prompt = (- f"Extract the email from the following text:\n{answer}"- f"\n\nOutput just the email. "- f"The email should be directly assignable to a string "- f"variable. No explanation is required. If you cannot "- f'extract the email, output "NA".'- )- structured_output[prompt_name] = LegacyExecutor._convert_scalar_answer(- answer, llm, answer_prompt_svc, email_prompt- )+ if answer.lower() == "na":+ structured_output[prompt_name] = answer+ else:+ email_prompt = (+ f"Extract the email from the following text:\n{answer}"+ f"\n\nOutput just the email. "+ f"The email should be directly assignable to a string "+ f"variable. No explanation is required. If you cannot "+ f'extract the email, output "NA".'+ )+ structured_output[prompt_name] = answer_prompt_svc.run_completion(+ llm=llm, prompt=email_prompt+ )elif output_type == PSKeys.DATE:date_prompt = (diff --git a/workers/executor/worker.py b/workers/executor/worker.pyindex ecef4e687..9b92466aa 100644--- a/workers/executor/worker.py+++ b/workers/executor/worker.py@@ -5,6 +5,7 @@ Routes execute_extraction tasks to registered executors."""import logging+import osfrom shared.enums.worker_enums import WorkerTypefrom shared.infrastructure.config.builder import WorkerBuilder@@ -43,7 +44,9 @@ def check_executor_health():"worker_type": "executor","registered_executors": executors,"executor_count": len(executors),- "queues": ["celery_executor_legacy"],+ "queues": os.environ.get(+ "CELERY_QUEUES_EXECUTOR", "celery_executor_legacy"+ ).split(","),},)diff --git a/workers/shared/enums/worker_enums_base.py b/workers/shared/enums/worker_enums_base.pyindex 215aaa75d..9c4dfbdb4 100644--- a/workers/shared/enums/worker_enums_base.py+++ b/workers/shared/enums/worker_enums_base.py@@ -155,6 +155,7 @@ class QueueName(str, Enum):# The dispatcher derives queue names as ``celery_executor_{executor_name}``.# The "legacy" executor is the default OSS executor.EXECUTOR = "celery_executor_legacy"+ EXECUTOR_AGENTIC = "celery_executor_agentic"# IDE callback queue (prompt studio post-execution callbacks)IDE_CALLBACK = "ide_callback"diff --git a/workers/shared/infrastructure/config/registry.py b/workers/shared/infrastructure/config/registry.pyindex b1967ba62..bfdfcf2e5 100644--- a/workers/shared/infrastructure/config/registry.py+++ b/workers/shared/infrastructure/config/registry.py@@ -66,6 +66,7 @@ class WorkerRegistry:),WorkerType.EXECUTOR: WorkerQueueConfig(primary_queue=QueueName.EXECUTOR,+ additional_queues=[QueueName.EXECUTOR_AGENTIC],),WorkerType.IDE_CALLBACK: WorkerQueueConfig(primary_queue=QueueName.IDE_CALLBACK,diff --git a/workers/tests/test_answer_prompt.py b/workers/tests/test_answer_prompt.pyindex ebe1675d3..af539318d 100644--- a/workers/tests/test_answer_prompt.py+++ b/workers/tests/test_answer_prompt.py@@ -562,8 +562,8 @@ class TestHandleAnswerPromptRetrieval:)result = executor._handle_answer_prompt(ctx)- # Answer stays "NA" which gets sanitized to None- assert result.data[PSKeys.OUTPUT]["field_a"] is None+ # Answer stays "NA" (top-level NA is preserved, not sanitized)+ assert result.data[PSKeys.OUTPUT]["field_a"] == "NA"class TestHandleAnswerPromptMultiPrompt:@@ -687,21 +687,21 @@ class TestHandleAnswerPromptMetrics:class TestNullSanitization:"""Tests for _sanitize_null_values."""- def test_na_string_becomes_none(self):- """Top-level 'NA' string → None."""+ def test_na_string_preserved(self):+ """Top-level 'NA' string is preserved (not sanitized to None)."""from executor.executors.legacy_executor import LegacyExecutoroutput = {"field": "NA"}result = LegacyExecutor._sanitize_null_values(output)- assert result["field"] is None+ assert result["field"] == "NA"- def test_na_case_insensitive(self):- """'na' (lowercase) → None."""+ def test_na_case_insensitive_preserved(self):+ """Top-level 'na' (lowercase) is preserved (not sanitized to None)."""from executor.executors.legacy_executor import LegacyExecutoroutput = {"field": "na"}result = LegacyExecutor._sanitize_null_values(output)- assert result["field"] is None+ assert result["field"] == "na"def test_nested_list_na(self):"""NA in nested list items → None."""