Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

[FIX] Executor Queue for Agentic extraction (#1893)

harini-venkataraman7983c98fc1b8
diff --git a/backend/prompt_studio/prompt_studio_core_v2/static/select_choices.json b/backend/prompt_studio/prompt_studio_core_v2/static/select_choices.json
index ea4fdfca2..1ba7fa8e4 100644
--- a/backend/prompt_studio/prompt_studio_core_v2/static/select_choices.json
+++ b/backend/prompt_studio/prompt_studio_core_v2/static/select_choices.json
@@ -14,7 +14,6 @@
"date":"date",
"boolean":"boolean",
"json":"json",
- "line_item":"line-item",
"table":"table"
},
"output_processing":{
diff --git a/backend/workflow_manager/workflow_v2/workflow_helper.py b/backend/workflow_manager/workflow_v2/workflow_helper.py
index 74662a65f..abe20703e 100644
--- a/backend/workflow_manager/workflow_v2/workflow_helper.py
+++ b/backend/workflow_manager/workflow_v2/workflow_helper.py
@@ -598,11 +598,7 @@ class WorkflowHelper:
@staticmethod
@celery_app.task(
name="async_execute_bin",
- autoretry_for=(Exception,),
max_retries=0,
- retry_backoff=True,
- retry_backoff_max=500,
- retry_jitter=True,
)
def execute_bin(
schema_name: str,
diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml
index 3d7777fe6..4dd8c3866 100644
--- a/docker/docker-compose.yaml
+++ b/docker/docker-compose.yaml
@@ -529,7 +529,7 @@ services:
- EXECUTOR_METRICS_PORT=8088
- HEALTH_PORT=8088
# Configurable Celery options
- - CELERY_QUEUES_EXECUTOR=${CELERY_QUEUES_EXECUTOR:-celery_executor_legacy}
+ - CELERY_QUEUES_EXECUTOR=${CELERY_QUEUES_EXECUTOR:-celery_executor_legacy,celery_executor_agentic}
- CELERY_POOL=${WORKER_EXECUTOR_POOL:-prefork}
- CELERY_PREFETCH_MULTIPLIER=${WORKER_EXECUTOR_PREFETCH_MULTIPLIER:-1}
- CELERY_CONCURRENCY=${WORKER_EXECUTOR_CONCURRENCY:-2}
diff --git a/workers/executor/executors/legacy_executor.py b/workers/executor/executors/legacy_executor.py
index 6a2c1be67..cf33c4321 100644
--- a/workers/executor/executors/legacy_executor.py
+++ b/workers/executor/executors/legacy_executor.py
@@ -1059,9 +1059,7 @@ class LegacyExecutor(BaseExecutor):
) -> dict[str, Any]:
"""Replace 'NA' strings with None in structured output."""
for k, v in structured_output.items():
- if isinstance(v, str) and v.lower() == "na":
- structured_output[k] = None
- elif isinstance(v, list):
+ if isinstance(v, list):
for i, item in enumerate(v):
if isinstance(item, str) and item.lower() == "na":
v[i] = None
@@ -1698,16 +1696,19 @@ class LegacyExecutor(BaseExecutor):
)
elif output_type == PSKeys.EMAIL:
- email_prompt = (
- f"Extract the email from the following text:\n{answer}"
- f"\n\nOutput just the email. "
- f"The email should be directly assignable to a string "
- f"variable. No explanation is required. If you cannot "
- f'extract the email, output "NA".'
- )
- structured_output[prompt_name] = LegacyExecutor._convert_scalar_answer(
- answer, llm, answer_prompt_svc, email_prompt
- )
+ if answer.lower() == "na":
+ structured_output[prompt_name] = answer
+ else:
+ email_prompt = (
+ f"Extract the email from the following text:\n{answer}"
+ f"\n\nOutput just the email. "
+ f"The email should be directly assignable to a string "
+ f"variable. No explanation is required. If you cannot "
+ f'extract the email, output "NA".'
+ )
+ structured_output[prompt_name] = answer_prompt_svc.run_completion(
+ llm=llm, prompt=email_prompt
+ )
elif output_type == PSKeys.DATE:
date_prompt = (
diff --git a/workers/executor/worker.py b/workers/executor/worker.py
index ecef4e687..9b92466aa 100644
--- a/workers/executor/worker.py
+++ b/workers/executor/worker.py
@@ -5,6 +5,7 @@ Routes execute_extraction tasks to registered executors.
"""
import logging
+import os
from shared.enums.worker_enums import WorkerType
from shared.infrastructure.config.builder import WorkerBuilder
@@ -43,7 +44,9 @@ def check_executor_health():
"worker_type": "executor",
"registered_executors": executors,
"executor_count": len(executors),
- "queues": ["celery_executor_legacy"],
+ "queues": os.environ.get(
+ "CELERY_QUEUES_EXECUTOR", "celery_executor_legacy"
+ ).split(","),
},
)
diff --git a/workers/shared/enums/worker_enums_base.py b/workers/shared/enums/worker_enums_base.py
index 215aaa75d..9c4dfbdb4 100644
--- a/workers/shared/enums/worker_enums_base.py
+++ b/workers/shared/enums/worker_enums_base.py
@@ -155,6 +155,7 @@ class QueueName(str, Enum):
# The dispatcher derives queue names as ``celery_executor_{executor_name}``.
# The "legacy" executor is the default OSS executor.
EXECUTOR = "celery_executor_legacy"
+ EXECUTOR_AGENTIC = "celery_executor_agentic"
# IDE callback queue (prompt studio post-execution callbacks)
IDE_CALLBACK = "ide_callback"
diff --git a/workers/shared/infrastructure/config/registry.py b/workers/shared/infrastructure/config/registry.py
index b1967ba62..bfdfcf2e5 100644
--- a/workers/shared/infrastructure/config/registry.py
+++ b/workers/shared/infrastructure/config/registry.py
@@ -66,6 +66,7 @@ class WorkerRegistry:
),
WorkerType.EXECUTOR: WorkerQueueConfig(
primary_queue=QueueName.EXECUTOR,
+ additional_queues=[QueueName.EXECUTOR_AGENTIC],
),
WorkerType.IDE_CALLBACK: WorkerQueueConfig(
primary_queue=QueueName.IDE_CALLBACK,
diff --git a/workers/tests/test_answer_prompt.py b/workers/tests/test_answer_prompt.py
index ebe1675d3..af539318d 100644
--- a/workers/tests/test_answer_prompt.py
+++ b/workers/tests/test_answer_prompt.py
@@ -562,8 +562,8 @@ class TestHandleAnswerPromptRetrieval:
)
result = executor._handle_answer_prompt(ctx)
- # Answer stays "NA" which gets sanitized to None
- assert result.data[PSKeys.OUTPUT]["field_a"] is None
+ # Answer stays "NA" (top-level NA is preserved, not sanitized)
+ assert result.data[PSKeys.OUTPUT]["field_a"] == "NA"
class TestHandleAnswerPromptMultiPrompt:
@@ -687,21 +687,21 @@ class TestHandleAnswerPromptMetrics:
class TestNullSanitization:
"""Tests for _sanitize_null_values."""
- def test_na_string_becomes_none(self):
- """Top-level 'NA' string → None."""
+ def test_na_string_preserved(self):
+ """Top-level 'NA' string is preserved (not sanitized to None)."""
from executor.executors.legacy_executor import LegacyExecutor
output = {"field": "NA"}
result = LegacyExecutor._sanitize_null_values(output)
- assert result["field"] is None
+ assert result["field"] == "NA"
- def test_na_case_insensitive(self):
- """'na' (lowercase) → None."""
+ def test_na_case_insensitive_preserved(self):
+ """Top-level 'na' (lowercase) is preserved (not sanitized to None)."""
from executor.executors.legacy_executor import LegacyExecutor
output = {"field": "na"}
result = LegacyExecutor._sanitize_null_values(output)
- assert result["field"] is None
+ assert result["field"] == "na"
def test_nested_list_na(self):
"""NA in nested list items → None."""