Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

[FIX] Use importlib.util.find_spec for pluggable worker discovery (#1923)

Chandrasekharan M139bc7593502
diff --git a/workers/shared/infrastructure/config/builder.py b/workers/shared/infrastructure/config/builder.py
index 2d9491177..859d823b9 100644
--- a/workers/shared/infrastructure/config/builder.py
+++ b/workers/shared/infrastructure/config/builder.py
@@ -323,27 +323,26 @@ class WorkerBuilder:
try:
import importlib
- from pathlib import Path
-
- # Check if the worker.py file exists
- # Path resolution: builder.py is at /app/workers/shared/infrastructure/config/
- # We need to get to /app/workers/, so go up 3 levels
- pluggable_worker_path = (
- Path(__file__).resolve().parents[3]
- / "pluggable_worker"
- / worker_type.value
- / "worker.py"
- )
+ import importlib.util
+
+ # Ask Python's import system whether the module is resolvable.
+ # find_spec() consults all registered finders and handles every
+ # module representation (source .py, Nuitka/Cython .so, .pyc,
+ # namespace packages, zipimports) — unlike a filesystem check
+ # for a specific file extension, which breaks for compiled plugins.
+ module_path = f"pluggable_worker.{worker_type.value}.worker"
- if not pluggable_worker_path.exists():
- logger.error(f"Pluggable worker file not found: {pluggable_worker_path}")
+ if importlib.util.find_spec(module_path) is None:
+ logger.error(f"Pluggable worker module not importable: {module_path}")
return False
- # Try to import the module to verify it's valid
- module_path = f"pluggable_worker.{worker_type.value}.worker"
+ # Load it to catch findable-but-broken modules (e.g. import errors
+ # inside worker.py that find_spec wouldn't surface).
importlib.import_module(module_path)
- except ImportError:
+ except (ImportError, ValueError):
+ # ValueError: find_spec raises this when sys.modules[module_path] is
+ # populated but has __spec__ = None (from a prior failed import).
logger.exception(f"Failed to import pluggable worker {worker_type.value}")
return False
except (OSError, AttributeError):
diff --git a/workers/worker.py b/workers/worker.py
index ed2a60584..3822e81a1 100755
--- a/workers/worker.py
+++ b/workers/worker.py
@@ -448,8 +448,10 @@ if worker_type.is_pluggable():
worker_directory = os.path.join("pluggable_worker", worker_type.value)
worker_path = os.path.join(base_dir, worker_directory)
else:
- # Core workers use their value directly (with hyphens converted to underscores where needed)
- worker_directory = worker_type.value.replace("-", "_")
+ # Enum values use underscores (Python module names); a few on-disk dirs
+ # still use hyphens (e.g. api-deployment). Derive the directory from the
+ # authoritative import-path map on WorkerType instead of a blind replace.
+ worker_directory = worker_type.to_import_path().rsplit(".", 1)[0]
worker_path = os.path.join(base_dir, worker_directory)
# Add worker directory to path for task imports