Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

UN-3368 [FIX] Ignore SIGTERM in worker child processes to fix warm shutdown (#1885)

ali813a43fad64e
diff --git a/workers/worker.py b/workers/worker.py
index ca5dce202..ed2a60584 100755
--- a/workers/worker.py
+++ b/workers/worker.py
@@ -337,33 +337,40 @@ app, config = WorkerBuilder.build_celery_app(worker_type)
# ============= WORKER PROCESS INIT HOOK FOR GRPC FORK-SAFETY =============
@signals.worker_process_init.connect
def on_worker_process_init(**kwargs):
- """Initialize worker process after fork to fix gRPC fork-safety issues.
-
- This signal handler runs in each forked worker process AFTER the fork,
- ensuring that gRPC connections and Google API clients are created fresh
- in the child process, not inherited from the parent.
-
- Without this, Google Cloud libraries (Drive, GCS, BigQuery, Secret Manager)
- crash with SIGSEGV because they inherit stale gRPC connections from the
- parent process.
-
- The GRPC_ENABLE_FORK_SUPPORT environment variable set at the top of this
- file enables gRPC's experimental fork support, and this handler ensures
- proper reinitialization after fork.
+ """Initialize worker process after fork.
+
+ This signal handler runs in each forked worker process AFTER the fork.
+
+ 1. SIGTERM immunity: Ignore SIGTERM in child workers so the MainProcess
+ can orchestrate graceful shutdown via pool.close()/pool.join().
+ Without this, container runtimes (e.g. containerd 2.0) that send
+ SIGTERM to all processes in the cgroup kill children immediately,
+ causing WorkerLostError and orphaned tool containers.
+ billiard's reset_signals() explicitly preserves SIG_IGN, so this
+ survives into task execution. SIGUSR1 (soft timeout) and SIGKILL
+ (hard timeout) are unaffected.
+
+ 2. gRPC fork-safety: Ensure gRPC connections and Google API clients are
+ created fresh in the child process, not inherited from the parent.
+ Without this, Google Cloud libraries (Drive, GCS, BigQuery, Secret
+ Manager) crash with SIGSEGV because they inherit stale gRPC
+ connections from the parent process.
"""
import gc
+ import signal
+
+ # Ignore SIGTERM in child workers - let MainProcess handle shutdown
+ signal.signal(signal.SIGTERM, signal.SIG_IGN)
- logger.info("🔄 Worker process initialized after fork (PID: %s)", os.getpid())
- logger.info("🔒 gRPC fork support enabled: GRPC_ENABLE_FORK_SUPPORT=1")
logger.info(
- "📡 gRPC poll strategy: %s", os.environ.get("GRPC_POLL_STRATEGY", "default")
+ "Worker process initialized after fork (PID: %s) - SIGTERM ignored, "
+ "gRPC fork support enabled",
+ os.getpid(),
)
# Force garbage collection to clean up any stale gRPC state from parent
gc.collect()
- logger.info("✅ Worker process ready for gRPC-based operations (Google APIs)")
-
# ============= END OF WORKER PROCESS INIT HOOK =============