Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

UN-4238 [FIX] Keep Prompt Studio results reaching the UI after a Redis restart (#2319)

alid3a59959bec0
diff --git a/backend/pyproject.toml b/backend/pyproject.toml
index c669d77c0..b36f08b63 100644
--- a/backend/pyproject.toml
+++ b/backend/pyproject.toml
@@ -33,7 +33,7 @@ dependencies = [
"psycopg2-binary==2.9.9",
"python-dotenv==1.2.2",
"python-magic==0.4.27", # For file upload/download
- "python-socketio==5.9.0", # For log_events
+ "python-socketio==5.17.0", # For log_events
"social-auth-app-django==5.3.0", # For OAuth
"social-auth-core==4.4.2", # For OAuth
"azure-identity==1.16.0",
diff --git a/backend/utils/log_events.py b/backend/utils/log_events.py
index dcb66ed81..e8261866a 100644
--- a/backend/utils/log_events.py
+++ b/backend/utils/log_events.py
@@ -1,6 +1,7 @@
import http
import logging
import os
+import uuid
from typing import Any
import socketio
@@ -36,6 +37,23 @@ sio = socketio.Server(
client_manager=socketio.KombuManager(**_kombu_kwargs),
)
+
+def _reset_pubsub_host_id() -> None:
+ """Give each forked worker its own pub/sub identity.
+
+ python-socketio (5.11+) drops pub/sub messages carrying its own
+ ``host_id``, on the assumption the publisher already delivered them
+ locally. With ``gunicorn --preload`` this module runs once in the master,
+ so every forked worker would inherit the same ``host_id`` and silently
+ discard events published by a sibling worker — e.g. a Prompt Studio result
+ whose ``/internal/emit-websocket/`` call landed on the other worker than
+ the browser's socket.
+ """
+ sio.manager.host_id = uuid.uuid4().hex
+
+
+os.register_at_fork(after_in_child=_reset_pubsub_host_id)
+
_redis_conn = None
diff --git a/backend/utils/tests/test_socketio_redis_reconnect.py b/backend/utils/tests/test_socketio_redis_reconnect.py
new file mode 100644
index 000000000..0ceaa5a3a
--- /dev/null
+++ b/backend/utils/tests/test_socketio_redis_reconnect.py
@@ -0,0 +1,114 @@
+"""Regression test: the Socket.IO pub/sub listener must survive a Redis restart.
+
+log_events.py wires ``socketio.Server`` to Redis through ``KombuManager``. Each
+backend process runs one listener thread that relays events published by
+other processes (e.g. the pod that handled ``/internal/emit-websocket/``) to
+the browsers connected to it. python-socketio 5.9.0 retried only on
+``OSError`` and ``KombuError``, so the ``redis.exceptions.ConnectionError``
+raised when Redis restarts killed that thread for the life of the process:
+the pod kept accepting sockets but never delivered another event, and Prompt
+Studio results showed only after a manual refresh.
+
+The fix is the dependency floor (5.16.3+ retries on any exception). This test
+pins that behaviour so a downgrade cannot silently reopen the gap.
+"""
+
+from __future__ import annotations
+
+import os
+from unittest import mock
+
+import redis
+import socketio
+
+
+class _Message:
+ def __init__(self, payload):
+ self.payload = payload
+
+ def ack(self):
+ pass
+
+
+class _Queue:
+ def __init__(self, items):
+ self._items = list(items)
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, *exc):
+ return False
+
+ def get(self, block=True):
+ item = self._items.pop(0)
+ if isinstance(item, Exception):
+ raise item
+ return item
+
+
+class _Connection:
+ def __init__(self, queue):
+ self._queue = queue
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, *exc):
+ return False
+
+ def SimpleQueue(self, reader_queue): # noqa: N802 - kombu's API name
+ return self._queue
+
+
+def test_listener_reconnects_after_redis_connection_error():
+ manager = socketio.KombuManager(url="redis://localhost:6379/0", write_only=True)
+ payload = {"method": "emit", "event": "prompt_studio_result"}
+ connections = [
+ # First connection: Redis goes away mid-read, as on a restart.
+ _Connection(
+ _Queue([redis.exceptions.ConnectionError("Connection closed by server.")])
+ ),
+ # Second connection: Redis is back and the next event arrives.
+ _Connection(_Queue([_Message(payload)])),
+ ]
+
+ with (
+ mock.patch.object(manager, "_queue", return_value=object()),
+ mock.patch.object(manager, "_connection", side_effect=connections),
+ mock.patch("socketio.kombu_manager.time.sleep"),
+ ):
+ listener = manager._listen()
+ assert next(listener) == payload
+
+
+def test_forked_worker_gets_its_own_pubsub_host_id():
+ """A forked worker must not inherit the master's pub/sub host_id.
+
+ With gunicorn --preload, ``sio`` is built in the master and the workers
+ are forked from it. python-socketio drops pub/sub messages carrying its own
+ host_id, so two workers sharing one would silently discard each other's
+ events (a result emitted by one worker never reaches a browser connected
+ to the other).
+ """
+ from utils.log_events import sio
+
+ parent_host_id = sio.manager.host_id
+ read_fd, write_fd = os.pipe()
+ pid = os.fork()
+ if pid == 0: # child: report its host_id; never return into pytest
+ exit_code = 1
+ try:
+ os.close(read_fd)
+ os.write(write_fd, sio.manager.host_id.encode())
+ exit_code = 0
+ finally:
+ os._exit(exit_code)
+ os.close(write_fd)
+ child_host_id = os.read(read_fd, 64).decode()
+ os.close(read_fd)
+ _, status = os.waitpid(pid, 0)
+
+ assert os.waitstatus_to_exitcode(status) == 0
+ assert child_host_id
+ assert child_host_id != parent_host_id
diff --git a/backend/uv.lock b/backend/uv.lock
index e13d83abf..cd3f948a1 100644
--- a/backend/uv.lock
+++ b/backend/uv.lock
@@ -3050,14 +3050,14 @@ wheels = [
[[package]]
name = "python-engineio"
-version = "4.13.1"
+version = "4.14.0"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "simple-websocket" },
]
-sdist = { url = "https://files.pythonhosted.org/packages/34/12/bdef9dbeedbe2cdeba2a2056ad27b1fb081557d34b69a97f574843462cae/python_engineio-4.13.1.tar.gz", hash = "sha256:0a853fcef52f5b345425d8c2b921ac85023a04dfcf75d7b74696c61e940fd066", size = 92348, upload-time = "2026-02-06T23:38:06.12Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/fc/65/f8bae11b228647e2e2f45b63dec7448efaddb7cb51f529de1fdba69e63b5/python_engineio-4.14.0.tar.gz", hash = "sha256:eaa1e386baf9c2c7959eef7f9d9165c5ea910c5b392f5316e78d29ed073cb43d", size = 80863, upload-time = "2026-08-30T19:52:01.32Z" }
wheels = [
- { url = "https://files.pythonhosted.org/packages/aa/54/0cce26da03a981f949bb8449c9778537f75f5917c172e1d2992ff25cb57d/python_engineio-4.13.1-py3-none-any.whl", hash = "sha256:f32ad10589859c11053ad7d9bb3c9695cdf862113bfb0d20bc4d890198287399", size = 59847, upload-time = "2026-02-06T23:38:04.861Z" },
+ { url = "https://files.pythonhosted.org/packages/e5/de/07cfd386974c2a26a7bde41f2111be29bbfc92b9ea0bb76694415a4a1a78/python_engineio-4.14.0-py3-none-any.whl", hash = "sha256:9f0fe275fb7d67bfc1a632421adf22949fd4843bd9c458c004b0a89cede302a2", size = 60291, upload-time = "2026-08-30T19:51:59.776Z" },
]
[[package]]
@@ -3071,15 +3071,15 @@ wheels = [
[[package]]
name = "python-socketio"
-version = "5.9.0"
+version = "5.17.0"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "bidict" },
{ name = "python-engineio" },
]
-sdist = { url = "https://files.pythonhosted.org/packages/12/b5/5084ce6c772ee2f7d20714a116976cdb5d24c850745b1364d43e024e5a89/python-socketio-5.9.0.tar.gz", hash = "sha256:dc42735f65534187f381fde291ebf620216a4960001370f32de940229b2e7f8f", size = 47982, upload-time = "2023-09-03T19:22:47.083Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/b9/04/8647675c93b5e74a3daa41a2a03930bac0cbdcfcf307900f0441ae6550ba/python_socketio-5.17.0.tar.gz", hash = "sha256:c3bbfc4937dcfea7c4d1b182afa94d4a30335d153987e8f2078b344beacf95a0", size = 134574, upload-time = "2026-09-14T22:51:02.968Z" }
wheels = [
- { url = "https://files.pythonhosted.org/packages/2b/31/fd41960cf365177f323d8c10d774a0b017336556c6f0ba0a38403240914d/python_socketio-5.9.0-py3-none-any.whl", hash = "sha256:c20f12e4ed0cba57581af26bbeea9998bc2eeebb3b952fa92493a1e051cfe9dc", size = 57630, upload-time = "2023-09-03T19:22:45.482Z" },
+ { url = "https://files.pythonhosted.org/packages/ee/be/44b558c944bc16618483967ecd3424c578705aa33ceee7df8c1e4ab43ea0/python_socketio-5.17.0-py3-none-any.whl", hash = "sha256:b5826fd2f8aa02e11347816349b74ac6b53e8a4f4e4b1cf1388e1aff19b7f3f4", size = 82548, upload-time = "2026-09-14T22:51:01.405Z" },
]
[[package]]
@@ -3794,7 +3794,7 @@ requires-dist = [
{ name = "psycopg2-binary", specifier = "==2.9.9" },
{ name = "python-dotenv", specifier = "==1.2.2" },
{ name = "python-magic", specifier = "==0.4.27" },
- { name = "python-socketio", specifier = "==5.9.0" },
+ { name = "python-socketio", specifier = "==5.17.0" },
{ name = "social-auth-app-django", specifier = "==5.3.0" },
{ name = "social-auth-core", specifier = "==4.4.2" },
{ name = "stripe", specifier = ">=12.5.1" },