LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
import http
import logging
import os
import re
import uuid
from importlib import import_module
from typing import Any
import socketio
from django.conf import settings
from django.contrib.auth import (
BACKEND_SESSION_KEY,
HASH_SESSION_KEY,
SESSION_KEY,
load_backend,
)
from django.core.wsgi import WSGIHandler
from django.db import close_old_connections
from django.utils.crypto import constant_time_compare
from unstract.core.cache.redis_client import create_redis_client
from unstract.core.data_models import LogDataDTO
from unstract.core.log_utils import get_validated_log_data, store_execution_log
from utils.constants import ExecutionLogConstants
from utils.cors_origin import RegexOrigin
logger = logging.getLogger(__name__)
_SESSION_KEY_RE = re.compile(r"[a-z0-9]{32}")
_cors_allowed_origins: list[Any] = list(settings.CORS_ALLOWED_ORIGINS)
for _pattern in getattr(settings, "CORS_ALLOWED_ORIGIN_REGEXES", []):
_cors_allowed_origins.append(RegexOrigin(_pattern))
_kombu_kwargs: dict[str, Any] = {"url": settings.SOCKET_IO_MANAGER_URL}
if getattr(settings, "SOCKET_IO_TRANSPORT_OPTIONS", None):
_kombu_kwargs["connection_options"] = {
"transport_options": settings.SOCKET_IO_TRANSPORT_OPTIONS
}
sio = socketio.Server(
# Allowed values: {threading, eventlet, gevent, gevent_uwsgi}
async_mode="threading",
cors_allowed_origins=_cors_allowed_origins,
logger=False,
engineio_logger=False,
always_connect=True,
client_manager=socketio.KombuManager(**_kombu_kwargs),
)
def _reset_pubsub_host_id() -> None:
"""Give each forked worker its own pub/sub identity.
python-socketio (5.11+) drops pub/sub messages carrying its own
``host_id``, on the assumption the publisher already delivered them
locally. With ``gunicorn --preload`` this module runs once in the master,
so every forked worker would inherit the same ``host_id`` and silently
discard events published by a sibling worker — e.g. a Prompt Studio result
whose ``/internal/emit-websocket/`` call landed on the other worker than
the browser's socket.
"""
sio.manager.host_id = uuid.uuid4().hex
os.register_at_fork(after_in_child=_reset_pubsub_host_id)
_redis_conn = None
def _get_redis_conn():
global _redis_conn
if _redis_conn is None:
_redis_conn = create_redis_client(decode_responses=False)
return _redis_conn
@sio.event
def connect(sid: str, environ: Any, auth: Any) -> None:
"""This function is called when a client connects to the server.
It handles the connection and authentication of the client.
"""
logger.info(f"[{os.getpid()}] Client with SID:{sid} connected")
session_id = _get_user_session_id_from_cookies(sid, environ)
if session_id:
sio.enter_room(sid, session_id)
logger.info(f"Entered room {session_id} for socket {sid}")
user_id = _get_user_id(sid, session_id)
if user_id:
# Per-user events, such as the inbox unread count.
sio.enter_room(sid, f"user:{user_id}")
else:
logger.info(
f"No signed-in user for the session of socket {sid}; no user room"
)
else:
sio.disconnect(sid)
@sio.event
def disconnect(sid: str) -> None:
logger.info(f"[{os.getpid()}] Client with SID:{sid} disconnected")
def _get_user_session_id_from_cookies(sid: str, environ: Any) -> str | None:
"""Get the user session ID from cookies.
Args:
sid (str): The socket ID of the client.
environ (Any): The environment variables of the client.
Returns:
Optional[str]: The user session ID.
"""
cookie_str = environ.get("HTTP_COOKIE")
if not cookie_str:
logger.warning(f"No cookies found in {environ} for the sid {sid}")
return None
cookie = http.cookies.SimpleCookie(cookie_str)
session_id = cookie.get(settings.SESSION_COOKIE_NAME)
if not session_id:
logger.warning(f"No session ID found in cookies for SID {sid}")
return None
# Django session keys are 32 lowercase letters and digits. Anything else,
# such as "user:5", could name another room, so it is refused before any
# session lookup.
if not _SESSION_KEY_RE.fullmatch(session_id.value):
logger.warning(f"Malformed session ID in cookies for SID {sid}")
return None
return session_id.value
def _get_user_id(sid: str, session_id: str) -> str | None:
"""The signed-in user of a session, or None.
Applies the checks django.contrib.auth.get_user applies to an HTTP request
with the same cookie: the session's backend is still configured and still
returns the user (a deactivated user is refused), and the session hash still
matches (a password change signs the session out). Unlike get_user, it never
flushes or rotates the session, which the browser's cookie still names.
"""
# Socket handlers run outside Django's request cycle, which is what normally
# closes stale connections.
close_old_connections()
try:
store = import_module(settings.SESSION_ENGINE).SessionStore(
session_key=session_id
)
user_id = store.get(SESSION_KEY)
backend_path = store.get(BACKEND_SESSION_KEY)
if user_id is None or backend_path not in settings.AUTHENTICATION_BACKENDS:
return None
user = load_backend(backend_path).get_user(user_id)
if user is None:
return None
if hasattr(user, "get_session_auth_hash"):
session_hash = store.get(HASH_SESSION_KEY)
hashes = [
user.get_session_auth_hash(),
*user.get_session_auth_fallback_hash(),
]
if not session_hash or not any(
constant_time_compare(session_hash, h) for h in hashes
):
return None
return user_id
except Exception:
logger.exception(f"Could not read the session of socket {sid}")
return None
finally:
close_old_connections()
# Functions moved to unstract.core.log_utils for sharing with workers
# Keep these as wrapper functions for backward compatibility
def _get_validated_log_data(json_data: Any) -> LogDataDTO | None:
"""Validate log data to persist history (backward compatibility wrapper)."""
return get_validated_log_data(json_data)
def _store_execution_log(data: dict[str, Any]) -> None:
"""Store execution log in database (backward compatibility wrapper)."""
store_execution_log(
data=data,
redis_client=_get_redis_conn(),
log_queue_name=ExecutionLogConstants.LOG_QUEUE_NAME,
is_enabled=ExecutionLogConstants.IS_ENABLED,
)
def _emit_websocket_event(room: str, event: str, data: dict[str, Any]) -> None:
"""Emit websocket event
Args:
room (str): Room to emit event to
event (str): Event name
channel (str): Channel name
data (bytes): Data to emit
"""
payload = {"data": data}
try:
logger.debug(f"[{os.getpid()}] Push websocket event: {event}, {payload}")
sio.emit(event, data=payload, room=room)
except Exception as e:
logger.error(f"Error emitting WebSocket event: {e}")
def handle_user_logs(room: str, event: str, message: dict[str, Any]) -> None:
"""Handle user logs from applications
Args:
message (dict[str, Any]): log message
"""
if not room or not event:
logger.warning(f"Message received without room and event: {message}")
return
_store_execution_log(message)
_emit_websocket_event(room, event, message)
def start_server(django_app: WSGIHandler, namespace: str) -> WSGIHandler:
django_app = socketio.WSGIApp(sio, django_app, socketio_path=namespace)
return django_app