Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

Fix/socket io design changes (#385)

ali0ee58329340b
diff --git a/backend/backend/__init__.py b/backend/backend/__init__.py
index 370372af0..e921073a8 100644
--- a/backend/backend/__init__.py
+++ b/backend/backend/__init__.py
@@ -1,3 +1,3 @@
-from .celery import app as celery_app
+from .celery_service import app as celery_app
__all__ = ["celery_app"]
diff --git a/backend/backend/celery.py b/backend/backend/celery_service.py
similarity index 92%
rename from backend/backend/celery.py
rename to backend/backend/celery_service.py
index 57f387391..99effcd78 100644
--- a/backend/backend/celery.py
+++ b/backend/backend/celery_service.py
@@ -6,6 +6,8 @@ from celery import Celery
from django.conf import settings
from utils.constants import ExecutionLogConstants
+from backend.celery_task import TaskRegistry
+
# Set the default Django settings module for the 'celery' program.
os.environ.setdefault(
"DJANGO_SETTINGS_MODULE",
@@ -15,10 +17,14 @@ os.environ.setdefault(
# Create a Celery instance. Default time zone is UTC.
app = Celery("backend")
+# To register the custom tasks.
+TaskRegistry()
+
# Use Redis as the message broker.
app.conf.broker_url = settings.CELERY_BROKER_URL
app.conf.result_backend = settings.CELERY_RESULT_BACKEND
+
# Load task modules from all registered Django app configs.
app.config_from_object("django.conf:settings", namespace="CELERY")
diff --git a/backend/backend/celery_task.py b/backend/backend/celery_task.py
new file mode 100644
index 000000000..bc3b709e3
--- /dev/null
+++ b/backend/backend/celery_task.py
@@ -0,0 +1,34 @@
+import logging
+import os
+from typing import Any
+
+from celery import shared_task
+from utils.log_events import handle_user_logs
+
+from unstract.core.constants import LogEventArgument, LogProcessingTask
+
+logger = logging.getLogger(__name__)
+
+
+class TaskRegistry:
+ @staticmethod
+ @shared_task(
+ name=LogProcessingTask.TASK_NAME,
+ )
+ def log_consumer(**kwargs: Any) -> None:
+ """Task to process the logs from log publisher.
+
+ Args:
+ kwargs: The arguments to process the logs.
+ Expected arguments:
+ USER_SESSION_ID: The room to be processed.
+ EVENT: The event to be processed Ex: logs:{session_id}.
+ MESSAGE: The message to be processed Ex: execution log.
+ """
+ log_message = kwargs.get(LogEventArgument.MESSAGE)
+ room = kwargs.get(LogEventArgument.USER_SESSION_ID)
+ event = kwargs.get(LogEventArgument.EVENT)
+ logger.debug(
+ f"[{os.getpid()}] Log message received: {log_message} for the room {room}"
+ )
+ handle_user_logs(room=room, event=event, message=log_message)
diff --git a/backend/backend/settings/base.py b/backend/backend/settings/base.py
index d043d1664..d18b3655d 100644
--- a/backend/backend/settings/base.py
+++ b/backend/backend/settings/base.py
@@ -160,6 +160,9 @@ LOG_HISTORY_CONSUMER_INTERVAL = int(
get_required_setting("LOG_HISTORY_CONSUMER_INTERVAL", "60")
)
LOGS_BATCH_LIMIT = int(get_required_setting("LOGS_BATCH_LIMIT", "30"))
+CELERY_BROKER_URL = get_required_setting(
+ "CELERY_BROKER_URL", f"redis://{REDIS_HOST}:{REDIS_PORT}"
+)
# Flag to Enable django admin
ADMIN_ENABLED = False
@@ -309,6 +312,9 @@ DATABASES = {
}
}
+# SocketIO connection manager
+SOCKET_IO_MANAGER_URL = f"redis://{REDIS_HOST}:{REDIS_PORT}"
+
DATABASE_ROUTERS = ("django_tenants.routers.TenantSyncRouter",)
CACHES = {
@@ -332,9 +338,6 @@ RQ_QUEUES = {
"default": {"USE_REDIS_CACHE": "default"},
}
-# Used for asynchronous/Queued execution
-# Celery based scheduler
-CELERY_BROKER_URL = f"redis://{REDIS_HOST}:{REDIS_PORT}"
# CELERY_RESULT_BACKEND = f"redis://{REDIS_HOST}:{REDIS_PORT}/1"
# Postgres as result backend
diff --git a/backend/pdm.lock b/backend/pdm.lock
index 57c972c42..d9fb2b2ae 100644
--- a/backend/pdm.lock
+++ b/backend/pdm.lock
@@ -4669,6 +4669,7 @@ groups = ["default"]
dependencies = [
"boto3~=1.28.17",
"botocore~=1.31.17",
+ "kombu==5.3.7",
"llama-index-llms-azure-openai==0.1.5",
"llama-index==0.10.28",
"redis~=5.0.1",
diff --git a/backend/pyproject.toml b/backend/pyproject.toml
index ef6957776..6708a047b 100644
--- a/backend/pyproject.toml
+++ b/backend/pyproject.toml
@@ -78,7 +78,7 @@ migrate_db.cmd = "python manage.py migrate"
migrate_db.env_file = ".env"
migrate_db.help = "Performs DB migrations for Unstract backend"
# Commands for backend execution
-consumer.cmd = "celery -A backend worker --loglevel=info -Q celery,celery_periodic_logs"
+consumer.cmd = "celery -A backend worker --loglevel=info -Q celery,celery_periodic_logs,celery_log_task_queue --autoscale 8,1"
consumer.env_file = ".env"
consumer.help = "Runs the Unstract consumer"
# Celery Flower
diff --git a/backend/sample.env b/backend/sample.env
index 765331cbe..cc8991a38 100644
--- a/backend/sample.env
+++ b/backend/sample.env
@@ -137,3 +137,6 @@ ENABLE_LOG_HISTORY=True
LOG_HISTORY_CONSUMER_INTERVAL=30
# Maximum number of logs to insert in a single batch.
LOGS_BATCH_LIMIT=30
+
+# Celery Configuration
+CELERY_BROKER_URL = "redis://unstract-redis:6379"
diff --git a/backend/utils/log_events.py b/backend/utils/log_events.py
index c8ecfb989..309a879a7 100644
--- a/backend/utils/log_events.py
+++ b/backend/utils/log_events.py
@@ -1,8 +1,7 @@
+import http
import json
import logging
import os
-import threading
-import time
from typing import Any, Optional
import redis
@@ -24,7 +23,9 @@ sio = socketio.Server(
logger=False,
engineio_logger=False,
always_connect=True,
+ client_manager=socketio.KombuManager(url=settings.SOCKET_IO_MANAGER_URL),
)
+
redis_conn = redis.Redis(
host=settings.REDIS_HOST,
port=int(settings.REDIS_PORT),
@@ -35,8 +36,17 @@ redis_conn = redis.Redis(
@sio.event
def connect(sid: str, environ: Any, auth: Any) -> None:
- # TODO Authenticate websocket connections
+ """This function is called when a client connects to the server.
+
+ It handles the connection and authentication of the client.
+ """
logger.info(f"[{os.getpid()}] Client with SID:{sid} connected")
+ session_id = _get_user_session_id_from_cookies(sid, environ)
+ if session_id:
+ sio.enter_room(sid, session_id)
+ logger.info(f"Entered room {session_id} for socket {sid}")
+ else:
+ sio.disconnect(sid)
@sio.event
@@ -44,6 +54,31 @@ def disconnect(sid: str) -> None:
logger.info(f"[{os.getpid()}] Client with SID:{sid} disconnected")
+def _get_user_session_id_from_cookies(sid: str, environ: Any) -> Optional[str]:
+ """Get the user session ID from cookies.
+
+ Args:
+ sid (str): The socket ID of the client.
+ environ (Any): The environment variables of the client.
+
+ Returns:
+ Optional[str]: The user session ID.
+ """
+ cookie_str = environ.get("HTTP_COOKIE")
+ if not cookie_str:
+ logger.warning(f"No cookies found in {environ} for the sid {sid}")
+ return None
+
+ cookie = http.cookies.SimpleCookie(cookie_str)
+ session_id = cookie.get(settings.SESSION_COOKIE_NAME)
+
+ if not session_id:
+ logger.warning(f"No session ID found in cookies for SID {sid}")
+ return None
+
+ return session_id.value
+
+
def _get_validated_log_data(json_data: Any) -> Optional[LogDataDTO]:
"""Validate log data to persist history. This function takes log data in
JSON format, validates it, and returns a `LogDataDTO` object if the data is
@@ -82,7 +117,7 @@ def _get_validated_log_data(json_data: Any) -> Optional[LogDataDTO]:
# Check if all required fields are present
if not all((execution_id, organization_id, timestamp)):
- logger.warning(f"Missing required fields while validating {json_data}")
+ logger.debug(f"Missing required fields while validating {json_data}")
return
return LogDataDTO(
@@ -94,10 +129,10 @@ def _get_validated_log_data(json_data: Any) -> Optional[LogDataDTO]:
)
-def _store_execution_log(data: bytes) -> None:
+def _store_execution_log(data: dict[str, Any]) -> None:
"""Store execution log in database
Args:
- data (bytes): Execution log data in bytes format
+ data (dict[str, Any]): Execution log data
"""
if not ExecutionLogConstants.IS_ENABLED:
return
@@ -109,69 +144,36 @@ def _store_execution_log(data: bytes) -> None:
logger.error(f"Error storing execution log: {e}")
-def _emit_websocket_event(channel: str, data: bytes) -> None:
+def _emit_websocket_event(room: str, event: str, data: dict[str, Any]) -> None:
"""Emit websocket event
Args:
- channel (str): WebSocket channel
- data (bytes): Execution log data in bytes format
+ room (str): Room to emit event to
+ event (str): Event name
+ channel (str): Channel name
+ data (bytes): Data to emit
"""
payload = {"data": data}
try:
- logger.debug(f"[{os.getpid()}] Push websocket event: {channel}, {payload}")
- sio.emit(channel, payload)
+ logger.debug(f"[{os.getpid()}] Push websocket event: {event}, {payload}")
+ sio.emit(event, data=payload, room=room)
except Exception as e:
logger.error(f"Error emitting WebSocket event: {e}")
-def _handle_pubsub_messages(message: dict[str, Any]) -> None:
- """Handle pubsub messages
+def handle_user_logs(room: str, event: str, message: dict[str, Any]) -> None:
+ """Handle user logs from applications
Args:
- message (dict[str, Any]): Pub sub message
+ message (dict[str, Any]): log message
"""
- channel = message.get("channel")
- data = message.get("data")
- if not channel or not data:
- logger.warning(f"Invalid message received: {message}")
+ if not room or not event:
+ logger.warning(f"Message received without room and event: {message}")
return
- try:
- channel_str = channel.decode("utf-8")
- except UnicodeDecodeError as e:
- logger.error(f"Error decoding channel: {e}")
- return
-
- _store_execution_log(data)
- _emit_websocket_event(channel_str, data)
-
-
-def _pubsub_listen_forever() -> None:
- global shutdown
-
- try:
- pubsub = redis_conn.pubsub(ignore_subscribe_messages=True)
- pubsub.psubscribe("logs:*")
-
- logger.info(f"[{os.getpid()}] Listening for pub sub messages...")
- while True:
- message = pubsub.get_message()
-
- if message:
- logger.debug(f"[{os.getpid()}] Pub sub message received: {message}")
- if message["type"] == "pmessage":
- _handle_pubsub_messages(message)
-
- # TODO Add graceful shutdown
-
- time.sleep(0.01)
- except Exception as e:
- logger.error(f"[{os.getpid()}] Failed to do pubsub: {e}")
+ _store_execution_log(message)
+ _emit_websocket_event(room, event, message)
def start_server(django_app: WSGIHandler, namespace: str) -> WSGIHandler:
django_app = socketio.WSGIApp(sio, django_app, socketio_path=namespace)
-
- pubsub_listener = threading.Thread(target=_pubsub_listen_forever, daemon=True)
- pubsub_listener.start()
-
return django_app
diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml
index bcca7c6c4..33de0fed5 100644
--- a/docker/docker-compose.yaml
+++ b/docker/docker-compose.yaml
@@ -39,7 +39,7 @@ services:
container_name: unstract-execution-consumer
restart: unless-stopped
entrypoint: .venv/bin/celery
- command: "-A backend worker --loglevel=info -Q celery,celery_periodic_logs --autoscale=8,1"
+ command: "-A backend worker --loglevel=info -Q celery,celery_periodic_logs,celery_log_task_queue --autoscale=8,1"
env_file:
- ../backend/.env
depends_on:
diff --git a/frontend/src/components/helpers/socket-messages/SocketMessages.js b/frontend/src/components/helpers/socket-messages/SocketMessages.js
index 207120ef8..72bb2d3af 100644
--- a/frontend/src/components/helpers/socket-messages/SocketMessages.js
+++ b/frontend/src/components/helpers/socket-messages/SocketMessages.js
@@ -32,7 +32,16 @@ function SocketMessages() {
const onMessage = (data) => {
try {
- const msg = JSON.parse(new TextDecoder().decode(data.data));
+ let msg = data.data;
+ // Attempt to decode data as JSON if it's in encoded state
+ if (typeof msg === "string" || msg instanceof Uint8Array) {
+ if (typeof msg === "string") {
+ msg = JSON.parse(msg);
+ } else {
+ msg = JSON.parse(new TextDecoder().decode(msg));
+ }
+ }
+
if (
(msg?.type === "LOG" || msg?.type === "COST") &&
msg?.service !== "prompt"
diff --git a/unstract/core/pdm.lock b/unstract/core/pdm.lock
index 0b4807be4..18b4d29a1 100644
--- a/unstract/core/pdm.lock
+++ b/unstract/core/pdm.lock
@@ -5,7 +5,7 @@
groups = ["default"]
strategy = ["cross_platform", "inherit_metadata"]
lock_version = "4.4.1"
-content_hash = "sha256:f8589435b9bfce4e1b1aca6600ec970edfe816e0ba7900ef67ab176f8c7dd4d3"
+content_hash = "sha256:19044e2ce12da08c83b008c16d2de4d46531c7b91a36296429729d6340c77eae"
[[package]]
name = "aiohttp"
@@ -84,6 +84,20 @@ files = [
{file = "aiosignal-1.3.1.tar.gz", hash = "sha256:54cd96e15e1649b75d6c87526a6ff0b6c1b0dd3459f43d9ca11d48c339b68cfc"},
]
+[[package]]
+name = "amqp"
+version = "5.2.0"
+requires_python = ">=3.6"
+summary = "Low-level AMQP client for Python (fork of amqplib)."
+groups = ["default"]
+dependencies = [
+ "vine<6.0.0,>=5.0.0",
+]
+files = [
+ {file = "amqp-5.2.0-py3-none-any.whl", hash = "sha256:827cb12fb0baa892aad844fd95258143bce4027fdac4fccddbc43330fd281637"},
+ {file = "amqp-5.2.0.tar.gz", hash = "sha256:a1ecff425ad063ad42a486c902807d1482311481c8ad95a72694b2975e75f7fd"},
+]
+
[[package]]
name = "annotated-types"
version = "0.6.0"
@@ -643,6 +657,22 @@ files = [
{file = "joblib-1.4.0.tar.gz", hash = "sha256:1eb0dc091919cd384490de890cb5dfd538410a6d4b3b54eef09fb8c50b409b1c"},
]
+[[package]]
+name = "kombu"
+version = "5.3.7"
+requires_python = ">=3.8"
+summary = "Messaging library for Python."
+groups = ["default"]
+dependencies = [
+ "amqp<6.0.0,>=5.1.1",
+ "typing-extensions; python_version < \"3.10\"",
+ "vine",
+]
+files = [
+ {file = "kombu-5.3.7-py3-none-any.whl", hash = "sha256:5634c511926309c7f9789f1433e9ed402616b56836ef9878f01bd59267b4c7a9"},
+ {file = "kombu-5.3.7.tar.gz", hash = "sha256:011c4cd9a355c14a1de8d35d257314a1d2456d52b7140388561acac3cf1a97bf"},
+]
+
[[package]]
name = "llama-index"
version = "0.10.28"
@@ -1850,6 +1880,17 @@ files = [
{file = "urllib3-1.26.18.tar.gz", hash = "sha256:f8ecc1bba5667413457c529ab955bf8c67b45db799d159066261719e328580a0"},
]
+[[package]]
+name = "vine"
+version = "5.1.0"
+requires_python = ">=3.6"
+summary = "Python promises."
+groups = ["default"]
+files = [
+ {file = "vine-5.1.0-py3-none-any.whl", hash = "sha256:40fdf3c48b2cfe1c38a49e9ae2da6fda88e4794c810050a728bd7413811fb1dc"},
+ {file = "vine-5.1.0.tar.gz", hash = "sha256:8b62e981d35c41049211cf62a0a1242d8c1ee9bd15bb196ce38aefd6799e61e0"},
+]
+
[[package]]
name = "wrapt"
version = "1.16.0"
diff --git a/unstract/core/pyproject.toml b/unstract/core/pyproject.toml
index a2ea5c5b6..c9b9928a3 100644
--- a/unstract/core/pyproject.toml
+++ b/unstract/core/pyproject.toml
@@ -16,6 +16,7 @@ dependencies = [
"llama-index-llms-azure-openai==0.1.5",
"redis~=5.0.1",
"requests==2.31.0",
+ "kombu==5.3.7",
]
# <3.11.1 due to resolution error from Unstract SDK
requires-python = ">=3.9,<3.11.1"
diff --git a/unstract/core/src/unstract/core/constants.py b/unstract/core/src/unstract/core/constants.py
index 5f397e7f6..73303df0c 100644
--- a/unstract/core/src/unstract/core/constants.py
+++ b/unstract/core/src/unstract/core/constants.py
@@ -5,3 +5,14 @@ class LogFieldName:
TYPE = "type"
DATA = "data"
EVENT_TIME = "event_time"
+
+
+class LogEventArgument:
+ EVENT = "event"
+ MESSAGE = "message"
+ USER_SESSION_ID = "user_session_id"
+
+
+class LogProcessingTask:
+ TASK_NAME = "logs_consumer"
+ QUEUE_NAME = "celery_log_task_queue"
diff --git a/unstract/core/src/unstract/core/pubsub_helper.py b/unstract/core/src/unstract/core/pubsub_helper.py
index 5658c0076..9011c40a0 100644
--- a/unstract/core/src/unstract/core/pubsub_helper.py
+++ b/unstract/core/src/unstract/core/pubsub_helper.py
@@ -1,19 +1,16 @@
-import json
import logging
import os
from datetime import datetime, timezone
from typing import Any, Optional
-import redis
+from kombu import Connection
+
+from unstract.core.constants import LogEventArgument, LogProcessingTask
class LogPublisher:
- r = redis.Redis(
- host=os.environ.get("REDIS_HOST", "http://localhost"),
- port=os.environ.get("REDIS_PORT", "6379"),
- username=os.environ.get("REDIS_USER", ""),
- password=os.environ.get("REDIS_PASSWORD", ""),
- )
+
+ kombu_conn = Connection(os.environ.get("CELERY_BROKER_URL"))
@staticmethod
def log_usage(
@@ -93,12 +90,53 @@ class LogPublisher:
"message": message,
}
+ @classmethod
+ def _get_task_message(
+ cls, user_session_id: str, event: str, message: Any
+ ) -> dict[str, Any]:
+
+ task_kwargs = {
+ LogEventArgument.EVENT: event,
+ LogEventArgument.MESSAGE: message,
+ LogEventArgument.USER_SESSION_ID: user_session_id,
+ }
+ task_message = {
+ "args": [],
+ "kwargs": task_kwargs,
+ "retries": 0,
+ "utc": True,
+ }
+ return task_message
+
+ @classmethod
+ def _get_task_header(cls, task_name: str) -> dict[str, Any]:
+ return {
+ "task": task_name,
+ }
+
@classmethod
def publish(cls, channel_id: str, payload: dict[str, Any]) -> bool:
- channel = f"logs:{channel_id}"
+ """Publish a message to the queue."""
try:
- cls.r.publish(channel, json.dumps(payload))
+ with cls.kombu_conn.Producer(serializer="json") as producer:
+ event = f"logs:{channel_id}"
+ task_message = cls._get_task_message(
+ user_session_id=channel_id,
+ event=event,
+ message=payload,
+ )
+ headers = cls._get_task_header(LogProcessingTask.TASK_NAME)
+ # Publish the message to the queue
+ producer.publish(
+ body=task_message,
+ exchange="",
+ headers=headers,
+ routing_key=LogProcessingTask.QUEUE_NAME,
+ compression=None,
+ retry=True,
+ )
+ logging.debug(f"Published '{channel_id}' <= {payload}")
except Exception as e:
- logging.error(f"Failed to publish '{channel}' <= {payload}: {e}")
+ logging.error(f"Failed to publish '{channel_id}' <= {payload}: {e}")
return False
return True
diff --git a/worker/README.md b/worker/README.md
index f29ff471e..0cc49d739 100644
--- a/worker/README.md
+++ b/worker/README.md
@@ -39,12 +39,9 @@ sudo ln -s "$HOME/.docker/run/docker.sock" /var/run/docker.sock
| Variable | Description |
| -------------------------- | ---------------------------------------------------------------------------------------|
-| `REDIS_HOST` | Host address of the Redis server. |
-| `REDIS_PORT` | Port number for the Redis server. |
-| `REDIS_PASSWORD` | Password for accessing the Redis server. (Leave empty if not required) |
-| `REDIS_USER` | User for accessing the Redis server. (If applicable) |
+| `CELERY_BROKER_URL` | URL for Celery's message broker, used to queue tasks. Must match backend configuration.|
| `TOOL_CONTAINER_NETWORK` | Network used for running tool containers. |
-| `TOOL_CONTAINER_LABELS` | Labels applied to tool containers for observability [Optional]. |
+| `TOOL_CONTAINER_LABELS` | Labels applied to tool containers for observability [Optional]. |
| `WORKFLOW_DATA_DIR` | Source mount bind directory for tool containers to access input files. |
| `TOOL_DATA_DIR` | Target mount directory within tool containers. (Default: "/data") |
| `LOG_LEVEL` | Log level for worker (Options: INFO, WARNING, ERROR, DEBUG, etc.) |
diff --git a/worker/pdm.lock b/worker/pdm.lock
index a273d0cc7..750f29977 100644
--- a/worker/pdm.lock
+++ b/worker/pdm.lock
@@ -82,6 +82,19 @@ files = [
{file = "aiosignal-1.3.1.tar.gz", hash = "sha256:54cd96e15e1649b75d6c87526a6ff0b6c1b0dd3459f43d9ca11d48c339b68cfc"},
]
+[[package]]
+name = "amqp"
+version = "5.2.0"
+requires_python = ">=3.6"
+summary = "Low-level AMQP client for Python (fork of amqplib)."
+dependencies = [
+ "vine<6.0.0,>=5.0.0",
+]
+files = [
+ {file = "amqp-5.2.0-py3-none-any.whl", hash = "sha256:827cb12fb0baa892aad844fd95258143bce4027fdac4fccddbc43330fd281637"},
+ {file = "amqp-5.2.0.tar.gz", hash = "sha256:a1ecff425ad063ad42a486c902807d1482311481c8ad95a72694b2975e75f7fd"},
+]
+
[[package]]
name = "annotated-types"
version = "0.7.0"
@@ -702,6 +715,21 @@ files = [
{file = "joblib-1.4.2.tar.gz", hash = "sha256:2382c5816b2636fbd20a09e0f4e9dad4736765fdfb7dca582943b9c1366b3f0e"},
]
+[[package]]
+name = "kombu"
+version = "5.3.7"
+requires_python = ">=3.8"
+summary = "Messaging library for Python."
+dependencies = [
+ "amqp<6.0.0,>=5.1.1",
+ "typing-extensions; python_version < \"3.10\"",
+ "vine",
+]
+files = [
+ {file = "kombu-5.3.7-py3-none-any.whl", hash = "sha256:5634c511926309c7f9789f1433e9ed402616b56836ef9878f01bd59267b4c7a9"},
+ {file = "kombu-5.3.7.tar.gz", hash = "sha256:011c4cd9a355c14a1de8d35d257314a1d2456d52b7140388561acac3cf1a97bf"},
+]
+
[[package]]
name = "llama-index"
version = "0.10.28"
@@ -728,7 +756,7 @@ files = [
[[package]]
name = "llama-index-agent-openai"
-version = "0.2.6"
+version = "0.2.7"
requires_python = "<4.0,>=3.8.1"
summary = "llama-index agent openai integration"
dependencies = [
@@ -737,8 +765,8 @@ dependencies = [
"openai>=1.14.0",
]
files = [
- {file = "llama_index_agent_openai-0.2.6-py3-none-any.whl", hash = "sha256:fddf5a21b5cc584d4aab4517edcdd7ccdba7bf2b66ba6f1ae31d6b87800b4246"},
- {file = "llama_index_agent_openai-0.2.6.tar.gz", hash = "sha256:32d95ab8ebd8fb8174e34427e6cfcddb56003da7029be9a1d3fe002540d13b9a"},
+ {file = "llama_index_agent_openai-0.2.7-py3-none-any.whl", hash = "sha256:34be65011a508dd8cab0c9a606594f28075b98b0cebe69e3c543adc8564fee0d"},
+ {file = "llama_index_agent_openai-0.2.7.tar.gz", hash = "sha256:13ce535f03e32c821763c01e26af4222f3981178622414d3868013a1946e8124"},
]
[[package]]
@@ -758,7 +786,7 @@ files = [
[[package]]
name = "llama-index-core"
-version = "0.10.41"
+version = "0.10.43"
requires_python = "<4.0,>=3.8.1"
summary = "Interface between LLMs and your data"
dependencies = [
@@ -787,8 +815,8 @@ dependencies = [
"wrapt",
]
files = [
- {file = "llama_index_core-0.10.41-py3-none-any.whl", hash = "sha256:cc404f6ec3c9ba563b1acd9005b95e18ba6ca5da6e83ac64d555821bd7c6f574"},
- {file = "llama_index_core-0.10.41.tar.gz", hash = "sha256:e514ee5a070c405f4864f07a9fda46950e06797ee5a98c30f3c5592ca0203118"},
+ {file = "llama_index_core-0.10.43-py3-none-any.whl", hash = "sha256:355a5f7808dfa09dc27b6bd564926fa697e0279344b829732a9f91b8000157ac"},
+ {file = "llama_index_core-0.10.43.tar.gz", hash = "sha256:c7dce5158e112d3cbc8d331c303e830b2eabcae55c9636e8050dc85320522c1b"},
]
[[package]]
@@ -866,15 +894,15 @@ files = [
[[package]]
name = "llama-index-llms-openai"
-version = "0.1.21"
+version = "0.1.22"
requires_python = "<4.0,>=3.8.1"
summary = "llama-index llms openai integration"
dependencies = [
"llama-index-core<0.11.0,>=0.10.24",
]
files = [
- {file = "llama_index_llms_openai-0.1.21-py3-none-any.whl", hash = "sha256:fd2fa240238c7dca0170bf55957bb088d9b6acf63c80924eb59498f3a5ac0df3"},
- {file = "llama_index_llms_openai-0.1.21.tar.gz", hash = "sha256:e9d631135160ac87093cd7d5ae3e32c4debf8eeaafa29ee4739675867155d773"},
+ {file = "llama_index_llms_openai-0.1.22-py3-none-any.whl", hash = "sha256:84a8c910671460ad724ed818192f209f7481e71bcc6528553ba7e66db2e14bcd"},
+ {file = "llama_index_llms_openai-0.1.22.tar.gz", hash = "sha256:729bf2ea7043517465e1d585089512b77d8b3ce92233a67c138d5d621061ed56"},
]
[[package]]
@@ -1061,6 +1089,37 @@ files = [
{file = "msal_extensions-1.1.0-py3-none-any.whl", hash = "sha256:01be9711b4c0b1a151450068eeb2c4f0997df3bba085ac299de3a66f585e382f"},
]
+[[package]]
+name = "msal"
+version = "1.28.0"
+requires_python = ">=3.7"
+summary = "The Microsoft Authentication Library (MSAL) for Python library enables your app to access the Microsoft Cloud by supporting authentication of users with Microsoft Azure Active Directory accounts (AAD) and Microsoft Accounts (MSA) using industry standard OAuth2 and OpenID Connect."
+dependencies = [
+ "PyJWT[crypto]<3,>=1.0.0",
+ "cryptography<45,>=0.6",
+ "requests<3,>=2.0.0",
+]
+files = [
+ {file = "msal-1.28.0-py3-none-any.whl", hash = "sha256:3064f80221a21cd535ad8c3fafbb3a3582cd9c7e9af0bb789ae14f726a0ca99b"},
+ {file = "msal-1.28.0.tar.gz", hash = "sha256:80bbabe34567cb734efd2ec1869b2d98195c927455369d8077b3c542088c5c9d"},
+]
+
+[[package]]
+name = "msal-extensions"
+version = "1.1.0"
+requires_python = ">=3.7"
+summary = "Microsoft Authentication Library extensions (MSAL EX) provides a persistence API that can save your data on disk, encrypted on Windows, macOS and Linux. Concurrent data access will be coordinated by a file lock mechanism."
+dependencies = [
+ "msal<2.0.0,>=0.4.1",
+ "packaging",
+ "portalocker<3,>=1.0; platform_system != \"Windows\"",
+ "portalocker<3,>=1.6; platform_system == \"Windows\"",
+]
+files = [
+ {file = "msal-extensions-1.1.0.tar.gz", hash = "sha256:6ab357867062db7b253d0bd2df6d411c7891a0ee7308d54d1e4317c1d1c54252"},
+ {file = "msal_extensions-1.1.0-py3-none-any.whl", hash = "sha256:01be9711b4c0b1a151450068eeb2c4f0997df3bba085ac299de3a66f585e382f"},
+]
+
[[package]]
name = "multidict"
version = "6.0.5"
@@ -1343,6 +1402,85 @@ files = [
{file = "pycparser-2.22.tar.gz", hash = "sha256:491c8be9c040f5390f5bf44a5b07752bd07f56edf992381b05c701439eec10f6"},
]
+[[package]]
+name = "pillow"
+version = "10.3.0"
+requires_python = ">=3.8"
+summary = "Python Imaging Library (Fork)"
+files = [
+ {file = "pillow-10.3.0-cp310-cp310-macosx_10_10_x86_64.whl", hash = "sha256:90b9e29824800e90c84e4022dd5cc16eb2d9605ee13f05d47641eb183cd73d45"},
+ {file = "pillow-10.3.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a2c405445c79c3f5a124573a051062300936b0281fee57637e706453e452746c"},
+ {file = "pillow-10.3.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:78618cdbccaa74d3f88d0ad6cb8ac3007f1a6fa5c6f19af64b55ca170bfa1edf"},
+ {file = "pillow-10.3.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:261ddb7ca91fcf71757979534fb4c128448b5b4c55cb6152d280312062f69599"},
+ {file = "pillow-10.3.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:ce49c67f4ea0609933d01c0731b34b8695a7a748d6c8d186f95e7d085d2fe475"},
+ {file = "pillow-10.3.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:b14f16f94cbc61215115b9b1236f9c18403c15dd3c52cf629072afa9d54c1cbf"},
+ {file = "pillow-10.3.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:d33891be6df59d93df4d846640f0e46f1a807339f09e79a8040bc887bdcd7ed3"},
+ {file = "pillow-10.3.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:b50811d664d392f02f7761621303eba9d1b056fb1868c8cdf4231279645c25f5"},
+ {file = "pillow-10.3.0-cp310-cp310-win32.whl", hash = "sha256:ca2870d5d10d8726a27396d3ca4cf7976cec0f3cb706debe88e3a5bd4610f7d2"},
+ {file = "pillow-10.3.0-cp310-cp310-win_amd64.whl", hash = "sha256:f0d0591a0aeaefdaf9a5e545e7485f89910c977087e7de2b6c388aec32011e9f"},
+ {file = "pillow-10.3.0-cp310-cp310-win_arm64.whl", hash = "sha256:ccce24b7ad89adb5a1e34a6ba96ac2530046763912806ad4c247356a8f33a67b"},
+ {file = "pillow-10.3.0-cp311-cp311-macosx_10_10_x86_64.whl", hash = "sha256:5f77cf66e96ae734717d341c145c5949c63180842a545c47a0ce7ae52ca83795"},
+ {file = "pillow-10.3.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e4b878386c4bf293578b48fc570b84ecfe477d3b77ba39a6e87150af77f40c57"},
+ {file = "pillow-10.3.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fdcbb4068117dfd9ce0138d068ac512843c52295ed996ae6dd1faf537b6dbc27"},
+ {file = "pillow-10.3.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9797a6c8fe16f25749b371c02e2ade0efb51155e767a971c61734b1bf6293994"},
+ {file = "pillow-10.3.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:9e91179a242bbc99be65e139e30690e081fe6cb91a8e77faf4c409653de39451"},
+ {file = "pillow-10.3.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:1b87bd9d81d179bd8ab871603bd80d8645729939f90b71e62914e816a76fc6bd"},
+ {file = "pillow-10.3.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:81d09caa7b27ef4e61cb7d8fbf1714f5aec1c6b6c5270ee53504981e6e9121ad"},
+ {file = "pillow-10.3.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:048ad577748b9fa4a99a0548c64f2cb8d672d5bf2e643a739ac8faff1164238c"},
+ {file = "pillow-10.3.0-cp311-cp311-win32.whl", hash = "sha256:7161ec49ef0800947dc5570f86568a7bb36fa97dd09e9827dc02b718c5643f09"},
+ {file = "pillow-10.3.0-cp311-cp311-win_amd64.whl", hash = "sha256:8eb0908e954d093b02a543dc963984d6e99ad2b5e36503d8a0aaf040505f747d"},
+ {file = "pillow-10.3.0-cp311-cp311-win_arm64.whl", hash = "sha256:4e6f7d1c414191c1199f8996d3f2282b9ebea0945693fb67392c75a3a320941f"},
+ {file = "pillow-10.3.0-cp39-cp39-macosx_10_10_x86_64.whl", hash = "sha256:2ed854e716a89b1afcedea551cd85f2eb2a807613752ab997b9974aaa0d56936"},
+ {file = "pillow-10.3.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:dc1a390a82755a8c26c9964d457d4c9cbec5405896cba94cf51f36ea0d855002"},
+ {file = "pillow-10.3.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4203efca580f0dd6f882ca211f923168548f7ba334c189e9eab1178ab840bf60"},
+ {file = "pillow-10.3.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3102045a10945173d38336f6e71a8dc71bcaeed55c3123ad4af82c52807b9375"},
+ {file = "pillow-10.3.0-cp39-cp39-manylinux_2_28_aarch64.whl", hash = "sha256:6fb1b30043271ec92dc65f6d9f0b7a830c210b8a96423074b15c7bc999975f57"},
+ {file = "pillow-10.3.0-cp39-cp39-manylinux_2_28_x86_64.whl", hash = "sha256:1dfc94946bc60ea375cc39cff0b8da6c7e5f8fcdc1d946beb8da5c216156ddd8"},
+ {file = "pillow-10.3.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:b09b86b27a064c9624d0a6c54da01c1beaf5b6cadfa609cf63789b1d08a797b9"},
+ {file = "pillow-10.3.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:d3b2348a78bc939b4fed6552abfd2e7988e0f81443ef3911a4b8498ca084f6eb"},
+ {file = "pillow-10.3.0-cp39-cp39-win32.whl", hash = "sha256:45ebc7b45406febf07fef35d856f0293a92e7417ae7933207e90bf9090b70572"},
+ {file = "pillow-10.3.0-cp39-cp39-win_amd64.whl", hash = "sha256:0ba26351b137ca4e0db0342d5d00d2e355eb29372c05afd544ebf47c0956ffeb"},
+ {file = "pillow-10.3.0-cp39-cp39-win_arm64.whl", hash = "sha256:50fd3f6b26e3441ae07b7c979309638b72abc1a25da31a81a7fbd9495713ef4f"},
+ {file = "pillow-10.3.0-pp310-pypy310_pp73-macosx_10_10_x86_64.whl", hash = "sha256:6b02471b72526ab8a18c39cb7967b72d194ec53c1fd0a70b050565a0f366d355"},
+ {file = "pillow-10.3.0-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:8ab74c06ffdab957d7670c2a5a6e1a70181cd10b727cd788c4dd9005b6a8acd9"},
+ {file = "pillow-10.3.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:048eeade4c33fdf7e08da40ef402e748df113fd0b4584e32c4af74fe78baaeb2"},
+ {file = "pillow-10.3.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9e2ec1e921fd07c7cda7962bad283acc2f2a9ccc1b971ee4b216b75fad6f0463"},
+ {file = "pillow-10.3.0-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:4c8e73e99da7db1b4cad7f8d682cf6abad7844da39834c288fbfa394a47bbced"},
+ {file = "pillow-10.3.0-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:16563993329b79513f59142a6b02055e10514c1a8e86dca8b48a893e33cf91e3"},
+ {file = "pillow-10.3.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:dd78700f5788ae180b5ee8902c6aea5a5726bac7c364b202b4b3e3ba2d293170"},
+ {file = "pillow-10.3.0-pp39-pypy39_pp73-macosx_10_10_x86_64.whl", hash = "sha256:aff76a55a8aa8364d25400a210a65ff59d0168e0b4285ba6bf2bd83cf675ba32"},
+ {file = "pillow-10.3.0-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:b7bc2176354defba3edc2b9a777744462da2f8e921fbaf61e52acb95bafa9828"},
+ {file = "pillow-10.3.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:793b4e24db2e8742ca6423d3fde8396db336698c55cd34b660663ee9e45ed37f"},
+ {file = "pillow-10.3.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d93480005693d247f8346bc8ee28c72a2191bdf1f6b5db469c096c0c867ac015"},
+ {file = "pillow-10.3.0-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:c83341b89884e2b2e55886e8fbbf37c3fa5efd6c8907124aeb72f285ae5696e5"},
+ {file = "pillow-10.3.0-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:1a1d1915db1a4fdb2754b9de292642a39a7fb28f1736699527bb649484fb966a"},
+ {file = "pillow-10.3.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:a0eaa93d054751ee9964afa21c06247779b90440ca41d184aeb5d410f20ff591"},
+ {file = "pillow-10.3.0.tar.gz", hash = "sha256:9d2455fbf44c914840c793e89aa82d0e1763a14253a000743719ae5946814b2d"},
+]
+
+[[package]]
+name = "portalocker"
+version = "2.8.2"
+requires_python = ">=3.8"
+summary = "Wraps the portalocker recipe for easy usage"
+dependencies = [
+ "pywin32>=226; platform_system == \"Windows\"",
+]
+files = [
+ {file = "portalocker-2.8.2-py3-none-any.whl", hash = "sha256:cfb86acc09b9aa7c3b43594e19be1345b9d16af3feb08bf92f23d4dce513a28e"},
+ {file = "portalocker-2.8.2.tar.gz", hash = "sha256:2b035aa7828e46c58e9b31390ee1f169b98e1066ab10b9a6a861fe7e25ee4f33"},
+]
+
+[[package]]
+name = "pycparser"
+version = "2.22"
+requires_python = ">=3.8"
+summary = "C parser in Python"
+files = [
+ {file = "pycparser-2.22-py3-none-any.whl", hash = "sha256:c3702b6d3dd8c7abc1afa565d7e63d53a1d0bd86cdc24edd75470f4de499cfcc"},
+ {file = "pycparser-2.22.tar.gz", hash = "sha256:491c8be9c040f5390f5bf44a5b07752bd07f56edf992381b05c701439eec10f6"},
+]
+
[[package]]
name = "pydantic"
version = "2.7.2"
@@ -1461,6 +1599,44 @@ files = [
{file = "pypdf-4.2.0.tar.gz", hash = "sha256:fe63f3f7d1dcda1c9374421a94c1bba6c6f8c4a62173a59b64ffd52058f846b1"},
]
+[[package]]
+name = "pyjwt"
+version = "2.8.0"
+requires_python = ">=3.7"
+summary = "JSON Web Token implementation in Python"
+files = [
+ {file = "PyJWT-2.8.0-py3-none-any.whl", hash = "sha256:59127c392cc44c2da5bb3192169a91f429924e17aff6534d70fdc02ab3e04320"},
+ {file = "PyJWT-2.8.0.tar.gz", hash = "sha256:57e28d156e3d5c10088e0c68abb90bfac3df82b40a71bd0daa20c65ccd5c23de"},
+]
+
+[[package]]
+name = "pyjwt"
+version = "2.8.0"
+extras = ["crypto"]
+requires_python = ">=3.7"
+summary = "JSON Web Token implementation in Python"
+dependencies = [
+ "PyJWT==2.8.0",
+ "cryptography>=3.4.0",
+]
+files = [
+ {file = "PyJWT-2.8.0-py3-none-any.whl", hash = "sha256:59127c392cc44c2da5bb3192169a91f429924e17aff6534d70fdc02ab3e04320"},
+ {file = "PyJWT-2.8.0.tar.gz", hash = "sha256:57e28d156e3d5c10088e0c68abb90bfac3df82b40a71bd0daa20c65ccd5c23de"},
+]
+
+[[package]]
+name = "pypdf"
+version = "4.2.0"
+requires_python = ">=3.6"
+summary = "A pure-python PDF library capable of splitting, merging, cropping, and transforming PDF files"
+dependencies = [
+ "typing-extensions>=4.0; python_version < \"3.11\"",
+]
+files = [
+ {file = "pypdf-4.2.0-py3-none-any.whl", hash = "sha256:dc035581664e0ad717e3492acebc1a5fc23dba759e788e3d4a9fc9b1a32e72c1"},
+ {file = "pypdf-4.2.0.tar.gz", hash = "sha256:fe63f3f7d1dcda1c9374421a94c1bba6c6f8c4a62173a59b64ffd52058f846b1"},
+]
+
[[package]]
name = "python-dateutil"
version = "2.9.0.post0"
@@ -1754,6 +1930,15 @@ files = [
{file = "striprtf-0.0.26.tar.gz", hash = "sha256:fdb2bba7ac440072d1c41eab50d8d74ae88f60a8b6575c6e2c7805dc462093aa"},
]
+[[package]]
+name = "striprtf"
+version = "0.0.26"
+summary = "A simple library to convert rtf to text"
+files = [
+ {file = "striprtf-0.0.26-py3-none-any.whl", hash = "sha256:8c8f9d32083cdc2e8bfb149455aa1cc5a4e0a035893bedc75db8b73becb3a1bb"},
+ {file = "striprtf-0.0.26.tar.gz", hash = "sha256:fdb2bba7ac440072d1c41eab50d8d74ae88f60a8b6575c6e2c7805dc462093aa"},
+]
+
[[package]]
name = "tenacity"
version = "8.3.0"
@@ -1853,6 +2038,7 @@ summary = "Core library that helps with executing workflows."
dependencies = [
"boto3~=1.28.17",
"botocore~=1.31.17",
+ "kombu==5.3.7",
"llama-index-llms-azure-openai==0.1.5",
"llama-index==0.10.28",
"redis~=5.0.1",
@@ -1869,6 +2055,16 @@ files = [
{file = "urllib3-1.26.18.tar.gz", hash = "sha256:f8ecc1bba5667413457c529ab955bf8c67b45db799d159066261719e328580a0"},
]
+[[package]]
+name = "vine"
+version = "5.1.0"
+requires_python = ">=3.6"
+summary = "Python promises."
+files = [
+ {file = "vine-5.1.0-py3-none-any.whl", hash = "sha256:40fdf3c48b2cfe1c38a49e9ae2da6fda88e4794c810050a728bd7413811fb1dc"},
+ {file = "vine-5.1.0.tar.gz", hash = "sha256:8b62e981d35c41049211cf62a0a1242d8c1ee9bd15bb196ce38aefd6799e61e0"},
+]
+
[[package]]
name = "websocket-client"
version = "1.8.0"
diff --git a/worker/sample.env b/worker/sample.env
index c43173fcf..b9de45c17 100644
--- a/worker/sample.env
+++ b/worker/sample.env
@@ -1,7 +1,4 @@
-REDIS_HOST="unstract-redis"
-REDIS_PORT=6379
-REDIS_PASSWORD=""
-REDIS_USER=default
+CELERY_BROKER_URL = "redis://unstract-redis:6379"
TOOL_CONTAINER_NETWORK="unstract-network"
TOOL_CONTAINER_LABELS="[]"