LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
Fix/socket io design changes (#385)
ali0ee58329340b
diff --git a/backend/backend/__init__.py b/backend/backend/__init__.pyindex 370372af0..e921073a8 100644--- a/backend/backend/__init__.py+++ b/backend/backend/__init__.py@@ -1,3 +1,3 @@-from .celery import app as celery_app+from .celery_service import app as celery_app__all__ = ["celery_app"]diff --git a/backend/backend/celery.py b/backend/backend/celery_service.pysimilarity index 92%rename from backend/backend/celery.pyrename to backend/backend/celery_service.pyindex 57f387391..99effcd78 100644--- a/backend/backend/celery.py+++ b/backend/backend/celery_service.py@@ -6,6 +6,8 @@ from celery import Celeryfrom django.conf import settingsfrom utils.constants import ExecutionLogConstants+from backend.celery_task import TaskRegistry+# Set the default Django settings module for the 'celery' program.os.environ.setdefault("DJANGO_SETTINGS_MODULE",@@ -15,10 +17,14 @@ os.environ.setdefault(# Create a Celery instance. Default time zone is UTC.app = Celery("backend")+# To register the custom tasks.+TaskRegistry()+# Use Redis as the message broker.app.conf.broker_url = settings.CELERY_BROKER_URLapp.conf.result_backend = settings.CELERY_RESULT_BACKEND+# Load task modules from all registered Django app configs.app.config_from_object("django.conf:settings", namespace="CELERY")diff --git a/backend/backend/celery_task.py b/backend/backend/celery_task.pynew file mode 100644index 000000000..bc3b709e3--- /dev/null+++ b/backend/backend/celery_task.py@@ -0,0 +1,34 @@+import logging+import os+from typing import Any++from celery import shared_task+from utils.log_events import handle_user_logs++from unstract.core.constants import LogEventArgument, LogProcessingTask++logger = logging.getLogger(__name__)+++class TaskRegistry:+ @staticmethod+ @shared_task(+ name=LogProcessingTask.TASK_NAME,+ )+ def log_consumer(**kwargs: Any) -> None:+ """Task to process the logs from log publisher.++ Args:+ kwargs: The arguments to process the logs.+ Expected arguments:+ USER_SESSION_ID: The room to be processed.+ EVENT: The event to be processed Ex: logs:{session_id}.+ MESSAGE: The message to be processed Ex: execution log.+ """+ log_message = kwargs.get(LogEventArgument.MESSAGE)+ room = kwargs.get(LogEventArgument.USER_SESSION_ID)+ event = kwargs.get(LogEventArgument.EVENT)+ logger.debug(+ f"[{os.getpid()}] Log message received: {log_message} for the room {room}"+ )+ handle_user_logs(room=room, event=event, message=log_message)diff --git a/backend/backend/settings/base.py b/backend/backend/settings/base.pyindex d043d1664..d18b3655d 100644--- a/backend/backend/settings/base.py+++ b/backend/backend/settings/base.py@@ -160,6 +160,9 @@ LOG_HISTORY_CONSUMER_INTERVAL = int(get_required_setting("LOG_HISTORY_CONSUMER_INTERVAL", "60"))LOGS_BATCH_LIMIT = int(get_required_setting("LOGS_BATCH_LIMIT", "30"))+CELERY_BROKER_URL = get_required_setting(+ "CELERY_BROKER_URL", f"redis://{REDIS_HOST}:{REDIS_PORT}"+)# Flag to Enable django adminADMIN_ENABLED = False@@ -309,6 +312,9 @@ DATABASES = {}}+# SocketIO connection manager+SOCKET_IO_MANAGER_URL = f"redis://{REDIS_HOST}:{REDIS_PORT}"+DATABASE_ROUTERS = ("django_tenants.routers.TenantSyncRouter",)CACHES = {@@ -332,9 +338,6 @@ RQ_QUEUES = {"default": {"USE_REDIS_CACHE": "default"},}-# Used for asynchronous/Queued execution-# Celery based scheduler-CELERY_BROKER_URL = f"redis://{REDIS_HOST}:{REDIS_PORT}"# CELERY_RESULT_BACKEND = f"redis://{REDIS_HOST}:{REDIS_PORT}/1"# Postgres as result backenddiff --git a/backend/pdm.lock b/backend/pdm.lockindex 57c972c42..d9fb2b2ae 100644--- a/backend/pdm.lock+++ b/backend/pdm.lock@@ -4669,6 +4669,7 @@ groups = ["default"]dependencies = ["boto3~=1.28.17","botocore~=1.31.17",+ "kombu==5.3.7","llama-index-llms-azure-openai==0.1.5","llama-index==0.10.28","redis~=5.0.1",diff --git a/backend/pyproject.toml b/backend/pyproject.tomlindex ef6957776..6708a047b 100644--- a/backend/pyproject.toml+++ b/backend/pyproject.toml@@ -78,7 +78,7 @@ migrate_db.cmd = "python manage.py migrate"migrate_db.env_file = ".env"migrate_db.help = "Performs DB migrations for Unstract backend"# Commands for backend execution-consumer.cmd = "celery -A backend worker --loglevel=info -Q celery,celery_periodic_logs"+consumer.cmd = "celery -A backend worker --loglevel=info -Q celery,celery_periodic_logs,celery_log_task_queue --autoscale 8,1"consumer.env_file = ".env"consumer.help = "Runs the Unstract consumer"# Celery Flowerdiff --git a/backend/sample.env b/backend/sample.envindex 765331cbe..cc8991a38 100644--- a/backend/sample.env+++ b/backend/sample.env@@ -137,3 +137,6 @@ ENABLE_LOG_HISTORY=TrueLOG_HISTORY_CONSUMER_INTERVAL=30# Maximum number of logs to insert in a single batch.LOGS_BATCH_LIMIT=30++# Celery Configuration+CELERY_BROKER_URL = "redis://unstract-redis:6379"diff --git a/backend/utils/log_events.py b/backend/utils/log_events.pyindex c8ecfb989..309a879a7 100644--- a/backend/utils/log_events.py+++ b/backend/utils/log_events.py@@ -1,8 +1,7 @@+import httpimport jsonimport loggingimport os-import threading-import timefrom typing import Any, Optionalimport redis@@ -24,7 +23,9 @@ sio = socketio.Server(logger=False,engineio_logger=False,always_connect=True,+ client_manager=socketio.KombuManager(url=settings.SOCKET_IO_MANAGER_URL),)+redis_conn = redis.Redis(host=settings.REDIS_HOST,port=int(settings.REDIS_PORT),@@ -35,8 +36,17 @@ redis_conn = redis.Redis(@sio.eventdef connect(sid: str, environ: Any, auth: Any) -> None:- # TODO Authenticate websocket connections+ """This function is called when a client connects to the server.++ It handles the connection and authentication of the client.+ """logger.info(f"[{os.getpid()}] Client with SID:{sid} connected")+ session_id = _get_user_session_id_from_cookies(sid, environ)+ if session_id:+ sio.enter_room(sid, session_id)+ logger.info(f"Entered room {session_id} for socket {sid}")+ else:+ sio.disconnect(sid)@sio.event@@ -44,6 +54,31 @@ def disconnect(sid: str) -> None:logger.info(f"[{os.getpid()}] Client with SID:{sid} disconnected")+def _get_user_session_id_from_cookies(sid: str, environ: Any) -> Optional[str]:+ """Get the user session ID from cookies.++ Args:+ sid (str): The socket ID of the client.+ environ (Any): The environment variables of the client.++ Returns:+ Optional[str]: The user session ID.+ """+ cookie_str = environ.get("HTTP_COOKIE")+ if not cookie_str:+ logger.warning(f"No cookies found in {environ} for the sid {sid}")+ return None++ cookie = http.cookies.SimpleCookie(cookie_str)+ session_id = cookie.get(settings.SESSION_COOKIE_NAME)++ if not session_id:+ logger.warning(f"No session ID found in cookies for SID {sid}")+ return None++ return session_id.value++def _get_validated_log_data(json_data: Any) -> Optional[LogDataDTO]:"""Validate log data to persist history. This function takes log data inJSON format, validates it, and returns a `LogDataDTO` object if the data is@@ -82,7 +117,7 @@ def _get_validated_log_data(json_data: Any) -> Optional[LogDataDTO]:# Check if all required fields are presentif not all((execution_id, organization_id, timestamp)):- logger.warning(f"Missing required fields while validating {json_data}")+ logger.debug(f"Missing required fields while validating {json_data}")returnreturn LogDataDTO(@@ -94,10 +129,10 @@ def _get_validated_log_data(json_data: Any) -> Optional[LogDataDTO]:)-def _store_execution_log(data: bytes) -> None:+def _store_execution_log(data: dict[str, Any]) -> None:"""Store execution log in databaseArgs:- data (bytes): Execution log data in bytes format+ data (dict[str, Any]): Execution log data"""if not ExecutionLogConstants.IS_ENABLED:return@@ -109,69 +144,36 @@ def _store_execution_log(data: bytes) -> None:logger.error(f"Error storing execution log: {e}")-def _emit_websocket_event(channel: str, data: bytes) -> None:+def _emit_websocket_event(room: str, event: str, data: dict[str, Any]) -> None:"""Emit websocket eventArgs:- channel (str): WebSocket channel- data (bytes): Execution log data in bytes format+ room (str): Room to emit event to+ event (str): Event name+ channel (str): Channel name+ data (bytes): Data to emit"""payload = {"data": data}try:- logger.debug(f"[{os.getpid()}] Push websocket event: {channel}, {payload}")- sio.emit(channel, payload)+ logger.debug(f"[{os.getpid()}] Push websocket event: {event}, {payload}")+ sio.emit(event, data=payload, room=room)except Exception as e:logger.error(f"Error emitting WebSocket event: {e}")-def _handle_pubsub_messages(message: dict[str, Any]) -> None:- """Handle pubsub messages+def handle_user_logs(room: str, event: str, message: dict[str, Any]) -> None:+ """Handle user logs from applicationsArgs:- message (dict[str, Any]): Pub sub message+ message (dict[str, Any]): log message"""- channel = message.get("channel")- data = message.get("data")- if not channel or not data:- logger.warning(f"Invalid message received: {message}")+ if not room or not event:+ logger.warning(f"Message received without room and event: {message}")return- try:- channel_str = channel.decode("utf-8")- except UnicodeDecodeError as e:- logger.error(f"Error decoding channel: {e}")- return-- _store_execution_log(data)- _emit_websocket_event(channel_str, data)---def _pubsub_listen_forever() -> None:- global shutdown-- try:- pubsub = redis_conn.pubsub(ignore_subscribe_messages=True)- pubsub.psubscribe("logs:*")-- logger.info(f"[{os.getpid()}] Listening for pub sub messages...")- while True:- message = pubsub.get_message()-- if message:- logger.debug(f"[{os.getpid()}] Pub sub message received: {message}")- if message["type"] == "pmessage":- _handle_pubsub_messages(message)-- # TODO Add graceful shutdown-- time.sleep(0.01)- except Exception as e:- logger.error(f"[{os.getpid()}] Failed to do pubsub: {e}")+ _store_execution_log(message)+ _emit_websocket_event(room, event, message)def start_server(django_app: WSGIHandler, namespace: str) -> WSGIHandler:django_app = socketio.WSGIApp(sio, django_app, socketio_path=namespace)-- pubsub_listener = threading.Thread(target=_pubsub_listen_forever, daemon=True)- pubsub_listener.start()-return django_appdiff --git a/docker/docker-compose.yaml b/docker/docker-compose.yamlindex bcca7c6c4..33de0fed5 100644--- a/docker/docker-compose.yaml+++ b/docker/docker-compose.yaml@@ -39,7 +39,7 @@ services:container_name: unstract-execution-consumerrestart: unless-stoppedentrypoint: .venv/bin/celery- command: "-A backend worker --loglevel=info -Q celery,celery_periodic_logs --autoscale=8,1"+ command: "-A backend worker --loglevel=info -Q celery,celery_periodic_logs,celery_log_task_queue --autoscale=8,1"env_file:- ../backend/.envdepends_on:diff --git a/frontend/src/components/helpers/socket-messages/SocketMessages.js b/frontend/src/components/helpers/socket-messages/SocketMessages.jsindex 207120ef8..72bb2d3af 100644--- a/frontend/src/components/helpers/socket-messages/SocketMessages.js+++ b/frontend/src/components/helpers/socket-messages/SocketMessages.js@@ -32,7 +32,16 @@ function SocketMessages() {const onMessage = (data) => {try {- const msg = JSON.parse(new TextDecoder().decode(data.data));+ let msg = data.data;+ // Attempt to decode data as JSON if it's in encoded state+ if (typeof msg === "string" || msg instanceof Uint8Array) {+ if (typeof msg === "string") {+ msg = JSON.parse(msg);+ } else {+ msg = JSON.parse(new TextDecoder().decode(msg));+ }+ }+if ((msg?.type === "LOG" || msg?.type === "COST") &&msg?.service !== "prompt"diff --git a/unstract/core/pdm.lock b/unstract/core/pdm.lockindex 0b4807be4..18b4d29a1 100644--- a/unstract/core/pdm.lock+++ b/unstract/core/pdm.lock@@ -5,7 +5,7 @@groups = ["default"]strategy = ["cross_platform", "inherit_metadata"]lock_version = "4.4.1"-content_hash = "sha256:f8589435b9bfce4e1b1aca6600ec970edfe816e0ba7900ef67ab176f8c7dd4d3"+content_hash = "sha256:19044e2ce12da08c83b008c16d2de4d46531c7b91a36296429729d6340c77eae"[[package]]name = "aiohttp"@@ -84,6 +84,20 @@ files = [{file = "aiosignal-1.3.1.tar.gz", hash = "sha256:54cd96e15e1649b75d6c87526a6ff0b6c1b0dd3459f43d9ca11d48c339b68cfc"},]+[[package]]+name = "amqp"+version = "5.2.0"+requires_python = ">=3.6"+summary = "Low-level AMQP client for Python (fork of amqplib)."+groups = ["default"]+dependencies = [+ "vine<6.0.0,>=5.0.0",+]+files = [+ {file = "amqp-5.2.0-py3-none-any.whl", hash = "sha256:827cb12fb0baa892aad844fd95258143bce4027fdac4fccddbc43330fd281637"},+ {file = "amqp-5.2.0.tar.gz", hash = "sha256:a1ecff425ad063ad42a486c902807d1482311481c8ad95a72694b2975e75f7fd"},+]+[[package]]name = "annotated-types"version = "0.6.0"@@ -643,6 +657,22 @@ files = [{file = "joblib-1.4.0.tar.gz", hash = "sha256:1eb0dc091919cd384490de890cb5dfd538410a6d4b3b54eef09fb8c50b409b1c"},]+[[package]]+name = "kombu"+version = "5.3.7"+requires_python = ">=3.8"+summary = "Messaging library for Python."+groups = ["default"]+dependencies = [+ "amqp<6.0.0,>=5.1.1",+ "typing-extensions; python_version < \"3.10\"",+ "vine",+]+files = [+ {file = "kombu-5.3.7-py3-none-any.whl", hash = "sha256:5634c511926309c7f9789f1433e9ed402616b56836ef9878f01bd59267b4c7a9"},+ {file = "kombu-5.3.7.tar.gz", hash = "sha256:011c4cd9a355c14a1de8d35d257314a1d2456d52b7140388561acac3cf1a97bf"},+]+[[package]]name = "llama-index"version = "0.10.28"@@ -1850,6 +1880,17 @@ files = [{file = "urllib3-1.26.18.tar.gz", hash = "sha256:f8ecc1bba5667413457c529ab955bf8c67b45db799d159066261719e328580a0"},]+[[package]]+name = "vine"+version = "5.1.0"+requires_python = ">=3.6"+summary = "Python promises."+groups = ["default"]+files = [+ {file = "vine-5.1.0-py3-none-any.whl", hash = "sha256:40fdf3c48b2cfe1c38a49e9ae2da6fda88e4794c810050a728bd7413811fb1dc"},+ {file = "vine-5.1.0.tar.gz", hash = "sha256:8b62e981d35c41049211cf62a0a1242d8c1ee9bd15bb196ce38aefd6799e61e0"},+]+[[package]]name = "wrapt"version = "1.16.0"diff --git a/unstract/core/pyproject.toml b/unstract/core/pyproject.tomlindex a2ea5c5b6..c9b9928a3 100644--- a/unstract/core/pyproject.toml+++ b/unstract/core/pyproject.toml@@ -16,6 +16,7 @@ dependencies = ["llama-index-llms-azure-openai==0.1.5","redis~=5.0.1","requests==2.31.0",+ "kombu==5.3.7",]# <3.11.1 due to resolution error from Unstract SDKrequires-python = ">=3.9,<3.11.1"diff --git a/unstract/core/src/unstract/core/constants.py b/unstract/core/src/unstract/core/constants.pyindex 5f397e7f6..73303df0c 100644--- a/unstract/core/src/unstract/core/constants.py+++ b/unstract/core/src/unstract/core/constants.py@@ -5,3 +5,14 @@ class LogFieldName:TYPE = "type"DATA = "data"EVENT_TIME = "event_time"+++class LogEventArgument:+ EVENT = "event"+ MESSAGE = "message"+ USER_SESSION_ID = "user_session_id"+++class LogProcessingTask:+ TASK_NAME = "logs_consumer"+ QUEUE_NAME = "celery_log_task_queue"diff --git a/unstract/core/src/unstract/core/pubsub_helper.py b/unstract/core/src/unstract/core/pubsub_helper.pyindex 5658c0076..9011c40a0 100644--- a/unstract/core/src/unstract/core/pubsub_helper.py+++ b/unstract/core/src/unstract/core/pubsub_helper.py@@ -1,19 +1,16 @@-import jsonimport loggingimport osfrom datetime import datetime, timezonefrom typing import Any, Optional-import redis+from kombu import Connection++from unstract.core.constants import LogEventArgument, LogProcessingTaskclass LogPublisher:- r = redis.Redis(- host=os.environ.get("REDIS_HOST", "http://localhost"),- port=os.environ.get("REDIS_PORT", "6379"),- username=os.environ.get("REDIS_USER", ""),- password=os.environ.get("REDIS_PASSWORD", ""),- )++ kombu_conn = Connection(os.environ.get("CELERY_BROKER_URL"))@staticmethoddef log_usage(@@ -93,12 +90,53 @@ class LogPublisher:"message": message,}+ @classmethod+ def _get_task_message(+ cls, user_session_id: str, event: str, message: Any+ ) -> dict[str, Any]:++ task_kwargs = {+ LogEventArgument.EVENT: event,+ LogEventArgument.MESSAGE: message,+ LogEventArgument.USER_SESSION_ID: user_session_id,+ }+ task_message = {+ "args": [],+ "kwargs": task_kwargs,+ "retries": 0,+ "utc": True,+ }+ return task_message++ @classmethod+ def _get_task_header(cls, task_name: str) -> dict[str, Any]:+ return {+ "task": task_name,+ }+@classmethoddef publish(cls, channel_id: str, payload: dict[str, Any]) -> bool:- channel = f"logs:{channel_id}"+ """Publish a message to the queue."""try:- cls.r.publish(channel, json.dumps(payload))+ with cls.kombu_conn.Producer(serializer="json") as producer:+ event = f"logs:{channel_id}"+ task_message = cls._get_task_message(+ user_session_id=channel_id,+ event=event,+ message=payload,+ )+ headers = cls._get_task_header(LogProcessingTask.TASK_NAME)+ # Publish the message to the queue+ producer.publish(+ body=task_message,+ exchange="",+ headers=headers,+ routing_key=LogProcessingTask.QUEUE_NAME,+ compression=None,+ retry=True,+ )+ logging.debug(f"Published '{channel_id}' <= {payload}")except Exception as e:- logging.error(f"Failed to publish '{channel}' <= {payload}: {e}")+ logging.error(f"Failed to publish '{channel_id}' <= {payload}: {e}")return Falsereturn Truediff --git a/worker/README.md b/worker/README.mdindex f29ff471e..0cc49d739 100644--- a/worker/README.md+++ b/worker/README.md@@ -39,12 +39,9 @@ sudo ln -s "$HOME/.docker/run/docker.sock" /var/run/docker.sock| Variable | Description || -------------------------- | ---------------------------------------------------------------------------------------|-| `REDIS_HOST` | Host address of the Redis server. |-| `REDIS_PORT` | Port number for the Redis server. |-| `REDIS_PASSWORD` | Password for accessing the Redis server. (Leave empty if not required) |-| `REDIS_USER` | User for accessing the Redis server. (If applicable) |+| `CELERY_BROKER_URL` | URL for Celery's message broker, used to queue tasks. Must match backend configuration.|| `TOOL_CONTAINER_NETWORK` | Network used for running tool containers. |-| `TOOL_CONTAINER_LABELS` | Labels applied to tool containers for observability [Optional]. |+| `TOOL_CONTAINER_LABELS` | Labels applied to tool containers for observability [Optional]. || `WORKFLOW_DATA_DIR` | Source mount bind directory for tool containers to access input files. || `TOOL_DATA_DIR` | Target mount directory within tool containers. (Default: "/data") || `LOG_LEVEL` | Log level for worker (Options: INFO, WARNING, ERROR, DEBUG, etc.) |diff --git a/worker/pdm.lock b/worker/pdm.lockindex a273d0cc7..750f29977 100644--- a/worker/pdm.lock+++ b/worker/pdm.lock@@ -82,6 +82,19 @@ files = [{file = "aiosignal-1.3.1.tar.gz", hash = "sha256:54cd96e15e1649b75d6c87526a6ff0b6c1b0dd3459f43d9ca11d48c339b68cfc"},]+[[package]]+name = "amqp"+version = "5.2.0"+requires_python = ">=3.6"+summary = "Low-level AMQP client for Python (fork of amqplib)."+dependencies = [+ "vine<6.0.0,>=5.0.0",+]+files = [+ {file = "amqp-5.2.0-py3-none-any.whl", hash = "sha256:827cb12fb0baa892aad844fd95258143bce4027fdac4fccddbc43330fd281637"},+ {file = "amqp-5.2.0.tar.gz", hash = "sha256:a1ecff425ad063ad42a486c902807d1482311481c8ad95a72694b2975e75f7fd"},+]+[[package]]name = "annotated-types"version = "0.7.0"@@ -702,6 +715,21 @@ files = [{file = "joblib-1.4.2.tar.gz", hash = "sha256:2382c5816b2636fbd20a09e0f4e9dad4736765fdfb7dca582943b9c1366b3f0e"},]+[[package]]+name = "kombu"+version = "5.3.7"+requires_python = ">=3.8"+summary = "Messaging library for Python."+dependencies = [+ "amqp<6.0.0,>=5.1.1",+ "typing-extensions; python_version < \"3.10\"",+ "vine",+]+files = [+ {file = "kombu-5.3.7-py3-none-any.whl", hash = "sha256:5634c511926309c7f9789f1433e9ed402616b56836ef9878f01bd59267b4c7a9"},+ {file = "kombu-5.3.7.tar.gz", hash = "sha256:011c4cd9a355c14a1de8d35d257314a1d2456d52b7140388561acac3cf1a97bf"},+]+[[package]]name = "llama-index"version = "0.10.28"@@ -728,7 +756,7 @@ files = [[[package]]name = "llama-index-agent-openai"-version = "0.2.6"+version = "0.2.7"requires_python = "<4.0,>=3.8.1"summary = "llama-index agent openai integration"dependencies = [@@ -737,8 +765,8 @@ dependencies = ["openai>=1.14.0",]files = [- {file = "llama_index_agent_openai-0.2.6-py3-none-any.whl", hash = "sha256:fddf5a21b5cc584d4aab4517edcdd7ccdba7bf2b66ba6f1ae31d6b87800b4246"},- {file = "llama_index_agent_openai-0.2.6.tar.gz", hash = "sha256:32d95ab8ebd8fb8174e34427e6cfcddb56003da7029be9a1d3fe002540d13b9a"},+ {file = "llama_index_agent_openai-0.2.7-py3-none-any.whl", hash = "sha256:34be65011a508dd8cab0c9a606594f28075b98b0cebe69e3c543adc8564fee0d"},+ {file = "llama_index_agent_openai-0.2.7.tar.gz", hash = "sha256:13ce535f03e32c821763c01e26af4222f3981178622414d3868013a1946e8124"},][[package]]@@ -758,7 +786,7 @@ files = [[[package]]name = "llama-index-core"-version = "0.10.41"+version = "0.10.43"requires_python = "<4.0,>=3.8.1"summary = "Interface between LLMs and your data"dependencies = [@@ -787,8 +815,8 @@ dependencies = ["wrapt",]files = [- {file = "llama_index_core-0.10.41-py3-none-any.whl", hash = "sha256:cc404f6ec3c9ba563b1acd9005b95e18ba6ca5da6e83ac64d555821bd7c6f574"},- {file = "llama_index_core-0.10.41.tar.gz", hash = "sha256:e514ee5a070c405f4864f07a9fda46950e06797ee5a98c30f3c5592ca0203118"},+ {file = "llama_index_core-0.10.43-py3-none-any.whl", hash = "sha256:355a5f7808dfa09dc27b6bd564926fa697e0279344b829732a9f91b8000157ac"},+ {file = "llama_index_core-0.10.43.tar.gz", hash = "sha256:c7dce5158e112d3cbc8d331c303e830b2eabcae55c9636e8050dc85320522c1b"},][[package]]@@ -866,15 +894,15 @@ files = [[[package]]name = "llama-index-llms-openai"-version = "0.1.21"+version = "0.1.22"requires_python = "<4.0,>=3.8.1"summary = "llama-index llms openai integration"dependencies = ["llama-index-core<0.11.0,>=0.10.24",]files = [- {file = "llama_index_llms_openai-0.1.21-py3-none-any.whl", hash = "sha256:fd2fa240238c7dca0170bf55957bb088d9b6acf63c80924eb59498f3a5ac0df3"},- {file = "llama_index_llms_openai-0.1.21.tar.gz", hash = "sha256:e9d631135160ac87093cd7d5ae3e32c4debf8eeaafa29ee4739675867155d773"},+ {file = "llama_index_llms_openai-0.1.22-py3-none-any.whl", hash = "sha256:84a8c910671460ad724ed818192f209f7481e71bcc6528553ba7e66db2e14bcd"},+ {file = "llama_index_llms_openai-0.1.22.tar.gz", hash = "sha256:729bf2ea7043517465e1d585089512b77d8b3ce92233a67c138d5d621061ed56"},][[package]]@@ -1061,6 +1089,37 @@ files = [{file = "msal_extensions-1.1.0-py3-none-any.whl", hash = "sha256:01be9711b4c0b1a151450068eeb2c4f0997df3bba085ac299de3a66f585e382f"},]+[[package]]+name = "msal"+version = "1.28.0"+requires_python = ">=3.7"+summary = "The Microsoft Authentication Library (MSAL) for Python library enables your app to access the Microsoft Cloud by supporting authentication of users with Microsoft Azure Active Directory accounts (AAD) and Microsoft Accounts (MSA) using industry standard OAuth2 and OpenID Connect."+dependencies = [+ "PyJWT[crypto]<3,>=1.0.0",+ "cryptography<45,>=0.6",+ "requests<3,>=2.0.0",+]+files = [+ {file = "msal-1.28.0-py3-none-any.whl", hash = "sha256:3064f80221a21cd535ad8c3fafbb3a3582cd9c7e9af0bb789ae14f726a0ca99b"},+ {file = "msal-1.28.0.tar.gz", hash = "sha256:80bbabe34567cb734efd2ec1869b2d98195c927455369d8077b3c542088c5c9d"},+]++[[package]]+name = "msal-extensions"+version = "1.1.0"+requires_python = ">=3.7"+summary = "Microsoft Authentication Library extensions (MSAL EX) provides a persistence API that can save your data on disk, encrypted on Windows, macOS and Linux. Concurrent data access will be coordinated by a file lock mechanism."+dependencies = [+ "msal<2.0.0,>=0.4.1",+ "packaging",+ "portalocker<3,>=1.0; platform_system != \"Windows\"",+ "portalocker<3,>=1.6; platform_system == \"Windows\"",+]+files = [+ {file = "msal-extensions-1.1.0.tar.gz", hash = "sha256:6ab357867062db7b253d0bd2df6d411c7891a0ee7308d54d1e4317c1d1c54252"},+ {file = "msal_extensions-1.1.0-py3-none-any.whl", hash = "sha256:01be9711b4c0b1a151450068eeb2c4f0997df3bba085ac299de3a66f585e382f"},+]+[[package]]name = "multidict"version = "6.0.5"@@ -1343,6 +1402,85 @@ files = [{file = "pycparser-2.22.tar.gz", hash = "sha256:491c8be9c040f5390f5bf44a5b07752bd07f56edf992381b05c701439eec10f6"},]+[[package]]+name = "pillow"+version = "10.3.0"+requires_python = ">=3.8"+summary = "Python Imaging Library (Fork)"+files = [+ {file = "pillow-10.3.0-cp310-cp310-macosx_10_10_x86_64.whl", hash = "sha256:90b9e29824800e90c84e4022dd5cc16eb2d9605ee13f05d47641eb183cd73d45"},+ {file = "pillow-10.3.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a2c405445c79c3f5a124573a051062300936b0281fee57637e706453e452746c"},+ {file = "pillow-10.3.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:78618cdbccaa74d3f88d0ad6cb8ac3007f1a6fa5c6f19af64b55ca170bfa1edf"},+ {file = "pillow-10.3.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:261ddb7ca91fcf71757979534fb4c128448b5b4c55cb6152d280312062f69599"},+ {file = "pillow-10.3.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:ce49c67f4ea0609933d01c0731b34b8695a7a748d6c8d186f95e7d085d2fe475"},+ {file = "pillow-10.3.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:b14f16f94cbc61215115b9b1236f9c18403c15dd3c52cf629072afa9d54c1cbf"},+ {file = "pillow-10.3.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:d33891be6df59d93df4d846640f0e46f1a807339f09e79a8040bc887bdcd7ed3"},+ {file = "pillow-10.3.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:b50811d664d392f02f7761621303eba9d1b056fb1868c8cdf4231279645c25f5"},+ {file = "pillow-10.3.0-cp310-cp310-win32.whl", hash = "sha256:ca2870d5d10d8726a27396d3ca4cf7976cec0f3cb706debe88e3a5bd4610f7d2"},+ {file = "pillow-10.3.0-cp310-cp310-win_amd64.whl", hash = "sha256:f0d0591a0aeaefdaf9a5e545e7485f89910c977087e7de2b6c388aec32011e9f"},+ {file = "pillow-10.3.0-cp310-cp310-win_arm64.whl", hash = "sha256:ccce24b7ad89adb5a1e34a6ba96ac2530046763912806ad4c247356a8f33a67b"},+ {file = "pillow-10.3.0-cp311-cp311-macosx_10_10_x86_64.whl", hash = "sha256:5f77cf66e96ae734717d341c145c5949c63180842a545c47a0ce7ae52ca83795"},+ {file = "pillow-10.3.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e4b878386c4bf293578b48fc570b84ecfe477d3b77ba39a6e87150af77f40c57"},+ {file = "pillow-10.3.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fdcbb4068117dfd9ce0138d068ac512843c52295ed996ae6dd1faf537b6dbc27"},+ {file = "pillow-10.3.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9797a6c8fe16f25749b371c02e2ade0efb51155e767a971c61734b1bf6293994"},+ {file = "pillow-10.3.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:9e91179a242bbc99be65e139e30690e081fe6cb91a8e77faf4c409653de39451"},+ {file = "pillow-10.3.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:1b87bd9d81d179bd8ab871603bd80d8645729939f90b71e62914e816a76fc6bd"},+ {file = "pillow-10.3.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:81d09caa7b27ef4e61cb7d8fbf1714f5aec1c6b6c5270ee53504981e6e9121ad"},+ {file = "pillow-10.3.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:048ad577748b9fa4a99a0548c64f2cb8d672d5bf2e643a739ac8faff1164238c"},+ {file = "pillow-10.3.0-cp311-cp311-win32.whl", hash = "sha256:7161ec49ef0800947dc5570f86568a7bb36fa97dd09e9827dc02b718c5643f09"},+ {file = "pillow-10.3.0-cp311-cp311-win_amd64.whl", hash = "sha256:8eb0908e954d093b02a543dc963984d6e99ad2b5e36503d8a0aaf040505f747d"},+ {file = "pillow-10.3.0-cp311-cp311-win_arm64.whl", hash = "sha256:4e6f7d1c414191c1199f8996d3f2282b9ebea0945693fb67392c75a3a320941f"},+ {file = "pillow-10.3.0-cp39-cp39-macosx_10_10_x86_64.whl", hash = "sha256:2ed854e716a89b1afcedea551cd85f2eb2a807613752ab997b9974aaa0d56936"},+ {file = "pillow-10.3.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:dc1a390a82755a8c26c9964d457d4c9cbec5405896cba94cf51f36ea0d855002"},+ {file = "pillow-10.3.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4203efca580f0dd6f882ca211f923168548f7ba334c189e9eab1178ab840bf60"},+ {file = "pillow-10.3.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3102045a10945173d38336f6e71a8dc71bcaeed55c3123ad4af82c52807b9375"},+ {file = "pillow-10.3.0-cp39-cp39-manylinux_2_28_aarch64.whl", hash = "sha256:6fb1b30043271ec92dc65f6d9f0b7a830c210b8a96423074b15c7bc999975f57"},+ {file = "pillow-10.3.0-cp39-cp39-manylinux_2_28_x86_64.whl", hash = "sha256:1dfc94946bc60ea375cc39cff0b8da6c7e5f8fcdc1d946beb8da5c216156ddd8"},+ {file = "pillow-10.3.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:b09b86b27a064c9624d0a6c54da01c1beaf5b6cadfa609cf63789b1d08a797b9"},+ {file = "pillow-10.3.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:d3b2348a78bc939b4fed6552abfd2e7988e0f81443ef3911a4b8498ca084f6eb"},+ {file = "pillow-10.3.0-cp39-cp39-win32.whl", hash = "sha256:45ebc7b45406febf07fef35d856f0293a92e7417ae7933207e90bf9090b70572"},+ {file = "pillow-10.3.0-cp39-cp39-win_amd64.whl", hash = "sha256:0ba26351b137ca4e0db0342d5d00d2e355eb29372c05afd544ebf47c0956ffeb"},+ {file = "pillow-10.3.0-cp39-cp39-win_arm64.whl", hash = "sha256:50fd3f6b26e3441ae07b7c979309638b72abc1a25da31a81a7fbd9495713ef4f"},+ {file = "pillow-10.3.0-pp310-pypy310_pp73-macosx_10_10_x86_64.whl", hash = "sha256:6b02471b72526ab8a18c39cb7967b72d194ec53c1fd0a70b050565a0f366d355"},+ {file = "pillow-10.3.0-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:8ab74c06ffdab957d7670c2a5a6e1a70181cd10b727cd788c4dd9005b6a8acd9"},+ {file = "pillow-10.3.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:048eeade4c33fdf7e08da40ef402e748df113fd0b4584e32c4af74fe78baaeb2"},+ {file = "pillow-10.3.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9e2ec1e921fd07c7cda7962bad283acc2f2a9ccc1b971ee4b216b75fad6f0463"},+ {file = "pillow-10.3.0-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:4c8e73e99da7db1b4cad7f8d682cf6abad7844da39834c288fbfa394a47bbced"},+ {file = "pillow-10.3.0-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:16563993329b79513f59142a6b02055e10514c1a8e86dca8b48a893e33cf91e3"},+ {file = "pillow-10.3.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:dd78700f5788ae180b5ee8902c6aea5a5726bac7c364b202b4b3e3ba2d293170"},+ {file = "pillow-10.3.0-pp39-pypy39_pp73-macosx_10_10_x86_64.whl", hash = "sha256:aff76a55a8aa8364d25400a210a65ff59d0168e0b4285ba6bf2bd83cf675ba32"},+ {file = "pillow-10.3.0-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:b7bc2176354defba3edc2b9a777744462da2f8e921fbaf61e52acb95bafa9828"},+ {file = "pillow-10.3.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:793b4e24db2e8742ca6423d3fde8396db336698c55cd34b660663ee9e45ed37f"},+ {file = "pillow-10.3.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d93480005693d247f8346bc8ee28c72a2191bdf1f6b5db469c096c0c867ac015"},+ {file = "pillow-10.3.0-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:c83341b89884e2b2e55886e8fbbf37c3fa5efd6c8907124aeb72f285ae5696e5"},+ {file = "pillow-10.3.0-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:1a1d1915db1a4fdb2754b9de292642a39a7fb28f1736699527bb649484fb966a"},+ {file = "pillow-10.3.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:a0eaa93d054751ee9964afa21c06247779b90440ca41d184aeb5d410f20ff591"},+ {file = "pillow-10.3.0.tar.gz", hash = "sha256:9d2455fbf44c914840c793e89aa82d0e1763a14253a000743719ae5946814b2d"},+]++[[package]]+name = "portalocker"+version = "2.8.2"+requires_python = ">=3.8"+summary = "Wraps the portalocker recipe for easy usage"+dependencies = [+ "pywin32>=226; platform_system == \"Windows\"",+]+files = [+ {file = "portalocker-2.8.2-py3-none-any.whl", hash = "sha256:cfb86acc09b9aa7c3b43594e19be1345b9d16af3feb08bf92f23d4dce513a28e"},+ {file = "portalocker-2.8.2.tar.gz", hash = "sha256:2b035aa7828e46c58e9b31390ee1f169b98e1066ab10b9a6a861fe7e25ee4f33"},+]++[[package]]+name = "pycparser"+version = "2.22"+requires_python = ">=3.8"+summary = "C parser in Python"+files = [+ {file = "pycparser-2.22-py3-none-any.whl", hash = "sha256:c3702b6d3dd8c7abc1afa565d7e63d53a1d0bd86cdc24edd75470f4de499cfcc"},+ {file = "pycparser-2.22.tar.gz", hash = "sha256:491c8be9c040f5390f5bf44a5b07752bd07f56edf992381b05c701439eec10f6"},+]+[[package]]name = "pydantic"version = "2.7.2"@@ -1461,6 +1599,44 @@ files = [{file = "pypdf-4.2.0.tar.gz", hash = "sha256:fe63f3f7d1dcda1c9374421a94c1bba6c6f8c4a62173a59b64ffd52058f846b1"},]+[[package]]+name = "pyjwt"+version = "2.8.0"+requires_python = ">=3.7"+summary = "JSON Web Token implementation in Python"+files = [+ {file = "PyJWT-2.8.0-py3-none-any.whl", hash = "sha256:59127c392cc44c2da5bb3192169a91f429924e17aff6534d70fdc02ab3e04320"},+ {file = "PyJWT-2.8.0.tar.gz", hash = "sha256:57e28d156e3d5c10088e0c68abb90bfac3df82b40a71bd0daa20c65ccd5c23de"},+]++[[package]]+name = "pyjwt"+version = "2.8.0"+extras = ["crypto"]+requires_python = ">=3.7"+summary = "JSON Web Token implementation in Python"+dependencies = [+ "PyJWT==2.8.0",+ "cryptography>=3.4.0",+]+files = [+ {file = "PyJWT-2.8.0-py3-none-any.whl", hash = "sha256:59127c392cc44c2da5bb3192169a91f429924e17aff6534d70fdc02ab3e04320"},+ {file = "PyJWT-2.8.0.tar.gz", hash = "sha256:57e28d156e3d5c10088e0c68abb90bfac3df82b40a71bd0daa20c65ccd5c23de"},+]++[[package]]+name = "pypdf"+version = "4.2.0"+requires_python = ">=3.6"+summary = "A pure-python PDF library capable of splitting, merging, cropping, and transforming PDF files"+dependencies = [+ "typing-extensions>=4.0; python_version < \"3.11\"",+]+files = [+ {file = "pypdf-4.2.0-py3-none-any.whl", hash = "sha256:dc035581664e0ad717e3492acebc1a5fc23dba759e788e3d4a9fc9b1a32e72c1"},+ {file = "pypdf-4.2.0.tar.gz", hash = "sha256:fe63f3f7d1dcda1c9374421a94c1bba6c6f8c4a62173a59b64ffd52058f846b1"},+]+[[package]]name = "python-dateutil"version = "2.9.0.post0"@@ -1754,6 +1930,15 @@ files = [{file = "striprtf-0.0.26.tar.gz", hash = "sha256:fdb2bba7ac440072d1c41eab50d8d74ae88f60a8b6575c6e2c7805dc462093aa"},]+[[package]]+name = "striprtf"+version = "0.0.26"+summary = "A simple library to convert rtf to text"+files = [+ {file = "striprtf-0.0.26-py3-none-any.whl", hash = "sha256:8c8f9d32083cdc2e8bfb149455aa1cc5a4e0a035893bedc75db8b73becb3a1bb"},+ {file = "striprtf-0.0.26.tar.gz", hash = "sha256:fdb2bba7ac440072d1c41eab50d8d74ae88f60a8b6575c6e2c7805dc462093aa"},+]+[[package]]name = "tenacity"version = "8.3.0"@@ -1853,6 +2038,7 @@ summary = "Core library that helps with executing workflows."dependencies = ["boto3~=1.28.17","botocore~=1.31.17",+ "kombu==5.3.7","llama-index-llms-azure-openai==0.1.5","llama-index==0.10.28","redis~=5.0.1",@@ -1869,6 +2055,16 @@ files = [{file = "urllib3-1.26.18.tar.gz", hash = "sha256:f8ecc1bba5667413457c529ab955bf8c67b45db799d159066261719e328580a0"},]+[[package]]+name = "vine"+version = "5.1.0"+requires_python = ">=3.6"+summary = "Python promises."+files = [+ {file = "vine-5.1.0-py3-none-any.whl", hash = "sha256:40fdf3c48b2cfe1c38a49e9ae2da6fda88e4794c810050a728bd7413811fb1dc"},+ {file = "vine-5.1.0.tar.gz", hash = "sha256:8b62e981d35c41049211cf62a0a1242d8c1ee9bd15bb196ce38aefd6799e61e0"},+]+[[package]]name = "websocket-client"version = "1.8.0"diff --git a/worker/sample.env b/worker/sample.envindex c43173fcf..b9de45c17 100644--- a/worker/sample.env+++ b/worker/sample.env@@ -1,7 +1,4 @@-REDIS_HOST="unstract-redis"-REDIS_PORT=6379-REDIS_PASSWORD=""-REDIS_USER=default+CELERY_BROKER_URL = "redis://unstract-redis:6379"TOOL_CONTAINER_NETWORK="unstract-network"TOOL_CONTAINER_LABELS="[]"