LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
Request id in logs and docker memory optimizations (#338)
Ritwik G79ef6d7f50c4
diff --git a/backend/backend/settings/base.py b/backend/backend/settings/base.pyindex 82ce2d481..4b5bb2990 100644--- a/backend/backend/settings/base.py+++ b/backend/backend/settings/base.py@@ -50,7 +50,14 @@ DEFAULT_LOG_LEVEL = os.environ.get("DEFAULT_LOG_LEVEL", "INFO")LOGGING = {"version": 1,"disable_existing_loggers": False,+ "filters": {"request_id": {"()": "log_request_id.filters.RequestIDFilter"}},"formatters": {+ "enriched": {+ "format": (+ "%(levelname)s : [%(asctime)s] {module:%(module)s process:%(process)d "+ "thread:%(thread)d request_id:%(request_id)s} :- %(message)s"+ ),+ },"verbose": {"format": "[%(asctime)s] %(levelname)s %(name)s: %(message)s","datefmt": "%d/%b/%Y %H:%M:%S",@@ -64,7 +71,8 @@ LOGGING = {"console": {"level": DEFAULT_LOG_LEVEL, # Set the desired logging level here"class": "logging.StreamHandler",- "formatter": "verbose",+ "filters": ["request_id"],+ "formatter": "enriched",},},"root": {@@ -241,6 +249,7 @@ AUTH_USER_MODEL = "account.User"PUBLIC_ORG_ID = "public"MIDDLEWARE = [+ "log_request_id.middleware.RequestIDMiddleware","corsheaders.middleware.CorsMiddleware","django_tenants.middleware.TenantSubfolderMiddleware","django.middleware.security.SecurityMiddleware",diff --git a/backend/entrypoint.sh b/backend/entrypoint.shindex 4493ee7f7..ebbb38c3f 100755--- a/backend/entrypoint.sh+++ b/backend/entrypoint.sh@@ -12,7 +12,7 @@ fi.venv/bin/gunicorn \--bind 0.0.0.0:8000 \--workers 2 \- --threads 4 \+ --threads 2 \--log-level debug \--timeout 600 \--access-logfile - \diff --git a/backend/pdm.lock b/backend/pdm.lockindex 560eda057..3ba5f0a9a 100644--- a/backend/pdm.lock+++ b/backend/pdm.lock@@ -5,7 +5,7 @@groups = ["default", "deploy", "test"]strategy = ["cross_platform", "inherit_metadata"]lock_version = "4.4.1"-content_hash = "sha256:aa4a58ff9b6bd73adc1fda8f8931d4a1a3954d5a19d032bd4dc51c309c287eae"+content_hash = "sha256:046a2f7554318d18df2ec2167dfddf3e4662aeb8bf34aa8c784af5003b215161"[[package]]name = "adlfs"@@ -965,6 +965,19 @@ files = [{file = "django_cors_headers-4.3.1-py3-none-any.whl", hash = "sha256:0b1fd19297e37417fc9f835d39e45c8c642938ddba1acce0c1753d3edef04f36"},]+[[package]]+name = "django-log-request-id"+version = "2.1.0"+summary = "Django middleware and log filter to attach a unique ID to every log message generated as part of a request"+groups = ["default"]+dependencies = [+ "django>=1.8",+]+files = [+ {file = "django-log-request-id-2.1.0.tar.gz", hash = "sha256:b11fc821774b54b820ef2e732dbaa52304711de8d5c4f9031ca48578f6121ab6"},+ {file = "django_log_request_id-2.1.0-py3-none-any.whl", hash = "sha256:c0793c030cfe9f673eff3f89667711f65c39509c016d85e436f9d48507ee778c"},+]+[[package]]name = "django-redis"version = "5.4.0"diff --git a/backend/pyproject.toml b/backend/pyproject.tomlindex 705aca9bc..be3c1d781 100644--- a/backend/pyproject.toml+++ b/backend/pyproject.toml@@ -20,6 +20,7 @@ dependencies = ["djangorestframework==3.14.0","django-cors-headers==4.3.1","django-celery-beat>=2.5.0",+ "django-log-request-id>=2.1.0","django-redis==5.4.0","django-tenants==3.5.0","drf-standardized-errors>=0.12.6",diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yamlindex 8390a9552..c5a736754 100644--- a/docker/docker-compose.yaml+++ b/docker/docker-compose.yaml@@ -39,7 +39,7 @@ services:container_name: unstract-execution-consumerrestart: unless-stoppedentrypoint: .venv/bin/celery- command: "-A backend worker --loglevel=info"+ command: "-A backend worker --loglevel=info --autoscale=8,1"env_file:- ../backend/.envdepends_on:@@ -57,7 +57,7 @@ services:container_name: unstract-celery-flowerrestart: unless-stoppedentrypoint: .venv/bin/celery- command: "-A backend flower --port=5555"+ command: "-A backend flower --port=5555 --purge_offline_workers=5"env_file:- ../backend/.envdepends_on:@@ -71,6 +71,8 @@ services:- ENVIRONMENT=developmentvolumes:- unstract_data:/data+ profiles:+ - optional# Celery Beatcelery-beat:diff --git a/worker/entrypoint.sh b/worker/entrypoint.shindex d36b03db4..89cac2d02 100755--- a/worker/entrypoint.sh+++ b/worker/entrypoint.sh@@ -3,7 +3,7 @@# 'src' layout is detected from pdm settings in pyproject.toml.venv/bin/gunicorn \--bind 0.0.0.0:5002 \- --workers 5 \+ --workers 2 \--threads 2 \--worker-class gevent\--log-level debug \