Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

Request id in logs and docker memory optimizations (#338)

Ritwik G79ef6d7f50c4
diff --git a/backend/backend/settings/base.py b/backend/backend/settings/base.py
index 82ce2d481..4b5bb2990 100644
--- a/backend/backend/settings/base.py
+++ b/backend/backend/settings/base.py
@@ -50,7 +50,14 @@ DEFAULT_LOG_LEVEL = os.environ.get("DEFAULT_LOG_LEVEL", "INFO")
LOGGING = {
"version": 1,
"disable_existing_loggers": False,
+ "filters": {"request_id": {"()": "log_request_id.filters.RequestIDFilter"}},
"formatters": {
+ "enriched": {
+ "format": (
+ "%(levelname)s : [%(asctime)s] {module:%(module)s process:%(process)d "
+ "thread:%(thread)d request_id:%(request_id)s} :- %(message)s"
+ ),
+ },
"verbose": {
"format": "[%(asctime)s] %(levelname)s %(name)s: %(message)s",
"datefmt": "%d/%b/%Y %H:%M:%S",
@@ -64,7 +71,8 @@ LOGGING = {
"console": {
"level": DEFAULT_LOG_LEVEL, # Set the desired logging level here
"class": "logging.StreamHandler",
- "formatter": "verbose",
+ "filters": ["request_id"],
+ "formatter": "enriched",
},
},
"root": {
@@ -241,6 +249,7 @@ AUTH_USER_MODEL = "account.User"
PUBLIC_ORG_ID = "public"
MIDDLEWARE = [
+ "log_request_id.middleware.RequestIDMiddleware",
"corsheaders.middleware.CorsMiddleware",
"django_tenants.middleware.TenantSubfolderMiddleware",
"django.middleware.security.SecurityMiddleware",
diff --git a/backend/entrypoint.sh b/backend/entrypoint.sh
index 4493ee7f7..ebbb38c3f 100755
--- a/backend/entrypoint.sh
+++ b/backend/entrypoint.sh
@@ -12,7 +12,7 @@ fi
.venv/bin/gunicorn \
--bind 0.0.0.0:8000 \
--workers 2 \
- --threads 4 \
+ --threads 2 \
--log-level debug \
--timeout 600 \
--access-logfile - \
diff --git a/backend/pdm.lock b/backend/pdm.lock
index 560eda057..3ba5f0a9a 100644
--- a/backend/pdm.lock
+++ b/backend/pdm.lock
@@ -5,7 +5,7 @@
groups = ["default", "deploy", "test"]
strategy = ["cross_platform", "inherit_metadata"]
lock_version = "4.4.1"
-content_hash = "sha256:aa4a58ff9b6bd73adc1fda8f8931d4a1a3954d5a19d032bd4dc51c309c287eae"
+content_hash = "sha256:046a2f7554318d18df2ec2167dfddf3e4662aeb8bf34aa8c784af5003b215161"
[[package]]
name = "adlfs"
@@ -965,6 +965,19 @@ files = [
{file = "django_cors_headers-4.3.1-py3-none-any.whl", hash = "sha256:0b1fd19297e37417fc9f835d39e45c8c642938ddba1acce0c1753d3edef04f36"},
]
+[[package]]
+name = "django-log-request-id"
+version = "2.1.0"
+summary = "Django middleware and log filter to attach a unique ID to every log message generated as part of a request"
+groups = ["default"]
+dependencies = [
+ "django>=1.8",
+]
+files = [
+ {file = "django-log-request-id-2.1.0.tar.gz", hash = "sha256:b11fc821774b54b820ef2e732dbaa52304711de8d5c4f9031ca48578f6121ab6"},
+ {file = "django_log_request_id-2.1.0-py3-none-any.whl", hash = "sha256:c0793c030cfe9f673eff3f89667711f65c39509c016d85e436f9d48507ee778c"},
+]
+
[[package]]
name = "django-redis"
version = "5.4.0"
diff --git a/backend/pyproject.toml b/backend/pyproject.toml
index 705aca9bc..be3c1d781 100644
--- a/backend/pyproject.toml
+++ b/backend/pyproject.toml
@@ -20,6 +20,7 @@ dependencies = [
"djangorestframework==3.14.0",
"django-cors-headers==4.3.1",
"django-celery-beat>=2.5.0",
+ "django-log-request-id>=2.1.0",
"django-redis==5.4.0",
"django-tenants==3.5.0",
"drf-standardized-errors>=0.12.6",
diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml
index 8390a9552..c5a736754 100644
--- a/docker/docker-compose.yaml
+++ b/docker/docker-compose.yaml
@@ -39,7 +39,7 @@ services:
container_name: unstract-execution-consumer
restart: unless-stopped
entrypoint: .venv/bin/celery
- command: "-A backend worker --loglevel=info"
+ command: "-A backend worker --loglevel=info --autoscale=8,1"
env_file:
- ../backend/.env
depends_on:
@@ -57,7 +57,7 @@ services:
container_name: unstract-celery-flower
restart: unless-stopped
entrypoint: .venv/bin/celery
- command: "-A backend flower --port=5555"
+ command: "-A backend flower --port=5555 --purge_offline_workers=5"
env_file:
- ../backend/.env
depends_on:
@@ -71,6 +71,8 @@ services:
- ENVIRONMENT=development
volumes:
- unstract_data:/data
+ profiles:
+ - optional
# Celery Beat
celery-beat:
diff --git a/worker/entrypoint.sh b/worker/entrypoint.sh
index d36b03db4..89cac2d02 100755
--- a/worker/entrypoint.sh
+++ b/worker/entrypoint.sh
@@ -3,7 +3,7 @@
# 'src' layout is detected from pdm settings in pyproject.toml
.venv/bin/gunicorn \
--bind 0.0.0.0:5002 \
- --workers 5 \
+ --workers 2 \
--threads 2 \
--worker-class gevent\
--log-level debug \