Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

5.2 KB
Raw Blame
[project]
name = "unstract-backend"
version = "0.0.1"
description = "Unstract backend built with Django to build and schedule ETL pipelines around unstructured data."
authors = [{ name = "Zipstack Inc.", email = "[email protected]" }]

requires-python = ">=3.12,<3.13"
readme = "README.md"
classifiers = [
    "Private :: Do Not Upload",
    "Framework :: Django",
    "Programming Language :: Python",
]

dependencies = [
    "Authlib==1.6.12", # For Auth plugins
    "boto3==1.43.106", # For Unstract-cloud-storage
    "celery[amqp]>=5.3.4", # For Celery
    "cron-descriptor==1.4.0", # For cron string description
    "cryptography>=48.0.1",
    "django==4.2.30",
    "djangorestframework==3.17.1",
    "django-cors-headers==4.3.1",
    # Pinning django-celery-beat to avoid build issues
    "django-celery-beat==2.5.0",
    "django-log-request-id>=2.1.0",
    "django-redis==5.4.0",
    "django-tenants==3.5.0",
    "drf-standardized-errors>=0.12.6",
    # Pinned: its rendering is the committed spec, so an upgrade rewrites the
    # contract published clients are generated from.
    "drf-spectacular==0.30.0",
    "psycopg2-binary==2.9.9",
    "python-dotenv==1.2.2",
    "python-magic==0.4.27", # For file upload/download
    "python-socketio==5.17.0", # For log_events
    "social-auth-app-django==5.3.0", # For OAuth
    "social-auth-core==4.4.2", # For OAuth
    "azure-identity==1.16.0",
    "azure-mgmt-apimanagement==3.0.0",
    "croniter>=3.0.3",
    "django-filter>=24.3",
    "httpx>=0.27.0",
    # Hence required to add all indirect local dependencies too here.
    "unstract-agent-kv-schema",
    "unstract-connectors",
    "unstract-core",
    "unstract-filesystem",
    "unstract-flags",
    "unstract-sdk1[aws,gcs,azure]",
    "unstract-tool-registry",
    "unstract-tool-sandbox",
    "unstract-workflow-execution",
    "unstract-filesystem",
    "google-cloud-recaptcha-enterprise>=1.28.2",
    "gunicorn>=23.0.0",
    "stripe>=12.5.1",
]

[dependency-groups]
dev = [
    "unstract-connectors",
    "unstract-core",
    "unstract-flags",
    "unstract-tool-registry",
    "unstract-tool-sandbox",
    "unstract-workflow-execution",
    "unstract-filesystem",
    # For file watching
    "inotify>=0.2.10",
    "poethepoet>=0.33.1",
    "debugpy>=1.8.14",
    "pytest>=9.0.3",
    "responses>=0.25.7",
    "psutil>=7.0.0",
]
# jsonschema also arrives via unstract-tool-registry, but the schema tests
# import it directly - declare it so the suite fails loudly rather than
# skipping if that transitive path ever changes.
test = [
    "pytest>=8.0.1",
    "pytest-django>=4.12.0",
    "jsonschema>=4.18.6,<5.0",
]
deploy = [
    "gunicorn~=23.0", # For serving the application
    # Keep versions empty and let uv decide version
    # since we use no code instrumentation and don't use in code
    "opentelemetry-distro",
    "opentelemetry-exporter-otlp",
]

[tool.uv.sources]
unstract-agent-kv-schema = { path = "../unstract/agent-kv-schema", editable = true }
unstract-connectors = { path = "../unstract/connectors", editable = true }
unstract-core = { path = "../unstract/core", editable = true }
unstract-filesystem = { path = "../unstract/filesystem", editable = true }
unstract-flags = { path = "../unstract/flags", editable = true }
unstract-sdk1 = { path = "../unstract/sdk1", editable = true }
unstract-tool-registry = { path = "../unstract/tool-registry", editable = true }
unstract-tool-sandbox = { path = "../unstract/tool-sandbox", editable = true }
unstract-workflow-execution = { path = "../unstract/workflow-execution", editable = true }

[tool.uv]
constraint-dependencies = [
    # (unstract-connectors) To address issue with Python 3.12 and Numpy 2.0
    # https://github.com/numpy/numpy/issues/26710
    "numpy<2.0.0", "pandas<2.2.0"
]

[tool.pytest.ini_options]
# Note: test.env is loaded by backend/conftest.py via python-dotenv directly
# (replaces the unmaintained pytest-dotenv plugin).
# --no-migrations builds the schema straight from the models: the migration
# history is replayed once per xdist worker otherwise, and nothing here asserts
# on it.
addopts = "--no-migrations"
python_files = ["test_*.py", "*_test.py", "*_tests.py", "tests.py"]
markers = [
    "integration: needs live infra (Postgres/Redis); runs in the rig's integration tier, not unit (select with -m integration / exclude with -m 'not integration')",
    "critical_path(path_id): attests coverage of declared critical path(s) from tests/critical_paths.yaml when this test passes",
]

[tool.poe]
envfile = ".env"

[tool.poe.tasks.backend]
cmd = "./entrypoint.sh"
help = "Runs the Unstract backend (Gunicorn)"

[tool.poe.tasks.migrate-db]
cmd = "uv run manage.py migrate"
help = "Performs DB migrations for Unstract backend"

[tool.poe.tasks.worker-metrics]
cmd = "celery -A backend worker --loglevel=info -Q dashboard_metric_events --autoscale 4,1"
help = "Runs the Unstract dashboard metrics worker"

[tool.poe.tasks.flower]
cmd = "celery -A backend flower --port=5555"
help = "Runs the Unstract Celery Monitoring Tool"

[tool.poe.tasks.beat]
cmd = "celery -A backend beat --loglevel=info"
help = "Runs the Unstract Celery Beat service"

# [build-system]
# requires = ["hatchling"]
# build-backend = "hatchling.build"

# [tool.hatch.build.targets.wheel]
# packages = ["unstract"]