LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
[project]
name = "unstract-backend"
version = "0.0.1"
description = "Unstract backend built with Django to build and schedule ETL pipelines around unstructured data."
authors = [{ name = "Zipstack Inc.", email = "[email protected]" }]
requires-python = ">=3.12,<3.13"
readme = "README.md"
classifiers = [
"Private :: Do Not Upload",
"Framework :: Django",
"Programming Language :: Python",
]
dependencies = [
"Authlib==1.6.12", # For Auth plugins
"boto3==1.43.106", # For Unstract-cloud-storage
"celery[amqp]>=5.3.4", # For Celery
"cron-descriptor==1.4.0", # For cron string description
"cryptography>=48.0.1",
"django==4.2.30",
"djangorestframework==3.17.1",
"django-cors-headers==4.3.1",
# Pinning django-celery-beat to avoid build issues
"django-celery-beat==2.5.0",
"django-log-request-id>=2.1.0",
"django-redis==5.4.0",
"django-tenants==3.5.0",
"drf-standardized-errors>=0.12.6",
# Pinned: its rendering is the committed spec, so an upgrade rewrites the
# contract published clients are generated from.
"drf-spectacular==0.30.0",
"psycopg2-binary==2.9.9",
"python-dotenv==1.2.2",
"python-magic==0.4.27", # For file upload/download
"python-socketio==5.17.0", # For log_events
"social-auth-app-django==5.3.0", # For OAuth
"social-auth-core==4.4.2", # For OAuth
"azure-identity==1.16.0",
"azure-mgmt-apimanagement==3.0.0",
"croniter>=3.0.3",
"django-filter>=24.3",
"httpx>=0.27.0",
# Hence required to add all indirect local dependencies too here.
"unstract-agent-kv-schema",
"unstract-connectors",
"unstract-core",
"unstract-filesystem",
"unstract-flags",
"unstract-sdk1[aws,gcs,azure]",
"unstract-tool-registry",
"unstract-tool-sandbox",
"unstract-workflow-execution",
"unstract-filesystem",
"google-cloud-recaptcha-enterprise>=1.28.2",
"gunicorn>=23.0.0",
"stripe>=12.5.1",
]
[dependency-groups]
dev = [
"unstract-connectors",
"unstract-core",
"unstract-flags",
"unstract-tool-registry",
"unstract-tool-sandbox",
"unstract-workflow-execution",
"unstract-filesystem",
# For file watching
"inotify>=0.2.10",
"poethepoet>=0.33.1",
"debugpy>=1.8.14",
"pytest>=9.0.3",
"responses>=0.25.7",
"psutil>=7.0.0",
]
# jsonschema also arrives via unstract-tool-registry, but the schema tests
# import it directly - declare it so the suite fails loudly rather than
# skipping if that transitive path ever changes.
test = [
"pytest>=8.0.1",
"pytest-django>=4.12.0",
"jsonschema>=4.18.6,<5.0",
]
deploy = [
"gunicorn~=23.0", # For serving the application
# Keep versions empty and let uv decide version
# since we use no code instrumentation and don't use in code
"opentelemetry-distro",
"opentelemetry-exporter-otlp",
]
[tool.uv.sources]
unstract-agent-kv-schema = { path = "../unstract/agent-kv-schema", editable = true }
unstract-connectors = { path = "../unstract/connectors", editable = true }
unstract-core = { path = "../unstract/core", editable = true }
unstract-filesystem = { path = "../unstract/filesystem", editable = true }
unstract-flags = { path = "../unstract/flags", editable = true }
unstract-sdk1 = { path = "../unstract/sdk1", editable = true }
unstract-tool-registry = { path = "../unstract/tool-registry", editable = true }
unstract-tool-sandbox = { path = "../unstract/tool-sandbox", editable = true }
unstract-workflow-execution = { path = "../unstract/workflow-execution", editable = true }
[tool.uv]
constraint-dependencies = [
# (unstract-connectors) To address issue with Python 3.12 and Numpy 2.0
# https://github.com/numpy/numpy/issues/26710
"numpy<2.0.0", "pandas<2.2.0"
]
[tool.pytest.ini_options]
# Note: test.env is loaded by backend/conftest.py via python-dotenv directly
# (replaces the unmaintained pytest-dotenv plugin).
# --no-migrations builds the schema straight from the models: the migration
# history is replayed once per xdist worker otherwise, and nothing here asserts
# on it.
addopts = "--no-migrations"
python_files = ["test_*.py", "*_test.py", "*_tests.py", "tests.py"]
markers = [
"integration: needs live infra (Postgres/Redis); runs in the rig's integration tier, not unit (select with -m integration / exclude with -m 'not integration')",
"critical_path(path_id): attests coverage of declared critical path(s) from tests/critical_paths.yaml when this test passes",
]
[tool.poe]
envfile = ".env"
[tool.poe.tasks.backend]
cmd = "./entrypoint.sh"
help = "Runs the Unstract backend (Gunicorn)"
[tool.poe.tasks.migrate-db]
cmd = "uv run manage.py migrate"
help = "Performs DB migrations for Unstract backend"
[tool.poe.tasks.worker-metrics]
cmd = "celery -A backend worker --loglevel=info -Q dashboard_metric_events --autoscale 4,1"
help = "Runs the Unstract dashboard metrics worker"
[tool.poe.tasks.flower]
cmd = "celery -A backend flower --port=5555"
help = "Runs the Unstract Celery Monitoring Tool"
[tool.poe.tasks.beat]
cmd = "celery -A backend beat --loglevel=info"
help = "Runs the Unstract Celery Beat service"
# [build-system]
# requires = ["hatchling"]
# build-backend = "hatchling.build"
# [tool.hatch.build.targets.wheel]
# packages = ["unstract"]