LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
[project]
name = "unstract"
version = "0.1.0"
requires-python = ">=3.12,<3.13"
dependencies = []
[dependency-groups]
dev = [
"docutils~=0.20.1",
"mypy~=1.9.0",
"pre-commit~=3.6.2",
"yamllint>=1.35.1",
"ruff<1.0.0,>=0.2.2",
"pytest>=9.0.3",
"pycln>=2.6.0",
"types-PyMySQL~=1.1.0.1",
"types-pyOpenSSL~=24.0.0.20240311",
"types-PyYAML~=6.0.12.12",
"types-redis~=4.6.0.3",
"types-requests~=2.31.0.6",
"types-tzlocal~=5.1.0.1",
]
workers = [
"unstract-workers",
]
test-rig = [
# Test runner foundation. Installed by tox into the rig env.
"pytest>=8.0.1",
"pytest-xdist>=3.5.0",
"pytest-cov>=4.1.0",
"pytest-md-report>=0.6.2",
"pytest-mock>=3.12.0",
"pytest-timeout>=2.3.1",
"pytest-django>=4.8.0",
"coverage[toml]>=7.4.0",
"pyyaml>=6.0.1",
"testcontainers[postgres,redis,minio,rabbitmq]>=4.7.0",
"requests>=2.31.0",
]
[tool.uv.sources]
unstract-agent-kv-schema = { path = "./unstract/agent-kv-schema", editable = true }
unstract-filesystem = { path = "./unstract/filesystem", editable = true }
unstract-workflow-execution = { path = "./unstract/workflow-execution", editable = true }
unstract-tool-sandbox = { path = "./unstract/tool-sandbox", editable = true }
unstract-tool-registry = { path = "./unstract/tool-registry", editable = true }
unstract-flags = { path = "./unstract/flags", editable = true }
unstract-core = { path = "./unstract/core", editable = true }
unstract-connectors = { path = "./unstract/connectors", editable = true }
# Workers
unstract-workers = { path = "./workers", editable = true }
# === Development tool configurations ===
[tool.ruff]
line-length = 90
target-version = "py312"
exclude = [
".bzr",
".direnv",
".eggs",
".git",
".git-rewrite",
".hg",
".mypy_cache",
".nox",
".pants.d",
".pytype",
".ruff_cache",
".svn",
".tox",
".venv",
"__pypackages__",
"_build",
"buck-out",
"build",
"dist",
"node_modules",
"venv",
".*migrations/.*.py",
"tests",
"*_pb2.py",
"*_pb2_grpc.py",
]
[tool.ruff.lint]
select = [
"E", # pycodestyle (formatting)
"F", # Pyflakes (static analysis)
"I", # isort (import sorting)
"B", # bugbear (security/performance)
"W", # warnings
"C90", # mccabe complexity
"N", # pep8-naming
"D", # pydocstyle
"UP", # pyupgrade
"ANN", # flake8-annotations
"TCH", # flake8-type-checking
"PYI", # flake8-pyi
]
fixable = ["ALL"]
ignore = [
"ANN001", # Missing type annotation for function argument `value`
"ANN002", # Missing type annotation for `*args`"
"ANN003", # Missing type annotation for `**kwargs`"
"ANN101", # Missing type annotation for self
"ANN102", # Missing type annotation for cls
"ANN201", # Missing return type annotation for public function
"ANN204", # Missing return type annotation for special method `__init__`,
"ANN205", # Missing return type annotation for staticmethod `list`
"ANN401", # Dynamically typed expressions (typing.Any) are disallowed in `user_login`
"B006", # Do not use mutable data structures for argument defaults
"B007", # Loop control variable `dirs` not used within loop body
"B008", # Do not perform function call `FileStorage`"
"B015", # Pointless comparison. Did you mean to assign a value? Otherwise, prepend `assert` or remove it."
"B018", # Found useless expression. Either assign it to a variable or remove it.
"B024", # `AuthService` is an abstract base class, but it has no abstract methods or properties"
"B904", # Within an `except` clause, raise exceptions with
"C901", # `handle_output` is too complex (11 > 10)"
"D100", # Missing docstring in public module
"D101", # Missing docstring in public class
"D102", # Missing docstring in public method
"D103", # Missing docstring in public function
"D104", # Missing docstring in public package
"D105", # Missing docstring in magic method
"D106", # Missing docstring in public nested class
"D107", # Missing docstring in `__init__`
"D205", # 1 blank line required between summary line and description
"D402", # First line should not be the function's signature"
"D415", # First line should end with a period, question mark, or exclamation point"
"D417", # Missing argument description in the docstring for `user_organizations`: `request`
"E501", # Line too long (91 > 90)
"N806", # Variable `organizationData` in function should be lowercase
"N802", # Function name `makeSignupRequestParams` should be lowercase
"N804", # First argument of a class method should be named `cls`"
"N805", # First argument of a method should be named `self`"
"N812", # Lowercase `exceptions` imported as non-lowercase `GoogleApiException`"
"N813", # Camelcase `Common` imported as lowercase `common`
"N816", # Variable `zipProject` in global scope should not be mixedCase"
"N817", # CamelCase `FieldLengthConstants` imported as acronym `FLC`,
"N818", # Exception name `InvalidToolPropertiesException` should be named with an Error suffix
"N999", # Invalid module name: 'FileValidator'"
"TCH001", #Move application import xxx into a type-checking block
"TCH002", # Move third-party import `rest_framework.serializers.Serializer` into a type-checking block
"TCH003", # Move standard library import `uuid` into a type-checking block"
# "TC006", # [*] Add quotes to type expression in `typing.cast()"
"ANN206", #Missing return type annotation
"ANN202", # Missing return type annotation for private function"
"B027", # `NotificationProvider.validate` is an empty method in an abstract base class, but has no abstract decorator"
"F405", # `os` may be undefined, or defined from star imports"
"UP038", # Use `X | Y` in `isinstance` call instead of `(X, Y)`"
]
[tool.ruff.lint.mccabe]
max-complexity = 10
[tool.ruff.lint.pydocstyle]
convention = "google"
[tool.ruff.format]
quote-style = "double"
indent-style = "space"
line-ending = "auto"
docstring-code-format = true
[tool.pycln]
all = true
expand-stars = true
no-gitignore = false
verbose = true
[tool.pyupgrade]
keep-runtime-typing = true
py39-plus = true
keep-dict-typing = true
[tool.pytest.ini_options]
python_files = ["tests.py", "test_*.py", "*_tests.py"]
testpaths = ["tests"]
markers = [
"slow: marks tests as slow (deselect with '-m \"not slow\"')",
"integration: marks tests as integration (deselect with '-m \"not integration\"')",
"unit: marks tests as unit (no external services required)",
"e2e: marks tests as end-to-end (require full platform stack)",
"critical: marks tests covering a critical path (see tests/critical_paths.yaml)",
"critical_path(path_id): attests coverage of the named critical path(s) when this test passes",
]
[tool.coverage.run]
branch = false
relative_files = true
omit = [
# Test modules shouldn't count themselves as covered code; that inflates
# the percentage and hides real gaps in the modules under test.
# Keep the patterns narrow: target test FILES, not "anything in a tests/
# directory" (which would exclude tests/rig/*.py — those ARE the
# production modules for the unit-rig group).
"**/test_*.py",
"**/*_tests.py",
"**/tests.py",
]
[tool.coverage.paths]
# Each group runs pytest from its own workdir, so per-group .coverage files
# store paths anchored at that workdir (e.g. "src/unstract/sdk1/foo.py" from
# unstract/sdk1/; "shared/foo.py" from workers/). `coverage combine` uses
# these mappings to fold parallel workdir paths back onto a single
# repo-relative path, otherwise xml/html show duplicate or orphaned files.
sdk1 = ["unstract/sdk1/src/unstract/sdk1", "*/unstract/sdk1/src/unstract/sdk1"]
runner = ["runner/src/unstract/runner", "*/runner/src/unstract/runner"]
prompt_service = [
"prompt-service/src/unstract/prompt_service",
"*/prompt-service/src/unstract/prompt_service",
]
platform_service = ["platform-service/src", "*/platform-service/src"]
workers = ["workers/shared", "*/workers/shared"]
connectors = ["unstract/connectors/src", "*/unstract/connectors/src"]
core = ["unstract/core/src", "*/unstract/core/src"]
tool_registry = ["unstract/tool-registry/src", "*/unstract/tool-registry/src"]
backend = ["backend", "*/backend"]
rig = ["tests/rig", "*/tests/rig"]
[tool.mypy]
python_version = "3.12"
warn_return_any = true
warn_unused_configs = true
disallow_untyped_defs = true
disallow_incomplete_defs = true
check_untyped_defs = true
disallow_untyped_decorators = false
no_implicit_optional = true
strict_optional = true
warn_redundant_casts = true
warn_unused_ignores = true
warn_no_return = true
warn_unreachable = true
allow_untyped_globals = false
allow_redefinition = false
local_partial_types = true
implicit_reexport = true
follow_imports = "silent"
ignore_missing_imports = true
pretty = true
show_column_numbers = true
show_error_codes = true
strict = true
exclude = '''(?x)^(
.*migrations/.*\.py|
backend/prompt/.*|
unstract/connectors/tests/.*|
unstract/core/.*|
unstract/flags/src/unstract/flags/.*|
__pypackages__/.*|
)$'''
[tool.yamllint]
relaxed = true