LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
feat: Updated indexer,structure,text-extractor to use 0.13.0 SDK (#41)
Chandrasekharan Mb202c7f287fc
diff --git a/.gitignore b/.gitignoreindex 9e300b9e3..8a520edfa 100644--- a/.gitignore+++ b/.gitignore@@ -603,7 +603,7 @@ $RECYCLE.BIN/### Unstract ###-# Plugins+# Authentication Pluginsbackend/plugins/authentication/*!backend/plugins/authentication/auth_sample@@ -624,9 +624,15 @@ docker/*.env!docker/sample.*.envdocker/public_tools.jsondocker/proxy_overrides.yaml+docker/workflow_data/# Tool development-tools/*/sdks/+tools/*/unstract-sdk/tools/*/data_dir/-docker/workflow_data/+# Backend settings ignore (allows user overrides)+backend/backend/settings/*+!backend/backend/settings/__init__.py+!backend/backend/settings/base.py+!backend/backend/settings/dev.py+!backend/backend/settings/test.pydiff --git a/backend/backend/settings/test_cases.py b/backend/backend/settings/test.pysimilarity index 100%rename from backend/backend/settings/test_cases.pyrename to backend/backend/settings/test.pydiff --git a/backend/pdm.lock b/backend/pdm.lockindex 569bc61c2..cdc291482 100644--- a/backend/pdm.lock+++ b/backend/pdm.lock@@ -5,7 +5,7 @@groups = ["default", "deploy", "test"]strategy = ["cross_platform", "inherit_metadata"]lock_version = "4.4.1"-content_hash = "sha256:27911cc8da87345bc67ffc964493ec9434a770ed2e380403ef87094e8a55e409"+content_hash = "sha256:e78a3068766ec116281b2ad59a89ffab883c035e32bd00ef919f30595eb38626"[[package]]name = "aiobotocore"@@ -1312,7 +1312,7 @@ files = [[[package]]name = "google-api-python-client"-version = "2.119.0"+version = "2.120.0"requires_python = ">=3.7"summary = "Google API Client Library for Python"groups = ["default"]@@ -1324,8 +1324,8 @@ dependencies = ["uritemplate<5,>=3.0.1",]files = [- {file = "google-api-python-client-2.119.0.tar.gz", hash = "sha256:ff9ef7539eaf7e088a481b25d1af4704210b07863e1d51b5ee498b910a3a46a3"},- {file = "google_api_python_client-2.119.0-py2.py3-none-any.whl", hash = "sha256:84e43bdb58dd8d2301669513863996378ffe9a3bf6d23b5ccd4f1e021323dbeb"},+ {file = "google-api-python-client-2.120.0.tar.gz", hash = "sha256:a0c8769cad9576768bcb3191cb1f550f6ab3290cba042badb0fb17bba03f70cc"},+ {file = "google_api_python_client-2.120.0-py2.py3-none-any.whl", hash = "sha256:e2cdf4497bfc758fb44a4b487920cc1ca0571c2428187697a8e43e3b9feba1c9"},][[package]]@@ -3266,7 +3266,7 @@ files = [[[package]]name = "python-dateutil"-version = "2.8.2"+version = "2.9.0"requires_python = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7"summary = "Extensions to the standard Python datetime module"groups = ["default"]@@ -3274,8 +3274,8 @@ dependencies = ["six>=1.5",]files = [- {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"},- {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"},+ {file = "python-dateutil-2.9.0.tar.gz", hash = "sha256:78e73e19c63f5b20ffa567001531680d939dc042bf7850431877645523c66709"},+ {file = "python_dateutil-2.9.0-py2.py3-none-any.whl", hash = "sha256:cbf2f1da5e6083ac2fbfd4da39a25f34312230110440f424a14c7558bb85d82e"},][[package]]@@ -4582,7 +4582,7 @@ files = [[[package]]name = "unstract-adapters"-version = "0.2.2"+version = "0.3.0"requires_python = "<3.12,>=3.9"summary = "Unstract Adapters"groups = ["default"]@@ -4610,8 +4610,8 @@ dependencies = ["weaviate-client==3.25.3",]files = [- {file = "unstract_adapters-0.2.2-py3-none-any.whl", hash = "sha256:aaf48036f844f4e0295c36596b1772111ab18229246b8677227b983b3c4226e4"},- {file = "unstract_adapters-0.2.2.tar.gz", hash = "sha256:465d7859a0eb7a905d2c626939f2fed3a501600f832c93160b3e27a559e5ab22"},+ {file = "unstract_adapters-0.3.0-py3-none-any.whl", hash = "sha256:ac2e6d6902bb54c0ef4d223b68365d4f044fb92b83c05bcd92208155e548253c"},+ {file = "unstract_adapters-0.3.0.tar.gz", hash = "sha256:3c4662a73dfaa0b9c0b947da8e3e351221b876c6dd505d054c70a3067fe8d9dc"},][[package]]@@ -4666,7 +4666,7 @@ dependencies = [[[package]]name = "unstract-sdk"-version = "0.12.1"+version = "0.13.0"requires_python = "<3.11.1,>=3.9"summary = "A framework for writing Unstract Tools/Apps"groups = ["default"]@@ -4680,11 +4680,11 @@ dependencies = ["python-magic~=0.4.27","tiktoken~=0.4.0","transformers==4.37.0",- "unstract-adapters~=0.2.2",+ "unstract-adapters~=0.3.0",]files = [- {file = "unstract_sdk-0.12.1-py3-none-any.whl", hash = "sha256:960d8a7b92a1ad32894650edd9254aaf7c3f7555053991b3860e55ab684624a8"},- {file = "unstract_sdk-0.12.1.tar.gz", hash = "sha256:06b1a7bf17948d548137502127458d83046c2e0b596ff326a292dbe94041a13a"},+ {file = "unstract_sdk-0.13.0-py3-none-any.whl", hash = "sha256:15c2aa60a93654fa9eab8b90024ca11b7fc5c9e3e6b8cdde97e11279dde0b2a3"},+ {file = "unstract_sdk-0.13.0.tar.gz", hash = "sha256:64e97366dd742964620201ed4a3aeb6495c74efd4ba482266f0308bafa08e70d"},][[package]]@@ -4698,7 +4698,7 @@ dependencies = ["PyYAML~=6.0.1","docker~=6.1.3","jsonschema~=4.18.2",- "unstract-adapters~=0.2.1",+ "unstract-adapters~=0.3.0","unstract-tool-sandbox",]diff --git a/backend/prompt_studio/prompt_studio_core/prompt_studio_helper.py b/backend/prompt_studio/prompt_studio_core/prompt_studio_helper.pyindex 33bc189ce..249fb53db 100644--- a/backend/prompt_studio/prompt_studio_core/prompt_studio_helper.py+++ b/backend/prompt_studio/prompt_studio_core/prompt_studio_helper.py@@ -2,7 +2,7 @@ import jsonimport loggingimport osfrom pathlib import Path-from typing import Any+from typing import Any, Optionalfrom django.conf import settingsfrom file_management.file_management_helper import FileManagerHelper@@ -388,10 +388,10 @@ class PromptStudioHelper:vector_db = str(profile_manager.vector_store.id)x2text_adapter = str(profile_manager.x2text.id)file_hash = ToolUtils.get_hash_from_file(file_path=file_name)- extract_file_path = None+ extract_file_path: Optional[str] = Noneif not is_summary:directory, filename = os.path.split(file_name)- extract_file_path: str = os.path.join(+ extract_file_path = os.path.join(directory, "extract", os.path.splitext(filename)[0] + ".txt")return str(diff --git a/backend/pyproject.toml b/backend/pyproject.tomlindex faf134db2..857d5a4a5 100644--- a/backend/pyproject.toml+++ b/backend/pyproject.toml@@ -30,8 +30,8 @@ dependencies = ["python-socketio==5.9.0", # For log_events"social-auth-app-django==5.3.0", # For OAuth"social-auth-core==4.4.2", # For OAuth- "unstract-sdk~=0.12.1",- "unstract-adapters~=0.2.2",+ "unstract-sdk~=0.13.0",+ "unstract-adapters~=0.3.0",# ! IMPORTANT!# Indirect local dependencies usually need to be added in their own projects# as: https://pdm-project.org/latest/usage/dependency/#local-dependencies.diff --git a/backend/sample.env b/backend/sample.envindex 20c96f0fd..1ffd6a168 100644--- a/backend/sample.env+++ b/backend/sample.env@@ -86,9 +86,9 @@ PROMPT_PORT=3003PROMPT_STUDIO_FILE_PATH=/app/prompt-studio-data# Structure Tool-STRUCTURE_TOOL_IMAGE_URL="docker:unstract/tool-structure:0.0.1"+STRUCTURE_TOOL_IMAGE_URL="docker:unstract/tool-structure:0.0.2"STRUCTURE_TOOL_IMAGE_NAME="unstract/tool-structure"-STRUCTURE_TOOL_IMAGE_TAG="0.0.1"+STRUCTURE_TOOL_IMAGE_TAG="0.0.2"# Feature FlagsEVALUATION_SERVER_IP=localhostdiff --git a/pdm.lock b/pdm.lockindex 8fe2528d8..c9a4b8c11 100644--- a/pdm.lock+++ b/pdm.lock@@ -5,7 +5,7 @@groups = ["default", "hook-check-django-migrations", "lint"]strategy = ["cross_platform", "inherit_metadata"]lock_version = "4.4.1"-content_hash = "sha256:7b566c4c5f996943e1437b7c15b44c44f6d4e2c6c30374202688b21f945afb63"+content_hash = "sha256:c97af56947d9808e3d1b1394577ad58c7adae9da9b1b70949c500e5e7badac34"[[package]]name = "absolufy-imports"@@ -330,7 +330,6 @@ version = "4.0.3"requires_python = ">=3.7"summary = "Timeout context manager for asyncio programs"groups = ["hook-check-django-migrations"]-marker = "python_version < \"3.12.0\""files = [{file = "async-timeout-4.0.3.tar.gz", hash = "sha256:4640d96be84d82d02ed59ea2b7105a0f7b33abe8703703cd0ab0bf87c427522f"},{file = "async_timeout-4.0.3-py3-none-any.whl", hash = "sha256:7405140ff1230c310e51dc27b3145b9092d659ce68ff733fb0cefe3ee42be028"},@@ -1393,7 +1392,7 @@ files = [[[package]]name = "google-api-python-client"-version = "2.119.0"+version = "2.120.0"requires_python = ">=3.7"summary = "Google API Client Library for Python"groups = ["hook-check-django-migrations"]@@ -1405,8 +1404,8 @@ dependencies = ["uritemplate<5,>=3.0.1",]files = [- {file = "google-api-python-client-2.119.0.tar.gz", hash = "sha256:ff9ef7539eaf7e088a481b25d1af4704210b07863e1d51b5ee498b910a3a46a3"},- {file = "google_api_python_client-2.119.0-py2.py3-none-any.whl", hash = "sha256:84e43bdb58dd8d2301669513863996378ffe9a3bf6d23b5ccd4f1e021323dbeb"},+ {file = "google-api-python-client-2.120.0.tar.gz", hash = "sha256:a0c8769cad9576768bcb3191cb1f550f6ab3290cba042badb0fb17bba03f70cc"},+ {file = "google_api_python_client-2.120.0-py2.py3-none-any.whl", hash = "sha256:e2cdf4497bfc758fb44a4b487920cc1ca0571c2428187697a8e43e3b9feba1c9"},][[package]]@@ -1673,7 +1672,7 @@ files = [[[package]]name = "gotrue"-version = "2.1.0"+version = "2.4.1"requires_python = ">=3.8,<4.0"summary = "Python Client Library for GoTrue"groups = ["hook-check-django-migrations"]@@ -1682,8 +1681,8 @@ dependencies = ["pydantic<3,>=1.10",]files = [- {file = "gotrue-2.1.0-py3-none-any.whl", hash = "sha256:6483d9a3ac9be1d1ad510be24171e133aa1cec702cc10a8f323b9e7519642447"},- {file = "gotrue-2.1.0.tar.gz", hash = "sha256:b21d48ee64f0f6a1ed111efe4871a83e542529f1a75a264833b50e6433cd3c98"},+ {file = "gotrue-2.4.1-py3-none-any.whl", hash = "sha256:9647bb7a585c969d26667df21168fa20b18f91c5d6afe286af08d7a0610fd2cc"},+ {file = "gotrue-2.4.1.tar.gz", hash = "sha256:8b260ef285f45a3a2f9b5a006f12afb9fad7a36a28fa277f19e733f22eb88584"},][[package]]@@ -3399,7 +3398,7 @@ files = [[[package]]name = "python-dateutil"-version = "2.8.2"+version = "2.9.0"requires_python = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7"summary = "Extensions to the standard Python datetime module"groups = ["hook-check-django-migrations"]@@ -3407,8 +3406,8 @@ dependencies = ["six>=1.5",]files = [- {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"},- {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"},+ {file = "python-dateutil-2.9.0.tar.gz", hash = "sha256:78e73e19c63f5b20ffa567001531680d939dc042bf7850431877645523c66709"},+ {file = "python_dateutil-2.9.0-py2.py3-none-any.whl", hash = "sha256:cbf2f1da5e6083ac2fbfd4da39a25f34312230110440f424a14c7558bb85d82e"},][[package]]@@ -3544,16 +3543,16 @@ files = [[[package]]name = "redis"-version = "5.0.1"+version = "5.0.2"requires_python = ">=3.7"summary = "Python client for Redis database and key-value store"groups = ["hook-check-django-migrations"]dependencies = [- "async-timeout>=4.0.2; python_full_version <= \"3.11.2\"",+ "async-timeout>=4.0.3",]files = [- {file = "redis-5.0.1-py3-none-any.whl", hash = "sha256:ed4802971884ae19d640775ba3b03aa2e7bd5e8fb8dfaed2decce4d0fc48391f"},- {file = "redis-5.0.1.tar.gz", hash = "sha256:0dab495cd5753069d3bc650a0dde8a8f9edde16fc5691b689a566eda58100d0f"},+ {file = "redis-5.0.2-py3-none-any.whl", hash = "sha256:4caa8e1fcb6f3c0ef28dba99535101d80934b7d4cd541bbb47f4a3826ee472d1"},+ {file = "redis-5.0.2.tar.gz", hash = "sha256:3f82cc80d350e93042c8e6e7a5d0596e4dd68715babffba79492733e1f367037"},][[package]]@@ -3693,7 +3692,7 @@ files = [[[package]]name = "rich"-version = "13.7.0"+version = "13.7.1"requires_python = ">=3.7.0"summary = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal"groups = ["hook-check-django-migrations"]@@ -3702,8 +3701,8 @@ dependencies = ["pygments<3.0.0,>=2.13.0",]files = [- {file = "rich-13.7.0-py3-none-any.whl", hash = "sha256:6da14c108c4866ee9520bbffa71f6fe3962e193b7da68720583850cd4548e235"},- {file = "rich-13.7.0.tar.gz", hash = "sha256:5cb5123b5cf9ee70584244246816e9114227e0b98ad9176eede6ad54bf5403fa"},+ {file = "rich-13.7.1-py3-none-any.whl", hash = "sha256:4edbae314f59eb482f54e9e30bf00d33350aaa94f4bfcd4e9e3110e64d0d7222"},+ {file = "rich-13.7.1.tar.gz", hash = "sha256:9be308cb1fe2f1f57d67ce99e95af38a1e2bc71ad9813b0e247cf7ffbcc3a432"},][[package]]@@ -4749,7 +4748,7 @@ files = [[[package]]name = "unstract-adapters"-version = "0.2.2"+version = "0.3.0"requires_python = "<3.12,>=3.9"summary = "Unstract Adapters"groups = ["hook-check-django-migrations"]@@ -4777,8 +4776,8 @@ dependencies = ["weaviate-client==3.25.3",]files = [- {file = "unstract_adapters-0.2.2-py3-none-any.whl", hash = "sha256:aaf48036f844f4e0295c36596b1772111ab18229246b8677227b983b3c4226e4"},- {file = "unstract_adapters-0.2.2.tar.gz", hash = "sha256:465d7859a0eb7a905d2c626939f2fed3a501600f832c93160b3e27a559e5ab22"},+ {file = "unstract_adapters-0.3.0-py3-none-any.whl", hash = "sha256:ac2e6d6902bb54c0ef4d223b68365d4f044fb92b83c05bcd92208155e548253c"},+ {file = "unstract_adapters-0.3.0.tar.gz", hash = "sha256:3c4662a73dfaa0b9c0b947da8e3e351221b876c6dd505d054c70a3067fe8d9dc"},][[package]]@@ -4836,7 +4835,7 @@ dependencies = [[[package]]name = "unstract-sdk"-version = "0.12.1"+version = "0.13.0"requires_python = "<3.11.1,>=3.9"summary = "A framework for writing Unstract Tools/Apps"groups = ["hook-check-django-migrations"]@@ -4850,11 +4849,11 @@ dependencies = ["python-magic~=0.4.27","tiktoken~=0.4.0","transformers==4.37.0",- "unstract-adapters~=0.2.2",+ "unstract-adapters~=0.3.0",]files = [- {file = "unstract_sdk-0.12.1-py3-none-any.whl", hash = "sha256:960d8a7b92a1ad32894650edd9254aaf7c3f7555053991b3860e55ab684624a8"},- {file = "unstract_sdk-0.12.1.tar.gz", hash = "sha256:06b1a7bf17948d548137502127458d83046c2e0b596ff326a292dbe94041a13a"},+ {file = "unstract_sdk-0.13.0-py3-none-any.whl", hash = "sha256:15c2aa60a93654fa9eab8b90024ca11b7fc5c9e3e6b8cdde97e11279dde0b2a3"},+ {file = "unstract_sdk-0.13.0.tar.gz", hash = "sha256:64e97366dd742964620201ed4a3aeb6495c74efd4ba482266f0308bafa08e70d"},][[package]]@@ -4869,8 +4868,8 @@ dependencies = ["PyYAML~=6.0.1","docker~=6.1.3","jsonschema~=4.18.2",- "unstract-adapters~=0.2.1",- "unstract-tool-sandbox @ file:///${PROJECT_ROOT}/unstract/tool-sandbox",+ "unstract-adapters~=0.3.0",+ "unstract-tool-sandbox",][[package]]diff --git a/prompt-service/pdm.lock b/prompt-service/pdm.lockindex d59328445..6ee67fdbf 100644--- a/prompt-service/pdm.lock+++ b/prompt-service/pdm.lock@@ -5,7 +5,7 @@groups = ["default", "deploy"]strategy = ["cross_platform", "inherit_metadata"]lock_version = "4.4.1"-content_hash = "sha256:6877a7e5b2b53d33a6f17b473de66b9d607119780e13c420f9216f7fcadd1186"+content_hash = "sha256:f70592e5a8efe531f8e1b05ab6be734d0e13fe53818c9ee1bd38a51e53a729a3"[[package]]name = "aiohttp"@@ -330,23 +330,23 @@ files = [[[package]]name = "boto3"-version = "1.34.51"+version = "1.34.53"requires_python = ">= 3.8"summary = "The AWS SDK for Python"groups = ["default"]dependencies = [- "botocore<1.35.0,>=1.34.51",+ "botocore<1.35.0,>=1.34.53","jmespath<2.0.0,>=0.7.1","s3transfer<0.11.0,>=0.10.0",]files = [- {file = "boto3-1.34.51-py3-none-any.whl", hash = "sha256:67732634dc7d0afda879bd9a5e2d0818a2c14a98bef766b95a3e253ea5104cb9"},- {file = "boto3-1.34.51.tar.gz", hash = "sha256:2cd9463e738a184cbce8a6824027c22163c5f73e277a35ff5aa0fb0e845b4301"},+ {file = "boto3-1.34.53-py3-none-any.whl", hash = "sha256:340c73f57fcca6f503403e2e13a0a4ad44bec218feee2e0896be612324394afd"},+ {file = "boto3-1.34.53.tar.gz", hash = "sha256:cd30261a782824ce543a628ae524480abb4ca6ab4e4a2631477e48baed43b5f2"},][[package]]name = "botocore"-version = "1.34.51"+version = "1.34.53"requires_python = ">= 3.8"summary = "Low-level, data-driven core of boto 3."groups = ["default"]@@ -357,8 +357,8 @@ dependencies = ["urllib3<2.1,>=1.25.4; python_version >= \"3.10\"",]files = [- {file = "botocore-1.34.51-py3-none-any.whl", hash = "sha256:01d5156247f991b3466a8404e3d7460a9ecbd9b214f9992d6ba797d9ddc6f120"},- {file = "botocore-1.34.51.tar.gz", hash = "sha256:5086217442e67dd9de36ec7e87a0c663f76b7790d5fb6a12de565af95e87e319"},+ {file = "botocore-1.34.53-py3-none-any.whl", hash = "sha256:cbbcaddc35738d32df55d26ed5561cf3fa32751a6b22e7e342be87b5e3f55eec"},+ {file = "botocore-1.34.53.tar.gz", hash = "sha256:3d243781e994dfc5b20036d9fb92672bfaef4dbe388eaa79dae6440ea56c53eb"},][[package]]@@ -1114,7 +1114,7 @@ files = [[[package]]name = "gotrue"-version = "2.1.0"+version = "2.4.1"requires_python = ">=3.8,<4.0"summary = "Python Client Library for GoTrue"groups = ["default"]@@ -1123,8 +1123,8 @@ dependencies = ["pydantic<3,>=1.10",]files = [- {file = "gotrue-2.1.0-py3-none-any.whl", hash = "sha256:6483d9a3ac9be1d1ad510be24171e133aa1cec702cc10a8f323b9e7519642447"},- {file = "gotrue-2.1.0.tar.gz", hash = "sha256:b21d48ee64f0f6a1ed111efe4871a83e542529f1a75a264833b50e6433cd3c98"},+ {file = "gotrue-2.4.1-py3-none-any.whl", hash = "sha256:9647bb7a585c969d26667df21168fa20b18f91c5d6afe286af08d7a0610fd2cc"},+ {file = "gotrue-2.4.1.tar.gz", hash = "sha256:8b260ef285f45a3a2f9b5a006f12afb9fad7a36a28fa277f19e733f22eb88584"},][[package]]@@ -2646,7 +2646,7 @@ files = [[[package]]name = "python-dateutil"-version = "2.8.2"+version = "2.9.0"requires_python = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7"summary = "Extensions to the standard Python datetime module"groups = ["default"]@@ -2654,8 +2654,8 @@ dependencies = ["six>=1.5",]files = [- {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"},- {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"},+ {file = "python-dateutil-2.9.0.tar.gz", hash = "sha256:78e73e19c63f5b20ffa567001531680d939dc042bf7850431877645523c66709"},+ {file = "python_dateutil-2.9.0-py2.py3-none-any.whl", hash = "sha256:cbf2f1da5e6083ac2fbfd4da39a25f34312230110440f424a14c7558bb85d82e"},][[package]]@@ -2884,7 +2884,7 @@ files = [[[package]]name = "rich"-version = "13.7.0"+version = "13.7.1"requires_python = ">=3.7.0"summary = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal"groups = ["default"]@@ -2893,8 +2893,8 @@ dependencies = ["pygments<3.0.0,>=2.13.0",]files = [- {file = "rich-13.7.0-py3-none-any.whl", hash = "sha256:6da14c108c4866ee9520bbffa71f6fe3962e193b7da68720583850cd4548e235"},- {file = "rich-13.7.0.tar.gz", hash = "sha256:5cb5123b5cf9ee70584244246816e9114227e0b98ad9176eede6ad54bf5403fa"},+ {file = "rich-13.7.1-py3-none-any.whl", hash = "sha256:4edbae314f59eb482f54e9e30bf00d33350aaa94f4bfcd4e9e3110e64d0d7222"},+ {file = "rich-13.7.1.tar.gz", hash = "sha256:9be308cb1fe2f1f57d67ce99e95af38a1e2bc71ad9813b0e247cf7ffbcc3a432"},][[package]]@@ -3643,7 +3643,7 @@ files = [[[package]]name = "unstract-adapters"-version = "0.2.2"+version = "0.3.0"requires_python = "<3.12,>=3.9"summary = "Unstract Adapters"groups = ["default"]@@ -3671,13 +3671,13 @@ dependencies = ["weaviate-client==3.25.3",]files = [- {file = "unstract_adapters-0.2.2-py3-none-any.whl", hash = "sha256:aaf48036f844f4e0295c36596b1772111ab18229246b8677227b983b3c4226e4"},- {file = "unstract_adapters-0.2.2.tar.gz", hash = "sha256:465d7859a0eb7a905d2c626939f2fed3a501600f832c93160b3e27a559e5ab22"},+ {file = "unstract_adapters-0.3.0-py3-none-any.whl", hash = "sha256:ac2e6d6902bb54c0ef4d223b68365d4f044fb92b83c05bcd92208155e548253c"},+ {file = "unstract_adapters-0.3.0.tar.gz", hash = "sha256:3c4662a73dfaa0b9c0b947da8e3e351221b876c6dd505d054c70a3067fe8d9dc"},][[package]]name = "unstract-sdk"-version = "0.12.1"+version = "0.13.0"requires_python = "<3.11.1,>=3.9"summary = "A framework for writing Unstract Tools/Apps"groups = ["default"]@@ -3691,11 +3691,11 @@ dependencies = ["python-magic~=0.4.27","tiktoken~=0.4.0","transformers==4.37.0",- "unstract-adapters~=0.2.2",+ "unstract-adapters~=0.3.0",]files = [- {file = "unstract_sdk-0.12.1-py3-none-any.whl", hash = "sha256:960d8a7b92a1ad32894650edd9254aaf7c3f7555053991b3860e55ab684624a8"},- {file = "unstract_sdk-0.12.1.tar.gz", hash = "sha256:06b1a7bf17948d548137502127458d83046c2e0b596ff326a292dbe94041a13a"},+ {file = "unstract_sdk-0.13.0-py3-none-any.whl", hash = "sha256:15c2aa60a93654fa9eab8b90024ca11b7fc5c9e3e6b8cdde97e11279dde0b2a3"},+ {file = "unstract_sdk-0.13.0.tar.gz", hash = "sha256:64e97366dd742964620201ed4a3aeb6495c74efd4ba482266f0308bafa08e70d"},][[package]]diff --git a/prompt-service/pyproject.toml b/prompt-service/pyproject.tomlindex 315787a51..b5d92cff6 100644--- a/prompt-service/pyproject.toml+++ b/prompt-service/pyproject.toml@@ -17,7 +17,7 @@ dependencies = ["flask~=3.0","llama-index==0.9.28","python-dotenv==1.0.0",- "unstract-sdk~=0.12.1",+ "unstract-sdk~=0.13.0",]requires-python = ">=3.9,<3.11.1"readme = "README.md"diff --git a/pyproject.toml b/pyproject.tomlindex 922af43df..94ce339ca 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -41,8 +41,8 @@ hook-check-django-migrations = ["psycopg2-binary==2.9.9","python-dotenv==1.0.0","python-magic==0.4.27",- "unstract-sdk~=0.12.1",- "unstract-adapters~=0.2.2",+ "unstract-sdk~=0.13.0",+ "unstract-adapters~=0.3.0","-e unstract-connectors @ file:///${PROJECT_ROOT}/unstract/connectors","-e unstract-core @ file:///${PROJECT_ROOT}/unstract/core","-e unstract-flags @ file:///${PROJECT_ROOT}/unstract/flags",diff --git a/tools/indexer/requirements.txt b/tools/indexer/requirements.txtindex 94a83b7d9..9faca9b40 100644--- a/tools/indexer/requirements.txt+++ b/tools/indexer/requirements.txt@@ -1,4 +1,4 @@# Add your dependencies here# Required for all unstract tools-unstract-sdk~=0.11.1+unstract-sdk~=0.13.0diff --git a/tools/indexer/src/config/properties.json b/tools/indexer/src/config/properties.jsonindex b3ed95dc9..d2376b378 100644--- a/tools/indexer/src/config/properties.json+++ b/tools/indexer/src/config/properties.json@@ -2,7 +2,7 @@"schemaVersion": "0.0.1","displayName": "Document Indexer","functionName": "document_indexer",- "toolVersion": "0.0.1",+ "toolVersion": "0.0.2","description": "Used to index documents in the specified vector DB / embedding.","input": {"description": "Document that needs to be indexed"diff --git a/tools/indexer/src/main.py b/tools/indexer/src/main.pyindex 05a7edfa1..eb8e81bf6 100644--- a/tools/indexer/src/main.py+++ b/tools/indexer/src/main.py@@ -34,7 +34,7 @@ class DocumentIndexer(BaseTool):tool_index = ToolIndex(tool=self)self.stream_log("Indexing document...")try:- tool_index.index_file(+ index_key = tool_index.index_file(tool_id=self.workflow_id,embedding_type=settings[ToolSettingsKey.EMBEDDING_ADAPTER_ID],vector_db=settings[ToolSettingsKey.VECTOR_DB_ADAPTER_ID],@@ -47,15 +47,6 @@ class DocumentIndexer(BaseTool):)except Exception as e:self.stream_error_and_exit(f"Error fetching data and indexing: {e}")- index_key = ToolIndex.generate_file_id(- tool_id=self.workflow_id,- file_hash=file_hash,- vector_db=settings[ToolSettingsKey.VECTOR_DB_ADAPTER_ID],- embedding=settings[ToolSettingsKey.EMBEDDING_ADAPTER_ID],- x2text=settings[ToolSettingsKey.X2TEXT_ADAPTER_ID],- chunk_size=settings[SettingsKeys.CHUNK_SIZE],- chunk_overlap=settings[SettingsKeys.CHUNK_OVERLAP],- )# Update GUIinput_log = ("### Indexing file\n"diff --git a/tools/structure/requirements.txt b/tools/structure/requirements.txtindex 94a83b7d9..9faca9b40 100644--- a/tools/structure/requirements.txt+++ b/tools/structure/requirements.txt@@ -1,4 +1,4 @@# Add your dependencies here# Required for all unstract tools-unstract-sdk~=0.11.1+unstract-sdk~=0.13.0diff --git a/tools/structure/src/config/properties.json b/tools/structure/src/config/properties.jsonindex 7ce7b0094..20ebd1e7c 100644--- a/tools/structure/src/config/properties.json+++ b/tools/structure/src/config/properties.json@@ -2,7 +2,7 @@"schemaVersion": "0.0.1","displayName": "Structure Tool","functionName": "structure_tool",- "toolVersion": "0.0.1",+ "toolVersion": "0.0.2","description": "This is a template tool which can answer set of input prompts designed in the Prompt Studio","input": {"description": "File that needs to be indexed and parsed for answers"diff --git a/tools/text_extractor/requirements.txt b/tools/text_extractor/requirements.txtindex 94a83b7d9..9faca9b40 100644--- a/tools/text_extractor/requirements.txt+++ b/tools/text_extractor/requirements.txt@@ -1,4 +1,4 @@# Add your dependencies here# Required for all unstract tools-unstract-sdk~=0.11.1+unstract-sdk~=0.13.0diff --git a/tools/text_extractor/src/config/properties.json b/tools/text_extractor/src/config/properties.jsonindex c06041a30..fe9a32571 100644--- a/tools/text_extractor/src/config/properties.json+++ b/tools/text_extractor/src/config/properties.json@@ -2,7 +2,7 @@"schemaVersion": "0.0.1","displayName": "Text Extractor","functionName": "text_extractor",- "toolVersion": "0.0.2-v2",+ "toolVersion": "0.0.2","description": "The Text Extractor is a powerful tool designed to convert documents to its text form or Extract texts from documents","input": {"description": "Document"diff --git a/unstract/tool-registry/pyproject.toml b/unstract/tool-registry/pyproject.tomlindex febac458e..1db7a1ccf 100644--- a/unstract/tool-registry/pyproject.toml+++ b/unstract/tool-registry/pyproject.toml@@ -13,7 +13,7 @@ dependencies = ["docker~=6.1.3","jsonschema~=4.18.2","PyYAML~=6.0.1",- "unstract-adapters~=0.2.1",+ "unstract-adapters~=0.3.0",# ! IMPORTANT!# Local dependencies usually need to be added as:# https://pdm-project.org/latest/usage/dependency/#local-dependenciesdiff --git a/unstract/tool-registry/src/unstract/tool_registry/public_tools.json b/unstract/tool-registry/src/unstract/tool_registry/public_tools.jsonindex 65afe0b0e..0e9a215dc 100644--- a/unstract/tool-registry/src/unstract/tool_registry/public_tools.json+++ b/unstract/tool-registry/src/unstract/tool_registry/public_tools.json@@ -140,7 +140,7 @@"schemaVersion": "0.0.1","displayName": "Document Indexer","functionName": "document_indexer",- "toolVersion": "0.0.1",+ "toolVersion": "0.0.2","description": "Used to index documents in the specified vector DB / embedding.","input": {"description": "Document that needs to be indexed"@@ -249,9 +249,9 @@"properties": {}},"icon": "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n<svg\n enable-background=\"new 0 0 24 24\"\n height=\"48\"\n viewBox=\"0 0 24 24\"\n width=\"48\"\n fill=\"#000000\"\n version=\"1.1\"\n id=\"svg552\"\n sodipodi:docname=\"summarize_black_48dp.svg\"\n inkscape:export-filename=\"summarize_black_48dp.png\"\n inkscape:export-xdpi=\"96\"\n inkscape:export-ydpi=\"96\"\n xmlns:inkscape=\"http://www.inkscape.org/namespaces/inkscape\"\n xmlns:sodipodi=\"http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd\"\n xmlns=\"http://www.w3.org/2000/svg\"\n xmlns:svg=\"http://www.w3.org/2000/svg\">\n <defs\n id=\"defs556\" />\n <sodipodi:namedview\n id=\"namedview554\"\n pagecolor=\"#ffffff\"\n bordercolor=\"#000000\"\n borderopacity=\"0.25\"\n inkscape:showpageshadow=\"2\"\n inkscape:pageopacity=\"0.0\"\n inkscape:pagecheckerboard=\"0\"\n inkscape:deskcolor=\"#d1d1d1\"\n showgrid=\"false\" />\n <g\n id=\"g544\">\n <path\n d=\"M 0,0 H 24 V 24 H 0 Z\"\n fill=\"none\"\n id=\"path542\" />\n </g>\n <g\n id=\"g550\"\n style=\"fill:#ff4d6d;fill-opacity:1\">\n <g\n id=\"g548\"\n style=\"fill:#ff4d6d;fill-opacity:1\">\n <path\n d=\"M 15,3 H 5 C 3.9,3 3.01,3.9 3.01,5 L 3,19 c 0,1.1 0.89,2 1.99,2 H 19 c 1.1,0 2,-0.9 2,-2 V 9 Z M 5,19 V 5 h 9 v 5 h 5 v 9 z M 9,8 C 9,8.55 8.55,9 8,9 7.45,9 7,8.55 7,8 7,7.45 7.45,7 8,7 8.55,7 9,7.45 9,8 Z m 0,4 c 0,0.55 -0.45,1 -1,1 -0.55,0 -1,-0.45 -1,-1 0,-0.55 0.45,-1 1,-1 0.55,0 1,0.45 1,1 z m 0,4 c 0,0.55 -0.45,1 -1,1 -0.55,0 -1,-0.45 -1,-1 0,-0.55 0.45,-1 1,-1 0.55,0 1,0.45 1,1 z\"\n id=\"path546\"\n style=\"fill:#ff4d6d;fill-opacity:1\" />\n </g>\n </g>\n</svg>\n",- "image_url": "docker:unstract/tool-indexer:0.0.1",+ "image_url": "docker:unstract/tool-indexer:0.0.2","image_name": "unstract/tool-indexer",- "image_tag": "0.0.1"+ "image_tag": "0.0.2"},"text_extractor": {"tool_uid": "text_extractor",@@ -259,7 +259,7 @@"schemaVersion": "0.0.1","displayName": "Text Extractor","functionName": "text_extractor",- "toolVersion": "0.0.1",+ "toolVersion": "0.0.2","description": "The Text Extractor is a powerful tool designed to convert documents to its text form or Extract texts from documents","input": {"description": "Document"@@ -334,9 +334,9 @@}},"icon": "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n<svg\n enable-background=\"new 0 0 20 20\"\n height=\"48\"\n viewBox=\"0 0 20 20\"\n width=\"48\"\n fill=\"#000000\"\n version=\"1.1\"\n id=\"svg8109\"\n sodipodi:docname=\"folder_copy_black_48dp.svg\"\n xmlns:inkscape=\"http://www.inkscape.org/namespaces/inkscape\"\n xmlns:sodipodi=\"http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd\"\n xmlns=\"http://www.w3.org/2000/svg\"\n xmlns:svg=\"http://www.w3.org/2000/svg\">\n <defs\n id=\"defs8113\" />\n <sodipodi:namedview\n id=\"namedview8111\"\n pagecolor=\"#ffffff\"\n bordercolor=\"#000000\"\n borderopacity=\"0.25\"\n inkscape:showpageshadow=\"2\"\n inkscape:pageopacity=\"0.0\"\n inkscape:pagecheckerboard=\"0\"\n inkscape:deskcolor=\"#d1d1d1\"\n showgrid=\"false\" />\n <g\n id=\"g8099\">\n <rect\n fill=\"none\"\n height=\"20\"\n width=\"20\"\n x=\"0\"\n id=\"rect8097\"\n y=\"0\" />\n </g>\n <g\n id=\"g8107\"\n style=\"fill:#ff4d6d;fill-opacity:1\">\n <g\n id=\"g8105\"\n style=\"fill:#ff4d6d;fill-opacity:1\">\n <path\n d=\"M 2.5,5 H 1 V 15.5 C 1,16.33 1.67,17 2.5,17 H 15.68 V 15.5 H 2.5 Z\"\n id=\"path8101\"\n style=\"fill:#ff4d6d;fill-opacity:1\" />\n <path\n d=\"M 16.5,4 H 11 L 9,2 H 5.5 C 4.67,2 4,2.67 4,3.5 v 9 C 4,13.33 4.67,14 5.5,14 h 11 c 0.83,0 1.5,-0.67 1.5,-1.5 v -7 C 18,4.67 17.33,4 16.5,4 Z m 0,8.5 h -11 v -9 h 2.88 l 2,2 h 6.12 z\"\n id=\"path8103\"\n style=\"fill:#ff4d6d;fill-opacity:1\" />\n </g>\n </g>\n</svg>\n",- "image_url": "docker:unstract/tool-text-extractor:0.0.1",+ "image_url": "docker:unstract/tool-text-extractor:0.0.2","image_name": "unstract/tool-text-extractor",- "image_tag": "0.0.1"+ "image_tag": "0.0.2"},"ocr": {"tool_uid": "ocr",