Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

fix: Bumped SDK for backend and structure tool (#635)

Chandrasekharan M2fc43aa934bc
diff --git a/backend/pdm.lock b/backend/pdm.lock
index 78d606630..1fcb54158 100644
--- a/backend/pdm.lock
+++ b/backend/pdm.lock
@@ -5,7 +5,7 @@
groups = ["default", "deploy", "dev", "test"]
strategy = ["inherit_metadata"]
lock_version = "4.4.2"
-content_hash = "sha256:5a41f7fb855a552c750c23bda694bd4c8058dc87d912a06fd0e1933878e92d86"
+content_hash = "sha256:73f9c170bdb20c4d3dbefec017dce3cf3bdd90966c3ac31a5bdbcdba985240bf"
[[package]]
name = "adlfs"
@@ -92,16 +92,16 @@ files = [
[[package]]
name = "aioitertools"
-version = "0.11.0"
-requires_python = ">=3.6"
+version = "0.12.0"
+requires_python = ">=3.8"
summary = "itertools and builtins for AsyncIO and mixed iterables"
groups = ["default", "dev"]
dependencies = [
"typing-extensions>=4.0; python_version < \"3.10\"",
]
files = [
- {file = "aioitertools-0.11.0-py3-none-any.whl", hash = "sha256:04b95e3dab25b449def24d7df809411c10e62aab0cbe31a50ca4e68748c43394"},
- {file = "aioitertools-0.11.0.tar.gz", hash = "sha256:42c68b8dd3a69c2bf7f2233bf7df4bb58b557bca5252ac02ed5187bbc67d6831"},
+ {file = "aioitertools-0.12.0-py3-none-any.whl", hash = "sha256:fc1f5fac3d737354de8831cbba3eb04f79dd649d8f3afb4c5b114925e662a796"},
+ {file = "aioitertools-0.12.0.tar.gz", hash = "sha256:c2a9055b4fbb7705f561b9d86053e8af5d10cc845d22c32008c43490b2d8dd6b"},
]
[[package]]
@@ -521,13 +521,13 @@ files = [
[[package]]
name = "certifi"
-version = "2024.7.4"
+version = "2024.8.30"
requires_python = ">=3.6"
summary = "Python package for providing Mozilla's CA Bundle."
groups = ["default", "dev"]
files = [
- {file = "certifi-2024.7.4-py3-none-any.whl", hash = "sha256:c198e21b1289c2ab85ee4e67bb4b4ef3ead0892059901a8d5b622f24a1101e90"},
- {file = "certifi-2024.7.4.tar.gz", hash = "sha256:5a1e7645bc0ec61a09e26c36f6106dd4cf40c6db3a1fb6352b0244e7fb057c7b"},
+ {file = "certifi-2024.8.30-py3-none-any.whl", hash = "sha256:922820b53db7a7257ffbda3f597266d435245903d80737e34f8a45ff3e3230d8"},
+ {file = "certifi-2024.8.30.tar.gz", hash = "sha256:bec941d2aa8195e248a60b31ff9f0558284cf01a52591ceda73ea9afffd69fd9"},
]
[[package]]
@@ -2468,7 +2468,7 @@ files = [
[[package]]
name = "openai"
-version = "1.42.0"
+version = "1.43.0"
requires_python = ">=3.7.1"
summary = "The official Python library for the openai API"
groups = ["default", "dev"]
@@ -2483,8 +2483,8 @@ dependencies = [
"typing-extensions<5,>=4.11",
]
files = [
- {file = "openai-1.42.0-py3-none-any.whl", hash = "sha256:dc91e0307033a4f94931e5d03cc3b29b9717014ad5e73f9f2051b6cb5eda4d80"},
- {file = "openai-1.42.0.tar.gz", hash = "sha256:c9d31853b4e0bc2dc8bd08003b462a006035655a701471695d0bfdc08529cde3"},
+ {file = "openai-1.43.0-py3-none-any.whl", hash = "sha256:1a748c2728edd3a738a72a0212ba866f4fdbe39c9ae03813508b267d45104abe"},
+ {file = "openai-1.43.0.tar.gz", hash = "sha256:e607aff9fc3e28eade107e5edd8ca95a910a4b12589336d3cbb6bfe2ac306b3c"},
]
[[package]]
@@ -2850,13 +2850,13 @@ files = [
[[package]]
name = "pymilvus"
-version = "2.4.5"
+version = "2.4.6"
requires_python = ">=3.8"
summary = "Python Sdk for Milvus"
groups = ["default", "dev"]
dependencies = [
"environs<=9.5.0",
- "grpcio<=1.63.0,>=1.49.1",
+ "grpcio>=1.49.1",
"milvus-lite<2.5.0,>=2.4.0; sys_platform != \"win32\"",
"pandas>=1.2.4",
"protobuf>=3.20.0",
@@ -2864,8 +2864,8 @@ dependencies = [
"ujson>=2.0.0",
]
files = [
- {file = "pymilvus-2.4.5-py3-none-any.whl", hash = "sha256:dc4f2d1eac8db9cf3951de39566a1a244695760bb94d8310fbfc73d6d62bb267"},
- {file = "pymilvus-2.4.5.tar.gz", hash = "sha256:1a497fe9b41d6bf62b1d5e1c412960922dde1598576fcbb8818040c8af11149f"},
+ {file = "pymilvus-2.4.6-py3-none-any.whl", hash = "sha256:b4c43472edc313b845d313be50610e19054e6954b2c5c3b515565c596c2d3d97"},
+ {file = "pymilvus-2.4.6.tar.gz", hash = "sha256:6ac3eb91c92cc01bbe444fe83f895f02d7b2546d96ac67998630bf31ac074d66"},
]
[[package]]
@@ -3731,7 +3731,7 @@ dependencies = [
[[package]]
name = "unstract-sdk"
-version = "0.46.0"
+version = "0.48.0"
requires_python = "<3.11.1,>=3.9"
summary = "A framework for writing Unstract Tools/Apps"
groups = ["default", "dev"]
@@ -3770,8 +3770,8 @@ dependencies = [
"transformers==4.37.0",
]
files = [
- {file = "unstract_sdk-0.46.0-py3-none-any.whl", hash = "sha256:09b13289c3384578b6e9598cfe2ae1013705d9e394218682aa24bbb6406422b4"},
- {file = "unstract_sdk-0.46.0.tar.gz", hash = "sha256:4fa969eb078b5f1bb0eb05142519a08d307b5ea95afaaf06fb8383285857d037"},
+ {file = "unstract_sdk-0.48.0-py3-none-any.whl", hash = "sha256:3b795be2ad1b695b342b7a6c675f99ff5a3acf98a66bce9f93a016f80af9e580"},
+ {file = "unstract_sdk-0.48.0.tar.gz", hash = "sha256:a00574d17cf06ac461b479b42f518efaf52356e4ad58ea523c4608faf17337e3"},
]
[[package]]
@@ -3786,7 +3786,7 @@ dependencies = [
"PyYAML~=6.0.1",
"docker~=6.1.3",
"jsonschema~=4.18.2",
- "unstract-sdk~=0.46.0",
+ "unstract-sdk~=0.48.0",
"unstract-tool-sandbox",
]
@@ -3829,13 +3829,13 @@ files = [
[[package]]
name = "urllib3"
-version = "1.26.19"
+version = "1.26.20"
requires_python = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7"
summary = "HTTP library with thread-safe connection pooling, file post, and more."
groups = ["default", "dev"]
files = [
- {file = "urllib3-1.26.19-py2.py3-none-any.whl", hash = "sha256:37a0344459b199fce0e80b0d3569837ec6b6937435c5244e7fd73fa6006830f3"},
- {file = "urllib3-1.26.19.tar.gz", hash = "sha256:3e3d753a8618b86d7de333b4223005f68720bcd6a7d2bcb9fbd2229ec7c1e429"},
+ {file = "urllib3-1.26.20-py2.py3-none-any.whl", hash = "sha256:0ed14ccfbf1c30a9072c7ca157e4319b70d65f623e91e7b32fadb2853431016e"},
+ {file = "urllib3-1.26.20.tar.gz", hash = "sha256:40c2dc0c681e47eb8f90e7e27bf6ff7df2e677421fd46756da1161c39ca70d32"},
]
[[package]]
@@ -3930,8 +3930,8 @@ files = [
[[package]]
name = "yarl"
-version = "1.9.4"
-requires_python = ">=3.7"
+version = "1.9.7"
+requires_python = ">=3.8"
summary = "Yet another URL library"
groups = ["default", "dev"]
dependencies = [
@@ -3939,7 +3939,7 @@ dependencies = [
"multidict>=4.0",
]
files = [
- {file = "yarl-1.9.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:23d32a2594cb5d565d358a92e151315d1b2268bc10f4610d098f96b147370136"},
- {file = "yarl-1.9.4-py3-none-any.whl", hash = "sha256:928cecb0ef9d5a7946eb6ff58417ad2fe9375762382f1bf5c55e61645f2c43ad"},
- {file = "yarl-1.9.4.tar.gz", hash = "sha256:566db86717cf8080b99b58b083b773a908ae40f06681e87e589a976faf8246bf"},
+ {file = "yarl-1.9.7-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:29c80890e0a64fb0e5f71350d48da330995073881f8b8e623154aef631febfb0"},
+ {file = "yarl-1.9.7-py3-none-any.whl", hash = "sha256:49935cc51d272264358962d050d726c3e5603a616f53e52ea88e9df1728aa2ee"},
+ {file = "yarl-1.9.7.tar.gz", hash = "sha256:f28e602edeeec01fc96daf7728e8052bc2e12a672e2a138561a1ebaf30fd9df7"},
]
diff --git a/backend/pyproject.toml b/backend/pyproject.toml
index f3d7d756d..910032e94 100644
--- a/backend/pyproject.toml
+++ b/backend/pyproject.toml
@@ -32,7 +32,7 @@ dependencies = [
"python-socketio==5.9.0", # For log_events
"social-auth-app-django==5.3.0", # For OAuth
"social-auth-core==4.4.2", # For OAuth
- "unstract-sdk~=0.46.0",
+ "unstract-sdk~=0.48.0",
# ! IMPORTANT!
# Indirect local dependencies usually need to be added in their own projects
# as: https://pdm-project.org/latest/usage/dependency/#local-dependencies.
diff --git a/backend/sample.env b/backend/sample.env
index 83c7be821..c089a42b9 100644
--- a/backend/sample.env
+++ b/backend/sample.env
@@ -90,9 +90,9 @@ PROMPT_STUDIO_FILE_PATH=/app/prompt-studio-data
# Structure Tool Image (Runs prompt studio exported tools)
# https://hub.docker.com/r/unstract/tool-structure
-STRUCTURE_TOOL_IMAGE_URL="docker:unstract/tool-structure:0.0.40"
+STRUCTURE_TOOL_IMAGE_URL="docker:unstract/tool-structure:0.0.41"
STRUCTURE_TOOL_IMAGE_NAME="unstract/tool-structure"
-STRUCTURE_TOOL_IMAGE_TAG="0.0.40"
+STRUCTURE_TOOL_IMAGE_TAG="0.0.41"
# Feature Flags
EVALUATION_SERVER_IP=unstract-flipt
diff --git a/prompt-service/src/unstract/prompt_service/main.py b/prompt-service/src/unstract/prompt_service/main.py
index 90d824a20..14d24ecbe 100644
--- a/prompt-service/src/unstract/prompt_service/main.py
+++ b/prompt-service/src/unstract/prompt_service/main.py
@@ -92,7 +92,7 @@ def prompt_processor() -> Any:
doc_name = str(payload.get(PSKeys.FILE_NAME, ""))
log_events_id: str = payload.get(PSKeys.LOG_EVENTS_ID, "")
structured_output: dict[str, Any] = {}
- metadata: Optional[dict[str, Any]] = {
+ metadata: dict[str, Any] = {
PSKeys.RUN_ID: run_id,
PSKeys.CONTEXT: {},
}
@@ -276,8 +276,6 @@ def prompt_processor() -> Any:
)
try:
- vector_index = vector_db.get_vector_store_index()
-
context = ""
if output[PSKeys.CHUNK_SIZE] == 0:
# We can do this only for chunkless indexes
@@ -374,6 +372,7 @@ def prompt_processor() -> Any:
retrieval_strategy = output.get(PSKeys.RETRIEVAL_STRATEGY)
if retrieval_strategy in {PSKeys.SIMPLE, PSKeys.SUBQUESTION}:
+ vector_index = vector_db.get_vector_store_index()
answer, context = run_retrieval(
tool_settings=tool_settings,
output=output,
diff --git a/tools/structure/requirements.txt b/tools/structure/requirements.txt
index 0c4e48aea..1cbc6f9cc 100644
--- a/tools/structure/requirements.txt
+++ b/tools/structure/requirements.txt
@@ -1,4 +1,4 @@
# Add your dependencies here
# Required for all unstract tools
-unstract-sdk~=0.46.0
+unstract-sdk~=0.48.0
diff --git a/tools/structure/src/config/properties.json b/tools/structure/src/config/properties.json
index 1ae89a631..d3719c7f9 100644
--- a/tools/structure/src/config/properties.json
+++ b/tools/structure/src/config/properties.json
@@ -2,7 +2,7 @@
"schemaVersion": "0.0.1",
"displayName": "Structure Tool",
"functionName": "structure_tool",
- "toolVersion": "0.0.40",
+ "toolVersion": "0.0.41",
"description": "This is a template tool which can answer set of input prompts designed in the Prompt Studio",
"input": {
"description": "File that needs to be indexed and parsed for answers"
diff --git a/unstract/tool-registry/pyproject.toml b/unstract/tool-registry/pyproject.toml
index f73d8a525..3bfa2831a 100644
--- a/unstract/tool-registry/pyproject.toml
+++ b/unstract/tool-registry/pyproject.toml
@@ -13,7 +13,7 @@ dependencies = [
"docker~=6.1.3",
"jsonschema~=4.18.2",
"PyYAML~=6.0.1",
- "unstract-sdk~=0.46.0",
+ "unstract-sdk~=0.48.0",
# ! IMPORTANT!
# Local dependencies usually need to be added as:
# https://pdm-project.org/latest/usage/dependency/#local-dependencies