Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

Updated sdk version and resolved review comments

Deepaka41f5f8b02cd
diff --git a/backend/file_management/file_management_helper.py b/backend/file_management/file_management_helper.py
index c4c0160f2..a6b9b600d 100644
--- a/backend/file_management/file_management_helper.py
+++ b/backend/file_management/file_management_helper.py
@@ -195,8 +195,8 @@ class FileManagerHelper:
elif file_content_type == "text/plain":
with fs.open(file_path, "r") as file:
FileManagerHelper.logger.info(f"Reading text file: {file_path}")
- encoded_string = file.read()
- return encoded_string
+ text_content = file.read()
+ return text_content
else:
raise InvalidFileType
diff --git a/backend/pdm.lock b/backend/pdm.lock
index da10551b7..7fce08129 100644
--- a/backend/pdm.lock
+++ b/backend/pdm.lock
@@ -5,7 +5,7 @@
groups = ["default", "test", "deploy"]
strategy = ["cross_platform", "inherit_metadata"]
lock_version = "4.4.1"
-content_hash = "sha256:73a791002f5f7bab1afe9eaee07e8fc946774da3448f9daabedbccfc9be4f202"
+content_hash = "sha256:51aa34c4f085355e4448d1cd7c60c9b1a703e12d932c36b70a2fd5c960b340c3"
[[package]]
name = "aiobotocore"
@@ -4667,7 +4667,7 @@ dependencies = [
[[package]]
name = "unstract-sdk"
-version = "0.11.2"
+version = "0.12.0"
requires_python = "<3.11.1,>=3.9"
summary = "A framework for writing Unstract Tools/Apps"
groups = ["default"]
@@ -4684,8 +4684,8 @@ dependencies = [
"unstract-adapters~=0.2.2",
]
files = [
- {file = "unstract_sdk-0.11.2-py3-none-any.whl", hash = "sha256:bcab064d87dc2985e5a4340e438a8ef71ef165b3e5078aba0ed2da3bda77f9e0"},
- {file = "unstract_sdk-0.11.2.tar.gz", hash = "sha256:0d6e3e00c9bbc6e569cc79922a2a45a54870c482cfa61287ea89a22ebd7fc606"},
+ {file = "unstract_sdk-0.12.0-py3-none-any.whl", hash = "sha256:3e7b57a94df8cbdebe13998fface9eaf4b9eba667cd9f323d555b785599b51a6"},
+ {file = "unstract_sdk-0.12.0.tar.gz", hash = "sha256:0208a653a59600fcbfaa9606341a11bb7e4381f64435754389fc46b735684395"},
]
[[package]]
@@ -4700,7 +4700,7 @@ dependencies = [
"docker~=6.1.3",
"jsonschema~=4.18.2",
"unstract-adapters~=0.2.1",
- "unstract-tool-sandbox @ file:///home/gayathrivijayakumar/code/Unstract/unstract/unstract/tool-registry/../tool-sandbox",
+ "unstract-tool-sandbox @ file:///home/ghost/Documents/zipstack/unstract-oss/unstract/tool-registry/../tool-sandbox",
]
[[package]]
diff --git a/backend/prompt_studio/prompt_studio_core/models.py b/backend/prompt_studio/prompt_studio_core/models.py
index 4a2327e7e..82d41948f 100644
--- a/backend/prompt_studio/prompt_studio_core/models.py
+++ b/backend/prompt_studio/prompt_studio_core/models.py
@@ -44,6 +44,7 @@ class CustomTool(BaseModel):
related_name="default_profile",
null=True,
blank=True,
+ db_comment="Default LLM Profile used in prompt",
)
summarize_llm_profile = models.ForeignKey(
ProfileManager,
@@ -51,9 +52,14 @@ class CustomTool(BaseModel):
related_name="summarize_llm_profile",
null=True,
blank=True,
+ db_comment="LLM Profile used for summarize",
+ )
+ summarize_context = models.BooleanField(
+ default=True, db_comment="Flag to summarize content"
+ )
+ summarize_as_source = models.BooleanField(
+ default=True, db_comment="Flag to use summarized content as source"
)
- summarize_context = models.BooleanField(default=True)
- summarize_as_source = models.BooleanField(default=True)
summarize_prompt = models.TextField(
blank=True,
db_comment="Field to store the summarize prompt",
diff --git a/backend/prompt_studio/prompt_studio_core/prompt_studio_helper.py b/backend/prompt_studio/prompt_studio_core/prompt_studio_helper.py
index 191618cb5..3b7dced4a 100644
--- a/backend/prompt_studio/prompt_studio_core/prompt_studio_helper.py
+++ b/backend/prompt_studio/prompt_studio_core/prompt_studio_helper.py
@@ -81,30 +81,37 @@ class PromptStudioHelper:
@staticmethod
def index_document(
- tool_id: str, file_name: str, org_id: str, user_id: str, is_summary: bool = False
+ tool_id: str,
+ file_name: str,
+ org_id: str,
+ user_id: str,
+ is_summary: bool = False,
) -> Any:
"""Method to index a document.
Args:
- tool_id (str):Id of the tool
+ tool_id (str): Id of the tool
file_name (str): File to parse
+ org_id (str): The ID of the organization to which the user belongs.
+ user_id (str): The ID of the user who uploaded the document.
+ is_summary (bool, optional): Whether the document is a summary
+ or not. Defaults to False.
Raises:
ToolNotValid
IndexingError
"""
tool: CustomTool = CustomTool.objects.get(pk=tool_id)
-
if is_summary:
- default_profile: ProfileManager = tool.summarize_llm_profile
+ default_profile = tool.summarize_llm_profile
file_path = file_name
else:
- default_profile: ProfileManager = tool.default_profile
+ default_profile = tool.default_profile
file_path = FileManagerHelper.handle_sub_directory_for_tenants(
org_id, is_create=False, user_id=user_id, tool_id=tool_id
)
file_path = str(Path(file_path) / file_name)
-
+
if not default_profile:
raise DefaultProfileError()
stream_log.publish(
@@ -185,16 +192,15 @@ class PromptStudioHelper:
prompts: list[ToolStudioPrompt] = []
prompts.append(prompt_instance)
tool: CustomTool = prompt_instance.tool_id
-
-
+
if tool.summarize_as_source:
directory, filename = os.path.split(file_path)
- file_path: str = os.path.join(
+ file_path = os.path.join(
directory,
TSPKeys.SUMMARIZE,
os.path.splitext(filename)[0] + ".txt",
)
-
+
stream_log.publish(
tool.tool_id,
stream_log.log(
@@ -290,7 +296,7 @@ class PromptStudioHelper:
file_name=path,
tool_id=str(tool.tool_id),
org_id=org_id,
- is_summary=tool.summarize_as_source
+ is_summary=tool.summarize_as_source,
)
output: dict[str, Any] = {}
diff --git a/backend/pyproject.toml b/backend/pyproject.toml
index 9494a1fda..3a1e962ff 100644
--- a/backend/pyproject.toml
+++ b/backend/pyproject.toml
@@ -30,7 +30,7 @@ dependencies = [
"python-socketio==5.9.0", # For log_events
"social-auth-app-django==5.3.0", # For OAuth
"social-auth-core==4.4.2", # For OAuth
- "unstract-sdk~=0.11.2",
+ "unstract-sdk~=0.12.0",
"unstract-adapters~=0.2.2",
# ! IMPORTANT!
# Indirect local dependencies usually need to be added in their own projects
diff --git a/pdm.lock b/pdm.lock
index a617b0be0..e071baeb4 100644
--- a/pdm.lock
+++ b/pdm.lock
@@ -5,7 +5,7 @@
groups = ["default", "hook-check-django-migrations", "lint"]
strategy = ["cross_platform", "inherit_metadata"]
lock_version = "4.4.1"
-content_hash = "sha256:34bd617283cb8620b48d2ff82e83a4721d9464962bb6ccebd4d9f860efaed221"
+content_hash = "sha256:32fcfebc305b03ed156f6a208a4edcf40cd407f555fb695d3cb932aaa43ece4f"
[[package]]
name = "absolufy-imports"
@@ -4836,7 +4836,7 @@ dependencies = [
[[package]]
name = "unstract-sdk"
-version = "0.11.2"
+version = "0.12.0"
requires_python = "<3.11.1,>=3.9"
summary = "A framework for writing Unstract Tools/Apps"
groups = ["hook-check-django-migrations"]
@@ -4853,8 +4853,8 @@ dependencies = [
"unstract-adapters~=0.2.2",
]
files = [
- {file = "unstract_sdk-0.11.2-py3-none-any.whl", hash = "sha256:bcab064d87dc2985e5a4340e438a8ef71ef165b3e5078aba0ed2da3bda77f9e0"},
- {file = "unstract_sdk-0.11.2.tar.gz", hash = "sha256:0d6e3e00c9bbc6e569cc79922a2a45a54870c482cfa61287ea89a22ebd7fc606"},
+ {file = "unstract_sdk-0.12.0-py3-none-any.whl", hash = "sha256:3e7b57a94df8cbdebe13998fface9eaf4b9eba667cd9f323d555b785599b51a6"},
+ {file = "unstract_sdk-0.12.0.tar.gz", hash = "sha256:0208a653a59600fcbfaa9606341a11bb7e4381f64435754389fc46b735684395"},
]
[[package]]
diff --git a/prompt-service/pdm.lock b/prompt-service/pdm.lock
index 82dec09a0..652afff29 100644
--- a/prompt-service/pdm.lock
+++ b/prompt-service/pdm.lock
@@ -5,7 +5,7 @@
groups = ["default", "deploy"]
strategy = ["cross_platform", "inherit_metadata"]
lock_version = "4.4.1"
-content_hash = "sha256:c02615c1fe1a21cddaa8ee726e7d3166446a694f45debed47272c2d8ee1504e0"
+content_hash = "sha256:64a1667d589e799eb7d6cbcb67e2bc4adcfdfdc8335f21ae6168562e3db368e5"
[[package]]
name = "aiohttp"
@@ -3677,7 +3677,7 @@ files = [
[[package]]
name = "unstract-sdk"
-version = "0.11.2"
+version = "0.12.0"
requires_python = "<3.11.1,>=3.9"
summary = "A framework for writing Unstract Tools/Apps"
groups = ["default"]
@@ -3694,8 +3694,8 @@ dependencies = [
"unstract-adapters~=0.2.2",
]
files = [
- {file = "unstract_sdk-0.11.2-py3-none-any.whl", hash = "sha256:bcab064d87dc2985e5a4340e438a8ef71ef165b3e5078aba0ed2da3bda77f9e0"},
- {file = "unstract_sdk-0.11.2.tar.gz", hash = "sha256:0d6e3e00c9bbc6e569cc79922a2a45a54870c482cfa61287ea89a22ebd7fc606"},
+ {file = "unstract_sdk-0.12.0-py3-none-any.whl", hash = "sha256:3e7b57a94df8cbdebe13998fface9eaf4b9eba667cd9f323d555b785599b51a6"},
+ {file = "unstract_sdk-0.12.0.tar.gz", hash = "sha256:0208a653a59600fcbfaa9606341a11bb7e4381f64435754389fc46b735684395"},
]
[[package]]
diff --git a/prompt-service/pyproject.toml b/prompt-service/pyproject.toml
index ae624cbee..4cc1facce 100644
--- a/prompt-service/pyproject.toml
+++ b/prompt-service/pyproject.toml
@@ -17,7 +17,7 @@ dependencies = [
"flask~=3.0",
"llama-index==0.9.28",
"python-dotenv==1.0.0",
- "unstract-sdk~=0.11.2",
+ "unstract-sdk~=0.12.0",
]
requires-python = ">=3.9,<3.11.1"
readme = "README.md"
diff --git a/pyproject.toml b/pyproject.toml
index e8c3160f4..ee6a68e15 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -41,7 +41,7 @@ hook-check-django-migrations = [
"psycopg2-binary==2.9.9",
"python-dotenv==1.0.0",
"python-magic==0.4.27",
- "unstract-sdk~=0.11.2",
+ "unstract-sdk~=0.12.0",
"unstract-adapters~=0.2.2",
"-e unstract-connectors @ file:///${PROJECT_ROOT}/unstract/connectors",
"-e unstract-core @ file:///${PROJECT_ROOT}/unstract/core",