LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
Updated sdk version and resolved review comments
Deepaka41f5f8b02cd
diff --git a/backend/file_management/file_management_helper.py b/backend/file_management/file_management_helper.pyindex c4c0160f2..a6b9b600d 100644--- a/backend/file_management/file_management_helper.py+++ b/backend/file_management/file_management_helper.py@@ -195,8 +195,8 @@ class FileManagerHelper:elif file_content_type == "text/plain":with fs.open(file_path, "r") as file:FileManagerHelper.logger.info(f"Reading text file: {file_path}")- encoded_string = file.read()- return encoded_string+ text_content = file.read()+ return text_contentelse:raise InvalidFileTypediff --git a/backend/pdm.lock b/backend/pdm.lockindex da10551b7..7fce08129 100644--- a/backend/pdm.lock+++ b/backend/pdm.lock@@ -5,7 +5,7 @@groups = ["default", "test", "deploy"]strategy = ["cross_platform", "inherit_metadata"]lock_version = "4.4.1"-content_hash = "sha256:73a791002f5f7bab1afe9eaee07e8fc946774da3448f9daabedbccfc9be4f202"+content_hash = "sha256:51aa34c4f085355e4448d1cd7c60c9b1a703e12d932c36b70a2fd5c960b340c3"[[package]]name = "aiobotocore"@@ -4667,7 +4667,7 @@ dependencies = [[[package]]name = "unstract-sdk"-version = "0.11.2"+version = "0.12.0"requires_python = "<3.11.1,>=3.9"summary = "A framework for writing Unstract Tools/Apps"groups = ["default"]@@ -4684,8 +4684,8 @@ dependencies = ["unstract-adapters~=0.2.2",]files = [- {file = "unstract_sdk-0.11.2-py3-none-any.whl", hash = "sha256:bcab064d87dc2985e5a4340e438a8ef71ef165b3e5078aba0ed2da3bda77f9e0"},- {file = "unstract_sdk-0.11.2.tar.gz", hash = "sha256:0d6e3e00c9bbc6e569cc79922a2a45a54870c482cfa61287ea89a22ebd7fc606"},+ {file = "unstract_sdk-0.12.0-py3-none-any.whl", hash = "sha256:3e7b57a94df8cbdebe13998fface9eaf4b9eba667cd9f323d555b785599b51a6"},+ {file = "unstract_sdk-0.12.0.tar.gz", hash = "sha256:0208a653a59600fcbfaa9606341a11bb7e4381f64435754389fc46b735684395"},][[package]]@@ -4700,7 +4700,7 @@ dependencies = ["docker~=6.1.3","jsonschema~=4.18.2","unstract-adapters~=0.2.1",- "unstract-tool-sandbox @ file:///home/gayathrivijayakumar/code/Unstract/unstract/unstract/tool-registry/../tool-sandbox",+ "unstract-tool-sandbox @ file:///home/ghost/Documents/zipstack/unstract-oss/unstract/tool-registry/../tool-sandbox",][[package]]diff --git a/backend/prompt_studio/prompt_studio_core/models.py b/backend/prompt_studio/prompt_studio_core/models.pyindex 4a2327e7e..82d41948f 100644--- a/backend/prompt_studio/prompt_studio_core/models.py+++ b/backend/prompt_studio/prompt_studio_core/models.py@@ -44,6 +44,7 @@ class CustomTool(BaseModel):related_name="default_profile",null=True,blank=True,+ db_comment="Default LLM Profile used in prompt",)summarize_llm_profile = models.ForeignKey(ProfileManager,@@ -51,9 +52,14 @@ class CustomTool(BaseModel):related_name="summarize_llm_profile",null=True,blank=True,+ db_comment="LLM Profile used for summarize",+ )+ summarize_context = models.BooleanField(+ default=True, db_comment="Flag to summarize content"+ )+ summarize_as_source = models.BooleanField(+ default=True, db_comment="Flag to use summarized content as source")- summarize_context = models.BooleanField(default=True)- summarize_as_source = models.BooleanField(default=True)summarize_prompt = models.TextField(blank=True,db_comment="Field to store the summarize prompt",diff --git a/backend/prompt_studio/prompt_studio_core/prompt_studio_helper.py b/backend/prompt_studio/prompt_studio_core/prompt_studio_helper.pyindex 191618cb5..3b7dced4a 100644--- a/backend/prompt_studio/prompt_studio_core/prompt_studio_helper.py+++ b/backend/prompt_studio/prompt_studio_core/prompt_studio_helper.py@@ -81,30 +81,37 @@ class PromptStudioHelper:@staticmethoddef index_document(- tool_id: str, file_name: str, org_id: str, user_id: str, is_summary: bool = False+ tool_id: str,+ file_name: str,+ org_id: str,+ user_id: str,+ is_summary: bool = False,) -> Any:"""Method to index a document.Args:- tool_id (str):Id of the tool+ tool_id (str): Id of the toolfile_name (str): File to parse+ org_id (str): The ID of the organization to which the user belongs.+ user_id (str): The ID of the user who uploaded the document.+ is_summary (bool, optional): Whether the document is a summary+ or not. Defaults to False.Raises:ToolNotValidIndexingError"""tool: CustomTool = CustomTool.objects.get(pk=tool_id)-if is_summary:- default_profile: ProfileManager = tool.summarize_llm_profile+ default_profile = tool.summarize_llm_profilefile_path = file_nameelse:- default_profile: ProfileManager = tool.default_profile+ default_profile = tool.default_profilefile_path = FileManagerHelper.handle_sub_directory_for_tenants(org_id, is_create=False, user_id=user_id, tool_id=tool_id)file_path = str(Path(file_path) / file_name)-+if not default_profile:raise DefaultProfileError()stream_log.publish(@@ -185,16 +192,15 @@ class PromptStudioHelper:prompts: list[ToolStudioPrompt] = []prompts.append(prompt_instance)tool: CustomTool = prompt_instance.tool_id--+if tool.summarize_as_source:directory, filename = os.path.split(file_path)- file_path: str = os.path.join(+ file_path = os.path.join(directory,TSPKeys.SUMMARIZE,os.path.splitext(filename)[0] + ".txt",)-+stream_log.publish(tool.tool_id,stream_log.log(@@ -290,7 +296,7 @@ class PromptStudioHelper:file_name=path,tool_id=str(tool.tool_id),org_id=org_id,- is_summary=tool.summarize_as_source+ is_summary=tool.summarize_as_source,)output: dict[str, Any] = {}diff --git a/backend/pyproject.toml b/backend/pyproject.tomlindex 9494a1fda..3a1e962ff 100644--- a/backend/pyproject.toml+++ b/backend/pyproject.toml@@ -30,7 +30,7 @@ dependencies = ["python-socketio==5.9.0", # For log_events"social-auth-app-django==5.3.0", # For OAuth"social-auth-core==4.4.2", # For OAuth- "unstract-sdk~=0.11.2",+ "unstract-sdk~=0.12.0","unstract-adapters~=0.2.2",# ! IMPORTANT!# Indirect local dependencies usually need to be added in their own projectsdiff --git a/pdm.lock b/pdm.lockindex a617b0be0..e071baeb4 100644--- a/pdm.lock+++ b/pdm.lock@@ -5,7 +5,7 @@groups = ["default", "hook-check-django-migrations", "lint"]strategy = ["cross_platform", "inherit_metadata"]lock_version = "4.4.1"-content_hash = "sha256:34bd617283cb8620b48d2ff82e83a4721d9464962bb6ccebd4d9f860efaed221"+content_hash = "sha256:32fcfebc305b03ed156f6a208a4edcf40cd407f555fb695d3cb932aaa43ece4f"[[package]]name = "absolufy-imports"@@ -4836,7 +4836,7 @@ dependencies = [[[package]]name = "unstract-sdk"-version = "0.11.2"+version = "0.12.0"requires_python = "<3.11.1,>=3.9"summary = "A framework for writing Unstract Tools/Apps"groups = ["hook-check-django-migrations"]@@ -4853,8 +4853,8 @@ dependencies = ["unstract-adapters~=0.2.2",]files = [- {file = "unstract_sdk-0.11.2-py3-none-any.whl", hash = "sha256:bcab064d87dc2985e5a4340e438a8ef71ef165b3e5078aba0ed2da3bda77f9e0"},- {file = "unstract_sdk-0.11.2.tar.gz", hash = "sha256:0d6e3e00c9bbc6e569cc79922a2a45a54870c482cfa61287ea89a22ebd7fc606"},+ {file = "unstract_sdk-0.12.0-py3-none-any.whl", hash = "sha256:3e7b57a94df8cbdebe13998fface9eaf4b9eba667cd9f323d555b785599b51a6"},+ {file = "unstract_sdk-0.12.0.tar.gz", hash = "sha256:0208a653a59600fcbfaa9606341a11bb7e4381f64435754389fc46b735684395"},][[package]]diff --git a/prompt-service/pdm.lock b/prompt-service/pdm.lockindex 82dec09a0..652afff29 100644--- a/prompt-service/pdm.lock+++ b/prompt-service/pdm.lock@@ -5,7 +5,7 @@groups = ["default", "deploy"]strategy = ["cross_platform", "inherit_metadata"]lock_version = "4.4.1"-content_hash = "sha256:c02615c1fe1a21cddaa8ee726e7d3166446a694f45debed47272c2d8ee1504e0"+content_hash = "sha256:64a1667d589e799eb7d6cbcb67e2bc4adcfdfdc8335f21ae6168562e3db368e5"[[package]]name = "aiohttp"@@ -3677,7 +3677,7 @@ files = [[[package]]name = "unstract-sdk"-version = "0.11.2"+version = "0.12.0"requires_python = "<3.11.1,>=3.9"summary = "A framework for writing Unstract Tools/Apps"groups = ["default"]@@ -3694,8 +3694,8 @@ dependencies = ["unstract-adapters~=0.2.2",]files = [- {file = "unstract_sdk-0.11.2-py3-none-any.whl", hash = "sha256:bcab064d87dc2985e5a4340e438a8ef71ef165b3e5078aba0ed2da3bda77f9e0"},- {file = "unstract_sdk-0.11.2.tar.gz", hash = "sha256:0d6e3e00c9bbc6e569cc79922a2a45a54870c482cfa61287ea89a22ebd7fc606"},+ {file = "unstract_sdk-0.12.0-py3-none-any.whl", hash = "sha256:3e7b57a94df8cbdebe13998fface9eaf4b9eba667cd9f323d555b785599b51a6"},+ {file = "unstract_sdk-0.12.0.tar.gz", hash = "sha256:0208a653a59600fcbfaa9606341a11bb7e4381f64435754389fc46b735684395"},][[package]]diff --git a/prompt-service/pyproject.toml b/prompt-service/pyproject.tomlindex ae624cbee..4cc1facce 100644--- a/prompt-service/pyproject.toml+++ b/prompt-service/pyproject.toml@@ -17,7 +17,7 @@ dependencies = ["flask~=3.0","llama-index==0.9.28","python-dotenv==1.0.0",- "unstract-sdk~=0.11.2",+ "unstract-sdk~=0.12.0",]requires-python = ">=3.9,<3.11.1"readme = "README.md"diff --git a/pyproject.toml b/pyproject.tomlindex e8c3160f4..ee6a68e15 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -41,7 +41,7 @@ hook-check-django-migrations = ["psycopg2-binary==2.9.9","python-dotenv==1.0.0","python-magic==0.4.27",- "unstract-sdk~=0.11.2",+ "unstract-sdk~=0.12.0","unstract-adapters~=0.2.2","-e unstract-connectors @ file:///${PROJECT_ROOT}/unstract/connectors","-e unstract-core @ file:///${PROJECT_ROOT}/unstract/core",