LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
feat: Cleaned up code in prompt service and prompt studio (#151)
Chandrasekharan Mcd5743177e74
diff --git a/backend/prompt_studio/prompt_profile_manager/migrations/0010_alter_profilemanager_retrieval_strategy_and_more.py b/backend/prompt_studio/prompt_profile_manager/migrations/0010_alter_profilemanager_retrieval_strategy_and_more.pynew file mode 100644index 000000000..301d17ffe--- /dev/null+++ b/backend/prompt_studio/prompt_profile_manager/migrations/0010_alter_profilemanager_retrieval_strategy_and_more.py@@ -0,0 +1,33 @@+# Generated by Django 4.2.1 on 2024-03-21 08:36++from django.db import migrations, models+++class Migration(migrations.Migration):+ dependencies = [+ (+ "prompt_profile_manager",+ "0009_alter_profilemanager_prompt_studio_tool",+ ),+ ]++ operations = [+ migrations.AlterField(+ model_name="profilemanager",+ name="retrieval_strategy",+ field=models.TextField(+ blank=True,+ choices=[("simple", "Simple retrieval")],+ db_comment="Field to store the retrieval strategy for prompts",+ ),+ ),+ migrations.AlterField(+ model_name="profilemanager",+ name="similarity_top_k",+ field=models.IntegerField(+ blank=True,+ db_comment="Field to store number of top embeddings to take into context",+ null=True,+ ),+ ),+ ]diff --git a/backend/prompt_studio/prompt_profile_manager/models.py b/backend/prompt_studio/prompt_profile_manager/models.pyindex 529f50a76..adfffe2f1 100644--- a/backend/prompt_studio/prompt_profile_manager/models.py+++ b/backend/prompt_studio/prompt_profile_manager/models.py@@ -3,6 +3,7 @@ import uuidfrom account.models import Userfrom adapter_processor.models import AdapterInstancefrom django.db import models+from prompt_studio.prompt_studio_core.exceptions import DefaultProfileErrorfrom prompt_studio.prompt_studio_core.models import CustomToolfrom utils.models.base_model import BaseModel@@ -12,8 +13,6 @@ class ProfileManager(BaseModel):class RetrievalStrategy(models.TextChoices):SIMPLE = "simple", "Simple retrieval"- SUBQUESTION = "subquestion", "Subquestion from prompt"- VECTOR = "vector+keyword", "Uses vector for retrieval"profile_id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False@@ -59,7 +58,9 @@ class ProfileManager(BaseModel):db_comment="Field to store the retrieval strategy for prompts",)similarity_top_k = models.IntegerField(- blank=True, null=True, db_comment="Field to store matching count"+ blank=True,+ null=True,+ db_comment="Field to store number of top embeddings to take into context", # noqa: E501)section = models.TextField(blank=True, null=True, db_comment="Field to store limit to section"@@ -101,3 +102,12 @@ class ProfileManager(BaseModel):name="unique_prompt_studio_tool_profile_name",),]++ @staticmethod+ def get_default_llm_profile(tool: CustomTool) -> "ProfileManager":+ try:+ return ProfileManager.objects.get( # type: ignore+ prompt_studio_tool=tool, is_default=True+ )+ except ProfileManager.DoesNotExist:+ raise DefaultProfileErrordiff --git a/backend/prompt_studio/prompt_studio_core/constants.py b/backend/prompt_studio/prompt_studio_core/constants.pyindex a40ce2fac..0bb31987f 100644--- a/backend/prompt_studio/prompt_studio_core/constants.py+++ b/backend/prompt_studio/prompt_studio_core/constants.py@@ -45,8 +45,6 @@ class ToolStudioPromptKeys:ASSERTION_FAILURE_PROMPT = "assertion_failure_prompt"RETRIEVAL_STRATEGY = "retrieval-strategy"SIMPLE = "simple"- VECTOR_KEYWORD = "vector+keyword"- SUBQUESTION = "subquestion"TYPE = "type"NUMBER = "number"EMAIL = "email"diff --git a/backend/prompt_studio/prompt_studio_core/models.py b/backend/prompt_studio/prompt_studio_core/models.pyindex c3677dbb7..291ff9c73 100644--- a/backend/prompt_studio/prompt_studio_core/models.py+++ b/backend/prompt_studio/prompt_studio_core/models.py@@ -2,9 +2,7 @@ import uuidfrom account.models import Userfrom adapter_processor.models import AdapterInstance-from django.core.exceptions import ObjectDoesNotExistfrom django.db import models-from prompt_studio.prompt_studio_core.exceptions import DefaultProfileErrorfrom utils.models.base_model import BaseModel@@ -79,10 +77,3 @@ class CustomTool(BaseModel):editable=False,)exclude_failed = models.BooleanField(default=True)-- # TODO: Add ProfileManager to return type- def get_default_llm_profile(self): # type: ignore- try:- return self.profilemanager_set.filter(is_default=True).first()- except ObjectDoesNotExist:- raise DefaultProfileErrordiff --git a/backend/prompt_studio/prompt_studio_core/prompt_studio_helper.py b/backend/prompt_studio/prompt_studio_core/prompt_studio_helper.pyindex 0a16f6feb..6b5a41bfd 100644--- a/backend/prompt_studio/prompt_studio_core/prompt_studio_helper.py+++ b/backend/prompt_studio/prompt_studio_core/prompt_studio_helper.py@@ -135,6 +135,7 @@ class PromptStudioHelper:raise PermissionError(error_msg)+ @staticmethoddef _publish_log(component: dict[str, str], level: str, state: str, message: str) -> None:@@ -179,9 +180,9 @@ class PromptStudioHelper:Returns:List[ToolStudioPrompt]: List of instance of the model"""- prompt_instances: list[- ToolStudioPrompt- ] = ToolStudioPrompt.objects.filter(tool_id=tool_id)+ prompt_instances: list[ToolStudioPrompt] = (+ ToolStudioPrompt.objects.filter(tool_id=tool_id)+ )return prompt_instances@staticmethod@@ -215,10 +216,7 @@ class PromptStudioHelper:default_profile = profile_managerfile_path = file_nameelse:- profile_manager = ProfileManager.objects.get(- prompt_studio_tool=tool, is_default=True- )- default_profile = profile_manager+ default_profile = ProfileManager.get_default_llm_profile(tool)file_path = FileManagerHelper.handle_sub_directory_for_tenants(org_id, is_create=False, user_id=user_id, tool_id=tool_id)@@ -442,14 +440,11 @@ class PromptStudioHelper:grammar_list = []# Using default profile manager llm if monitor_llm is None- if monitor_llm:+ if monitor_llm_instance:monitor_llm = str(monitor_llm_instance.id)else:- # TODO: Use CustomTool model to get profile_manager- profile_manager = ProfileManager.objects.get(- prompt_studio_tool=tool, is_default=True- )- monitor_llm = str(profile_manager.llm.id)+ default_profile = ProfileManager.get_default_llm_profile(tool)+ monitor_llm = str(default_profile.llm.id)# Adding validationsif prompt_grammer:@@ -484,9 +479,9 @@ class PromptStudioHelper:)output: dict[str, Any] = {}- output[- TSPKeys.ASSERTION_FAILURE_PROMPT- ] = prompt.assertion_failure_prompt+ output[TSPKeys.ASSERTION_FAILURE_PROMPT] = (+ prompt.assertion_failure_prompt+ )output[TSPKeys.ASSERT_PROMPT] = prompt.assert_promptoutput[TSPKeys.IS_ASSERT] = prompt.is_assertoutput[TSPKeys.PROMPT] = prompt.prompt@@ -501,12 +496,12 @@ class PromptStudioHelper:output[TSPKeys.GRAMMAR] = grammar_listoutput[TSPKeys.TYPE] = prompt.enforce_typeoutput[TSPKeys.NAME] = prompt.prompt_key- output[- TSPKeys.RETRIEVAL_STRATEGY- ] = prompt.profile_manager.retrieval_strategy- output[- TSPKeys.SIMILARITY_TOP_K- ] = prompt.profile_manager.similarity_top_k+ output[TSPKeys.RETRIEVAL_STRATEGY] = (+ prompt.profile_manager.retrieval_strategy+ )+ output[TSPKeys.SIMILARITY_TOP_K] = (+ prompt.profile_manager.similarity_top_k+ )output[TSPKeys.SECTION] = prompt.profile_manager.sectionoutput[TSPKeys.X2TEXT_ADAPTER] = x2text@@ -635,9 +630,7 @@ class PromptStudioHelper:outputs: list[dict[str, Any]] = []grammar: list[dict[str, Any]] = []prompt_grammar = tool.prompt_grammer- default_profile: ProfileManager = ProfileManager.objects.get(- prompt_studio_tool=tool, is_default=True- )+ default_profile = ProfileManager.get_default_llm_profile(tool)# Need to check the user who created profile manager# has access to adapters configured in profile managerPromptStudioHelper.validate_profile_manager_owner_access(@@ -674,9 +667,9 @@ class PromptStudioHelper:llm_profile_manager[TSPKeys.VECTOR_DB] = vector_dbllm_profile_manager[TSPKeys.EMBEDDING] = embedding_modelllm_profile_manager[TSPKeys.CHUNK_SIZE] = default_profile.chunk_size- llm_profile_manager[- TSPKeys.CHUNK_OVERLAP- ] = default_profile.chunk_overlap+ llm_profile_manager[TSPKeys.CHUNK_OVERLAP] = (+ default_profile.chunk_overlap+ )for prompt in prompts:output: dict[str, Any] = {}diff --git a/backend/prompt_studio/prompt_studio_core/serializers.py b/backend/prompt_studio/prompt_studio_core/serializers.pyindex 32ef5624e..bf232951f 100644--- a/backend/prompt_studio/prompt_studio_core/serializers.py+++ b/backend/prompt_studio/prompt_studio_core/serializers.py@@ -33,9 +33,7 @@ class CustomToolSerializer(AuditSerializer):str(instance.tool_id),)try:- profile_manager = ProfileManager.objects.get(- prompt_studio_tool=instance, is_default=True- )+ profile_manager = ProfileManager.get_default_llm_profile(instance)data[TSKeys.DEFAULT_PROFILE] = profile_manager.profile_idexcept ObjectDoesNotExist:logger.info(@@ -57,9 +55,9 @@ class CustomToolSerializer(AuditSerializer):except Exception as e:logger.error(f"Error occured while appending prompts {e}")return data-+data["created_by_email"] = instance.created_by.email-+return datadiff --git a/backend/prompt_studio/prompt_studio_core/static/select_choices.json b/backend/prompt_studio/prompt_studio_core/static/select_choices.jsonindex 1e3746674..80acabee4 100644--- a/backend/prompt_studio/prompt_studio_core/static/select_choices.json+++ b/backend/prompt_studio/prompt_studio_core/static/select_choices.json@@ -23,9 +23,7 @@"openai_embedding":"openai_embedding"},"retrieval_strategy":{- "simple":"simple",- "subquestion":"subquestion",- "vector+keyword":"vector+keyword"+ "simple":"simple"},"vector_store":{"Postgres pg_vector":"Postgres pg_vector",diff --git a/backend/prompt_studio/prompt_studio_registry/constants.py b/backend/prompt_studio/prompt_studio_registry/constants.pyindex f33deb303..73eee1e9f 100644--- a/backend/prompt_studio/prompt_studio_registry/constants.py+++ b/backend/prompt_studio/prompt_studio_registry/constants.py@@ -62,8 +62,6 @@ class JsonSchemaKey:ASSERTION_FAILURE_PROMPT = "assertion_failure_prompt"RETRIEVAL_STRATEGY = "retrieval-strategy"SIMPLE = "simple"- VECTOR_KEYWORD = "vector+keyword"- SUBQUESTION = "subquestion"TYPE = "type"NUMBER = "number"EMAIL = "email"diff --git a/backend/prompt_studio/prompt_studio_registry/exceptions.py b/backend/prompt_studio/prompt_studio_registry/exceptions.pyindex b46dad0eb..038878cb2 100644--- a/backend/prompt_studio/prompt_studio_registry/exceptions.py+++ b/backend/prompt_studio/prompt_studio_registry/exceptions.py@@ -14,19 +14,3 @@ class ToolDoesNotExist(APIException):class ToolSaveError(APIException):status_code = 500default_detail = "Error while saving the tool."---class MandatoryFieldMissingError(APIException):- status_code = 400- default_detail = "Mandatory field missing."---class ProfileErrors(APIException):- status_code = 400- default_detail = f"""Looks like some default values are- not selected. Please check profile managers."""---class DuplicateData(APIException):- status_code = 400- default_detail = "Duplicate Data"diff --git a/backend/prompt_studio/prompt_studio_registry/prompt_studio_registry_helper.py b/backend/prompt_studio/prompt_studio_registry/prompt_studio_registry_helper.pyindex 384573f6f..fd33d8da9 100644--- a/backend/prompt_studio/prompt_studio_registry/prompt_studio_registry_helper.py+++ b/backend/prompt_studio/prompt_studio_registry/prompt_studio_registry_helper.py@@ -4,6 +4,7 @@ from typing import Any, Optionalfrom account.models import Userfrom django.conf import settingsfrom django.db import IntegrityError+from prompt_studio.prompt_profile_manager.models import ProfileManagerfrom prompt_studio.prompt_studio.models import ToolStudioPromptfrom prompt_studio.prompt_studio_core.models import CustomToolfrom prompt_studio.prompt_studio_core.prompt_studio_helper import (@@ -114,10 +115,10 @@ class PromptStudioRegistryHelper:PromptStudioRegistryHelper.frame_properties(tool=custom_tool))spec: Spec = PromptStudioRegistryHelper.frame_spec(tool=custom_tool)- prompts: list[- ToolStudioPrompt- ] = PromptStudioHelper.fetch_prompt_from_tool(- tool_id=custom_tool.tool_id+ prompts: list[ToolStudioPrompt] = (+ PromptStudioHelper.fetch_prompt_from_tool(+ tool_id=custom_tool.tool_id+ ))metadata = PromptStudioRegistryHelper.frame_export_json(tool=custom_tool, prompts=prompts@@ -180,7 +181,7 @@ class PromptStudioRegistryHelper:llm = ""embedding_model = ""- default_llm_profile = tool.get_default_llm_profile() # type: ignore+ default_llm_profile = ProfileManager.get_default_llm_profile(tool)for prompt in prompts:if not prompt.profile_manager:prompt.profile_manager = default_llm_profile@@ -192,9 +193,9 @@ class PromptStudioRegistryHelper:adapter_id = str(prompt.profile_manager.embedding_model.adapter_id)embedding_suffix = adapter_id.split("|")[0]- output[- JsonSchemaKey.ASSERTION_FAILURE_PROMPT- ] = prompt.assertion_failure_prompt+ output[JsonSchemaKey.ASSERTION_FAILURE_PROMPT] = (+ prompt.assertion_failure_prompt+ )output[JsonSchemaKey.ASSERT_PROMPT] = prompt.assert_promptoutput[JsonSchemaKey.IS_ASSERT] = prompt.is_assertoutput[JsonSchemaKey.PROMPT] = prompt.prompt@@ -203,21 +204,21 @@ class PromptStudioRegistryHelper:output[JsonSchemaKey.VECTOR_DB] = vector_dboutput[JsonSchemaKey.EMBEDDING] = embedding_modeloutput[JsonSchemaKey.X2TEXT_ADAPTER] = x2text- output[- JsonSchemaKey.CHUNK_OVERLAP- ] = prompt.profile_manager.chunk_overlap+ output[JsonSchemaKey.CHUNK_OVERLAP] = (+ prompt.profile_manager.chunk_overlap+ )output[JsonSchemaKey.LLM] = llmoutput[JsonSchemaKey.PREAMBLE] = tool.preambleoutput[JsonSchemaKey.POSTAMBLE] = tool.postambleoutput[JsonSchemaKey.GRAMMAR] = grammar_listoutput[JsonSchemaKey.TYPE] = prompt.enforce_typeoutput[JsonSchemaKey.NAME] = prompt.prompt_key- output[- JsonSchemaKey.RETRIEVAL_STRATEGY- ] = prompt.profile_manager.retrieval_strategy- output[- JsonSchemaKey.SIMILARITY_TOP_K- ] = prompt.profile_manager.similarity_top_k+ output[JsonSchemaKey.RETRIEVAL_STRATEGY] = (+ prompt.profile_manager.retrieval_strategy+ )+ output[JsonSchemaKey.SIMILARITY_TOP_K] = (+ prompt.profile_manager.similarity_top_k+ )output[JsonSchemaKey.SECTION] = prompt.profile_manager.sectionoutput[JsonSchemaKey.REINDEX] = prompt.profile_manager.reindexoutput[JsonSchemaKey.EMBEDDING_SUFFIX] = embedding_suffixdiff --git a/backend/prompt_studio/prompt_studio_registry/static/select_choices.json b/backend/prompt_studio/prompt_studio_registry/static/select_choices.jsondeleted file mode 100644index e2217bef7..000000000--- a/backend/prompt_studio/prompt_studio_registry/static/select_choices.json+++ /dev/null@@ -1,31 +0,0 @@-{- "combined_output":- {- "JSON":"JSON",- "YAML":"YAML"- },- "choose_llm":{- "AZURE":"Azure OpenAI"- },- "output_type":{- "string":"Text",- "float":"Number",- "int":"Integer"- },- "output_processing":{- "DEFAULT":"Default"- },- "embedding":{- "azure_openai_embedding":"azure_openai_embedding",- "openai_embedding":"openai_embedding"- },- "retrieval_strategy":{- "simple":"simple",- "subquestion":"subquestion",- "vector_keyword":"vector_keyword"- },- "vector_store":{- "Postgres pg_vector":"Postgres pg_vector",- "qdrant":"qdrant"- }-}diff --git a/backend/tenant_account/serializer.py b/backend/tenant_account/serializer.pyindex 305efa749..7983ea5f1 100644--- a/backend/tenant_account/serializer.py+++ b/backend/tenant_account/serializer.py@@ -91,7 +91,7 @@ class InviteUserSerializer(serializers.Serializer):def get_users(self, validated_data: dict[str, Any]) -> list[dict[str, Union[str, None]]]:- return validated_data.get("users", [])+ return validated_data.get("users", []) # type: ignoreclass RemoveUserFromOrganizationSerializer(serializers.Serializer):diff --git a/backend/tool_instance/tool_instance_helper.py b/backend/tool_instance/tool_instance_helper.pyindex fd3ce63b5..bfb804d19 100644--- a/backend/tool_instance/tool_instance_helper.py+++ b/backend/tool_instance/tool_instance_helper.py@@ -413,7 +413,7 @@ class ToolInstanceHelper:@staticmethoddef validate_adapter_access(user: User,- adapter_ids: list[str],+ adapter_ids: set[str],) -> None:adapter_instances = AdapterInstance.objects.filter(id__in=adapter_idsdiff --git a/pdm.lock b/pdm.lockindex 99822b962..449778736 100644--- a/pdm.lock+++ b/pdm.lock@@ -5,7 +5,7 @@groups = ["default", "hook-check-django-migrations", "lint"]strategy = ["cross_platform", "inherit_metadata"]lock_version = "4.4.1"-content_hash = "sha256:6e3e1de75a1e85f4e17024312e238e0ebb40b2e8e1b9c51f1a94e99b5634b27a"+content_hash = "sha256:ce8dc58a791735289e20a118db667c687497a171f3001d736c3f7a13e744f397"[[package]]name = "absolufy-imports"@@ -2535,33 +2535,33 @@ files = [[[package]]name = "mypy"-version = "1.2.0"-requires_python = ">=3.7"+version = "1.9.0"+requires_python = ">=3.8"summary = "Optional static typing for Python"groups = ["lint"]dependencies = ["mypy-extensions>=1.0.0","tomli>=1.1.0; python_version < \"3.11\"",- "typing-extensions>=3.10",-]-files = [- {file = "mypy-1.2.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:701189408b460a2ff42b984e6bd45c3f41f0ac9f5f58b8873bbedc511900086d"},- {file = "mypy-1.2.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fe91be1c51c90e2afe6827601ca14353bbf3953f343c2129fa1e247d55fd95ba"},- {file = "mypy-1.2.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8d26b513225ffd3eacece727f4387bdce6469192ef029ca9dd469940158bc89e"},- {file = "mypy-1.2.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:3a2d219775a120581a0ae8ca392b31f238d452729adbcb6892fa89688cb8306a"},- {file = "mypy-1.2.0-cp310-cp310-win_amd64.whl", hash = "sha256:2e93a8a553e0394b26c4ca683923b85a69f7ccdc0139e6acd1354cc884fe0128"},- {file = "mypy-1.2.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:3efde4af6f2d3ccf58ae825495dbb8d74abd6d176ee686ce2ab19bd025273f41"},- {file = "mypy-1.2.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:695c45cea7e8abb6f088a34a6034b1d273122e5530aeebb9c09626cea6dca4cb"},- {file = "mypy-1.2.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d0e9464a0af6715852267bf29c9553e4555b61f5904a4fc538547a4d67617937"},- {file = "mypy-1.2.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:8293a216e902ac12779eb7a08f2bc39ec6c878d7c6025aa59464e0c4c16f7eb9"},- {file = "mypy-1.2.0-cp311-cp311-win_amd64.whl", hash = "sha256:f46af8d162f3d470d8ffc997aaf7a269996d205f9d746124a179d3abe05ac602"},- {file = "mypy-1.2.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:70894c5345bea98321a2fe84df35f43ee7bb0feec117a71420c60459fc3e1eed"},- {file = "mypy-1.2.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:4a99fe1768925e4a139aace8f3fb66db3576ee1c30b9c0f70f744ead7e329c9f"},- {file = "mypy-1.2.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:023fe9e618182ca6317ae89833ba422c411469156b690fde6a315ad10695a521"},- {file = "mypy-1.2.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:4d19f1a239d59f10fdc31263d48b7937c585810288376671eaf75380b074f238"},- {file = "mypy-1.2.0-cp39-cp39-win_amd64.whl", hash = "sha256:2de7babe398cb7a85ac7f1fd5c42f396c215ab3eff731b4d761d68d0f6a80f48"},- {file = "mypy-1.2.0-py3-none-any.whl", hash = "sha256:d8e9187bfcd5ffedbe87403195e1fc340189a68463903c39e2b63307c9fa0394"},- {file = "mypy-1.2.0.tar.gz", hash = "sha256:f70a40410d774ae23fcb4afbbeca652905a04de7948eaf0b1789c8d1426b72d1"},+ "typing-extensions>=4.1.0",+]+files = [+ {file = "mypy-1.9.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:f8a67616990062232ee4c3952f41c779afac41405806042a8126fe96e098419f"},+ {file = "mypy-1.9.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d357423fa57a489e8c47b7c85dfb96698caba13d66e086b412298a1a0ea3b0ed"},+ {file = "mypy-1.9.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:49c87c15aed320de9b438ae7b00c1ac91cd393c1b854c2ce538e2a72d55df150"},+ {file = "mypy-1.9.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:48533cdd345c3c2e5ef48ba3b0d3880b257b423e7995dada04248725c6f77374"},+ {file = "mypy-1.9.0-cp310-cp310-win_amd64.whl", hash = "sha256:4d3dbd346cfec7cb98e6cbb6e0f3c23618af826316188d587d1c1bc34f0ede03"},+ {file = "mypy-1.9.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:653265f9a2784db65bfca694d1edd23093ce49740b2244cde583aeb134c008f3"},+ {file = "mypy-1.9.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:3a3c007ff3ee90f69cf0a15cbcdf0995749569b86b6d2f327af01fd1b8aee9dc"},+ {file = "mypy-1.9.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2418488264eb41f69cc64a69a745fad4a8f86649af4b1041a4c64ee61fc61129"},+ {file = "mypy-1.9.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:68edad3dc7d70f2f17ae4c6c1b9471a56138ca22722487eebacfd1eb5321d612"},+ {file = "mypy-1.9.0-cp311-cp311-win_amd64.whl", hash = "sha256:85ca5fcc24f0b4aeedc1d02f93707bccc04733f21d41c88334c5482219b1ccb3"},+ {file = "mypy-1.9.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:587ce887f75dd9700252a3abbc9c97bbe165a4a630597845c61279cf32dfbf02"},+ {file = "mypy-1.9.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:f88566144752999351725ac623471661c9d1cd8caa0134ff98cceeea181789f4"},+ {file = "mypy-1.9.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:61758fabd58ce4b0720ae1e2fea5cfd4431591d6d590b197775329264f86311d"},+ {file = "mypy-1.9.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:e49499be624dead83927e70c756970a0bc8240e9f769389cdf5714b0784ca6bf"},+ {file = "mypy-1.9.0-cp39-cp39-win_amd64.whl", hash = "sha256:571741dc4194b4f82d344b15e8837e8c5fcc462d66d076748142327626a1b6e9"},+ {file = "mypy-1.9.0-py3-none-any.whl", hash = "sha256:a260627a570559181a9ea5de61ac6297aa5af202f06fd7ab093ce74e7181e43e"},+ {file = "mypy-1.9.0.tar.gz", hash = "sha256:3cc5da0127e6a478cddd906068496a97a7618a21ce9b54bde5bf7e539c7af974"},][[package]]diff --git a/prompt-service/src/unstract/prompt_service/constants.py b/prompt-service/src/unstract/prompt_service/constants.pyindex c115a04a5..5afe5a9bc 100644--- a/prompt-service/src/unstract/prompt_service/constants.py+++ b/prompt-service/src/unstract/prompt_service/constants.py@@ -2,48 +2,6 @@ from enum import Enumclass PromptServiceContants:- HELP = "help"- disallowed_words = [- "which",- "what",- "how",- "when",- "where",- "who",- "why",- "is",- "are",- "was",- "were",- "do",- "does",- "seem",- "have",- "has",- "had",- "can",- "could",- "may",- "might",- "will",- "would",- "should",- "must",- "shall",- "did",- "would",- "be",- "many",- "being",- "(",- ")",- ",",- ]- AND = "and"- TO = "to"- OR = "or"- IS = "is"- DOC_ID = "doc_id"WORD = "word"SYNONYMS = "synonyms"OUTPUTS = "outputs"@@ -64,8 +22,6 @@ class PromptServiceContants:ASSERTION_FAILURE_PROMPT = "assertion_failure_prompt"RETRIEVAL_STRATEGY = "retrieval-strategy"SIMPLE = "simple"- VECTOR_KEYWORD = "vector+keyword"- SUBQUESTION = "subquestion"TYPE = "type"NUMBER = "number"EMAIL = "email"@@ -90,19 +46,6 @@ class PromptServiceContants:LOG_EVENTS_ID = "log_events_id"-class Query:- QUERY = "Query"- DROP_TABLE = "DROP TABLE IF EXISTS nodes;"- INSERT_INTO = "INSERT INTO nodes VALUES (?, ?, ?)"- SELECT = "SELECT *,rank FROM nodes WHERE "- NODE_MATCH = " nodes MATCH ?"- ORDER_BY = " ORDER BY RANK LIMIT 2;"---class Prompt:- CONTEXT = "Context"--class LogLevel(Enum):DEBUG = "DEBUG"INFO = "INFO"diff --git a/prompt-service/src/unstract/prompt_service/main.py b/prompt-service/src/unstract/prompt_service/main.pyindex 5ccaed569..72c40057b 100644--- a/prompt-service/src/unstract/prompt_service/main.py+++ b/prompt-service/src/unstract/prompt_service/main.py@@ -1,47 +1,24 @@import jsonimport logging-import re-import sqlite3from enum import Enumfrom typing import Any, Optional-import nltkimport peeweefrom flask import Flask, request-from llama_index import (- QueryBundle,- ServiceContext,- VectorStoreIndex,- get_response_synthesizer,-)-from llama_index.indices.base_retriever import BaseRetriever+from llama_index import VectorStoreIndexfrom llama_index.llms import LLM-from llama_index.query_engine import (- RetrieverQueryEngine,- SubQuestionQueryEngine,-)-from llama_index.schema import NodeWithScore-from llama_index.tools import QueryEngineTool, ToolMetadata-from llama_index.vector_stores.types import (- ExactMatchFilter,- MetadataFilters,- VectorStore,- VectorStoreQuery,- VectorStoreQueryResult,-)-from nltk import ngrams+from llama_index.vector_stores.types import ExactMatchFilter, MetadataFiltersfrom unstract.prompt_service.authentication_middleware import (AuthenticationMiddleware,)from unstract.prompt_service.constants import PromptServiceContants as PSKeys-from unstract.prompt_service.constants import Query, RunLevel+from unstract.prompt_service.constants import RunLevelfrom unstract.prompt_service.helper import EnvLoader, plugin_loaderfrom unstract.prompt_service.prompt_ide_base_tool import PromptServiceBaseToolfrom unstract.sdk.constants import LogLevelfrom unstract.sdk.embedding import ToolEmbeddingfrom unstract.sdk.index import ToolIndexfrom unstract.sdk.llm import ToolLLM-from unstract.sdk.tool.base import BaseToolfrom unstract.sdk.utils.service_context import (ServiceContext as UNServiceContext,)@@ -94,207 +71,6 @@ def _publish_log()-def get_keywords_from_pos(text: str) -> list[Any]:- text = text.lower()- keywords = []- sentences = nltk.sent_tokenize(text)-- words_allowed_only_in_middle = [PSKeys.AND, PSKeys.TO, PSKeys.OR, PSKeys.IS]- pos_lookup: dict[str, Any] = {- "NN": [],- "VB": [],- "JJ": [],- "IN": [],- "DT": [],- ".": [],- "X": [],- "PRP": [],- "RB": [],- "EX": [],- "WDT": [],- "WP": [],- "MD": [],- }- for sentence in sentences:- # TODO : Revisit pos.txt -> non generic usecase- with open(POS_TEXT_PATH, "w") as f:- f.write("***********\n")- pos = nltk.pos_tag(nltk.word_tokenize(str(sentence)))-- for word, posx in pos:- if posx.startswith("NN"):- posx = "NN"- if posx.startswith("VB"):- posx = "VB"- if posx.endswith("$"):- posx = "X"- if posx.startswith("RB"):- posx = "RB"- if posx not in pos_lookup:- pos_lookup[posx] = []- pos_lookup[posx].append(word)- # with open("samples/pos.txt", "a") as f:- # f.write(str(pos_lookup) + "\n")- words = nltk.word_tokenize(sentence)- trigrams = list(ngrams(words, 3))- for trigram in trigrams:- allowed = False- override_allowed = False- p = 0- for word in trigram:- if (- word in pos_lookup["NN"]- or word in pos_lookup["VB"]- or word in pos_lookup["JJ"]- ):- allowed = True- if (- word in pos_lookup["IN"]- or word in pos_lookup["DT"]- or word in pos_lookup["."]- or word in pos_lookup["X"]- or word in pos_lookup["PRP"]- or word in pos_lookup["RB"]- or word in pos_lookup["EX"]- or word in pos_lookup["WDT"]- or word in pos_lookup["WP"]- or word in pos_lookup["MD"]- or word in PSKeys.disallowed_words- ):- override_allowed = True- if p == 0 or p == 2:- if word in words_allowed_only_in_middle:- override_allowed = True- p += 1- if allowed and not override_allowed:- keywords.append(" ".join(trigram))- bigrams = list(ngrams(words, 2))- for bigram in bigrams:- allowed = False- override_allowed = False- for word in bigram:- if (- word in pos_lookup["NN"]- or word in pos_lookup["VB"]- or word in pos_lookup["JJ"]- ):- allowed = True- if (- word in pos_lookup["IN"]- or word in pos_lookup["DT"]- or word in pos_lookup["."]- or word in pos_lookup["X"]- or word in pos_lookup["PRP"]- or word in pos_lookup["RB"]- or word in pos_lookup["EX"]- or word in pos_lookup["WDT"]- or word in pos_lookup["WP"]- or word in pos_lookup["MD"]- or word in PSKeys.disallowed_words- ):- override_allowed = True- # In bigrams, these words cannot be in the middle- # so if they are preset, remove the bigram- if word in words_allowed_only_in_middle:- override_allowed = True-- if allowed and not override_allowed:- keywords.append(" ".join(bigram))- with open(POS_TEXT_PATH, "a") as f:- f.write(str(keywords) + "\n")- return keywords---class UnstractRetriever_V_K(BaseRetriever):- def __init__(- self,- index: VectorStoreIndex,- doc_id: str,- vector_db: VectorStore,- collection: str,- service_context: ServiceContext,- tool: BaseTool,- ):- self.index = index- self.db_name = f"/tmp/{doc_id}.db"- self.service_context = service_context- self.doc_id = doc_id- self.collection = collection- self.vector_db = vector_db- self.tool = tool-- def _retrieve(self, query_bundle: QueryBundle) -> list[NodeWithScore]:- print(f"Query: {query_bundle.query_str}")- # vec_retriever = self.index.as_retriever(- # similarity_top_k=2,- # filters=MetadataFilters(- # filters=[- # ExactMatchFilter(key=PSKeys.DOC_ID, value=self.doc_id)- # ],- # ),- # )- keywords = get_keywords_from_pos(query_bundle.query_str)- print(f"Keywords: {keywords}")-- db = sqlite3.connect(self.db_name)- cursor = db.cursor()- cursor.execute(Query.DROP_TABLE)- db.commit()- cursor.execute(- "CREATE VIRTUAL TABLE IF NOT EXISTS nodes "- "USING fts5(doc_id, node_id, text, tokenize='porter unicode61');"- )- db.commit()-- try:- embedding_li = self.service_context.embed_model- q = VectorStoreQuery(- query_embedding=embedding_li.get_query_embedding(" "),- doc_ids=[self.doc_id],- similarity_top_k=10000,- )- except Exception as e:- self.tool.stream_log(f"Error creating querying : {e}")- raise Exception(f"Error creating querying : {e}")-- n: VectorStoreQueryResult = self.vector_db.query(query=q)- # all_nodes = n.nodes- all_nodes = []- for node in n.nodes: # type:ignore- all_nodes.append(NodeWithScore(node=node, score=0.8))-- if len(n.nodes) > 0: # type:ignore- for node in n.nodes: # type:ignore- node_chunk_text = re.sub(" +", " ", node.get_content()).replace(- "\n", " "- )- node_text = node_chunk_text- cursor.execute(- Query.INSERT_INTO,- (self.doc_id, node.node_id, node_text),- )- db.commit()- else:- self.tool.stream_log(f"No nodes found for {self.doc_id}")-- keyword_nodes_metadata = get_nodes_with_keywords(self.db_name, keywords)- keyword_node_ids = []- for node in keyword_nodes_metadata:- keyword_node_ids.append(node[1])-- # Get node from index using node_id- keyword_nodes = []-- for node in all_nodes:- if node.node_id in keyword_node_ids:- keyword_nodes.append(node)- # print(node)-- return keyword_nodes- # return keyword_nodes + vec_nodes- # return vec_nodes--def construct_prompt(preamble: str,prompt: str,@@ -350,7 +126,6 @@ def construct_prompt_for_engine(grammar_list: list[dict[str, Any]],) -> str:# Let's cleanup the context. Remove if 3 consecutive newlines are found-prompt = f"{preamble}\n\nQuestion or Instruction: {prompt}\n"if grammar_list is not None and len(grammar_list) > 0:prompt += "\n"@@ -369,27 +144,6 @@ def construct_prompt_for_engine(return prompt-def get_nodes_with_keywords(db_name: str, keywords: list[Any]) -> list[Any]:- if len(keywords) == 0:- return []- db = sqlite3.connect(db_name)- cursor = db.cursor()- keywords = ['"' + k + '"' for k in keywords]- # keywords = ['dosing schedules']- query = Query.SELECT- for i in range(len(keywords)):- if i == 0:- query += Query.NODE_MATCH- else:- query += " OR " + Query.NODE_MATCH- query += Query.ORDER_BY- res = cursor.execute(query, keywords)- nodes = []- for r in res:- nodes.append(r)- return nodes--def authentication_middleware(func: Any) -> Any:def wrapper(*args: Any, **kwargs: Any) -> Any:token = AuthenticationMiddleware.get_token_from_auth_header(request)@@ -639,36 +393,11 @@ def prompt_processor() -> Any:llm_li,vector_index,)-- # query_engine = vector_index.as_query_engine(- # filters=MetadataFilters(- # filters=[ExactMatchFilter(key="doc_id", value=doc_id)], # noqa- # ),- # similarity_top_k=output['similarity-top-k'],- # )- # r = query_engine.query(output['promptx'])- # print(r)- # answer = r.response-- elif output[PSKeys.RETRIEVAL_STRATEGY] == PSKeys.VECTOR_KEYWORD:- # TODO: Currently the retriever is restricted to keywords only. # noqa- # TODO: We need to add the vector retriever as well (removed due to context length) # noqa- answer, context = vector_keyword_retriver(- output,- util,- doc_id,- service_context,- vector_db_li,- vector_index,- )- elif output[PSKeys.RETRIEVAL_STRATEGY] == PSKeys.SUBQUESTION:- answer, context = subquestion_retriver(- output, doc_id, service_context, vector_index- )- # nodes = response.source_nodes- # print(nodes)else:- app.logger.info("No retrieval strategy matched")+ app.logger.info(+ "Invalid retrieval strategy "+ f"passed {output[PSKeys.RETRIEVAL_STRATEGY]}"+ )_publish_log(log_events_id,@@ -836,9 +565,6 @@ def prompt_processor() -> Any:app.logger.info(f'No eval plugin found to evaluate prompt: {output["name"]}' # noqa: E501)- #- #- #_publish_log(log_events_id,@@ -873,72 +599,6 @@ def prompt_processor() -> Any:return structured_output-def subquestion_retriver(- output: dict[str, Any],- doc_id: str,- service_context: ServiceContext,- vector_index: VectorStoreIndex,-) -> tuple[Any, str]:- query_engine = vector_index.as_query_engine(- filters=MetadataFilters(- filters=[ExactMatchFilter(key=PSKeys.DOC_ID, value=doc_id)],- ),- similarity_top_k=output[PSKeys.SIMILARITY_TOP_K],- )- query_engine_tools = [- QueryEngineTool(- query_engine=query_engine,- metadata=ToolMetadata(- name="unstract-subquestion",- description="Subquestion query engine",- ),- ),- ]- query_engine = SubQuestionQueryEngine.from_defaults(- query_engine_tools=query_engine_tools,- service_context=service_context,- use_async=True,- )-- prompt = f"{output[PSKeys.PREAMBLE]}\n\n{output[PSKeys.PROMPTX]}"- response = query_engine.query(prompt)- answer = response.response # type:ignore- # Retrieves all the source nodes contents truncated to input length.- sources_text = response.get_formatted_sources(10000)- return (answer, sources_text)---def vector_keyword_retriver(- output: dict[str, Any],- util: BaseTool,- doc_id: str,- service_context: ServiceContext,- vector_db_li: VectorStore,- vector_index: VectorStoreIndex,-) -> tuple[Any, str]:- retriever = UnstractRetriever_V_K(- vector_index,- doc_id,- vector_db_li,- "unstract_vector_db",- service_context,- util,- )- response_synthesizer = get_response_synthesizer(- service_context=service_context,- verbose=True,- )- custom_query_engine = RetrieverQueryEngine(- retriever=retriever,- response_synthesizer=response_synthesizer,- )- response = custom_query_engine.query(output[PSKeys.PROMPTX])- answer = response.response # type:ignore- # Retrieves all the source nodes contents truncated to input length.- sources_text = response.get_formatted_sources(10000)- return (answer, sources_text)--def simple_retriver( # type:ignoreoutput: dict[str, Any],doc_id: str,@@ -1061,9 +721,6 @@ def extract_variable(if promptx != output[PSKeys.PROMPT]:app.logger.info(f"Prompt after variable replacement: {promptx}")return promptx- # app.logger.info(f"Total Tokens: {total_extraction_tokens}")- # with open(f"/tmp/json_of_{file_name_without_path}.json", "w") as f:- # f.write(json.dumps(structured_output, indent=2))def enable_single_pass_extraction() -> None:diff --git a/pyproject.toml b/pyproject.tomlindex c8ef69fb1..8573ce75c 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -17,7 +17,7 @@ lint = ["flake8~=6.0.0","flake8-pyproject~=1.2.2","isort~=5.12.0",- "mypy~=1.2.0",+ "mypy~=1.9.0","pre-commit~=3.3.1","types-PyMySQL~=1.1.0.1","types-pyOpenSSL~=23.2.0.2",diff --git a/tools/structure/src/constants.py b/tools/structure/src/constants.pyindex 0650fc2de..21d2af4ba 100644--- a/tools/structure/src/constants.py+++ b/tools/structure/src/constants.py@@ -21,8 +21,6 @@ class SettingsKeys:ASSERTION_FAILURE_PROMPT = "assertion_failure_prompt"RETRIEVAL_STRATEGY = "retrieval-strategy"SIMPLE = "simple"- VECTOR_KEYWORD = "vector+keyword"- SUBQUESTION = "subquestion"TYPE = "type"NUMBER = "number"EMAIL = "email"