LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
Feat/mrq filter by low confidence (#850)
vishnuszipstack8a01ca29519a
diff --git a/backend/prompt_studio/prompt_studio_registry_v2/urls.py b/backend/prompt_studio/prompt_studio_registry_v2/urls.pyindex 9cc93e05e..b2c303289 100644--- a/backend/prompt_studio/prompt_studio_registry_v2/urls.py+++ b/backend/prompt_studio/prompt_studio_registry_v2/urls.py@@ -1,3 +1,15 @@+from django.urls import pathfrom rest_framework.urlpatterns import format_suffix_patterns-urlpatterns = format_suffix_patterns([])+from .views import PromptStudioRegistryView++urlpatterns = [+ path(+ "registry/",+ PromptStudioRegistryView.as_view({"get": "list"}),+ name="prompt_studio_registry_list",+ ),+]++# Optional: Apply format suffix patterns+urlpatterns = format_suffix_patterns(urlpatterns)diff --git a/backend/workflow_manager/endpoint_v2/destination.py b/backend/workflow_manager/endpoint_v2/destination.pyindex 627b254c9..45047419f 100644--- a/backend/workflow_manager/endpoint_v2/destination.py+++ b/backend/workflow_manager/endpoint_v2/destination.py@@ -37,6 +37,7 @@ from workflow_manager.workflow_v2.execution import WorkflowExecutionServiceHelpefrom workflow_manager.workflow_v2.file_history_helper import FileHistoryHelperfrom workflow_manager.workflow_v2.models.file_history import FileHistoryfrom workflow_manager.workflow_v2.models.workflow import Workflow+from workflow_manager.workflow_v2.utils import WorkflowUtilfrom backend.constants import FeatureFlagfrom backend.exceptions import UnstractFSException@@ -183,9 +184,9 @@ class DestinationConnector(BaseConnector):if connection_type == WorkflowEndpoint.ConnectionType.FILESYSTEM:self.copy_output_to_output_directory()elif connection_type == WorkflowEndpoint.ConnectionType.DATABASE:- if (- file_hash.file_destination- == WorkflowEndpoint.ConnectionType.MANUALREVIEW+ result = self.get_result(file_history)+ if WorkflowUtil.validate_db_rule(+ result, workflow, file_hash.file_destination):self._push_data_to_queue(file_name, workflow, input_file_path)else:@@ -288,6 +289,7 @@ class DestinationConnector(BaseConnector):# If data is None, don't execute CREATE or INSERT queryif not data:return+# Remove metadata from result# Tool text-extractor returns data in the form of string.# Don't pop out metadata in this case.diff --git a/backend/workflow_manager/workflow_v2/constants.py b/backend/workflow_manager/workflow_v2/constants.pyindex a1cf08b43..c36a435da 100644--- a/backend/workflow_manager/workflow_v2/constants.py+++ b/backend/workflow_manager/workflow_v2/constants.py@@ -58,3 +58,8 @@ class WorkflowMessages:FILE_MARKER_CLEAR_SUCCESS = "File marker cleared successfully."FILE_MARKER_CLEAR_FAILED = "Failed to clear file marker."WORKFLOW_EXECUTION_NOT_FOUND = "Workflow execution not found."+++class ResultKeys:+ METADATA = "metadata"+ CONFIDENCE_DATA = "confidence_data"diff --git a/backend/workflow_manager/workflow_v2/enums.py b/backend/workflow_manager/workflow_v2/enums.pyindex cc13a1e35..37e4c01bf 100644--- a/backend/workflow_manager/workflow_v2/enums.py+++ b/backend/workflow_manager/workflow_v2/enums.py@@ -71,3 +71,8 @@ class ColumnModes(Enum):class AgentName(Enum):UNSTRACT_DBWRITER = "Unstract/DBWriter"+++class RuleLogic(Enum):+ AND = "AND"+ OR = "OR"diff --git a/backend/workflow_manager/workflow_v2/utils.py b/backend/workflow_manager/workflow_v2/utils.pyindex 1c19fec69..a88d95dc7 100644--- a/backend/workflow_manager/workflow_v2/utils.py+++ b/backend/workflow_manager/workflow_v2/utils.py@@ -1,4 +1,4 @@-from typing import Any+from typing import Any, Optionalfrom workflow_manager.endpoint_v2.dto import FileHashfrom workflow_manager.workflow_v2.models.workflow import Workflow@@ -59,3 +59,24 @@ class WorkflowUtil:destination modified."""return file_hash++ @staticmethod+ def validate_db_rule(+ result: Optional[Any],+ workflow_id: Workflow,+ file_destination: Optional[tuple[str, str]],+ ) -> bool:+ """Placeholder method to check the db rules - MRQ++ Args:+ result (Optional[Any]): The result dictionary containing+ confidence_data.+ workflow_id (str): The ID of the workflow to retrieve the rules.+ file_destination (Optional[tuple[str, str]]): The destination of+ the file (e.g., MANUALREVIEW or other).++ Returns:+ bool: True if the field_key conditions are met based on rule logic+ and file destination.+ """+ return Falsediff --git a/frontend/src/components/agency/configure-connector-modal/ConfigureConnectorModal.jsx b/frontend/src/components/agency/configure-connector-modal/ConfigureConnectorModal.jsxindex b5026d09a..00d625238 100644--- a/frontend/src/components/agency/configure-connector-modal/ConfigureConnectorModal.jsx+++ b/frontend/src/components/agency/configure-connector-modal/ConfigureConnectorModal.jsx@@ -34,6 +34,7 @@ function ConfigureConnectorModal({connType,selectedItemName,setSelectedItemName,+ workflowDetails,}) {const [activeKey, setActiveKey] = useState("1");useEffect(() => {@@ -190,8 +191,11 @@ function ConfigureConnectorModal({{activeKey === "2" && connType === "FILESYSTEM" && (<ManageFiles selectedItem={connectorId} />)}- {activeKey === "MANUALREVIEW" && (- <DBRules connDetails={connDetails} />+ {activeKey === "MANUALREVIEW" && DBRules && (+ <DBRules+ connDetails={connDetails}+ workflowDetails={workflowDetails}+ />)}</Col></Row>@@ -218,6 +222,7 @@ ConfigureConnectorModal.propTypes = {connType: PropTypes.string.isRequired,selectedItemName: PropTypes.string,setSelectedItemName: PropTypes.func.isRequired,+ workflowDetails: PropTypes.object,};export { ConfigureConnectorModal };diff --git a/frontend/src/components/agency/ds-settings-card/DsSettingsCard.jsx b/frontend/src/components/agency/ds-settings-card/DsSettingsCard.jsxindex 9e6d49126..8f79d1c17 100644--- a/frontend/src/components/agency/ds-settings-card/DsSettingsCard.jsx+++ b/frontend/src/components/agency/ds-settings-card/DsSettingsCard.jsx@@ -47,7 +47,7 @@ const needToRemove = {function DsSettingsCard({ type, endpointDetails, message }) {const workflowStore = useWorkflowStore();- const { source, destination, allowChangeEndpoint } = workflowStore;+ const { source, destination, allowChangeEndpoint, details } = workflowStore;const [options, setOptions] = useState({});const [openModal, setOpenModal] = useState(false);@@ -490,6 +490,7 @@ function DsSettingsCard({ type, endpointDetails, message }) {connType={connType}selectedItemName={selectedItemName}setSelectedItemName={setSelectedItemName}+ workflowDetails={details}/></>);