LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
UN-2684 [FEAT] Automatic removal of processed file name history hashes after configurable retention period (#1500)
Kirtiman Mishraec092977fba4
diff --git a/backend/workflow_manager/endpoint_v2/serializers.py b/backend/workflow_manager/endpoint_v2/serializers.pyindex 27990f408..de3fb12a6 100644--- a/backend/workflow_manager/endpoint_v2/serializers.py+++ b/backend/workflow_manager/endpoint_v2/serializers.py@@ -1,11 +1,10 @@import logging+from typing import Anyfrom connector_v2.models import ConnectorInstancefrom connector_v2.serializers import ConnectorInstanceSerializerfrom rest_framework import serializers-from rest_framework.serializers import (- ModelSerializer,-)+from rest_framework.serializers import ModelSerializerfrom workflow_manager.endpoint_v2.models import WorkflowEndpointlogger = logging.getLogger(__name__)@@ -25,7 +24,7 @@ class WorkflowEndpointSerializer(ModelSerializer):model = WorkflowEndpointfields = "__all__"- def get_fields(self):+ def get_fields(self) -> Any:"""Override get_fields to dynamically set the connector_instance_id queryset.This is needed to ensure that the queryset is set after the organizationdiff --git a/backend/workflow_manager/endpoint_v2/source.py b/backend/workflow_manager/endpoint_v2/source.pyindex f1cc4d72b..b0852fc42 100644--- a/backend/workflow_manager/endpoint_v2/source.py+++ b/backend/workflow_manager/endpoint_v2/source.py@@ -212,7 +212,12 @@ class SourceConnector(BaseConnector):for input_directory in valid_directories:logger.debug(f"Listing files from: {input_directory}")matched_files, count = self._get_matched_files(- source_fs, input_directory, patterns, recursive, limit, unique_file_paths+ source_fs,+ input_directory,+ patterns,+ recursive,+ limit,+ unique_file_paths,)self.publish_user_sys_log(f"Matched '{count}' files from '{input_directory}'")total_matched_files.update(matched_files)@@ -671,6 +676,7 @@ class SourceConnector(BaseConnector):workflow=workflow,provider_file_uuid=provider_file_uuid,file_path=file_hash.file_path,+ workflow_log=self.workflow_log,)return Nonediff --git a/backend/workflow_manager/endpoint_v2/static/src/file.json b/backend/workflow_manager/endpoint_v2/static/src/file.jsonindex c84bc68db..dc8d6ef0a 100644--- a/backend/workflow_manager/endpoint_v2/static/src/file.json+++ b/backend/workflow_manager/endpoint_v2/static/src/file.json@@ -36,8 +36,50 @@"type": "integer","title": "Max files to process","default": 100,- "description": "The maximum number of files to process",- "minimum": 1+ "minimum": 1,+ "description": "The maximum number of files to process"+ },+ "fileReprocessingHandling": {+ "type": "string",+ "title": "File metadata deletion behavior",+ "description": "File metadata will not be deleted until the expiration interval passes",+ "default": "skip_duplicates",+ "enum": ["skip_duplicates", "reprocess_after_interval"],+ "enumNames": ["Never delete", "Delete after"]+ }+ },+ "dependencies": {+ "fileReprocessingHandling": {+ "oneOf": [+ {+ "properties": {+ "fileReprocessingHandling": {+ "enum": ["skip_duplicates"]+ }+ }+ },+ {+ "properties": {+ "fileReprocessingHandling": {+ "enum": ["reprocess_after_interval"]+ },+ "reprocessInterval": {+ "type": "integer",+ "title": "Duration",+ "default": 30,+ "minimum": 1+ },+ "intervalUnit": {+ "type": "string",+ "title": "Duration Type",+ "default": "days",+ "enum": ["days", "months"],+ "enumNames": ["Days", "Months"]+ }+ },+ "required": ["reprocessInterval", "intervalUnit"]+ }+ ]}}}diff --git a/backend/workflow_manager/workflow_v2/file_history_helper.py b/backend/workflow_manager/workflow_v2/file_history_helper.pyindex e4d266502..3d730f7f9 100644--- a/backend/workflow_manager/workflow_v2/file_history_helper.py+++ b/backend/workflow_manager/workflow_v2/file_history_helper.py@@ -1,11 +1,15 @@import logging+from datetime import timedeltafrom typing import Anyfrom django.db.models import Qfrom django.db.utils import IntegrityError+from django.utils import timezonefrom workflow_manager.endpoint_v2.dto import FileHash+from workflow_manager.endpoint_v2.models import WorkflowEndpointfrom workflow_manager.file_execution.models import WorkflowFileExecution+from workflow_manager.utils.workflow_log import WorkflowLogfrom workflow_manager.workflow_v2.enums import ExecutionStatusfrom workflow_manager.workflow_v2.models.file_history import FileHistoryfrom workflow_manager.workflow_v2.models.workflow import Workflow@@ -23,14 +27,19 @@ class FileHistoryHelper:cache_key: str | None = None,provider_file_uuid: str | None = None,file_path: str | None = None,+ workflow_log: WorkflowLog | None = None,) -> FileHistory | None:"""Retrieve a file history record based on the cache key.+ First deletes expired file histories based on reprocessing interval,+ then returns the remaining file history if it exists.+Args:workflow (Workflow): The workflow associated with the file history.cache_key (Optional[str]): The cache key to search for.provider_file_uuid (Optional[str]): The provider file UUID to search for.file_path (Optional[str]): The file path to search for.+ workflow_log (Optional[WorkflowLog]): The workflow log for user notifications.Returns:Optional[FileHistory]: The matching file history record, if found.@@ -41,16 +50,23 @@ class FileHistoryHelper:f"while fetching file history for {workflow}")return None++ # Delete expired file histories before querying based on reprocessing interval+ cls._delete_expired_file_histories(workflow, workflow_log)+filters = Q(workflow=workflow)if cache_key:filters &= Q(cache_key=cache_key)elif provider_file_uuid:filters &= Q(provider_file_uuid=provider_file_uuid)+ file_history: FileHistory | Nonetry:if file_path:- return FileHistory.objects.get(filters & Q(file_path=file_path))- return FileHistory.objects.get(filters & Q(file_path__isnull=True))+ file_history = FileHistory.objects.get(filters & Q(file_path=file_path))+ return file_history+ file_history = FileHistory.objects.get(filters & Q(file_path__isnull=True))+ return file_historyexcept FileHistory.DoesNotExist:if file_path:# Legacy fallback: file_path was not stored in older file history records; fallback ensures backward compatibility.@@ -124,6 +140,102 @@ class FileHistoryHelper:)return None+ @classmethod+ def _delete_expired_file_histories(+ cls, workflow: Workflow, workflow_log: WorkflowLog | None = None+ ) -> None:+ """Delete expired file histories based on reprocessing interval from WorkflowEndpoint configuration.++ TODO: Move deletion logic to background job instead+ of calling during get_file_history().+ 1. This cleanup should run periodically in the+ background (cron/Celery) rather than blocking file+ queries.+ 2. Current approach impacts performance since every+ get_file_history() call triggers deletion.++ Args:+ workflow: The workflow to check for expired file histories.+ workflow_log: Optional workflow log for user notifications.+ """+ try:+ reprocessing_interval = cls._get_reprocessing_interval_from_config(+ workflow, workflow_log+ )+ if reprocessing_interval is None or reprocessing_interval <= 0:+ return # No reprocessing configured, keep all histories++ now = timezone.now()+ expiry_date = now - timedelta(days=reprocessing_interval)+ deleted_count, _ = FileHistory.objects.filter(+ workflow=workflow, created_at__lt=expiry_date+ ).delete()++ if deleted_count > 0:+ logger.info(+ f"Deleted {deleted_count} expired file histories for workflow {workflow.id}"+ )++ except Exception as ex:+ error_msg = (+ "Unable to clean up expired file metadata. "+ "files may not be reprocessed if its expected to. "+ )+ logger.exception(f"{error_msg}:{ex}")+ if workflow_log:+ workflow_log.log_error(logger=logger, message=error_msg)++ @staticmethod+ def _get_reprocessing_interval_from_config(+ workflow: Workflow, workflow_log: WorkflowLog | None = None+ ) -> int | None:+ """Get reprocessing interval in days from workflow configuration.++ Args:+ workflow: The workflow to get configuration from.+ workflow_log: Optional workflow log for user notifications.++ Returns:+ int | None: Reprocessing interval in days, or None if no reprocessing.+ """+ try:+ source_endpoint = WorkflowEndpoint.objects.get(+ workflow=workflow,+ endpoint_type=WorkflowEndpoint.EndpointType.SOURCE,+ )++ if not source_endpoint.configuration:+ return None++ duplicate_handling = source_endpoint.configuration.get(+ "fileReprocessingHandling"+ )+ if duplicate_handling != "reprocess_after_interval":+ return None # Skip duplicates++ interval_value: int = source_endpoint.configuration.get(+ "reprocessInterval", 0+ )+ interval_unit: str = source_endpoint.configuration.get("intervalUnit", "days")++ if interval_value <= 0:+ return None++ # Convert to days+ if interval_unit == "months":+ return interval_value * 30+ return interval_value++ except Exception as ex:+ error_msg = (+ "Unable to fetch file metadata deletion settings, "+ "file may not be reprocessed if its expected to. "+ )+ logger.exception(f"{error_msg}:{ex}")+ if workflow_log:+ workflow_log.log_error(logger=logger, message=error_msg)+ return None+@staticmethoddef create_file_history(file_hash: FileHash,@@ -143,9 +255,11 @@ class FileHistoryHelper:result (Any): The result from the execution.metadata (str | None): The metadata from the execution.error (str | None): The error from the execution.+ is_api (bool): Whether this is an API call."""try:file_path = file_hash.file_path if not is_api else None+FileHistory.objects.create(workflow=workflow,cache_key=file_hash.file_hash,diff --git a/backend/workflow_manager/workflow_v2/migrations/0016_add_filehistory_deletion_index.py b/backend/workflow_manager/workflow_v2/migrations/0016_add_filehistory_deletion_index.pynew file mode 100644index 000000000..05c35d638--- /dev/null+++ b/backend/workflow_manager/workflow_v2/migrations/0016_add_filehistory_deletion_index.py@@ -0,0 +1,18 @@+# Generated by Django 4.2.1 on 2025-09-04 06:53++from django.db import migrations, models+++class Migration(migrations.Migration):+ dependencies = [+ ("workflow_v2", "0015_executionlog_idx_wf_execution_event_time_and_more"),+ ]++ operations = [+ migrations.AddIndex(+ model_name="filehistory",+ index=models.Index(+ fields=["workflow", "created_at"], name="idx_fh_workflow_created"+ ),+ ),+ ]diff --git a/backend/workflow_manager/workflow_v2/models/file_history.py b/backend/workflow_manager/workflow_v2/models/file_history.pyindex d2ccfcc87..34a9f44a3 100644--- a/backend/workflow_manager/workflow_v2/models/file_history.py+++ b/backend/workflow_manager/workflow_v2/models/file_history.py@@ -55,6 +55,11 @@ class FileHistory(BaseModel):verbose_name = "File History"verbose_name_plural = "File Histories"db_table = "file_history"+ indexes = [+ models.Index(+ fields=["workflow", "created_at"], name="idx_fh_workflow_created"+ ),+ ]constraints = [# Legacy behavior: file_path is not present or is nullmodels.UniqueConstraint(diff --git a/frontend/src/components/agency/configure-connector-modal/ConfigureConnectorModal.jsx b/frontend/src/components/agency/configure-connector-modal/ConfigureConnectorModal.jsxindex cc6fe810e..b146bc5d6 100644--- a/frontend/src/components/agency/configure-connector-modal/ConfigureConnectorModal.jsx+++ b/frontend/src/components/agency/configure-connector-modal/ConfigureConnectorModal.jsx@@ -1,28 +1,28 @@+import { CloudDownloadOutlined, CloudUploadOutlined } from "@ant-design/icons";import {+ Button,Col,+ Image,Modal,Row,- Typography,Select,Space,- Image,Tabs,- Button,+ Typography,} from "antd";-import { CloudDownloadOutlined, CloudUploadOutlined } from "@ant-design/icons";+import { cloneDeep, isEqual } from "lodash";import PropTypes from "prop-types";-import { useEffect, useState, useRef, useCallback } from "react";-import { isEqual, cloneDeep } from "lodash";+import { useCallback, useEffect, useRef, useState } from "react";-import { useAlertStore } from "../../../store/alert-store";import { useAxiosPrivate } from "../../../hooks/useAxiosPrivate";+import { useExceptionHandler } from "../../../hooks/useExceptionHandler";+import usePostHogEvents from "../../../hooks/usePostHogEvents";import useRequestUrl from "../../../hooks/useRequestUrl";+import { useAlertStore } from "../../../store/alert-store";import { AddSourceModal } from "../../input-output/add-source-modal/AddSourceModal";import { ManageFiles } from "../../input-output/manage-files/ManageFiles";-import { ConfigureFormsLayout } from "../configure-forms-layout/ConfigureFormsLayout";-import { useExceptionHandler } from "../../../hooks/useExceptionHandler";-import usePostHogEvents from "../../../hooks/usePostHogEvents";import { CustomButton } from "../../widgets/custom-button/CustomButton";+import { ConfigureFormsLayout } from "../configure-forms-layout/ConfigureFormsLayout";import "./ConfigureConnectorModal.css";let DBRules;diff --git a/frontend/src/layouts/rjsf-form-layout/CustomFieldTemplate.css b/frontend/src/layouts/rjsf-form-layout/CustomFieldTemplate.cssindex 2faae1072..097d7a731 100644--- a/frontend/src/layouts/rjsf-form-layout/CustomFieldTemplate.css+++ b/frontend/src/layouts/rjsf-form-layout/CustomFieldTemplate.css@@ -13,3 +13,15 @@margin: 0;padding-left: 1.2em;}++.compact-expiry-fields {+ width: 100%;+}++.compact-expiry-fields .ant-col:first-child {+ padding-right: 2px;+}++.compact-expiry-fields .ant-col:last-child {+ padding-left: 2px;+}diff --git a/frontend/src/layouts/rjsf-form-layout/CustomObjectFieldTemplate.jsx b/frontend/src/layouts/rjsf-form-layout/CustomObjectFieldTemplate.jsxnew file mode 100644index 000000000..4c72ad2a2--- /dev/null+++ b/frontend/src/layouts/rjsf-form-layout/CustomObjectFieldTemplate.jsx@@ -0,0 +1,102 @@+import { Row, Col } from "antd";+import PropTypes from "prop-types";+import "./CustomFieldTemplate.css";++const CustomObjectFieldTemplate = (props) => {+ const { properties, title, description, formData } = props;++ // Create a map of properties by name for easy lookup+ const propertyMap = {};+ properties.forEach((prop) => {+ propertyMap[prop.name] = prop;+ });++ // Define field order for file connector forms (this template is only used for forms with file expiry fields)+ const fieldOrder = [+ "folders",+ "processSubDirectories",+ "fileExtensions",+ "maxFiles",+ "fileReprocessingHandling",+ // Conditional fields at bottom:+ "reprocessInterval",+ "intervalUnit",+ ];++ // Auto-detect if we have file expiry fields that should be rendered side-by-side+ const hasFileExpiryFields =+ propertyMap["reprocessInterval"] && propertyMap["intervalUnit"];++ // Filter fields based on conditions+ const shouldShowConditionalFields =+ formData?.fileReprocessingHandling === "reprocess_after_interval";++ return (+ <div>+ {title && <h3>{title}</h3>}+ {description && <p>{description}</p>}++ {fieldOrder.map((fieldName) => {+ const property = propertyMap[fieldName];+ if (!property) return null;++ // Skip conditional fields when they shouldn't be shown+ if (+ (fieldName === "reprocessInterval" || fieldName === "intervalUnit") &&+ !shouldShowConditionalFields+ ) {+ return null;+ }++ // Handle file expiry fields side-by-side layout+ if (+ hasFileExpiryFields &&+ (fieldName === "reprocessInterval" || fieldName === "intervalUnit")+ ) {+ // Render both fields together when we hit reprocessInterval+ if (+ fieldName === "reprocessInterval" &&+ shouldShowConditionalFields+ ) {+ const intervalProperty = propertyMap["reprocessInterval"];+ const unitProperty = propertyMap["intervalUnit"];++ if (intervalProperty && unitProperty) {+ return (+ <Row+ key="conditional-fields"+ gutter={[4, 2]}+ className="compact-expiry-fields"+ >+ <Col span={12}>{intervalProperty.content}</Col>+ <Col span={12}>{unitProperty.content}</Col>+ </Row>+ );+ }+ }+ // Skip intervalUnit as it's already rendered with reprocessInterval+ if (fieldName === "intervalUnit") {+ return null;+ }+ }++ // Default rendering for regular fields+ return (+ <div key={fieldName} className="property-wrapper">+ {property.content}+ </div>+ );+ })}+ </div>+ );+};++CustomObjectFieldTemplate.propTypes = {+ properties: PropTypes.array.isRequired,+ uiSchema: PropTypes.object,+ title: PropTypes.string,+ description: PropTypes.string,+ formData: PropTypes.object,+};++export { CustomObjectFieldTemplate };diff --git a/frontend/src/layouts/rjsf-form-layout/RjsfFormLayout.css b/frontend/src/layouts/rjsf-form-layout/RjsfFormLayout.cssindex 46a09663b..6b2edf1ab 100644--- a/frontend/src/layouts/rjsf-form-layout/RjsfFormLayout.css+++ b/frontend/src/layouts/rjsf-form-layout/RjsfFormLayout.css@@ -23,3 +23,22 @@.rjsf-helper-font {font-size: 12px;}++/* Compact styling for file expiry fields */+.compact-expiry-fields {+ margin-top: 8px !important;+ margin-bottom: 16px !important;+}++.compact-expiry-fields .ant-col {+ padding-bottom: 0 !important;+}++.compact-expiry-fields .ant-form-item {+ margin-bottom: 12px !important;+}++/* Make the expires after input narrower */+.compact-expiry-fields .ant-col:first-child .ant-input {+ max-width: 120px;+}diff --git a/frontend/src/layouts/rjsf-form-layout/RjsfFormLayout.jsx b/frontend/src/layouts/rjsf-form-layout/RjsfFormLayout.jsxindex 020d61260..2f6c944bb 100644--- a/frontend/src/layouts/rjsf-form-layout/RjsfFormLayout.jsx+++ b/frontend/src/layouts/rjsf-form-layout/RjsfFormLayout.jsx@@ -1,9 +1,10 @@-import { useCallback, useMemo } from "react";import Form from "@rjsf/antd";import validator from "@rjsf/validator-ajv8";-import PropTypes from "prop-types";import { Alert, Space } from "antd";+import PropTypes from "prop-types";+import { useCallback, useMemo } from "react";+import CustomMarkdown from "../../components/helpers/custom-markdown/CustomMarkdown.jsx";import { AltDateTimeWidget } from "../../components/rjsf-custom-widgets/alt-date-time-widget/AltDateTimeWidget.jsx";import { AltDateWidget } from "../../components/rjsf-custom-widgets/alt-date-widget/AltDateWidget.jsx";import { ArrayField } from "../../components/rjsf-custom-widgets/array-field/ArrayField.jsx";@@ -15,15 +16,15 @@ import { DateWidget } from "../../components/rjsf-custom-widgets/date-widget/Datimport { EmailWidget } from "../../components/rjsf-custom-widgets/email-widget/EmailWidget.jsx";import { FileWidget } from "../../components/rjsf-custom-widgets/file-widget/FileWidget.jsx";import { HiddenWidget } from "../../components/rjsf-custom-widgets/hidden-widget/HiddenWidget.jsx";+import { PasswordWidget } from "../../components/rjsf-custom-widgets/password-widget/PasswordWidget.jsx";import { SelectWidget } from "../../components/rjsf-custom-widgets/select-widget/SelectWidget.jsx";+import { TextWidget } from "../../components/rjsf-custom-widgets/text-widget/TextWidget.jsx";import { TimeWidget } from "../../components/rjsf-custom-widgets/time-widget/TimeWidget.jsx";+import { UpDownWidget } from "../../components/rjsf-custom-widgets/up-down-widget/UpDownWidget.jsx";import { URLWidget } from "../../components/rjsf-custom-widgets/url-widget/URLWidget.jsx";import { SpinnerLoader } from "../../components/widgets/spinner-loader/SpinnerLoader.jsx";-import { TextWidget } from "../../components/rjsf-custom-widgets/text-widget/TextWidget.jsx";-import { PasswordWidget } from "../../components/rjsf-custom-widgets/password-widget/PasswordWidget.jsx";-import { UpDownWidget } from "../../components/rjsf-custom-widgets/up-down-widget/UpDownWidget.jsx";import { CustomFieldTemplate } from "./CustomFieldTemplate.jsx";-import CustomMarkdown from "../../components/helpers/custom-markdown/CustomMarkdown.jsx";+import { CustomObjectFieldTemplate } from "./CustomObjectFieldTemplate.jsx";import "./RjsfFormLayout.css";function RjsfFormLayout({@@ -248,6 +249,21 @@ function RjsfFormLayout({[isStateUpdateRequired, setFormData]);+ // Apply CustomObjectFieldTemplate for file connector forms (those with fileReprocessingHandling)+ const templates = useMemo(() => {+ const isFileConnectorForm =+ !!formSchema?.properties?.fileReprocessingHandling;++ return isFileConnectorForm+ ? {+ FieldTemplate: CustomFieldTemplate,+ ObjectFieldTemplate: CustomObjectFieldTemplate,+ }+ : {+ FieldTemplate: CustomFieldTemplate,+ };+ }, [formSchema]);+return (<>{isLoading ? (@@ -273,9 +289,7 @@ function RjsfFormLayout({onSubmit={(e) => validateAndSubmit?.(e.formData)}showErrorList={false}onChange={handleChange}- templates={{- FieldTemplate: CustomFieldTemplate,- }}+ templates={templates}>{children || <></>}</Form>