Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

UN-2684 [FEAT] Automatic removal of processed file name history hashes after configurable retention period (#1500)

Kirtiman Mishraec092977fba4
diff --git a/backend/workflow_manager/endpoint_v2/serializers.py b/backend/workflow_manager/endpoint_v2/serializers.py
index 27990f408..de3fb12a6 100644
--- a/backend/workflow_manager/endpoint_v2/serializers.py
+++ b/backend/workflow_manager/endpoint_v2/serializers.py
@@ -1,11 +1,10 @@
import logging
+from typing import Any
from connector_v2.models import ConnectorInstance
from connector_v2.serializers import ConnectorInstanceSerializer
from rest_framework import serializers
-from rest_framework.serializers import (
- ModelSerializer,
-)
+from rest_framework.serializers import ModelSerializer
from workflow_manager.endpoint_v2.models import WorkflowEndpoint
logger = logging.getLogger(__name__)
@@ -25,7 +24,7 @@ class WorkflowEndpointSerializer(ModelSerializer):
model = WorkflowEndpoint
fields = "__all__"
- def get_fields(self):
+ def get_fields(self) -> Any:
"""Override get_fields to dynamically set the connector_instance_id queryset.
This is needed to ensure that the queryset is set after the organization
diff --git a/backend/workflow_manager/endpoint_v2/source.py b/backend/workflow_manager/endpoint_v2/source.py
index f1cc4d72b..b0852fc42 100644
--- a/backend/workflow_manager/endpoint_v2/source.py
+++ b/backend/workflow_manager/endpoint_v2/source.py
@@ -212,7 +212,12 @@ class SourceConnector(BaseConnector):
for input_directory in valid_directories:
logger.debug(f"Listing files from: {input_directory}")
matched_files, count = self._get_matched_files(
- source_fs, input_directory, patterns, recursive, limit, unique_file_paths
+ source_fs,
+ input_directory,
+ patterns,
+ recursive,
+ limit,
+ unique_file_paths,
)
self.publish_user_sys_log(f"Matched '{count}' files from '{input_directory}'")
total_matched_files.update(matched_files)
@@ -671,6 +676,7 @@ class SourceConnector(BaseConnector):
workflow=workflow,
provider_file_uuid=provider_file_uuid,
file_path=file_hash.file_path,
+ workflow_log=self.workflow_log,
)
return None
diff --git a/backend/workflow_manager/endpoint_v2/static/src/file.json b/backend/workflow_manager/endpoint_v2/static/src/file.json
index c84bc68db..dc8d6ef0a 100644
--- a/backend/workflow_manager/endpoint_v2/static/src/file.json
+++ b/backend/workflow_manager/endpoint_v2/static/src/file.json
@@ -36,8 +36,50 @@
"type": "integer",
"title": "Max files to process",
"default": 100,
- "description": "The maximum number of files to process",
- "minimum": 1
+ "minimum": 1,
+ "description": "The maximum number of files to process"
+ },
+ "fileReprocessingHandling": {
+ "type": "string",
+ "title": "File metadata deletion behavior",
+ "description": "File metadata will not be deleted until the expiration interval passes",
+ "default": "skip_duplicates",
+ "enum": ["skip_duplicates", "reprocess_after_interval"],
+ "enumNames": ["Never delete", "Delete after"]
+ }
+ },
+ "dependencies": {
+ "fileReprocessingHandling": {
+ "oneOf": [
+ {
+ "properties": {
+ "fileReprocessingHandling": {
+ "enum": ["skip_duplicates"]
+ }
+ }
+ },
+ {
+ "properties": {
+ "fileReprocessingHandling": {
+ "enum": ["reprocess_after_interval"]
+ },
+ "reprocessInterval": {
+ "type": "integer",
+ "title": "Duration",
+ "default": 30,
+ "minimum": 1
+ },
+ "intervalUnit": {
+ "type": "string",
+ "title": "Duration Type",
+ "default": "days",
+ "enum": ["days", "months"],
+ "enumNames": ["Days", "Months"]
+ }
+ },
+ "required": ["reprocessInterval", "intervalUnit"]
+ }
+ ]
}
}
}
diff --git a/backend/workflow_manager/workflow_v2/file_history_helper.py b/backend/workflow_manager/workflow_v2/file_history_helper.py
index e4d266502..3d730f7f9 100644
--- a/backend/workflow_manager/workflow_v2/file_history_helper.py
+++ b/backend/workflow_manager/workflow_v2/file_history_helper.py
@@ -1,11 +1,15 @@
import logging
+from datetime import timedelta
from typing import Any
from django.db.models import Q
from django.db.utils import IntegrityError
+from django.utils import timezone
from workflow_manager.endpoint_v2.dto import FileHash
+from workflow_manager.endpoint_v2.models import WorkflowEndpoint
from workflow_manager.file_execution.models import WorkflowFileExecution
+from workflow_manager.utils.workflow_log import WorkflowLog
from workflow_manager.workflow_v2.enums import ExecutionStatus
from workflow_manager.workflow_v2.models.file_history import FileHistory
from workflow_manager.workflow_v2.models.workflow import Workflow
@@ -23,14 +27,19 @@ class FileHistoryHelper:
cache_key: str | None = None,
provider_file_uuid: str | None = None,
file_path: str | None = None,
+ workflow_log: WorkflowLog | None = None,
) -> FileHistory | None:
"""Retrieve a file history record based on the cache key.
+ First deletes expired file histories based on reprocessing interval,
+ then returns the remaining file history if it exists.
+
Args:
workflow (Workflow): The workflow associated with the file history.
cache_key (Optional[str]): The cache key to search for.
provider_file_uuid (Optional[str]): The provider file UUID to search for.
file_path (Optional[str]): The file path to search for.
+ workflow_log (Optional[WorkflowLog]): The workflow log for user notifications.
Returns:
Optional[FileHistory]: The matching file history record, if found.
@@ -41,16 +50,23 @@ class FileHistoryHelper:
f"while fetching file history for {workflow}"
)
return None
+
+ # Delete expired file histories before querying based on reprocessing interval
+ cls._delete_expired_file_histories(workflow, workflow_log)
+
filters = Q(workflow=workflow)
if cache_key:
filters &= Q(cache_key=cache_key)
elif provider_file_uuid:
filters &= Q(provider_file_uuid=provider_file_uuid)
+ file_history: FileHistory | None
try:
if file_path:
- return FileHistory.objects.get(filters & Q(file_path=file_path))
- return FileHistory.objects.get(filters & Q(file_path__isnull=True))
+ file_history = FileHistory.objects.get(filters & Q(file_path=file_path))
+ return file_history
+ file_history = FileHistory.objects.get(filters & Q(file_path__isnull=True))
+ return file_history
except FileHistory.DoesNotExist:
if file_path:
# Legacy fallback: file_path was not stored in older file history records; fallback ensures backward compatibility.
@@ -124,6 +140,102 @@ class FileHistoryHelper:
)
return None
+ @classmethod
+ def _delete_expired_file_histories(
+ cls, workflow: Workflow, workflow_log: WorkflowLog | None = None
+ ) -> None:
+ """Delete expired file histories based on reprocessing interval from WorkflowEndpoint configuration.
+
+ TODO: Move deletion logic to background job instead
+ of calling during get_file_history().
+ 1. This cleanup should run periodically in the
+ background (cron/Celery) rather than blocking file
+ queries.
+ 2. Current approach impacts performance since every
+ get_file_history() call triggers deletion.
+
+ Args:
+ workflow: The workflow to check for expired file histories.
+ workflow_log: Optional workflow log for user notifications.
+ """
+ try:
+ reprocessing_interval = cls._get_reprocessing_interval_from_config(
+ workflow, workflow_log
+ )
+ if reprocessing_interval is None or reprocessing_interval <= 0:
+ return # No reprocessing configured, keep all histories
+
+ now = timezone.now()
+ expiry_date = now - timedelta(days=reprocessing_interval)
+ deleted_count, _ = FileHistory.objects.filter(
+ workflow=workflow, created_at__lt=expiry_date
+ ).delete()
+
+ if deleted_count > 0:
+ logger.info(
+ f"Deleted {deleted_count} expired file histories for workflow {workflow.id}"
+ )
+
+ except Exception as ex:
+ error_msg = (
+ "Unable to clean up expired file metadata. "
+ "files may not be reprocessed if its expected to. "
+ )
+ logger.exception(f"{error_msg}:{ex}")
+ if workflow_log:
+ workflow_log.log_error(logger=logger, message=error_msg)
+
+ @staticmethod
+ def _get_reprocessing_interval_from_config(
+ workflow: Workflow, workflow_log: WorkflowLog | None = None
+ ) -> int | None:
+ """Get reprocessing interval in days from workflow configuration.
+
+ Args:
+ workflow: The workflow to get configuration from.
+ workflow_log: Optional workflow log for user notifications.
+
+ Returns:
+ int | None: Reprocessing interval in days, or None if no reprocessing.
+ """
+ try:
+ source_endpoint = WorkflowEndpoint.objects.get(
+ workflow=workflow,
+ endpoint_type=WorkflowEndpoint.EndpointType.SOURCE,
+ )
+
+ if not source_endpoint.configuration:
+ return None
+
+ duplicate_handling = source_endpoint.configuration.get(
+ "fileReprocessingHandling"
+ )
+ if duplicate_handling != "reprocess_after_interval":
+ return None # Skip duplicates
+
+ interval_value: int = source_endpoint.configuration.get(
+ "reprocessInterval", 0
+ )
+ interval_unit: str = source_endpoint.configuration.get("intervalUnit", "days")
+
+ if interval_value <= 0:
+ return None
+
+ # Convert to days
+ if interval_unit == "months":
+ return interval_value * 30
+ return interval_value
+
+ except Exception as ex:
+ error_msg = (
+ "Unable to fetch file metadata deletion settings, "
+ "file may not be reprocessed if its expected to. "
+ )
+ logger.exception(f"{error_msg}:{ex}")
+ if workflow_log:
+ workflow_log.log_error(logger=logger, message=error_msg)
+ return None
+
@staticmethod
def create_file_history(
file_hash: FileHash,
@@ -143,9 +255,11 @@ class FileHistoryHelper:
result (Any): The result from the execution.
metadata (str | None): The metadata from the execution.
error (str | None): The error from the execution.
+ is_api (bool): Whether this is an API call.
"""
try:
file_path = file_hash.file_path if not is_api else None
+
FileHistory.objects.create(
workflow=workflow,
cache_key=file_hash.file_hash,
diff --git a/backend/workflow_manager/workflow_v2/migrations/0016_add_filehistory_deletion_index.py b/backend/workflow_manager/workflow_v2/migrations/0016_add_filehistory_deletion_index.py
new file mode 100644
index 000000000..05c35d638
--- /dev/null
+++ b/backend/workflow_manager/workflow_v2/migrations/0016_add_filehistory_deletion_index.py
@@ -0,0 +1,18 @@
+# Generated by Django 4.2.1 on 2025-09-04 06:53
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+ dependencies = [
+ ("workflow_v2", "0015_executionlog_idx_wf_execution_event_time_and_more"),
+ ]
+
+ operations = [
+ migrations.AddIndex(
+ model_name="filehistory",
+ index=models.Index(
+ fields=["workflow", "created_at"], name="idx_fh_workflow_created"
+ ),
+ ),
+ ]
diff --git a/backend/workflow_manager/workflow_v2/models/file_history.py b/backend/workflow_manager/workflow_v2/models/file_history.py
index d2ccfcc87..34a9f44a3 100644
--- a/backend/workflow_manager/workflow_v2/models/file_history.py
+++ b/backend/workflow_manager/workflow_v2/models/file_history.py
@@ -55,6 +55,11 @@ class FileHistory(BaseModel):
verbose_name = "File History"
verbose_name_plural = "File Histories"
db_table = "file_history"
+ indexes = [
+ models.Index(
+ fields=["workflow", "created_at"], name="idx_fh_workflow_created"
+ ),
+ ]
constraints = [
# Legacy behavior: file_path is not present or is null
models.UniqueConstraint(
diff --git a/frontend/src/components/agency/configure-connector-modal/ConfigureConnectorModal.jsx b/frontend/src/components/agency/configure-connector-modal/ConfigureConnectorModal.jsx
index cc6fe810e..b146bc5d6 100644
--- a/frontend/src/components/agency/configure-connector-modal/ConfigureConnectorModal.jsx
+++ b/frontend/src/components/agency/configure-connector-modal/ConfigureConnectorModal.jsx
@@ -1,28 +1,28 @@
+import { CloudDownloadOutlined, CloudUploadOutlined } from "@ant-design/icons";
import {
+ Button,
Col,
+ Image,
Modal,
Row,
- Typography,
Select,
Space,
- Image,
Tabs,
- Button,
+ Typography,
} from "antd";
-import { CloudDownloadOutlined, CloudUploadOutlined } from "@ant-design/icons";
+import { cloneDeep, isEqual } from "lodash";
import PropTypes from "prop-types";
-import { useEffect, useState, useRef, useCallback } from "react";
-import { isEqual, cloneDeep } from "lodash";
+import { useCallback, useEffect, useRef, useState } from "react";
-import { useAlertStore } from "../../../store/alert-store";
import { useAxiosPrivate } from "../../../hooks/useAxiosPrivate";
+import { useExceptionHandler } from "../../../hooks/useExceptionHandler";
+import usePostHogEvents from "../../../hooks/usePostHogEvents";
import useRequestUrl from "../../../hooks/useRequestUrl";
+import { useAlertStore } from "../../../store/alert-store";
import { AddSourceModal } from "../../input-output/add-source-modal/AddSourceModal";
import { ManageFiles } from "../../input-output/manage-files/ManageFiles";
-import { ConfigureFormsLayout } from "../configure-forms-layout/ConfigureFormsLayout";
-import { useExceptionHandler } from "../../../hooks/useExceptionHandler";
-import usePostHogEvents from "../../../hooks/usePostHogEvents";
import { CustomButton } from "../../widgets/custom-button/CustomButton";
+import { ConfigureFormsLayout } from "../configure-forms-layout/ConfigureFormsLayout";
import "./ConfigureConnectorModal.css";
let DBRules;
diff --git a/frontend/src/layouts/rjsf-form-layout/CustomFieldTemplate.css b/frontend/src/layouts/rjsf-form-layout/CustomFieldTemplate.css
index 2faae1072..097d7a731 100644
--- a/frontend/src/layouts/rjsf-form-layout/CustomFieldTemplate.css
+++ b/frontend/src/layouts/rjsf-form-layout/CustomFieldTemplate.css
@@ -13,3 +13,15 @@
margin: 0;
padding-left: 1.2em;
}
+
+.compact-expiry-fields {
+ width: 100%;
+}
+
+.compact-expiry-fields .ant-col:first-child {
+ padding-right: 2px;
+}
+
+.compact-expiry-fields .ant-col:last-child {
+ padding-left: 2px;
+}
diff --git a/frontend/src/layouts/rjsf-form-layout/CustomObjectFieldTemplate.jsx b/frontend/src/layouts/rjsf-form-layout/CustomObjectFieldTemplate.jsx
new file mode 100644
index 000000000..4c72ad2a2
--- /dev/null
+++ b/frontend/src/layouts/rjsf-form-layout/CustomObjectFieldTemplate.jsx
@@ -0,0 +1,102 @@
+import { Row, Col } from "antd";
+import PropTypes from "prop-types";
+import "./CustomFieldTemplate.css";
+
+const CustomObjectFieldTemplate = (props) => {
+ const { properties, title, description, formData } = props;
+
+ // Create a map of properties by name for easy lookup
+ const propertyMap = {};
+ properties.forEach((prop) => {
+ propertyMap[prop.name] = prop;
+ });
+
+ // Define field order for file connector forms (this template is only used for forms with file expiry fields)
+ const fieldOrder = [
+ "folders",
+ "processSubDirectories",
+ "fileExtensions",
+ "maxFiles",
+ "fileReprocessingHandling",
+ // Conditional fields at bottom:
+ "reprocessInterval",
+ "intervalUnit",
+ ];
+
+ // Auto-detect if we have file expiry fields that should be rendered side-by-side
+ const hasFileExpiryFields =
+ propertyMap["reprocessInterval"] && propertyMap["intervalUnit"];
+
+ // Filter fields based on conditions
+ const shouldShowConditionalFields =
+ formData?.fileReprocessingHandling === "reprocess_after_interval";
+
+ return (
+ <div>
+ {title && <h3>{title}</h3>}
+ {description && <p>{description}</p>}
+
+ {fieldOrder.map((fieldName) => {
+ const property = propertyMap[fieldName];
+ if (!property) return null;
+
+ // Skip conditional fields when they shouldn't be shown
+ if (
+ (fieldName === "reprocessInterval" || fieldName === "intervalUnit") &&
+ !shouldShowConditionalFields
+ ) {
+ return null;
+ }
+
+ // Handle file expiry fields side-by-side layout
+ if (
+ hasFileExpiryFields &&
+ (fieldName === "reprocessInterval" || fieldName === "intervalUnit")
+ ) {
+ // Render both fields together when we hit reprocessInterval
+ if (
+ fieldName === "reprocessInterval" &&
+ shouldShowConditionalFields
+ ) {
+ const intervalProperty = propertyMap["reprocessInterval"];
+ const unitProperty = propertyMap["intervalUnit"];
+
+ if (intervalProperty && unitProperty) {
+ return (
+ <Row
+ key="conditional-fields"
+ gutter={[4, 2]}
+ className="compact-expiry-fields"
+ >
+ <Col span={12}>{intervalProperty.content}</Col>
+ <Col span={12}>{unitProperty.content}</Col>
+ </Row>
+ );
+ }
+ }
+ // Skip intervalUnit as it's already rendered with reprocessInterval
+ if (fieldName === "intervalUnit") {
+ return null;
+ }
+ }
+
+ // Default rendering for regular fields
+ return (
+ <div key={fieldName} className="property-wrapper">
+ {property.content}
+ </div>
+ );
+ })}
+ </div>
+ );
+};
+
+CustomObjectFieldTemplate.propTypes = {
+ properties: PropTypes.array.isRequired,
+ uiSchema: PropTypes.object,
+ title: PropTypes.string,
+ description: PropTypes.string,
+ formData: PropTypes.object,
+};
+
+export { CustomObjectFieldTemplate };
diff --git a/frontend/src/layouts/rjsf-form-layout/RjsfFormLayout.css b/frontend/src/layouts/rjsf-form-layout/RjsfFormLayout.css
index 46a09663b..6b2edf1ab 100644
--- a/frontend/src/layouts/rjsf-form-layout/RjsfFormLayout.css
+++ b/frontend/src/layouts/rjsf-form-layout/RjsfFormLayout.css
@@ -23,3 +23,22 @@
.rjsf-helper-font {
font-size: 12px;
}
+
+/* Compact styling for file expiry fields */
+.compact-expiry-fields {
+ margin-top: 8px !important;
+ margin-bottom: 16px !important;
+}
+
+.compact-expiry-fields .ant-col {
+ padding-bottom: 0 !important;
+}
+
+.compact-expiry-fields .ant-form-item {
+ margin-bottom: 12px !important;
+}
+
+/* Make the expires after input narrower */
+.compact-expiry-fields .ant-col:first-child .ant-input {
+ max-width: 120px;
+}
diff --git a/frontend/src/layouts/rjsf-form-layout/RjsfFormLayout.jsx b/frontend/src/layouts/rjsf-form-layout/RjsfFormLayout.jsx
index 020d61260..2f6c944bb 100644
--- a/frontend/src/layouts/rjsf-form-layout/RjsfFormLayout.jsx
+++ b/frontend/src/layouts/rjsf-form-layout/RjsfFormLayout.jsx
@@ -1,9 +1,10 @@
-import { useCallback, useMemo } from "react";
import Form from "@rjsf/antd";
import validator from "@rjsf/validator-ajv8";
-import PropTypes from "prop-types";
import { Alert, Space } from "antd";
+import PropTypes from "prop-types";
+import { useCallback, useMemo } from "react";
+import CustomMarkdown from "../../components/helpers/custom-markdown/CustomMarkdown.jsx";
import { AltDateTimeWidget } from "../../components/rjsf-custom-widgets/alt-date-time-widget/AltDateTimeWidget.jsx";
import { AltDateWidget } from "../../components/rjsf-custom-widgets/alt-date-widget/AltDateWidget.jsx";
import { ArrayField } from "../../components/rjsf-custom-widgets/array-field/ArrayField.jsx";
@@ -15,15 +16,15 @@ import { DateWidget } from "../../components/rjsf-custom-widgets/date-widget/Dat
import { EmailWidget } from "../../components/rjsf-custom-widgets/email-widget/EmailWidget.jsx";
import { FileWidget } from "../../components/rjsf-custom-widgets/file-widget/FileWidget.jsx";
import { HiddenWidget } from "../../components/rjsf-custom-widgets/hidden-widget/HiddenWidget.jsx";
+import { PasswordWidget } from "../../components/rjsf-custom-widgets/password-widget/PasswordWidget.jsx";
import { SelectWidget } from "../../components/rjsf-custom-widgets/select-widget/SelectWidget.jsx";
+import { TextWidget } from "../../components/rjsf-custom-widgets/text-widget/TextWidget.jsx";
import { TimeWidget } from "../../components/rjsf-custom-widgets/time-widget/TimeWidget.jsx";
+import { UpDownWidget } from "../../components/rjsf-custom-widgets/up-down-widget/UpDownWidget.jsx";
import { URLWidget } from "../../components/rjsf-custom-widgets/url-widget/URLWidget.jsx";
import { SpinnerLoader } from "../../components/widgets/spinner-loader/SpinnerLoader.jsx";
-import { TextWidget } from "../../components/rjsf-custom-widgets/text-widget/TextWidget.jsx";
-import { PasswordWidget } from "../../components/rjsf-custom-widgets/password-widget/PasswordWidget.jsx";
-import { UpDownWidget } from "../../components/rjsf-custom-widgets/up-down-widget/UpDownWidget.jsx";
import { CustomFieldTemplate } from "./CustomFieldTemplate.jsx";
-import CustomMarkdown from "../../components/helpers/custom-markdown/CustomMarkdown.jsx";
+import { CustomObjectFieldTemplate } from "./CustomObjectFieldTemplate.jsx";
import "./RjsfFormLayout.css";
function RjsfFormLayout({
@@ -248,6 +249,21 @@ function RjsfFormLayout({
[isStateUpdateRequired, setFormData]
);
+ // Apply CustomObjectFieldTemplate for file connector forms (those with fileReprocessingHandling)
+ const templates = useMemo(() => {
+ const isFileConnectorForm =
+ !!formSchema?.properties?.fileReprocessingHandling;
+
+ return isFileConnectorForm
+ ? {
+ FieldTemplate: CustomFieldTemplate,
+ ObjectFieldTemplate: CustomObjectFieldTemplate,
+ }
+ : {
+ FieldTemplate: CustomFieldTemplate,
+ };
+ }, [formSchema]);
+
return (
<>
{isLoading ? (
@@ -273,9 +289,7 @@ function RjsfFormLayout({
onSubmit={(e) => validateAndSubmit?.(e.formData)}
showErrorList={false}
onChange={handleChange}
- templates={{
- FieldTemplate: CustomFieldTemplate,
- }}
+ templates={templates}
>
{children || <></>}
</Form>