LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
UN-2966 [FEAT] Auto-refresh and controls for execution logs (#1687)
Chandrasekharan M3d4415d79284
diff --git a/backend/workflow_manager/execution/filter.py b/backend/workflow_manager/execution/filter.pyindex 6664e3f8a..d62a96432 100644--- a/backend/workflow_manager/execution/filter.py+++ b/backend/workflow_manager/execution/filter.py@@ -10,6 +10,7 @@ from workflow_manager.workflow_v2.models import WorkflowExecutionclass ExecutionFilter(filters.FilterSet):+ id = filters.UUIDFilter()execution_entity = filters.ChoiceFilter(choices=[(ExecutionEntity.API.value, "API"),diff --git a/backend/workflow_manager/file_execution/views.py b/backend/workflow_manager/file_execution/views.pyindex 17885b1d0..8e3ef9607 100644--- a/backend/workflow_manager/file_execution/views.py+++ b/backend/workflow_manager/file_execution/views.py@@ -16,7 +16,7 @@ class FileCentricExecutionViewSet(viewsets.ReadOnlyModelViewSet):serializer_class = FileCentricExecutionSerializerpagination_class = CustomPaginationfilter_backends = [DjangoFilterBackend, OrderingFilter]- ordering_fields = ["created_at", "execution_time"]+ ordering_fields = ["created_at", "execution_time", "file_size"]ordering = ["created_at"]filterset_class = FileExecutionFilterdiff --git a/frontend/src/components/logging/detailed-logs/DetailedLogs.css b/frontend/src/components/logging/detailed-logs/DetailedLogs.cssindex a047dc690..668af2000 100644--- a/frontend/src/components/logging/detailed-logs/DetailedLogs.css+++ b/frontend/src/components/logging/detailed-logs/DetailedLogs.css@@ -1,3 +1,67 @@+/* Flexbox layout for responsive table */+.detailed-logs-container {+ display: flex;+ flex-direction: column;+ height: 100%;+ overflow: hidden;+}++.detailed-logs-header {+ flex: 0 0 auto;+ display: flex;+ justify-content: space-between;+ align-items: center;+ padding: 16px 24px;+}++.detailed-logs-cards {+ flex: 0 0 auto;+}++.detailed-logs-table-container {+ flex: 1;+ min-height: 0;+ padding: 0 12px 8px;+ display: flex;+ flex-direction: column;+}++.detailed-logs-table-container .ant-table-wrapper {+ flex: 1;+ min-height: 0;+ display: flex;+ flex-direction: column;+}++.detailed-logs-table-container .ant-spin-nested-loading,+.detailed-logs-table-container .ant-spin-container {+ flex: 1;+ min-height: 0;+ display: flex;+ flex-direction: column;+}++.detailed-logs-table-container .ant-table {+ flex: 1;+ min-height: 0;+ display: flex;+ flex-direction: column;+ overflow: auto;+}++.detailed-logs-table-container .ant-table-container {+ flex: 1;+ min-height: 0;+ display: flex;+ flex-direction: column;+}++.detailed-logs-table-container .ant-table-body {+ flex: 1;+ min-height: 0;+ overflow-y: auto !important;+}+.logging-card-title {font-weight: 600;font-size: 14px;@@ -16,11 +80,50 @@}.view-log-button {- margin-right: 20px;+ margin-right: 24px;+}++/* Action column header with settings icon */+.action-column-header {+ display: flex;+ align-items: center;+ justify-content: space-evenly;+ width: 100%;+}++.column-settings-trigger {+ display: flex;+ align-items: center;+ justify-content: center;+ cursor: pointer;+ border-radius: 4px;+}++.column-settings-trigger:hover {+ background-color: rgba(0, 0, 0, 0.04);+}++.column-settings-icon {+ font-size: 16px;+ color: rgba(0, 0, 0, 0.45);+}++.column-settings-trigger:hover .column-settings-icon {+ color: #1677ff;+}++/* Thin scrollbar */+.detailed-logs-table-container .ant-table-body::-webkit-scrollbar {+ width: 6px;+}++.detailed-logs-table-container .ant-table-body::-webkit-scrollbar-thumb {+ background: rgba(0, 0, 0, 0.2);+ border-radius: 3px;}-.column-filter-dropdown{- margin-bottom: 20px;+.detailed-logs-table-container .ant-table-body::-webkit-scrollbar-track {+ background: transparent;}.search-container {@@ -32,3 +135,37 @@margin-bottom: 8px;display: block;}++/* Fix cursor in pagination dropdown */+.detailed-logs-table-container .ant-pagination-options .ant-select-selector {+ cursor: pointer;+}++.detailed-logs-table-container+ .ant-pagination-options+ .ant-select-selection-item {+ cursor: pointer;+}++/* Status message text with ellipsis */+.status-message-text {+ display: block;+ overflow: hidden;+ text-overflow: ellipsis;+ white-space: nowrap;+}++/* Outlined copy button style - matches LLM Profiles / API Deployments */+.copy-btn-outlined {+ border: 1px solid #d9d9d9;+ background: #fff;+ color: rgba(0, 0, 0, 0.65);+ margin-left: 8px;+ height: auto;+ min-width: auto;+}++.copy-btn-outlined:hover {+ border-color: #1677ff;+ color: #1677ff;+}diff --git a/frontend/src/components/logging/detailed-logs/DetailedLogs.jsx b/frontend/src/components/logging/detailed-logs/DetailedLogs.jsxindex 4c7729d86..0c89b2afe 100644--- a/frontend/src/components/logging/detailed-logs/DetailedLogs.jsx+++ b/frontend/src/components/logging/detailed-logs/DetailedLogs.jsx@@ -1,15 +1,17 @@-import { useEffect, useState } from "react";+import { useEffect, useState, useRef } from "react";+import PropTypes from "prop-types";import { useNavigate, useParams } from "react-router-dom";import {ArrowLeftOutlined,CalendarOutlined,ClockCircleOutlined,CloseCircleFilled,- DownOutlined,+ CopyOutlined,EyeOutlined,FileTextOutlined,HourglassOutlined,InfoCircleFilled,+ MoreOutlined,SearchOutlined,} from "@ant-design/icons";import {@@ -35,8 +37,56 @@ import {} from "../../../helpers/GetStaticData";import { LogModal } from "../log-modal/LogModal";import { FilterIcon } from "../filter-dropdown/FilterDropdown";+import { LogsRefreshControls } from "../logs-refresh-controls/LogsRefreshControls";import useRequestUrl from "../../../hooks/useRequestUrl";+// Component for status message with conditional tooltip (only when truncated)+const StatusMessageCell = ({ text }) => {+ const textRef = useRef(null);+ const [isOverflowing, setIsOverflowing] = useState(false);++ useEffect(() => {+ const checkOverflow = () => {+ if (textRef.current) {+ setIsOverflowing(+ textRef.current.scrollWidth > textRef.current.clientWidth+ );+ }+ };+ checkOverflow();+ window.addEventListener("resize", checkOverflow);+ return () => window.removeEventListener("resize", checkOverflow);+ }, [text]);++ const content = (+ <span ref={textRef} className="status-message-text">+ {text}+ </span>+ );++ return isOverflowing ? <Tooltip title={text}>{content}</Tooltip> : content;+};++StatusMessageCell.propTypes = {+ text: PropTypes.string.isRequired,+};++// Action column header with visibility dropdown+const ActionColumnHeader = ({ menu }) => (+ <div className="action-column-header">+ <span>Action</span>+ <Dropdown menu={menu} trigger={["click"]} placement="bottomRight">+ <span className="column-settings-trigger">+ <MoreOutlined className="column-settings-icon" />+ </span>+ </Dropdown>+ </div>+);++ActionColumnHeader.propTypes = {+ menu: PropTypes.object.isRequired,+};+const DetailedLogs = () => {const { id, type } = useParams(); // Get the ID from the URLconst axiosPrivate = useAxiosPrivate();@@ -61,8 +111,40 @@ const DetailedLogs = () => {const [statusFilter, setStatusFilter] = useState(null);const [searchText, setSearchText] = useState("");const [searchTimeout, setSearchTimeout] = useState(null);+ const [autoRefresh, setAutoRefresh] = useState(false);+ const autoRefreshIntervalRef = useRef(null);const filterOptions = ["COMPLETED", "PENDING", "ERROR", "EXECUTING"];+ const TERMINAL_STATES = ["COMPLETED", "ERROR", "STOPPED"];++ // Check if execution is in a terminal state (no more updates expected)+ const isTerminalState =+ executionDetails?.status &&+ TERMINAL_STATES.includes(executionDetails.status.toUpperCase());++ const handleCopyToClipboard = (text, label = "Text") => {+ if (!navigator.clipboard) {+ setAlertDetails({+ type: "error",+ content: "Clipboard not available in this browser",+ });+ return;+ }+ navigator.clipboard.writeText(text).then(+ () => {+ setAlertDetails({+ type: "success",+ content: `${label} copied to clipboard`,+ });+ },+ (err) => {+ setAlertDetails({+ type: "error",+ content: `Failed to copy: ${err.message || "Unknown error"}`,+ });+ }+ );+ };const fetchExecutionDetails = async (id) => {try {@@ -128,17 +210,17 @@ const DetailedLogs = () => {const params = { ...defaultParams, ...customParams };const searchParams = new URLSearchParams();- Object.entries(params).forEach(([key, value]) => {+ for (const [key, value] of Object.entries(params)) {if (value !== null && value !== undefined) {searchParams.append(key, value);}- });+ }// Handle file status filter for MultipleChoiceFilterif (statusFilter && statusFilter.length > 0) {- statusFilter.forEach((status) => {+ for (const status of statusFilter) {searchParams.append("status", status);- });+ }}const response = await axiosPrivate.get(@@ -182,11 +264,16 @@ const DetailedLogs = () => {};}, [searchTimeout]);+ // Column width strategy:+ // - Fixed px for predictable content (timestamps, status, file size, execution time, action)+ // - Percentage for variable content (file name 12%, status message 30%, file path 20%)+ // - Status Message gets most space as it contains the most important variable infoconst columnsDetailedTable = [{title: "Executed At",dataIndex: "executedAt",key: "executedAt",+ width: 140,sorter: true,render: (_, record) => (<Tooltip title={record.executedAtWithSeconds}>@@ -198,6 +285,8 @@ const DetailedLogs = () => {title: "File Name",dataIndex: "fileName",key: "fileName",+ width: "12%",+ ellipsis: true,filterDropdown: () => (<div className="search-container"><Input@@ -216,11 +305,15 @@ const DetailedLogs = () => {title: "Status Message",dataIndex: "statusMessage",key: "statusMessage",+ width: "30%",+ ellipsis: true,+ render: (text) => <StatusMessageCell text={text} />,},{title: "Status",dataIndex: "status",key: "status",+ width: 110,filters: filterOptions.map((filter) => ({text: filter,value: filter,@@ -231,11 +324,8 @@ const DetailedLogs = () => {title: "File Size",dataIndex: "fileSize",key: "fileSize",- },- {- title: "Execution Time",- dataIndex: "executionTime",- key: "executionTime",+ width: 70,+ sorter: true,},...(type !== "API"? [@@ -243,13 +333,23 @@ const DetailedLogs = () => {title: "File Path",dataIndex: "filePath",key: "filePath",+ width: "20%",+ ellipsis: true,},]: []),+ {+ title: "Execution Time",+ dataIndex: "executionTime",+ key: "executionTime",+ width: 90,+ sorter: true,+ },{title: "Action",dataIndex: "action",key: "action",+ width: 60,render: (_, record) => (<Tooltip title="View logs"><Button@@ -261,13 +361,22 @@ const DetailedLogs = () => {},];- const handleTableChange = (pagination, filters, sorter) => {- setPagination((prev) => {- return { ...prev, ...pagination };- });+ const handleTableChange = (newPagination, filters, sorter) => {+ setPagination((prev) => ({+ ...prev,+ current: newPagination.current,+ pageSize: newPagination.pageSize,+ }));if (sorter.order) {- const order = sorter.order === "ascend" ? "created_at" : "-created_at";+ const fieldMap = {+ executedAt: "created_at",+ fileSize: "file_size",+ executionTime: "execution_time",+ };+ const backendField = fieldMap[sorter.field] || sorter.field;+ const order =+ sorter.order === "ascend" ? backendField : `-${backendField}`;setOrdering(order);} else {setOrdering(null);@@ -284,25 +393,44 @@ const DetailedLogs = () => {setSelectedRecord(record);};+ const handleRefresh = () => {+ fetchExecutionDetails(id);+ fetchExecutionFiles(id, pagination.current);+ };+useEffect(() => {fetchExecutionDetails(id);fetchExecutionFiles(id, pagination.current);- }, [pagination.current, ordering, statusFilter]);+ }, [id, pagination.current, pagination.pageSize, ordering, statusFilter]);++ // Auto-disable auto-refresh when execution reaches terminal state+ useEffect(() => {+ if (isTerminalState && autoRefresh) {+ setAutoRefresh(false);+ }+ }, [isTerminalState]);+ // Auto-refresh interval managementuseEffect(() => {- let interval = null;- if (executionDetails?.status === "EXECUTING") {- interval = setInterval(() => {+ if (autoRefreshIntervalRef.current) {+ clearInterval(autoRefreshIntervalRef.current);+ autoRefreshIntervalRef.current = null;+ }++ if (autoRefresh) {+ autoRefreshIntervalRef.current = setInterval(() => {fetchExecutionDetails(id);fetchExecutionFiles(id, pagination.current);- }, 5000);+ }, 30000);}+return () => {- if (interval) {- clearInterval(interval);+ if (autoRefreshIntervalRef.current) {+ clearInterval(autoRefreshIntervalRef.current);+ autoRefreshIntervalRef.current = null;}};- }, [executionDetails?.status, id, pagination.current]);+ }, [autoRefresh, id, pagination.current]);useEffect(() => {const initialColumns = columnsDetailedTable.reduce((acc, col) => {@@ -318,35 +446,65 @@ const DetailedLogs = () => {[key]: !prev[key],}));};- const columnsToShow = columnsDetailedTable.filter(- (col) => columnsVisibility[col.key]- );const menu = {- items: columnsDetailedTable.map((col) => ({- key: col.key,- label: (- <Checkbox- checked={columnsVisibility[col.key]}- onChange={() => handleColumnToggle(col.key)}- >- {col.title}- </Checkbox>- ),- })),+ items: columnsDetailedTable+ .filter((col) => col.key !== "action")+ .map((col) => ({+ key: col.key,+ label: (+ <Checkbox+ checked={columnsVisibility[col.key]}+ onChange={() => handleColumnToggle(col.key)}+ >+ {col.title}+ </Checkbox>+ ),+ })),};+ const columnsToShow = columnsDetailedTable+ .filter((col) => columnsVisibility[col.key])+ .map((col) => {+ if (col.key === "action") {+ return {+ ...col,+ width: 80,+ align: "center",+ title: <ActionColumnHeader menu={menu} />,+ };+ }+ return col;+ });+return (- <>- <Typography.Title className="logs-title" level={4}>- <Button- type="text"- shape="circle"- icon={<ArrowLeftOutlined />}- onClick={() => navigate(`/${sessionDetails?.orgName}/logs`)}+ <div className="detailed-logs-container">+ <div className="detailed-logs-header">+ <Typography.Title className="logs-title" level={4}>+ <Button+ type="text"+ shape="circle"+ icon={<ArrowLeftOutlined />}+ onClick={() => navigate(`/${sessionDetails?.orgName}/logs`)}+ />+ {type} Execution ID {id}+ <Button+ className="copy-btn-outlined"+ icon={<CopyOutlined />}+ onClick={() => handleCopyToClipboard(id, "Execution ID")}+ />+ </Typography.Title>+ <LogsRefreshControls+ autoRefresh={autoRefresh}+ setAutoRefresh={setAutoRefresh}+ onRefresh={handleRefresh}+ disabled={isTerminalState}/>- {type} Session ID - {id}{" "}- </Typography.Title>- <Flex align="center" justify="space-between">+ </div>+ <Flex+ align="center"+ justify="space-between"+ className="detailed-logs-cards"+ ><Flex className="pad-12"><Card className="logs-details-card"><Flex justify="flex-start" align="center">@@ -421,28 +579,28 @@ const DetailedLogs = () => {</Flex><ButtonclassName="view-log-button"- type="link"+ icon={<EyeOutlined />}onClick={() => handleLogsModalOpen({})}>View Logs</Button></Flex>- <div className="settings-layout">- <Dropdown- menu={menu}- trigger={["click"]}- className="column-filter-dropdown"- >- <Button icon={<DownOutlined />}>Show/Hide Columns</Button>- </Dropdown>+ <div className="detailed-logs-table-container"><TabledataSource={executionFiles}columns={columnsToShow}- pagination={{ ...pagination }}+ pagination={{+ ...pagination,+ showSizeChanger: true,+ pageSizeOptions: ["10", "20", "50", "100"],+ showTotal: (total, range) =>+ `${range[0]}-${range[1]} of ${total} files`,+ }}+ bordered+ size="small"loading={loading}onChange={handleTableChange}sortDirections={["ascend", "descend", "ascend"]}- scroll={{ y: 55 * 10 }}/></div><LogModal@@ -452,7 +610,7 @@ const DetailedLogs = () => {setLogDescModalOpen={setLogDescModalOpen}filterParams={{ executionTime: "event_time" }}/>- </>+ </div>);};export { DetailedLogs };diff --git a/frontend/src/components/logging/execution-logs/ExecutionLogs.css b/frontend/src/components/logging/execution-logs/ExecutionLogs.cssindex f1a4f9fad..1b059f550 100644--- a/frontend/src/components/logging/execution-logs/ExecutionLogs.css+++ b/frontend/src/components/logging/execution-logs/ExecutionLogs.css@@ -2,11 +2,12 @@background-color: var(--white);height: 100%;margin: 12px;- overflow-y: auto;+ overflow: hidden;+ display: flex;+ flex-direction: column;}.logs-title {- padding: 15px;margin: 0;}@@ -30,7 +31,7 @@height: 30px;}-.log-tab{+.log-tab {height: 30px;}@@ -47,3 +48,25 @@.log-tab .ant-tabs-ink-bar {top: 40px !important;}++.logs-header {+ flex: 0 0 auto;+ display: flex;+ justify-content: flex-end;+ align-items: center;+ padding: 16px 24px;+}++.logs-filter-controls {+ display: flex;+ align-items: center;+ gap: 10px;+}++.logs-table-container {+ flex: 1;+ min-height: 0;+ padding: 0 24px 8px;+ display: flex;+ flex-direction: column;+}diff --git a/frontend/src/components/logging/execution-logs/ExecutionLogs.jsx b/frontend/src/components/logging/execution-logs/ExecutionLogs.jsxindex f60bba85a..caa186e5a 100644--- a/frontend/src/components/logging/execution-logs/ExecutionLogs.jsx+++ b/frontend/src/components/logging/execution-logs/ExecutionLogs.jsx@@ -1,9 +1,10 @@-import { DatePicker, Flex, Tabs, Typography } from "antd";+import { DatePicker, Tabs } from "antd";import { useNavigate, useParams } from "react-router-dom";-import { useEffect, useState } from "react";+import { useEffect, useState, useRef } from "react";import { LogsTable } from "../logs-table/LogsTable";import { DetailedLogs } from "../detailed-logs/DetailedLogs";+import { LogsRefreshControls } from "../logs-refresh-controls/LogsRefreshControls";import {ApiDeployments,ETLIcon,@@ -43,12 +44,15 @@ function ExecutionLogs() {const [selectedDateRange, setSelectedDateRange] = useState([]);const [datePickerValue, setDatePickerValue] = useState(null);const [ordering, setOrdering] = useState(null);- const [pollingIds, setPollingIds] = useState(new Set());+ const [executionIdSearch, setExecutionIdSearch] = useState("");+ const [autoRefresh, setAutoRefresh] = useState(false);+ const autoRefreshIntervalRef = useRef(null);const currentPath = location.pathname !== `/${sessionDetails?.orgName}/logs`;+const items = [{key: "ETL",- label: "ETL Sessions",+ label: "ETL Executions",icon: (<ETLIconclassName={`log-tab-icon ${activeTab === "ETL" ? "active" : ""}`}@@ -57,7 +61,7 @@ function ExecutionLogs() {},{key: "API",- label: "API Sessions",+ label: "API Executions",icon: (<ApiDeploymentsclassName={`log-tab-icon ${activeTab === "API" ? "active" : ""}`}@@ -66,7 +70,7 @@ function ExecutionLogs() {},{key: "WF",- label: "Workflow Sessions",+ label: "Workflow Executions",icon: (<WorkflowsclassName={`log-tab-icon ${activeTab === "WF" ? "active" : ""}`}@@ -75,7 +79,7 @@ function ExecutionLogs() {},{key: "TASK",- label: "Task Sessions",+ label: "Task Executions",icon: (<TaskclassName={`log-tab-icon ${activeTab === "TASK" ? "active" : ""}`}@@ -94,74 +98,6 @@ function ExecutionLogs() {}};- const pollExecutingRecord = async (id) => {- try {- const url = getUrl(`/execution/${id}/`);- const response = await axiosPrivate.get(url);- const item = response?.data;-- // Update the record in the dataList- setDataList((prevData) => {- const newData = [...prevData];- const index = newData.findIndex((record) => record.key === id);- if (index !== -1) {- const total = item.total_files || 0;- const processed =- (item?.successful_files || 0) + (item?.failed_files || 0);- const progress =- total > 0 ? Math.round((processed / total) * 100) : 0;-- newData[index] = {- ...newData[index],- progress,- processed,- total,- success: item?.status === "COMPLETED",- isError: item?.status === "ERROR",- status: item?.status,- successfulFiles: item?.successful_files,- failedFiles: item?.failed_files,- execution_time: item?.execution_time,- };-- // If status is no longer executing, remove from polling- if (item?.status.toLowerCase() !== "executing") {- setPollingIds((prev) => {- const newSet = new Set(prev);- newSet.delete(id);- return newSet;- });- }- }- return newData;- });-- // Continue polling if still executing- if (item?.status === "EXECUTING") {- setTimeout(() => pollExecutingRecord(id), 5000); // Poll every 5 seconds- }- } catch (err) {- setPollingIds((prev) => {- const newSet = new Set(prev);- newSet.delete(id);- return newSet;- });- }- };-- const startPollingForExecuting = (records) => {- records.forEach((record) => {- if (record.status === "EXECUTING" && !pollingIds.has(record.key)) {- setPollingIds((prev) => {- const newSet = new Set(prev);- newSet.add(record.key);- return newSet;- });- pollExecutingRecord(record.key);- }- });- };-const fetchLogs = async (page) => {try {setLoading(true);@@ -174,6 +110,7 @@ function ExecutionLogs() {created_at_after: selectedDateRange[0] || null,created_at_before: selectedDateRange[1] || null,ordering,+ id: executionIdSearch || null,},});setPagination({@@ -205,7 +142,6 @@ function ExecutionLogs() {};});setDataList(formattedData);- startPollingForExecuting(formattedData);} catch (err) {setAlertDetails(handleException(err));} finally {@@ -215,28 +151,12 @@ function ExecutionLogs() {const customButtons = () => {return (- <Flex gap={10} className="log-controls">- <Tabs- activeKey={activeTab}- items={items}- onChange={onChange}- className="log-tab"- />- <RangePicker- showTime={{ format: "YYYY-MM-DDTHH:mm:ssZ[Z]" }}- value={datePickerValue}- onChange={(value) => {- setDatePickerValue(value);- setSelectedDateRange(- value ? [value[0].toISOString(), value[1].toISOString()] : []- );- }}- onOk={onOk}- className="logs-date-picker"- disabled={currentPath}- format="YYYY-MM-DD HH:mm:ss"- />- </Flex>+ <Tabs+ activeKey={activeTab}+ items={items}+ onChange={onChange}+ className="log-tab"+ />);};@@ -245,12 +165,46 @@ function ExecutionLogs() {setDataList([]);fetchLogs(pagination.current);}- }, [activeTab, pagination.current, selectedDateRange, ordering, currentPath]);+ }, [+ activeTab,+ pagination.current,+ pagination.pageSize,+ selectedDateRange,+ ordering,+ executionIdSearch,+ currentPath,+ ]);++ // Auto-refresh interval management+ useEffect(() => {+ if (autoRefreshIntervalRef.current) {+ clearInterval(autoRefreshIntervalRef.current);+ autoRefreshIntervalRef.current = null;+ }++ if (autoRefresh) {+ autoRefreshIntervalRef.current = setInterval(() => {+ fetchLogs(pagination.current);+ }, 30000);+ }++ return () => {+ if (autoRefreshIntervalRef.current) {+ clearInterval(autoRefreshIntervalRef.current);+ autoRefreshIntervalRef.current = null;+ }+ };+ }, [autoRefresh, pagination.current]);++ // Clear auto-refresh when tab changes+ useEffect(() => {+ setAutoRefresh(false);+ }, [activeTab]);return (<><ToolNavBar- title={"Logs"}+ title={"Execution Logs"}CustomButtons={customButtons}enableSearch={false}/>@@ -259,10 +213,31 @@ function ExecutionLogs() {<DetailedLogs />) : (<>- <Typography.Title className="logs-title" level={4}>- Execution Logs- </Typography.Title>- <div className="settings-layout">+ <div className="logs-header">+ <div className="logs-filter-controls">+ <RangePicker+ showTime={{ format: "YYYY-MM-DDTHH:mm:ssZ[Z]" }}+ value={datePickerValue}+ onChange={(value) => {+ setDatePickerValue(value);+ setSelectedDateRange(+ value?.[0] && value?.[1]+ ? [value[0].toISOString(), value[1].toISOString()]+ : []+ );+ }}+ onOk={onOk}+ className="logs-date-picker"+ format="YYYY-MM-DD HH:mm:ss"+ />+ <LogsRefreshControls+ autoRefresh={autoRefresh}+ setAutoRefresh={setAutoRefresh}+ onRefresh={() => fetchLogs(pagination.current)}+ />+ </div>+ </div>+ <div className="logs-table-container"><LogsTabletableData={dataList}loading={loading}@@ -270,6 +245,8 @@ function ExecutionLogs() {setPagination={setPagination}setOrdering={setOrdering}activeTab={activeTab}+ executionIdSearch={executionIdSearch}+ setExecutionIdSearch={setExecutionIdSearch}/></div></>diff --git a/frontend/src/components/logging/logs-refresh-controls/LogsRefreshControls.css b/frontend/src/components/logging/logs-refresh-controls/LogsRefreshControls.cssnew file mode 100644index 000000000..1ca0e0add--- /dev/null+++ b/frontend/src/components/logging/logs-refresh-controls/LogsRefreshControls.css@@ -0,0 +1,18 @@+.logs-refresh-controls {+ display: flex;+ align-items: center;+ gap: 8px;+}++.logs-auto-refresh-label {+ font-size: 13px;+ color: #8c8c8c;+}++.logs-refresh-btn {+ height: 30px;+}++.logs-refresh-controls.disabled .logs-auto-refresh-label {+ color: #bfbfbf;+}diff --git a/frontend/src/components/logging/logs-refresh-controls/LogsRefreshControls.jsx b/frontend/src/components/logging/logs-refresh-controls/LogsRefreshControls.jsxnew file mode 100644index 000000000..74ea1599f--- /dev/null+++ b/frontend/src/components/logging/logs-refresh-controls/LogsRefreshControls.jsx@@ -0,0 +1,44 @@+import { Switch, Button, Typography, Tooltip } from "antd";+import { ReloadOutlined } from "@ant-design/icons";+import PropTypes from "prop-types";+import "./LogsRefreshControls.css";++function LogsRefreshControls({+ autoRefresh,+ setAutoRefresh,+ onRefresh,+ disabled = false,+}) {+ return (+ <Tooltip title={disabled ? "Execution has completed" : ""}>+ <div className={`logs-refresh-controls ${disabled ? "disabled" : ""}`}>+ <Typography.Text className="logs-auto-refresh-label">+ Auto-refresh (30s)+ </Typography.Text>+ <Switch+ size="small"+ checked={autoRefresh}+ onChange={setAutoRefresh}+ disabled={disabled}+ />+ <Button+ icon={<ReloadOutlined />}+ onClick={onRefresh}+ className="logs-refresh-btn"+ disabled={disabled}+ >+ Refresh+ </Button>+ </div>+ </Tooltip>+ );+}++LogsRefreshControls.propTypes = {+ autoRefresh: PropTypes.bool.isRequired,+ setAutoRefresh: PropTypes.func.isRequired,+ onRefresh: PropTypes.func.isRequired,+ disabled: PropTypes.bool,+};++export { LogsRefreshControls };diff --git a/frontend/src/components/logging/logs-table/LogsTable.css b/frontend/src/components/logging/logs-table/LogsTable.cssindex 88fc1f854..407766768 100644--- a/frontend/src/components/logging/logs-table/LogsTable.css+++ b/frontend/src/components/logging/logs-table/LogsTable.css@@ -1,14 +1,60 @@-.title-name-redirect{+.title-name-redirect {word-break: break-all;}+.logs-table-container .ant-table-wrapper {+ flex: 1;+ min-height: 0;+ display: flex;+ flex-direction: column;+}++.logs-table-container .ant-spin-nested-loading,+.logs-table-container .ant-spin-container {+ flex: 1;+ min-height: 0;+ display: flex;+ flex-direction: column;+}++.logs-table-container .ant-table {+ flex: 1;+ min-height: 0;+ display: flex;+ flex-direction: column;+ overflow: auto;+}++.logs-table-container .ant-table-container {+ flex: 1;+ min-height: 0;+ display: flex;+ flex-direction: column;+}++.logs-table-container .ant-table-body {+ flex: 1;+ overflow-y: auto !important;+}++.logs-table-container .ant-table-thead > tr > th {+ position: sticky;+ top: 0;+ z-index: 1;+ background: #fafafa;+}++.logs-table-container .ant-pagination {+ margin-top: 8px;+ padding-bottom: 0;+}.gen-index-progress {- color: #FAAD14;+ color: #faad14;}.gen-index-success {- color: #52C41A;+ color: #52c41a;}.gen-index-fail {@@ -19,3 +65,17 @@display: inline-block;margin-right: 10px;}++.search-container {+ padding: 8px;+}++.search-input {+ width: 188px;+ margin-bottom: 8px;+ display: block;+}++.search-filter-icon-active {+ color: #1890ff;+}diff --git a/frontend/src/components/logging/logs-table/LogsTable.jsx b/frontend/src/components/logging/logs-table/LogsTable.jsxindex 639905de9..9273ce8e5 100644--- a/frontend/src/components/logging/logs-table/LogsTable.jsx+++ b/frontend/src/components/logging/logs-table/LogsTable.jsx@@ -1,4 +1,4 @@-import { Table, Tooltip, Typography } from "antd";+import { Input, Table, Tooltip, Typography } from "antd";import "./LogsTable.css";import { useNavigate } from "react-router-dom";import PropTypes from "prop-types";@@ -6,12 +6,39 @@ import {CloseCircleFilled,HourglassOutlined,InfoCircleFilled,+ SearchOutlined,} from "@ant-design/icons";import { EmptyState } from "../../widgets/empty-state/EmptyState";import { logsStaticContent } from "../../../helpers/GetStaticData";import { useSessionStore } from "../../../store/session-store";+// Search filter dropdown component for execution ID column+const SearchFilterDropdown = ({ value, onChange }) => (+ <div className="search-container">+ <Input+ placeholder="Search execution ID"+ value={value}+ onChange={(e) => onChange(e.target.value)}+ className="search-input"+ />+ </div>+);++SearchFilterDropdown.propTypes = {+ value: PropTypes.string,+ onChange: PropTypes.func.isRequired,+};++// Search filter icon component+const SearchFilterIcon = ({ isActive }) => (+ <SearchOutlined className={isActive ? "search-filter-icon-active" : ""} />+);++SearchFilterIcon.propTypes = {+ isActive: PropTypes.bool,+};+const LogsTable = ({tableData,loading,@@ -19,6 +46,8 @@ const LogsTable = ({setPagination,setOrdering,activeTab,+ executionIdSearch,+ setExecutionIdSearch,}) => {const navigate = useNavigate();const { sessionDetails } = useSessionStore();@@ -39,6 +68,13 @@ const LogsTable = ({title: "Execution ID",dataIndex: "executionId",key: "executionId",+ filterDropdown: (+ <SearchFilterDropdown+ value={executionIdSearch}+ onChange={setExecutionIdSearch}+ />+ ),+ filterIcon: <SearchFilterIcon isActive={!!executionIdSearch} />,render: (text) => (<Typography.LinkclassName="title-name-redirect"@@ -101,30 +137,36 @@ const LogsTable = ({title: "Files Processed",dataIndex: "filesProcessed",key: "filesProcessed",- render: (text, record) => `${record?.processed}/${record?.totalFiles}`,+ render: (_, record) => `${record?.processed}/${record?.totalFiles}`,},{title: "Execution Time",dataIndex: "executionTime",key: "executionTime",+ sorter: true,render: (_, record) => record?.execution_time || "-",},];- const handleTableChange = (pagination, filters, sorter) => {+ const handleTableChange = (pagination, _filters, sorter) => {setPagination((prev) => {return { ...prev, ...pagination };});if (sorter.order) {- // Determine ascending or descending order- const order = sorter.order === "ascend" ? "created_at" : "-created_at";+ const fieldMap = {+ executedAt: "created_at",+ executionTime: "execution_time",+ };+ const backendField = fieldMap[sorter.field] || sorter.field;+ const order =+ sorter.order === "ascend" ? backendField : `-${backendField}`;setOrdering(order);setPagination((prev) => {return { ...prev, ...pagination, current: 1 };});} else {- setOrdering(null); // Default ordering if sorting is cleared+ setOrdering(null);}};@@ -132,13 +174,18 @@ const LogsTable = ({<Tablecolumns={columns}dataSource={tableData}- pagination={pagination}+ pagination={{+ ...pagination,+ showSizeChanger: true,+ pageSizeOptions: ["10", "20", "50", "100"],+ showTotal: (total, range) =>+ `${range[0]}-${range[1]} of ${total} executions`,+ }}borderedsize="small"loading={loading}onChange={handleTableChange}sortDirections={["ascend", "descend", "ascend"]}- scroll={{ y: 55 * 10 }}locale={{emptyText: (<EmptyState@@ -163,6 +210,8 @@ LogsTable.propTypes = {setPagination: PropTypes.func,setOrdering: PropTypes.func,activeTab: PropTypes.string,+ executionIdSearch: PropTypes.string,+ setExecutionIdSearch: PropTypes.func,};export { LogsTable };diff --git a/frontend/src/store/alert-store.js b/frontend/src/store/alert-store.jsindex 95dec9b79..ea58005f1 100644--- a/frontend/src/store/alert-store.js+++ b/frontend/src/store/alert-store.js@@ -4,6 +4,7 @@ import { uniqueId } from "lodash";import { isNonNegativeNumber } from "../helpers/GetStaticData";const DEFAULT_DURATION = 6;+const SUCCESS_DURATION = 2;const STORE_VARIABLES = {alertDetails: {@@ -21,12 +22,14 @@ const useAlertStore = create((setState) => ({if (!details) return STORE_VARIABLES;const isErrorType = details?.type === "error";+ const isSuccessType = details?.type === "success";+ const defaultDuration = isSuccessType ? SUCCESS_DURATION : DEFAULT_DURATION;const updatedDetails = {...details,title: details.title || (isErrorType ? "Failed" : "Success"),duration: isNonNegativeNumber(details.duration)? details.duration- : DEFAULT_DURATION,+ : defaultDuration,key: `open${Date.now()}-${uniqueId()}`,};diff --git a/unstract/workflow-execution/src/unstract/workflow_execution/execution_file_handler.py b/unstract/workflow-execution/src/unstract/workflow_execution/execution_file_handler.pyindex ecf688686..c3c593971 100644--- a/unstract/workflow-execution/src/unstract/workflow_execution/execution_file_handler.py+++ b/unstract/workflow-execution/src/unstract/workflow_execution/execution_file_handler.py@@ -168,15 +168,16 @@ class ExecutionFileHandler:# Add custom_data to metadata if providedif custom_data:+ logger.info(+ f"Custom data is provided for file_execution_id {file_execution_id}"+ )content[MetaDataKey.CUSTOM_DATA] = custom_datafile_system = FileSystem(FileStorageType.WORKFLOW_EXECUTION)file_storage = file_system.get_file_storage()file_storage.json_dump(path=metadata_path, data=content)- logger.info(- f"metadata for {input_file_path} is " "added in to execution directory"- )+ logger.info(f"metadata for {input_file_path} is added in to execution directory")def _get_file_execution_dir(self) -> str | None:"""Get the directory path for a specific file execution.diff --git a/workers/shared/workflow/execution/context.py b/workers/shared/workflow/execution/context.pyindex 5fbdcd7f6..53e133480 100644--- a/workers/shared/workflow/execution/context.py+++ b/workers/shared/workflow/execution/context.py@@ -170,6 +170,7 @@ class WorkerExecutionContext:"password" in key.lower()or "secret" in key.lower()or "token" in key.lower()+ or "custom_data" in key.lower()):value = "***REDACTED***"log_parts.append(f"{key}={value}")