LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
UN-3770 [FEAT] Opt-in pagination for list endpoints; wire Workflows page (#2187)
Chandrasekharan Meda02f45f9e6
diff --git a/backend/adapter_processor_v2/views.py b/backend/adapter_processor_v2/views.pyindex 41897f4b8..7dfacf443 100644--- a/backend/adapter_processor_v2/views.py+++ b/backend/adapter_processor_v2/views.py@@ -28,6 +28,7 @@ from rest_framework.viewsets import GenericViewSet, ModelViewSetfrom tenant_account_v2.organization_member_service import OrganizationMemberServicefrom tool_instance_v2.models import ToolInstancefrom utils.filtering import FilterHelper+from utils.pagination import OptionalPaginationfrom utils.user_context import UserContextfrom adapter_processor_v2.adapter_processor import AdapterProcessor@@ -146,6 +147,7 @@ class AdapterInstanceViewSet(OwnerManagementMixin, ResourceShareManagementMixin, ModelViewSet):serializer_class = AdapterInstanceSerializer+ pagination_class = OptionalPaginationnotification_resource_name_field = "adapter_name"def get_notification_resource_type(self, resource: Any) -> str | None:@@ -187,7 +189,15 @@ class AdapterInstanceViewSet(constant.ADAPTER_NAME,):queryset = queryset.filter(**filter_args)- return queryset++ search = self.request.query_params.get("search")+ if search:+ queryset = queryset.filter(adapter_name__icontains=search)++ # Order by the DISTINCT ON field so pagination is deterministic and the+ # admin/service branch (no distinct) is ordered too. Not modified_at:+ # that would conflict with the DISTINCT ON in for_user().+ return queryset.order_by("id")def get_serializer_class(self,diff --git a/backend/connector_v2/views.py b/backend/connector_v2/views.pyindex 2ff7a18f3..c4a3741f2 100644--- a/backend/connector_v2/views.py+++ b/backend/connector_v2/views.py@@ -21,6 +21,7 @@ from rest_framework.response import Responsefrom rest_framework.versioning import URLPathVersioningfrom tenant_account_v2.organization_member_service import OrganizationMemberServicefrom utils.filtering import FilterHelper+from utils.pagination import OptionalPaginationfrom utils.user_context import UserContextfrom backend.constants import RequestKey@@ -45,6 +46,7 @@ class ConnectorInstanceViewSet():versioning_class = URLPathVersioningserializer_class = ConnectorInstanceSerializer+ pagination_class = OptionalPaginationnotification_resource_name_field = "connector_name"def get_notification_resource_type(self, resource: Any) -> str | None:@@ -103,6 +105,10 @@ class ConnectorInstanceViewSet(if filter_args:queryset = queryset.filter(**filter_args)+ search = self.request.query_params.get("search")+ if search:+ queryset = queryset.filter(connector_name__icontains=search)+# Filter by connector_modeconnector_mode_param = self.request.query_params.get("connector_mode")if connector_mode_param:@@ -121,7 +127,10 @@ class ConnectorInstanceViewSet()queryset = queryset.none()- return queryset+ # Order by the DISTINCT ON field so pagination is deterministic and the+ # admin/service branch (no distinct) is ordered too. Not modified_at:+ # that would conflict with the DISTINCT ON in for_user().+ return queryset.order_by("id")def _get_connector_metadata(self, connector_id: str) -> dict[str, str] | None:"""Gets connector metadata for the ConnectorInstance.diff --git a/backend/prompt_studio/prompt_studio_core_v2/views.py b/backend/prompt_studio/prompt_studio_core_v2/views.pyindex 7faf3a23c..c7251278e 100644--- a/backend/prompt_studio/prompt_studio_core_v2/views.py+++ b/backend/prompt_studio/prompt_studio_core_v2/views.py@@ -31,6 +31,7 @@ from rest_framework.versioning import URLPathVersioningfrom tool_instance_v2.models import ToolInstancefrom utils.file_storage.helpers.prompt_studio_file_helper import PromptStudioFileHelperfrom utils.hubspot_notify import notify_hubspot_event+from utils.pagination import OptionalPaginationfrom utils.user_context import UserContextfrom utils.user_session import UserSessionUtilsfrom workflow_manager.endpoint_v2.models import WorkflowEndpoint@@ -128,6 +129,7 @@ class PromptStudioCoreView("""Viewset to handle all Custom tool related operations."""versioning_class = URLPathVersioning+ pagination_class = OptionalPaginationserializer_class = CustomToolSerializernotification_resource_name_field = "tool_name"@@ -172,7 +174,12 @@ class PromptStudioCoreView(qs = qs.select_related("created_by").annotate(_prompt_count=Subquery(prompt_count_sq),)- return qs+ search = self.request.query_params.get("search")+ if search:+ qs = qs.filter(tool_name__icontains=search)+ # Order by the DISTINCT ON field so pagination is deterministic and the+ # admin/service branch (no distinct) is ordered too.+ return qs.order_by("tool_id")def get_object(self):"""Override get_object to trigger lazy migration when accessing tools."""diff --git a/backend/utils/pagination.py b/backend/utils/pagination.pyindex 2f87a93bb..be1c6765c 100644--- a/backend/utils/pagination.py+++ b/backend/utils/pagination.py@@ -7,3 +7,23 @@ class CustomPagination(PageNumberPagination):page_size = Pagination.PAGE_SIZEpage_size_query_param = Pagination.PAGE_SIZE_QUERY_PARAMmax_page_size = Pagination.MAX_PAGE_SIZE+++class OptionalPagination(CustomPagination):+ """Paginate only when the caller opts in via ?page / ?page_size.++ These list endpoints are shared: besides the listing page they feed+ dropdowns/selectors that expect a bare array. Returning None keeps that+ response untouched for callers that don't ask for a page, while the+ listing page opts in and gets the {count, next, previous, results} envelope.+ """++ def paginate_queryset(self, queryset, request, view=None):+ params = request.query_params+ # Blank values (?page=) count as not opting in, so shared callers keep+ # their bare-array response instead of getting an enveloped page.+ if not params.get(self.page_query_param) and not params.get(+ self.page_size_query_param+ ):+ return None+ return super().paginate_queryset(queryset, request, view=view)diff --git a/backend/utils/tests/test_pagination.py b/backend/utils/tests/test_pagination.pynew file mode 100644index 000000000..5bab48815--- /dev/null+++ b/backend/utils/tests/test_pagination.py@@ -0,0 +1,81 @@+"""Contract tests for utils.pagination.OptionalPagination.++The four listing endpoints it backs (workflows, prompt studio, adapters,+connectors) are shared with dropdown/selector consumers that expect a bare+array. The opt-in behaviour below is what keeps those consumers regression-free:+no page/page_size param -> paginate_queryset returns None -> DRF serialises the+bare list; only an explicit opt-in gets the {count, next, previous, results}+envelope.++Deliberately DB-free and Django-settings-free (a hand-rolled request stub, no+APIRequestFactory) so it runs in the rig's unit tier.+"""++from __future__ import annotations++from utils.pagination import OptionalPagination++QUERYSET = list(range(1, 101)) # 100 sliceable items stand in for a queryset+++class _Req:+ """Minimal stand-in exposing the surface DRF pagination touches."""++ def __init__(self, params: dict[str, str]):+ self.query_params = params++ def build_absolute_uri(self) -> str:+ return "https://testserver/things/"+++class TestOptionalPagination:+ def test_no_params_returns_none(self):+ """No opt-in -> None, so callers keep their bare-array response."""+ assert OptionalPagination().paginate_queryset(QUERYSET, _Req({})) is None++ def test_unrelated_params_do_not_trigger_pagination(self):+ req = _Req({"adapter_type": "LLM", "search": "foo"})+ assert OptionalPagination().paginate_queryset(QUERYSET, req) is None++ def test_blank_params_do_not_trigger_pagination(self):+ """Empty ?page= / ?page_size= keep the bare-array response."""+ req = _Req({"page": "", "page_size": ""})+ assert OptionalPagination().paginate_queryset(QUERYSET, req) is None++ def test_single_blank_param_does_not_trigger_pagination(self):+ for params in ({"page": ""}, {"page_size": ""}):+ assert OptionalPagination().paginate_queryset(QUERYSET, _Req(params)) is None++ def test_blank_page_with_explicit_page_size_serves_first_page(self):+ """A blank ?page= alongside a real ?page_size= is page 1, not an error.++ DRF reads the page number as `query_params.get("page") or 1`, so a blank+ value falls back to the first page instead of raising NotFound.+ """+ page = OptionalPagination().paginate_queryset(+ QUERYSET, _Req({"page": "", "page_size": "10"})+ )+ assert page == list(range(1, 11))++ def test_blank_page_size_with_explicit_page_uses_default_size(self):+ page = OptionalPagination().paginate_queryset(+ QUERYSET, _Req({"page": "2", "page_size": ""})+ )+ assert page == list(range(51, 101))++ def test_page_param_opts_in(self):+ paginator = OptionalPagination()+ page = paginator.paginate_queryset(QUERYSET, _Req({"page": "1"}))+ assert page == list(range(1, 51)) # default page_size == 50+ assert paginator.page.paginator.count == 100++ def test_page_size_param_alone_opts_in(self):+ page = OptionalPagination().paginate_queryset(QUERYSET, _Req({"page_size": "10"}))+ assert page == list(range(1, 11))++ def test_page_size_capped_at_max(self):+ page = OptionalPagination().paginate_queryset(+ QUERYSET, _Req({"page_size": "100000"})+ )+ # max_page_size (1000) caps the request; all 100 rows fit in one page+ assert page == QUERYSETdiff --git a/backend/workflow_manager/workflow_v2/views.py b/backend/workflow_manager/workflow_v2/views.pyindex 0db26a850..ef9f52f95 100644--- a/backend/workflow_manager/workflow_v2/views.py+++ b/backend/workflow_manager/workflow_v2/views.py@@ -22,6 +22,7 @@ from rest_framework.versioning import URLPathVersioningfrom rest_framework.views import APIViewfrom utils.filtering import FilterHelperfrom utils.organization_utils import filter_queryset_by_organization, resolve_organization+from utils.pagination import OptionalPaginationfrom backend.constants import RequestKeyfrom unstract.core.data_models import FileHistoryCreateRequest@@ -75,6 +76,7 @@ class WorkflowViewSet(OwnerManagementMixin, ResourceShareManagementMixin, viewsets.ModelViewSet):versioning_class = URLPathVersioning+ pagination_class = OptionalPaginationnotification_resource_name_field = "workflow_name"def get_notification_resource_type(self, resource: Any) -> str | None:@@ -112,11 +114,18 @@ class WorkflowViewSet(queryset = queryset.select_related("created_by").prefetch_related("memberships__user")++ search = self.request.query_params.get("search")+ if search:+ queryset = queryset.filter(workflow_name__icontains=search)++ # `id` tiebreaker keeps ordering deterministic across paginated requests+ # (the for_user() manager uses plain .distinct(), so there is no default)order_by = self.request.query_params.get("order_by")- if order_by == "desc":- queryset = queryset.order_by("-modified_at")- elif order_by == "asc":- queryset = queryset.order_by("modified_at")+ if order_by == "asc":+ queryset = queryset.order_by("modified_at", "id")+ else:+ queryset = queryset.order_by("-modified_at", "id")return querysetdiff --git a/frontend/src/components/workflows/workflow/Workflows.css b/frontend/src/components/workflows/workflow/Workflows.cssindex 22d6ec9e9..7a407a2fe 100644--- a/frontend/src/components/workflows/workflow/Workflows.css+++ b/frontend/src/components/workflows/workflow/Workflows.css@@ -85,3 +85,9 @@flex: 1;overflow-y: auto;}++.workflows-pagination {+ display: flex;+ justify-content: flex-end;+ padding: 12px 16px;+}diff --git a/frontend/src/components/workflows/workflow/Workflows.jsx b/frontend/src/components/workflows/workflow/Workflows.jsxindex 4160cde4f..7d5567c36 100644--- a/frontend/src/components/workflows/workflow/Workflows.jsx+++ b/frontend/src/components/workflows/workflow/Workflows.jsx@@ -1,12 +1,12 @@import { PlusOutlined, UserOutlined } from "@ant-design/icons";-import { Typography } from "antd";-import isEmpty from "lodash/isEmpty";+import { Pagination, Typography } from "antd";import PropTypes from "prop-types";import { useCallback, useEffect, useRef, useState } from "react";import { useLocation, useNavigate } from "react-router-dom";import { useCoOwnerManagement } from "../../../hooks/useCoOwnerManagement.jsx";import { useExceptionHandler } from "../../../hooks/useExceptionHandler.jsx";+import { usePaginatedList } from "../../../hooks/usePaginatedList";import usePostHogEvents from "../../../hooks/usePostHogEvents.js";import {useInitialFetchCount,@@ -32,6 +32,8 @@ import "./Workflows.css";const { Text } = Typography;+const DEFAULT_PAGE_SIZE = 10;+function Workflows() {const navigate = useNavigate();const location = useLocation();@@ -51,7 +53,8 @@ function Workflows() {const [editingProject, setEditProject] = useState();const [loading, setLoading] = useState(false);const [openModal, toggleModal] = useState(true);- const projectListRef = useRef();+ // Ref forwards the fetch fn to the pagination hook (avoids declaration ordering)+ const fetchListRef = useRef(null);const [backendErrors, setBackendErrors] = useState(null);const [shareOpen, setShareOpen] = useState(false);const [selectedWorkflow, setSelectedWorkflow] = useState();@@ -61,10 +64,27 @@ function Workflows() {const [allGroups, setAllGroups] = useState([]);const { setAlertDetails } = useAlertStore();++ const {+ pagination,+ setPagination,+ searchTerm,+ handlePaginationChange,+ handleSearch,+ } = usePaginatedList({+ fetchData: (...args) => fetchListRef.current?.(...args),+ defaultPageSize: DEFAULT_PAGE_SIZE,+ });++ // Refresh the current page (preserves page + active search) after mutationsconst handleListRefresh = useCallback(- () => getProjectList(),- // eslint-disable-next-line react-hooks/exhaustive-deps- [],+ () =>+ fetchListRef.current?.(+ pagination.current,+ pagination.pageSize,+ searchTerm,+ ),+ [pagination.current, pagination.pageSize, searchTerm],);const {coOwnerOpen,@@ -91,28 +111,47 @@ function Workflows() {}}, [location.pathname]);- function getProjectList() {+ const getProjectList = (+ page = 1,+ pageSize = DEFAULT_PAGE_SIZE,+ search = "",+ ) => {+ setLoading(true);+ const params = { page, page_size: pageSize };+ if (search) {+ params.search = search;+ }projectApiService- .getProjectList()+ .getProjectList(params).then((res) => {- projectListRef.current = res.data;- setProjectList(res.data);+ const data = res?.data;+ // Endpoint is opt-in paginated: envelope when we send ?page, else a+ // bare array. Handle both so nothing breaks if the opt-in is dropped.+ const results = data?.results ?? data ?? [];+ const total = data?.count ?? results.length;+ // Deleting the last row on a page leaves it empty; step back a page.+ if (results.length === 0 && page > 1 && total > 0) {+ getProjectList(page - 1, pageSize, search);+ return;+ }+ setProjectList(results);+ setPagination((prev) => ({+ ...prev,+ current: page,+ pageSize,+ total,+ }));}).catch(() => {console.error("Unable to get project list");+ // Avoid an indefinite spinner when the first fetch fails.+ setProjectList((prev) => prev ?? []);+ })+ .finally(() => {+ setLoading(false);});- }-- function onSearch(searchText, setSearchList) {- if (!searchText.trim()) {- setSearchList(projectListRef.current);- return;- }- const filteredList = projectListRef.current.filter((item) =>- item.workflow_name.toLowerCase().includes(searchText.toLowerCase()),- );- setSearchList(filteredList);- }+ };+ fetchListRef.current = getProjectList;function editProject(name, description) {setLoading(true);@@ -121,7 +160,7 @@ function Workflows() {.then((res) => {closeNewProject();if (editingProject?.name) {- getProjectList();+ handleListRefresh();} else {openProject(res.data);}@@ -129,7 +168,6 @@ function Workflows() {type: "success",content: "Workflow updated successfully",});- getProjectList();}).catch((err) => {setAlertDetails(@@ -220,7 +258,7 @@ function Workflows() {projectApiService.deleteProject(project.id).then(() => {- getProjectList();+ handleListRefresh();setAlertDetails({type: "success",content: "Workflow deleted successfully",@@ -307,7 +345,7 @@ function Workflows() {type: "success",content: "Workflow sharing updated successfully",});- getProjectList();+ handleListRefresh();// Close only on success; keep the modal open on failure so the user// can see the rejected entries and retry.setShareOpen(false);@@ -363,15 +401,13 @@ function Workflows() {<ToolNavBartitle="Workflows"enableSearch- searchList={projectList}- setSearchList={setProjectList}customButtons={newWorkflowButton}- onSearch={onSearch}+ onSearch={(value) => handleSearch(value)}/><div className="workflows-pg-layout"><div className="workflows-pg-body">- {!projectListRef.current && <SpinnerLoader />}- {projectListRef.current && isEmpty(projectListRef?.current) && (+ {projectList === undefined && <SpinnerLoader />}+ {projectList?.length === 0 && !searchTerm && (<div className="list-of-workflows-body"><EmptyStatetext="No Workflow available"@@ -383,24 +419,37 @@ function Workflows() {/></div>)}- {isEmpty(projectList) && !isEmpty(projectListRef?.current) && (+ {projectList?.length === 0 && searchTerm && (<EmptyState text="No results found for this search" />)}- {!isEmpty(projectList) && (- <ViewTools- isLoading={loading}- isEmpty={!projectList?.length}- listOfTools={projectList}- setOpenAddTool={toggleModal}- handleEdit={updateProject}- handleDelete={deleteProject}- handleShare={handleShare}- handleCoOwner={handleCoOwner}- titleProp="workflow_name"- descriptionProp="description"- idProp="id"- type="Workflow"- />+ {projectList?.length > 0 && (+ <>+ <ViewTools+ isLoading={loading}+ isEmpty={false}+ listOfTools={projectList}+ setOpenAddTool={toggleModal}+ handleEdit={updateProject}+ handleDelete={deleteProject}+ handleShare={handleShare}+ handleCoOwner={handleCoOwner}+ titleProp="workflow_name"+ descriptionProp="description"+ idProp="id"+ type="Workflow"+ />+ {pagination.total > pagination.pageSize && (+ <div className="workflows-pagination">+ <Pagination+ current={pagination.current}+ pageSize={pagination.pageSize}+ total={pagination.total}+ onChange={handlePaginationChange}+ showSizeChanger={false}+ />+ </div>+ )}+ </>)}{editingProject && (<LazyLoaderdiff --git a/frontend/src/components/workflows/workflow/workflow-service.js b/frontend/src/components/workflows/workflow/workflow-service.jsindex d5cce6307..9453aedc4 100644--- a/frontend/src/components/workflows/workflow/workflow-service.js+++ b/frontend/src/components/workflows/workflow/workflow-service.js@@ -24,8 +24,7 @@ function workflowService() {};return axiosPrivate(options);},- getProjectList: (myProjects = false) => {- const params = myProjects ? { created_by: sessionDetails?.id } : {};+ getProjectList: (params = {}) => {options = {url: `${path}/workflow/`,method: "GET",