Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

UN-3770 [FEAT] Opt-in pagination for list endpoints; wire Workflows page (#2187)

Chandrasekharan Meda02f45f9e6
diff --git a/backend/adapter_processor_v2/views.py b/backend/adapter_processor_v2/views.py
index 41897f4b8..7dfacf443 100644
--- a/backend/adapter_processor_v2/views.py
+++ b/backend/adapter_processor_v2/views.py
@@ -28,6 +28,7 @@ from rest_framework.viewsets import GenericViewSet, ModelViewSet
from tenant_account_v2.organization_member_service import OrganizationMemberService
from tool_instance_v2.models import ToolInstance
from utils.filtering import FilterHelper
+from utils.pagination import OptionalPagination
from utils.user_context import UserContext
from adapter_processor_v2.adapter_processor import AdapterProcessor
@@ -146,6 +147,7 @@ class AdapterInstanceViewSet(
OwnerManagementMixin, ResourceShareManagementMixin, ModelViewSet
):
serializer_class = AdapterInstanceSerializer
+ pagination_class = OptionalPagination
notification_resource_name_field = "adapter_name"
def get_notification_resource_type(self, resource: Any) -> str | None:
@@ -187,7 +189,15 @@ class AdapterInstanceViewSet(
constant.ADAPTER_NAME,
):
queryset = queryset.filter(**filter_args)
- return queryset
+
+ search = self.request.query_params.get("search")
+ if search:
+ queryset = queryset.filter(adapter_name__icontains=search)
+
+ # Order by the DISTINCT ON field so pagination is deterministic and the
+ # admin/service branch (no distinct) is ordered too. Not modified_at:
+ # that would conflict with the DISTINCT ON in for_user().
+ return queryset.order_by("id")
def get_serializer_class(
self,
diff --git a/backend/connector_v2/views.py b/backend/connector_v2/views.py
index 2ff7a18f3..c4a3741f2 100644
--- a/backend/connector_v2/views.py
+++ b/backend/connector_v2/views.py
@@ -21,6 +21,7 @@ from rest_framework.response import Response
from rest_framework.versioning import URLPathVersioning
from tenant_account_v2.organization_member_service import OrganizationMemberService
from utils.filtering import FilterHelper
+from utils.pagination import OptionalPagination
from utils.user_context import UserContext
from backend.constants import RequestKey
@@ -45,6 +46,7 @@ class ConnectorInstanceViewSet(
):
versioning_class = URLPathVersioning
serializer_class = ConnectorInstanceSerializer
+ pagination_class = OptionalPagination
notification_resource_name_field = "connector_name"
def get_notification_resource_type(self, resource: Any) -> str | None:
@@ -103,6 +105,10 @@ class ConnectorInstanceViewSet(
if filter_args:
queryset = queryset.filter(**filter_args)
+ search = self.request.query_params.get("search")
+ if search:
+ queryset = queryset.filter(connector_name__icontains=search)
+
# Filter by connector_mode
connector_mode_param = self.request.query_params.get("connector_mode")
if connector_mode_param:
@@ -121,7 +127,10 @@ class ConnectorInstanceViewSet(
)
queryset = queryset.none()
- return queryset
+ # Order by the DISTINCT ON field so pagination is deterministic and the
+ # admin/service branch (no distinct) is ordered too. Not modified_at:
+ # that would conflict with the DISTINCT ON in for_user().
+ return queryset.order_by("id")
def _get_connector_metadata(self, connector_id: str) -> dict[str, str] | None:
"""Gets connector metadata for the ConnectorInstance.
diff --git a/backend/prompt_studio/prompt_studio_core_v2/views.py b/backend/prompt_studio/prompt_studio_core_v2/views.py
index 7faf3a23c..c7251278e 100644
--- a/backend/prompt_studio/prompt_studio_core_v2/views.py
+++ b/backend/prompt_studio/prompt_studio_core_v2/views.py
@@ -31,6 +31,7 @@ from rest_framework.versioning import URLPathVersioning
from tool_instance_v2.models import ToolInstance
from utils.file_storage.helpers.prompt_studio_file_helper import PromptStudioFileHelper
from utils.hubspot_notify import notify_hubspot_event
+from utils.pagination import OptionalPagination
from utils.user_context import UserContext
from utils.user_session import UserSessionUtils
from workflow_manager.endpoint_v2.models import WorkflowEndpoint
@@ -128,6 +129,7 @@ class PromptStudioCoreView(
"""Viewset to handle all Custom tool related operations."""
versioning_class = URLPathVersioning
+ pagination_class = OptionalPagination
serializer_class = CustomToolSerializer
notification_resource_name_field = "tool_name"
@@ -172,7 +174,12 @@ class PromptStudioCoreView(
qs = qs.select_related("created_by").annotate(
_prompt_count=Subquery(prompt_count_sq),
)
- return qs
+ search = self.request.query_params.get("search")
+ if search:
+ qs = qs.filter(tool_name__icontains=search)
+ # Order by the DISTINCT ON field so pagination is deterministic and the
+ # admin/service branch (no distinct) is ordered too.
+ return qs.order_by("tool_id")
def get_object(self):
"""Override get_object to trigger lazy migration when accessing tools."""
diff --git a/backend/utils/pagination.py b/backend/utils/pagination.py
index 2f87a93bb..be1c6765c 100644
--- a/backend/utils/pagination.py
+++ b/backend/utils/pagination.py
@@ -7,3 +7,23 @@ class CustomPagination(PageNumberPagination):
page_size = Pagination.PAGE_SIZE
page_size_query_param = Pagination.PAGE_SIZE_QUERY_PARAM
max_page_size = Pagination.MAX_PAGE_SIZE
+
+
+class OptionalPagination(CustomPagination):
+ """Paginate only when the caller opts in via ?page / ?page_size.
+
+ These list endpoints are shared: besides the listing page they feed
+ dropdowns/selectors that expect a bare array. Returning None keeps that
+ response untouched for callers that don't ask for a page, while the
+ listing page opts in and gets the {count, next, previous, results} envelope.
+ """
+
+ def paginate_queryset(self, queryset, request, view=None):
+ params = request.query_params
+ # Blank values (?page=) count as not opting in, so shared callers keep
+ # their bare-array response instead of getting an enveloped page.
+ if not params.get(self.page_query_param) and not params.get(
+ self.page_size_query_param
+ ):
+ return None
+ return super().paginate_queryset(queryset, request, view=view)
diff --git a/backend/utils/tests/test_pagination.py b/backend/utils/tests/test_pagination.py
new file mode 100644
index 000000000..5bab48815
--- /dev/null
+++ b/backend/utils/tests/test_pagination.py
@@ -0,0 +1,81 @@
+"""Contract tests for utils.pagination.OptionalPagination.
+
+The four listing endpoints it backs (workflows, prompt studio, adapters,
+connectors) are shared with dropdown/selector consumers that expect a bare
+array. The opt-in behaviour below is what keeps those consumers regression-free:
+no page/page_size param -> paginate_queryset returns None -> DRF serialises the
+bare list; only an explicit opt-in gets the {count, next, previous, results}
+envelope.
+
+Deliberately DB-free and Django-settings-free (a hand-rolled request stub, no
+APIRequestFactory) so it runs in the rig's unit tier.
+"""
+
+from __future__ import annotations
+
+from utils.pagination import OptionalPagination
+
+QUERYSET = list(range(1, 101)) # 100 sliceable items stand in for a queryset
+
+
+class _Req:
+ """Minimal stand-in exposing the surface DRF pagination touches."""
+
+ def __init__(self, params: dict[str, str]):
+ self.query_params = params
+
+ def build_absolute_uri(self) -> str:
+ return "https://testserver/things/"
+
+
+class TestOptionalPagination:
+ def test_no_params_returns_none(self):
+ """No opt-in -> None, so callers keep their bare-array response."""
+ assert OptionalPagination().paginate_queryset(QUERYSET, _Req({})) is None
+
+ def test_unrelated_params_do_not_trigger_pagination(self):
+ req = _Req({"adapter_type": "LLM", "search": "foo"})
+ assert OptionalPagination().paginate_queryset(QUERYSET, req) is None
+
+ def test_blank_params_do_not_trigger_pagination(self):
+ """Empty ?page= / ?page_size= keep the bare-array response."""
+ req = _Req({"page": "", "page_size": ""})
+ assert OptionalPagination().paginate_queryset(QUERYSET, req) is None
+
+ def test_single_blank_param_does_not_trigger_pagination(self):
+ for params in ({"page": ""}, {"page_size": ""}):
+ assert OptionalPagination().paginate_queryset(QUERYSET, _Req(params)) is None
+
+ def test_blank_page_with_explicit_page_size_serves_first_page(self):
+ """A blank ?page= alongside a real ?page_size= is page 1, not an error.
+
+ DRF reads the page number as `query_params.get("page") or 1`, so a blank
+ value falls back to the first page instead of raising NotFound.
+ """
+ page = OptionalPagination().paginate_queryset(
+ QUERYSET, _Req({"page": "", "page_size": "10"})
+ )
+ assert page == list(range(1, 11))
+
+ def test_blank_page_size_with_explicit_page_uses_default_size(self):
+ page = OptionalPagination().paginate_queryset(
+ QUERYSET, _Req({"page": "2", "page_size": ""})
+ )
+ assert page == list(range(51, 101))
+
+ def test_page_param_opts_in(self):
+ paginator = OptionalPagination()
+ page = paginator.paginate_queryset(QUERYSET, _Req({"page": "1"}))
+ assert page == list(range(1, 51)) # default page_size == 50
+ assert paginator.page.paginator.count == 100
+
+ def test_page_size_param_alone_opts_in(self):
+ page = OptionalPagination().paginate_queryset(QUERYSET, _Req({"page_size": "10"}))
+ assert page == list(range(1, 11))
+
+ def test_page_size_capped_at_max(self):
+ page = OptionalPagination().paginate_queryset(
+ QUERYSET, _Req({"page_size": "100000"})
+ )
+ # max_page_size (1000) caps the request; all 100 rows fit in one page
+ assert page == QUERYSET
diff --git a/backend/workflow_manager/workflow_v2/views.py b/backend/workflow_manager/workflow_v2/views.py
index 0db26a850..ef9f52f95 100644
--- a/backend/workflow_manager/workflow_v2/views.py
+++ b/backend/workflow_manager/workflow_v2/views.py
@@ -22,6 +22,7 @@ from rest_framework.versioning import URLPathVersioning
from rest_framework.views import APIView
from utils.filtering import FilterHelper
from utils.organization_utils import filter_queryset_by_organization, resolve_organization
+from utils.pagination import OptionalPagination
from backend.constants import RequestKey
from unstract.core.data_models import FileHistoryCreateRequest
@@ -75,6 +76,7 @@ class WorkflowViewSet(
OwnerManagementMixin, ResourceShareManagementMixin, viewsets.ModelViewSet
):
versioning_class = URLPathVersioning
+ pagination_class = OptionalPagination
notification_resource_name_field = "workflow_name"
def get_notification_resource_type(self, resource: Any) -> str | None:
@@ -112,11 +114,18 @@ class WorkflowViewSet(
queryset = queryset.select_related("created_by").prefetch_related(
"memberships__user"
)
+
+ search = self.request.query_params.get("search")
+ if search:
+ queryset = queryset.filter(workflow_name__icontains=search)
+
+ # `id` tiebreaker keeps ordering deterministic across paginated requests
+ # (the for_user() manager uses plain .distinct(), so there is no default)
order_by = self.request.query_params.get("order_by")
- if order_by == "desc":
- queryset = queryset.order_by("-modified_at")
- elif order_by == "asc":
- queryset = queryset.order_by("modified_at")
+ if order_by == "asc":
+ queryset = queryset.order_by("modified_at", "id")
+ else:
+ queryset = queryset.order_by("-modified_at", "id")
return queryset
diff --git a/frontend/src/components/workflows/workflow/Workflows.css b/frontend/src/components/workflows/workflow/Workflows.css
index 22d6ec9e9..7a407a2fe 100644
--- a/frontend/src/components/workflows/workflow/Workflows.css
+++ b/frontend/src/components/workflows/workflow/Workflows.css
@@ -85,3 +85,9 @@
flex: 1;
overflow-y: auto;
}
+
+.workflows-pagination {
+ display: flex;
+ justify-content: flex-end;
+ padding: 12px 16px;
+}
diff --git a/frontend/src/components/workflows/workflow/Workflows.jsx b/frontend/src/components/workflows/workflow/Workflows.jsx
index 4160cde4f..7d5567c36 100644
--- a/frontend/src/components/workflows/workflow/Workflows.jsx
+++ b/frontend/src/components/workflows/workflow/Workflows.jsx
@@ -1,12 +1,12 @@
import { PlusOutlined, UserOutlined } from "@ant-design/icons";
-import { Typography } from "antd";
-import isEmpty from "lodash/isEmpty";
+import { Pagination, Typography } from "antd";
import PropTypes from "prop-types";
import { useCallback, useEffect, useRef, useState } from "react";
import { useLocation, useNavigate } from "react-router-dom";
import { useCoOwnerManagement } from "../../../hooks/useCoOwnerManagement.jsx";
import { useExceptionHandler } from "../../../hooks/useExceptionHandler.jsx";
+import { usePaginatedList } from "../../../hooks/usePaginatedList";
import usePostHogEvents from "../../../hooks/usePostHogEvents.js";
import {
useInitialFetchCount,
@@ -32,6 +32,8 @@ import "./Workflows.css";
const { Text } = Typography;
+const DEFAULT_PAGE_SIZE = 10;
+
function Workflows() {
const navigate = useNavigate();
const location = useLocation();
@@ -51,7 +53,8 @@ function Workflows() {
const [editingProject, setEditProject] = useState();
const [loading, setLoading] = useState(false);
const [openModal, toggleModal] = useState(true);
- const projectListRef = useRef();
+ // Ref forwards the fetch fn to the pagination hook (avoids declaration ordering)
+ const fetchListRef = useRef(null);
const [backendErrors, setBackendErrors] = useState(null);
const [shareOpen, setShareOpen] = useState(false);
const [selectedWorkflow, setSelectedWorkflow] = useState();
@@ -61,10 +64,27 @@ function Workflows() {
const [allGroups, setAllGroups] = useState([]);
const { setAlertDetails } = useAlertStore();
+
+ const {
+ pagination,
+ setPagination,
+ searchTerm,
+ handlePaginationChange,
+ handleSearch,
+ } = usePaginatedList({
+ fetchData: (...args) => fetchListRef.current?.(...args),
+ defaultPageSize: DEFAULT_PAGE_SIZE,
+ });
+
+ // Refresh the current page (preserves page + active search) after mutations
const handleListRefresh = useCallback(
- () => getProjectList(),
- // eslint-disable-next-line react-hooks/exhaustive-deps
- [],
+ () =>
+ fetchListRef.current?.(
+ pagination.current,
+ pagination.pageSize,
+ searchTerm,
+ ),
+ [pagination.current, pagination.pageSize, searchTerm],
);
const {
coOwnerOpen,
@@ -91,28 +111,47 @@ function Workflows() {
}
}, [location.pathname]);
- function getProjectList() {
+ const getProjectList = (
+ page = 1,
+ pageSize = DEFAULT_PAGE_SIZE,
+ search = "",
+ ) => {
+ setLoading(true);
+ const params = { page, page_size: pageSize };
+ if (search) {
+ params.search = search;
+ }
projectApiService
- .getProjectList()
+ .getProjectList(params)
.then((res) => {
- projectListRef.current = res.data;
- setProjectList(res.data);
+ const data = res?.data;
+ // Endpoint is opt-in paginated: envelope when we send ?page, else a
+ // bare array. Handle both so nothing breaks if the opt-in is dropped.
+ const results = data?.results ?? data ?? [];
+ const total = data?.count ?? results.length;
+ // Deleting the last row on a page leaves it empty; step back a page.
+ if (results.length === 0 && page > 1 && total > 0) {
+ getProjectList(page - 1, pageSize, search);
+ return;
+ }
+ setProjectList(results);
+ setPagination((prev) => ({
+ ...prev,
+ current: page,
+ pageSize,
+ total,
+ }));
})
.catch(() => {
console.error("Unable to get project list");
+ // Avoid an indefinite spinner when the first fetch fails.
+ setProjectList((prev) => prev ?? []);
+ })
+ .finally(() => {
+ setLoading(false);
});
- }
-
- function onSearch(searchText, setSearchList) {
- if (!searchText.trim()) {
- setSearchList(projectListRef.current);
- return;
- }
- const filteredList = projectListRef.current.filter((item) =>
- item.workflow_name.toLowerCase().includes(searchText.toLowerCase()),
- );
- setSearchList(filteredList);
- }
+ };
+ fetchListRef.current = getProjectList;
function editProject(name, description) {
setLoading(true);
@@ -121,7 +160,7 @@ function Workflows() {
.then((res) => {
closeNewProject();
if (editingProject?.name) {
- getProjectList();
+ handleListRefresh();
} else {
openProject(res.data);
}
@@ -129,7 +168,6 @@ function Workflows() {
type: "success",
content: "Workflow updated successfully",
});
- getProjectList();
})
.catch((err) => {
setAlertDetails(
@@ -220,7 +258,7 @@ function Workflows() {
projectApiService
.deleteProject(project.id)
.then(() => {
- getProjectList();
+ handleListRefresh();
setAlertDetails({
type: "success",
content: "Workflow deleted successfully",
@@ -307,7 +345,7 @@ function Workflows() {
type: "success",
content: "Workflow sharing updated successfully",
});
- getProjectList();
+ handleListRefresh();
// Close only on success; keep the modal open on failure so the user
// can see the rejected entries and retry.
setShareOpen(false);
@@ -363,15 +401,13 @@ function Workflows() {
<ToolNavBar
title="Workflows"
enableSearch
- searchList={projectList}
- setSearchList={setProjectList}
customButtons={newWorkflowButton}
- onSearch={onSearch}
+ onSearch={(value) => handleSearch(value)}
/>
<div className="workflows-pg-layout">
<div className="workflows-pg-body">
- {!projectListRef.current && <SpinnerLoader />}
- {projectListRef.current && isEmpty(projectListRef?.current) && (
+ {projectList === undefined && <SpinnerLoader />}
+ {projectList?.length === 0 && !searchTerm && (
<div className="list-of-workflows-body">
<EmptyState
text="No Workflow available"
@@ -383,24 +419,37 @@ function Workflows() {
/>
</div>
)}
- {isEmpty(projectList) && !isEmpty(projectListRef?.current) && (
+ {projectList?.length === 0 && searchTerm && (
<EmptyState text="No results found for this search" />
)}
- {!isEmpty(projectList) && (
- <ViewTools
- isLoading={loading}
- isEmpty={!projectList?.length}
- listOfTools={projectList}
- setOpenAddTool={toggleModal}
- handleEdit={updateProject}
- handleDelete={deleteProject}
- handleShare={handleShare}
- handleCoOwner={handleCoOwner}
- titleProp="workflow_name"
- descriptionProp="description"
- idProp="id"
- type="Workflow"
- />
+ {projectList?.length > 0 && (
+ <>
+ <ViewTools
+ isLoading={loading}
+ isEmpty={false}
+ listOfTools={projectList}
+ setOpenAddTool={toggleModal}
+ handleEdit={updateProject}
+ handleDelete={deleteProject}
+ handleShare={handleShare}
+ handleCoOwner={handleCoOwner}
+ titleProp="workflow_name"
+ descriptionProp="description"
+ idProp="id"
+ type="Workflow"
+ />
+ {pagination.total > pagination.pageSize && (
+ <div className="workflows-pagination">
+ <Pagination
+ current={pagination.current}
+ pageSize={pagination.pageSize}
+ total={pagination.total}
+ onChange={handlePaginationChange}
+ showSizeChanger={false}
+ />
+ </div>
+ )}
+ </>
)}
{editingProject && (
<LazyLoader
diff --git a/frontend/src/components/workflows/workflow/workflow-service.js b/frontend/src/components/workflows/workflow/workflow-service.js
index d5cce6307..9453aedc4 100644
--- a/frontend/src/components/workflows/workflow/workflow-service.js
+++ b/frontend/src/components/workflows/workflow/workflow-service.js
@@ -24,8 +24,7 @@ function workflowService() {
};
return axiosPrivate(options);
},
- getProjectList: (myProjects = false) => {
- const params = myProjects ? { created_by: sessionDetails?.id } : {};
+ getProjectList: (params = {}) => {
options = {
url: `${path}/workflow/`,
method: "GET",