diff --git a/apps/sim/app/api/workflows/route.test.ts b/apps/sim/app/api/workflows/route.test.ts index e1b83bdb0eb..5da81e4467c 100644 --- a/apps/sim/app/api/workflows/route.test.ts +++ b/apps/sim/app/api/workflows/route.test.ts @@ -11,17 +11,22 @@ const { mockWorkflowCreated, mockDbSelect, mockDbInsert, + mockWorkspaceExists, + mockVerifyWorkspaceMembership, } = vi.hoisted(() => ({ mockCheckSessionOrInternalAuth: vi.fn(), mockGetUserEntityPermissions: vi.fn(), mockWorkflowCreated: vi.fn(), mockDbSelect: vi.fn(), mockDbInsert: vi.fn(), + mockWorkspaceExists: vi.fn(), + mockVerifyWorkspaceMembership: vi.fn(), })) vi.mock('drizzle-orm', () => ({ ...drizzleOrmMock, min: vi.fn((field) => ({ type: 'min', field })), + count: vi.fn(() => ({ type: 'count' })), })) vi.mock('@sim/db', () => ({ @@ -71,11 +76,11 @@ vi.mock('@/lib/auth/hybrid', () => ({ vi.mock('@/lib/workspaces/permissions/utils', () => ({ getUserEntityPermissions: (...args: unknown[]) => mockGetUserEntityPermissions(...args), - workspaceExists: vi.fn(), + workspaceExists: (...args: unknown[]) => mockWorkspaceExists(...args), })) vi.mock('@/app/api/workflows/utils', () => ({ - verifyWorkspaceMembership: vi.fn(), + verifyWorkspaceMembership: (...args: unknown[]) => mockVerifyWorkspaceMembership(...args), })) vi.mock('@/lib/core/telemetry', () => ({ @@ -84,7 +89,7 @@ vi.mock('@/lib/core/telemetry', () => ({ }, })) -import { POST } from '@/app/api/workflows/route' +import { GET, POST } from '@/app/api/workflows/route' describe('Workflows API Route - POST ordering', () => { beforeEach(() => { @@ -171,3 +176,138 @@ describe('Workflows API Route - POST ordering', () => { expect(insertedValues?.sortOrder).toBe(0) }) }) + +describe('Workflows API Route - GET pagination', () => { + beforeEach(() => { + vi.clearAllMocks() + + mockCheckSessionOrInternalAuth.mockResolvedValue({ + success: true, + userId: 'user-123', + userName: 'Test User', + userEmail: 'test@example.com', + }) + mockWorkspaceExists.mockResolvedValue(true) + mockVerifyWorkspaceMembership.mockResolvedValue('member') + }) + + /** + * Builds a fluent mock chain for db.select() that terminates with the + * given resolved values. The chain supports arbitrary method calls + * (from, where, orderBy, limit, offset) in any order. + */ + function buildSelectChain(resolvedValues: unknown[]) { + const chain: Record = {} + const self = new Proxy(chain, { + get(_target, prop) { + if (prop === 'then') { + return (resolve: (v: unknown) => void) => resolve(resolvedValues) + } + return vi.fn().mockReturnValue(self) + }, + }) + return self + } + + it('returns pagination metadata with workspace workflows', async () => { + const mockWorkflows = [ + { id: 'wf-1', name: 'Workflow 1', workspaceId: 'ws-1' }, + { id: 'wf-2', name: 'Workflow 2', workspaceId: 'ws-1' }, + ] + + const selectCalls: unknown[][] = [] + mockDbSelect.mockImplementation((...args: unknown[]) => { + selectCalls.push(args) + if (selectCalls.length === 1) { + return buildSelectChain([{ count: 2 }]) + } + return buildSelectChain(mockWorkflows) + }) + + const req = createMockRequest( + 'GET', + undefined, + {}, + 'http://localhost:3000/api/workflows?workspaceId=ws-1' + ) + + const response = await GET(req as any) + const json = await response.json() + + expect(response.status).toBe(200) + expect(json.data).toHaveLength(2) + expect(json.pagination).toBeDefined() + expect(json.pagination.total).toBe(2) + expect(json.pagination.limit).toBe(200) + expect(json.pagination.offset).toBe(0) + expect(json.pagination.hasMore).toBe(false) + }) + + it('respects custom limit and offset params', async () => { + const mockWorkflows = [{ id: 'wf-1', name: 'Workflow 1', workspaceId: 'ws-1' }] + + const selectCalls: unknown[][] = [] + mockDbSelect.mockImplementation((...args: unknown[]) => { + selectCalls.push(args) + if (selectCalls.length === 1) { + return buildSelectChain([{ count: 5 }]) + } + return buildSelectChain(mockWorkflows) + }) + + const req = createMockRequest( + 'GET', + undefined, + {}, + 'http://localhost:3000/api/workflows?workspaceId=ws-1&limit=1&offset=2' + ) + + const response = await GET(req as any) + const json = await response.json() + + expect(response.status).toBe(200) + expect(json.pagination.limit).toBe(1) + expect(json.pagination.offset).toBe(2) + expect(json.pagination.total).toBe(5) + expect(json.pagination.hasMore).toBe(true) + }) + + it('clamps limit to MAX_PAGE_LIMIT', async () => { + const selectCalls: unknown[][] = [] + mockDbSelect.mockImplementation((...args: unknown[]) => { + selectCalls.push(args) + if (selectCalls.length === 1) { + return buildSelectChain([{ count: 0 }]) + } + return buildSelectChain([]) + }) + + const req = createMockRequest( + 'GET', + undefined, + {}, + 'http://localhost:3000/api/workflows?workspaceId=ws-1&limit=9999' + ) + + const response = await GET(req as any) + const json = await response.json() + + expect(response.status).toBe(200) + expect(json.pagination.limit).toBe(500) + }) + + it('returns pagination in empty workspace response for no-workspace query', async () => { + mockDbSelect.mockImplementation(() => buildSelectChain([])) + + const req = createMockRequest('GET', undefined, {}, 'http://localhost:3000/api/workflows') + + const response = await GET(req as any) + const json = await response.json() + + expect(response.status).toBe(200) + expect(json.data).toEqual([]) + expect(json.pagination).toBeDefined() + expect(json.pagination.total).toBe(0) + expect(json.pagination.hasMore).toBe(false) + }) +}) diff --git a/apps/sim/app/api/workflows/route.ts b/apps/sim/app/api/workflows/route.ts index 611d808cf61..40978110458 100644 --- a/apps/sim/app/api/workflows/route.ts +++ b/apps/sim/app/api/workflows/route.ts @@ -1,7 +1,7 @@ import { db } from '@sim/db' import { permissions, workflow, workflowFolder } from '@sim/db/schema' import { createLogger } from '@sim/logger' -import { and, asc, eq, inArray, isNull, min } from 'drizzle-orm' +import { and, asc, count, eq, inArray, isNull, min } from 'drizzle-orm' import { type NextRequest, NextResponse } from 'next/server' import { z } from 'zod' import { AuditAction, AuditResourceType, recordAudit } from '@/lib/audit/log' @@ -12,6 +12,9 @@ import { verifyWorkspaceMembership } from '@/app/api/workflows/utils' const logger = createLogger('WorkflowAPI') +const DEFAULT_PAGE_LIMIT = 200 +const MAX_PAGE_LIMIT = 500 + const CreateWorkflowSchema = z.object({ name: z.string().min(1, 'Name is required'), description: z.string().optional().default(''), @@ -28,6 +31,14 @@ export async function GET(request: NextRequest) { const url = new URL(request.url) const workspaceId = url.searchParams.get('workspaceId') + const rawLimit = url.searchParams.get('limit') + const rawOffset = url.searchParams.get('offset') + const limit = Math.min( + Math.max(1, rawLimit ? Number(rawLimit) : DEFAULT_PAGE_LIMIT), + MAX_PAGE_LIMIT + ) + const offset = Math.max(0, rawOffset ? Number(rawOffset) : 0) + try { const auth = await checkSessionOrInternalAuth(request, { requireWorkflowId: false }) if (!auth.success || !auth.userId) { @@ -63,15 +74,26 @@ export async function GET(request: NextRequest) { } let workflows + let total: number const orderByClause = [asc(workflow.sortOrder), asc(workflow.createdAt), asc(workflow.id)] if (workspaceId) { - workflows = await db - .select() - .from(workflow) - .where(eq(workflow.workspaceId, workspaceId)) - .orderBy(...orderByClause) + const whereCondition = eq(workflow.workspaceId, workspaceId) + + const [countResult, workflowRows] = await Promise.all([ + db.select({ count: count() }).from(workflow).where(whereCondition), + db + .select() + .from(workflow) + .where(whereCondition) + .orderBy(...orderByClause) + .limit(limit) + .offset(offset), + ]) + + total = countResult[0]?.count ?? 0 + workflows = workflowRows } else { const workspacePermissionRows = await db .select({ workspaceId: permissions.entityId }) @@ -79,16 +101,41 @@ export async function GET(request: NextRequest) { .where(and(eq(permissions.userId, userId), eq(permissions.entityType, 'workspace'))) const workspaceIds = workspacePermissionRows.map((row) => row.workspaceId) if (workspaceIds.length === 0) { - return NextResponse.json({ data: [] }, { status: 200 }) + return NextResponse.json( + { data: [], pagination: { total: 0, limit, offset, hasMore: false } }, + { status: 200 } + ) } - workflows = await db - .select() - .from(workflow) - .where(inArray(workflow.workspaceId, workspaceIds)) - .orderBy(...orderByClause) + + const whereCondition = inArray(workflow.workspaceId, workspaceIds) + + const [countResult, workflowRows] = await Promise.all([ + db.select({ count: count() }).from(workflow).where(whereCondition), + db + .select() + .from(workflow) + .where(whereCondition) + .orderBy(...orderByClause) + .limit(limit) + .offset(offset), + ]) + + total = countResult[0]?.count ?? 0 + workflows = workflowRows } - return NextResponse.json({ data: workflows }, { status: 200 }) + return NextResponse.json( + { + data: workflows, + pagination: { + total, + limit, + offset, + hasMore: offset + workflows.length < total, + }, + }, + { status: 200 } + ) } catch (error: any) { const elapsed = Date.now() - startTime logger.error(`[${requestId}] Workflow fetch error after ${elapsed}ms`, error) diff --git a/apps/sim/app/workspace/[workspaceId]/w/hooks/use-export-workspace.ts b/apps/sim/app/workspace/[workspaceId]/w/hooks/use-export-workspace.ts index 44fab2c685e..6318fcd8af6 100644 --- a/apps/sim/app/workspace/[workspaceId]/w/hooks/use-export-workspace.ts +++ b/apps/sim/app/workspace/[workspaceId]/w/hooks/use-export-workspace.ts @@ -8,6 +8,7 @@ import { sanitizePathSegment, type WorkflowExportData, } from '@/lib/workflows/operations/import-export' +import { fetchAllPages } from '@/hooks/queries/utils/paginated-fetch' const logger = createLogger('useExportWorkspace') @@ -32,11 +33,9 @@ export function useExportWorkspace({ onSuccess }: UseExportWorkspaceProps = {}) try { logger.info('Exporting workspace', { workspaceId }) - const workflowsResponse = await fetch(`/api/workflows?workspaceId=${workspaceId}`) - if (!workflowsResponse.ok) { - throw new Error('Failed to fetch workflows') - } - const { data: workflows } = await workflowsResponse.json() + const workflows = await fetchAllPages>( + `/api/workflows?workspaceId=${workspaceId}` + ) const foldersResponse = await fetch(`/api/folders?workspaceId=${workspaceId}`) if (!foldersResponse.ok) { diff --git a/apps/sim/hooks/queries/utils/paginated-fetch.ts b/apps/sim/hooks/queries/utils/paginated-fetch.ts new file mode 100644 index 00000000000..b4d445e9a9c --- /dev/null +++ b/apps/sim/hooks/queries/utils/paginated-fetch.ts @@ -0,0 +1,39 @@ +/** + * Fetches all pages from a paginated API endpoint. + * + * The endpoint is expected to return `{ data: T[], pagination: { hasMore: boolean } }`. + * Pages are fetched sequentially until `hasMore` is `false`. + * + * @param baseUrl - Base URL including any existing query params (e.g. `/api/workflows?workspaceId=ws-1`) + * @param pageSize - Number of items per page (default 200) + * @returns All items concatenated across pages + */ +const MAX_PAGES = 100 + +export async function fetchAllPages(baseUrl: string, pageSize = 200): Promise { + const allItems: T[] = [] + let offset = 0 + let pages = 0 + const separator = baseUrl.includes('?') ? '&' : '?' + + while (pages < MAX_PAGES) { + const response = await fetch(`${baseUrl}${separator}limit=${pageSize}&offset=${offset}`) + + if (!response.ok) { + throw new Error(`Failed to fetch from ${baseUrl}: ${response.statusText}`) + } + + const json = await response.json() + const data: T[] = Array.isArray(json.data) ? json.data : [] + allItems.push(...data) + + if (!json.pagination?.hasMore || data.length === 0) { + break + } + + offset += pageSize + pages++ + } + + return allItems +} diff --git a/apps/sim/hooks/queries/workflow-mcp-servers.ts b/apps/sim/hooks/queries/workflow-mcp-servers.ts index fe03a28edcb..ae9b714007b 100644 --- a/apps/sim/hooks/queries/workflow-mcp-servers.ts +++ b/apps/sim/hooks/queries/workflow-mcp-servers.ts @@ -1,5 +1,6 @@ import { createLogger } from '@sim/logger' import { keepPreviousData, useMutation, useQuery, useQueryClient } from '@tanstack/react-query' +import { fetchAllPages } from '@/hooks/queries/utils/paginated-fetch' const logger = createLogger('WorkflowMcpServerQueries') @@ -445,13 +446,7 @@ export function useDeleteWorkflowMcpTool() { * Fetch deployed workflows for a workspace */ async function fetchDeployedWorkflows(workspaceId: string): Promise { - const response = await fetch(`/api/workflows?workspaceId=${workspaceId}`) - - if (!response.ok) { - throw new Error('Failed to fetch workflows') - } - - const { data }: { data: any[] } = await response.json() + const data = await fetchAllPages>(`/api/workflows?workspaceId=${workspaceId}`) return data .filter((w) => w.isDeployed) diff --git a/apps/sim/hooks/queries/workflows.ts b/apps/sim/hooks/queries/workflows.ts index 9664cf86423..76c44afa56b 100644 --- a/apps/sim/hooks/queries/workflows.ts +++ b/apps/sim/hooks/queries/workflows.ts @@ -8,6 +8,7 @@ import { createOptimisticMutationHandlers, generateTempId, } from '@/hooks/queries/utils/optimistic-mutation' +import { fetchAllPages } from '@/hooks/queries/utils/paginated-fetch' import { getTopInsertionSortOrder } from '@/hooks/queries/utils/top-insertion-sort-order' import { useFolderStore } from '@/stores/folders/store' import { useWorkflowRegistry } from '@/stores/workflows/registry/store' @@ -74,13 +75,9 @@ function mapWorkflow(workflow: any): WorkflowMetadata { } async function fetchWorkflows(workspaceId: string): Promise { - const response = await fetch(`/api/workflows?workspaceId=${workspaceId}`) - - if (!response.ok) { - throw new Error('Failed to fetch workflows') - } - - const { data }: { data: any[] } = await response.json() + const data = await fetchAllPages>( + `/api/workflows?workspaceId=${workspaceId}` + ) return data.map(mapWorkflow) }