Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(api): end the crawl status cursor after a cancelled or failed job's last page (#4863)
Gergő Móricz7e4a959dca0e
diff --git a/.github/workflows/test-server.yml b/.github/workflows/test-server.ymlindex 8db3b596b..73a644bb1 100644--- a/.github/workflows/test-server.yml+++ b/.github/workflows/test-server.yml@@ -243,6 +243,15 @@ jobs:run: pnpm exec vitest run src/controllers/v2/__tests__/browser-delete.test.ts --reporter=defaultworking-directory: apps/api+ - name: Run crawl status cursor regression tests+ if: matrix.engine == 'fetch' && matrix.proxy == 'no-proxy' && matrix.nuq == 'postgres'+ run: >-+ pnpm exec vitest run+ src/controllers/v1/__tests__/crawl-status-next.test.ts+ src/controllers/v2/__tests__/crawl-status-next.test.ts+ --reporter=default+ working-directory: apps/api+- name: Run agent interop auth and concurrency regression testsif: matrix.engine == 'fetch' && matrix.proxy == 'no-proxy' && matrix.nuq == 'postgres'run: >-diff --git a/apps/api/src/__tests__/snips/v2/crawl-status-cursor.test.ts b/apps/api/src/__tests__/snips/v2/crawl-status-cursor.test.tsnew file mode 100644index 000000000..1bcb0c6f3--- /dev/null+++ b/apps/api/src/__tests__/snips/v2/crawl-status-cursor.test.ts@@ -0,0 +1,143 @@+import {+ ALLOW_TEST_SUITE_WEBSITE,+ concurrentIf,+ TEST_API_URL,+ TEST_SUITE_WEBSITE,+} from "../lib";+import { idmux, Identity, scrapeTimeout } from "./lib";+import request from "supertest";+import { describe, expect, beforeAll } from "vitest";++let identity: Identity;+let lowConcurrencyIdentity: Identity;++beforeAll(async () => {+ identity = await idmux({+ name: "crawl-status-cursor",+ concurrency: 100,+ credits: 1000000,+ });+ lowConcurrencyIdentity = await idmux({+ name: "crawl-status-cursor-cancel",+ concurrency: 1,+ credits: 1000000,+ });+}, 10000);++const sleep = (ms: number) => new Promise(r => setTimeout(r, ms));++async function getStatus(path: string, apiKey: string) {+ const res = await request(TEST_API_URL)+ .get(path)+ .set("Authorization", `Bearer ${apiKey}`)+ .send();+ expect(res.statusCode).toBe(200);+ return res.body;+}++// Follows `next` like the SDKs do, with a hop cap so a cursor that never ends+// fails the test instead of hanging it.+async function followCursor(firstPath: string, apiKey: string) {+ const docs: any[] = [];+ let path: string | undefined = firstPath;+ let hops = 0;+ let last: any;+ while (path !== undefined && hops < 20) {+ last = await getStatus(path, apiKey);+ docs.push(...last.data);+ hops++;+ path = last.next+ ? new URL(last.next).pathname + new URL(last.next).search+ : undefined;+ }+ return { docs, hops, last, ended: path === undefined };+}++async function startBatch(urls: string[], apiKey: string) {+ const res = await request(TEST_API_URL)+ .post("/v2/batch/scrape")+ .set("Authorization", `Bearer ${apiKey}`)+ .set("Content-Type", "application/json")+ .send({ urls });+ expect(res.statusCode).toBe(200);+ expect(res.body.success).toBe(true);+ return res.body.id as string;+}++describe("Crawl status cursor", () => {+ concurrentIf(ALLOW_TEST_SUITE_WEBSITE)(+ "ends after the last page of a completed job",+ async () => {+ const apiKey = identity.apiKey;+ const id = await startBatch(+ Array.from(+ { length: 5 },+ (_, i) => `${TEST_SUITE_WEBSITE}?cursorDone=${i}`,+ ),+ apiKey,+ );++ let status: any;+ do {+ if (status) await sleep(250);+ status = await getStatus(`/v2/batch/scrape/${id}`, apiKey);+ } while (status.status === "scraping");+ expect(status.status).toBe("completed");++ const result = await followCursor(+ `/v2/batch/scrape/${id}?limit=2`,+ apiKey,+ );+ expect(result.ended).toBe(true);+ expect(result.hops).toBe(Math.max(1, Math.ceil(status.completed / 2)));+ expect(result.docs).toHaveLength(status.completed);+ expect(result.last.next).toBeUndefined();+ },+ scrapeTimeout * 2,+ );++ concurrentIf(ALLOW_TEST_SUITE_WEBSITE)(+ "ends after the last page of a cancelled job",+ async () => {+ const apiKey = lowConcurrencyIdentity.apiKey;+ const id = await startBatch(+ Array.from(+ { length: 20 },+ (_, i) => `${TEST_SUITE_WEBSITE}?cursorCancel=${i}`,+ ),+ apiKey,+ );++ let status: any;+ do {+ if (status) await sleep(100);+ status = await getStatus(`/v2/batch/scrape/${id}`, apiKey);+ } while (status.status === "scraping" && status.completed < 1);+ expect(status.status).toBe("scraping");++ const cancel = await request(TEST_API_URL)+ .delete(`/v2/batch/scrape/${id}`)+ .set("Authorization", `Bearer ${apiKey}`)+ .send();+ expect(cancel.statusCode).toBe(200);++ const completedBeforeCancel = status.completed;+ status = await getStatus(`/v2/batch/scrape/${id}`, apiKey);+ expect(status.status).toBe("cancelled");++ const result = await followCursor(+ `/v2/batch/scrape/${id}?limit=2`,+ apiKey,+ );+ expect(result.ended).toBe(true);+ expect(result.last.next).toBeUndefined();+ expect(result.docs.length).toBeGreaterThanOrEqual(completedBeforeCancel);++ // No more than 20 documents can ever exist, so this page is past the end.+ const past = await getStatus(`/v2/batch/scrape/${id}?skip=20`, apiKey);+ expect(past.data).toHaveLength(0);+ expect(past.next).toBeUndefined();+ },+ scrapeTimeout * 2,+ );+});diff --git a/apps/api/src/controllers/v1/__tests__/crawl-status-next.test.ts b/apps/api/src/controllers/v1/__tests__/crawl-status-next.test.tsnew file mode 100644index 000000000..b9bdbb0c9--- /dev/null+++ b/apps/api/src/controllers/v1/__tests__/crawl-status-next.test.ts@@ -0,0 +1,230 @@+import { vi } from "vitest";++const mocks = vi.hoisted(() => ({+ getCrawl: vi.fn(),+ getGroup: vi.fn(),+ getGroupNumericStats: vi.fn(),+ getGroupJobs: vi.fn(),+ getCrawlError: vi.fn(),+}));++vi.mock("../../../lib/crawl-redis", () => ({+ getCrawl: mocks.getCrawl,+ getCrawlError: mocks.getCrawlError,+ getCrawlExpiry: vi.fn().mockResolvedValue(new Date(0)),+ getCrawlQualifiedJobCount: vi.fn(),+ getDoneJobsOrderedLength: vi.fn(),+ getDoneJobsOrderedUntil: vi.fn(),+ getLastDoneJobTimestamp: vi.fn().mockResolvedValue(null),+ isCrawlKickoffFinished: vi.fn(),+}));++vi.mock("../../../services/worker/nuq-router", () => ({+ crawlGroup: { getGroup: mocks.getGroup },+ scrapeQueue: {+ getGroupAnyJob: vi.fn().mockResolvedValue(null),+ getGroupNumericStats: mocks.getGroupNumericStats,+ getGroupJobs: mocks.getGroupJobs,+ },+}));++vi.mock("../../../services/redis", () => ({+ redisEvictConnection: { smembers: vi.fn().mockResolvedValue([]) },+}));++vi.mock("../../../lib/request-credits-store", () => ({+ readRequestCredits: vi.fn().mockResolvedValue(3),+}));++vi.mock("../../../lib/gcs-jobs", () => ({+ getJobFromGCS: vi.fn().mockResolvedValue(null),+}));++vi.mock("../../../lib/logger", () => ({+ logger: {+ info: vi.fn(),+ warn: vi.fn(),+ error: vi.fn(),+ debug: vi.fn(),+ child: vi.fn().mockReturnThis(),+ },+}));++import { crawlStatusController } from "../crawl-status";++const TEAM_ID = "11111111-1111-1111-1111-111111111111";+const JOB_ID = "22222222-2222-4222-8222-222222222222";++function makeReq(query: Record<string, string> = {}) {+ return {+ params: { jobId: JOB_ID },+ query,+ auth: { team_id: TEAM_ID },+ protocol: "http",+ host: "localhost",+ } as any;+}++function makeRes() {+ const res: any = { status: vi.fn(), json: vi.fn() };+ res.status.mockReturnValue(res);+ res.json.mockReturnValue(res);+ return res;+}++const doc = (url: string) => ({+ id: url,+ returnvalue: { markdown: url, metadata: { sourceURL: url } },+});++// Two completed pages. Pages 0-1 are returned by the first request and the+// page past them (skip=2) is empty. `pending` jobs are still queued or running.+function mockCrawl(options: {+ cancelled: boolean;+ groupStatus: string;+ pending?: number;+ completed?: number;+ crawlError?: string;+}) {+ mocks.getCrawlError.mockResolvedValue(options.crawlError ?? null);+ mocks.getGroup.mockResolvedValue({ status: options.groupStatus });+ mocks.getCrawl.mockResolvedValue({+ team_id: TEAM_ID,+ cancelled: options.cancelled,+ createdAt: Date.now(),+ });+ mocks.getGroupNumericStats.mockResolvedValue({+ completed: options.completed ?? 2,+ active: options.pending ?? 0,+ });+ mocks.getGroupJobs.mockImplementation(+ async (_id: string, _status: string, limit: number, offset: number) =>+ [doc("https://example.com/a"), doc("https://example.com/b")].slice(+ offset,+ offset + limit,+ ),+ );+}++async function getStatus(query: Record<string, string> = {}) {+ const res = makeRes();+ await crawlStatusController(makeReq(query), res);+ return res.json.mock.calls[0][0];+}++describe("crawl status next cursor", () => {+ it("ends the cursor after the last page of a cancelled crawl", async () => {+ mockCrawl({ cancelled: true, groupStatus: "active" });++ const first = await getStatus();+ expect(first.status).toBe("cancelled");+ expect(first.data).toHaveLength(2);+ expect(first.next).toBeUndefined();++ // Following a cursor past the end must not hand back the same cursor.+ const past = await getStatus({ skip: "2" });+ expect(past.data).toHaveLength(0);+ expect(past.next).toBeUndefined();+ });++ it("ends the cursor of a cancelled crawl whose pending jobs are still counted", async () => {+ // total is 5 (2 completed + 3 still queued or running), but only completed+ // jobs have pages, so the page past them is the last one.+ mockCrawl({ cancelled: true, groupStatus: "active", pending: 3 });++ const past = await getStatus({ skip: "2" });+ expect(past.total).toBe(5);+ expect(past.data).toHaveLength(0);+ expect(past.next).toBeUndefined();+ });++ it("ends the cursor after the last page of a completed crawl", async () => {+ mockCrawl({ cancelled: false, groupStatus: "completed" });++ const first = await getStatus();+ expect(first.status).toBe("completed");+ expect(first.next).toBeUndefined();+ });++ it("keeps the cursor while the crawl is still running", async () => {+ mockCrawl({ cancelled: false, groupStatus: "active" });++ const first = await getStatus();+ expect(first.status).toBe("scraping");+ expect(first.next).toBe(`http://localhost/v1/crawl/${JOB_ID}?skip=2`);+ });++ it("keeps the cursor while more completed pages remain", async () => {+ mockCrawl({ cancelled: true, groupStatus: "active" });++ const first = await getStatus({ limit: "1" });+ expect(first.data).toHaveLength(1);+ expect(first.next).toBe(+ `http://localhost/v1/crawl/${JOB_ID}?skip=1&limit=1`,+ );+ });++ it("keeps the cursor on a full page of a cancelled crawl whose jobs finished during the request", async () => {+ mockCrawl({ cancelled: true, groupStatus: "active", completed: 1 });++ const first = await getStatus({ limit: "2" });+ expect(first.data).toHaveLength(2);+ expect(first.next).toBe(+ `http://localhost/v1/crawl/${JOB_ID}?skip=2&limit=2`,+ );++ const past = await getStatus({ skip: "2", limit: "2" });+ expect(past.data).toHaveLength(0);+ expect(past.next).toBeUndefined();+ });++ it("keeps the cursor when the response size cap cuts a page short", async () => {+ mockCrawl({ cancelled: true, groupStatus: "active", completed: 1 });+ const big = {+ id: "big",+ returnvalue: { markdown: "x".repeat(11 * 1024 * 1024), metadata: {} },+ };+ mocks.getGroupJobs.mockImplementation(+ async (_id: string, _status: string, limit: number, offset: number) =>+ [big, doc("https://example.com/b")].slice(offset, offset + limit),+ );++ const first = await getStatus();+ expect(first.data).toHaveLength(1);+ expect(first.next).toBe(`http://localhost/v1/crawl/${JOB_ID}?skip=1`);+ });++ it("ends the cursor of a crawl whose group was cancelled", async () => {+ mockCrawl({ cancelled: false, groupStatus: "cancelled" });++ const first = await getStatus();+ expect(first.status).toBe("cancelled");+ expect(first.data).toHaveLength(2);+ expect(first.next).toBeUndefined();++ const past = await getStatus({ skip: "2" });+ expect(past.data).toHaveLength(0);+ expect(past.next).toBeUndefined();+ });++ it("returns no cursor for a crawl that failed during kickoff", async () => {+ mockCrawl({+ cancelled: false,+ groupStatus: "completed",+ completed: 0,+ crawlError: "queue full",+ });++ const first = await getStatus();+ expect(first.status).toBe("failed");+ expect(first.next).toBeUndefined();+ });++ it("ends the cursor after a full last page of a completed crawl", async () => {+ mockCrawl({ cancelled: false, groupStatus: "completed" });++ const first = await getStatus({ limit: "2" });+ expect(first.data).toHaveLength(2);+ expect(first.next).toBeUndefined();+ });+});diff --git a/apps/api/src/controllers/v1/crawl-status.ts b/apps/api/src/controllers/v1/crawl-status.tsindex 8a66ebf25..5499ea57a 100644--- a/apps/api/src/controllers/v1/crawl-status.ts+++ b/apps/api/src/controllers/v1/crawl-status.ts@@ -276,10 +276,11 @@ export async function crawlStatusController(next: string | undefined;};+ const pageSize = end !== undefined ? end - start + 1 : 100;const doneJobs = await scrapeQueue.getGroupJobs(req.params.jobId,"completed",- end !== undefined ? end - start + 1 : 100,+ pageSize,start,logger.child({ zeroDataRetention }),);@@ -322,9 +323,16 @@ export async function crawlStatusController(outputBulkB = {data: scrapes,+ // Only completed documents are paged, so a finished job's cursor ends after+ // its last one. `completed` is read before this page, so a full or+ // size-capped page keeps the cursor for jobs that finished meanwhile.next:- (outputBulkA.total ?? 0) > start + iteratedOver ||- outputBulkA.status !== "completed"+ outputBulkA.status === "scraping" ||+ (outputBulkA.completed ?? 0) > start + iteratedOver ||+ doneJobs.length > iteratedOver ||+ (outputBulkA.status !== "completed" &&+ doneJobs.length > 0 &&+ doneJobs.length === pageSize)? `${req.protocol}://${req.host}/v1/${isBatch ? "batch/scrape" : "crawl"}/${req.params.jobId}?skip=${start + iteratedOver}${req.query.limit ? `&limit=${req.query.limit}` : ""}`: undefined,};diff --git a/apps/api/src/controllers/v2/__tests__/crawl-status-next.test.ts b/apps/api/src/controllers/v2/__tests__/crawl-status-next.test.tsnew file mode 100644index 000000000..33b57c168--- /dev/null+++ b/apps/api/src/controllers/v2/__tests__/crawl-status-next.test.ts@@ -0,0 +1,230 @@+import { vi } from "vitest";++const mocks = vi.hoisted(() => ({+ getCrawl: vi.fn(),+ getGroup: vi.fn(),+ getGroupNumericStats: vi.fn(),+ getGroupJobs: vi.fn(),+ getCrawlError: vi.fn(),+}));++vi.mock("../../../lib/crawl-redis", () => ({+ getCrawl: mocks.getCrawl,+ getCrawlError: mocks.getCrawlError,+ getCrawlExpiry: vi.fn().mockResolvedValue(new Date(0)),+ getCrawlQualifiedJobCount: vi.fn(),+ getDoneJobsOrderedLength: vi.fn(),+ getDoneJobsOrderedUntil: vi.fn(),+ getLastDoneJobTimestamp: vi.fn().mockResolvedValue(null),+ isCrawlKickoffFinished: vi.fn(),+}));++vi.mock("../../../services/worker/nuq-router", () => ({+ crawlGroup: { getGroup: mocks.getGroup },+ scrapeQueue: {+ getGroupAnyJob: vi.fn().mockResolvedValue(null),+ getGroupNumericStats: mocks.getGroupNumericStats,+ getGroupJobs: mocks.getGroupJobs,+ },+}));++vi.mock("../../../services/redis", () => ({+ redisEvictConnection: { smembers: vi.fn().mockResolvedValue([]) },+}));++vi.mock("../../../lib/request-credits-store", () => ({+ readRequestCredits: vi.fn().mockResolvedValue(3),+}));++vi.mock("../../../lib/gcs-jobs", () => ({+ getJobFromGCS: vi.fn().mockResolvedValue(null),+}));++vi.mock("../../../lib/logger", () => ({+ logger: {+ info: vi.fn(),+ warn: vi.fn(),+ error: vi.fn(),+ debug: vi.fn(),+ child: vi.fn().mockReturnThis(),+ },+}));++import { crawlStatusController } from "../crawl-status";++const TEAM_ID = "11111111-1111-1111-1111-111111111111";+const JOB_ID = "22222222-2222-4222-8222-222222222222";++function makeReq(query: Record<string, string> = {}) {+ return {+ params: { jobId: JOB_ID },+ query,+ auth: { team_id: TEAM_ID },+ protocol: "http",+ host: "localhost",+ } as any;+}++function makeRes() {+ const res: any = { status: vi.fn(), json: vi.fn() };+ res.status.mockReturnValue(res);+ res.json.mockReturnValue(res);+ return res;+}++const doc = (url: string) => ({+ id: url,+ returnvalue: { markdown: url, metadata: { sourceURL: url } },+});++// Two completed pages. Pages 0-1 are returned by the first request and the+// page past them (skip=2) is empty. `pending` jobs are still queued or running.+function mockCrawl(options: {+ cancelled: boolean;+ groupStatus: string;+ pending?: number;+ completed?: number;+ crawlError?: string;+}) {+ mocks.getCrawlError.mockResolvedValue(options.crawlError ?? null);+ mocks.getGroup.mockResolvedValue({ status: options.groupStatus });+ mocks.getCrawl.mockResolvedValue({+ team_id: TEAM_ID,+ cancelled: options.cancelled,+ createdAt: Date.now(),+ });+ mocks.getGroupNumericStats.mockResolvedValue({+ completed: options.completed ?? 2,+ active: options.pending ?? 0,+ });+ mocks.getGroupJobs.mockImplementation(+ async (_id: string, _status: string, limit: number, offset: number) =>+ [doc("https://example.com/a"), doc("https://example.com/b")].slice(+ offset,+ offset + limit,+ ),+ );+}++async function getStatus(query: Record<string, string> = {}) {+ const res = makeRes();+ await crawlStatusController(makeReq(query), res);+ return res.json.mock.calls[0][0];+}++describe("crawl status next cursor", () => {+ it("ends the cursor after the last page of a cancelled crawl", async () => {+ mockCrawl({ cancelled: true, groupStatus: "active" });++ const first = await getStatus();+ expect(first.status).toBe("cancelled");+ expect(first.data).toHaveLength(2);+ expect(first.next).toBeUndefined();++ // Following a cursor past the end must not hand back the same cursor.+ const past = await getStatus({ skip: "2" });+ expect(past.data).toHaveLength(0);+ expect(past.next).toBeUndefined();+ });++ it("ends the cursor of a cancelled crawl whose pending jobs are still counted", async () => {+ // total is 5 (2 completed + 3 still queued or running), but only completed+ // jobs have pages, so the page past them is the last one.+ mockCrawl({ cancelled: true, groupStatus: "active", pending: 3 });++ const past = await getStatus({ skip: "2" });+ expect(past.total).toBe(5);+ expect(past.data).toHaveLength(0);+ expect(past.next).toBeUndefined();+ });++ it("ends the cursor after the last page of a completed crawl", async () => {+ mockCrawl({ cancelled: false, groupStatus: "completed" });++ const first = await getStatus();+ expect(first.status).toBe("completed");+ expect(first.next).toBeUndefined();+ });++ it("keeps the cursor while the crawl is still running", async () => {+ mockCrawl({ cancelled: false, groupStatus: "active" });++ const first = await getStatus();+ expect(first.status).toBe("scraping");+ expect(first.next).toBe(`http://localhost/v2/crawl/${JOB_ID}?skip=2`);+ });++ it("keeps the cursor while more completed pages remain", async () => {+ mockCrawl({ cancelled: true, groupStatus: "active" });++ const first = await getStatus({ limit: "1" });+ expect(first.data).toHaveLength(1);+ expect(first.next).toBe(+ `http://localhost/v2/crawl/${JOB_ID}?skip=1&limit=1`,+ );+ });++ it("keeps the cursor on a full page of a cancelled crawl whose jobs finished during the request", async () => {+ mockCrawl({ cancelled: true, groupStatus: "active", completed: 1 });++ const first = await getStatus({ limit: "2" });+ expect(first.data).toHaveLength(2);+ expect(first.next).toBe(+ `http://localhost/v2/crawl/${JOB_ID}?skip=2&limit=2`,+ );++ const past = await getStatus({ skip: "2", limit: "2" });+ expect(past.data).toHaveLength(0);+ expect(past.next).toBeUndefined();+ });++ it("keeps the cursor when the response size cap cuts a page short", async () => {+ mockCrawl({ cancelled: true, groupStatus: "active", completed: 1 });+ const big = {+ id: "big",+ returnvalue: { markdown: "x".repeat(11 * 1024 * 1024), metadata: {} },+ };+ mocks.getGroupJobs.mockImplementation(+ async (_id: string, _status: string, limit: number, offset: number) =>+ [big, doc("https://example.com/b")].slice(offset, offset + limit),+ );++ const first = await getStatus();+ expect(first.data).toHaveLength(1);+ expect(first.next).toBe(`http://localhost/v2/crawl/${JOB_ID}?skip=1`);+ });++ it("ends the cursor of a crawl whose group was cancelled", async () => {+ mockCrawl({ cancelled: false, groupStatus: "cancelled" });++ const first = await getStatus();+ expect(first.status).toBe("cancelled");+ expect(first.data).toHaveLength(2);+ expect(first.next).toBeUndefined();++ const past = await getStatus({ skip: "2" });+ expect(past.data).toHaveLength(0);+ expect(past.next).toBeUndefined();+ });++ it("returns no cursor for a crawl that failed during kickoff", async () => {+ mockCrawl({+ cancelled: false,+ groupStatus: "completed",+ completed: 0,+ crawlError: "queue full",+ });++ const first = await getStatus();+ expect(first.status).toBe("failed");+ expect(first.next).toBeUndefined();+ });++ it("ends the cursor after a full last page of a completed crawl", async () => {+ mockCrawl({ cancelled: false, groupStatus: "completed" });++ const first = await getStatus({ limit: "2" });+ expect(first.data).toHaveLength(2);+ expect(first.next).toBeUndefined();+ });+});diff --git a/apps/api/src/controllers/v2/crawl-status.ts b/apps/api/src/controllers/v2/crawl-status.tsindex 81678c947..89ab75914 100644--- a/apps/api/src/controllers/v2/crawl-status.ts+++ b/apps/api/src/controllers/v2/crawl-status.ts@@ -305,10 +305,11 @@ export async function crawlStatusController(next: string | undefined;};+ const pageSize = end !== undefined ? end - start + 1 : 100;const doneJobs = await scrapeQueue.getGroupJobs(req.params.jobId,"completed",- end !== undefined ? end - start + 1 : 100,+ pageSize,start,logger.child({ zeroDataRetention }),);@@ -351,9 +352,16 @@ export async function crawlStatusController(outputBulkB = {data: scrapes,+ // Only completed documents are paged, so a finished job's cursor ends after+ // its last one. `completed` is read before this page, so a full or+ // size-capped page keeps the cursor for jobs that finished meanwhile.next:- (outputBulkA.total ?? 0) > start + iteratedOver ||- outputBulkA.status !== "completed"+ outputBulkA.status === "scraping" ||+ (outputBulkA.completed ?? 0) > start + iteratedOver ||+ doneJobs.length > iteratedOver ||+ (outputBulkA.status !== "completed" &&+ doneJobs.length > 0 &&+ doneJobs.length === pageSize)? `${req.protocol}://${req.host}/v2/${isBatch ? "batch/scrape" : "crawl"}/${req.params.jobId}?skip=${start + iteratedOver}${req.query.limit ? `&limit=${req.query.limit}` : ""}`: undefined,};