Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(api): end the crawl status cursor after a cancelled or failed job's last page (#4863)

Gergő Móricz7e4a959dca0e
diff --git a/.github/workflows/test-server.yml b/.github/workflows/test-server.yml
index 8db3b596b..73a644bb1 100644
--- a/.github/workflows/test-server.yml
+++ b/.github/workflows/test-server.yml
@@ -243,6 +243,15 @@ jobs:
run: pnpm exec vitest run src/controllers/v2/__tests__/browser-delete.test.ts --reporter=default
working-directory: apps/api
+ - name: Run crawl status cursor regression tests
+ if: matrix.engine == 'fetch' && matrix.proxy == 'no-proxy' && matrix.nuq == 'postgres'
+ run: >-
+ pnpm exec vitest run
+ src/controllers/v1/__tests__/crawl-status-next.test.ts
+ src/controllers/v2/__tests__/crawl-status-next.test.ts
+ --reporter=default
+ working-directory: apps/api
+
- name: Run agent interop auth and concurrency regression tests
if: matrix.engine == 'fetch' && matrix.proxy == 'no-proxy' && matrix.nuq == 'postgres'
run: >-
diff --git a/apps/api/src/__tests__/snips/v2/crawl-status-cursor.test.ts b/apps/api/src/__tests__/snips/v2/crawl-status-cursor.test.ts
new file mode 100644
index 000000000..1bcb0c6f3
--- /dev/null
+++ b/apps/api/src/__tests__/snips/v2/crawl-status-cursor.test.ts
@@ -0,0 +1,143 @@
+import {
+ ALLOW_TEST_SUITE_WEBSITE,
+ concurrentIf,
+ TEST_API_URL,
+ TEST_SUITE_WEBSITE,
+} from "../lib";
+import { idmux, Identity, scrapeTimeout } from "./lib";
+import request from "supertest";
+import { describe, expect, beforeAll } from "vitest";
+
+let identity: Identity;
+let lowConcurrencyIdentity: Identity;
+
+beforeAll(async () => {
+ identity = await idmux({
+ name: "crawl-status-cursor",
+ concurrency: 100,
+ credits: 1000000,
+ });
+ lowConcurrencyIdentity = await idmux({
+ name: "crawl-status-cursor-cancel",
+ concurrency: 1,
+ credits: 1000000,
+ });
+}, 10000);
+
+const sleep = (ms: number) => new Promise(r => setTimeout(r, ms));
+
+async function getStatus(path: string, apiKey: string) {
+ const res = await request(TEST_API_URL)
+ .get(path)
+ .set("Authorization", `Bearer ${apiKey}`)
+ .send();
+ expect(res.statusCode).toBe(200);
+ return res.body;
+}
+
+// Follows `next` like the SDKs do, with a hop cap so a cursor that never ends
+// fails the test instead of hanging it.
+async function followCursor(firstPath: string, apiKey: string) {
+ const docs: any[] = [];
+ let path: string | undefined = firstPath;
+ let hops = 0;
+ let last: any;
+ while (path !== undefined && hops < 20) {
+ last = await getStatus(path, apiKey);
+ docs.push(...last.data);
+ hops++;
+ path = last.next
+ ? new URL(last.next).pathname + new URL(last.next).search
+ : undefined;
+ }
+ return { docs, hops, last, ended: path === undefined };
+}
+
+async function startBatch(urls: string[], apiKey: string) {
+ const res = await request(TEST_API_URL)
+ .post("/v2/batch/scrape")
+ .set("Authorization", `Bearer ${apiKey}`)
+ .set("Content-Type", "application/json")
+ .send({ urls });
+ expect(res.statusCode).toBe(200);
+ expect(res.body.success).toBe(true);
+ return res.body.id as string;
+}
+
+describe("Crawl status cursor", () => {
+ concurrentIf(ALLOW_TEST_SUITE_WEBSITE)(
+ "ends after the last page of a completed job",
+ async () => {
+ const apiKey = identity.apiKey;
+ const id = await startBatch(
+ Array.from(
+ { length: 5 },
+ (_, i) => `${TEST_SUITE_WEBSITE}?cursorDone=${i}`,
+ ),
+ apiKey,
+ );
+
+ let status: any;
+ do {
+ if (status) await sleep(250);
+ status = await getStatus(`/v2/batch/scrape/${id}`, apiKey);
+ } while (status.status === "scraping");
+ expect(status.status).toBe("completed");
+
+ const result = await followCursor(
+ `/v2/batch/scrape/${id}?limit=2`,
+ apiKey,
+ );
+ expect(result.ended).toBe(true);
+ expect(result.hops).toBe(Math.max(1, Math.ceil(status.completed / 2)));
+ expect(result.docs).toHaveLength(status.completed);
+ expect(result.last.next).toBeUndefined();
+ },
+ scrapeTimeout * 2,
+ );
+
+ concurrentIf(ALLOW_TEST_SUITE_WEBSITE)(
+ "ends after the last page of a cancelled job",
+ async () => {
+ const apiKey = lowConcurrencyIdentity.apiKey;
+ const id = await startBatch(
+ Array.from(
+ { length: 20 },
+ (_, i) => `${TEST_SUITE_WEBSITE}?cursorCancel=${i}`,
+ ),
+ apiKey,
+ );
+
+ let status: any;
+ do {
+ if (status) await sleep(100);
+ status = await getStatus(`/v2/batch/scrape/${id}`, apiKey);
+ } while (status.status === "scraping" && status.completed < 1);
+ expect(status.status).toBe("scraping");
+
+ const cancel = await request(TEST_API_URL)
+ .delete(`/v2/batch/scrape/${id}`)
+ .set("Authorization", `Bearer ${apiKey}`)
+ .send();
+ expect(cancel.statusCode).toBe(200);
+
+ const completedBeforeCancel = status.completed;
+ status = await getStatus(`/v2/batch/scrape/${id}`, apiKey);
+ expect(status.status).toBe("cancelled");
+
+ const result = await followCursor(
+ `/v2/batch/scrape/${id}?limit=2`,
+ apiKey,
+ );
+ expect(result.ended).toBe(true);
+ expect(result.last.next).toBeUndefined();
+ expect(result.docs.length).toBeGreaterThanOrEqual(completedBeforeCancel);
+
+ // No more than 20 documents can ever exist, so this page is past the end.
+ const past = await getStatus(`/v2/batch/scrape/${id}?skip=20`, apiKey);
+ expect(past.data).toHaveLength(0);
+ expect(past.next).toBeUndefined();
+ },
+ scrapeTimeout * 2,
+ );
+});
diff --git a/apps/api/src/controllers/v1/__tests__/crawl-status-next.test.ts b/apps/api/src/controllers/v1/__tests__/crawl-status-next.test.ts
new file mode 100644
index 000000000..b9bdbb0c9
--- /dev/null
+++ b/apps/api/src/controllers/v1/__tests__/crawl-status-next.test.ts
@@ -0,0 +1,230 @@
+import { vi } from "vitest";
+
+const mocks = vi.hoisted(() => ({
+ getCrawl: vi.fn(),
+ getGroup: vi.fn(),
+ getGroupNumericStats: vi.fn(),
+ getGroupJobs: vi.fn(),
+ getCrawlError: vi.fn(),
+}));
+
+vi.mock("../../../lib/crawl-redis", () => ({
+ getCrawl: mocks.getCrawl,
+ getCrawlError: mocks.getCrawlError,
+ getCrawlExpiry: vi.fn().mockResolvedValue(new Date(0)),
+ getCrawlQualifiedJobCount: vi.fn(),
+ getDoneJobsOrderedLength: vi.fn(),
+ getDoneJobsOrderedUntil: vi.fn(),
+ getLastDoneJobTimestamp: vi.fn().mockResolvedValue(null),
+ isCrawlKickoffFinished: vi.fn(),
+}));
+
+vi.mock("../../../services/worker/nuq-router", () => ({
+ crawlGroup: { getGroup: mocks.getGroup },
+ scrapeQueue: {
+ getGroupAnyJob: vi.fn().mockResolvedValue(null),
+ getGroupNumericStats: mocks.getGroupNumericStats,
+ getGroupJobs: mocks.getGroupJobs,
+ },
+}));
+
+vi.mock("../../../services/redis", () => ({
+ redisEvictConnection: { smembers: vi.fn().mockResolvedValue([]) },
+}));
+
+vi.mock("../../../lib/request-credits-store", () => ({
+ readRequestCredits: vi.fn().mockResolvedValue(3),
+}));
+
+vi.mock("../../../lib/gcs-jobs", () => ({
+ getJobFromGCS: vi.fn().mockResolvedValue(null),
+}));
+
+vi.mock("../../../lib/logger", () => ({
+ logger: {
+ info: vi.fn(),
+ warn: vi.fn(),
+ error: vi.fn(),
+ debug: vi.fn(),
+ child: vi.fn().mockReturnThis(),
+ },
+}));
+
+import { crawlStatusController } from "../crawl-status";
+
+const TEAM_ID = "11111111-1111-1111-1111-111111111111";
+const JOB_ID = "22222222-2222-4222-8222-222222222222";
+
+function makeReq(query: Record<string, string> = {}) {
+ return {
+ params: { jobId: JOB_ID },
+ query,
+ auth: { team_id: TEAM_ID },
+ protocol: "http",
+ host: "localhost",
+ } as any;
+}
+
+function makeRes() {
+ const res: any = { status: vi.fn(), json: vi.fn() };
+ res.status.mockReturnValue(res);
+ res.json.mockReturnValue(res);
+ return res;
+}
+
+const doc = (url: string) => ({
+ id: url,
+ returnvalue: { markdown: url, metadata: { sourceURL: url } },
+});
+
+// Two completed pages. Pages 0-1 are returned by the first request and the
+// page past them (skip=2) is empty. `pending` jobs are still queued or running.
+function mockCrawl(options: {
+ cancelled: boolean;
+ groupStatus: string;
+ pending?: number;
+ completed?: number;
+ crawlError?: string;
+}) {
+ mocks.getCrawlError.mockResolvedValue(options.crawlError ?? null);
+ mocks.getGroup.mockResolvedValue({ status: options.groupStatus });
+ mocks.getCrawl.mockResolvedValue({
+ team_id: TEAM_ID,
+ cancelled: options.cancelled,
+ createdAt: Date.now(),
+ });
+ mocks.getGroupNumericStats.mockResolvedValue({
+ completed: options.completed ?? 2,
+ active: options.pending ?? 0,
+ });
+ mocks.getGroupJobs.mockImplementation(
+ async (_id: string, _status: string, limit: number, offset: number) =>
+ [doc("https://example.com/a"), doc("https://example.com/b")].slice(
+ offset,
+ offset + limit,
+ ),
+ );
+}
+
+async function getStatus(query: Record<string, string> = {}) {
+ const res = makeRes();
+ await crawlStatusController(makeReq(query), res);
+ return res.json.mock.calls[0][0];
+}
+
+describe("crawl status next cursor", () => {
+ it("ends the cursor after the last page of a cancelled crawl", async () => {
+ mockCrawl({ cancelled: true, groupStatus: "active" });
+
+ const first = await getStatus();
+ expect(first.status).toBe("cancelled");
+ expect(first.data).toHaveLength(2);
+ expect(first.next).toBeUndefined();
+
+ // Following a cursor past the end must not hand back the same cursor.
+ const past = await getStatus({ skip: "2" });
+ expect(past.data).toHaveLength(0);
+ expect(past.next).toBeUndefined();
+ });
+
+ it("ends the cursor of a cancelled crawl whose pending jobs are still counted", async () => {
+ // total is 5 (2 completed + 3 still queued or running), but only completed
+ // jobs have pages, so the page past them is the last one.
+ mockCrawl({ cancelled: true, groupStatus: "active", pending: 3 });
+
+ const past = await getStatus({ skip: "2" });
+ expect(past.total).toBe(5);
+ expect(past.data).toHaveLength(0);
+ expect(past.next).toBeUndefined();
+ });
+
+ it("ends the cursor after the last page of a completed crawl", async () => {
+ mockCrawl({ cancelled: false, groupStatus: "completed" });
+
+ const first = await getStatus();
+ expect(first.status).toBe("completed");
+ expect(first.next).toBeUndefined();
+ });
+
+ it("keeps the cursor while the crawl is still running", async () => {
+ mockCrawl({ cancelled: false, groupStatus: "active" });
+
+ const first = await getStatus();
+ expect(first.status).toBe("scraping");
+ expect(first.next).toBe(`http://localhost/v1/crawl/${JOB_ID}?skip=2`);
+ });
+
+ it("keeps the cursor while more completed pages remain", async () => {
+ mockCrawl({ cancelled: true, groupStatus: "active" });
+
+ const first = await getStatus({ limit: "1" });
+ expect(first.data).toHaveLength(1);
+ expect(first.next).toBe(
+ `http://localhost/v1/crawl/${JOB_ID}?skip=1&limit=1`,
+ );
+ });
+
+ it("keeps the cursor on a full page of a cancelled crawl whose jobs finished during the request", async () => {
+ mockCrawl({ cancelled: true, groupStatus: "active", completed: 1 });
+
+ const first = await getStatus({ limit: "2" });
+ expect(first.data).toHaveLength(2);
+ expect(first.next).toBe(
+ `http://localhost/v1/crawl/${JOB_ID}?skip=2&limit=2`,
+ );
+
+ const past = await getStatus({ skip: "2", limit: "2" });
+ expect(past.data).toHaveLength(0);
+ expect(past.next).toBeUndefined();
+ });
+
+ it("keeps the cursor when the response size cap cuts a page short", async () => {
+ mockCrawl({ cancelled: true, groupStatus: "active", completed: 1 });
+ const big = {
+ id: "big",
+ returnvalue: { markdown: "x".repeat(11 * 1024 * 1024), metadata: {} },
+ };
+ mocks.getGroupJobs.mockImplementation(
+ async (_id: string, _status: string, limit: number, offset: number) =>
+ [big, doc("https://example.com/b")].slice(offset, offset + limit),
+ );
+
+ const first = await getStatus();
+ expect(first.data).toHaveLength(1);
+ expect(first.next).toBe(`http://localhost/v1/crawl/${JOB_ID}?skip=1`);
+ });
+
+ it("ends the cursor of a crawl whose group was cancelled", async () => {
+ mockCrawl({ cancelled: false, groupStatus: "cancelled" });
+
+ const first = await getStatus();
+ expect(first.status).toBe("cancelled");
+ expect(first.data).toHaveLength(2);
+ expect(first.next).toBeUndefined();
+
+ const past = await getStatus({ skip: "2" });
+ expect(past.data).toHaveLength(0);
+ expect(past.next).toBeUndefined();
+ });
+
+ it("returns no cursor for a crawl that failed during kickoff", async () => {
+ mockCrawl({
+ cancelled: false,
+ groupStatus: "completed",
+ completed: 0,
+ crawlError: "queue full",
+ });
+
+ const first = await getStatus();
+ expect(first.status).toBe("failed");
+ expect(first.next).toBeUndefined();
+ });
+
+ it("ends the cursor after a full last page of a completed crawl", async () => {
+ mockCrawl({ cancelled: false, groupStatus: "completed" });
+
+ const first = await getStatus({ limit: "2" });
+ expect(first.data).toHaveLength(2);
+ expect(first.next).toBeUndefined();
+ });
+});
diff --git a/apps/api/src/controllers/v1/crawl-status.ts b/apps/api/src/controllers/v1/crawl-status.ts
index 8a66ebf25..5499ea57a 100644
--- a/apps/api/src/controllers/v1/crawl-status.ts
+++ b/apps/api/src/controllers/v1/crawl-status.ts
@@ -276,10 +276,11 @@ export async function crawlStatusController(
next: string | undefined;
};
+ const pageSize = end !== undefined ? end - start + 1 : 100;
const doneJobs = await scrapeQueue.getGroupJobs(
req.params.jobId,
"completed",
- end !== undefined ? end - start + 1 : 100,
+ pageSize,
start,
logger.child({ zeroDataRetention }),
);
@@ -322,9 +323,16 @@ export async function crawlStatusController(
outputBulkB = {
data: scrapes,
+ // Only completed documents are paged, so a finished job's cursor ends after
+ // its last one. `completed` is read before this page, so a full or
+ // size-capped page keeps the cursor for jobs that finished meanwhile.
next:
- (outputBulkA.total ?? 0) > start + iteratedOver ||
- outputBulkA.status !== "completed"
+ outputBulkA.status === "scraping" ||
+ (outputBulkA.completed ?? 0) > start + iteratedOver ||
+ doneJobs.length > iteratedOver ||
+ (outputBulkA.status !== "completed" &&
+ doneJobs.length > 0 &&
+ doneJobs.length === pageSize)
? `${req.protocol}://${req.host}/v1/${isBatch ? "batch/scrape" : "crawl"}/${req.params.jobId}?skip=${start + iteratedOver}${req.query.limit ? `&limit=${req.query.limit}` : ""}`
: undefined,
};
diff --git a/apps/api/src/controllers/v2/__tests__/crawl-status-next.test.ts b/apps/api/src/controllers/v2/__tests__/crawl-status-next.test.ts
new file mode 100644
index 000000000..33b57c168
--- /dev/null
+++ b/apps/api/src/controllers/v2/__tests__/crawl-status-next.test.ts
@@ -0,0 +1,230 @@
+import { vi } from "vitest";
+
+const mocks = vi.hoisted(() => ({
+ getCrawl: vi.fn(),
+ getGroup: vi.fn(),
+ getGroupNumericStats: vi.fn(),
+ getGroupJobs: vi.fn(),
+ getCrawlError: vi.fn(),
+}));
+
+vi.mock("../../../lib/crawl-redis", () => ({
+ getCrawl: mocks.getCrawl,
+ getCrawlError: mocks.getCrawlError,
+ getCrawlExpiry: vi.fn().mockResolvedValue(new Date(0)),
+ getCrawlQualifiedJobCount: vi.fn(),
+ getDoneJobsOrderedLength: vi.fn(),
+ getDoneJobsOrderedUntil: vi.fn(),
+ getLastDoneJobTimestamp: vi.fn().mockResolvedValue(null),
+ isCrawlKickoffFinished: vi.fn(),
+}));
+
+vi.mock("../../../services/worker/nuq-router", () => ({
+ crawlGroup: { getGroup: mocks.getGroup },
+ scrapeQueue: {
+ getGroupAnyJob: vi.fn().mockResolvedValue(null),
+ getGroupNumericStats: mocks.getGroupNumericStats,
+ getGroupJobs: mocks.getGroupJobs,
+ },
+}));
+
+vi.mock("../../../services/redis", () => ({
+ redisEvictConnection: { smembers: vi.fn().mockResolvedValue([]) },
+}));
+
+vi.mock("../../../lib/request-credits-store", () => ({
+ readRequestCredits: vi.fn().mockResolvedValue(3),
+}));
+
+vi.mock("../../../lib/gcs-jobs", () => ({
+ getJobFromGCS: vi.fn().mockResolvedValue(null),
+}));
+
+vi.mock("../../../lib/logger", () => ({
+ logger: {
+ info: vi.fn(),
+ warn: vi.fn(),
+ error: vi.fn(),
+ debug: vi.fn(),
+ child: vi.fn().mockReturnThis(),
+ },
+}));
+
+import { crawlStatusController } from "../crawl-status";
+
+const TEAM_ID = "11111111-1111-1111-1111-111111111111";
+const JOB_ID = "22222222-2222-4222-8222-222222222222";
+
+function makeReq(query: Record<string, string> = {}) {
+ return {
+ params: { jobId: JOB_ID },
+ query,
+ auth: { team_id: TEAM_ID },
+ protocol: "http",
+ host: "localhost",
+ } as any;
+}
+
+function makeRes() {
+ const res: any = { status: vi.fn(), json: vi.fn() };
+ res.status.mockReturnValue(res);
+ res.json.mockReturnValue(res);
+ return res;
+}
+
+const doc = (url: string) => ({
+ id: url,
+ returnvalue: { markdown: url, metadata: { sourceURL: url } },
+});
+
+// Two completed pages. Pages 0-1 are returned by the first request and the
+// page past them (skip=2) is empty. `pending` jobs are still queued or running.
+function mockCrawl(options: {
+ cancelled: boolean;
+ groupStatus: string;
+ pending?: number;
+ completed?: number;
+ crawlError?: string;
+}) {
+ mocks.getCrawlError.mockResolvedValue(options.crawlError ?? null);
+ mocks.getGroup.mockResolvedValue({ status: options.groupStatus });
+ mocks.getCrawl.mockResolvedValue({
+ team_id: TEAM_ID,
+ cancelled: options.cancelled,
+ createdAt: Date.now(),
+ });
+ mocks.getGroupNumericStats.mockResolvedValue({
+ completed: options.completed ?? 2,
+ active: options.pending ?? 0,
+ });
+ mocks.getGroupJobs.mockImplementation(
+ async (_id: string, _status: string, limit: number, offset: number) =>
+ [doc("https://example.com/a"), doc("https://example.com/b")].slice(
+ offset,
+ offset + limit,
+ ),
+ );
+}
+
+async function getStatus(query: Record<string, string> = {}) {
+ const res = makeRes();
+ await crawlStatusController(makeReq(query), res);
+ return res.json.mock.calls[0][0];
+}
+
+describe("crawl status next cursor", () => {
+ it("ends the cursor after the last page of a cancelled crawl", async () => {
+ mockCrawl({ cancelled: true, groupStatus: "active" });
+
+ const first = await getStatus();
+ expect(first.status).toBe("cancelled");
+ expect(first.data).toHaveLength(2);
+ expect(first.next).toBeUndefined();
+
+ // Following a cursor past the end must not hand back the same cursor.
+ const past = await getStatus({ skip: "2" });
+ expect(past.data).toHaveLength(0);
+ expect(past.next).toBeUndefined();
+ });
+
+ it("ends the cursor of a cancelled crawl whose pending jobs are still counted", async () => {
+ // total is 5 (2 completed + 3 still queued or running), but only completed
+ // jobs have pages, so the page past them is the last one.
+ mockCrawl({ cancelled: true, groupStatus: "active", pending: 3 });
+
+ const past = await getStatus({ skip: "2" });
+ expect(past.total).toBe(5);
+ expect(past.data).toHaveLength(0);
+ expect(past.next).toBeUndefined();
+ });
+
+ it("ends the cursor after the last page of a completed crawl", async () => {
+ mockCrawl({ cancelled: false, groupStatus: "completed" });
+
+ const first = await getStatus();
+ expect(first.status).toBe("completed");
+ expect(first.next).toBeUndefined();
+ });
+
+ it("keeps the cursor while the crawl is still running", async () => {
+ mockCrawl({ cancelled: false, groupStatus: "active" });
+
+ const first = await getStatus();
+ expect(first.status).toBe("scraping");
+ expect(first.next).toBe(`http://localhost/v2/crawl/${JOB_ID}?skip=2`);
+ });
+
+ it("keeps the cursor while more completed pages remain", async () => {
+ mockCrawl({ cancelled: true, groupStatus: "active" });
+
+ const first = await getStatus({ limit: "1" });
+ expect(first.data).toHaveLength(1);
+ expect(first.next).toBe(
+ `http://localhost/v2/crawl/${JOB_ID}?skip=1&limit=1`,
+ );
+ });
+
+ it("keeps the cursor on a full page of a cancelled crawl whose jobs finished during the request", async () => {
+ mockCrawl({ cancelled: true, groupStatus: "active", completed: 1 });
+
+ const first = await getStatus({ limit: "2" });
+ expect(first.data).toHaveLength(2);
+ expect(first.next).toBe(
+ `http://localhost/v2/crawl/${JOB_ID}?skip=2&limit=2`,
+ );
+
+ const past = await getStatus({ skip: "2", limit: "2" });
+ expect(past.data).toHaveLength(0);
+ expect(past.next).toBeUndefined();
+ });
+
+ it("keeps the cursor when the response size cap cuts a page short", async () => {
+ mockCrawl({ cancelled: true, groupStatus: "active", completed: 1 });
+ const big = {
+ id: "big",
+ returnvalue: { markdown: "x".repeat(11 * 1024 * 1024), metadata: {} },
+ };
+ mocks.getGroupJobs.mockImplementation(
+ async (_id: string, _status: string, limit: number, offset: number) =>
+ [big, doc("https://example.com/b")].slice(offset, offset + limit),
+ );
+
+ const first = await getStatus();
+ expect(first.data).toHaveLength(1);
+ expect(first.next).toBe(`http://localhost/v2/crawl/${JOB_ID}?skip=1`);
+ });
+
+ it("ends the cursor of a crawl whose group was cancelled", async () => {
+ mockCrawl({ cancelled: false, groupStatus: "cancelled" });
+
+ const first = await getStatus();
+ expect(first.status).toBe("cancelled");
+ expect(first.data).toHaveLength(2);
+ expect(first.next).toBeUndefined();
+
+ const past = await getStatus({ skip: "2" });
+ expect(past.data).toHaveLength(0);
+ expect(past.next).toBeUndefined();
+ });
+
+ it("returns no cursor for a crawl that failed during kickoff", async () => {
+ mockCrawl({
+ cancelled: false,
+ groupStatus: "completed",
+ completed: 0,
+ crawlError: "queue full",
+ });
+
+ const first = await getStatus();
+ expect(first.status).toBe("failed");
+ expect(first.next).toBeUndefined();
+ });
+
+ it("ends the cursor after a full last page of a completed crawl", async () => {
+ mockCrawl({ cancelled: false, groupStatus: "completed" });
+
+ const first = await getStatus({ limit: "2" });
+ expect(first.data).toHaveLength(2);
+ expect(first.next).toBeUndefined();
+ });
+});
diff --git a/apps/api/src/controllers/v2/crawl-status.ts b/apps/api/src/controllers/v2/crawl-status.ts
index 81678c947..89ab75914 100644
--- a/apps/api/src/controllers/v2/crawl-status.ts
+++ b/apps/api/src/controllers/v2/crawl-status.ts
@@ -305,10 +305,11 @@ export async function crawlStatusController(
next: string | undefined;
};
+ const pageSize = end !== undefined ? end - start + 1 : 100;
const doneJobs = await scrapeQueue.getGroupJobs(
req.params.jobId,
"completed",
- end !== undefined ? end - start + 1 : 100,
+ pageSize,
start,
logger.child({ zeroDataRetention }),
);
@@ -351,9 +352,16 @@ export async function crawlStatusController(
outputBulkB = {
data: scrapes,
+ // Only completed documents are paged, so a finished job's cursor ends after
+ // its last one. `completed` is read before this page, so a full or
+ // size-capped page keeps the cursor for jobs that finished meanwhile.
next:
- (outputBulkA.total ?? 0) > start + iteratedOver ||
- outputBulkA.status !== "completed"
+ outputBulkA.status === "scraping" ||
+ (outputBulkA.completed ?? 0) > start + iteratedOver ||
+ doneJobs.length > iteratedOver ||
+ (outputBulkA.status !== "completed" &&
+ doneJobs.length > 0 &&
+ doneJobs.length === pageSize)
? `${req.protocol}://${req.host}/v2/${isBatch ? "batch/scrape" : "crawl"}/${req.params.jobId}?skip=${start + iteratedOver}${req.query.limit ? `&limit=${req.query.limit}` : ""}`
: undefined,
};