Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(api): drain concurrency-limit backlog on crawl/batch cancel (#3561)

tomsideguide23acf6e02c61
diff --git a/apps/api/src/__tests__/snips/v2/batch-scrape.test.ts b/apps/api/src/__tests__/snips/v2/batch-scrape.test.ts
index 05eeb104b..cab07958d 100644
--- a/apps/api/src/__tests__/snips/v2/batch-scrape.test.ts
+++ b/apps/api/src/__tests__/snips/v2/batch-scrape.test.ts
@@ -4,10 +4,13 @@ import {
describeIf,
TEST_PRODUCTION,
TEST_SUITE_WEBSITE,
+ TEST_API_URL,
} from "../lib";
import { batchScrape, scrapeTimeout, idmux, Identity } from "./lib";
+import request from "supertest";
let identity: Identity;
+let lowConcurrencyIdentity: Identity;
beforeAll(async () => {
identity = await idmux({
@@ -15,6 +18,11 @@ beforeAll(async () => {
concurrency: 100,
credits: 1000000,
});
+ lowConcurrencyIdentity = await idmux({
+ name: "batch-scrape-cancel",
+ concurrency: 2,
+ credits: 1000000,
+ });
}, 10000);
describe("Batch scrape tests", () => {
@@ -50,6 +58,50 @@ describe("Batch scrape tests", () => {
scrapeTimeout,
);
+ concurrentIf(ALLOW_TEST_SUITE_WEBSITE)(
+ "cancel flips batch status to cancelled immediately",
+ async () => {
+ const apiKey = lowConcurrencyIdentity.apiKey;
+ const urls = Array.from(
+ { length: 20 },
+ (_, i) => `${TEST_SUITE_WEBSITE}?cancelTest=${i}`,
+ );
+
+ const start = await request(TEST_API_URL)
+ .post("/v2/batch/scrape")
+ .set("Authorization", `Bearer ${apiKey}`)
+ .set("Content-Type", "application/json")
+ .send({ urls });
+ expect(start.statusCode).toBe(200);
+ expect(start.body.success).toBe(true);
+ const id = start.body.id as string;
+
+ const cancel = await request(TEST_API_URL)
+ .delete(`/v2/batch/scrape/${encodeURIComponent(id)}`)
+ .set("Authorization", `Bearer ${apiKey}`)
+ .send();
+ expect(cancel.statusCode).toBe(200);
+ expect(cancel.body.status).toBe("cancelled");
+
+ const statusAfter = await request(TEST_API_URL)
+ .get(`/v2/batch/scrape/${encodeURIComponent(id)}`)
+ .set("Authorization", `Bearer ${apiKey}`)
+ .send();
+ expect(statusAfter.statusCode).toBe(200);
+ expect(statusAfter.body.status).toBe("cancelled");
+ },
+ scrapeTimeout,
+ );
+
+ it.concurrent("cancel rejects unknown batch id with 404", async () => {
+ const unknownId = "00000000-0000-7000-8000-000000000000";
+ const res = await request(TEST_API_URL)
+ .delete(`/v2/batch/scrape/${unknownId}`)
+ .set("Authorization", `Bearer ${identity.apiKey}`)
+ .send();
+ expect(res.statusCode).toBe(404);
+ });
+
describeIf(TEST_PRODUCTION)("JSON format", () => {
it.concurrent(
"works",
diff --git a/apps/api/src/controllers/v1/crawl-cancel.ts b/apps/api/src/controllers/v1/crawl-cancel.ts
index c260ec3d1..cd783f824 100644
--- a/apps/api/src/controllers/v1/crawl-cancel.ts
+++ b/apps/api/src/controllers/v1/crawl-cancel.ts
@@ -1,10 +1,11 @@
import { Response } from "express";
import { logger } from "../../lib/logger";
-import { getCrawl, saveCrawl } from "../../lib/crawl-redis";
+import { getCrawl, getCrawlJobs, saveCrawl } from "../../lib/crawl-redis";
import * as Sentry from "@sentry/node";
import { configDotenv } from "dotenv";
import { RequestWithAuth } from "./types";
import { crawlGroup } from "../../services/worker/nuq";
+import { removeConcurrencyLimitedJobs } from "../../lib/concurrency-limit";
configDotenv();
export async function crawlCancelController(
@@ -37,6 +38,9 @@ export async function crawlCancelController(
logger.error(error);
}
+ const jobIds = await getCrawlJobs(req.params.jobId);
+ await removeConcurrencyLimitedJobs(sc.team_id, jobIds);
+
res.json({
status: "cancelled",
});
diff --git a/apps/api/src/controllers/v1/crawl-status.ts b/apps/api/src/controllers/v1/crawl-status.ts
index 27589a35f..619f67e26 100644
--- a/apps/api/src/controllers/v1/crawl-status.ts
+++ b/apps/api/src/controllers/v1/crawl-status.ts
@@ -215,7 +215,11 @@ export async function crawlStatusController(
total?: number;
creditsUsed?: number;
} = {
- status: group.status === "active" ? "scraping" : group.status,
+ status: sc?.cancelled
+ ? "cancelled"
+ : group.status === "active"
+ ? "scraping"
+ : group.status,
completed: numericStats.completed ?? 0,
total:
(numericStats.completed ?? 0) +
diff --git a/apps/api/src/controllers/v2/crawl-cancel.ts b/apps/api/src/controllers/v2/crawl-cancel.ts
index ba561ee6c..267904011 100644
--- a/apps/api/src/controllers/v2/crawl-cancel.ts
+++ b/apps/api/src/controllers/v2/crawl-cancel.ts
@@ -1,10 +1,11 @@
import { Response } from "express";
import { logger } from "../../lib/logger";
-import { getCrawl, saveCrawl } from "../../lib/crawl-redis";
+import { getCrawl, getCrawlJobs, saveCrawl } from "../../lib/crawl-redis";
import * as Sentry from "@sentry/node";
import { configDotenv } from "dotenv";
import { RequestWithAuth } from "./types";
import { crawlGroup } from "../../services/worker/nuq";
+import { removeConcurrencyLimitedJobs } from "../../lib/concurrency-limit";
configDotenv();
export async function crawlCancelController(
@@ -38,6 +39,9 @@ export async function crawlCancelController(
logger.error(error);
}
+ const jobIds = await getCrawlJobs(req.params.jobId);
+ await removeConcurrencyLimitedJobs(sc.team_id, jobIds);
+
res.json({
status: "cancelled",
});
diff --git a/apps/api/src/controllers/v2/crawl-status.ts b/apps/api/src/controllers/v2/crawl-status.ts
index f15b82f16..98ec8c32e 100644
--- a/apps/api/src/controllers/v2/crawl-status.ts
+++ b/apps/api/src/controllers/v2/crawl-status.ts
@@ -235,7 +235,11 @@ export async function crawlStatusController(
total?: number;
creditsUsed?: number;
} = {
- status: group.status === "active" ? "scraping" : group.status,
+ status: sc?.cancelled
+ ? "cancelled"
+ : group.status === "active"
+ ? "scraping"
+ : group.status,
completed: numericStats.completed ?? 0,
total:
(numericStats.completed ?? 0) +
diff --git a/apps/api/src/lib/concurrency-limit.ts b/apps/api/src/lib/concurrency-limit.ts
index d1f997457..029a67145 100644
--- a/apps/api/src/lib/concurrency-limit.ts
+++ b/apps/api/src/lib/concurrency-limit.ts
@@ -85,6 +85,25 @@ export async function removeConcurrencyLimitActiveJob(
await getRedisConnection().zrem(constructKey(team_id), id);
}
+export async function removeConcurrencyLimitedJobs(
+ team_id: string,
+ job_ids: string[],
+) {
+ if (job_ids.length === 0) return;
+ const redis = getRedisConnection();
+ const queueKey = constructQueueKey(team_id);
+ const chunkSize = 1000;
+ for (let i = 0; i < job_ids.length; i += chunkSize) {
+ const chunk = job_ids.slice(i, i + chunkSize);
+ const pipeline = redis.pipeline();
+ pipeline.zrem(queueKey, ...chunk);
+ for (const id of chunk) {
+ pipeline.del(constructJobKey(id));
+ }
+ await pipeline.exec();
+ }
+}
+
type ConcurrencyLimitedJob = {
id: string;
data: any;