Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(api): drain concurrency-limit backlog on crawl/batch cancel (#3561)
tomsideguide23acf6e02c61
diff --git a/apps/api/src/__tests__/snips/v2/batch-scrape.test.ts b/apps/api/src/__tests__/snips/v2/batch-scrape.test.tsindex 05eeb104b..cab07958d 100644--- a/apps/api/src/__tests__/snips/v2/batch-scrape.test.ts+++ b/apps/api/src/__tests__/snips/v2/batch-scrape.test.ts@@ -4,10 +4,13 @@ import {describeIf,TEST_PRODUCTION,TEST_SUITE_WEBSITE,+ TEST_API_URL,} from "../lib";import { batchScrape, scrapeTimeout, idmux, Identity } from "./lib";+import request from "supertest";let identity: Identity;+let lowConcurrencyIdentity: Identity;beforeAll(async () => {identity = await idmux({@@ -15,6 +18,11 @@ beforeAll(async () => {concurrency: 100,credits: 1000000,});+ lowConcurrencyIdentity = await idmux({+ name: "batch-scrape-cancel",+ concurrency: 2,+ credits: 1000000,+ });}, 10000);describe("Batch scrape tests", () => {@@ -50,6 +58,50 @@ describe("Batch scrape tests", () => {scrapeTimeout,);+ concurrentIf(ALLOW_TEST_SUITE_WEBSITE)(+ "cancel flips batch status to cancelled immediately",+ async () => {+ const apiKey = lowConcurrencyIdentity.apiKey;+ const urls = Array.from(+ { length: 20 },+ (_, i) => `${TEST_SUITE_WEBSITE}?cancelTest=${i}`,+ );++ const start = await request(TEST_API_URL)+ .post("/v2/batch/scrape")+ .set("Authorization", `Bearer ${apiKey}`)+ .set("Content-Type", "application/json")+ .send({ urls });+ expect(start.statusCode).toBe(200);+ expect(start.body.success).toBe(true);+ const id = start.body.id as string;++ const cancel = await request(TEST_API_URL)+ .delete(`/v2/batch/scrape/${encodeURIComponent(id)}`)+ .set("Authorization", `Bearer ${apiKey}`)+ .send();+ expect(cancel.statusCode).toBe(200);+ expect(cancel.body.status).toBe("cancelled");++ const statusAfter = await request(TEST_API_URL)+ .get(`/v2/batch/scrape/${encodeURIComponent(id)}`)+ .set("Authorization", `Bearer ${apiKey}`)+ .send();+ expect(statusAfter.statusCode).toBe(200);+ expect(statusAfter.body.status).toBe("cancelled");+ },+ scrapeTimeout,+ );++ it.concurrent("cancel rejects unknown batch id with 404", async () => {+ const unknownId = "00000000-0000-7000-8000-000000000000";+ const res = await request(TEST_API_URL)+ .delete(`/v2/batch/scrape/${unknownId}`)+ .set("Authorization", `Bearer ${identity.apiKey}`)+ .send();+ expect(res.statusCode).toBe(404);+ });+describeIf(TEST_PRODUCTION)("JSON format", () => {it.concurrent("works",diff --git a/apps/api/src/controllers/v1/crawl-cancel.ts b/apps/api/src/controllers/v1/crawl-cancel.tsindex c260ec3d1..cd783f824 100644--- a/apps/api/src/controllers/v1/crawl-cancel.ts+++ b/apps/api/src/controllers/v1/crawl-cancel.ts@@ -1,10 +1,11 @@import { Response } from "express";import { logger } from "../../lib/logger";-import { getCrawl, saveCrawl } from "../../lib/crawl-redis";+import { getCrawl, getCrawlJobs, saveCrawl } from "../../lib/crawl-redis";import * as Sentry from "@sentry/node";import { configDotenv } from "dotenv";import { RequestWithAuth } from "./types";import { crawlGroup } from "../../services/worker/nuq";+import { removeConcurrencyLimitedJobs } from "../../lib/concurrency-limit";configDotenv();export async function crawlCancelController(@@ -37,6 +38,9 @@ export async function crawlCancelController(logger.error(error);}+ const jobIds = await getCrawlJobs(req.params.jobId);+ await removeConcurrencyLimitedJobs(sc.team_id, jobIds);+res.json({status: "cancelled",});diff --git a/apps/api/src/controllers/v1/crawl-status.ts b/apps/api/src/controllers/v1/crawl-status.tsindex 27589a35f..619f67e26 100644--- a/apps/api/src/controllers/v1/crawl-status.ts+++ b/apps/api/src/controllers/v1/crawl-status.ts@@ -215,7 +215,11 @@ export async function crawlStatusController(total?: number;creditsUsed?: number;} = {- status: group.status === "active" ? "scraping" : group.status,+ status: sc?.cancelled+ ? "cancelled"+ : group.status === "active"+ ? "scraping"+ : group.status,completed: numericStats.completed ?? 0,total:(numericStats.completed ?? 0) +diff --git a/apps/api/src/controllers/v2/crawl-cancel.ts b/apps/api/src/controllers/v2/crawl-cancel.tsindex ba561ee6c..267904011 100644--- a/apps/api/src/controllers/v2/crawl-cancel.ts+++ b/apps/api/src/controllers/v2/crawl-cancel.ts@@ -1,10 +1,11 @@import { Response } from "express";import { logger } from "../../lib/logger";-import { getCrawl, saveCrawl } from "../../lib/crawl-redis";+import { getCrawl, getCrawlJobs, saveCrawl } from "../../lib/crawl-redis";import * as Sentry from "@sentry/node";import { configDotenv } from "dotenv";import { RequestWithAuth } from "./types";import { crawlGroup } from "../../services/worker/nuq";+import { removeConcurrencyLimitedJobs } from "../../lib/concurrency-limit";configDotenv();export async function crawlCancelController(@@ -38,6 +39,9 @@ export async function crawlCancelController(logger.error(error);}+ const jobIds = await getCrawlJobs(req.params.jobId);+ await removeConcurrencyLimitedJobs(sc.team_id, jobIds);+res.json({status: "cancelled",});diff --git a/apps/api/src/controllers/v2/crawl-status.ts b/apps/api/src/controllers/v2/crawl-status.tsindex f15b82f16..98ec8c32e 100644--- a/apps/api/src/controllers/v2/crawl-status.ts+++ b/apps/api/src/controllers/v2/crawl-status.ts@@ -235,7 +235,11 @@ export async function crawlStatusController(total?: number;creditsUsed?: number;} = {- status: group.status === "active" ? "scraping" : group.status,+ status: sc?.cancelled+ ? "cancelled"+ : group.status === "active"+ ? "scraping"+ : group.status,completed: numericStats.completed ?? 0,total:(numericStats.completed ?? 0) +diff --git a/apps/api/src/lib/concurrency-limit.ts b/apps/api/src/lib/concurrency-limit.tsindex d1f997457..029a67145 100644--- a/apps/api/src/lib/concurrency-limit.ts+++ b/apps/api/src/lib/concurrency-limit.ts@@ -85,6 +85,25 @@ export async function removeConcurrencyLimitActiveJob(await getRedisConnection().zrem(constructKey(team_id), id);}+export async function removeConcurrencyLimitedJobs(+ team_id: string,+ job_ids: string[],+) {+ if (job_ids.length === 0) return;+ const redis = getRedisConnection();+ const queueKey = constructQueueKey(team_id);+ const chunkSize = 1000;+ for (let i = 0; i < job_ids.length; i += chunkSize) {+ const chunk = job_ids.slice(i, i + chunkSize);+ const pipeline = redis.pipeline();+ pipeline.zrem(queueKey, ...chunk);+ for (const id of chunk) {+ pipeline.del(constructJobKey(id));+ }+ await pipeline.exec();+ }+}+type ConcurrencyLimitedJob = {id: string;data: any;