Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(queue-worker): proactive job cancel

Gergő Móriczcbe67d89a5df
diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.ts
index 81ed955ae..d13cb198e 100644
--- a/apps/api/src/services/queue-worker.ts
+++ b/apps/api/src/services/queue-worker.ts
@@ -739,6 +739,13 @@ async function processJob(job: Job & { id: string }, token: string) {
});
const start = Date.now();
+ if (job.data.crawl_id) {
+ const sc = (await getCrawl(job.data.crawl_id)) as StoredCrawl;
+ if (sc && sc.cancelled) {
+ throw new Error("Parent crawl/batch scrape was cancelled");
+ }
+ }
+
const pipeline = await Promise.race([
startWebScraperPipeline({
job,
@@ -983,11 +990,15 @@ async function processJob(job: Job & { id: string }, token: string) {
} catch (error) {
const isEarlyTimeout =
error instanceof Error && error.message === "timeout";
+ const isCancelled =
+ error instanceof Error && error.message === "Parent crawl/batch scrape was cancelled";
if (isEarlyTimeout) {
logger.error(`🐂 Job timed out ${job.id}`);
} else if (error instanceof RacedRedirectError) {
logger.warn(`🐂 Job got redirect raced ${job.id}, silently failing`);
+ } else if (isCancelled) {
+ logger.warn(`🐂 Job got cancelled, silently failing`);
} else {
logger.error(`🐂 Job errored ${job.id} - ${error}`, { error });