Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(queue-worker): proactive job cancel
Gergő Móriczcbe67d89a5df
diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.tsindex 81ed955ae..d13cb198e 100644--- a/apps/api/src/services/queue-worker.ts+++ b/apps/api/src/services/queue-worker.ts@@ -739,6 +739,13 @@ async function processJob(job: Job & { id: string }, token: string) {});const start = Date.now();+ if (job.data.crawl_id) {+ const sc = (await getCrawl(job.data.crawl_id)) as StoredCrawl;+ if (sc && sc.cancelled) {+ throw new Error("Parent crawl/batch scrape was cancelled");+ }+ }+const pipeline = await Promise.race([startWebScraperPipeline({job,@@ -983,11 +990,15 @@ async function processJob(job: Job & { id: string }, token: string) {} catch (error) {const isEarlyTimeout =error instanceof Error && error.message === "timeout";+ const isCancelled =+ error instanceof Error && error.message === "Parent crawl/batch scrape was cancelled";if (isEarlyTimeout) {logger.error(`🐂 Job timed out ${job.id}`);} else if (error instanceof RacedRedirectError) {logger.warn(`🐂 Job got redirect raced ${job.id}, silently failing`);+ } else if (isCancelled) {+ logger.warn(`🐂 Job got cancelled, silently failing`);} else {logger.error(`🐂 Job errored ${job.id} - ${error}`, { error });