Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(queue-worker): bring done add to earlier
Gergő Móriczdbc6d078710a
diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.tsindex 84c6d42e9..de8f5567c 100644--- a/apps/api/src/services/queue-worker.ts+++ b/apps/api/src/services/queue-worker.ts@@ -997,6 +997,19 @@ async function processJob(job: Job & { id: string }, token: string) {logger.info(`🐂 Job done ${job.id}`);return data;} catch (error) {+ if (job.data.crawl_id) {+ const sc = (await getCrawl(job.data.crawl_id)) as StoredCrawl;++ logger.debug("Declaring job as done...");+ await addCrawlJobDone(job.data.crawl_id, job.id, false);+ await redisConnection.srem(+ "crawl:" + job.data.crawl_id + ":visited_unique",+ normalizeURL(job.data.url, sc),+ );++ await finishCrawlIfNeeded(job, sc);+ }+const isEarlyTimeout =error instanceof Error && error.message === "timeout";const isCancelled =@@ -1076,39 +1089,6 @@ async function processJob(job: Job & { id: string }, token: string) {},true,);-- if (job.data.crawl_id) {- const sc = (await getCrawl(job.data.crawl_id)) as StoredCrawl;-- logger.debug("Declaring job as done...");- await addCrawlJobDone(job.data.crawl_id, job.id, false);- await redisConnection.srem(- "crawl:" + job.data.crawl_id + ":visited_unique",- normalizeURL(job.data.url, sc),- );-- await finishCrawlIfNeeded(job, sc);-- // await logJob({- // job_id: job.data.crawl_id,- // success: false,- // message:- // typeof error === "string"- // ? error- // : error.message ??- // "Something went wrong... Contact [email protected]",- // num_docs: 0,- // docs: [],- // time_taken: 0,- // team_id: job.data.team_id,- // mode: job.data.crawlerOptions !== null ? "crawl" : "batch_scrape",- // url: sc ? sc.originUrl ?? job.data.url : job.data.url,- // crawlerOptions: sc ? sc.crawlerOptions : undefined,- // scrapeOptions: sc ? sc.scrapeOptions : job.data.scrapeOptions,- // origin: job.data.origin,- // });- }- // done(null, data);return data;}}