Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(queue-worker): do not kill crawl on one-page error
Gergő Móricz93ac20f930e7
diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.tsindex 5a0b28dbf..3ea976d6b 100644--- a/apps/api/src/services/queue-worker.ts+++ b/apps/api/src/services/queue-worker.ts@@ -568,24 +568,24 @@ async function processJob(job: Job & { id: string }, token: string) {crawl_id: job.data.crawl_id,});- await logJob({- job_id: job.data.crawl_id,- success: false,- message:- typeof error === "string"- ? error- : error.message ??- "Something went wrong... Contact [email protected]",- num_docs: 0,- docs: [],- time_taken: 0,- team_id: job.data.team_id,- mode: job.data.crawlerOptions !== null ? "crawl" : "batch_scrape",- url: sc ? sc.originUrl ?? job.data.url : job.data.url,- crawlerOptions: sc ? sc.crawlerOptions : undefined,- scrapeOptions: sc ? sc.scrapeOptions : job.data.scrapeOptions,- origin: job.data.origin,- });+ // await logJob({+ // job_id: job.data.crawl_id,+ // success: false,+ // message:+ // typeof error === "string"+ // ? error+ // : error.message ??+ // "Something went wrong... Contact [email protected]",+ // num_docs: 0,+ // docs: [],+ // time_taken: 0,+ // team_id: job.data.team_id,+ // mode: job.data.crawlerOptions !== null ? "crawl" : "batch_scrape",+ // url: sc ? sc.originUrl ?? job.data.url : job.data.url,+ // crawlerOptions: sc ? sc.crawlerOptions : undefined,+ // scrapeOptions: sc ? sc.scrapeOptions : job.data.scrapeOptions,+ // origin: job.data.origin,+ // });}// done(null, data);return data;