Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(queue-worker): do not kill crawl on one-page error

Gergő Móricz93ac20f930e7
diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.ts
index 5a0b28dbf..3ea976d6b 100644
--- a/apps/api/src/services/queue-worker.ts
+++ b/apps/api/src/services/queue-worker.ts
@@ -568,24 +568,24 @@ async function processJob(job: Job & { id: string }, token: string) {
crawl_id: job.data.crawl_id,
});
- await logJob({
- job_id: job.data.crawl_id,
- success: false,
- message:
- typeof error === "string"
- ? error
- : error.message ??
- "Something went wrong... Contact [email protected]",
- num_docs: 0,
- docs: [],
- time_taken: 0,
- team_id: job.data.team_id,
- mode: job.data.crawlerOptions !== null ? "crawl" : "batch_scrape",
- url: sc ? sc.originUrl ?? job.data.url : job.data.url,
- crawlerOptions: sc ? sc.crawlerOptions : undefined,
- scrapeOptions: sc ? sc.scrapeOptions : job.data.scrapeOptions,
- origin: job.data.origin,
- });
+ // await logJob({
+ // job_id: job.data.crawl_id,
+ // success: false,
+ // message:
+ // typeof error === "string"
+ // ? error
+ // : error.message ??
+ // "Something went wrong... Contact [email protected]",
+ // num_docs: 0,
+ // docs: [],
+ // time_taken: 0,
+ // team_id: job.data.team_id,
+ // mode: job.data.crawlerOptions !== null ? "crawl" : "batch_scrape",
+ // url: sc ? sc.originUrl ?? job.data.url : job.data.url,
+ // crawlerOptions: sc ? sc.crawlerOptions : undefined,
+ // scrapeOptions: sc ? sc.scrapeOptions : job.data.scrapeOptions,
+ // origin: job.data.origin,
+ // });
}
// done(null, data);
return data;