Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(queue-worker): bring done add to earlier

Gergő Móriczdbc6d078710a
diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.ts
index 84c6d42e9..de8f5567c 100644
--- a/apps/api/src/services/queue-worker.ts
+++ b/apps/api/src/services/queue-worker.ts
@@ -997,6 +997,19 @@ async function processJob(job: Job & { id: string }, token: string) {
logger.info(`🐂 Job done ${job.id}`);
return data;
} catch (error) {
+ if (job.data.crawl_id) {
+ const sc = (await getCrawl(job.data.crawl_id)) as StoredCrawl;
+
+ logger.debug("Declaring job as done...");
+ await addCrawlJobDone(job.data.crawl_id, job.id, false);
+ await redisConnection.srem(
+ "crawl:" + job.data.crawl_id + ":visited_unique",
+ normalizeURL(job.data.url, sc),
+ );
+
+ await finishCrawlIfNeeded(job, sc);
+ }
+
const isEarlyTimeout =
error instanceof Error && error.message === "timeout";
const isCancelled =
@@ -1076,39 +1089,6 @@ async function processJob(job: Job & { id: string }, token: string) {
},
true,
);
-
- if (job.data.crawl_id) {
- const sc = (await getCrawl(job.data.crawl_id)) as StoredCrawl;
-
- logger.debug("Declaring job as done...");
- await addCrawlJobDone(job.data.crawl_id, job.id, false);
- await redisConnection.srem(
- "crawl:" + job.data.crawl_id + ":visited_unique",
- normalizeURL(job.data.url, sc),
- );
-
- await finishCrawlIfNeeded(job, sc);
-
- // await logJob({
- // job_id: job.data.crawl_id,
- // success: false,
- // message:
- // typeof error === "string"
- // ? error
- // : error.message ??
- // "Something went wrong... Contact [email protected]",
- // num_docs: 0,
- // docs: [],
- // time_taken: 0,
- // team_id: job.data.team_id,
- // mode: job.data.crawlerOptions !== null ? "crawl" : "batch_scrape",
- // url: sc ? sc.originUrl ?? job.data.url : job.data.url,
- // crawlerOptions: sc ? sc.crawlerOptions : undefined,
- // scrapeOptions: sc ? sc.scrapeOptions : job.data.scrapeOptions,
- // origin: job.data.origin,
- // });
- }
- // done(null, data);
return data;
}
}