Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(batchScrape): handle timeout
Móricz Gergő91f52287dbf2
diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.tsindex 21dd472ea..0959424a6 100644--- a/apps/api/src/services/queue-worker.ts+++ b/apps/api/src/services/queue-worker.ts@@ -276,10 +276,18 @@ async function processJob(job: Job & { id: string }, token: string) {});const start = Date.now();- const pipeline = await startWebScraperPipeline({- job,- token,- });+ const pipeline = await Promise.race([+ startWebScraperPipeline({+ job,+ token,+ }),+ ...(job.data.scrapeOptions.timeout !== undefined ? [+ (async () => {+ await sleep(job.data.scrapeOptions.timeout);+ throw new Error("timeout")+ })(),+ ] : [])+ ]);if (!pipeline.success) {// TODO: let's Not do this@@ -486,13 +494,19 @@ async function processJob(job: Job & { id: string }, token: string) {logger.info(`🐂 Job done ${job.id}`);return data;} catch (error) {- logger.error(`🐂 Job errored ${job.id} - ${error}`);+ const isEarlyTimeout = error instanceof Error && error.message === "timeout";- Sentry.captureException(error, {- data: {- job: job.id,- },- });+ if (!isEarlyTimeout) {+ logger.error(`🐂 Job errored ${job.id} - ${error}`);++ Sentry.captureException(error, {+ data: {+ job: job.id,+ },+ });+ } else {+ logger.error(`🐂 Job timed out ${job.id}`);+ }if (error instanceof CustomError) {// Here we handle the error, then save the failed job