Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(scrape): better timeout handling
Móricz Gergő3815d246288c
diff --git a/apps/api/src/controllers/v0/scrape.ts b/apps/api/src/controllers/v0/scrape.tsindex 05fb3c412..9bc41fc1c 100644--- a/apps/api/src/controllers/v0/scrape.ts+++ b/apps/api/src/controllers/v0/scrape.ts@@ -86,7 +86,7 @@ export async function scrapeHelper(try {doc = (await waitForJob<Document>(jobId, timeout));} catch (e) {- if (e instanceof Error && e.message.startsWith("Job wait")) {+ if (e instanceof Error && (e.message.startsWith("Job wait") || e.message === "timeout")) {span.setAttribute("timedOut", true);return {success: false,diff --git a/apps/api/src/controllers/v0/search.ts b/apps/api/src/controllers/v0/search.tsindex e0102406f..4dd38afd8 100644--- a/apps/api/src/controllers/v0/search.ts+++ b/apps/api/src/controllers/v0/search.ts@@ -196,7 +196,7 @@ export async function searchController(req: Request, res: Response) {});return res.status(result.returnCode).json(result);} catch (error) {- if (error instanceof Error && error.message.startsWith("Job wait")) {+ if (error instanceof Error && (error.message.startsWith("Job wait") || error.message === "timeout")) {return res.status(408).json({ error: "Request timed out" });}diff --git a/apps/api/src/controllers/v1/scrape.ts b/apps/api/src/controllers/v1/scrape.tsindex 76fab4333..9c85c91e8 100644--- a/apps/api/src/controllers/v1/scrape.ts+++ b/apps/api/src/controllers/v1/scrape.ts@@ -56,7 +56,7 @@ export async function scrapeController(doc = await waitForJob<Document>(jobId, timeout + totalWait); // TODO: better types for this} catch (e) {logger.error(`Error in scrapeController: ${e}`);- if (e instanceof Error && e.message.startsWith("Job wait")) {+ if (e instanceof Error && (e.message.startsWith("Job wait") || e.message === "timeout")) {return res.status(408).json({success: false,error: "Request timed out",