Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(api): dns + crawl denial errors (#2469)
delong36158ee03b86d
diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.tsindex cef02feef..72b012a47 100644--- a/apps/api/src/scraper/scrapeURL/index.ts+++ b/apps/api/src/scraper/scrapeURL/index.ts@@ -1238,6 +1238,9 @@ export async function scrapeURL(} else if (error instanceof ProxySelectionError) {errorType = "ProxySelectionError";meta.logger.warn("scrapeURL: Proxy selection error", { error });+ } else if (error instanceof DNSResolutionError) {+ errorType = "DNSResolutionError";+ meta.logger.warn("scrapeURL: DNS resolution error", { error });} else if (error instanceof AbortManagerThrownError) {errorType = "AbortManagerThrownError";throw error.inner;diff --git a/apps/api/src/services/worker/scrape-worker.ts b/apps/api/src/services/worker/scrape-worker.tsindex 5c3f01be2..fb1b9f226 100644--- a/apps/api/src/services/worker/scrape-worker.ts+++ b/apps/api/src/services/worker/scrape-worker.ts@@ -286,7 +286,7 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {const reason =filterResult.denialReason ||"Redirected target URL is not allowed by crawlOptions";- throw new Error(reason);+ throw new CrawlDenialError(reason);}// Only re-set originUrl if it's different from the current hostname@@ -309,7 +309,7 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {(await getACUCTeam(job.data.team_id))?.flags ?? null,)) {- throw new Error(BLOCKLISTED_URL_MESSAGE); // TODO: make this its own error type that is ignored by error tracking+ throw new CrawlDenialError(BLOCKLISTED_URL_MESSAGE); // TODO: make this its own error type that is ignored by error tracking}const p1 = generateURLPermutations(normalizeURL(doc.metadata.url, sc));@@ -433,7 +433,7 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {const reason =filterResult.denialReasons.get(url) ||"Source URL is not allowed by crawl configuration";- throw new Error(reason);+ throw new CrawlDenialError(reason);}}}