Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat: improve DNS resolution error message (#1724)
Gergő Móricz2b87ea65996f
diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.tsindex 850b4c964..4a26d8768 100644--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts@@ -9,6 +9,7 @@ import {SiteError,SSLError,UnsupportedFileError,+ DNSResolutionError} from "../../error";import { MockState } from "../../lib/mock";import { fireEngineURL } from "./scrape";@@ -187,6 +188,11 @@ export async function fireEngineCheckStatus(} else {throw new SiteError(code);}+ } else if (+ typeof status.error === "string" &&+ status.error.includes("Dns resolution error for hostname: ")+ ) {+ throw new DNSResolutionError(status.error.split("Dns resolution error for hostname: ")[1]);} else if (typeof status.error === "string" &&status.error.includes("File size exceeds")diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.tsindex b4babf308..25bf1c0c4 100644--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts@@ -16,6 +16,7 @@ import {import {ActionError,EngineError,+ DNSResolutionError,SiteError,SSLError,TimeoutError,@@ -99,6 +100,7 @@ async function performFireEngineScrape<error instanceof EngineError ||error instanceof SiteError ||error instanceof SSLError ||+ error instanceof DNSResolutionError ||error instanceof ActionError ||error instanceof UnsupportedFileError) {diff --git a/apps/api/src/scraper/scrapeURL/error.ts b/apps/api/src/scraper/scrapeURL/error.tsindex a47f5a86a..f139cfbd2 100644--- a/apps/api/src/scraper/scrapeURL/error.ts+++ b/apps/api/src/scraper/scrapeURL/error.ts@@ -98,6 +98,12 @@ export class PDFInsufficientTimeError extends Error {}}+export class DNSResolutionError extends Error {+ constructor(hostname: string) {+ super(`DNS resolution failed for hostname: ${hostname}. Please check if the domain is valid and accessible.`);+ }+}+export class IndexMissError extends Error {constructor() {super("Index doesn't have the page we're looking for");diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.tsindex ba8e3256c..5e0e97805 100644--- a/apps/api/src/scraper/scrapeURL/index.ts+++ b/apps/api/src/scraper/scrapeURL/index.ts@@ -24,6 +24,7 @@ import {SSLError,PDFInsufficientTimeError,IndexMissError,+ DNSResolutionError,} from "./error";import { executeTransformers } from "./transformers";import { LLMRefusalError } from "./transformers/llmExtract";@@ -371,6 +372,8 @@ async function scrapeURLLoop(meta: Meta): Promise<ScrapeUrlResponse> {throw error;} else if (error instanceof SSLError) {throw error;+ } else if (error instanceof DNSResolutionError) {+ throw error;} else if (error instanceof ActionError) {throw error;} else if (error instanceof UnsupportedFileError) {