Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat: improve DNS resolution error message (#1724)

Gergő Móricz2b87ea65996f
diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts
index 850b4c964..4a26d8768 100644
--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts
@@ -9,6 +9,7 @@ import {
SiteError,
SSLError,
UnsupportedFileError,
+ DNSResolutionError
} from "../../error";
import { MockState } from "../../lib/mock";
import { fireEngineURL } from "./scrape";
@@ -187,6 +188,11 @@ export async function fireEngineCheckStatus(
} else {
throw new SiteError(code);
}
+ } else if (
+ typeof status.error === "string" &&
+ status.error.includes("Dns resolution error for hostname: ")
+ ) {
+ throw new DNSResolutionError(status.error.split("Dns resolution error for hostname: ")[1]);
} else if (
typeof status.error === "string" &&
status.error.includes("File size exceeds")
diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts
index b4babf308..25bf1c0c4 100644
--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts
@@ -16,6 +16,7 @@ import {
import {
ActionError,
EngineError,
+ DNSResolutionError,
SiteError,
SSLError,
TimeoutError,
@@ -99,6 +100,7 @@ async function performFireEngineScrape<
error instanceof EngineError ||
error instanceof SiteError ||
error instanceof SSLError ||
+ error instanceof DNSResolutionError ||
error instanceof ActionError ||
error instanceof UnsupportedFileError
) {
diff --git a/apps/api/src/scraper/scrapeURL/error.ts b/apps/api/src/scraper/scrapeURL/error.ts
index a47f5a86a..f139cfbd2 100644
--- a/apps/api/src/scraper/scrapeURL/error.ts
+++ b/apps/api/src/scraper/scrapeURL/error.ts
@@ -98,6 +98,12 @@ export class PDFInsufficientTimeError extends Error {
}
}
+export class DNSResolutionError extends Error {
+ constructor(hostname: string) {
+ super(`DNS resolution failed for hostname: ${hostname}. Please check if the domain is valid and accessible.`);
+ }
+}
+
export class IndexMissError extends Error {
constructor() {
super("Index doesn't have the page we're looking for");
diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.ts
index ba8e3256c..5e0e97805 100644
--- a/apps/api/src/scraper/scrapeURL/index.ts
+++ b/apps/api/src/scraper/scrapeURL/index.ts
@@ -24,6 +24,7 @@ import {
SSLError,
PDFInsufficientTimeError,
IndexMissError,
+ DNSResolutionError,
} from "./error";
import { executeTransformers } from "./transformers";
import { LLMRefusalError } from "./transformers/llmExtract";
@@ -371,6 +372,8 @@ async function scrapeURLLoop(meta: Meta): Promise<ScrapeUrlResponse> {
throw error;
} else if (error instanceof SSLError) {
throw error;
+ } else if (error instanceof DNSResolutionError) {
+ throw error;
} else if (error instanceof ActionError) {
throw error;
} else if (error instanceof UnsupportedFileError) {