Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(api): dns + crawl denial errors (#2469)

delong36158ee03b86d
diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.ts
index cef02feef..72b012a47 100644
--- a/apps/api/src/scraper/scrapeURL/index.ts
+++ b/apps/api/src/scraper/scrapeURL/index.ts
@@ -1238,6 +1238,9 @@ export async function scrapeURL(
} else if (error instanceof ProxySelectionError) {
errorType = "ProxySelectionError";
meta.logger.warn("scrapeURL: Proxy selection error", { error });
+ } else if (error instanceof DNSResolutionError) {
+ errorType = "DNSResolutionError";
+ meta.logger.warn("scrapeURL: DNS resolution error", { error });
} else if (error instanceof AbortManagerThrownError) {
errorType = "AbortManagerThrownError";
throw error.inner;
diff --git a/apps/api/src/services/worker/scrape-worker.ts b/apps/api/src/services/worker/scrape-worker.ts
index 5c3f01be2..fb1b9f226 100644
--- a/apps/api/src/services/worker/scrape-worker.ts
+++ b/apps/api/src/services/worker/scrape-worker.ts
@@ -286,7 +286,7 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {
const reason =
filterResult.denialReason ||
"Redirected target URL is not allowed by crawlOptions";
- throw new Error(reason);
+ throw new CrawlDenialError(reason);
}
// Only re-set originUrl if it's different from the current hostname
@@ -309,7 +309,7 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {
(await getACUCTeam(job.data.team_id))?.flags ?? null,
)
) {
- throw new Error(BLOCKLISTED_URL_MESSAGE); // TODO: make this its own error type that is ignored by error tracking
+ throw new CrawlDenialError(BLOCKLISTED_URL_MESSAGE); // TODO: make this its own error type that is ignored by error tracking
}
const p1 = generateURLPermutations(normalizeURL(doc.metadata.url, sc));
@@ -433,7 +433,7 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {
const reason =
filterResult.denialReasons.get(url) ||
"Source URL is not allowed by crawl configuration";
- throw new Error(reason);
+ throw new CrawlDenialError(reason);
}
}
}