Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

Nick: (#2402)

Nicolase0a0fdf83cc4
diff --git a/apps/api/src/controllers/v1/scrape.ts b/apps/api/src/controllers/v1/scrape.ts
index 7f8faac27..84523d2da 100644
--- a/apps/api/src/controllers/v1/scrape.ts
+++ b/apps/api/src/controllers/v1/scrape.ts
@@ -136,6 +136,15 @@ export async function scrapeController(
}
if (e instanceof TransportableError) {
+ // DNS resolution errors should return 200 with success: false
+ if (e.code === "SCRAPE_DNS_RESOLUTION_ERROR") {
+ return res.status(200).json({
+ success: false,
+ code: e.code,
+ error: e.message,
+ });
+ }
+
return res.status(e.code === "SCRAPE_TIMEOUT" ? 408 : 500).json({
success: false,
code: e.code,
diff --git a/apps/api/src/controllers/v2/scrape.ts b/apps/api/src/controllers/v2/scrape.ts
index f56cc1f40..7d3bd08c7 100644
--- a/apps/api/src/controllers/v2/scrape.ts
+++ b/apps/api/src/controllers/v2/scrape.ts
@@ -206,6 +206,18 @@ export async function scrapeController(
}
if (e instanceof TransportableError) {
+ // DNS resolution errors should return 200 with success: false
+ if (e.code === "SCRAPE_DNS_RESOLUTION_ERROR") {
+ setSpanAttributes(span, {
+ "scrape.status_code": 200,
+ });
+ return res.status(200).json({
+ success: false,
+ code: e.code,
+ error: e.message,
+ });
+ }
+
const statusCode = e.code === "SCRAPE_TIMEOUT" ? 408 : 500;
setSpanAttributes(span, {
"scrape.status_code": statusCode,
diff --git a/apps/api/src/lib/scrape-billing.ts b/apps/api/src/lib/scrape-billing.ts
index 67f174af4..ed987eb41 100644
--- a/apps/api/src/lib/scrape-billing.ts
+++ b/apps/api/src/lib/scrape-billing.ts
@@ -7,6 +7,7 @@ import {
} from "../controllers/v2/types";
import { CostTracking } from "./cost-tracking";
import { hasFormatOfType } from "./format-utils";
+import { TransportableError } from "./error";
const creditsPerPDFPage = 1;
const stealthProxyCostBonus = 4;
@@ -17,6 +18,7 @@ export async function calculateCreditsToBeBilled(
document: Document | null,
costTracking: CostTracking | ReturnType<typeof CostTracking.prototype.toJSON>,
flags: TeamFlags,
+ error?: Error | null,
) {
const costTrackingJSON: ReturnType<typeof CostTracking.prototype.toJSON> =
costTracking instanceof CostTracking ? costTracking.toJSON() : costTracking;
@@ -32,6 +34,14 @@ export async function calculateCreditsToBeBilled(
creditsToBeBilled = Math.ceil((costTrackingJSON.totalCost ?? 1) * 1800);
}
+ // Bill for DNS resolution errors
+ if (
+ error instanceof TransportableError &&
+ error.code === "SCRAPE_DNS_RESOLUTION_ERROR"
+ ) {
+ creditsToBeBilled = 1;
+ }
+
return creditsToBeBilled;
}
diff --git a/apps/api/src/services/worker/scrape-worker.ts b/apps/api/src/services/worker/scrape-worker.ts
index db1b40f56..79c3d5c79 100644
--- a/apps/api/src/services/worker/scrape-worker.ts
+++ b/apps/api/src/services/worker/scrape-worker.ts
@@ -90,6 +90,7 @@ async function billScrapeJob(
logger: Logger,
costTracking: CostTracking,
flags: TeamFlags,
+ error?: Error | null,
) {
let creditsToBeBilled: number | null = null;
@@ -100,6 +101,7 @@ async function billScrapeJob(
document,
costTracking,
flags,
+ error,
);
if (
@@ -658,6 +660,7 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {
logger,
costTracking,
(await getACUCTeam(job.data.team_id))?.flags ?? null,
+ error instanceof Error ? error : null,
);
logger.debug("Logging job to DB...");