Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Nick: (#2402)
Nicolase0a0fdf83cc4
diff --git a/apps/api/src/controllers/v1/scrape.ts b/apps/api/src/controllers/v1/scrape.tsindex 7f8faac27..84523d2da 100644--- a/apps/api/src/controllers/v1/scrape.ts+++ b/apps/api/src/controllers/v1/scrape.ts@@ -136,6 +136,15 @@ export async function scrapeController(}if (e instanceof TransportableError) {+ // DNS resolution errors should return 200 with success: false+ if (e.code === "SCRAPE_DNS_RESOLUTION_ERROR") {+ return res.status(200).json({+ success: false,+ code: e.code,+ error: e.message,+ });+ }+return res.status(e.code === "SCRAPE_TIMEOUT" ? 408 : 500).json({success: false,code: e.code,diff --git a/apps/api/src/controllers/v2/scrape.ts b/apps/api/src/controllers/v2/scrape.tsindex f56cc1f40..7d3bd08c7 100644--- a/apps/api/src/controllers/v2/scrape.ts+++ b/apps/api/src/controllers/v2/scrape.ts@@ -206,6 +206,18 @@ export async function scrapeController(}if (e instanceof TransportableError) {+ // DNS resolution errors should return 200 with success: false+ if (e.code === "SCRAPE_DNS_RESOLUTION_ERROR") {+ setSpanAttributes(span, {+ "scrape.status_code": 200,+ });+ return res.status(200).json({+ success: false,+ code: e.code,+ error: e.message,+ });+ }+const statusCode = e.code === "SCRAPE_TIMEOUT" ? 408 : 500;setSpanAttributes(span, {"scrape.status_code": statusCode,diff --git a/apps/api/src/lib/scrape-billing.ts b/apps/api/src/lib/scrape-billing.tsindex 67f174af4..ed987eb41 100644--- a/apps/api/src/lib/scrape-billing.ts+++ b/apps/api/src/lib/scrape-billing.ts@@ -7,6 +7,7 @@ import {} from "../controllers/v2/types";import { CostTracking } from "./cost-tracking";import { hasFormatOfType } from "./format-utils";+import { TransportableError } from "./error";const creditsPerPDFPage = 1;const stealthProxyCostBonus = 4;@@ -17,6 +18,7 @@ export async function calculateCreditsToBeBilled(document: Document | null,costTracking: CostTracking | ReturnType<typeof CostTracking.prototype.toJSON>,flags: TeamFlags,+ error?: Error | null,) {const costTrackingJSON: ReturnType<typeof CostTracking.prototype.toJSON> =costTracking instanceof CostTracking ? costTracking.toJSON() : costTracking;@@ -32,6 +34,14 @@ export async function calculateCreditsToBeBilled(creditsToBeBilled = Math.ceil((costTrackingJSON.totalCost ?? 1) * 1800);}+ // Bill for DNS resolution errors+ if (+ error instanceof TransportableError &&+ error.code === "SCRAPE_DNS_RESOLUTION_ERROR"+ ) {+ creditsToBeBilled = 1;+ }+return creditsToBeBilled;}diff --git a/apps/api/src/services/worker/scrape-worker.ts b/apps/api/src/services/worker/scrape-worker.tsindex db1b40f56..79c3d5c79 100644--- a/apps/api/src/services/worker/scrape-worker.ts+++ b/apps/api/src/services/worker/scrape-worker.ts@@ -90,6 +90,7 @@ async function billScrapeJob(logger: Logger,costTracking: CostTracking,flags: TeamFlags,+ error?: Error | null,) {let creditsToBeBilled: number | null = null;@@ -100,6 +101,7 @@ async function billScrapeJob(document,costTracking,flags,+ error,);if (@@ -658,6 +660,7 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {logger,costTracking,(await getACUCTeam(job.data.team_id))?.flags ?? null,+ error instanceof Error ? error : null,);logger.debug("Logging job to DB...");