Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(api): don't bill where stealth proxy was unsupported (#2484)
tomdb1a4e2ba92d
diff --git a/apps/api/src/lib/scrape-billing.ts b/apps/api/src/lib/scrape-billing.tsindex ed987eb41..e28bdf240 100644--- a/apps/api/src/lib/scrape-billing.ts+++ b/apps/api/src/lib/scrape-billing.ts@@ -8,6 +8,7 @@ import {import { CostTracking } from "./cost-tracking";import { hasFormatOfType } from "./format-utils";import { TransportableError } from "./error";+import { FeatureFlag } from "../scraper/scrapeURL/engines";const creditsPerPDFPage = 1;const stealthProxyCostBonus = 4;@@ -19,6 +20,7 @@ export async function calculateCreditsToBeBilled(costTracking: CostTracking | ReturnType<typeof CostTracking.prototype.toJSON>,flags: TeamFlags,error?: Error | null,+ unsupportedFeatures?: Set<FeatureFlag>,) {const costTrackingJSON: ReturnType<typeof CostTracking.prototype.toJSON> =costTracking instanceof CostTracking ? costTracking.toJSON() : costTracking;@@ -77,7 +79,10 @@ export async function calculateCreditsToBeBilled(creditsToBeBilled += creditsPerPDFPage * (document.metadata.numPages - 1);}- if (document?.metadata?.proxyUsed === "stealth") {+ if (+ document?.metadata?.proxyUsed === "stealth" &&+ !unsupportedFeatures?.has("stealthProxy") // if stealth proxy was unsupported, don't bill for it+ ) {creditsToBeBilled += stealthProxyCostBonus;}diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.tsindex 3967437c1..0738f1f9c 100644--- a/apps/api/src/scraper/scrapeURL/index.ts+++ b/apps/api/src/scraper/scrapeURL/index.ts@@ -78,6 +78,7 @@ export type ScrapeUrlResponse =| {success: true;document: Document;+ unsupportedFeatures?: Set<FeatureFlag>;}| {success: false;@@ -838,6 +839,7 @@ async function scrapeURLLoop(meta: Meta): Promise<ScrapeUrlResponse> {return {success: true,document,+ unsupportedFeatures: result.unsupportedFeatures,};});}diff --git a/apps/api/src/services/worker/scrape-worker.ts b/apps/api/src/services/worker/scrape-worker.tsindex abeb11ffa..a15ea050f 100644--- a/apps/api/src/services/worker/scrape-worker.ts+++ b/apps/api/src/services/worker/scrape-worker.ts@@ -73,6 +73,7 @@ import {} from "../../lib/otel-tracer";import { ScrapeUrlResponse } from "../../scraper/scrapeURL";import { logScrape } from "../logging/log_job";+import { FeatureFlag } from "../../scraper/scrapeURL/engines";configDotenv();@@ -93,6 +94,7 @@ async function billScrapeJob(costTracking: CostTracking,flags: TeamFlags,error?: Error | null,+ unsupportedFeatures?: Set<FeatureFlag>,) {let creditsToBeBilled: number | null = null;@@ -104,6 +106,7 @@ async function billScrapeJob(costTracking,flags,error,+ unsupportedFeatures,);if (@@ -452,6 +455,8 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {logger,costTracking,(await getACUCTeam(job.data.team_id))?.flags ?? null,+ undefined,+ pipeline.unsupportedFeatures,);doc.metadata.creditsUsed = credits_billed ?? undefined;@@ -521,6 +526,8 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {logger,costTracking,(await getACUCTeam(job.data.team_id))?.flags ?? null,+ undefined,+ pipeline.unsupportedFeatures,);doc.metadata.creditsUsed = credits_billed ?? undefined;