Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(api): don't bill where stealth proxy was unsupported (#2484)

tomdb1a4e2ba92d
diff --git a/apps/api/src/lib/scrape-billing.ts b/apps/api/src/lib/scrape-billing.ts
index ed987eb41..e28bdf240 100644
--- a/apps/api/src/lib/scrape-billing.ts
+++ b/apps/api/src/lib/scrape-billing.ts
@@ -8,6 +8,7 @@ import {
import { CostTracking } from "./cost-tracking";
import { hasFormatOfType } from "./format-utils";
import { TransportableError } from "./error";
+import { FeatureFlag } from "../scraper/scrapeURL/engines";
const creditsPerPDFPage = 1;
const stealthProxyCostBonus = 4;
@@ -19,6 +20,7 @@ export async function calculateCreditsToBeBilled(
costTracking: CostTracking | ReturnType<typeof CostTracking.prototype.toJSON>,
flags: TeamFlags,
error?: Error | null,
+ unsupportedFeatures?: Set<FeatureFlag>,
) {
const costTrackingJSON: ReturnType<typeof CostTracking.prototype.toJSON> =
costTracking instanceof CostTracking ? costTracking.toJSON() : costTracking;
@@ -77,7 +79,10 @@ export async function calculateCreditsToBeBilled(
creditsToBeBilled += creditsPerPDFPage * (document.metadata.numPages - 1);
}
- if (document?.metadata?.proxyUsed === "stealth") {
+ if (
+ document?.metadata?.proxyUsed === "stealth" &&
+ !unsupportedFeatures?.has("stealthProxy") // if stealth proxy was unsupported, don't bill for it
+ ) {
creditsToBeBilled += stealthProxyCostBonus;
}
diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.ts
index 3967437c1..0738f1f9c 100644
--- a/apps/api/src/scraper/scrapeURL/index.ts
+++ b/apps/api/src/scraper/scrapeURL/index.ts
@@ -78,6 +78,7 @@ export type ScrapeUrlResponse =
| {
success: true;
document: Document;
+ unsupportedFeatures?: Set<FeatureFlag>;
}
| {
success: false;
@@ -838,6 +839,7 @@ async function scrapeURLLoop(meta: Meta): Promise<ScrapeUrlResponse> {
return {
success: true,
document,
+ unsupportedFeatures: result.unsupportedFeatures,
};
});
}
diff --git a/apps/api/src/services/worker/scrape-worker.ts b/apps/api/src/services/worker/scrape-worker.ts
index abeb11ffa..a15ea050f 100644
--- a/apps/api/src/services/worker/scrape-worker.ts
+++ b/apps/api/src/services/worker/scrape-worker.ts
@@ -73,6 +73,7 @@ import {
} from "../../lib/otel-tracer";
import { ScrapeUrlResponse } from "../../scraper/scrapeURL";
import { logScrape } from "../logging/log_job";
+import { FeatureFlag } from "../../scraper/scrapeURL/engines";
configDotenv();
@@ -93,6 +94,7 @@ async function billScrapeJob(
costTracking: CostTracking,
flags: TeamFlags,
error?: Error | null,
+ unsupportedFeatures?: Set<FeatureFlag>,
) {
let creditsToBeBilled: number | null = null;
@@ -104,6 +106,7 @@ async function billScrapeJob(
costTracking,
flags,
error,
+ unsupportedFeatures,
);
if (
@@ -452,6 +455,8 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {
logger,
costTracking,
(await getACUCTeam(job.data.team_id))?.flags ?? null,
+ undefined,
+ pipeline.unsupportedFeatures,
);
doc.metadata.creditsUsed = credits_billed ?? undefined;
@@ -521,6 +526,8 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {
logger,
costTracking,
(await getACUCTeam(job.data.team_id))?.flags ?? null,
+ undefined,
+ pipeline.unsupportedFeatures,
);
doc.metadata.creditsUsed = credits_billed ?? undefined;