Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(billing): keep the lockdown surcharge on json scrapes (#4486)
claude[bot]ef39d9751814
diff --git a/apps/api/src/lib/scrape-billing.test.ts b/apps/api/src/lib/scrape-billing.test.tsindex cab395609..32150383c 100644--- a/apps/api/src/lib/scrape-billing.test.ts+++ b/apps/api/src/lib/scrape-billing.test.ts@@ -221,6 +221,71 @@ describe("calculateCreditsToBeBilled", () => {expect(credits).toBe(3);});++ // =========================================+ // lockdown + json surcharge stacking+ // =========================================++ const promptInjectionGuardCall = {+ type: "other",+ model: "vertex/gemini",+ cost: 0,+ metadata: { module: "scrapeURL", method: "checkForPromptInjection" },+ };++ // `guard` means the caller asked for the prompt injection check and the+ // guard actually ran. Both conditions must hold for the +4 guard fee.+ const billScrape = (args: {+ lockdown?: boolean;+ json?: boolean;+ guard?: boolean;+ }) =>+ calculateCreditsToBeBilled(+ {+ lockdown: args.lockdown,+ formats: args.json+ ? [{ type: "json", schema: {}, checkPromptInjection: args.guard }]+ : [{ type: "markdown" }],+ } as any,+ {+ teamId: "team-id",+ orgId: null,+ },+ {+ metadata: {+ statusCode: 200,+ proxyUsed: "basic",+ },+ } as any,+ {+ totalCost: 0,+ calls: args.guard ? [promptInjectionGuardCall] : [],+ } as any,+ {} as any,+ );++ // Public billing docs quote 5 credits for a json scrape. Guard that figure.+ it("bills a json scrape alone at 5 credits", async () => {+ expect(await billScrape({ json: true })).toBe(5);+ });++ it("bills a lockdown scrape alone at 5 credits", async () => {+ expect(await billScrape({ lockdown: true })).toBe(5);+ });++ it("keeps the lockdown surcharge on a json scrape (9 credits)", async () => {+ expect(await billScrape({ lockdown: true, json: true })).toBe(9);+ });++ it("stacks lockdown, json, and the prompt injection guard (13 credits)", async () => {+ expect(await billScrape({ lockdown: true, json: true, guard: true })).toBe(+ 13,+ );+ });++ it("bills json plus the prompt injection guard at 9 credits", async () => {+ expect(await billScrape({ json: true, guard: true })).toBe(9);+ });});// =========================================diff --git a/apps/api/src/lib/scrape-billing.ts b/apps/api/src/lib/scrape-billing.tsindex 9741ef8dc..5f062c5af 100644--- a/apps/api/src/lib/scrape-billing.ts+++ b/apps/api/src/lib/scrape-billing.ts@@ -17,6 +17,7 @@ import { UnsafeDomainBlockedError } from "./threat-protection/error";const creditsPerPDFPage = 1;const unblockedDomainCostBonus = 4;const xTwitterCostBonus = 29;+const jsonCostBonus = 4;const redactPIICostBonus = 4;// Each additional PDF page also gets redacted through fire-privacy, so// the per-page surcharge mirrors the +4 base — same tier as lockdown.@@ -150,7 +151,9 @@ export async function calculateCreditsToBeBilled(hasFormatOfType(options.formats, "json") ||changeTrackingFormat?.modes?.includes("json")) {- creditsToBeBilled = 5;+ // Additive, so an earlier surcharge such as lockdown survives. A json+ // scrape on its own still totals 5 credits (1 base + 4).+ creditsToBeBilled += jsonCostBonus;}if (hasFormatOfType(options.formats, "json")?.checkPromptInjection) {diff --git a/apps/api/src/services/monitoring/store.test.ts b/apps/api/src/services/monitoring/store.test.tsindex bc82b8be0..c06003e2e 100644--- a/apps/api/src/services/monitoring/store.test.ts+++ b/apps/api/src/services/monitoring/store.test.ts@@ -45,6 +45,39 @@ describe("monitoring store credit helpers", () => {expect(estimateMonitorCreditsPerRun(targets, true)).toBe(6);});+ it("keeps the lockdown cost on a json monitor", () => {+ const targets: MonitorTarget[] = [+ {+ id: "target-1",+ type: "scrape",+ urls: ["https://example.com/a"],+ scrapeOptions: {+ lockdown: true,+ formats: [{ type: "json", schema: {} }],+ },+ },+ ];++ // 1 base + 4 lockdown + 4 json.+ expect(estimateMonitorCreditsPerRun(targets, false)).toBe(9);+ });++ it("estimates the prompt injection guard cost on a json monitor", () => {+ const targets: MonitorTarget[] = [+ {+ id: "target-1",+ type: "scrape",+ urls: ["https://example.com/a"],+ scrapeOptions: {+ formats: [{ type: "json", schema: {}, checkPromptInjection: true }],+ },+ },+ ];++ // 1 base + 4 json + 4 prompt injection guard.+ expect(estimateMonitorCreditsPerRun(targets, false)).toBe(9);+ });+it("uses target options when page rows do not have recorded scrape credits", () => {const targets: MonitorTarget[] = [{diff --git a/apps/api/src/services/monitoring/store.ts b/apps/api/src/services/monitoring/store.tsindex d6ddb13ef..d3adbed83 100644--- a/apps/api/src/services/monitoring/store.ts+++ b/apps/api/src/services/monitoring/store.ts@@ -37,7 +37,7 @@ function ensureTargetIds(targets: Array<Record<string, any>>): MonitorTarget[] {}const BASE_SCRAPE_CREDITS_PER_PAGE = 1;-const JSON_SCRAPE_CREDITS_PER_PAGE = 5;+const JSON_SCRAPE_CREDIT_BONUS = 4;const DETERMINISTIC_JSON_SCRAPE_CREDITS_PER_PAGE = 7;const SCRAPE_OPTION_CREDIT_BONUS = 4;const JUDGE_CREDITS_PER_PAGE = 1;@@ -76,6 +76,18 @@ function hasAnyFormatOfType(formats: unknown, types: string[]): boolean {return types.some(type => hasFormatOfType(formats, type));}+function requestsPromptInjectionCheck(formats: unknown): boolean {+ if (!Array.isArray(formats)) return false;+ return formats.some(+ format =>+ !!format &&+ typeof format === "object" &&+ formatType(format) === "json" &&+ "checkPromptInjection" in format &&+ format.checkPromptInjection === true,+ );+}+function requestsJsonChangeTracking(formats: unknown): boolean {if (!Array.isArray(formats)) return false;return formats.some(format => {@@ -105,11 +117,19 @@ function estimateBaseCreditsPerPage(credits += SCRAPE_OPTION_CREDIT_BONUS;}- // Deterministic JSON costs more than plain JSON; both override the base scrape credit.+ // Deterministic JSON is a flat per-page rate that overrides the base scrape+ // credit. Plain JSON adds its premium on top, so an earlier surcharge such+ // as lockdown survives. This mirrors calculateCreditsToBeBilled.if (usesDeterministicJson) {credits = DETERMINISTIC_JSON_SCRAPE_CREDITS_PER_PAGE;} else if (usesJsonCredits) {- credits = JSON_SCRAPE_CREDITS_PER_PAGE;+ credits += JSON_SCRAPE_CREDIT_BONUS;+ }++ // The prompt injection guard bills +4 in calculateCreditsToBeBilled. The+ // estimate cannot know whether the guard ran, so it assumes it does.+ if (requestsPromptInjectionCheck(formats)) {+ credits += SCRAPE_OPTION_CREDIT_BONUS;}if (hasAnyFormatOfType(formats, ["question", "query"])) {