Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(api): don't bill the prompt injection guard when it fails open (#4747)
Gergő Móriczfd9c74cc0e3a
diff --git a/apps/api/src/lib/scrape-billing.test.ts b/apps/api/src/lib/scrape-billing.test.tsindex 32150383c..6ca688fc0 100644--- a/apps/api/src/lib/scrape-billing.test.ts+++ b/apps/api/src/lib/scrape-billing.test.ts@@ -226,19 +226,27 @@ describe("calculateCreditsToBeBilled", () => {// lockdown + json surcharge stacking// =========================================- const promptInjectionGuardCall = {+ const promptInjectionGuardCall = (+ verdict: "clean" | "injection" | "none" = "clean",+ ) => ({type: "other",model: "vertex/gemini",cost: 0,- metadata: { module: "scrapeURL", method: "checkForPromptInjection" },- };+ metadata: {+ module: "scrapeURL",+ method: "checkForPromptInjection",+ verdict,+ },+ });// `guard` means the caller asked for the prompt injection check and the// guard actually ran. Both conditions must hold for the +4 guard fee.+ // `guardVerdicts` overrides the per-chunk verdicts the guard recorded.const billScrape = (args: {lockdown?: boolean;json?: boolean;guard?: boolean;+ guardVerdicts?: ("clean" | "injection" | "none")[];}) =>calculateCreditsToBeBilled({@@ -259,7 +267,9 @@ describe("calculateCreditsToBeBilled", () => {} as any,{totalCost: 0,- calls: args.guard ? [promptInjectionGuardCall] : [],+ calls: args.guard+ ? (args.guardVerdicts ?? ["clean"]).map(promptInjectionGuardCall)+ : [],} as any,{} as any,);@@ -286,6 +296,57 @@ describe("calculateCreditsToBeBilled", () => {it("bills json plus the prompt injection guard at 9 credits", async () => {expect(await billScrape({ json: true, guard: true })).toBe(9);});++ it("drops the guard fee when any chunk got no verdict", async () => {+ expect(+ await billScrape({+ json: true,+ guard: true,+ guardVerdicts: ["clean", "none", "clean"],+ }),+ ).toBe(5);+ });++ it("drops the guard fee when no chunk got a verdict", async () => {+ expect(+ await billScrape({ json: true, guard: true, guardVerdicts: ["none"] }),+ ).toBe(5);+ });++ // =========================================+ // prompt injection guard on failed scrapes+ // =========================================++ const billFailedGuardedScrape = (+ verdicts: ("clean" | "injection" | "none")[],+ ) =>+ calculateCreditsToBeBilled(+ {+ formats: [{ type: "json", schema: {}, checkPromptInjection: true }],+ } as any,+ {+ teamId: "team-id",+ orgId: null,+ },+ null,+ {+ totalCost: 0,+ calls: verdicts.map(promptInjectionGuardCall),+ } as any,+ {} as any,+ );++ it("bills 5 credits for a failed scrape whose guard scanned every chunk", async () => {+ expect(await billFailedGuardedScrape(["clean", "clean"])).toBe(5);+ });++ it("bills 5 credits for a scrape the guard blocked, even if a concurrent chunk failed", async () => {+ expect(await billFailedGuardedScrape(["injection", "none"])).toBe(5);+ });++ it("bills nothing for a failed scrape whose guard left a chunk unscanned", async () => {+ expect(await billFailedGuardedScrape(["clean", "none"])).toBe(0);+ });});// =========================================diff --git a/apps/api/src/lib/scrape-billing.ts b/apps/api/src/lib/scrape-billing.tsindex 3f43530c9..0ec7e8733 100644--- a/apps/api/src/lib/scrape-billing.ts+++ b/apps/api/src/lib/scrape-billing.ts@@ -62,6 +62,31 @@ export function calculateThreatScanCredits(return credits;}+/**+ * Whether the prompt injection guard gave this scrape's content a verdict, and+ * so whether its fee bills. The guard writes one cost-tracking record per+ * chunk it attempted and fails open on a chunk it could not classify+ * (verdict "none"): a scan that left any chunk unscanned does not bill. A+ * detection is a verdict for the whole page even if a concurrent chunk failed,+ * since it blocked the extraction.+ */+function promptInjectionGuardGaveVerdict(+ costTrackingJSON: ReturnType<typeof CostTracking.prototype.toJSON>,+): boolean {+ const guardCalls = (costTrackingJSON.calls ?? []).filter(+ call =>+ call.metadata?.module === "scrapeURL" &&+ call.metadata?.method === "checkForPromptInjection",+ );+ if (guardCalls.some(call => call.metadata.verdict === "injection")) {+ return true;+ }+ return (+ guardCalls.length > 0 &&+ guardCalls.every(call => call.metadata.verdict === "clean")+ );+}+export async function calculateCreditsToBeBilled(options: ScrapeOptions,internalOptions: InternalOptions,@@ -115,12 +140,10 @@ export async function calculateCreditsToBeBilled(creditsToBeBilled = 1;}- const promptInjectionGuardRan = costTrackingJSON.calls?.some(- call =>- call.metadata?.module === "scrapeURL" &&- call.metadata?.method === "checkForPromptInjection",- );- if (creditsToBeBilled === 0 && promptInjectionGuardRan) {+ if (+ creditsToBeBilled === 0 &&+ promptInjectionGuardGaveVerdict(costTrackingJSON)+ ) {creditsToBeBilled = 5;}@@ -157,15 +180,11 @@ export async function calculateCreditsToBeBilled(creditsToBeBilled += jsonCostBonus;}- if (hasFormatOfType(options.formats, "json")?.checkPromptInjection) {- const promptInjectionGuardRan = costTrackingJSON.calls?.some(- call =>- call.metadata?.module === "scrapeURL" &&- call.metadata?.method === "checkForPromptInjection",- );- if (promptInjectionGuardRan) {- creditsToBeBilled += 4;- }+ if (+ hasFormatOfType(options.formats, "json")?.checkPromptInjection &&+ promptInjectionGuardGaveVerdict(costTrackingJSON)+ ) {+ creditsToBeBilled += 4;}if (hasFormatOfType(options.formats, "deterministicJson")) {diff --git a/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.ts b/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.tsindex 61c968335..45424831a 100644--- a/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.ts+++ b/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.ts@@ -6,7 +6,10 @@ import {generateSchemaFromPrompt,} from "../transformers/llmExtract";import { smartScrape } from "./smartScrape";-import { checkForPromptInjection } from "./promptInjectionGuard";+import {+ checkForPromptInjection,+ createPromptInjectionGuardLimiter,+} from "./promptInjectionGuard";import { parseMarkdown } from "../../../lib/html-to-markdown";import { getModel } from "../../../lib/generic-ai";import { TokenUsage } from "../../../controllers/v1/types";@@ -266,6 +269,8 @@ export async function extractData({const logger = extractOptions.logger;const isSingleUrl = urls.length === 1;let costLimitExceededTokenUsage: number | null = null;+ // Set when the prompt injection guard failed open on part of the content.+ let promptInjectionScanIncomplete = false;if (extractOptions.markdown &&@@ -375,7 +380,7 @@ export async function extractData({metadata,zeroDataRetention: !!extractOptions.zeroDataRetention,})- : Promise.resolve(),+ : Promise.resolve(true),generateCompletions({...extractOptionsNewSchema,costTrackingOptions: {@@ -392,6 +397,9 @@ export async function extractData({if (guardSettled.status === "rejected") {throw guardSettled.reason;}+ if (!guardSettled.value) {+ promptInjectionScanIncomplete = true;+ }if (generateSettled.status === "fulfilled") {extract = generateSettled.value.extract;@@ -496,16 +504,23 @@ export async function extractData({),);// console.log("markdowns", markdowns);+ // Shared so the per-page scans below stay within one guard's+ // concurrency limit instead of each bursting its own.+ const guardLimiter = createPromptInjectionGuardLimiter();extractedData = await Promise.all(markdowns.map(async markdown => {if (extractOptions.options.checkPromptInjection) {- await checkForPromptInjection({+ const scannedFully = await checkForPromptInjection({markdown,logger,costTracking: extractOptions.costTrackingOptions.costTracking,metadata,zeroDataRetention: !!extractOptions.zeroDataRetention,+ limiter: guardLimiter,});+ if (!scannedFully) {+ promptInjectionScanIncomplete = true;+ }}const newExtractOptions = {@@ -543,6 +558,13 @@ export async function extractData({}}+ if (promptInjectionScanIncomplete) {+ // The guard fee does not bill in this case (see scrape-billing.ts).+ warning =+ "The prompt injection check could not scan all of the page content, so part of it went to JSON extraction unchecked. The prompt injection check was not billed." ++ (warning ? " " + warning : "");+ }+return {extractedDataArray: extractedData,warning: warning,diff --git a/apps/api/src/scraper/scrapeURL/lib/promptInjectionGuard.test.ts b/apps/api/src/scraper/scrapeURL/lib/promptInjectionGuard.test.tsindex fc4c45007..199589f19 100644--- a/apps/api/src/scraper/scrapeURL/lib/promptInjectionGuard.test.ts+++ b/apps/api/src/scraper/scrapeURL/lib/promptInjectionGuard.test.ts@@ -1,4 +1,19 @@-import { chunkByChars } from "./promptInjectionGuard";+import { generateObject, LoadAPIKeyError } from "ai";+import {+ chunkByChars,+ checkForPromptInjection,+ createPromptInjectionGuardLimiter,+} from "./promptInjectionGuard";+import { CostTracking } from "../../../lib/cost-tracking";+import { PromptInjectionDetectedError } from "../error";++vi.mock("ai", async importOriginal => ({+ ...(await importOriginal<typeof import("ai")>()),+ generateObject: vi.fn(),+}));+vi.mock("../../../lib/generic-ai", () => ({+ getModel: vi.fn().mockReturnValue({ modelId: "gpt-4o-mini" }),+}));describe("chunkByChars", () => {it("returns the whole text as a single chunk if within the limit", () => {@@ -43,8 +58,7 @@ describe("chunkByChars", () => {const overlap = 20;const marker = "INJECT-THIS-PHRASE";// Placed to straddle the boundary between the first and second chunk.- const text =- "x".repeat(maxCharsPerChunk - 5) + marker + "y".repeat(500);+ const text = "x".repeat(maxCharsPerChunk - 5) + marker + "y".repeat(500);const chunks = chunkByChars(text, maxCharsPerChunk, overlap);@@ -62,3 +76,121 @@ describe("chunkByChars", () => {expect(chunks.join("").length).toBeGreaterThanOrEqual(text.length);});});++describe("checkForPromptInjection", () => {+ const noopLogger = {+ warn: () => {},+ info: () => {},+ error: () => {},+ debug: () => {},+ } as any;++ const verdict = (isInjection: boolean) => ({+ object: { isInjection, reason: "test" },+ usage: { inputTokens: 100, outputTokens: 10 },+ });++ // Three chunks at the guard's 32k chunk size with 2k overlap.+ const threeChunkMarkdown = "x".repeat(80_000);++ const run = (costTracking: CostTracking, markdown = threeChunkMarkdown) =>+ checkForPromptInjection({+ markdown,+ logger: noopLogger,+ costTracking,+ metadata: { teamId: "test-team" },+ zeroDataRetention: false,+ });++ const guardVerdicts = (costTracking: CostTracking) =>+ costTracking.calls+ .filter(call => call.metadata.method === "checkForPromptInjection")+ .map(call => call.metadata.verdict);++ beforeEach(() => {+ vi.mocked(generateObject).mockReset();+ });++ it("resolves true and records a clean verdict per chunk when every chunk scans", async () => {+ vi.mocked(generateObject).mockResolvedValue(verdict(false) as any);+ const costTracking = new CostTracking();++ await expect(run(costTracking)).resolves.toBe(true);+ expect(guardVerdicts(costTracking)).toEqual(["clean", "clean", "clean"]);+ });++ it("fails open and resolves false when one chunk's classifier call errors", async () => {+ vi.mocked(generateObject)+ .mockResolvedValueOnce(verdict(false) as any)+ .mockRejectedValueOnce(new Error("timeout"))+ .mockResolvedValueOnce(verdict(false) as any);+ const costTracking = new CostTracking();++ await expect(run(costTracking)).resolves.toBe(false);+ expect(guardVerdicts(costTracking).sort()).toEqual([+ "clean",+ "clean",+ "none",+ ]);+ });++ it("records a no-verdict call when the request was never dispatched", async () => {+ vi.mocked(generateObject).mockRejectedValue(+ new LoadAPIKeyError({ message: "missing key" }),+ );+ const costTracking = new CostTracking();++ await expect(run(costTracking, "short page")).resolves.toBe(false);+ expect(guardVerdicts(costTracking)).toEqual(["none"]);+ expect(costTracking.toJSON().totalCost).toBe(0);+ });++ it("throws and records an injection verdict on a detection", async () => {+ vi.mocked(generateObject).mockResolvedValue(verdict(true) as any);+ const costTracking = new CostTracking();++ await expect(run(costTracking, "short page")).rejects.toBeInstanceOf(+ PromptInjectionDetectedError,+ );+ expect(guardVerdicts(costTracking)).toEqual(["injection"]);+ });++ it("keeps concurrent scans sharing a limiter within one guard's concurrency limit", async () => {+ let inFlight = 0;+ let maxInFlight = 0;+ vi.mocked(generateObject).mockImplementation((async () => {+ inFlight++;+ maxInFlight = Math.max(maxInFlight, inFlight);+ await new Promise(resolve => setTimeout(resolve, 5));+ inFlight--;+ return verdict(false);+ }) as any);+ const costTracking = new CostTracking();+ const limiter = createPromptInjectionGuardLimiter();++ // Four pages of three chunks each: 12 calls, 5 at most in flight.+ const results = await Promise.all(+ [0, 1, 2, 3].map(() =>+ checkForPromptInjection({+ markdown: threeChunkMarkdown,+ logger: noopLogger,+ costTracking,+ metadata: { teamId: "test-team" },+ zeroDataRetention: false,+ limiter,+ }),+ ),+ );++ expect(results).toEqual([true, true, true, true]);+ expect(generateObject).toHaveBeenCalledTimes(12);+ expect(maxInFlight).toBe(5);+ });++ it("resolves true without calling the classifier for empty content", async () => {+ const costTracking = new CostTracking();++ await expect(run(costTracking, " ")).resolves.toBe(true);+ expect(generateObject).not.toHaveBeenCalled();+ });+});diff --git a/apps/api/src/scraper/scrapeURL/lib/promptInjectionGuard.ts b/apps/api/src/scraper/scrapeURL/lib/promptInjectionGuard.tsindex ed7dabc4a..f32f2cdab 100644--- a/apps/api/src/scraper/scrapeURL/lib/promptInjectionGuard.ts+++ b/apps/api/src/scraper/scrapeURL/lib/promptInjectionGuard.ts@@ -1,15 +1,8 @@import crypto from "crypto";import { Logger } from "winston";import { z } from "zod";-import {- generateObject,- InvalidPromptError,- LoadAPIKeyError,- LoadSettingError,- NoObjectGeneratedError,- NoSuchModelError,- UnsupportedFunctionalityError,-} from "ai";+import { generateObject, NoObjectGeneratedError } from "ai";+import { Semaphore } from "async-mutex";import { getModel } from "../../../lib/generic-ai";import {CostLimitExceededError,@@ -65,17 +58,23 @@ function buildGuardSystemPrompt(tagName: string): string {].join(" ");}+// "none" marks a chunk the guard failed open on. Billing reads it: the guard+// fee is charged only when every chunk got a verdict (see scrape-billing.ts).+type GuardVerdict = "clean" | "injection" | "none";+function recordGuardCall(costTracking: CostTracking,modelId: string,inputTokens: number,outputTokens: number,+ verdict: GuardVerdict,) {costTracking.addCall({type: "other",metadata: {module: "scrapeURL",method: "checkForPromptInjection",+ verdict,},tokens: { input: inputTokens, output: outputTokens },model: modelId,@@ -91,7 +90,7 @@ async function classifyChunk(costTracking: CostTracking,metadata: { teamId: string; functionId?: string },zeroDataRetention: boolean,-): Promise<void> {+): Promise<boolean> {const tagName = `untrusted_page_content_${crypto.randomUUID()}`;try {@@ -123,6 +122,7 @@ async function classifyChunk(modelId,result.usage?.inputTokens ?? 0,result.usage?.outputTokens ?? 0,+ result.object.isInjection ? "injection" : "clean",);if (result.object.isInjection) {@@ -134,6 +134,7 @@ async function classifyChunk(`The scraped page content appears to contain a prompt injection attempt, so JSON extraction was aborted for safety. Guard verdict: ${result.object.reason.slice(0, 300)}`,);}+ return true;} catch (error) {if (error instanceof PromptInjectionDetectedError ||@@ -142,49 +143,53 @@ async function classifyChunk(throw error;}- // Denylist, not allowlist: unknown future SDK error types default to "billed".- const neverDispatched = [- InvalidPromptError,- NoSuchModelError,- UnsupportedFunctionalityError,- LoadAPIKeyError,- LoadSettingError,- ].some(ErrorClass => ErrorClass.isInstance(error));- if (!neverDispatched) {- const usage = NoObjectGeneratedError.isInstance(error)- ? error.usage- : undefined;- recordGuardCall(- costTracking,- modelId,- usage?.inputTokens ?? 0,- usage?.outputTokens ?? 0,- );- }+ const usage = NoObjectGeneratedError.isInstance(error)+ ? error.usage+ : undefined;+ recordGuardCall(+ costTracking,+ modelId,+ usage?.inputTokens ?? 0,+ usage?.outputTokens ?? 0,+ "none",+ );// Deliberately fail-open so a guard outage doesn't take down the whole scrape.logger.warn("Prompt injection guard call failed; proceeding without a guard verdict (fail-open)",{ error },);+ return false;}}+// Caps in-flight classifier calls across several concurrent scans, e.g. one+// per page of a smart-scrape result, which would otherwise each burst up to+// GUARD_CONCURRENCY_LIMIT calls at once.+export function createPromptInjectionGuardLimiter(): Semaphore {+ return new Semaphore(GUARD_CONCURRENCY_LIMIT);+}++// Throws PromptInjectionDetectedError on a detection. Otherwise resolves to+// false when the guard failed open on at least one chunk, i.e. part of the+// content reached extraction unscanned.export async function checkForPromptInjection({markdown,logger,costTracking,metadata,zeroDataRetention,+ limiter = createPromptInjectionGuardLimiter(),}: {markdown: string | undefined;logger: Logger;costTracking: CostTracking;metadata: { teamId: string; functionId?: string };zeroDataRetention: boolean;-}): Promise<void> {+ limiter?: Semaphore;+}): Promise<boolean> {if (!markdown || markdown.trim().length === 0) {- return;+ return true;}// Chunked and overlapping: a single trim window is bypassable, and a non-overlapping split could hide an injection at a boundary.@@ -198,20 +203,25 @@ export async function checkForPromptInjection({const modelId = typeof model === "string" ? model : model.modelId;// Batched so a detection stops later batches from ever being scheduled, and so a huge page can't burst past rate limits.+ let everyChunkScanned = true;for (let i = 0; i < chunks.length; i += GUARD_CONCURRENCY_LIMIT) {const batch = chunks.slice(i, i + GUARD_CONCURRENCY_LIMIT);- await Promise.all(+ const scanned = await Promise.all(batch.map(chunk =>- classifyChunk(- chunk,- model,- modelId,- logger,- costTracking,- metadata,- zeroDataRetention,+ limiter.runExclusive(() =>+ classifyChunk(+ chunk,+ model,+ modelId,+ logger,+ costTracking,+ metadata,+ zeroDataRetention,+ ),),),);+ if (scanned.includes(false)) everyChunkScanned = false;}+ return everyChunkScanned;}