Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(api): don't bill the prompt injection guard when it fails open (#4747)

Gergő Móriczfd9c74cc0e3a
diff --git a/apps/api/src/lib/scrape-billing.test.ts b/apps/api/src/lib/scrape-billing.test.ts
index 32150383c..6ca688fc0 100644
--- a/apps/api/src/lib/scrape-billing.test.ts
+++ b/apps/api/src/lib/scrape-billing.test.ts
@@ -226,19 +226,27 @@ describe("calculateCreditsToBeBilled", () => {
// lockdown + json surcharge stacking
// =========================================
- const promptInjectionGuardCall = {
+ const promptInjectionGuardCall = (
+ verdict: "clean" | "injection" | "none" = "clean",
+ ) => ({
type: "other",
model: "vertex/gemini",
cost: 0,
- metadata: { module: "scrapeURL", method: "checkForPromptInjection" },
- };
+ metadata: {
+ module: "scrapeURL",
+ method: "checkForPromptInjection",
+ verdict,
+ },
+ });
// `guard` means the caller asked for the prompt injection check and the
// guard actually ran. Both conditions must hold for the +4 guard fee.
+ // `guardVerdicts` overrides the per-chunk verdicts the guard recorded.
const billScrape = (args: {
lockdown?: boolean;
json?: boolean;
guard?: boolean;
+ guardVerdicts?: ("clean" | "injection" | "none")[];
}) =>
calculateCreditsToBeBilled(
{
@@ -259,7 +267,9 @@ describe("calculateCreditsToBeBilled", () => {
} as any,
{
totalCost: 0,
- calls: args.guard ? [promptInjectionGuardCall] : [],
+ calls: args.guard
+ ? (args.guardVerdicts ?? ["clean"]).map(promptInjectionGuardCall)
+ : [],
} as any,
{} as any,
);
@@ -286,6 +296,57 @@ describe("calculateCreditsToBeBilled", () => {
it("bills json plus the prompt injection guard at 9 credits", async () => {
expect(await billScrape({ json: true, guard: true })).toBe(9);
});
+
+ it("drops the guard fee when any chunk got no verdict", async () => {
+ expect(
+ await billScrape({
+ json: true,
+ guard: true,
+ guardVerdicts: ["clean", "none", "clean"],
+ }),
+ ).toBe(5);
+ });
+
+ it("drops the guard fee when no chunk got a verdict", async () => {
+ expect(
+ await billScrape({ json: true, guard: true, guardVerdicts: ["none"] }),
+ ).toBe(5);
+ });
+
+ // =========================================
+ // prompt injection guard on failed scrapes
+ // =========================================
+
+ const billFailedGuardedScrape = (
+ verdicts: ("clean" | "injection" | "none")[],
+ ) =>
+ calculateCreditsToBeBilled(
+ {
+ formats: [{ type: "json", schema: {}, checkPromptInjection: true }],
+ } as any,
+ {
+ teamId: "team-id",
+ orgId: null,
+ },
+ null,
+ {
+ totalCost: 0,
+ calls: verdicts.map(promptInjectionGuardCall),
+ } as any,
+ {} as any,
+ );
+
+ it("bills 5 credits for a failed scrape whose guard scanned every chunk", async () => {
+ expect(await billFailedGuardedScrape(["clean", "clean"])).toBe(5);
+ });
+
+ it("bills 5 credits for a scrape the guard blocked, even if a concurrent chunk failed", async () => {
+ expect(await billFailedGuardedScrape(["injection", "none"])).toBe(5);
+ });
+
+ it("bills nothing for a failed scrape whose guard left a chunk unscanned", async () => {
+ expect(await billFailedGuardedScrape(["clean", "none"])).toBe(0);
+ });
});
// =========================================
diff --git a/apps/api/src/lib/scrape-billing.ts b/apps/api/src/lib/scrape-billing.ts
index 3f43530c9..0ec7e8733 100644
--- a/apps/api/src/lib/scrape-billing.ts
+++ b/apps/api/src/lib/scrape-billing.ts
@@ -62,6 +62,31 @@ export function calculateThreatScanCredits(
return credits;
}
+/**
+ * Whether the prompt injection guard gave this scrape's content a verdict, and
+ * so whether its fee bills. The guard writes one cost-tracking record per
+ * chunk it attempted and fails open on a chunk it could not classify
+ * (verdict "none"): a scan that left any chunk unscanned does not bill. A
+ * detection is a verdict for the whole page even if a concurrent chunk failed,
+ * since it blocked the extraction.
+ */
+function promptInjectionGuardGaveVerdict(
+ costTrackingJSON: ReturnType<typeof CostTracking.prototype.toJSON>,
+): boolean {
+ const guardCalls = (costTrackingJSON.calls ?? []).filter(
+ call =>
+ call.metadata?.module === "scrapeURL" &&
+ call.metadata?.method === "checkForPromptInjection",
+ );
+ if (guardCalls.some(call => call.metadata.verdict === "injection")) {
+ return true;
+ }
+ return (
+ guardCalls.length > 0 &&
+ guardCalls.every(call => call.metadata.verdict === "clean")
+ );
+}
+
export async function calculateCreditsToBeBilled(
options: ScrapeOptions,
internalOptions: InternalOptions,
@@ -115,12 +140,10 @@ export async function calculateCreditsToBeBilled(
creditsToBeBilled = 1;
}
- const promptInjectionGuardRan = costTrackingJSON.calls?.some(
- call =>
- call.metadata?.module === "scrapeURL" &&
- call.metadata?.method === "checkForPromptInjection",
- );
- if (creditsToBeBilled === 0 && promptInjectionGuardRan) {
+ if (
+ creditsToBeBilled === 0 &&
+ promptInjectionGuardGaveVerdict(costTrackingJSON)
+ ) {
creditsToBeBilled = 5;
}
@@ -157,15 +180,11 @@ export async function calculateCreditsToBeBilled(
creditsToBeBilled += jsonCostBonus;
}
- if (hasFormatOfType(options.formats, "json")?.checkPromptInjection) {
- const promptInjectionGuardRan = costTrackingJSON.calls?.some(
- call =>
- call.metadata?.module === "scrapeURL" &&
- call.metadata?.method === "checkForPromptInjection",
- );
- if (promptInjectionGuardRan) {
- creditsToBeBilled += 4;
- }
+ if (
+ hasFormatOfType(options.formats, "json")?.checkPromptInjection &&
+ promptInjectionGuardGaveVerdict(costTrackingJSON)
+ ) {
+ creditsToBeBilled += 4;
}
if (hasFormatOfType(options.formats, "deterministicJson")) {
diff --git a/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.ts b/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.ts
index 61c968335..45424831a 100644
--- a/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.ts
+++ b/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.ts
@@ -6,7 +6,10 @@ import {
generateSchemaFromPrompt,
} from "../transformers/llmExtract";
import { smartScrape } from "./smartScrape";
-import { checkForPromptInjection } from "./promptInjectionGuard";
+import {
+ checkForPromptInjection,
+ createPromptInjectionGuardLimiter,
+} from "./promptInjectionGuard";
import { parseMarkdown } from "../../../lib/html-to-markdown";
import { getModel } from "../../../lib/generic-ai";
import { TokenUsage } from "../../../controllers/v1/types";
@@ -266,6 +269,8 @@ export async function extractData({
const logger = extractOptions.logger;
const isSingleUrl = urls.length === 1;
let costLimitExceededTokenUsage: number | null = null;
+ // Set when the prompt injection guard failed open on part of the content.
+ let promptInjectionScanIncomplete = false;
if (
extractOptions.markdown &&
@@ -375,7 +380,7 @@ export async function extractData({
metadata,
zeroDataRetention: !!extractOptions.zeroDataRetention,
})
- : Promise.resolve(),
+ : Promise.resolve(true),
generateCompletions({
...extractOptionsNewSchema,
costTrackingOptions: {
@@ -392,6 +397,9 @@ export async function extractData({
if (guardSettled.status === "rejected") {
throw guardSettled.reason;
}
+ if (!guardSettled.value) {
+ promptInjectionScanIncomplete = true;
+ }
if (generateSettled.status === "fulfilled") {
extract = generateSettled.value.extract;
@@ -496,16 +504,23 @@ export async function extractData({
),
);
// console.log("markdowns", markdowns);
+ // Shared so the per-page scans below stay within one guard's
+ // concurrency limit instead of each bursting its own.
+ const guardLimiter = createPromptInjectionGuardLimiter();
extractedData = await Promise.all(
markdowns.map(async markdown => {
if (extractOptions.options.checkPromptInjection) {
- await checkForPromptInjection({
+ const scannedFully = await checkForPromptInjection({
markdown,
logger,
costTracking: extractOptions.costTrackingOptions.costTracking,
metadata,
zeroDataRetention: !!extractOptions.zeroDataRetention,
+ limiter: guardLimiter,
});
+ if (!scannedFully) {
+ promptInjectionScanIncomplete = true;
+ }
}
const newExtractOptions = {
@@ -543,6 +558,13 @@ export async function extractData({
}
}
+ if (promptInjectionScanIncomplete) {
+ // The guard fee does not bill in this case (see scrape-billing.ts).
+ warning =
+ "The prompt injection check could not scan all of the page content, so part of it went to JSON extraction unchecked. The prompt injection check was not billed." +
+ (warning ? " " + warning : "");
+ }
+
return {
extractedDataArray: extractedData,
warning: warning,
diff --git a/apps/api/src/scraper/scrapeURL/lib/promptInjectionGuard.test.ts b/apps/api/src/scraper/scrapeURL/lib/promptInjectionGuard.test.ts
index fc4c45007..199589f19 100644
--- a/apps/api/src/scraper/scrapeURL/lib/promptInjectionGuard.test.ts
+++ b/apps/api/src/scraper/scrapeURL/lib/promptInjectionGuard.test.ts
@@ -1,4 +1,19 @@
-import { chunkByChars } from "./promptInjectionGuard";
+import { generateObject, LoadAPIKeyError } from "ai";
+import {
+ chunkByChars,
+ checkForPromptInjection,
+ createPromptInjectionGuardLimiter,
+} from "./promptInjectionGuard";
+import { CostTracking } from "../../../lib/cost-tracking";
+import { PromptInjectionDetectedError } from "../error";
+
+vi.mock("ai", async importOriginal => ({
+ ...(await importOriginal<typeof import("ai")>()),
+ generateObject: vi.fn(),
+}));
+vi.mock("../../../lib/generic-ai", () => ({
+ getModel: vi.fn().mockReturnValue({ modelId: "gpt-4o-mini" }),
+}));
describe("chunkByChars", () => {
it("returns the whole text as a single chunk if within the limit", () => {
@@ -43,8 +58,7 @@ describe("chunkByChars", () => {
const overlap = 20;
const marker = "INJECT-THIS-PHRASE";
// Placed to straddle the boundary between the first and second chunk.
- const text =
- "x".repeat(maxCharsPerChunk - 5) + marker + "y".repeat(500);
+ const text = "x".repeat(maxCharsPerChunk - 5) + marker + "y".repeat(500);
const chunks = chunkByChars(text, maxCharsPerChunk, overlap);
@@ -62,3 +76,121 @@ describe("chunkByChars", () => {
expect(chunks.join("").length).toBeGreaterThanOrEqual(text.length);
});
});
+
+describe("checkForPromptInjection", () => {
+ const noopLogger = {
+ warn: () => {},
+ info: () => {},
+ error: () => {},
+ debug: () => {},
+ } as any;
+
+ const verdict = (isInjection: boolean) => ({
+ object: { isInjection, reason: "test" },
+ usage: { inputTokens: 100, outputTokens: 10 },
+ });
+
+ // Three chunks at the guard's 32k chunk size with 2k overlap.
+ const threeChunkMarkdown = "x".repeat(80_000);
+
+ const run = (costTracking: CostTracking, markdown = threeChunkMarkdown) =>
+ checkForPromptInjection({
+ markdown,
+ logger: noopLogger,
+ costTracking,
+ metadata: { teamId: "test-team" },
+ zeroDataRetention: false,
+ });
+
+ const guardVerdicts = (costTracking: CostTracking) =>
+ costTracking.calls
+ .filter(call => call.metadata.method === "checkForPromptInjection")
+ .map(call => call.metadata.verdict);
+
+ beforeEach(() => {
+ vi.mocked(generateObject).mockReset();
+ });
+
+ it("resolves true and records a clean verdict per chunk when every chunk scans", async () => {
+ vi.mocked(generateObject).mockResolvedValue(verdict(false) as any);
+ const costTracking = new CostTracking();
+
+ await expect(run(costTracking)).resolves.toBe(true);
+ expect(guardVerdicts(costTracking)).toEqual(["clean", "clean", "clean"]);
+ });
+
+ it("fails open and resolves false when one chunk's classifier call errors", async () => {
+ vi.mocked(generateObject)
+ .mockResolvedValueOnce(verdict(false) as any)
+ .mockRejectedValueOnce(new Error("timeout"))
+ .mockResolvedValueOnce(verdict(false) as any);
+ const costTracking = new CostTracking();
+
+ await expect(run(costTracking)).resolves.toBe(false);
+ expect(guardVerdicts(costTracking).sort()).toEqual([
+ "clean",
+ "clean",
+ "none",
+ ]);
+ });
+
+ it("records a no-verdict call when the request was never dispatched", async () => {
+ vi.mocked(generateObject).mockRejectedValue(
+ new LoadAPIKeyError({ message: "missing key" }),
+ );
+ const costTracking = new CostTracking();
+
+ await expect(run(costTracking, "short page")).resolves.toBe(false);
+ expect(guardVerdicts(costTracking)).toEqual(["none"]);
+ expect(costTracking.toJSON().totalCost).toBe(0);
+ });
+
+ it("throws and records an injection verdict on a detection", async () => {
+ vi.mocked(generateObject).mockResolvedValue(verdict(true) as any);
+ const costTracking = new CostTracking();
+
+ await expect(run(costTracking, "short page")).rejects.toBeInstanceOf(
+ PromptInjectionDetectedError,
+ );
+ expect(guardVerdicts(costTracking)).toEqual(["injection"]);
+ });
+
+ it("keeps concurrent scans sharing a limiter within one guard's concurrency limit", async () => {
+ let inFlight = 0;
+ let maxInFlight = 0;
+ vi.mocked(generateObject).mockImplementation((async () => {
+ inFlight++;
+ maxInFlight = Math.max(maxInFlight, inFlight);
+ await new Promise(resolve => setTimeout(resolve, 5));
+ inFlight--;
+ return verdict(false);
+ }) as any);
+ const costTracking = new CostTracking();
+ const limiter = createPromptInjectionGuardLimiter();
+
+ // Four pages of three chunks each: 12 calls, 5 at most in flight.
+ const results = await Promise.all(
+ [0, 1, 2, 3].map(() =>
+ checkForPromptInjection({
+ markdown: threeChunkMarkdown,
+ logger: noopLogger,
+ costTracking,
+ metadata: { teamId: "test-team" },
+ zeroDataRetention: false,
+ limiter,
+ }),
+ ),
+ );
+
+ expect(results).toEqual([true, true, true, true]);
+ expect(generateObject).toHaveBeenCalledTimes(12);
+ expect(maxInFlight).toBe(5);
+ });
+
+ it("resolves true without calling the classifier for empty content", async () => {
+ const costTracking = new CostTracking();
+
+ await expect(run(costTracking, " ")).resolves.toBe(true);
+ expect(generateObject).not.toHaveBeenCalled();
+ });
+});
diff --git a/apps/api/src/scraper/scrapeURL/lib/promptInjectionGuard.ts b/apps/api/src/scraper/scrapeURL/lib/promptInjectionGuard.ts
index ed7dabc4a..f32f2cdab 100644
--- a/apps/api/src/scraper/scrapeURL/lib/promptInjectionGuard.ts
+++ b/apps/api/src/scraper/scrapeURL/lib/promptInjectionGuard.ts
@@ -1,15 +1,8 @@
import crypto from "crypto";
import { Logger } from "winston";
import { z } from "zod";
-import {
- generateObject,
- InvalidPromptError,
- LoadAPIKeyError,
- LoadSettingError,
- NoObjectGeneratedError,
- NoSuchModelError,
- UnsupportedFunctionalityError,
-} from "ai";
+import { generateObject, NoObjectGeneratedError } from "ai";
+import { Semaphore } from "async-mutex";
import { getModel } from "../../../lib/generic-ai";
import {
CostLimitExceededError,
@@ -65,17 +58,23 @@ function buildGuardSystemPrompt(tagName: string): string {
].join(" ");
}
+// "none" marks a chunk the guard failed open on. Billing reads it: the guard
+// fee is charged only when every chunk got a verdict (see scrape-billing.ts).
+type GuardVerdict = "clean" | "injection" | "none";
+
function recordGuardCall(
costTracking: CostTracking,
modelId: string,
inputTokens: number,
outputTokens: number,
+ verdict: GuardVerdict,
) {
costTracking.addCall({
type: "other",
metadata: {
module: "scrapeURL",
method: "checkForPromptInjection",
+ verdict,
},
tokens: { input: inputTokens, output: outputTokens },
model: modelId,
@@ -91,7 +90,7 @@ async function classifyChunk(
costTracking: CostTracking,
metadata: { teamId: string; functionId?: string },
zeroDataRetention: boolean,
-): Promise<void> {
+): Promise<boolean> {
const tagName = `untrusted_page_content_${crypto.randomUUID()}`;
try {
@@ -123,6 +122,7 @@ async function classifyChunk(
modelId,
result.usage?.inputTokens ?? 0,
result.usage?.outputTokens ?? 0,
+ result.object.isInjection ? "injection" : "clean",
);
if (result.object.isInjection) {
@@ -134,6 +134,7 @@ async function classifyChunk(
`The scraped page content appears to contain a prompt injection attempt, so JSON extraction was aborted for safety. Guard verdict: ${result.object.reason.slice(0, 300)}`,
);
}
+ return true;
} catch (error) {
if (
error instanceof PromptInjectionDetectedError ||
@@ -142,49 +143,53 @@ async function classifyChunk(
throw error;
}
- // Denylist, not allowlist: unknown future SDK error types default to "billed".
- const neverDispatched = [
- InvalidPromptError,
- NoSuchModelError,
- UnsupportedFunctionalityError,
- LoadAPIKeyError,
- LoadSettingError,
- ].some(ErrorClass => ErrorClass.isInstance(error));
- if (!neverDispatched) {
- const usage = NoObjectGeneratedError.isInstance(error)
- ? error.usage
- : undefined;
- recordGuardCall(
- costTracking,
- modelId,
- usage?.inputTokens ?? 0,
- usage?.outputTokens ?? 0,
- );
- }
+ const usage = NoObjectGeneratedError.isInstance(error)
+ ? error.usage
+ : undefined;
+ recordGuardCall(
+ costTracking,
+ modelId,
+ usage?.inputTokens ?? 0,
+ usage?.outputTokens ?? 0,
+ "none",
+ );
// Deliberately fail-open so a guard outage doesn't take down the whole scrape.
logger.warn(
"Prompt injection guard call failed; proceeding without a guard verdict (fail-open)",
{ error },
);
+ return false;
}
}
+// Caps in-flight classifier calls across several concurrent scans, e.g. one
+// per page of a smart-scrape result, which would otherwise each burst up to
+// GUARD_CONCURRENCY_LIMIT calls at once.
+export function createPromptInjectionGuardLimiter(): Semaphore {
+ return new Semaphore(GUARD_CONCURRENCY_LIMIT);
+}
+
+// Throws PromptInjectionDetectedError on a detection. Otherwise resolves to
+// false when the guard failed open on at least one chunk, i.e. part of the
+// content reached extraction unscanned.
export async function checkForPromptInjection({
markdown,
logger,
costTracking,
metadata,
zeroDataRetention,
+ limiter = createPromptInjectionGuardLimiter(),
}: {
markdown: string | undefined;
logger: Logger;
costTracking: CostTracking;
metadata: { teamId: string; functionId?: string };
zeroDataRetention: boolean;
-}): Promise<void> {
+ limiter?: Semaphore;
+}): Promise<boolean> {
if (!markdown || markdown.trim().length === 0) {
- return;
+ return true;
}
// Chunked and overlapping: a single trim window is bypassable, and a non-overlapping split could hide an injection at a boundary.
@@ -198,20 +203,25 @@ export async function checkForPromptInjection({
const modelId = typeof model === "string" ? model : model.modelId;
// Batched so a detection stops later batches from ever being scheduled, and so a huge page can't burst past rate limits.
+ let everyChunkScanned = true;
for (let i = 0; i < chunks.length; i += GUARD_CONCURRENCY_LIMIT) {
const batch = chunks.slice(i, i + GUARD_CONCURRENCY_LIMIT);
- await Promise.all(
+ const scanned = await Promise.all(
batch.map(chunk =>
- classifyChunk(
- chunk,
- model,
- modelId,
- logger,
- costTracking,
- metadata,
- zeroDataRetention,
+ limiter.runExclusive(() =>
+ classifyChunk(
+ chunk,
+ model,
+ modelId,
+ logger,
+ costTracking,
+ metadata,
+ zeroDataRetention,
+ ),
),
),
);
+ if (scanned.includes(false)) everyChunkScanned = false;
}
+ return everyChunkScanned;
}