Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api): make checkPromptInjection a top-level scrape option (#5024)
Gergő Móriczfa7ec5f0a954
diff --git a/apps/api/src/__tests__/snips/v2/scrape.test.ts b/apps/api/src/__tests__/snips/v2/scrape.test.tsindex a14d00ea6..3990910f7 100644--- a/apps/api/src/__tests__/snips/v2/scrape.test.ts+++ b/apps/api/src/__tests__/snips/v2/scrape.test.ts@@ -445,6 +445,46 @@ describe("Scrape tests", () => {scrapeTimeout,);+ itIf(TEST_PRODUCTION || (HAS_AI && ALLOW_TEST_SUITE_WEBSITE))(+ "runs the top-level checkPromptInjection on a markdown scrape and bills it",+ async () => {+ const response = await scrape(+ {+ url: `${TEST_SUITE_WEBSITE}/product`,+ formats: ["markdown"],+ checkPromptInjection: true,+ timeout: scrapeTimeout,+ },+ identity,+ );++ expect(response.markdown).toBeDefined();+ expect(response.metadata.creditsUsed).toBe(5);+ },+ scrapeTimeout,+ );++ itIf(TEST_PRODUCTION || (HAS_AI && ALLOW_TEST_SUITE_WEBSITE))(+ "blocks a markdown scrape when the top-level checkPromptInjection detects a prompt injection",+ async () => {+ const raw = await scrapeRaw(+ {+ url: `${TEST_SUITE_WEBSITE}/prompt-injection`,+ formats: ["markdown"],+ checkPromptInjection: true,+ timeout: scrapeTimeout,+ },+ identity,+ );++ expect(raw.statusCode).toBe(403);+ expect(raw.body.success).toBe(false);+ expect(raw.body.code).toBe("SCRAPE_PROMPT_INJECTION_DETECTED");+ expect(typeof raw.body.error).toBe("string");+ },+ scrapeTimeout,+ );+itIf(TEST_PRODUCTION || (HAS_AI && ALLOW_TEST_SUITE_WEBSITE))("blocks when checkPromptInjection detects a prompt injection",async () => {diff --git a/apps/api/src/controllers/v2/check-prompt-injection-schema.test.ts b/apps/api/src/controllers/v2/check-prompt-injection-schema.test.tsnew file mode 100644index 000000000..ce2ee28e2--- /dev/null+++ b/apps/api/src/controllers/v2/check-prompt-injection-schema.test.ts@@ -0,0 +1,106 @@+import { describe, expect, it } from "vitest";+import {+ crawlRequestSchema,+ fromV1ScrapeOptions,+ scrapeOptions,+ scrapeRequestSchema,+} from "./types";+import { scrapeOptions as v1ScrapeOptions } from "../v1/types";++const url = "https://example.com";+const jsonFormat = {+ type: "json" as const,+ schema: { type: "object", properties: { title: { type: "string" } } },+};++describe("checkPromptInjection", () => {+ it("accepts the top-level flag with any format", () => {+ const parsed = scrapeRequestSchema.parse({+ url,+ checkPromptInjection: true,+ formats: ["markdown"],+ });++ expect(parsed.checkPromptInjection).toBe(true);+ });++ it("lifts the json-format flag to the top level and drops it from the format", () => {+ const parsed = scrapeRequestSchema.parse({+ url,+ formats: [{ ...jsonFormat, checkPromptInjection: true }],+ });++ expect(parsed.checkPromptInjection).toBe(true);+ expect(parsed.formats[0]).not.toHaveProperty("checkPromptInjection");+ });++ it("enables the check when either level sets it", () => {+ const parsed = scrapeRequestSchema.parse({+ url,+ checkPromptInjection: true,+ formats: [{ ...jsonFormat, checkPromptInjection: false }],+ });++ expect(parsed.checkPromptInjection).toBe(true);+ });++ it("defaults to false", () => {+ expect(scrapeRequestSchema.parse({ url }).checkPromptInjection).toBe(false);+ });++ it("still validates the json-format flag", () => {+ expect(+ scrapeRequestSchema.safeParse({+ url,+ formats: [{ ...jsonFormat, checkPromptInjection: "yes" }],+ }).success,+ ).toBe(false);+ });++ it("is stable when parsed options are parsed again", () => {+ const once = scrapeOptions.parse({+ formats: [{ ...jsonFormat, checkPromptInjection: true }],+ });++ expect(scrapeOptions.parse(once)).toEqual(once);+ });++ it("lifts the json-format flag inside crawl scrapeOptions", () => {+ const parsed = crawlRequestSchema.parse({+ url,+ scrapeOptions: {+ formats: [{ ...jsonFormat, checkPromptInjection: true }],+ },+ });++ expect(parsed.scrapeOptions.checkPromptInjection).toBe(true);+ });++ it("rejects the rawBase64 format, which has no markdown to scan", () => {+ expect(+ scrapeRequestSchema.safeParse({+ url,+ checkPromptInjection: true,+ formats: ["rawBase64"],+ }).success,+ ).toBe(false);+ });++ it("maps v1 jsonOptions.checkPromptInjection to the top-level flag", () => {+ const v1 = v1ScrapeOptions.parse({+ formats: ["json"],+ jsonOptions: { schema: jsonFormat.schema, checkPromptInjection: true },+ });++ const { scrapeOptions: converted } = fromV1ScrapeOptions(+ v1,+ undefined,+ "team-id",+ );++ expect(converted.checkPromptInjection).toBe(true);+ expect(converted.formats.find(f => f.type === "json")).not.toHaveProperty(+ "checkPromptInjection",+ );+ });+});diff --git a/apps/api/src/controllers/v2/parse.ts b/apps/api/src/controllers/v2/parse.tsindex 2f5c47043..e0b370dff 100644--- a/apps/api/src/controllers/v2/parse.ts+++ b/apps/api/src/controllers/v2/parse.ts@@ -720,7 +720,8 @@ export async function parseController(!!hasFormatOfType(req.body.formats, "summary") ||!!hasFormatOfType(req.body.formats, "question") ||!!hasFormatOfType(req.body.formats, "highlights") ||- !!hasFormatOfType(req.body.formats, "query");+ !!hasFormatOfType(req.body.formats, "query") ||+ req.body.checkPromptInjection;if (!usedLlm) {const ct = hasFormatOfType(req.body.formats, "changeTracking");diff --git a/apps/api/src/controllers/v2/scrape.ts b/apps/api/src/controllers/v2/scrape.tsindex 59600a8a2..b871c7e52 100644--- a/apps/api/src/controllers/v2/scrape.ts+++ b/apps/api/src/controllers/v2/scrape.ts@@ -722,7 +722,8 @@ export async function scrapeController(!!hasFormatOfType(req.body.formats, "branding") ||!!hasFormatOfType(req.body.formats, "question") ||!!hasFormatOfType(req.body.formats, "highlights") ||- !!hasFormatOfType(req.body.formats, "query");+ !!hasFormatOfType(req.body.formats, "query") ||+ req.body.checkPromptInjection;if (!usedLlm) {const ct = hasFormatOfType(req.body.formats, "changeTracking");diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.tsindex 727bc1f1c..b68374592 100644--- a/apps/api/src/controllers/v2/types.ts+++ b/apps/api/src/controllers/v2/types.ts@@ -365,10 +365,14 @@ const jsonFormatWithOptions = z.strictObject({}).superRefine(addStrictSchemaIssue),prompt: z.string().max(10000).optional(),+ // Deprecated: lifted into the top-level checkPromptInjection at parse time.checkPromptInjection: z.boolean().optional(),});-export type JsonFormatWithOptions = z.output<typeof jsonFormatWithOptions>;+export type JsonFormatWithOptions = Omit<+ z.output<typeof jsonFormatWithOptions>,+ "checkPromptInjection"+>;// "Deterministic JSON" — same shape as json mode, but extraction is performed by// reusable-json-mode (a cached, reusable JS extractor run in the code-sandbox)@@ -844,6 +848,7 @@ const scrapeOptionFields = z.strictObject({minAge: z.int().gte(0).optional(),storeInCache: z.boolean().prefault(true),lockdown: z.boolean().prefault(false),+ checkPromptInjection: z.boolean().prefault(false),safeMode: z.boolean().optional(),redactPII: redactPIISchema,// Enterprise: per-request field-level override of the org's threat@@ -866,13 +871,23 @@ const scrapeOptionFields = z.strictObject({__forceFirePDF: z.boolean().prefault(false).optional(),});-const baseScrapeOptions = scrapeOptionFields.refine(- options => !(options.profile && options.lockdown),- {+const baseScrapeOptions = scrapeOptionFields+ .refine(options => !(options.profile && options.lockdown), {message: "Profiles require live browsing and cannot be used with lockdown.",path: ["profile"],- },-);+ })+ .refine(+ options =>+ !(+ options.checkPromptInjection &&+ options.formats.some(f => f.type === "rawBase64")+ ),+ {+ message:+ "checkPromptInjection scans page markdown and cannot be used with the rawBase64 format.",+ path: ["checkPromptInjection"],+ },+ );type ScrapeOptionsBase = z.infer<typeof baseScrapeOptions>;@@ -902,6 +917,34 @@ export const applyScrapeOptionsDefaults = <T extends ScrapeOptionsBase>(: true),});+/**+ * Folds the deprecated json-format checkPromptInjection into the top-level+ * flag and drops it from the format. Also accepts raw stored monitor options.+ */+export function liftCheckPromptInjection<+ T extends { formats?: unknown[]; checkPromptInjection?: unknown },+>(options: T): T {+ let requestedOnFormat = false;+ const formats = options.formats?.map(format => {+ if (+ typeof format !== "object" ||+ format === null ||+ !("checkPromptInjection" in format)+ ) {+ return format;+ }+ const { checkPromptInjection, ...rest } = format;+ requestedOnFormat ||= checkPromptInjection === true;+ return rest;+ });+ return {+ ...options,+ formats,+ checkPromptInjection:+ options.checkPromptInjection === true || requestedOnFormat,+ };+}+// Base transform function that handles both nullable and non-nullable cases// Uses generic type to preserve all fields from extended schemasconst extractTransformImpl = <T extends ScrapeOptionsBase | undefined>(@@ -909,7 +952,7 @@ const extractTransformImpl = <T extends ScrapeOptionsBase | undefined>(): T extends undefined ? undefined : T => {if (!obj) return obj as T extends undefined ? undefined : T;// Handle timeout- let result = applyScrapeOptionsDefaults(obj);+ let result = liftCheckPromptInjection(applyScrapeOptionsDefaults(obj));if (obj.formats.find(x => typeof x === "object" && x.type === "json") &&obj.timeout === 30000@@ -2327,6 +2370,9 @@ export function fromV1ScrapeOptions(}: {}),location: v1ScrapeOptions.location ?? v1ScrapeOptions.geolocation,+ checkPromptInjection: (+ v1ScrapeOptions.jsonOptions || v1ScrapeOptions.extract+ )?.checkPromptInjection,formats: v1ScrapeOptions.formats.map(x => {// json and extract is standardized down to extract fmt in v1 -- fine to take one and dismiss the other@@ -2336,7 +2382,6 @@ export function fromV1ScrapeOptions(type: "json",schema: opts?.schema,prompt: opts?.prompt,- checkPromptInjection: opts?.checkPromptInjection ?? false,};return fmt;} else if (x === "json") {@@ -2347,7 +2392,6 @@ export function fromV1ScrapeOptions(type: "json",schema: opts.schema,prompt: opts.prompt,- checkPromptInjection: opts.checkPromptInjection ?? false,};return includesFormat(v1ScrapeOptions.formats as any, "extract")? nulldiff --git a/apps/api/src/lib/keyless-credit-projection.test.ts b/apps/api/src/lib/keyless-credit-projection.test.tsnew file mode 100644index 000000000..02a122697--- /dev/null+++ b/apps/api/src/lib/keyless-credit-projection.test.ts@@ -0,0 +1,33 @@+import { describe, expect, it } from "vitest";+import { projectScrapeCredits } from "./keyless-credit-projection";+import { scrapeOptions } from "../controllers/v2/types";++describe("projectScrapeCredits", () => {+ it("projects the prompt injection guard fee on a markdown scrape", () => {+ expect(+ projectScrapeCredits(+ scrapeOptions.parse({ checkPromptInjection: true }),+ null,+ false,+ ),+ ).toBe(5);+ });++ it("projects the guard fee on top of json", () => {+ expect(+ projectScrapeCredits(+ scrapeOptions.parse({+ formats: [+ {+ type: "json",+ schema: { type: "object", properties: {} },+ checkPromptInjection: true,+ },+ ],+ }),+ null,+ false,+ ),+ ).toBe(9);+ });+});diff --git a/apps/api/src/lib/keyless-credit-projection.ts b/apps/api/src/lib/keyless-credit-projection.tsindex 9037f558a..0239c4524 100644--- a/apps/api/src/lib/keyless-credit-projection.ts+++ b/apps/api/src/lib/keyless-credit-projection.ts@@ -27,6 +27,10 @@ export function projectScrapeCredits(credits = 10;}+ if (options.checkPromptInjection) {+ credits += 4;+ }+if (hasFormatOfType(options.formats, "question") ||hasFormatOfType(options.formats, "query")diff --git a/apps/api/src/lib/scrape-billing.test.ts b/apps/api/src/lib/scrape-billing.test.tsindex a798a4a89..8d4148e36 100644--- a/apps/api/src/lib/scrape-billing.test.ts+++ b/apps/api/src/lib/scrape-billing.test.ts@@ -273,15 +273,19 @@ describe("calculateCreditsToBeBilled", () => {const billScrape = (args: {lockdown?: boolean;json?: boolean;+ deterministicJson?: boolean;guard?: boolean;guardVerdicts?: ("clean" | "injection" | "none")[];}) =>calculateCreditsToBeBilled({lockdown: args.lockdown,+ checkPromptInjection: args.guard,formats: args.json- ? [{ type: "json", schema: {}, checkPromptInjection: args.guard }]- : [{ type: "markdown" }],+ ? [{ type: "json", schema: {} }]+ : args.deterministicJson+ ? [{ type: "deterministicJson", schema: {} }]+ : [{ type: "markdown" }],} as any,{teamId: "team-id",@@ -325,6 +329,14 @@ describe("calculateCreditsToBeBilled", () => {expect(await billScrape({ json: true, guard: true })).toBe(9);});+ it("bills the prompt injection guard on a markdown-only scrape (5 credits)", async () => {+ expect(await billScrape({ guard: true })).toBe(5);+ });++ it("keeps the guard fee on top of the deterministicJson flat rate", async () => {+ expect(await billScrape({ deterministicJson: true, guard: true })).toBe(7);+ });+it("drops the guard fee when any chunk got no verdict", async () => {expect(await billScrape({@@ -350,7 +362,8 @@ describe("calculateCreditsToBeBilled", () => {) =>calculateCreditsToBeBilled({- formats: [{ type: "json", schema: {}, checkPromptInjection: true }],+ checkPromptInjection: true,+ formats: [{ type: "markdown" }],} as any,{teamId: "team-id",diff --git a/apps/api/src/lib/scrape-billing.ts b/apps/api/src/lib/scrape-billing.tsindex 2de29a4a5..7df98515e 100644--- a/apps/api/src/lib/scrape-billing.ts+++ b/apps/api/src/lib/scrape-billing.ts@@ -68,7 +68,7 @@ export function calculateThreatScanCredits(* chunk it attempted and fails open on a chunk it could not classify* (verdict "none"): a scan that left any chunk unscanned does not bill. A* detection is a verdict for the whole page even if a concurrent chunk failed,- * since it blocked the extraction.+ * since it blocked the scrape.*/function promptInjectionGuardGaveVerdict(costTrackingJSON: ReturnType<typeof CostTracking.prototype.toJSON>,@@ -178,13 +178,6 @@ export async function calculateCreditsToBeBilled(creditsToBeBilled += jsonCostBonus;}- if (- hasFormatOfType(options.formats, "json")?.checkPromptInjection &&- promptInjectionGuardGaveVerdict(costTrackingJSON)- ) {- creditsToBeBilled += 4;- }-if (hasFormatOfType(options.formats, "deterministicJson")) {// 10 when this run generated the extractor script, 3 when it reused a// cached one. The codegen call is tagged in deterministicJson/llm/client.ts.@@ -196,6 +189,13 @@ export async function calculateCreditsToBeBilled(creditsToBeBilled = generatedScript ? 10 : 3;}+ if (+ options.checkPromptInjection &&+ promptInjectionGuardGaveVerdict(costTrackingJSON)+ ) {+ creditsToBeBilled += 4;+ }+if (internalOptions.v1Agent?.model === "fire-1" ||internalOptions.v1JSONAgent?.model?.toLowerCase() === "fire-1"diff --git a/apps/api/src/scraper/scrapeURL/error.ts b/apps/api/src/scraper/scrapeURL/error.tsindex cfe3894ec..6f8a7d79c 100644--- a/apps/api/src/scraper/scrapeURL/error.ts+++ b/apps/api/src/scraper/scrapeURL/error.ts@@ -664,7 +664,7 @@ export class PromptInjectionDetectedError extends TransportableError {super("SCRAPE_PROMPT_INJECTION_DETECTED",message ??- "The scraped page content appears to contain a prompt injection attempt, so JSON extraction was aborted for safety.",+ "The scraped page content appears to contain a prompt injection attempt, so the scrape was aborted for safety.",);}diff --git a/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.test.ts b/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.test.tsindex 4f36fbe65..81307ed6a 100644--- a/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.test.ts+++ b/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.test.ts@@ -10,10 +10,19 @@ vi.mock("./promptInjectionGuard", () => ({createPromptInjectionGuardLimiter: vi.fn(),}));+vi.mock("./smartScrape", () => ({+ smartScrape: vi.fn(),+}));++vi.mock("../../../lib/html-to-markdown", () => ({+ parseMarkdown: vi.fn(async () => "smartscraped page"),+}));+import type { Mock } from "vitest";import { extractData } from "./extractSmartScrape";import { generateCompletions } from "../transformers/llmExtract";import { checkForPromptInjection } from "./promptInjectionGuard";+import { smartScrape } from "./smartScrape";import { JsonExtractionContentTooLargeError } from "../error";import { CostTracking } from "../../../lib/cost-tracking";@@ -48,19 +57,18 @@ describe("extractData", () => {});describe("prompt injection guard", () => {- function run(ids: { scrapeId?: string; extractId?: string }) {+ it("leaves the page itself to the scrape-level guard", async () => {+ (checkForPromptInjection as Mock).mockClear();(generateCompletions as Mock).mockResolvedValueOnce({extract: { ok: true },warning: undefined,totalUsage: { promptTokens: 1, completionTokens: 1, totalTokens: 2 },});- return extractData({++ await extractData({extractOptions: {logger: noopLogger,- options: {- schema: { type: "object", properties: {} },- checkPromptInjection: true,- },+ options: { schema: { type: "object", properties: {} } },markdown: "page",costTrackingOptions: {costTracking: new CostTracking(),@@ -70,35 +78,63 @@ describe("extractData", () => {} as any,urls: ["https://example.com"],useAgent: false,- ...ids,- metadata: { teamId: "test-team", functionId: "performLLMExtract" },+ checkPromptInjection: true,+ scrapeId: "test-scrape",+ metadata: { teamId: "test-team" },});- }- beforeEach(() => {- (checkForPromptInjection as Mock).mockClear();+ expect(checkForPromptInjection).not.toHaveBeenCalled();});- it("passes the scrape id to the guard for its span metadata", async () => {- await run({ scrapeId: "test-scrape" });+ it("scans each page SmartScrape fetches", async () => {+ (checkForPromptInjection as Mock).mockClear();+ (smartScrape as Mock).mockResolvedValueOnce({+ scrapedPages: [{ html: "<p>page</p>" }],+ });+ (generateCompletions as Mock)+ .mockResolvedValueOnce({+ extract: {+ extractedData: null,+ shouldUseSmartscrape: true,+ smartscrape_prompt: "find the title",+ },+ warning: undefined,+ totalUsage: { promptTokens: 1, completionTokens: 1, totalTokens: 2 },+ })+ .mockResolvedValueOnce({ extract: { title: "t" } });- expect(checkForPromptInjection).toHaveBeenCalledTimes(1);- expect(- (checkForPromptInjection as Mock).mock.calls[0][0].metadata,- ).toEqual({- teamId: "test-team",- functionId: "performLLMExtract",+ await extractData({+ extractOptions: {+ logger: noopLogger,+ options: {+ schema: {+ type: "object",+ properties: { title: { type: "string" } },+ },+ },+ markdown: "page",+ costTrackingOptions: {+ costTracking: new CostTracking(),+ metadata: {},+ },+ metadata: { teamId: "test-team" },+ } as any,+ urls: ["https://example.com"],+ useAgent: true,+ checkPromptInjection: true,scrapeId: "test-scrape",- extractId: undefined,+ metadata: { teamId: "test-team", functionId: "performLLMExtract" },});- });- it("passes the extract id to the guard for its span metadata", async () => {- await run({ extractId: "test-extract" });-- expect(- (checkForPromptInjection as Mock).mock.calls[0][0].metadata,- ).toMatchObject({ teamId: "test-team", extractId: "test-extract" });+ expect(checkForPromptInjection).toHaveBeenCalledTimes(1);+ expect((checkForPromptInjection as Mock).mock.calls[0][0]).toMatchObject({+ markdown: "smartscraped page",+ metadata: {+ teamId: "test-team",+ functionId: "performLLMExtract",+ scrapeId: "test-scrape",+ },+ });});});diff --git a/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.ts b/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.tsindex 6ec4ef35f..452dcc64a 100644--- a/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.ts+++ b/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.ts@@ -22,7 +22,7 @@ import { JsonExtractionContentTooLargeError } from "../error";import { toRootSchema, typeIncludes } from "../../../lib/openai-strict-schema";// ~2MB of markdown, well past typical page sizes -- caps worst-case JSON extraction cost/latency.-const MAX_JSON_EXTRACTION_MARKDOWN_CHARS = 2_000_000;+export const MAX_JSON_EXTRACTION_MARKDOWN_CHARS = 2_000_000;const commonSmartScrapeProperties = {shouldUseSmartscrape: {@@ -265,6 +265,7 @@ export async function extractData({extractOptions,urls,useAgent,+ checkPromptInjection,extractId,sessionId,scrapeId,@@ -273,6 +274,8 @@ export async function extractData({extractOptions: GenerateCompletionsOptions;urls: string[];useAgent: boolean;+ /** Scans the pages SmartScrape fetches; the page itself is scanned upstream. */+ checkPromptInjection?: boolean;extractId?: string;sessionId?: string;scrapeId?: string;@@ -286,7 +289,7 @@ export async function extractData({const logger = extractOptions.logger;const isSingleUrl = urls.length === 1;let costLimitExceededTokenUsage: number | null = null;- // Set when the prompt injection guard failed open on part of the content.+ // Set when the prompt injection guard failed open on a SmartScrape page.let promptInjectionScanIncomplete = false;if (@@ -396,18 +399,8 @@ export async function extractData({warning: string | undefined,totalUsage: TokenUsage | undefined;- // Runs concurrently with extraction; guard verdict is checked before `extract` is used.- const [guardSettled, generateSettled] = await Promise.allSettled([- extractOptions.options.checkPromptInjection- ? checkForPromptInjection({- markdown: extractOptions.markdown,- logger,- costTracking: extractOptions.costTrackingOptions.costTracking,- metadata: { ...metadata, scrapeId, extractId },- zeroDataRetention: !!extractOptions.zeroDataRetention,- })- : Promise.resolve(true),- generateCompletions({+ try {+ const completion = await generateCompletions({...extractOptionsNewSchema,costTrackingOptions: {costTracking: extractOptions.costTrackingOptions.costTracking,@@ -417,22 +410,11 @@ export async function extractData({description: "Check if using smartScrape is needed for this case",},},- }),- ]);-- if (guardSettled.status === "rejected") {- throw guardSettled.reason;- }- if (!guardSettled.value) {- promptInjectionScanIncomplete = true;- }-- if (generateSettled.status === "fulfilled") {- extract = generateSettled.value.extract;- warning = generateSettled.value.warning;- totalUsage = generateSettled.value.totalUsage;- } else {- const error = generateSettled.reason;+ });+ extract = completion.extract;+ warning = completion.warning;+ totalUsage = completion.totalUsage;+ } catch (error) {if (error instanceof CostLimitExceededError) {throw error;}@@ -537,7 +519,7 @@ export async function extractData({const guardLimiter = createPromptInjectionGuardLimiter();extractedData = await Promise.all(markdowns.map(async markdown => {- if (extractOptions.options.checkPromptInjection) {+ if (checkPromptInjection) {const scannedFully = await checkForPromptInjection({markdown,logger,diff --git a/apps/api/src/scraper/scrapeURL/lib/promptInjectionGuard.ts b/apps/api/src/scraper/scrapeURL/lib/promptInjectionGuard.tsindex e580df14d..bdead397e 100644--- a/apps/api/src/scraper/scrapeURL/lib/promptInjectionGuard.ts+++ b/apps/api/src/scraper/scrapeURL/lib/promptInjectionGuard.ts@@ -138,11 +138,11 @@ async function classifyChunk(if (result.object.isInjection) {logger.warn(- "Prompt injection detected in scraped content; blocking JSON extraction",+ "Prompt injection detected in scraped content; blocking the scrape",{ reason: result.object.reason },);throw new PromptInjectionDetectedError(- `The scraped page content appears to contain a prompt injection attempt, so JSON extraction was aborted for safety. Guard verdict: ${result.object.reason.slice(0, 300)}`,+ `The scraped page content appears to contain a prompt injection attempt, so the scrape was aborted for safety. Guard verdict: ${result.object.reason.slice(0, 300)}`,);}return true;@@ -183,7 +183,7 @@ export function createPromptInjectionGuardLimiter(): Semaphore {// Throws PromptInjectionDetectedError on a detection. Otherwise resolves to// false when the guard failed open on at least one chunk, i.e. part of the-// content reached extraction unscanned.+// content went unscanned.export async function checkForPromptInjection({markdown,logger,diff --git a/apps/api/src/scraper/scrapeURL/transformers/index.ts b/apps/api/src/scraper/scrapeURL/transformers/index.tsindex c899263c4..8f4c81237 100644--- a/apps/api/src/scraper/scrapeURL/transformers/index.ts+++ b/apps/api/src/scraper/scrapeURL/transformers/index.ts@@ -23,6 +23,7 @@ import { fetchProduct } from "./product";import { fetchMenu } from "./menu";import { fetchVideo } from "./video";import { performRedactPII } from "./redactPII";+import { performPromptInjectionGuard } from "./promptInjectionGuard";import { useIndex, useSearchIndex } from "../../../services/index";import { sendDocumentToIndex } from "../engines/index/index";import { sendDocumentToSearchIndex } from "./sendToSearchIndex";@@ -99,6 +100,7 @@ async function deriveMarkdownFromHTML(// - summary format requires markdown (for summarization)// - question/highlights/query formats require markdown (for page-level answers)// - redactPII needs markdown as its source text (spans are markdown char offsets)+ // - checkPromptInjection scans markdownconst hasMarkdown = hasFormatOfType(meta.options.formats, "markdown");const hasChangeTracking = hasFormatOfType(meta.options.formats,@@ -123,6 +125,7 @@ async function deriveMarkdownFromHTML(!hasHighlights &&!hasQuery &&!hasRedactPII &&+ !meta.options.checkPromptInjection &&!meta.options.onlyCleanContent) {return document;@@ -621,6 +624,7 @@ const transformerStack: Transformer[] = [deriveMarkdownFromHTML,performCleanContent,performRedactPII,+ performPromptInjectionGuard,deriveLinksFromHTML,deriveImagesFromHTML,deriveBrandingFromActions,diff --git a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.tsindex f26ad67f5..4a45b5085 100644--- a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts+++ b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts@@ -1062,6 +1062,7 @@ export async function performLLMExtract(extractOptions: generationOptions,urls: [meta.rewrittenUrl ?? meta.url],useAgent,+ checkPromptInjection: meta.options.checkPromptInjection,scrapeId: meta.id,metadata: {teamId: meta.internalOptions.teamId,diff --git a/apps/api/src/scraper/scrapeURL/transformers/promptInjectionGuard.test.ts b/apps/api/src/scraper/scrapeURL/transformers/promptInjectionGuard.test.tsnew file mode 100644index 000000000..21c3ad472--- /dev/null+++ b/apps/api/src/scraper/scrapeURL/transformers/promptInjectionGuard.test.ts@@ -0,0 +1,92 @@+import type { Mock } from "vitest";+import { performPromptInjectionGuard } from "./promptInjectionGuard";+import { checkForPromptInjection } from "../lib/promptInjectionGuard";+import { PromptInjectionDetectedError } from "../error";+import { CostTracking } from "../../../lib/cost-tracking";+import { MAX_JSON_EXTRACTION_MARKDOWN_CHARS } from "../lib/extractSmartScrape";++vi.mock("../lib/promptInjectionGuard", () => ({+ checkForPromptInjection: vi.fn(),+}));++const mockedGuard = checkForPromptInjection as Mock;++const meta = (checkPromptInjection: boolean) =>+ ({+ id: "test-scrape",+ url: "https://example.com",+ options: { formats: [{ type: "markdown" }], checkPromptInjection },+ internalOptions: {+ teamId: "test-team",+ zeroDataRetention: false,+ llmTelemetry: { functionId: "test-function", metadata: {} },+ },+ costTracking: new CostTracking(),+ logger: { debug: vi.fn(), info: vi.fn(), warn: vi.fn(), error: vi.fn() },+ }) as any;++const document = (markdown: string) =>+ ({ markdown, metadata: {}, warning: "earlier warning" }) as any;++describe("performPromptInjectionGuard", () => {+ beforeEach(() => {+ mockedGuard.mockReset();+ });++ it("does nothing when the check was not requested", async () => {+ const doc = document("page");++ expect(await performPromptInjectionGuard(meta(false), doc)).toBe(doc);+ expect(mockedGuard).not.toHaveBeenCalled();+ });++ it("scans the page markdown with the scrape's ids", async () => {+ mockedGuard.mockResolvedValue(true);+ const doc = document("page");++ expect(await performPromptInjectionGuard(meta(true), doc)).toBe(doc);+ expect(mockedGuard).toHaveBeenCalledTimes(1);+ expect(mockedGuard.mock.calls[0][0]).toMatchObject({+ markdown: "page",+ metadata: {+ teamId: "test-team",+ functionId: "test-function",+ scrapeId: "test-scrape",+ },+ zeroDataRetention: false,+ });+ });++ it("blocks the scrape when the guard detects an injection", async () => {+ mockedGuard.mockRejectedValue(new PromptInjectionDetectedError());++ await expect(+ performPromptInjectionGuard(meta(true), document("page")),+ ).rejects.toBeInstanceOf(PromptInjectionDetectedError);+ });++ it("warns when the guard could not scan every chunk", async () => {+ mockedGuard.mockResolvedValue(false);++ const result = await performPromptInjectionGuard(+ meta(true),+ document("page"),+ );++ expect(result.warning).toMatch(+ /^The prompt injection check could not scan all of the page content.* earlier warning$/,+ );+ });++ it("skips markdown over the json extraction cap with the same warning", async () => {+ const result = await performPromptInjectionGuard(+ meta(true),+ document("x".repeat(MAX_JSON_EXTRACTION_MARKDOWN_CHARS + 1)),+ );++ expect(mockedGuard).not.toHaveBeenCalled();+ expect(result.warning).toMatch(+ /^The prompt injection check could not scan all of the page content/,+ );+ });+});diff --git a/apps/api/src/scraper/scrapeURL/transformers/promptInjectionGuard.ts b/apps/api/src/scraper/scrapeURL/transformers/promptInjectionGuard.tsnew file mode 100644index 000000000..b62527e20--- /dev/null+++ b/apps/api/src/scraper/scrapeURL/transformers/promptInjectionGuard.ts@@ -0,0 +1,41 @@+import { Meta } from "..";+import { Document } from "../../../controllers/v2/types";+import { checkForPromptInjection } from "../lib/promptInjectionGuard";+import { MAX_JSON_EXTRACTION_MARKDOWN_CHARS } from "../lib/extractSmartScrape";++// Runs before every LLM-backed format, so blocked content never reaches one.+export async function performPromptInjectionGuard(+ meta: Meta,+ document: Document,+): Promise<Document> {+ if (!meta.options.checkPromptInjection) {+ return document;+ }++ const markdown = document.markdown ?? "";+ const scannedFully =+ markdown.length <= MAX_JSON_EXTRACTION_MARKDOWN_CHARS &&+ (await checkForPromptInjection({+ markdown,+ logger: meta.logger,+ costTracking: meta.costTracking,+ metadata: {+ teamId: meta.internalOptions.teamId,+ functionId: meta.internalOptions.llmTelemetry?.functionId,+ scrapeId: meta.id,+ },+ zeroDataRetention: !!meta.internalOptions.zeroDataRetention,+ }));++ if (scannedFully) {+ return document;+ }++ // The guard fee does not bill in this case (see scrape-billing.ts).+ return {+ ...document,+ warning:+ "The prompt injection check could not scan all of the page content, so some of it was not checked. The prompt injection check was not billed." ++ (document.warning ? " " + document.warning : ""),+ };+}diff --git a/apps/api/src/services/monitoring/store.test.ts b/apps/api/src/services/monitoring/store.test.tsindex fc4abf47a..c6bcefaad 100644--- a/apps/api/src/services/monitoring/store.test.ts+++ b/apps/api/src/services/monitoring/store.test.ts@@ -290,6 +290,23 @@ describe("monitoring store credit helpers", () => {expect(estimateMonitorCreditsPerRun(targets, false)).toBe(9);});+ it("estimates the top-level prompt injection guard on a markdown monitor", () => {+ const targets: MonitorTarget[] = [+ {+ id: "target-1",+ type: "scrape",+ urls: ["https://example.com/a"],+ scrapeOptions: {+ formats: ["markdown"],+ checkPromptInjection: true,+ },+ },+ ];++ // 1 base + 4 prompt injection guard.+ expect(estimateMonitorCreditsPerRun(targets, false)).toBe(5);+ });+it("uses target options when page rows do not have recorded scrape credits", () => {const targets: MonitorTarget[] = [{diff --git a/apps/api/src/services/monitoring/store.ts b/apps/api/src/services/monitoring/store.tsindex 4b4851f53..472b5411d 100644--- a/apps/api/src/services/monitoring/store.ts+++ b/apps/api/src/services/monitoring/store.ts@@ -15,7 +15,11 @@ import { db, dbRr } from "../../db/connection";import * as schema from "../../db/schema";import { monitoringClaimDueMonitors } from "../../db/rpc";import { config } from "../../config";-import { getPDFMaxPages, shouldParsePDF } from "../../controllers/v2/types";+import {+ getPDFMaxPages,+ liftCheckPromptInjection,+ shouldParsePDF,+} from "../../controllers/v2/types";import { isXTwitterUrl } from "../../scraper/scrapeURL/engines/x-twitter/url";import {getNextMonitorRunAt,@@ -88,18 +92,6 @@ function hasAnyFormatOfType(formats: unknown, types: string[]): boolean {return types.some(type => hasFormatOfType(formats, type));}-function requestsPromptInjectionCheck(formats: unknown): boolean {- if (!Array.isArray(formats)) return false;- return formats.some(- format =>- !!format &&- typeof format === "object" &&- formatType(format) === "json" &&- "checkPromptInjection" in format &&- format.checkPromptInjection === true,- );-}-function requestsJsonChangeTracking(formats: unknown): boolean {if (!Array.isArray(formats)) return false;return formats.some(format => {@@ -140,7 +132,7 @@ function estimateBaseCreditsPerPage(// The prompt injection guard bills +4 in calculateCreditsToBeBilled. The// estimate cannot know whether the guard ran, so it assumes it does.- if (requestsPromptInjectionCheck(formats)) {+ if (liftCheckPromptInjection(options ?? {}).checkPromptInjection === true) {credits += SCRAPE_OPTION_CREDIT_BONUS;}diff --git a/apps/js-sdk/firecrawl/package.json b/apps/js-sdk/firecrawl/package.jsonindex a9cd9976d..36c7326a2 100644--- a/apps/js-sdk/firecrawl/package.json+++ b/apps/js-sdk/firecrawl/package.json@@ -1,6 +1,6 @@{"name": "@mendable/firecrawl-js",- "version": "4.45.0",+ "version": "4.46.0","description": "JavaScript SDK for the Firecrawl API: web scraping, crawling, web search, and scientific literature search over a research paper index of PubMed, bioRxiv, medRxiv and arXiv abstracts","main": "dist/index.js","types": "dist/index.d.ts",diff --git a/apps/js-sdk/firecrawl/src/v2/types.ts b/apps/js-sdk/firecrawl/src/v2/types.tsindex ee0cd0a36..5df514c9c 100644--- a/apps/js-sdk/firecrawl/src/v2/types.ts+++ b/apps/js-sdk/firecrawl/src/v2/types.ts@@ -31,6 +31,7 @@ export interface JsonFormat extends Format {type: "json";prompt?: string;schema?: Record<string, unknown> | ZodTypeAny;+ /** @deprecated Use the top-level `checkPromptInjection` scrape option. */checkPromptInjection?: boolean;}@@ -276,6 +277,13 @@ export interface ScrapeOptions {minAge?: number;storeInCache?: boolean;lockdown?: boolean;+ /**+ * Scans the page content for prompt injection with any format except+ * rawBase64, before LLM-backed formats run. A detection fails the scrape+ * with SCRAPE_PROMPT_INJECTION_DETECTED.+ * Adds 4 credits when the check scans the whole page.+ */+ checkPromptInjection?: boolean;redactPII?: boolean | RedactPIIOptions;threatProtection?: ThreatProtectionOptions;auditMetadata?: AuditMetadata;diff --git a/apps/python-sdk/firecrawl/__init__.py b/apps/python-sdk/firecrawl/__init__.pyindex 3e443fccd..22651aa68 100644--- a/apps/python-sdk/firecrawl/__init__.py+++ b/apps/python-sdk/firecrawl/__init__.py@@ -41,7 +41,7 @@ from .v1 import (V1ChangeTrackingOptions,)-__version__ = "4.49.3"+__version__ = "4.50.0"# Define the logger for the Firecrawl projectlogger: logging.Logger = logging.getLogger("firecrawl")diff --git a/apps/python-sdk/firecrawl/__tests__/unit/v2/utils/test_validation.py b/apps/python-sdk/firecrawl/__tests__/unit/v2/utils/test_validation.pyindex 24d1e0bc1..97370c81f 100644--- a/apps/python-sdk/firecrawl/__tests__/unit/v2/utils/test_validation.py+++ b/apps/python-sdk/firecrawl/__tests__/unit/v2/utils/test_validation.py@@ -116,6 +116,25 @@ class TestPrepareScrapeOptions:assert result["timeout"] == 30000assert result["waitFor"] == 2000+ def test_prepare_top_level_check_prompt_injection(self):+ """check_prompt_injection is sent as the top-level checkPromptInjection."""+ result = prepare_scrape_options(+ ScrapeOptions(formats=["markdown"], check_prompt_injection=True)+ )++ assert result["checkPromptInjection"] is True+ assert "check_prompt_injection" not in result++ def test_prepare_json_format_check_prompt_injection_still_sent(self):+ """The deprecated json-format field keeps working."""+ result = prepare_scrape_options(+ ScrapeOptions(+ formats=[JsonFormat(prompt="Extract", check_prompt_injection=True)]+ )+ )++ assert result["formats"][0]["checkPromptInjection"] is True+def test_prepare_snake_case_conversion(self):"""Test snake_case to camelCase conversion."""options = ScrapeOptions(diff --git a/apps/python-sdk/firecrawl/v2/client.py b/apps/python-sdk/firecrawl/v2/client.pyindex d21a50116..8af7917dc 100644--- a/apps/python-sdk/firecrawl/v2/client.py+++ b/apps/python-sdk/firecrawl/v2/client.py@@ -97,7 +97,8 @@ _SCRAPE_OPTION_KEYS = frozenset({"only_main_content", "timeout", "wait_for", "mobile","parsers", "actions", "location", "skip_tls_verification","remove_base64_images", "fast_mode", "use_mock", "block_ads",- "proxy", "max_age", "store_in_cache", "lockdown", "threat_protection",+ "proxy", "max_age", "store_in_cache", "lockdown", "check_prompt_injection",+ "threat_protection","profile", "audit_metadata",})@@ -186,6 +187,7 @@ class FirecrawlClient:max_age: Optional[int] = None,store_in_cache: Optional[bool] = None,lockdown: Optional[bool] = None,+ check_prompt_injection: Optional[bool] = None,threat_protection: Optional[ThreatProtectionOptions] = None,profile: Optional[Dict[str, Any]] = None,audit_metadata: Optional[AuditMetadata] = None,@@ -217,6 +219,7 @@ class FirecrawlClient:max_age: Maximum age of the cachestore_in_cache: Whether to store the result in the cachelockdown: Serve only previously cached results; never make outbound requests. Returns 404 SCRAPE_LOCKDOWN_CACHE_MISS on cache miss.+ check_prompt_injection: Scan the page content for prompt injection with any format except rawBase64, before LLM-backed formats run. A detection fails the scrape. Adds 4 credits when the check scans the whole page.threat_protection: Enterprise per-request override of the team's threat protection policyprofile: Browser profile for persistent state (e.g. {"name": "my-profile", "saveChanges": True})tool_detail: "compact" returns provider, capability and description; "summary" (default) adds metadata; "full" includes contracts when domain discovery is enabled.@@ -246,6 +249,7 @@ class FirecrawlClient:max_age=max_age,store_in_cache=store_in_cache,lockdown=lockdown,+ check_prompt_injection=check_prompt_injection,threat_protection=threat_protection,profile=profile,audit_metadata=audit_metadata,@@ -253,7 +257,7 @@ class FirecrawlClient:domain_tools=domain_tools,tool_detail=tool_detail,).items() if v is not None}- ) if any(v is not None for v in [formats, headers, include_tags, exclude_tags, only_main_content, timeout, wait_for, mobile, parsers, actions, location, skip_tls_verification, remove_base64_images, fast_mode, use_mock, block_ads, proxy, max_age, store_in_cache, lockdown, threat_protection, profile, audit_metadata, integration, domain_tools, tool_detail]) else None+ ) if any(v is not None for v in [formats, headers, include_tags, exclude_tags, only_main_content, timeout, wait_for, mobile, parsers, actions, location, skip_tls_verification, remove_base64_images, fast_mode, use_mock, block_ads, proxy, max_age, store_in_cache, lockdown, check_prompt_injection, threat_protection, profile, audit_metadata, integration, domain_tools, tool_detail]) else Noneif alexandria is not None:if url is not None or auto_resume is not None or (options and set(options.model_dump(exclude_none=True, exclude_unset=True)) - {"timeout", "integration"}):raise ValueError("alexandria cannot be combined with URL scrape options")@@ -636,6 +640,7 @@ class FirecrawlClient:max_age: Optional[int] = None,store_in_cache: Optional[bool] = None,lockdown: Optional[bool] = None,+ check_prompt_injection: Optional[bool] = None,threat_protection: Optional[ThreatProtectionOptions] = None,profile: Optional[Dict[str, Any]] = None,audit_metadata: Optional[AuditMetadata] = None,@@ -687,6 +692,7 @@ class FirecrawlClient:max_age: Cache max age (convenience kwarg)store_in_cache: Cache results (convenience kwarg)lockdown: Serve only cached results (convenience kwarg)+ check_prompt_injection: Scan the page content for prompt injection with any format except rawBase64, before LLM-backed formats run. A detection fails the scrape. Adds 4 credits when the check scans the whole page.threat_protection: Enterprise threat protection override (convenience kwarg)profile: Browser profile (convenience kwarg)audit_metadata: Metadata to include in SIEM logging events@@ -714,6 +720,7 @@ class FirecrawlClient:remove_base64_images=remove_base64_images, fast_mode=fast_mode,use_mock=use_mock, block_ads=block_ads, proxy=proxy,max_age=max_age, store_in_cache=store_in_cache, lockdown=lockdown,+ check_prompt_injection=check_prompt_injection,threat_protection=threat_protection, profile=profile,audit_metadata=audit_metadata,).items() if v is not None}@@ -798,6 +805,7 @@ class FirecrawlClient:max_age: Optional[int] = None,store_in_cache: Optional[bool] = None,lockdown: Optional[bool] = None,+ check_prompt_injection: Optional[bool] = None,threat_protection: Optional[ThreatProtectionOptions] = None,profile: Optional[Dict[str, Any]] = None,audit_metadata: Optional[AuditMetadata] = None,@@ -847,6 +855,7 @@ class FirecrawlClient:max_age: Cache max age (convenience kwarg)store_in_cache: Cache results (convenience kwarg)lockdown: Serve only cached results (convenience kwarg)+ check_prompt_injection: Scan the page content for prompt injection with any format except rawBase64, before LLM-backed formats run. A detection fails the scrape. Adds 4 credits when the check scans the whole page.threat_protection: Enterprise threat protection override (convenience kwarg)profile: Browser profile (convenience kwarg)audit_metadata: Metadata to include in SIEM logging events@@ -871,6 +880,7 @@ class FirecrawlClient:remove_base64_images=remove_base64_images, fast_mode=fast_mode,use_mock=use_mock, block_ads=block_ads, proxy=proxy,max_age=max_age, store_in_cache=store_in_cache, lockdown=lockdown,+ check_prompt_injection=check_prompt_injection,threat_protection=threat_protection, profile=profile,audit_metadata=audit_metadata,).items() if v is not None}@@ -1352,6 +1362,7 @@ class FirecrawlClient:max_age: Optional[int] = None,store_in_cache: Optional[bool] = None,lockdown: Optional[bool] = None,+ check_prompt_injection: Optional[bool] = None,threat_protection: Optional[ThreatProtectionOptions] = None,audit_metadata: Optional[AuditMetadata] = None,webhook: Optional[Union[str, WebhookConfig]] = None,@@ -1386,6 +1397,7 @@ class FirecrawlClient:max_age: Cache max agestore_in_cache: Whether to store results in cachelockdown: Serve only previously cached results; never make outbound requests.+ check_prompt_injection: Scan the page content for prompt injection with any format except rawBase64, before LLM-backed formats run. A detection fails the scrape. Adds 4 credits when the check scans the whole page.threat_protection: Enterprise per-request override of the team's threat protection policyaudit_metadata: Metadata to include in SIEM logging eventswebhook: Webhook configuration@@ -1421,9 +1433,10 @@ class FirecrawlClient:max_age=max_age,store_in_cache=store_in_cache,lockdown=lockdown,+ check_prompt_injection=check_prompt_injection,threat_protection=threat_protection,).items() if v is not None}- ) if any(v is not None for v in [formats, headers, include_tags, exclude_tags, only_main_content, timeout, wait_for, mobile, parsers, actions, location, skip_tls_verification, remove_base64_images, fast_mode, use_mock, block_ads, proxy, max_age, store_in_cache, lockdown, threat_protection]) else None+ ) if any(v is not None for v in [formats, headers, include_tags, exclude_tags, only_main_content, timeout, wait_for, mobile, parsers, actions, location, skip_tls_verification, remove_base64_images, fast_mode, use_mock, block_ads, proxy, max_age, store_in_cache, lockdown, check_prompt_injection, threat_protection]) else Nonereturn batch_module.start_batch_scrape(self.http_client,@@ -1868,6 +1881,7 @@ class FirecrawlClient:max_age: Optional[int] = None,store_in_cache: Optional[bool] = None,lockdown: Optional[bool] = None,+ check_prompt_injection: Optional[bool] = None,threat_protection: Optional[ThreatProtectionOptions] = None,audit_metadata: Optional[AuditMetadata] = None,webhook: Optional[Union[str, WebhookConfig]] = None,@@ -1905,9 +1919,10 @@ class FirecrawlClient:max_age=max_age,store_in_cache=store_in_cache,lockdown=lockdown,+ check_prompt_injection=check_prompt_injection,threat_protection=threat_protection,).items() if v is not None}- ) if any(v is not None for v in [formats, headers, include_tags, exclude_tags, only_main_content, timeout, wait_for, mobile, parsers, actions, location, skip_tls_verification, remove_base64_images, fast_mode, use_mock, block_ads, proxy, max_age, store_in_cache, lockdown, threat_protection]) else None+ ) if any(v is not None for v in [formats, headers, include_tags, exclude_tags, only_main_content, timeout, wait_for, mobile, parsers, actions, location, skip_tls_verification, remove_base64_images, fast_mode, use_mock, block_ads, proxy, max_age, store_in_cache, lockdown, check_prompt_injection, threat_protection]) else Nonereturn batch_module.batch_scrape(self.http_client,diff --git a/apps/python-sdk/firecrawl/v2/types.py b/apps/python-sdk/firecrawl/v2/types.pyindex 6e6bc2a72..0ef9ca63e 100644--- a/apps/python-sdk/firecrawl/v2/types.py+++ b/apps/python-sdk/firecrawl/v2/types.py@@ -698,6 +698,7 @@ class JsonFormat(Format):type: Literal["json"] = "json"prompt: Optional[str] = Noneschema: Optional[Any] = None+ # Deprecated: use the top-level ScrapeOptions.check_prompt_injection.check_prompt_injection: Optional[bool] = None@@ -919,6 +920,12 @@ class ScrapeOptions(BaseModel):min_age: Optional[int] = Nonestore_in_cache: Optional[bool] = Nonelockdown: Optional[bool] = None+ # Scans the page content for prompt injection with any format except+ # rawBase64, before LLM-backed formats run. A detection fails the scrape.+ # Adds 4 credits when the check scans the whole page.+ check_prompt_injection: Optional[bool] = Field(+ default=None, alias="checkPromptInjection"+ )redact_pii: Optional[Union[bool, RedactPIIOptions]] = Field(default=None, alias="redactPII")diff --git a/apps/python-sdk/firecrawl/v2/utils/validation.py b/apps/python-sdk/firecrawl/v2/utils/validation.pyindex 093aa2ce2..4bbed9cd9 100644--- a/apps/python-sdk/firecrawl/v2/utils/validation.py+++ b/apps/python-sdk/firecrawl/v2/utils/validation.py@@ -570,6 +570,7 @@ def prepare_scrape_options(options: Optional[ScrapeOptions]) -> Optional[Dict[st"max_age": "maxAge","min_age": "minAge","redact_pii": "redactPII",+ "check_prompt_injection": "checkPromptInjection","threat_protection": "threatProtection","audit_metadata": "auditMetadata",}