Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(api): make checkPromptInjection a top-level scrape option (#5024)

Gergő Móriczfa7ec5f0a954
diff --git a/apps/api/src/__tests__/snips/v2/scrape.test.ts b/apps/api/src/__tests__/snips/v2/scrape.test.ts
index a14d00ea6..3990910f7 100644
--- a/apps/api/src/__tests__/snips/v2/scrape.test.ts
+++ b/apps/api/src/__tests__/snips/v2/scrape.test.ts
@@ -445,6 +445,46 @@ describe("Scrape tests", () => {
scrapeTimeout,
);
+ itIf(TEST_PRODUCTION || (HAS_AI && ALLOW_TEST_SUITE_WEBSITE))(
+ "runs the top-level checkPromptInjection on a markdown scrape and bills it",
+ async () => {
+ const response = await scrape(
+ {
+ url: `${TEST_SUITE_WEBSITE}/product`,
+ formats: ["markdown"],
+ checkPromptInjection: true,
+ timeout: scrapeTimeout,
+ },
+ identity,
+ );
+
+ expect(response.markdown).toBeDefined();
+ expect(response.metadata.creditsUsed).toBe(5);
+ },
+ scrapeTimeout,
+ );
+
+ itIf(TEST_PRODUCTION || (HAS_AI && ALLOW_TEST_SUITE_WEBSITE))(
+ "blocks a markdown scrape when the top-level checkPromptInjection detects a prompt injection",
+ async () => {
+ const raw = await scrapeRaw(
+ {
+ url: `${TEST_SUITE_WEBSITE}/prompt-injection`,
+ formats: ["markdown"],
+ checkPromptInjection: true,
+ timeout: scrapeTimeout,
+ },
+ identity,
+ );
+
+ expect(raw.statusCode).toBe(403);
+ expect(raw.body.success).toBe(false);
+ expect(raw.body.code).toBe("SCRAPE_PROMPT_INJECTION_DETECTED");
+ expect(typeof raw.body.error).toBe("string");
+ },
+ scrapeTimeout,
+ );
+
itIf(TEST_PRODUCTION || (HAS_AI && ALLOW_TEST_SUITE_WEBSITE))(
"blocks when checkPromptInjection detects a prompt injection",
async () => {
diff --git a/apps/api/src/controllers/v2/check-prompt-injection-schema.test.ts b/apps/api/src/controllers/v2/check-prompt-injection-schema.test.ts
new file mode 100644
index 000000000..ce2ee28e2
--- /dev/null
+++ b/apps/api/src/controllers/v2/check-prompt-injection-schema.test.ts
@@ -0,0 +1,106 @@
+import { describe, expect, it } from "vitest";
+import {
+ crawlRequestSchema,
+ fromV1ScrapeOptions,
+ scrapeOptions,
+ scrapeRequestSchema,
+} from "./types";
+import { scrapeOptions as v1ScrapeOptions } from "../v1/types";
+
+const url = "https://example.com";
+const jsonFormat = {
+ type: "json" as const,
+ schema: { type: "object", properties: { title: { type: "string" } } },
+};
+
+describe("checkPromptInjection", () => {
+ it("accepts the top-level flag with any format", () => {
+ const parsed = scrapeRequestSchema.parse({
+ url,
+ checkPromptInjection: true,
+ formats: ["markdown"],
+ });
+
+ expect(parsed.checkPromptInjection).toBe(true);
+ });
+
+ it("lifts the json-format flag to the top level and drops it from the format", () => {
+ const parsed = scrapeRequestSchema.parse({
+ url,
+ formats: [{ ...jsonFormat, checkPromptInjection: true }],
+ });
+
+ expect(parsed.checkPromptInjection).toBe(true);
+ expect(parsed.formats[0]).not.toHaveProperty("checkPromptInjection");
+ });
+
+ it("enables the check when either level sets it", () => {
+ const parsed = scrapeRequestSchema.parse({
+ url,
+ checkPromptInjection: true,
+ formats: [{ ...jsonFormat, checkPromptInjection: false }],
+ });
+
+ expect(parsed.checkPromptInjection).toBe(true);
+ });
+
+ it("defaults to false", () => {
+ expect(scrapeRequestSchema.parse({ url }).checkPromptInjection).toBe(false);
+ });
+
+ it("still validates the json-format flag", () => {
+ expect(
+ scrapeRequestSchema.safeParse({
+ url,
+ formats: [{ ...jsonFormat, checkPromptInjection: "yes" }],
+ }).success,
+ ).toBe(false);
+ });
+
+ it("is stable when parsed options are parsed again", () => {
+ const once = scrapeOptions.parse({
+ formats: [{ ...jsonFormat, checkPromptInjection: true }],
+ });
+
+ expect(scrapeOptions.parse(once)).toEqual(once);
+ });
+
+ it("lifts the json-format flag inside crawl scrapeOptions", () => {
+ const parsed = crawlRequestSchema.parse({
+ url,
+ scrapeOptions: {
+ formats: [{ ...jsonFormat, checkPromptInjection: true }],
+ },
+ });
+
+ expect(parsed.scrapeOptions.checkPromptInjection).toBe(true);
+ });
+
+ it("rejects the rawBase64 format, which has no markdown to scan", () => {
+ expect(
+ scrapeRequestSchema.safeParse({
+ url,
+ checkPromptInjection: true,
+ formats: ["rawBase64"],
+ }).success,
+ ).toBe(false);
+ });
+
+ it("maps v1 jsonOptions.checkPromptInjection to the top-level flag", () => {
+ const v1 = v1ScrapeOptions.parse({
+ formats: ["json"],
+ jsonOptions: { schema: jsonFormat.schema, checkPromptInjection: true },
+ });
+
+ const { scrapeOptions: converted } = fromV1ScrapeOptions(
+ v1,
+ undefined,
+ "team-id",
+ );
+
+ expect(converted.checkPromptInjection).toBe(true);
+ expect(converted.formats.find(f => f.type === "json")).not.toHaveProperty(
+ "checkPromptInjection",
+ );
+ });
+});
diff --git a/apps/api/src/controllers/v2/parse.ts b/apps/api/src/controllers/v2/parse.ts
index 2f5c47043..e0b370dff 100644
--- a/apps/api/src/controllers/v2/parse.ts
+++ b/apps/api/src/controllers/v2/parse.ts
@@ -720,7 +720,8 @@ export async function parseController(
!!hasFormatOfType(req.body.formats, "summary") ||
!!hasFormatOfType(req.body.formats, "question") ||
!!hasFormatOfType(req.body.formats, "highlights") ||
- !!hasFormatOfType(req.body.formats, "query");
+ !!hasFormatOfType(req.body.formats, "query") ||
+ req.body.checkPromptInjection;
if (!usedLlm) {
const ct = hasFormatOfType(req.body.formats, "changeTracking");
diff --git a/apps/api/src/controllers/v2/scrape.ts b/apps/api/src/controllers/v2/scrape.ts
index 59600a8a2..b871c7e52 100644
--- a/apps/api/src/controllers/v2/scrape.ts
+++ b/apps/api/src/controllers/v2/scrape.ts
@@ -722,7 +722,8 @@ export async function scrapeController(
!!hasFormatOfType(req.body.formats, "branding") ||
!!hasFormatOfType(req.body.formats, "question") ||
!!hasFormatOfType(req.body.formats, "highlights") ||
- !!hasFormatOfType(req.body.formats, "query");
+ !!hasFormatOfType(req.body.formats, "query") ||
+ req.body.checkPromptInjection;
if (!usedLlm) {
const ct = hasFormatOfType(req.body.formats, "changeTracking");
diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.ts
index 727bc1f1c..b68374592 100644
--- a/apps/api/src/controllers/v2/types.ts
+++ b/apps/api/src/controllers/v2/types.ts
@@ -365,10 +365,14 @@ const jsonFormatWithOptions = z.strictObject({
})
.superRefine(addStrictSchemaIssue),
prompt: z.string().max(10000).optional(),
+ // Deprecated: lifted into the top-level checkPromptInjection at parse time.
checkPromptInjection: z.boolean().optional(),
});
-export type JsonFormatWithOptions = z.output<typeof jsonFormatWithOptions>;
+export type JsonFormatWithOptions = Omit<
+ z.output<typeof jsonFormatWithOptions>,
+ "checkPromptInjection"
+>;
// "Deterministic JSON" — same shape as json mode, but extraction is performed by
// reusable-json-mode (a cached, reusable JS extractor run in the code-sandbox)
@@ -844,6 +848,7 @@ const scrapeOptionFields = z.strictObject({
minAge: z.int().gte(0).optional(),
storeInCache: z.boolean().prefault(true),
lockdown: z.boolean().prefault(false),
+ checkPromptInjection: z.boolean().prefault(false),
safeMode: z.boolean().optional(),
redactPII: redactPIISchema,
// Enterprise: per-request field-level override of the org's threat
@@ -866,13 +871,23 @@ const scrapeOptionFields = z.strictObject({
__forceFirePDF: z.boolean().prefault(false).optional(),
});
-const baseScrapeOptions = scrapeOptionFields.refine(
- options => !(options.profile && options.lockdown),
- {
+const baseScrapeOptions = scrapeOptionFields
+ .refine(options => !(options.profile && options.lockdown), {
message: "Profiles require live browsing and cannot be used with lockdown.",
path: ["profile"],
- },
-);
+ })
+ .refine(
+ options =>
+ !(
+ options.checkPromptInjection &&
+ options.formats.some(f => f.type === "rawBase64")
+ ),
+ {
+ message:
+ "checkPromptInjection scans page markdown and cannot be used with the rawBase64 format.",
+ path: ["checkPromptInjection"],
+ },
+ );
type ScrapeOptionsBase = z.infer<typeof baseScrapeOptions>;
@@ -902,6 +917,34 @@ export const applyScrapeOptionsDefaults = <T extends ScrapeOptionsBase>(
: true),
});
+/**
+ * Folds the deprecated json-format checkPromptInjection into the top-level
+ * flag and drops it from the format. Also accepts raw stored monitor options.
+ */
+export function liftCheckPromptInjection<
+ T extends { formats?: unknown[]; checkPromptInjection?: unknown },
+>(options: T): T {
+ let requestedOnFormat = false;
+ const formats = options.formats?.map(format => {
+ if (
+ typeof format !== "object" ||
+ format === null ||
+ !("checkPromptInjection" in format)
+ ) {
+ return format;
+ }
+ const { checkPromptInjection, ...rest } = format;
+ requestedOnFormat ||= checkPromptInjection === true;
+ return rest;
+ });
+ return {
+ ...options,
+ formats,
+ checkPromptInjection:
+ options.checkPromptInjection === true || requestedOnFormat,
+ };
+}
+
// Base transform function that handles both nullable and non-nullable cases
// Uses generic type to preserve all fields from extended schemas
const extractTransformImpl = <T extends ScrapeOptionsBase | undefined>(
@@ -909,7 +952,7 @@ const extractTransformImpl = <T extends ScrapeOptionsBase | undefined>(
): T extends undefined ? undefined : T => {
if (!obj) return obj as T extends undefined ? undefined : T;
// Handle timeout
- let result = applyScrapeOptionsDefaults(obj);
+ let result = liftCheckPromptInjection(applyScrapeOptionsDefaults(obj));
if (
obj.formats.find(x => typeof x === "object" && x.type === "json") &&
obj.timeout === 30000
@@ -2327,6 +2370,9 @@ export function fromV1ScrapeOptions(
}
: {}),
location: v1ScrapeOptions.location ?? v1ScrapeOptions.geolocation,
+ checkPromptInjection: (
+ v1ScrapeOptions.jsonOptions || v1ScrapeOptions.extract
+ )?.checkPromptInjection,
formats: v1ScrapeOptions.formats
.map(x => {
// json and extract is standardized down to extract fmt in v1 -- fine to take one and dismiss the other
@@ -2336,7 +2382,6 @@ export function fromV1ScrapeOptions(
type: "json",
schema: opts?.schema,
prompt: opts?.prompt,
- checkPromptInjection: opts?.checkPromptInjection ?? false,
};
return fmt;
} else if (x === "json") {
@@ -2347,7 +2392,6 @@ export function fromV1ScrapeOptions(
type: "json",
schema: opts.schema,
prompt: opts.prompt,
- checkPromptInjection: opts.checkPromptInjection ?? false,
};
return includesFormat(v1ScrapeOptions.formats as any, "extract")
? null
diff --git a/apps/api/src/lib/keyless-credit-projection.test.ts b/apps/api/src/lib/keyless-credit-projection.test.ts
new file mode 100644
index 000000000..02a122697
--- /dev/null
+++ b/apps/api/src/lib/keyless-credit-projection.test.ts
@@ -0,0 +1,33 @@
+import { describe, expect, it } from "vitest";
+import { projectScrapeCredits } from "./keyless-credit-projection";
+import { scrapeOptions } from "../controllers/v2/types";
+
+describe("projectScrapeCredits", () => {
+ it("projects the prompt injection guard fee on a markdown scrape", () => {
+ expect(
+ projectScrapeCredits(
+ scrapeOptions.parse({ checkPromptInjection: true }),
+ null,
+ false,
+ ),
+ ).toBe(5);
+ });
+
+ it("projects the guard fee on top of json", () => {
+ expect(
+ projectScrapeCredits(
+ scrapeOptions.parse({
+ formats: [
+ {
+ type: "json",
+ schema: { type: "object", properties: {} },
+ checkPromptInjection: true,
+ },
+ ],
+ }),
+ null,
+ false,
+ ),
+ ).toBe(9);
+ });
+});
diff --git a/apps/api/src/lib/keyless-credit-projection.ts b/apps/api/src/lib/keyless-credit-projection.ts
index 9037f558a..0239c4524 100644
--- a/apps/api/src/lib/keyless-credit-projection.ts
+++ b/apps/api/src/lib/keyless-credit-projection.ts
@@ -27,6 +27,10 @@ export function projectScrapeCredits(
credits = 10;
}
+ if (options.checkPromptInjection) {
+ credits += 4;
+ }
+
if (
hasFormatOfType(options.formats, "question") ||
hasFormatOfType(options.formats, "query")
diff --git a/apps/api/src/lib/scrape-billing.test.ts b/apps/api/src/lib/scrape-billing.test.ts
index a798a4a89..8d4148e36 100644
--- a/apps/api/src/lib/scrape-billing.test.ts
+++ b/apps/api/src/lib/scrape-billing.test.ts
@@ -273,15 +273,19 @@ describe("calculateCreditsToBeBilled", () => {
const billScrape = (args: {
lockdown?: boolean;
json?: boolean;
+ deterministicJson?: boolean;
guard?: boolean;
guardVerdicts?: ("clean" | "injection" | "none")[];
}) =>
calculateCreditsToBeBilled(
{
lockdown: args.lockdown,
+ checkPromptInjection: args.guard,
formats: args.json
- ? [{ type: "json", schema: {}, checkPromptInjection: args.guard }]
- : [{ type: "markdown" }],
+ ? [{ type: "json", schema: {} }]
+ : args.deterministicJson
+ ? [{ type: "deterministicJson", schema: {} }]
+ : [{ type: "markdown" }],
} as any,
{
teamId: "team-id",
@@ -325,6 +329,14 @@ describe("calculateCreditsToBeBilled", () => {
expect(await billScrape({ json: true, guard: true })).toBe(9);
});
+ it("bills the prompt injection guard on a markdown-only scrape (5 credits)", async () => {
+ expect(await billScrape({ guard: true })).toBe(5);
+ });
+
+ it("keeps the guard fee on top of the deterministicJson flat rate", async () => {
+ expect(await billScrape({ deterministicJson: true, guard: true })).toBe(7);
+ });
+
it("drops the guard fee when any chunk got no verdict", async () => {
expect(
await billScrape({
@@ -350,7 +362,8 @@ describe("calculateCreditsToBeBilled", () => {
) =>
calculateCreditsToBeBilled(
{
- formats: [{ type: "json", schema: {}, checkPromptInjection: true }],
+ checkPromptInjection: true,
+ formats: [{ type: "markdown" }],
} as any,
{
teamId: "team-id",
diff --git a/apps/api/src/lib/scrape-billing.ts b/apps/api/src/lib/scrape-billing.ts
index 2de29a4a5..7df98515e 100644
--- a/apps/api/src/lib/scrape-billing.ts
+++ b/apps/api/src/lib/scrape-billing.ts
@@ -68,7 +68,7 @@ export function calculateThreatScanCredits(
* chunk it attempted and fails open on a chunk it could not classify
* (verdict "none"): a scan that left any chunk unscanned does not bill. A
* detection is a verdict for the whole page even if a concurrent chunk failed,
- * since it blocked the extraction.
+ * since it blocked the scrape.
*/
function promptInjectionGuardGaveVerdict(
costTrackingJSON: ReturnType<typeof CostTracking.prototype.toJSON>,
@@ -178,13 +178,6 @@ export async function calculateCreditsToBeBilled(
creditsToBeBilled += jsonCostBonus;
}
- if (
- hasFormatOfType(options.formats, "json")?.checkPromptInjection &&
- promptInjectionGuardGaveVerdict(costTrackingJSON)
- ) {
- creditsToBeBilled += 4;
- }
-
if (hasFormatOfType(options.formats, "deterministicJson")) {
// 10 when this run generated the extractor script, 3 when it reused a
// cached one. The codegen call is tagged in deterministicJson/llm/client.ts.
@@ -196,6 +189,13 @@ export async function calculateCreditsToBeBilled(
creditsToBeBilled = generatedScript ? 10 : 3;
}
+ if (
+ options.checkPromptInjection &&
+ promptInjectionGuardGaveVerdict(costTrackingJSON)
+ ) {
+ creditsToBeBilled += 4;
+ }
+
if (
internalOptions.v1Agent?.model === "fire-1" ||
internalOptions.v1JSONAgent?.model?.toLowerCase() === "fire-1"
diff --git a/apps/api/src/scraper/scrapeURL/error.ts b/apps/api/src/scraper/scrapeURL/error.ts
index cfe3894ec..6f8a7d79c 100644
--- a/apps/api/src/scraper/scrapeURL/error.ts
+++ b/apps/api/src/scraper/scrapeURL/error.ts
@@ -664,7 +664,7 @@ export class PromptInjectionDetectedError extends TransportableError {
super(
"SCRAPE_PROMPT_INJECTION_DETECTED",
message ??
- "The scraped page content appears to contain a prompt injection attempt, so JSON extraction was aborted for safety.",
+ "The scraped page content appears to contain a prompt injection attempt, so the scrape was aborted for safety.",
);
}
diff --git a/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.test.ts b/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.test.ts
index 4f36fbe65..81307ed6a 100644
--- a/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.test.ts
+++ b/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.test.ts
@@ -10,10 +10,19 @@ vi.mock("./promptInjectionGuard", () => ({
createPromptInjectionGuardLimiter: vi.fn(),
}));
+vi.mock("./smartScrape", () => ({
+ smartScrape: vi.fn(),
+}));
+
+vi.mock("../../../lib/html-to-markdown", () => ({
+ parseMarkdown: vi.fn(async () => "smartscraped page"),
+}));
+
import type { Mock } from "vitest";
import { extractData } from "./extractSmartScrape";
import { generateCompletions } from "../transformers/llmExtract";
import { checkForPromptInjection } from "./promptInjectionGuard";
+import { smartScrape } from "./smartScrape";
import { JsonExtractionContentTooLargeError } from "../error";
import { CostTracking } from "../../../lib/cost-tracking";
@@ -48,19 +57,18 @@ describe("extractData", () => {
});
describe("prompt injection guard", () => {
- function run(ids: { scrapeId?: string; extractId?: string }) {
+ it("leaves the page itself to the scrape-level guard", async () => {
+ (checkForPromptInjection as Mock).mockClear();
(generateCompletions as Mock).mockResolvedValueOnce({
extract: { ok: true },
warning: undefined,
totalUsage: { promptTokens: 1, completionTokens: 1, totalTokens: 2 },
});
- return extractData({
+
+ await extractData({
extractOptions: {
logger: noopLogger,
- options: {
- schema: { type: "object", properties: {} },
- checkPromptInjection: true,
- },
+ options: { schema: { type: "object", properties: {} } },
markdown: "page",
costTrackingOptions: {
costTracking: new CostTracking(),
@@ -70,35 +78,63 @@ describe("extractData", () => {
} as any,
urls: ["https://example.com"],
useAgent: false,
- ...ids,
- metadata: { teamId: "test-team", functionId: "performLLMExtract" },
+ checkPromptInjection: true,
+ scrapeId: "test-scrape",
+ metadata: { teamId: "test-team" },
});
- }
- beforeEach(() => {
- (checkForPromptInjection as Mock).mockClear();
+ expect(checkForPromptInjection).not.toHaveBeenCalled();
});
- it("passes the scrape id to the guard for its span metadata", async () => {
- await run({ scrapeId: "test-scrape" });
+ it("scans each page SmartScrape fetches", async () => {
+ (checkForPromptInjection as Mock).mockClear();
+ (smartScrape as Mock).mockResolvedValueOnce({
+ scrapedPages: [{ html: "<p>page</p>" }],
+ });
+ (generateCompletions as Mock)
+ .mockResolvedValueOnce({
+ extract: {
+ extractedData: null,
+ shouldUseSmartscrape: true,
+ smartscrape_prompt: "find the title",
+ },
+ warning: undefined,
+ totalUsage: { promptTokens: 1, completionTokens: 1, totalTokens: 2 },
+ })
+ .mockResolvedValueOnce({ extract: { title: "t" } });
- expect(checkForPromptInjection).toHaveBeenCalledTimes(1);
- expect(
- (checkForPromptInjection as Mock).mock.calls[0][0].metadata,
- ).toEqual({
- teamId: "test-team",
- functionId: "performLLMExtract",
+ await extractData({
+ extractOptions: {
+ logger: noopLogger,
+ options: {
+ schema: {
+ type: "object",
+ properties: { title: { type: "string" } },
+ },
+ },
+ markdown: "page",
+ costTrackingOptions: {
+ costTracking: new CostTracking(),
+ metadata: {},
+ },
+ metadata: { teamId: "test-team" },
+ } as any,
+ urls: ["https://example.com"],
+ useAgent: true,
+ checkPromptInjection: true,
scrapeId: "test-scrape",
- extractId: undefined,
+ metadata: { teamId: "test-team", functionId: "performLLMExtract" },
});
- });
- it("passes the extract id to the guard for its span metadata", async () => {
- await run({ extractId: "test-extract" });
-
- expect(
- (checkForPromptInjection as Mock).mock.calls[0][0].metadata,
- ).toMatchObject({ teamId: "test-team", extractId: "test-extract" });
+ expect(checkForPromptInjection).toHaveBeenCalledTimes(1);
+ expect((checkForPromptInjection as Mock).mock.calls[0][0]).toMatchObject({
+ markdown: "smartscraped page",
+ metadata: {
+ teamId: "test-team",
+ functionId: "performLLMExtract",
+ scrapeId: "test-scrape",
+ },
+ });
});
});
diff --git a/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.ts b/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.ts
index 6ec4ef35f..452dcc64a 100644
--- a/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.ts
+++ b/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.ts
@@ -22,7 +22,7 @@ import { JsonExtractionContentTooLargeError } from "../error";
import { toRootSchema, typeIncludes } from "../../../lib/openai-strict-schema";
// ~2MB of markdown, well past typical page sizes -- caps worst-case JSON extraction cost/latency.
-const MAX_JSON_EXTRACTION_MARKDOWN_CHARS = 2_000_000;
+export const MAX_JSON_EXTRACTION_MARKDOWN_CHARS = 2_000_000;
const commonSmartScrapeProperties = {
shouldUseSmartscrape: {
@@ -265,6 +265,7 @@ export async function extractData({
extractOptions,
urls,
useAgent,
+ checkPromptInjection,
extractId,
sessionId,
scrapeId,
@@ -273,6 +274,8 @@ export async function extractData({
extractOptions: GenerateCompletionsOptions;
urls: string[];
useAgent: boolean;
+ /** Scans the pages SmartScrape fetches; the page itself is scanned upstream. */
+ checkPromptInjection?: boolean;
extractId?: string;
sessionId?: string;
scrapeId?: string;
@@ -286,7 +289,7 @@ export async function extractData({
const logger = extractOptions.logger;
const isSingleUrl = urls.length === 1;
let costLimitExceededTokenUsage: number | null = null;
- // Set when the prompt injection guard failed open on part of the content.
+ // Set when the prompt injection guard failed open on a SmartScrape page.
let promptInjectionScanIncomplete = false;
if (
@@ -396,18 +399,8 @@ export async function extractData({
warning: string | undefined,
totalUsage: TokenUsage | undefined;
- // Runs concurrently with extraction; guard verdict is checked before `extract` is used.
- const [guardSettled, generateSettled] = await Promise.allSettled([
- extractOptions.options.checkPromptInjection
- ? checkForPromptInjection({
- markdown: extractOptions.markdown,
- logger,
- costTracking: extractOptions.costTrackingOptions.costTracking,
- metadata: { ...metadata, scrapeId, extractId },
- zeroDataRetention: !!extractOptions.zeroDataRetention,
- })
- : Promise.resolve(true),
- generateCompletions({
+ try {
+ const completion = await generateCompletions({
...extractOptionsNewSchema,
costTrackingOptions: {
costTracking: extractOptions.costTrackingOptions.costTracking,
@@ -417,22 +410,11 @@ export async function extractData({
description: "Check if using smartScrape is needed for this case",
},
},
- }),
- ]);
-
- if (guardSettled.status === "rejected") {
- throw guardSettled.reason;
- }
- if (!guardSettled.value) {
- promptInjectionScanIncomplete = true;
- }
-
- if (generateSettled.status === "fulfilled") {
- extract = generateSettled.value.extract;
- warning = generateSettled.value.warning;
- totalUsage = generateSettled.value.totalUsage;
- } else {
- const error = generateSettled.reason;
+ });
+ extract = completion.extract;
+ warning = completion.warning;
+ totalUsage = completion.totalUsage;
+ } catch (error) {
if (error instanceof CostLimitExceededError) {
throw error;
}
@@ -537,7 +519,7 @@ export async function extractData({
const guardLimiter = createPromptInjectionGuardLimiter();
extractedData = await Promise.all(
markdowns.map(async markdown => {
- if (extractOptions.options.checkPromptInjection) {
+ if (checkPromptInjection) {
const scannedFully = await checkForPromptInjection({
markdown,
logger,
diff --git a/apps/api/src/scraper/scrapeURL/lib/promptInjectionGuard.ts b/apps/api/src/scraper/scrapeURL/lib/promptInjectionGuard.ts
index e580df14d..bdead397e 100644
--- a/apps/api/src/scraper/scrapeURL/lib/promptInjectionGuard.ts
+++ b/apps/api/src/scraper/scrapeURL/lib/promptInjectionGuard.ts
@@ -138,11 +138,11 @@ async function classifyChunk(
if (result.object.isInjection) {
logger.warn(
- "Prompt injection detected in scraped content; blocking JSON extraction",
+ "Prompt injection detected in scraped content; blocking the scrape",
{ reason: result.object.reason },
);
throw new PromptInjectionDetectedError(
- `The scraped page content appears to contain a prompt injection attempt, so JSON extraction was aborted for safety. Guard verdict: ${result.object.reason.slice(0, 300)}`,
+ `The scraped page content appears to contain a prompt injection attempt, so the scrape was aborted for safety. Guard verdict: ${result.object.reason.slice(0, 300)}`,
);
}
return true;
@@ -183,7 +183,7 @@ export function createPromptInjectionGuardLimiter(): Semaphore {
// Throws PromptInjectionDetectedError on a detection. Otherwise resolves to
// false when the guard failed open on at least one chunk, i.e. part of the
-// content reached extraction unscanned.
+// content went unscanned.
export async function checkForPromptInjection({
markdown,
logger,
diff --git a/apps/api/src/scraper/scrapeURL/transformers/index.ts b/apps/api/src/scraper/scrapeURL/transformers/index.ts
index c899263c4..8f4c81237 100644
--- a/apps/api/src/scraper/scrapeURL/transformers/index.ts
+++ b/apps/api/src/scraper/scrapeURL/transformers/index.ts
@@ -23,6 +23,7 @@ import { fetchProduct } from "./product";
import { fetchMenu } from "./menu";
import { fetchVideo } from "./video";
import { performRedactPII } from "./redactPII";
+import { performPromptInjectionGuard } from "./promptInjectionGuard";
import { useIndex, useSearchIndex } from "../../../services/index";
import { sendDocumentToIndex } from "../engines/index/index";
import { sendDocumentToSearchIndex } from "./sendToSearchIndex";
@@ -99,6 +100,7 @@ async function deriveMarkdownFromHTML(
// - summary format requires markdown (for summarization)
// - question/highlights/query formats require markdown (for page-level answers)
// - redactPII needs markdown as its source text (spans are markdown char offsets)
+ // - checkPromptInjection scans markdown
const hasMarkdown = hasFormatOfType(meta.options.formats, "markdown");
const hasChangeTracking = hasFormatOfType(
meta.options.formats,
@@ -123,6 +125,7 @@ async function deriveMarkdownFromHTML(
!hasHighlights &&
!hasQuery &&
!hasRedactPII &&
+ !meta.options.checkPromptInjection &&
!meta.options.onlyCleanContent
) {
return document;
@@ -621,6 +624,7 @@ const transformerStack: Transformer[] = [
deriveMarkdownFromHTML,
performCleanContent,
performRedactPII,
+ performPromptInjectionGuard,
deriveLinksFromHTML,
deriveImagesFromHTML,
deriveBrandingFromActions,
diff --git a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts
index f26ad67f5..4a45b5085 100644
--- a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts
+++ b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts
@@ -1062,6 +1062,7 @@ export async function performLLMExtract(
extractOptions: generationOptions,
urls: [meta.rewrittenUrl ?? meta.url],
useAgent,
+ checkPromptInjection: meta.options.checkPromptInjection,
scrapeId: meta.id,
metadata: {
teamId: meta.internalOptions.teamId,
diff --git a/apps/api/src/scraper/scrapeURL/transformers/promptInjectionGuard.test.ts b/apps/api/src/scraper/scrapeURL/transformers/promptInjectionGuard.test.ts
new file mode 100644
index 000000000..21c3ad472
--- /dev/null
+++ b/apps/api/src/scraper/scrapeURL/transformers/promptInjectionGuard.test.ts
@@ -0,0 +1,92 @@
+import type { Mock } from "vitest";
+import { performPromptInjectionGuard } from "./promptInjectionGuard";
+import { checkForPromptInjection } from "../lib/promptInjectionGuard";
+import { PromptInjectionDetectedError } from "../error";
+import { CostTracking } from "../../../lib/cost-tracking";
+import { MAX_JSON_EXTRACTION_MARKDOWN_CHARS } from "../lib/extractSmartScrape";
+
+vi.mock("../lib/promptInjectionGuard", () => ({
+ checkForPromptInjection: vi.fn(),
+}));
+
+const mockedGuard = checkForPromptInjection as Mock;
+
+const meta = (checkPromptInjection: boolean) =>
+ ({
+ id: "test-scrape",
+ url: "https://example.com",
+ options: { formats: [{ type: "markdown" }], checkPromptInjection },
+ internalOptions: {
+ teamId: "test-team",
+ zeroDataRetention: false,
+ llmTelemetry: { functionId: "test-function", metadata: {} },
+ },
+ costTracking: new CostTracking(),
+ logger: { debug: vi.fn(), info: vi.fn(), warn: vi.fn(), error: vi.fn() },
+ }) as any;
+
+const document = (markdown: string) =>
+ ({ markdown, metadata: {}, warning: "earlier warning" }) as any;
+
+describe("performPromptInjectionGuard", () => {
+ beforeEach(() => {
+ mockedGuard.mockReset();
+ });
+
+ it("does nothing when the check was not requested", async () => {
+ const doc = document("page");
+
+ expect(await performPromptInjectionGuard(meta(false), doc)).toBe(doc);
+ expect(mockedGuard).not.toHaveBeenCalled();
+ });
+
+ it("scans the page markdown with the scrape's ids", async () => {
+ mockedGuard.mockResolvedValue(true);
+ const doc = document("page");
+
+ expect(await performPromptInjectionGuard(meta(true), doc)).toBe(doc);
+ expect(mockedGuard).toHaveBeenCalledTimes(1);
+ expect(mockedGuard.mock.calls[0][0]).toMatchObject({
+ markdown: "page",
+ metadata: {
+ teamId: "test-team",
+ functionId: "test-function",
+ scrapeId: "test-scrape",
+ },
+ zeroDataRetention: false,
+ });
+ });
+
+ it("blocks the scrape when the guard detects an injection", async () => {
+ mockedGuard.mockRejectedValue(new PromptInjectionDetectedError());
+
+ await expect(
+ performPromptInjectionGuard(meta(true), document("page")),
+ ).rejects.toBeInstanceOf(PromptInjectionDetectedError);
+ });
+
+ it("warns when the guard could not scan every chunk", async () => {
+ mockedGuard.mockResolvedValue(false);
+
+ const result = await performPromptInjectionGuard(
+ meta(true),
+ document("page"),
+ );
+
+ expect(result.warning).toMatch(
+ /^The prompt injection check could not scan all of the page content.* earlier warning$/,
+ );
+ });
+
+ it("skips markdown over the json extraction cap with the same warning", async () => {
+ const result = await performPromptInjectionGuard(
+ meta(true),
+ document("x".repeat(MAX_JSON_EXTRACTION_MARKDOWN_CHARS + 1)),
+ );
+
+ expect(mockedGuard).not.toHaveBeenCalled();
+ expect(result.warning).toMatch(
+ /^The prompt injection check could not scan all of the page content/,
+ );
+ });
+});
diff --git a/apps/api/src/scraper/scrapeURL/transformers/promptInjectionGuard.ts b/apps/api/src/scraper/scrapeURL/transformers/promptInjectionGuard.ts
new file mode 100644
index 000000000..b62527e20
--- /dev/null
+++ b/apps/api/src/scraper/scrapeURL/transformers/promptInjectionGuard.ts
@@ -0,0 +1,41 @@
+import { Meta } from "..";
+import { Document } from "../../../controllers/v2/types";
+import { checkForPromptInjection } from "../lib/promptInjectionGuard";
+import { MAX_JSON_EXTRACTION_MARKDOWN_CHARS } from "../lib/extractSmartScrape";
+
+// Runs before every LLM-backed format, so blocked content never reaches one.
+export async function performPromptInjectionGuard(
+ meta: Meta,
+ document: Document,
+): Promise<Document> {
+ if (!meta.options.checkPromptInjection) {
+ return document;
+ }
+
+ const markdown = document.markdown ?? "";
+ const scannedFully =
+ markdown.length <= MAX_JSON_EXTRACTION_MARKDOWN_CHARS &&
+ (await checkForPromptInjection({
+ markdown,
+ logger: meta.logger,
+ costTracking: meta.costTracking,
+ metadata: {
+ teamId: meta.internalOptions.teamId,
+ functionId: meta.internalOptions.llmTelemetry?.functionId,
+ scrapeId: meta.id,
+ },
+ zeroDataRetention: !!meta.internalOptions.zeroDataRetention,
+ }));
+
+ if (scannedFully) {
+ return document;
+ }
+
+ // The guard fee does not bill in this case (see scrape-billing.ts).
+ return {
+ ...document,
+ warning:
+ "The prompt injection check could not scan all of the page content, so some of it was not checked. The prompt injection check was not billed." +
+ (document.warning ? " " + document.warning : ""),
+ };
+}
diff --git a/apps/api/src/services/monitoring/store.test.ts b/apps/api/src/services/monitoring/store.test.ts
index fc4abf47a..c6bcefaad 100644
--- a/apps/api/src/services/monitoring/store.test.ts
+++ b/apps/api/src/services/monitoring/store.test.ts
@@ -290,6 +290,23 @@ describe("monitoring store credit helpers", () => {
expect(estimateMonitorCreditsPerRun(targets, false)).toBe(9);
});
+ it("estimates the top-level prompt injection guard on a markdown monitor", () => {
+ const targets: MonitorTarget[] = [
+ {
+ id: "target-1",
+ type: "scrape",
+ urls: ["https://example.com/a"],
+ scrapeOptions: {
+ formats: ["markdown"],
+ checkPromptInjection: true,
+ },
+ },
+ ];
+
+ // 1 base + 4 prompt injection guard.
+ expect(estimateMonitorCreditsPerRun(targets, false)).toBe(5);
+ });
+
it("uses target options when page rows do not have recorded scrape credits", () => {
const targets: MonitorTarget[] = [
{
diff --git a/apps/api/src/services/monitoring/store.ts b/apps/api/src/services/monitoring/store.ts
index 4b4851f53..472b5411d 100644
--- a/apps/api/src/services/monitoring/store.ts
+++ b/apps/api/src/services/monitoring/store.ts
@@ -15,7 +15,11 @@ import { db, dbRr } from "../../db/connection";
import * as schema from "../../db/schema";
import { monitoringClaimDueMonitors } from "../../db/rpc";
import { config } from "../../config";
-import { getPDFMaxPages, shouldParsePDF } from "../../controllers/v2/types";
+import {
+ getPDFMaxPages,
+ liftCheckPromptInjection,
+ shouldParsePDF,
+} from "../../controllers/v2/types";
import { isXTwitterUrl } from "../../scraper/scrapeURL/engines/x-twitter/url";
import {
getNextMonitorRunAt,
@@ -88,18 +92,6 @@ function hasAnyFormatOfType(formats: unknown, types: string[]): boolean {
return types.some(type => hasFormatOfType(formats, type));
}
-function requestsPromptInjectionCheck(formats: unknown): boolean {
- if (!Array.isArray(formats)) return false;
- return formats.some(
- format =>
- !!format &&
- typeof format === "object" &&
- formatType(format) === "json" &&
- "checkPromptInjection" in format &&
- format.checkPromptInjection === true,
- );
-}
-
function requestsJsonChangeTracking(formats: unknown): boolean {
if (!Array.isArray(formats)) return false;
return formats.some(format => {
@@ -140,7 +132,7 @@ function estimateBaseCreditsPerPage(
// The prompt injection guard bills +4 in calculateCreditsToBeBilled. The
// estimate cannot know whether the guard ran, so it assumes it does.
- if (requestsPromptInjectionCheck(formats)) {
+ if (liftCheckPromptInjection(options ?? {}).checkPromptInjection === true) {
credits += SCRAPE_OPTION_CREDIT_BONUS;
}
diff --git a/apps/js-sdk/firecrawl/package.json b/apps/js-sdk/firecrawl/package.json
index a9cd9976d..36c7326a2 100644
--- a/apps/js-sdk/firecrawl/package.json
+++ b/apps/js-sdk/firecrawl/package.json
@@ -1,6 +1,6 @@
{
"name": "@mendable/firecrawl-js",
- "version": "4.45.0",
+ "version": "4.46.0",
"description": "JavaScript SDK for the Firecrawl API: web scraping, crawling, web search, and scientific literature search over a research paper index of PubMed, bioRxiv, medRxiv and arXiv abstracts",
"main": "dist/index.js",
"types": "dist/index.d.ts",
diff --git a/apps/js-sdk/firecrawl/src/v2/types.ts b/apps/js-sdk/firecrawl/src/v2/types.ts
index ee0cd0a36..5df514c9c 100644
--- a/apps/js-sdk/firecrawl/src/v2/types.ts
+++ b/apps/js-sdk/firecrawl/src/v2/types.ts
@@ -31,6 +31,7 @@ export interface JsonFormat extends Format {
type: "json";
prompt?: string;
schema?: Record<string, unknown> | ZodTypeAny;
+ /** @deprecated Use the top-level `checkPromptInjection` scrape option. */
checkPromptInjection?: boolean;
}
@@ -276,6 +277,13 @@ export interface ScrapeOptions {
minAge?: number;
storeInCache?: boolean;
lockdown?: boolean;
+ /**
+ * Scans the page content for prompt injection with any format except
+ * rawBase64, before LLM-backed formats run. A detection fails the scrape
+ * with SCRAPE_PROMPT_INJECTION_DETECTED.
+ * Adds 4 credits when the check scans the whole page.
+ */
+ checkPromptInjection?: boolean;
redactPII?: boolean | RedactPIIOptions;
threatProtection?: ThreatProtectionOptions;
auditMetadata?: AuditMetadata;
diff --git a/apps/python-sdk/firecrawl/__init__.py b/apps/python-sdk/firecrawl/__init__.py
index 3e443fccd..22651aa68 100644
--- a/apps/python-sdk/firecrawl/__init__.py
+++ b/apps/python-sdk/firecrawl/__init__.py
@@ -41,7 +41,7 @@ from .v1 import (
V1ChangeTrackingOptions,
)
-__version__ = "4.49.3"
+__version__ = "4.50.0"
# Define the logger for the Firecrawl project
logger: logging.Logger = logging.getLogger("firecrawl")
diff --git a/apps/python-sdk/firecrawl/__tests__/unit/v2/utils/test_validation.py b/apps/python-sdk/firecrawl/__tests__/unit/v2/utils/test_validation.py
index 24d1e0bc1..97370c81f 100644
--- a/apps/python-sdk/firecrawl/__tests__/unit/v2/utils/test_validation.py
+++ b/apps/python-sdk/firecrawl/__tests__/unit/v2/utils/test_validation.py
@@ -116,6 +116,25 @@ class TestPrepareScrapeOptions:
assert result["timeout"] == 30000
assert result["waitFor"] == 2000
+ def test_prepare_top_level_check_prompt_injection(self):
+ """check_prompt_injection is sent as the top-level checkPromptInjection."""
+ result = prepare_scrape_options(
+ ScrapeOptions(formats=["markdown"], check_prompt_injection=True)
+ )
+
+ assert result["checkPromptInjection"] is True
+ assert "check_prompt_injection" not in result
+
+ def test_prepare_json_format_check_prompt_injection_still_sent(self):
+ """The deprecated json-format field keeps working."""
+ result = prepare_scrape_options(
+ ScrapeOptions(
+ formats=[JsonFormat(prompt="Extract", check_prompt_injection=True)]
+ )
+ )
+
+ assert result["formats"][0]["checkPromptInjection"] is True
+
def test_prepare_snake_case_conversion(self):
"""Test snake_case to camelCase conversion."""
options = ScrapeOptions(
diff --git a/apps/python-sdk/firecrawl/v2/client.py b/apps/python-sdk/firecrawl/v2/client.py
index d21a50116..8af7917dc 100644
--- a/apps/python-sdk/firecrawl/v2/client.py
+++ b/apps/python-sdk/firecrawl/v2/client.py
@@ -97,7 +97,8 @@ _SCRAPE_OPTION_KEYS = frozenset({
"only_main_content", "timeout", "wait_for", "mobile",
"parsers", "actions", "location", "skip_tls_verification",
"remove_base64_images", "fast_mode", "use_mock", "block_ads",
- "proxy", "max_age", "store_in_cache", "lockdown", "threat_protection",
+ "proxy", "max_age", "store_in_cache", "lockdown", "check_prompt_injection",
+ "threat_protection",
"profile", "audit_metadata",
})
@@ -186,6 +187,7 @@ class FirecrawlClient:
max_age: Optional[int] = None,
store_in_cache: Optional[bool] = None,
lockdown: Optional[bool] = None,
+ check_prompt_injection: Optional[bool] = None,
threat_protection: Optional[ThreatProtectionOptions] = None,
profile: Optional[Dict[str, Any]] = None,
audit_metadata: Optional[AuditMetadata] = None,
@@ -217,6 +219,7 @@ class FirecrawlClient:
max_age: Maximum age of the cache
store_in_cache: Whether to store the result in the cache
lockdown: Serve only previously cached results; never make outbound requests. Returns 404 SCRAPE_LOCKDOWN_CACHE_MISS on cache miss.
+ check_prompt_injection: Scan the page content for prompt injection with any format except rawBase64, before LLM-backed formats run. A detection fails the scrape. Adds 4 credits when the check scans the whole page.
threat_protection: Enterprise per-request override of the team's threat protection policy
profile: Browser profile for persistent state (e.g. {"name": "my-profile", "saveChanges": True})
tool_detail: "compact" returns provider, capability and description; "summary" (default) adds metadata; "full" includes contracts when domain discovery is enabled.
@@ -246,6 +249,7 @@ class FirecrawlClient:
max_age=max_age,
store_in_cache=store_in_cache,
lockdown=lockdown,
+ check_prompt_injection=check_prompt_injection,
threat_protection=threat_protection,
profile=profile,
audit_metadata=audit_metadata,
@@ -253,7 +257,7 @@ class FirecrawlClient:
domain_tools=domain_tools,
tool_detail=tool_detail,
).items() if v is not None}
- ) if any(v is not None for v in [formats, headers, include_tags, exclude_tags, only_main_content, timeout, wait_for, mobile, parsers, actions, location, skip_tls_verification, remove_base64_images, fast_mode, use_mock, block_ads, proxy, max_age, store_in_cache, lockdown, threat_protection, profile, audit_metadata, integration, domain_tools, tool_detail]) else None
+ ) if any(v is not None for v in [formats, headers, include_tags, exclude_tags, only_main_content, timeout, wait_for, mobile, parsers, actions, location, skip_tls_verification, remove_base64_images, fast_mode, use_mock, block_ads, proxy, max_age, store_in_cache, lockdown, check_prompt_injection, threat_protection, profile, audit_metadata, integration, domain_tools, tool_detail]) else None
if alexandria is not None:
if url is not None or auto_resume is not None or (options and set(options.model_dump(exclude_none=True, exclude_unset=True)) - {"timeout", "integration"}):
raise ValueError("alexandria cannot be combined with URL scrape options")
@@ -636,6 +640,7 @@ class FirecrawlClient:
max_age: Optional[int] = None,
store_in_cache: Optional[bool] = None,
lockdown: Optional[bool] = None,
+ check_prompt_injection: Optional[bool] = None,
threat_protection: Optional[ThreatProtectionOptions] = None,
profile: Optional[Dict[str, Any]] = None,
audit_metadata: Optional[AuditMetadata] = None,
@@ -687,6 +692,7 @@ class FirecrawlClient:
max_age: Cache max age (convenience kwarg)
store_in_cache: Cache results (convenience kwarg)
lockdown: Serve only cached results (convenience kwarg)
+ check_prompt_injection: Scan the page content for prompt injection with any format except rawBase64, before LLM-backed formats run. A detection fails the scrape. Adds 4 credits when the check scans the whole page.
threat_protection: Enterprise threat protection override (convenience kwarg)
profile: Browser profile (convenience kwarg)
audit_metadata: Metadata to include in SIEM logging events
@@ -714,6 +720,7 @@ class FirecrawlClient:
remove_base64_images=remove_base64_images, fast_mode=fast_mode,
use_mock=use_mock, block_ads=block_ads, proxy=proxy,
max_age=max_age, store_in_cache=store_in_cache, lockdown=lockdown,
+ check_prompt_injection=check_prompt_injection,
threat_protection=threat_protection, profile=profile,
audit_metadata=audit_metadata,
).items() if v is not None}
@@ -798,6 +805,7 @@ class FirecrawlClient:
max_age: Optional[int] = None,
store_in_cache: Optional[bool] = None,
lockdown: Optional[bool] = None,
+ check_prompt_injection: Optional[bool] = None,
threat_protection: Optional[ThreatProtectionOptions] = None,
profile: Optional[Dict[str, Any]] = None,
audit_metadata: Optional[AuditMetadata] = None,
@@ -847,6 +855,7 @@ class FirecrawlClient:
max_age: Cache max age (convenience kwarg)
store_in_cache: Cache results (convenience kwarg)
lockdown: Serve only cached results (convenience kwarg)
+ check_prompt_injection: Scan the page content for prompt injection with any format except rawBase64, before LLM-backed formats run. A detection fails the scrape. Adds 4 credits when the check scans the whole page.
threat_protection: Enterprise threat protection override (convenience kwarg)
profile: Browser profile (convenience kwarg)
audit_metadata: Metadata to include in SIEM logging events
@@ -871,6 +880,7 @@ class FirecrawlClient:
remove_base64_images=remove_base64_images, fast_mode=fast_mode,
use_mock=use_mock, block_ads=block_ads, proxy=proxy,
max_age=max_age, store_in_cache=store_in_cache, lockdown=lockdown,
+ check_prompt_injection=check_prompt_injection,
threat_protection=threat_protection, profile=profile,
audit_metadata=audit_metadata,
).items() if v is not None}
@@ -1352,6 +1362,7 @@ class FirecrawlClient:
max_age: Optional[int] = None,
store_in_cache: Optional[bool] = None,
lockdown: Optional[bool] = None,
+ check_prompt_injection: Optional[bool] = None,
threat_protection: Optional[ThreatProtectionOptions] = None,
audit_metadata: Optional[AuditMetadata] = None,
webhook: Optional[Union[str, WebhookConfig]] = None,
@@ -1386,6 +1397,7 @@ class FirecrawlClient:
max_age: Cache max age
store_in_cache: Whether to store results in cache
lockdown: Serve only previously cached results; never make outbound requests.
+ check_prompt_injection: Scan the page content for prompt injection with any format except rawBase64, before LLM-backed formats run. A detection fails the scrape. Adds 4 credits when the check scans the whole page.
threat_protection: Enterprise per-request override of the team's threat protection policy
audit_metadata: Metadata to include in SIEM logging events
webhook: Webhook configuration
@@ -1421,9 +1433,10 @@ class FirecrawlClient:
max_age=max_age,
store_in_cache=store_in_cache,
lockdown=lockdown,
+ check_prompt_injection=check_prompt_injection,
threat_protection=threat_protection,
).items() if v is not None}
- ) if any(v is not None for v in [formats, headers, include_tags, exclude_tags, only_main_content, timeout, wait_for, mobile, parsers, actions, location, skip_tls_verification, remove_base64_images, fast_mode, use_mock, block_ads, proxy, max_age, store_in_cache, lockdown, threat_protection]) else None
+ ) if any(v is not None for v in [formats, headers, include_tags, exclude_tags, only_main_content, timeout, wait_for, mobile, parsers, actions, location, skip_tls_verification, remove_base64_images, fast_mode, use_mock, block_ads, proxy, max_age, store_in_cache, lockdown, check_prompt_injection, threat_protection]) else None
return batch_module.start_batch_scrape(
self.http_client,
@@ -1868,6 +1881,7 @@ class FirecrawlClient:
max_age: Optional[int] = None,
store_in_cache: Optional[bool] = None,
lockdown: Optional[bool] = None,
+ check_prompt_injection: Optional[bool] = None,
threat_protection: Optional[ThreatProtectionOptions] = None,
audit_metadata: Optional[AuditMetadata] = None,
webhook: Optional[Union[str, WebhookConfig]] = None,
@@ -1905,9 +1919,10 @@ class FirecrawlClient:
max_age=max_age,
store_in_cache=store_in_cache,
lockdown=lockdown,
+ check_prompt_injection=check_prompt_injection,
threat_protection=threat_protection,
).items() if v is not None}
- ) if any(v is not None for v in [formats, headers, include_tags, exclude_tags, only_main_content, timeout, wait_for, mobile, parsers, actions, location, skip_tls_verification, remove_base64_images, fast_mode, use_mock, block_ads, proxy, max_age, store_in_cache, lockdown, threat_protection]) else None
+ ) if any(v is not None for v in [formats, headers, include_tags, exclude_tags, only_main_content, timeout, wait_for, mobile, parsers, actions, location, skip_tls_verification, remove_base64_images, fast_mode, use_mock, block_ads, proxy, max_age, store_in_cache, lockdown, check_prompt_injection, threat_protection]) else None
return batch_module.batch_scrape(
self.http_client,
diff --git a/apps/python-sdk/firecrawl/v2/types.py b/apps/python-sdk/firecrawl/v2/types.py
index 6e6bc2a72..0ef9ca63e 100644
--- a/apps/python-sdk/firecrawl/v2/types.py
+++ b/apps/python-sdk/firecrawl/v2/types.py
@@ -698,6 +698,7 @@ class JsonFormat(Format):
type: Literal["json"] = "json"
prompt: Optional[str] = None
schema: Optional[Any] = None
+ # Deprecated: use the top-level ScrapeOptions.check_prompt_injection.
check_prompt_injection: Optional[bool] = None
@@ -919,6 +920,12 @@ class ScrapeOptions(BaseModel):
min_age: Optional[int] = None
store_in_cache: Optional[bool] = None
lockdown: Optional[bool] = None
+ # Scans the page content for prompt injection with any format except
+ # rawBase64, before LLM-backed formats run. A detection fails the scrape.
+ # Adds 4 credits when the check scans the whole page.
+ check_prompt_injection: Optional[bool] = Field(
+ default=None, alias="checkPromptInjection"
+ )
redact_pii: Optional[Union[bool, RedactPIIOptions]] = Field(
default=None, alias="redactPII"
)
diff --git a/apps/python-sdk/firecrawl/v2/utils/validation.py b/apps/python-sdk/firecrawl/v2/utils/validation.py
index 093aa2ce2..4bbed9cd9 100644
--- a/apps/python-sdk/firecrawl/v2/utils/validation.py
+++ b/apps/python-sdk/firecrawl/v2/utils/validation.py
@@ -570,6 +570,7 @@ def prepare_scrape_options(options: Optional[ScrapeOptions]) -> Optional[Dict[st
"max_age": "maxAge",
"min_age": "minAge",
"redact_pii": "redactPII",
+ "check_prompt_injection": "checkPromptInjection",
"threat_protection": "threatProtection",
"audit_metadata": "auditMetadata",
}