Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. π₯
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Simplify redactPII API (#3691)
Abimael Martelld2672e72c1e0
diff --git a/apps/api/src/__tests__/snips/v2/scrape-pii.test.ts b/apps/api/src/__tests__/snips/v2/scrape-pii.test.tsindex 8d7dc43dd..a7653c813 100644--- a/apps/api/src/__tests__/snips/v2/scrape-pii.test.ts+++ b/apps/api/src/__tests__/snips/v2/scrape-pii.test.ts@@ -1,12 +1,5 @@import { ALLOW_TEST_SUITE_WEBSITE, describeIf, TEST_PRODUCTION } from "../lib";-import {- Identity,- idmux,- scrape,- scrapeRaw,- scrapeTimeout,- scrapeWithFailure,-} from "./lib";+import { Identity, idmux, scrape, scrapeRaw, scrapeTimeout } from "./lib";let identity: Identity;@@ -20,9 +13,9 @@ beforeAll(async () => {describeIf(ALLOW_TEST_SUITE_WEBSITE)("V2 Scrape redactPII (schema)", () => {it.concurrent(- "rejects redactPII: true when `pii` is not in formats",+ "accepts redactPII: true when `pii` is not in formats",async () => {- const res = await scrapeWithFailure(+ const res = await scrapeRaw({url: "https://firecrawl.dev",formats: ["markdown"],@@ -31,8 +24,9 @@ describeIf(ALLOW_TEST_SUITE_WEBSITE)("V2 Scrape redactPII (schema)", () => {identity,);- expect(res.success).toBe(false);- expect(res.error).toMatch(/redactPII requires `pii`/);+ expect(res.statusCode).toBe(200);+ expect(res.body.success).toBe(true);+ expect(res.body.data.pii).toBeUndefined();},scrapeTimeout,);diff --git a/apps/api/src/controllers/v2/__tests__/redactPII-schema.test.ts b/apps/api/src/controllers/v2/__tests__/redactPII-schema.test.tsindex 12e27c1d1..32d4c7275 100644--- a/apps/api/src/controllers/v2/__tests__/redactPII-schema.test.ts+++ b/apps/api/src/controllers/v2/__tests__/redactPII-schema.test.ts@@ -5,10 +5,10 @@ describe("v2 scrapeRequestSchema β redactPII", () => {// ---- boolean form -------------------------------------------------------- it("accepts redactPII: true with `pii` in formats", () => {+ it("accepts redactPII: true without requiring `pii` in formats", () => {const result = scrapeRequestSchema.safeParse({url: baseUrl,- formats: ["markdown", "pii"],+ formats: ["markdown"],redactPII: true,});expect(result.success).toBe(true);@@ -44,16 +44,18 @@ describe("v2 scrapeRequestSchema β redactPII", () => {}});- it("rejects redactPII: true without `pii` in formats", () => {+ it("accepts redactPII: true with default formats", () => {const result = scrapeRequestSchema.safeParse({url: baseUrl,- formats: ["markdown"],redactPII: true,});- expect(result.success).toBe(false);- if (!result.success) {- const messages = result.error.issues.map(i => i.message).join("\n");- expect(messages).toMatch(/redactPII requires `pii`/);+ expect(result.success).toBe(true);+ if (result.success) {+ expect(result.data.formats).toEqual([{ type: "markdown" }]);+ expect(result.data.redactPII).toMatchObject({+ mode: "accurate",+ replaceStyle: "tag",+ });}});@@ -141,16 +143,18 @@ describe("v2 scrapeRequestSchema β redactPII", () => {expect(result.success).toBe(false);});- it("requires `pii` in formats even with the object form", () => {+ it("accepts the object form without requiring `pii` in formats", () => {const result = scrapeRequestSchema.safeParse({url: baseUrl,formats: ["markdown"],redactPII: { mode: "aggressive" },});- expect(result.success).toBe(false);- if (!result.success) {- const messages = result.error.issues.map(i => i.message).join("\n");- expect(messages).toMatch(/redactPII requires `pii`/);+ expect(result.success).toBe(true);+ if (result.success) {+ expect(result.data.redactPII).toMatchObject({+ mode: "aggressive",+ replaceStyle: "tag",+ });}});@@ -179,7 +183,7 @@ describe("v2 scrapeRequestSchema β redactPII", () => {it("preserves explicit onlyMainContent: false when redactPII is enabled", () => {const result = scrapeRequestSchema.safeParse({url: baseUrl,- formats: ["markdown", "pii"],+ formats: ["markdown"],redactPII: true,onlyMainContent: false,});@@ -192,7 +196,6 @@ describe("v2 scrapeRequestSchema β redactPII", () => {it("keeps the default onlyMainContent: true when redactPII is enabled", () => {const result = scrapeRequestSchema.safeParse({url: baseUrl,- formats: ["markdown", "pii"],redactPII: true,});expect(result.success).toBe(true);diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.tsindex 95ba5fb52..7537775cd 100644--- a/apps/api/src/controllers/v2/types.ts+++ b/apps/api/src/controllers/v2/types.ts@@ -692,17 +692,6 @@ const waitForRefineOpts = {path: ["waitFor"],};-const redactPIIRefine = (- obj?: Pick<ScrapeOptionsBase, "redactPII" | "formats">,-): boolean => {- if (!obj || !obj.redactPII) return true;- return !!obj.formats?.some(f => f.type === "pii");-};-const redactPIIRefineOpts = {- message: "redactPII requires `pii` to be included in formats",- path: ["redactPII"],-};-// Base transform function that handles both nullable and non-nullable cases// Uses generic type to preserve all fields from extended schemasconst extractTransformImpl = <T extends ScrapeOptionsBase | undefined>(@@ -776,7 +765,6 @@ export const scrapeOptions = strictWithMessage(baseScrapeOptions)},).refine(waitForRefine, waitForRefineOpts)- .refine(redactPIIRefine, redactPIIRefineOpts).transform(extractTransformRequired);export type BaseScrapeOptions = z.infer<typeof baseScrapeOptions>;@@ -860,10 +848,6 @@ const extractOptions = zx => (x.scrapeOptions ? waitForRefine(x.scrapeOptions) : true),waitForRefineOpts,)- .refine(- x => (x.scrapeOptions ? redactPIIRefine(x.scrapeOptions) : true),- redactPIIRefineOpts,- ).transform(x => ({...x,scrapeOptions: extractTransform(x.scrapeOptions),@@ -934,7 +918,6 @@ const scrapeRequestSchemaBase = baseScrapeOptions.extend({export const scrapeRequestSchema = strictWithMessage(scrapeRequestSchemaBase).refine(waitForRefine, waitForRefineOpts)- .refine(redactPIIRefine, redactPIIRefineOpts).transform(extractTransformRequired);export type ScrapeRequest = z.infer<typeof scrapeRequestSchema>;@@ -992,7 +975,6 @@ const parseRequestSchemaBase = baseScrapeOptions.extend({export const parseRequestSchema = strictWithMessage(parseRequestSchemaBase).refine(waitForRefine, waitForRefineOpts)- .refine(redactPIIRefine, redactPIIRefineOpts).transform(x => {const { file, ...scrapeLike } = x;return {@@ -1026,7 +1008,6 @@ export const batchScrapeRequestSchema = strictWithMessage(batchScrapeRequestSchemaBase,).refine(waitForRefine, waitForRefineOpts)- .refine(redactPIIRefine, redactPIIRefineOpts).transform(extractTransformRequired);const batchScrapeRequestSchemaNoURLValidationBase = baseScrapeOptions.extend({@@ -1051,7 +1032,6 @@ export const batchScrapeRequestSchemaNoURLValidation = strictWithMessage(batchScrapeRequestSchemaNoURLValidationBase,).refine(waitForRefine, waitForRefineOpts)- .refine(redactPIIRefine, redactPIIRefineOpts).transform(extractTransformRequired);export type BatchScrapeRequest = z.infer<typeof batchScrapeRequestSchema>;@@ -1116,7 +1096,6 @@ const crawlRequestSchemaBase = crawlerOptions.extend({export const crawlRequestSchema = strictWithMessage(crawlRequestSchemaBase).refine(x => waitForRefine(x.scrapeOptions), waitForRefineOpts)- .refine(x => redactPIIRefine(x.scrapeOptions), redactPIIRefineOpts).transform(x => {const scrapeOptionsValue = x.scrapeOptions ?? baseScrapeOptions.parse({});return {diff --git a/apps/api/src/scraper/scrapeURL/transformers/index.ts b/apps/api/src/scraper/scrapeURL/transformers/index.tsindex 105bd7d90..3c2389781 100644--- a/apps/api/src/scraper/scrapeURL/transformers/index.ts+++ b/apps/api/src/scraper/scrapeURL/transformers/index.ts@@ -86,9 +86,7 @@ async function deriveMarkdownFromHTML(// - json format requires markdown (for LLM extraction)// - summary format requires markdown (for summarization)// - question/highlights/query formats require markdown (for page-level answers)- // - pii format redacts markdown (spans are markdown char offsets) β but- // only when redactPII is actually enabled; otherwise performRedactPII- // bails immediately and the derived markdown is wasted work.+ // - redactPII needs markdown as its source text (spans are markdown char offsets)const hasMarkdown = hasFormatOfType(meta.options.formats, "markdown");const hasChangeTracking = hasFormatOfType(meta.options.formats,@@ -99,8 +97,7 @@ async function deriveMarkdownFromHTML(const hasQuestion = hasFormatOfType(meta.options.formats, "question");const hasHighlights = hasFormatOfType(meta.options.formats, "highlights");const hasQuery = hasFormatOfType(meta.options.formats, "query");- const hasPii =- hasFormatOfType(meta.options.formats, "pii") && !!meta.options.redactPII;+ const hasRedactPII = !!meta.options.redactPII;if (!hasMarkdown &&!hasChangeTracking &&@@ -109,7 +106,7 @@ async function deriveMarkdownFromHTML(!hasQuestion &&!hasHighlights &&!hasQuery &&- !hasPii &&+ !hasRedactPII &&!meta.options.onlyCleanContent) {return document;@@ -501,9 +498,8 @@ function coerceFieldsToFormats(meta: Meta, document: Document): Document {);}- // pii is only surfaced when both `pii` is in formats and `redactPII: true`.- // Drop any stale block (e.g. from a cached doc) otherwise so the response- // matches the documented contract.+ // pii is a diagnostic/report format. Redaction itself is controlled by+ // redactPII; include `pii` only when callers want spans/counts in the response.const hasPii = hasFormatOfType(meta.options.formats, "pii");const wantPii = !!(hasPii && meta.options.redactPII);if (!wantPii && document.pii !== undefined) {diff --git a/apps/api/src/scraper/scrapeURL/transformers/redactPII.test.ts b/apps/api/src/scraper/scrapeURL/transformers/redactPII.test.tsindex 878752b5f..9f539384f 100644--- a/apps/api/src/scraper/scrapeURL/transformers/redactPII.test.ts+++ b/apps/api/src/scraper/scrapeURL/transformers/redactPII.test.ts@@ -54,6 +54,29 @@ describe("performRedactPII", () => {expect(document.pii?.redactedMarkdown).toBe("Hello <PERSON>");});+ it("runs when redactPII is enabled without the pii format", async () => {+ mockedRedactText.mockResolvedValue({+ status: "ok",+ redactedMarkdown: "Hello <PERSON>",+ spans: [],+ counts: {},+ });++ const meta = baseMeta();+ meta.options.formats = [{ type: "markdown" }];++ const document = await performRedactPII(+ meta,+ baseDocument({+ markdown: "Hello Alice",+ }),+ );++ expect(mockedRedactText).toHaveBeenCalledTimes(1);+ expect(document.markdown).toBe("Hello <PERSON>");+ expect(document.pii?.redactedMarkdown).toBe("Hello <PERSON>");+ });+it("keeps markdown as an empty string when redaction cannot produce safe output", async () => {mockedRedactText.mockResolvedValue({status: "failed",diff --git a/apps/api/src/scraper/scrapeURL/transformers/redactPII.ts b/apps/api/src/scraper/scrapeURL/transformers/redactPII.tsindex 6384305a9..b043614fd 100644--- a/apps/api/src/scraper/scrapeURL/transformers/redactPII.ts+++ b/apps/api/src/scraper/scrapeURL/transformers/redactPII.ts@@ -1,6 +1,5 @@import { Meta } from "..";import { Document } from "../../../controllers/v2/types";-import { hasFormatOfType } from "../../../lib/format-utils";import { redactText } from "../../../lib/fire-privacy-client";export async function performRedactPII(@@ -8,9 +7,8 @@ export async function performRedactPII(document: Document,): Promise<Document> {if (!meta.options.redactPII) return document;- if (!hasFormatOfType(meta.options.formats, "pii")) return document;- // pii format requires markdown to redact. If the markdown derivation step+ // PII redaction requires markdown to redact. If the markdown derivation step// ran and produced nothing, we surface that as `skipped` rather than calling// fire-privacy with an empty body.if (typeof document.markdown !== "string") {