Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. π₯
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(api): remove pii format surface (#3741)
Abimael Martellb88d363e6073
diff --git a/apps/api/src/controllers/v2/__tests__/redactPII-schema.test.ts b/apps/api/src/controllers/v2/__tests__/redactPII-schema.test.tsindex c4458ad0a..5670d58e5 100644--- a/apps/api/src/controllers/v2/__tests__/redactPII-schema.test.ts+++ b/apps/api/src/controllers/v2/__tests__/redactPII-schema.test.ts@@ -59,6 +59,15 @@ describe("v2 scrapeRequestSchema β redactPII", () => {}});+ it("rejects the removed pii format", () => {+ const result = scrapeRequestSchema.safeParse({+ url: baseUrl,+ formats: ["pii"],+ redactPII: true,+ });+ expect(result.success).toBe(false);+ });+// ---- object form --------------------------------------------------------it("accepts an explicit mode in the object form", () => {diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.tsindex c65684a82..22ac101ba 100644--- a/apps/api/src/controllers/v2/types.ts+++ b/apps/api/src/controllers/v2/types.ts@@ -459,8 +459,7 @@ export type FormatObject =| QueryFormatWithOptions| { type: "branding" }| { type: "audio" }- | { type: "video" }- | { type: "pii" };+ | { type: "video" };const pdfModeSchema = z.enum(["fast", "auto", "ocr"]);@@ -635,7 +634,6 @@ const baseScrapeOptions = z.strictObject({queryFormatWithOptions,z.strictObject({ type: z.literal("audio") }),z.strictObject({ type: z.literal("video") }),- z.strictObject({ type: z.literal("pii") }),]).array().optional()@@ -1187,58 +1185,6 @@ export const mapRequestSchema = strictWithMessage(mapRequestSchemaBase);export type MapRequest = z.infer<typeof mapRequestSchema>;export type MapRequestInput = z.input<typeof mapRequestSchema>;-export type PIISource = "model" | "heuristics" | "unknown";--export type PIISpan = {- start: number;- end: number;- // Unified entity bucket. Present when `kind` maps onto one of the public- // entity buckets; omitted when fire-privacy returned a recognizer kind we- // don't expose (e.g. ORGANIZATION, DATE_TIME). Spans without an entity- // are dropped under any `entities` allowlist.- entity?: RedactPIIEntity;- // Granular recognizer label from fire-privacy (e.g. PRIVATE_PERSON,- // EMAIL_ADDRESS, ACCOUNT_NUMBER). Useful for audit / debugging; prefer- // `entity` for taxonomy-level checks.- kind: string;- source: PIISource;- // Confidence in [0, 1] when fire-privacy returned a score. Omitted when- // the recognizer didn't supply one.- score?: number;-};--// `ok` β redaction completed; redactedMarkdown is the result.-// `skipped` β redaction was not performed; see `reason` for why.-// redactedMarkdown may be the original text or null.-// `failed` β redaction was attempted but did not produce a usable result;-// see `reason`. redactedMarkdown is null.-type PIIStatus = "ok" | "skipped" | "failed";--// Why redaction was skipped or failed. Always set when status !== "ok".-// empty_input β no markdown to redact, or markdown was whitespace-// too_large β input exceeded the redaction-side byte ceiling-// upstream_skipped β fire-privacy reported model_status: "skipped"-// service_unavailable β fire-privacy returned 503-// timeout β request exceeded the redaction timeout budget-// error β any other failure (5xx, invalid JSON, network)-export type PIIReason =- | "empty_input"- | "too_large"- | "upstream_skipped"- | "service_unavailable"- | "timeout"- | "error";--export type PIIBlock = {- status: PIIStatus;- reason?: PIIReason;- redactedMarkdown: string | null;- spans: PIISpan[];- // Count of spans per public entity bucket. Spans whose `kind` doesn't- // map onto a bucket are not counted. Only non-zero entries are present.- counts: Partial<Record<RedactPIIEntity, number>>;-};-export type Document = {title?: string;description?: string;@@ -1258,7 +1204,6 @@ export type Document = {highlights?: string;branding?: BrandingProfile;warning?: string;- pii?: PIIBlock;attributes?: {selector: string;attribute: string;diff --git a/apps/api/src/lib/fire-privacy-client.ts b/apps/api/src/lib/fire-privacy-client.tsindex 211110e79..8649a1697 100644--- a/apps/api/src/lib/fire-privacy-client.ts+++ b/apps/api/src/lib/fire-privacy-client.ts@@ -1,10 +1,6 @@import { Logger } from "winston";import { config } from "../config";import {- PIIBlock,- PIIReason,- PIISource,- PIISpan,RedactPIIOptions,type RedactPIIEntity,} from "../controllers/v2/types";@@ -28,6 +24,35 @@ type RedactOptions = {options?: RedactPIIOptions;};+type RedactionSource = "model" | "heuristics" | "unknown";++type RedactionSpan = {+ start: number;+ end: number;+ entity?: RedactPIIEntity;+ kind: string;+ source: RedactionSource;+ score?: number;+};++type RedactionStatus = "ok" | "skipped" | "failed";++type RedactionReason =+ | "empty_input"+ | "too_large"+ | "upstream_skipped"+ | "service_unavailable"+ | "timeout"+ | "error";++type RedactionResult = {+ status: RedactionStatus;+ reason?: RedactionReason;+ redactedMarkdown: string | null;+ spans: RedactionSpan[];+ counts: Partial<Record<RedactPIIEntity, number>>;+};+// Mode + replaceStyle map to fire-privacy's `mode` and `operator` fields.// Keep both sides in sync if either side changes.const MODE_MAP = {@@ -60,9 +85,8 @@ const MAX_REDACT_BYTES = 250_000;const CHUNK_CONCURRENCY = 3;// Maps a span's `kind` (as returned by either OPF or Presidio) onto the-// unified entity bucket we expose to callers. Kinds we don't recognize-// fall through unmapped β entity filtering treats them as "not in any-// bucket" and drops them when an entity allowlist is in play.+// unified entity bucket used by redactPII options. Kinds we don't recognize+// fall through unmapped and drop when an entity allowlist is in play.const KIND_TO_ENTITY: Record<string, RedactPIIEntity> = {// PersonPRIVATE_PERSON: "PERSON",@@ -94,19 +118,19 @@ const KIND_TO_ENTITY: Record<string, RedactPIIEntity> = {MEDICAL_LICENSE: "SECRET",};-// Classify fire-privacy's per-span `source` string into the public-// taxonomy. OPF spans always carry `openai-privacy-filter`; Presidio-// recognizer names end in `Recognizer`. Anything else is opaque.-function classifySource(raw: unknown): PIISource {+// Classify fire-privacy's per-span `source` string into an internal taxonomy.+// OPF spans always carry `openai-privacy-filter`; Presidio recognizer names+// end in `Recognizer`. Anything else is opaque.+function classifySource(raw: unknown): RedactionSource {if (typeof raw !== "string") return "unknown";if (raw === "openai-privacy-filter") return "model";if (raw.endsWith("Recognizer")) return "heuristics";return "unknown";}-function coerceSpans(value: unknown): PIISpan[] {+function coerceSpans(value: unknown): RedactionSpan[] {if (!Array.isArray(value)) return [];- const out: PIISpan[] = [];+ const out: RedactionSpan[] = [];for (const raw of value) {if (typeof raw !== "object" || raw === null) continue;const r = raw as Record<string, unknown>;@@ -118,7 +142,7 @@ function coerceSpans(value: unknown): PIISpan[] {continue;}const entity = KIND_TO_ENTITY[r.kind];- const span: PIISpan = {+ const span: RedactionSpan = {start: r.start,end: r.end,kind: r.kind,@@ -135,9 +159,9 @@ function coerceSpans(value: unknown): PIISpan[] {// spans unchanged. When set, keeps only spans with a mapped `entity`// that's in the allowlist β unmapped spans drop.function filterByEntities(- spans: PIISpan[],+ spans: RedactionSpan[],entities: readonly RedactPIIEntity[] | undefined,-): PIISpan[] {+): RedactionSpan[] {if (!entities || entities.length === 0) return spans;const allow = new Set(entities);return spans.filter(s => s.entity !== undefined && allow.has(s.entity));@@ -152,7 +176,7 @@ function filterByEntities(// remove β drop the chars entirelyfunction renderRedacted(text: string,- spans: PIISpan[],+ spans: RedactionSpan[],replaceStyle: RedactPIIOptions["replaceStyle"],): string {if (spans.length === 0) return text;@@ -180,7 +204,7 @@ function renderRedacted(}function countByEntity(- spans: PIISpan[],+ spans: RedactionSpan[],): Partial<Record<RedactPIIEntity, number>> {const out: Partial<Record<RedactPIIEntity, number>> = {};for (const span of spans) {@@ -195,14 +219,14 @@ function countByEntity(type ChunkResult =| {ok: true;- spans: PIISpan[];+ spans: RedactionSpan[];redactedText: string;// Sticky upstream-skip flag: true if the model returned// model_status === "skipped" for this chunk. Surfaces on the// merged block when any chunk was skipped upstream.upstreamSkipped: boolean;}- | { ok: false; reason: PIIReason };+ | { ok: false; reason: RedactionReason };async function redactOnce(chunk: Chunk,@@ -233,7 +257,7 @@ async function redactOnce(});} catch (err) {clearTimeout(timer);- const reason: PIIReason = timedOut ? "timeout" : "error";+ const reason: RedactionReason = timedOut ? "timeout" : "error";logger?.warn("fire-privacy request failed", {reason,url,@@ -246,7 +270,7 @@ async function redactOnce(clearTimeout(timer);if (!response.ok) {- const reason: PIIReason =+ const reason: RedactionReason =response.status === 503 ? "service_unavailable" : "error";logger?.warn("fire-privacy returned non-2xx", {reason,@@ -321,7 +345,9 @@ async function runChunks(return results;}-export async function redactText(opts: RedactOptions): Promise<PIIBlock> {+export async function redactText(+ opts: RedactOptions,+): Promise<RedactionResult> {const { text, logger } = opts;const timeoutMs = opts.timeoutMs ?? config.FIRE_PRIVACY_TIMEOUT_MS;const options: RedactPIIOptions = opts.options ?? {diff --git a/apps/api/src/scraper/scrapeURL/transformers/index.ts b/apps/api/src/scraper/scrapeURL/transformers/index.tsindex e5488d5fe..44481a40c 100644--- a/apps/api/src/scraper/scrapeURL/transformers/index.ts+++ b/apps/api/src/scraper/scrapeURL/transformers/index.ts@@ -506,18 +506,6 @@ function coerceFieldsToFormats(meta: Meta, document: Document): Document {);}- // Redaction itself is controlled by redactPII. Keep internal redaction- // details only when explicitly requested.- const hasPii = hasFormatOfType(meta.options.formats, "pii");- const wantPii = !!(hasPii && meta.options.redactPII);- if (!wantPii && document.pii !== undefined) {- delete document.pii;- } else if (wantPii && document.pii === undefined) {- meta.logger.warn(- "Redaction details were requested, but there was no pii field in the result.",- );- }-if (!hasChangeTracking && document.changeTracking !== undefined) {meta.logger.warn("Removed changeTracking from Document because it wasn't in formats -- this is extremely wasteful and indicates a bug.",diff --git a/apps/api/src/scraper/scrapeURL/transformers/redactPII.test.ts b/apps/api/src/scraper/scrapeURL/transformers/redactPII.test.tsindex 282e2fc68..0959b039d 100644--- a/apps/api/src/scraper/scrapeURL/transformers/redactPII.test.ts+++ b/apps/api/src/scraper/scrapeURL/transformers/redactPII.test.ts@@ -51,7 +51,6 @@ describe("performRedactPII", () => {);expect(document.markdown).toBe("Hello <PERSON>");- expect(document.pii?.redactedMarkdown).toBe("Hello <PERSON>");});it("runs when redactPII is enabled with markdown output", async () => {@@ -74,7 +73,6 @@ describe("performRedactPII", () => {expect(mockedRedactText).toHaveBeenCalledTimes(1);expect(document.markdown).toBe("Hello <PERSON>");- expect(document.pii?.redactedMarkdown).toBe("Hello <PERSON>");});it("keeps markdown as an empty string when redaction cannot produce safe output", async () => {@@ -94,7 +92,6 @@ describe("performRedactPII", () => {);expect(document.markdown).toBe("");- expect(document.pii?.redactedMarkdown).toBeNull();});it("keeps downstream markdown consumers safe when source markdown is missing", async () => {@@ -102,12 +99,5 @@ describe("performRedactPII", () => {expect(mockedRedactText).not.toHaveBeenCalled();expect(document.markdown).toBe("");- expect(document.pii).toEqual({- status: "skipped",- reason: "empty_input",- redactedMarkdown: null,- spans: [],- counts: {},- });});});diff --git a/apps/api/src/scraper/scrapeURL/transformers/redactPII.ts b/apps/api/src/scraper/scrapeURL/transformers/redactPII.tsindex b043614fd..d0e62b3cb 100644--- a/apps/api/src/scraper/scrapeURL/transformers/redactPII.ts+++ b/apps/api/src/scraper/scrapeURL/transformers/redactPII.ts@@ -9,21 +9,13 @@ export async function performRedactPII(if (!meta.options.redactPII) return document;// PII redaction requires markdown to redact. If the markdown derivation step- // ran and produced nothing, we surface that as `skipped` rather than calling- // fire-privacy with an empty body.+ // ran and produced nothing, fail closed without calling fire-privacy.if (typeof document.markdown !== "string") {- document.pii = {- status: "skipped",- reason: "empty_input",- redactedMarkdown: null,- spans: [],- counts: {},- };document.markdown = "";return document;}- document.pii = await redactText({+ const result = await redactText({text: document.markdown,url: meta.url,logger: meta.logger,@@ -34,11 +26,10 @@ export async function performRedactPII(});// Swap raw markdown for the redacted version. Caller asked for PII- // redaction; leaving the leaky one in `document.markdown` next to- // `document.pii.redactedMarkdown` is a footgun. On `failed` /- // `skipped: too_large` (redactedMarkdown === null), fail closed with+ // redaction; leaving the leaky one in `document.markdown` is a footgun.+ // On `failed` / `skipped: too_large` (redactedMarkdown === null), fail closed with// an empty string so later transformers still receive markdown-shaped input.- document.markdown = document.pii.redactedMarkdown ?? "";+ document.markdown = result.redactedMarkdown ?? "";return document;}diff --git a/apps/js-sdk/firecrawl/src/v2/types.ts b/apps/js-sdk/firecrawl/src/v2/types.tsindex 9580c7764..c6e5e4408 100644--- a/apps/js-sdk/firecrawl/src/v2/types.ts+++ b/apps/js-sdk/firecrawl/src/v2/types.ts@@ -14,8 +14,7 @@ export type FormatString =| "attributes"| "branding"| "audio"- | "video"- | "pii";+ | "video";export interface Viewport {width: number;@@ -240,45 +239,6 @@ export interface RedactPIIOptions {replaceStyle?: "tag" | "mask" | "remove";}-export type PIISource = "model" | "heuristics" | "unknown";--export interface PIISpan {- start: number;- end: number;- /** Unified entity bucket. Omitted when `kind` doesn't map onto one. */- entity?: RedactPIIEntity;- /** Granular recognizer label from fire-privacy. */- kind: string;- source: PIISource;- /** Confidence in [0, 1] when supplied. */- score?: number;-}--/**- * - ok: redaction completed; redactedMarkdown is the result.- * - skipped: redaction was not performed; see `reason`.- * - failed: redaction was attempted but did not produce a usable result.- */-export type PIIStatus = "ok" | "skipped" | "failed";--/** Always set when status !== "ok". */-export type PIIReason =- | "empty_input"- | "too_large"- | "upstream_skipped"- | "service_unavailable"- | "timeout"- | "error";--export interface PIIBlock {- status: PIIStatus;- reason?: PIIReason;- redactedMarkdown: string | null;- spans: PIISpan[];- /** Span count per entity bucket. Only non-zero entries are present. */- counts: Partial<Record<RedactPIIEntity, number>>;-}-export type ParseFileData =| Blob| File@@ -549,7 +509,6 @@ export interface Document {warning?: string;changeTracking?: Record<string, unknown>;branding?: BrandingProfile;- pii?: PIIBlock;}// Pagination configuration for auto-fetching pages from v2 endpoints that return a `next` URLdiff --git a/apps/python-sdk/firecrawl/v2/types.py b/apps/python-sdk/firecrawl/v2/types.pyindex 8dcec63d2..2bdbb0dfe 100644--- a/apps/python-sdk/firecrawl/v2/types.py+++ b/apps/python-sdk/firecrawl/v2/types.py@@ -271,56 +271,6 @@ RedactPIIEntity = Literal["SECRET",]-PIISource = Literal["model", "heuristics", "unknown"]---class PIISpan(BaseModel):- """A single PII detection in the source markdown."""-- start: int- end: int- # Unified entity bucket. Present when `kind` maps onto one of the- # public entity buckets; omitted when fire-privacy returned a- # recognizer kind that doesn't map.- entity: Optional[RedactPIIEntity] = None- # Granular recognizer label from fire-privacy (e.g. PRIVATE_PERSON,- # EMAIL_ADDRESS). Prefer `entity` for taxonomy-level checks.- kind: str- source: PIISource- # Confidence in [0, 1] when the recognizer supplied one.- score: Optional[float] = None---# ok β redaction completed; redactedMarkdown is the result.-# skipped β redaction was not performed; see `reason`.-# failed β redaction was attempted but did not produce a usable result;-# see `reason`. redactedMarkdown is None.-PIIStatus = Literal["ok", "skipped", "failed"]--# Always set when status != "ok".-PIIReason = Literal[- "empty_input",- "too_large",- "upstream_skipped",- "service_unavailable",- "timeout",- "error",-]---class PIIBlock(BaseModel):- """Result of the PII redaction step."""-- status: PIIStatus- reason: Optional[PIIReason] = None- redacted_markdown: Optional[str] = Field(default=None, alias="redactedMarkdown")- spans: List[PIISpan] = []- # Span count per public entity bucket. Spans whose `kind` doesn't- # map onto a bucket are not counted.- counts: Dict[RedactPIIEntity, int] = {}-- model_config = {"populate_by_name": True}-class Document(BaseModel):"""A scraped document."""@@ -342,7 +292,6 @@ class Document(BaseModel):warning: Optional[str] = Nonechange_tracking: Optional[Dict[str, Any]] = Nonebranding: Optional[BrandingProfile] = None- pii: Optional[PIIBlock] = None@propertydef metadata_typed(self) -> DocumentMetadata:@@ -469,7 +418,6 @@ FormatString = Literal["query","audio","video",- "pii",# snake_case versions (user-friendly)"raw_html","change_tracking",