Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. πŸ”₯

15.1 KB
Raw Blame
import { Logger } from "winston";
import { config } from "../config";
import {
  RedactPIIOptions,
  type RedactPIIEntity,
} from "../controllers/v2/types";
import { chunkMarkdown, type Chunk } from "./fire-privacy-chunker";
import { UsOnlyUnavailableError } from "../scraper/scrapeURL/error";
import { isRegionUnavailable, US_DATA_REGION } from "./us-only";

type FirePrivacyResponse = {
  redacted_text?: unknown;
  spans?: unknown;
  model_status?: unknown;
  data_region?: unknown;
};

type RedactOptions = {
  text: string;
  url?: string;
  timeoutMs?: number;
  logger?: Logger;
  // Caller-provided config. Boolean form is normalized to a defaults
  // object via the Zod transform before reaching here; an unset value
  // wouldn't trigger this code path at all (transformer skips when
  // meta.options.redactPII is falsy).
  options?: RedactPIIOptions;
  // Pins the model call to US capacity; a refusal or a reply without the
  // region echo throws UsOnlyUnavailableError.
  usOnly?: boolean;
};

type RedactionSource = "model" | "heuristics" | "unknown";

type RedactionSpan = {
  start: number;
  end: number;
  entity?: RedactPIIEntity;
  kind: string;
  source: RedactionSource;
  score?: number;
};

type RedactionStatus = "ok" | "skipped" | "failed";

type RedactionReason =
  | "empty_input"
  | "too_large"
  | "upstream_skipped"
  | "service_unavailable"
  | "timeout"
  | "error"
  | "region_unavailable"
  | "region_unconfirmed";

type RedactionResult = {
  status: RedactionStatus;
  reason?: RedactionReason;
  redactedMarkdown: string | null;
  spans: RedactionSpan[];
  counts: Partial<Record<RedactPIIEntity, number>>;
};

// Mode + replaceStyle map to fire-privacy's `mode` and `operator` fields.
// Keep both sides in sync if either side changes.
const MODE_MAP = {
  accurate: "model",
  aggressive: "both",
  fast: "heuristics",
} as const;

const REPLACE_MAP = {
  tag: "replace",
  mask: "mask",
  remove: "redact",
} as const;

const DEFAULTS = {
  mode: "accurate",
  replaceStyle: "tag",
  language: "en",
} as const;

// Hard ceiling: above this byte count we refuse to redact and return
// `skipped` with reason `too_large` rather than ship partial results.
// Sized from `eval/scaling/` measurements in fire-privacy: 250KB β‰ˆ ~80
// PDF pages, ~10 chunks, ~60s wall at c=3 with the model on. Anything
// larger pushes past the typical scrape budget and starves the fleet.
const MAX_REDACT_BYTES = 250_000;
// Chunks fan out at this concurrency to fire-privacy. The fleet has 6
// pods at saturation; c=3 keeps a single call under 50% of capacity so
// other tenants aren't starved.
const CHUNK_CONCURRENCY = 3;

// Maps a span's `kind` (as returned by either OPF or Presidio) onto the
// unified entity bucket used by redactPII options. Kinds we don't recognize
// fall through unmapped and drop when an entity allowlist is in play.
const KIND_TO_ENTITY: Record<string, RedactPIIEntity> = {
  // Person
  PRIVATE_PERSON: "PERSON",
  PERSON: "PERSON",
  // Email
  PRIVATE_EMAIL: "EMAIL",
  EMAIL_ADDRESS: "EMAIL",
  // Phone
  PRIVATE_PHONE: "PHONE",
  PHONE_NUMBER: "PHONE",
  PHONEIMEI: "PHONE",
  // Location
  PRIVATE_ADDRESS: "LOCATION",
  LOCATION: "LOCATION",
  // Financial
  ACCOUNT_NUMBER: "FINANCIAL",
  CREDIT_CARD: "FINANCIAL",
  IBAN_CODE: "FINANCIAL",
  US_BANK_NUMBER: "FINANCIAL",
  US_SSN: "FINANCIAL",
  US_ITIN: "FINANCIAL",
  CRYPTO: "FINANCIAL",
  // Secret
  SECRET: "SECRET",
  API_KEY: "SECRET",
  PASSWORD: "SECRET",
  US_DRIVER_LICENSE: "SECRET",
  US_PASSPORT: "SECRET",
  MEDICAL_LICENSE: "SECRET",
};

// Classify fire-privacy's per-span `source` string into an internal taxonomy.
// OPF spans always carry `openai-privacy-filter`; Presidio recognizer names
// end in `Recognizer`. Anything else is opaque.
function classifySource(raw: unknown): RedactionSource {
  if (typeof raw !== "string") return "unknown";
  if (raw === "openai-privacy-filter") return "model";
  if (raw.endsWith("Recognizer")) return "heuristics";
  return "unknown";
}

function coerceSpans(value: unknown): RedactionSpan[] {
  if (!Array.isArray(value)) return [];
  const out: RedactionSpan[] = [];
  for (const raw of value) {
    if (typeof raw !== "object" || raw === null) continue;
    const r = raw as Record<string, unknown>;
    if (
      typeof r.start !== "number" ||
      typeof r.end !== "number" ||
      typeof r.kind !== "string"
    ) {
      continue;
    }
    const entity = KIND_TO_ENTITY[r.kind];
    const span: RedactionSpan = {
      start: r.start,
      end: r.end,
      kind: r.kind,
      source: classifySource(r.source),
    };
    if (entity !== undefined) span.entity = entity;
    if (typeof r.score === "number") span.score = r.score;
    out.push(span);
  }
  return out;
}

// Apply an entity allowlist to the span set. When unset, returns the
// spans unchanged. When set, keeps only spans with a mapped `entity`
// that's in the allowlist β€” unmapped spans drop.
function filterByEntities(
  spans: RedactionSpan[],
  entities: readonly RedactPIIEntity[] | undefined,
): RedactionSpan[] {
  if (!entities || entities.length === 0) return spans;
  const allow = new Set(entities);
  return spans.filter(s => s.entity !== undefined && allow.has(s.entity));
}

// Re-render redacted text from the original + a filtered span set when
// fire-privacy's `redacted_text` no longer matches what we want to return
// (i.e. we narrowed the spans via entity filter). Same operator semantics
// as fire-privacy:
//   tag    β†’ `<KIND>` placeholder per span
//   mask   β†’ '*' Γ— span length
//   remove β†’ drop the chars entirely
function renderRedacted(
  text: string,
  spans: RedactionSpan[],
  replaceStyle: RedactPIIOptions["replaceStyle"],
): string {
  if (spans.length === 0) return text;
  const sorted = [...spans].sort((a, b) => a.start - b.start);
  let out = "";
  let cursor = 0;
  for (const span of sorted) {
    if (span.start < cursor) continue; // overlap with prior span; skip
    if (span.start > text.length) break;
    out += text.slice(cursor, span.start);
    switch (replaceStyle) {
      case "tag":
        out += `<${span.kind}>`;
        break;
      case "mask":
        out += "*".repeat(Math.max(0, span.end - span.start));
        break;
      case "remove":
        break;
    }
    cursor = Math.min(span.end, text.length);
  }
  out += text.slice(cursor);
  return out;
}

function countByEntity(
  spans: RedactionSpan[],
): Partial<Record<RedactPIIEntity, number>> {
  const out: Partial<Record<RedactPIIEntity, number>> = {};
  for (const span of spans) {
    if (span.entity === undefined) continue;
    out[span.entity] = (out[span.entity] ?? 0) + 1;
  }
  return out;
}

// Result of one /redact call, in source-coordinate space (spans already
// offset by the chunk's start).
type ChunkResult =
  | {
      ok: true;
      spans: RedactionSpan[];
      redactedText: string;
      // Sticky upstream-skip flag: true if the model returned
      // model_status === "skipped" for this chunk. Surfaces on the
      // merged block when any chunk was skipped upstream.
      upstreamSkipped: boolean;
    }
  | { ok: false; reason: RedactionReason };

async function redactOnce(
  chunk: Chunk,
  options: RedactPIIOptions,
  url: string | undefined,
  timeoutMs: number,
  logger: Logger | undefined,
  usOnly: boolean,
): Promise<ChunkResult> {
  const controller = new AbortController();
  let timedOut = false;
  const timer = setTimeout(() => {
    timedOut = true;
    controller.abort();
  }, timeoutMs);

  let response: Response;
  try {
    response = await fetch(`${config.FIRE_PRIVACY_URL}/redact`, {
      method: "POST",
      headers: { "content-type": "application/json" },
      body: JSON.stringify({
        text: chunk.text,
        mode: MODE_MAP[options.mode],
        operator: REPLACE_MAP[options.replaceStyle],
        language: DEFAULTS.language,
        ...(usOnly && { data_region: US_DATA_REGION }),
      }),
      signal: controller.signal,
    });
  } catch (err) {
    clearTimeout(timer);
    const reason: RedactionReason = timedOut ? "timeout" : "error";
    logger?.warn("fire-privacy request failed", {
      reason,
      url,
      mode: options.mode,
      chunkStart: chunk.start,
      error: err instanceof Error ? err.message : String(err),
    });
    return { ok: false, reason };
  }
  // Read under the request deadline so a stalled body cannot hang the chunk.
  const refusal: unknown =
    usOnly && !response.ok ? await response.json().catch(() => null) : null;
  clearTimeout(timer);

  if (!response.ok) {
    const reason: RedactionReason = isRegionUnavailable(refusal)
      ? "region_unavailable"
      : response.status === 503
        ? "service_unavailable"
        : "error";
    logger?.warn("fire-privacy returned non-2xx", {
      reason,
      httpStatus: response.status,
      url,
      mode: options.mode,
      chunkStart: chunk.start,
    });
    return { ok: false, reason };
  }

  let body: FirePrivacyResponse;
  try {
    body = (await response.json()) as FirePrivacyResponse;
  } catch (err) {
    logger?.warn("fire-privacy returned invalid JSON", {
      url,
      chunkStart: chunk.start,
      error: err instanceof Error ? err.message : String(err),
    });
    return { ok: false, reason: "error" };
  }

  const upstreamRedacted =
    typeof body.redacted_text === "string" ? body.redacted_text : null;
  const rawSpans = coerceSpans(body.spans);

  if (body.model_status === "error" || upstreamRedacted === null) {
    return { ok: false, reason: "error" };
  }

  // A build that ignores the pin answers without the echo; its model call
  // may have run outside the US.
  if (usOnly && body.data_region !== US_DATA_REGION) {
    logger?.warn("fire-privacy did not confirm US-only processing", {
      url,
      chunkStart: chunk.start,
    });
    return { ok: false, reason: "region_unconfirmed" };
  }

  // Lift spans into source coordinates.
  const spans = rawSpans.map(s => ({
    ...s,
    start: s.start + chunk.start,
    end: s.end + chunk.start,
  }));

  return {
    ok: true,
    spans,
    redactedText: upstreamRedacted,
    upstreamSkipped: body.model_status === "skipped",
  };
}

// Run chunks against fire-privacy with bounded concurrency. Returns
// results in chunk order (results[i] corresponds to chunks[i]).
async function runChunks(
  chunks: Chunk[],
  options: RedactPIIOptions,
  url: string | undefined,
  timeoutMs: number,
  logger: Logger | undefined,
  usOnly: boolean,
): Promise<ChunkResult[]> {
  const results = new Array<ChunkResult>(chunks.length);
  let next = 0;

  const worker = async (): Promise<void> => {
    while (true) {
      const i = next++;
      if (i >= chunks.length) return;
      results[i] = await redactOnce(
        chunks[i],
        options,
        url,
        timeoutMs,
        logger,
        usOnly,
      );
    }
  };

  const workers = Array.from(
    { length: Math.min(CHUNK_CONCURRENCY, chunks.length) },
    () => worker(),
  );
  await Promise.all(workers);
  return results;
}

export async function redactText(
  opts: RedactOptions,
): Promise<RedactionResult> {
  const { text, logger } = opts;
  const timeoutMs = opts.timeoutMs ?? config.FIRE_PRIVACY_TIMEOUT_MS;
  const options: RedactPIIOptions = opts.options ?? {
    mode: DEFAULTS.mode,
    replaceStyle: DEFAULTS.replaceStyle,
  };

  // Fire-privacy is an optional service. If it's not configured for this
  // deployment, surface a clear `failed/error` block rather than making
  // a request to `undefined/redact` and letting it fail opaquely.
  if (!config.FIRE_PRIVACY_URL) {
    logger?.warn("redactPII requested but FIRE_PRIVACY_URL is not configured", {
      url: opts.url,
    });
    return {
      status: "failed",
      reason: "error",
      redactedMarkdown: null,
      spans: [],
      counts: {},
    };
  }

  // Empty/whitespace input is a no-op locally β€” saves a round trip and matches
  // fire-privacy's own "skipped" semantics. We pass the original text through
  // as `redactedMarkdown` since there's nothing to remove.
  if (text.trim().length === 0) {
    return {
      status: "skipped",
      reason: "empty_input",
      redactedMarkdown: text,
      spans: [],
      counts: {},
    };
  }

  // Hard byte ceiling. Anything above this is refused with a dedicated
  // reason so callers can tell "we declined" apart from "we tried and
  // it broke." We measure bytes, not chars, because fire-privacy's
  // request cap is byte-based.
  const inputBytes = new TextEncoder().encode(text).length;
  if (inputBytes > MAX_REDACT_BYTES) {
    logger?.info("fire-privacy input exceeds redaction ceiling", {
      url: opts.url,
      inputBytes,
      maxBytes: MAX_REDACT_BYTES,
    });
    return {
      status: "skipped",
      reason: "too_large",
      redactedMarkdown: null,
      spans: [],
      counts: {},
    };
  }

  const chunks = chunkMarkdown(text);
  const results = await runChunks(
    chunks,
    options,
    opts.url,
    timeoutMs,
    logger,
    opts.usOnly === true,
  );

  const regionFailure = results.find(
    r =>
      !r.ok &&
      (r.reason === "region_unavailable" || r.reason === "region_unconfirmed"),
  );
  if (regionFailure && !regionFailure.ok) {
    const detail =
      regionFailure.reason === "region_unconfirmed"
        ? "the redaction service did not confirm US-only processing"
        : "no US capacity could take this request";
    throw new UsOnlyUnavailableError(
      `PII redaction is not available for teams restricted to US-only processing right now: ${detail}. Try again later.`,
    );
  }

  // All-or-nothing: any chunk failure poisons the whole response. Partial
  // redaction is worse than no redaction β€” callers can't tell which
  // sections of their markdown are clean. Pick the first non-ok reason
  // so the failure surface (timeout / service_unavailable / error) is
  // preserved end-to-end.
  const firstFailure = results.find(r => !r.ok);
  if (firstFailure && !firstFailure.ok) {
    return {
      status: "failed",
      reason: firstFailure.reason,
      redactedMarkdown: null,
      spans: [],
      counts: {},
    };
  }

  const successes = results.filter(
    (r): r is Extract<ChunkResult, { ok: true }> => r.ok,
  );

  // Merge: spans already lifted into source coordinates by redactOnce.
  const allSpans = successes.flatMap(r => r.spans);
  const spans = filterByEntities(allSpans, options.entities);

  // Re-render when the entity filter pruned spans OR when we have
  // multiple chunks (per-chunk redacted_text concatenations are valid
  // since chunks are non-overlapping, but re-rendering with the same
  // operator yields identical output and keeps one code path for the
  // edge cases β€” e.g. chunk boundaries inside a span are impossible
  // by construction since spans are produced after we split).
  const concatRedacted = successes.map(r => r.redactedText).join("");
  const redactedMarkdown =
    spans.length === allSpans.length
      ? concatRedacted
      : renderRedacted(text, spans, options.replaceStyle);

  const upstreamSkipped = successes.some(r => r.upstreamSkipped);

  if (upstreamSkipped) {
    return {
      status: "skipped",
      reason: "upstream_skipped",
      redactedMarkdown,
      spans,
      counts: countByEntity(spans),
    };
  }

  return {
    status: "ok",
    redactedMarkdown,
    spans,
    counts: countByEntity(spans),
  };
}