Skip to content

Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active

🌫️ feat: Add Bounded PII Text Transformation (#16295)

lia-by-librechat[bot]7d5cbc9ebf49
diff --git a/librechat.example.yaml b/librechat.example.yaml
index e407c300dd..3f25483564 100644
--- a/librechat.example.yaml
+++ b/librechat.example.yaml
@@ -1390,7 +1390,11 @@ endpoints:
# messages:
# unattributedAssistantContent: model_output # `model_output` (default) or `inspect`
# pii:
-# action: audit # `block` (default) or `audit`
+# action: audit # `block` (default), `audit`, or `redact` (reserved for transformation-aware ingress; current routes still block)
+# # `redact` requires the later encrypted owner-view and canonical-message integration.
+# # maxCharacters: 65536 # bounded total inspected text per transformation session
+# # maxMatches: 256 # bounded matches per transformation session
+# # Custom patterns may set category: email, phone, name, credential, or custom.
# fields: [name, text, summary, quote, answer, decision_response, decision_reason, content_part, attachment_reference, assembled_context]
# starterPatterns: [sk_prefix, bearer_header, api_key_header]
# customPatterns:
diff --git a/packages/api/src/protection/detectors/pattern.spec.ts b/packages/api/src/protection/detectors/pattern.spec.ts
index 992eb6e593..cc84a1a6ad 100644
--- a/packages/api/src/protection/detectors/pattern.spec.ts
+++ b/packages/api/src/protection/detectors/pattern.spec.ts
@@ -72,6 +72,32 @@ describe('pattern content inspector', () => {
).toMatchObject({ ruleId: 'bearer_header' });
});
+ it.each([
+ { config: { starterPatterns: ['bearer_header'] }, text: 'Bearer contract-token' },
+ {
+ config: {
+ starterPatterns: [],
+ customPatterns: [{ id: 'custom', label: 'Custom', regex: 'ORG-[A-Z]+' }],
+ },
+ text: 'ORG-SECRET',
+ },
+ ])('enforces a zero remaining allowance when locating $text', ({ config, text }) => {
+ const inspector = createPatternContentInspector(config, { linearTime: true });
+
+ expect(inspector.locate('clean text', 0)).toEqual([]);
+ expect(inspector.locate('', 0)).toEqual([]);
+ expect(() => inspector.locate(text, 0)).toThrow(PatternConfigurationError);
+ expect(inspector.locate(text, 1)).toHaveLength(1);
+ });
+
+ it.each([-1, 0.5, Number.NaN, Number.POSITIVE_INFINITY, Number.MAX_SAFE_INTEGER + 1])(
+ 'rejects invalid remaining match allowance %s before inspecting',
+ (allowance) => {
+ const inspector = createPatternContentInspector({}, { linearTime: true });
+ expect(() => inspector.locate('clean text', allowance)).toThrow(PatternConfigurationError);
+ },
+ );
+
it('memoizes a compiled set by config identity and memory limit', () => {
const compile = jest.spyOn(RE2Set.prototype, 'compile');
const config: MessageFilterPiiConfig = {
diff --git a/packages/api/src/protection/detectors/pattern.ts b/packages/api/src/protection/detectors/pattern.ts
index 373028a3a2..6456b931be 100644
--- a/packages/api/src/protection/detectors/pattern.ts
+++ b/packages/api/src/protection/detectors/pattern.ts
@@ -1,4 +1,4 @@
-import { RE2Set } from 're2js';
+import { RE2JS, RE2Set } from 're2js';
import { logger } from '@librechat/data-schemas';
import {
MAX_PII_CUSTOM_REGEX_CHARACTERS,
@@ -9,23 +9,25 @@ import {
MAX_PII_PATTERNS_PER_SOURCE,
getPiiRegexProgramSize,
} from 'librechat-data-provider';
-import type { MessageFilterPiiConfig, FilterPiiCustomPatternConfig } from 'librechat-data-provider';
+import type {
+ FilterPiiCategory,
+ MessageFilterPiiConfig,
+ FilterPiiCustomPatternConfig,
+} from 'librechat-data-provider';
import type { ProtectionFinding, TextContentFragment } from '../types';
-interface TestablePattern {
- test(input: string): boolean;
-}
-
interface CompiledPattern {
readonly id: string;
readonly label: string;
- readonly pattern: TestablePattern;
+ readonly pattern: RegExp;
+ readonly category: FilterPiiCategory;
}
interface PreparedCustomPattern {
readonly id: string;
readonly label: string;
readonly regex: string;
+ readonly category: FilterPiiCategory;
}
interface SnapshotPatternContentInspectorConfig {
@@ -67,10 +69,18 @@ export interface PatternContentInspectorPreflightCost {
readonly regexes: readonly string[];
}
+export interface PatternTextMatch {
+ readonly start: number;
+ readonly end: number;
+ readonly category: FilterPiiCategory;
+}
+
export interface PatternContentInspector {
readonly active: boolean;
inspectFragment(fragment: TextContentFragment): ProtectionFinding | null;
inspect(fragments: Iterable<TextContentFragment>): ProtectionFinding | null;
+ /** Internal match locations, never safe to return as public finding metadata. */
+ locate(text: string, maxMatches: number): readonly PatternTextMatch[];
}
export interface PatternContentInspectorConfig {
@@ -86,9 +96,24 @@ export interface PatternContentInspectorOptions {
}
const STARTER_PATTERNS: readonly CompiledPattern[] = [
- { id: 'sk_prefix', label: 'sk- prefix token', pattern: /\b(sk-)[a-zA-Z0-9_-]+/ },
- { id: 'bearer_header', label: 'Bearer token', pattern: /\b(Bearer )[^\s"']+/i },
- { id: 'api_key_header', label: 'api-key header', pattern: /\b(api-key:?\s+)[^\s"']+/i },
+ {
+ id: 'sk_prefix',
+ label: 'sk- prefix token',
+ pattern: /\b(sk-)[a-zA-Z0-9_-]+/,
+ category: 'credential',
+ },
+ {
+ id: 'bearer_header',
+ label: 'Bearer token',
+ pattern: /\b(Bearer )[^\s"']+/i,
+ category: 'credential',
+ },
+ {
+ id: 'api_key_header',
+ label: 'api-key header',
+ pattern: /\b(api-key:?\s+)[^\s"']+/i,
+ category: 'credential',
+ },
];
const STARTER_BY_ID = new Map(STARTER_PATTERNS.map((pattern) => [pattern.id, pattern]));
@@ -201,7 +226,25 @@ function readCustomPattern(candidate: unknown, index: number): FilterPiiCustomPa
if (typeof regex !== 'string' || regex.length === 0 || regex.length > MAX_PII_PATTERN_LENGTH) {
throw configurationError(`customPatterns[${index}].regex is invalid`);
}
- return { id, label, regex };
+ let category: unknown;
+ try {
+ category = (candidate as FilterPiiCustomPatternConfig).category;
+ } catch {
+ throw configurationError(`customPatterns[${index}] could not be read safely`);
+ }
+ if (
+ category != null &&
+ (typeof category !== 'string' ||
+ !['email', 'phone', 'name', 'credential', 'custom'].includes(category))
+ ) {
+ throw configurationError(`customPatterns[${index}].category is invalid`);
+ }
+ return {
+ id,
+ label,
+ regex,
+ ...(category == null ? {} : { category: category as FilterPiiCategory }),
+ };
}
function snapshotConfig(
@@ -240,7 +283,7 @@ function snapshotConfig(
throw configurationError('customPatterns could not be read safely');
}
const pattern = readCustomPattern(patternCandidate, index);
- custom.push(pattern);
+ custom.push({ ...pattern, category: pattern.category ?? 'custom' });
regexes.push(pattern.regex);
regexCharacters += pattern.regex.length;
if (regexCharacters > MAX_PII_CUSTOM_REGEX_CHARACTERS) {
@@ -357,10 +400,17 @@ function createSequentialInspector(patterns: readonly CompiledPattern[]): Patter
function createInspector(
active: boolean,
inspectFragment: (fragment: TextContentFragment) => ProtectionFinding | null,
+ locate?: (text: string, maxMatches: number) => readonly PatternTextMatch[],
): PatternContentInspector {
return {
active,
inspectFragment,
+ locate(text, maxMatches) {
+ if (locate == null) {
+ throw configurationError('redaction requires a linear-time pattern inspector');
+ }
+ return locate(text, maxMatches);
+ },
inspect(fragments) {
for (const fragment of fragments) {
const finding = inspectFragment(fragment);
@@ -416,7 +466,44 @@ function createLinearInspector(
return pattern == null ? null : findingFor(pattern, fragment);
};
- return createInspector(prepared.cost.active, inspectFragment);
+ const compiled = new Map<string, RE2JS>();
+ const locate = (text: string, maxMatches: number): readonly PatternTextMatch[] => {
+ if (!Number.isSafeInteger(maxMatches) || maxMatches < 0) {
+ throw configurationError('redaction match limit must be a non-negative safe integer');
+ }
+ const matches: PatternTextMatch[] = [];
+ const selected = customSet == null ? [] : customSet.match(text);
+ const candidates: Array<readonly [string, FilterPiiCategory]> = prepared.starter.map(
+ (pattern) => [
+ `${pattern.pattern.ignoreCase ? '(?i)' : ''}${pattern.pattern.source}`,
+ pattern.category,
+ ],
+ );
+ for (const index of selected) {
+ const pattern = prepared.custom[index];
+ if (pattern != null) {
+ candidates.push([pattern.regex, pattern.category]);
+ }
+ }
+ for (const [expression, category] of candidates) {
+ let regex = compiled.get(expression);
+ if (regex == null) {
+ regex = RE2JS.compile(expression);
+ compiled.set(expression, regex);
+ }
+ const matcher = regex.matcher(text);
+ while (matcher.find()) {
+ const start = matcher.start();
+ const end = matcher.end();
+ if (end <= start || matches.length >= maxMatches) {
+ throw configurationError('redaction match limit or empty match encountered');
+ }
+ matches.push({ start, end, category });
+ }
+ }
+ return matches;
+ };
+ return createInspector(prepared.cost.active, inspectFragment, locate);
}
function cacheLinearInspector(
@@ -480,6 +567,7 @@ export function createPatternContentInspector(
id: pattern.id,
label: pattern.label,
pattern: new RegExp(pattern.regex),
+ category: pattern.category,
}));
const inspector = createSequentialInspector([...prepared.starter, ...custom]);
if (options.cacheResult !== false) {
diff --git a/packages/api/src/protection/index.ts b/packages/api/src/protection/index.ts
index 18e325633a..264f49b381 100644
--- a/packages/api/src/protection/index.ts
+++ b/packages/api/src/protection/index.ts
@@ -1,5 +1,6 @@
export * from './types';
export * from './runtime';
+export * from './transform';
export * from './title';
export * from './legacy';
export * from './provenance';
diff --git a/packages/api/src/protection/transform.spec.ts b/packages/api/src/protection/transform.spec.ts
new file mode 100644
index 0000000000..88022c2981
--- /dev/null
+++ b/packages/api/src/protection/transform.spec.ts
@@ -0,0 +1,227 @@
+import type { FiltersConfig } from 'librechat-data-provider';
+import type { TextContentFragment } from './types';
+import { createPiiTextTransformer, PiiTransformationError } from './transform';
+import { inspectContent } from './runtime';
+
+jest.mock('@librechat/data-schemas', () => ({
+ logger: { warn: jest.fn(), error: jest.fn(), info: jest.fn(), debug: jest.fn() },
+}));
+
+const pii: NonNullable<NonNullable<FiltersConfig['messages']>['pii']> = {
+ action: 'redact' as const,
+ starterPatterns: [],
+ fields: ['text'],
+ customPatterns: [
+ {
+ id: 'email',
+ label: 'Email address',
+ regex: '(?i)[a-z]+@[a-z]+\\.[a-z]+',
+ category: 'email' as const,
+ },
+ ],
+};
+
+function fragment(text: string, extras: Partial<TextContentFragment> = {}): TextContentFragment {
+ return {
+ id: 'chat.text',
+ path: '/text',
+ text,
+ source: 'message',
+ field: 'text',
+ format: 'plain',
+ treatment: 'replaceable',
+ provenance: 'user',
+ ...extras,
+ } as TextContentFragment;
+}
+
+describe('opt-in PII text transformation', () => {
+ it('leaves the existing inspection path blocking until a caller integrates transformation', () => {
+ const filters: FiltersConfig = { messages: { pii } };
+ expect(inspectContent([fragment('[email protected]')], { filters })).toMatchObject({
+ detectorId: 'pii-pattern',
+ ruleId: 'email',
+ });
+ });
+
+ it('uses typed placeholders and returns no matched values in metadata', () => {
+ const session = createPiiTextTransformer(pii).createSession();
+ const first = session.transform(fragment('😀 [email protected], [email protected]'));
+ const second = session.transform(fragment('Next: [email protected] and [email protected]'));
+
+ expect(first).toEqual({
+ version: 1,
+ content: '😀 [EMAIL_1], [EMAIL_1]',
+ replacements: 2,
+ categories: [{ category: 'EMAIL', count: 2 }],
+ });
+ expect(second.content).toBe('Next: [EMAIL_2] and [EMAIL_1]');
+ expect(JSON.stringify(first.categories)).not.toContain('Alice');
+ expect(JSON.stringify(second.categories)).not.toContain('Bob');
+ const newTurn = createPiiTextTransformer(pii).createSession();
+ expect(newTurn.transform(fragment('[email protected]')).content).toBe('[EMAIL_1]');
+ });
+
+ it('never aliases a user-supplied placeholder to another private value', () => {
+ const session = createPiiTextTransformer(pii).createSession();
+ expect(session.transform(fragment('[EMAIL_1] [email protected]')).content).toBe(
+ '[EMAIL_1] [EMAIL_2]',
+ );
+ expect(() => session.transform(fragment('A forged [EMAIL_2] marker'))).toThrow(
+ PiiTransformationError,
+ );
+ });
+
+ it('reserves literal markers across selected fragments without sharing session state', () => {
+ const transformer = createPiiTextTransformer(pii);
+ const session = transformer.createSession();
+ const independent = transformer.createSession();
+
+ expect(session.transform(fragment('[EMAIL_1] [EMAIL_3]'))).toEqual({
+ version: 1,
+ content: '[EMAIL_1] [EMAIL_3]',
+ replacements: 0,
+ categories: [],
+ });
+ expect(session.transform(fragment('[email protected]')).content).toBe('[EMAIL_2]');
+ expect(independent.transform(fragment('[email protected]')).content).toBe('[EMAIL_1]');
+ expect(session.transform(fragment('[email protected] [email protected]')).content).toBe(
+ '[EMAIL_4] [EMAIL_2]',
+ );
+ expect(session.transform(fragment('[EMAIL_1]'))).toMatchObject({ replacements: 0 });
+ expect(() => session.transform(fragment('[EMAIL_4]'))).toThrow(PiiTransformationError);
+ });
+
+ it('still charges selected literal markers against the character budget', () => {
+ const session = createPiiTextTransformer({ ...pii, maxCharacters: 18 }).createSession();
+ expect(session.transform(fragment('[EMAIL_1]')).replacements).toBe(0);
+ expect(session.transform(fragment('[EMAIL_2]')).replacements).toBe(0);
+ expect(() => session.transform(fragment('[EMAIL_3]'))).toThrow(PiiTransformationError);
+ });
+
+ it('does not charge an excluded field against the selected character budget', () => {
+ const text = '[email protected]';
+ const session = createPiiTextTransformer({
+ ...pii,
+ maxCharacters: text.length,
+ }).createSession();
+ const excluded = fragment(text.repeat(100), { field: 'summary' });
+
+ expect(session.transform(excluded)).toEqual({
+ version: 1,
+ content: excluded.text,
+ replacements: 0,
+ categories: [],
+ });
+ expect(session.transform(fragment(text)).content).toBe('[EMAIL_1]');
+ });
+
+ it('neither validates nor reserves excluded markers before or after selected fragments', () => {
+ const session = createPiiTextTransformer(pii).createSession();
+ const excluded = fragment('[EMAIL_1] [EMAIL_2]', { field: 'summary' });
+
+ expect(session.transform(excluded).content).toBe(excluded.text);
+ expect(session.transform(fragment('[email protected]')).content).toBe('[EMAIL_1]');
+ expect(session.transform(excluded).content).toBe(excluded.text);
+ expect(session.transform(fragment('[email protected]')).content).toBe('[EMAIL_2]');
+ });
+
+ it('accepts clean and empty fragments at the match limit but rejects one more match', () => {
+ const session = createPiiTextTransformer({ ...pii, maxMatches: 1 }).createSession();
+
+ expect(session.transform(fragment('[email protected]')).replacements).toBe(1);
+ for (const text of ['No private details here.', '']) {
+ expect(session.transform(fragment(text))).toEqual({
+ version: 1,
+ content: text,
+ replacements: 0,
+ categories: [],
+ });
+ }
+ expect(() => session.transform(fragment('[email protected]'))).toThrow(PiiTransformationError);
+ });
+
+ it('redacts selected built-in credential headers without leaking the value', () => {
+ const session = createPiiTextTransformer({
+ action: 'redact',
+ starterPatterns: ['bearer_header'],
+ }).createSession();
+ const result = session.transform(fragment('Authorization: bEaReR contract-token'));
+ expect(result.content).toBe('Authorization: [CREDENTIAL_1]');
+ expect(JSON.stringify(result.categories)).not.toContain('contract-token');
+ });
+
+ it('merges overlapping patterns so partial sensitive values never survive', () => {
+ const config = {
+ action: 'redact' as const,
+ starterPatterns: [],
+ customPatterns: [
+ { id: 'early', label: 'early', regex: 'ORG-SECRET', category: 'credential' as const },
+ { id: 'later', label: 'later', regex: 'SECRET-DATA', category: 'custom' as const },
+ ],
+ };
+ const result = createPiiTextTransformer(config)
+ .createSession()
+ .transform(fragment('prefix ORG-SECRET-DATA suffix'));
+ expect(result.content).toBe('prefix [CREDENTIAL_1] suffix');
+ expect(result.replacements).toBe(1);
+ expect(result.content).not.toContain('SECRET');
+ });
+
+ it('respects configured fields without attempting to replace other fields', () => {
+ const session = createPiiTextTransformer(pii).createSession();
+ expect(session.transform(fragment('[email protected]', { field: 'summary' })).content).toBe(
+ );
+ expect(session.transform(fragment('[email protected]')).content).toBe('[EMAIL_1]');
+ });
+
+ it('fails closed for non-replaceable matches and structured or URI content', () => {
+ for (const extras of [
+ { treatment: 'inspect_only' as const },
+ { format: 'json' as const },
+ { format: 'uri' as const },
+ ]) {
+ expect(() =>
+ createPiiTextTransformer(pii)
+ .createSession()
+ .transform(fragment('[email protected]', extras)),
+ ).toThrow(PiiTransformationError);
+ }
+ });
+
+ it('bounds characters and matches across all fragments in one session', () => {
+ const config = {
+ ...pii,
+ maxCharacters: 50,
+ maxMatches: 2,
+ };
+ const session = createPiiTextTransformer(config).createSession();
+ expect(session.transform(fragment('[email protected] [email protected]')).replacements).toBe(2);
+ expect(() => session.transform(fragment('[email protected]'))).toThrow(PiiTransformationError);
+ expect(() =>
+ createPiiTextTransformer({ ...config, maxCharacters: 4 })
+ .createSession()
+ .transform(fragment('[email protected]')),
+ ).toThrow(PiiTransformationError);
+ });
+
+ it('rejects empty regex matches, invalid limits and missing redact action without echoing input', () => {
+ expect(() => createPiiTextTransformer({ ...pii, maxMatches: -1 })).toThrow();
+ expect(() => createPiiTextTransformer({ ...pii, maxMatches: 0 })).toThrow();
+ expect(() => createPiiTextTransformer({ ...pii, action: 'block' })).toThrow();
+ const session = createPiiTextTransformer({
+ action: 'redact',
+ starterPatterns: [],
+ customPatterns: [{ id: 'empty', label: 'empty', regex: 'a*' }],
+ }).createSession();
+ let failure: Error | undefined;
+ try {
+ session.transform(fragment('secret value'));
+ } catch (error) {
+ failure = error as Error;
+ }
+ expect(failure).toBeInstanceOf(PiiTransformationError);
+ expect(failure?.message).not.toContain('secret value');
+ });
+});
diff --git a/packages/api/src/protection/transform.ts b/packages/api/src/protection/transform.ts
new file mode 100644
index 0000000000..ddfd9695ae
--- /dev/null
+++ b/packages/api/src/protection/transform.ts
@@ -0,0 +1,184 @@
+import {
+ DEFAULT_PII_REDACTION_MAX_CHARACTERS,
+ DEFAULT_PII_REDACTION_MAX_MATCHES,
+ MAX_PII_PATTERNS_PER_SOURCE,
+} from 'librechat-data-provider';
+import type { FilterPiiAction, FilterPiiCategory } from 'librechat-data-provider';
+import type { PatternContentInspectorConfig, PatternTextMatch } from './detectors/pattern';
+import type { TextContentFragment } from './types';
+import { createPatternContentInspector, PatternConfigurationError } from './detectors/pattern';
+
+export type PiiPlaceholderType = Uppercase<FilterPiiCategory>;
+
+export interface PiiRedactionConfig extends PatternContentInspectorConfig {
+ readonly action?: FilterPiiAction;
+ readonly fields?: readonly string[];
+ readonly maxCharacters?: number;
+ readonly maxMatches?: number;
+}
+
+export interface PiiRedactionMetadata {
+ readonly category: PiiPlaceholderType;
+ readonly count: number;
+}
+
+export interface PiiTransformationResult {
+ readonly version: 1;
+ readonly content: string;
+ readonly replacements: number;
+ readonly categories: readonly PiiRedactionMetadata[];
+}
+
+export class PiiTransformationError extends Error {
+ constructor(reason: 'limit' | 'unreplaceable' | 'inspection') {
+ super(`PII transformation failed: ${reason}`);
+ this.name = 'PiiTransformationError';
+ }
+}
+
+export interface PiiTransformationSession {
+ /** Applies this rule to selected text fields; the caller must still enforce all other policies. */
+ transform(fragment: TextContentFragment): PiiTransformationResult;
+}
+
+function readLimit(value: number | undefined, fallback: number, maximum: number): number {
+ if (value == null) {
+ return fallback;
+ }
+ if (!Number.isSafeInteger(value) || value <= 0 || value > maximum) {
+ throw new PatternConfigurationError('Invalid PII transformation limit');
+ }
+ return value;
+}
+
+function mergedMatches(matches: readonly PatternTextMatch[]): PatternTextMatch[] {
+ const sorted = [...matches].sort(
+ (left, right) =>
+ left.start - right.start ||
+ right.end - left.end ||
+ left.category.localeCompare(right.category),
+ );
+ const merged: PatternTextMatch[] = [];
+ for (const match of sorted) {
+ const previous = merged[merged.length - 1];
+ if (previous == null || match.start >= previous.end) {
+ merged.push(match);
+ } else if (match.end > previous.end) {
+ merged[merged.length - 1] = { ...previous, end: match.end };
+ }
+ }
+ return merged;
+}
+
+/**
+ * Explicit, opt-in transformation. Legacy inspectors still reject `redact` findings
+ * until the authenticated submission and storage pipeline consumes this API.
+ * A session owns only ephemeral, bounded mappings; never log or persist it.
+ */
+export function createPiiTextTransformer(config: PiiRedactionConfig): {
+ createSession(): PiiTransformationSession;
+} {
+ if (config.action !== 'redact') {
+ throw new PatternConfigurationError('PII transformation requires the redact action');
+ }
+ const maxCharacters = readLimit(
+ config.maxCharacters,
+ DEFAULT_PII_REDACTION_MAX_CHARACTERS,
+ 262_144,
+ );
+ const maxMatches = readLimit(config.maxMatches, DEFAULT_PII_REDACTION_MAX_MATCHES, 4_096);
+ let fields: ReadonlySet<string> | undefined;
+ if (config.fields != null) {
+ try {
+ if (!Array.isArray(config.fields) || config.fields.length > MAX_PII_PATTERNS_PER_SOURCE) {
+ throw new PatternConfigurationError('Invalid PII transformation fields');
+ }
+ fields = new Set(config.fields);
+ if (fields.size === 0 || [...fields].some((field) => typeof field !== 'string')) {
+ throw new PatternConfigurationError('Invalid PII transformation fields');
+ }
+ } catch {
+ throw new PatternConfigurationError('Invalid PII transformation fields');
+ }
+ }
+ const inspector = createPatternContentInspector(config, { linearTime: true });
+
+ return {
+ createSession() {
+ const placeholders = new Map<string, string>();
+ const issued = new Set<string>();
+ const reserved = new Set<string>();
+ const nextByCategory = new Map<PiiPlaceholderType, number>();
+ let remainingCharacters = maxCharacters;
+ let remainingMatches = maxMatches;
+ return {
+ transform(fragment) {
+ let matches: readonly PatternTextMatch[];
+ try {
+ if (fields != null && !fields.has(fragment.field)) {
+ return { version: 1, content: fragment.text, replacements: 0, categories: [] };
+ }
+ if (typeof fragment.text !== 'string' || fragment.text.length > remainingCharacters) {
+ throw new PiiTransformationError('limit');
+ }
+ remainingCharacters -= fragment.text.length;
+ for (const marker of fragment.text.matchAll(
+ /\[(?:EMAIL|PHONE|NAME|CREDENTIAL|CUSTOM)_[1-9]\d*\]/g,
+ )) {
+ reserved.add(marker[0]);
+ if (issued.has(marker[0])) {
+ throw new PiiTransformationError('inspection');
+ }
+ }
+ matches = inspector.locate(fragment.text, remainingMatches);
+ } catch (error) {
+ if (error instanceof PiiTransformationError) {
+ throw error;
+ }
+ throw new PiiTransformationError('inspection');
+ }
+ if (matches.length === 0) {
+ return { version: 1, content: fragment.text, replacements: 0, categories: [] };
+ }
+ if (
+ fragment.treatment !== 'replaceable' ||
+ !['plain', 'markdown'].includes(fragment.format)
+ ) {
+ throw new PiiTransformationError('unreplaceable');
+ }
+ const merged = mergedMatches(matches);
+ remainingMatches -= matches.length;
+ const categories = new Map<PiiPlaceholderType, number>();
+ const parts: string[] = [];
+ let offset = 0;
+ for (const { start, end, category } of merged) {
+ const type = category.toUpperCase() as PiiPlaceholderType;
+ const value = fragment.text.slice(start, end);
+ const key = JSON.stringify([type, value]);
+ let placeholder = placeholders.get(key);
+ if (placeholder == null) {
+ let next = nextByCategory.get(type) ?? 0;
+ do {
+ next++;
+ placeholder = `[${type}_${next}]`;
+ } while (reserved.has(placeholder) || issued.has(placeholder));
+ nextByCategory.set(type, next);
+ placeholders.set(key, placeholder);
+ issued.add(placeholder);
+ }
+ categories.set(type, (categories.get(type) ?? 0) + 1);
+ parts.push(fragment.text.slice(offset, start), placeholder);
+ offset = end;
+ }
+ parts.push(fragment.text.slice(offset));
+ return {
+ version: 1,
+ content: parts.join(''),
+ replacements: merged.length,
+ categories: [...categories].map(([category, count]) => ({ category, count })),
+ };
+ },
+ };
+ },
+ };
+}
diff --git a/packages/data-provider/src/filters.spec.ts b/packages/data-provider/src/filters.spec.ts
index c92df1ea8a..eaae6af784 100644
--- a/packages/data-provider/src/filters.spec.ts
+++ b/packages/data-provider/src/filters.spec.ts
@@ -57,7 +57,7 @@ describe('filtersConfigSchema', () => {
).toBe(false);
});
- it('accepts omitted, audit, and block actions while rejecting unsupported actions', () => {
+ it('accepts opt-in redact, audit, and block actions while rejecting unsupported actions', () => {
expect(filtersConfigSchema.parse({ messages: { pii: {} } })).toEqual({
messages: { pii: {} },
});
@@ -71,8 +71,44 @@ describe('filtersConfigSchema', () => {
prompts: { pii: { action: 'block' } },
});
expect(filtersConfigSchema.safeParse({ messages: { pii: { action: 'redact' } } }).success).toBe(
+ true,
+ );
+ expect(
+ filtersConfigSchema.safeParse({ messages: { pii: { action: 'forward' } } }).success,
+ ).toBe(false);
+ });
+
+ it('bounds opt-in transformation controls and validates placeholder categories', () => {
+ expect(filtersConfigSchema.parse({ messages: { pii: { action: 'redact' } } })).toEqual({
+ messages: { pii: { action: 'redact' } },
+ });
+ expect(filtersConfigSchema.safeParse({ messages: { pii: { maxCharacters: 0 } } }).success).toBe(
false,
);
+ expect(
+ filtersConfigSchema.safeParse({ messages: { pii: { maxMatches: 4_097 } } }).success,
+ ).toBe(false);
+ expect(
+ filtersConfigSchema.safeParse({
+ messages: {
+ pii: {
+ action: 'redact',
+ customPatterns: [{ id: 'email', label: 'Email', regex: 'a@b', category: 'email' }],
+ },
+ },
+ }).success,
+ ).toBe(true);
+ expect(
+ filtersConfigSchema.safeParse({
+ messages: {
+ pii: {
+ customPatterns: [
+ { id: 'email', label: 'Email', regex: 'a@b', category: 'secret-value' },
+ ],
+ },
+ },
+ }).success,
+ ).toBe(false);
});
it('combines active-pattern and selected-field checks', () => {
diff --git a/packages/data-provider/src/filters.ts b/packages/data-provider/src/filters.ts
index 9d010c727e..4368777d19 100644
--- a/packages/data-provider/src/filters.ts
+++ b/packages/data-provider/src/filters.ts
@@ -162,7 +162,10 @@ export const fileFilterFieldSchema = z.enum(FILE_FILTER_FIELDS);
export const toolArgumentFilterFieldSchema = z.enum(TOOL_ARGUMENT_FILTER_FIELDS);
export const modelParameterFilterFieldSchema = z.enum(MODEL_PARAMETER_FILTER_FIELDS);
export const filterPiiStarterPatternSchema = z.enum(FILTER_PII_STARTER_PATTERNS);
-export const filterPiiActionSchema = z.enum(['block', 'audit']);
+export const filterPiiActionSchema = z.enum(['block', 'audit', 'redact']);
+export const filterPiiCategorySchema = z.enum(['email', 'phone', 'name', 'credential', 'custom']);
+export const DEFAULT_PII_REDACTION_MAX_CHARACTERS = 65_536;
+export const DEFAULT_PII_REDACTION_MAX_MATCHES = 256;
export const actionMetadataFilterFieldSchema = z.enum(ACTION_METADATA_FILTER_FIELDS);
export const unattributedAssistantContentSchema = z.enum(['model_output', 'inspect']);
export type UnattributedAssistantContent = z.infer<typeof unattributedAssistantContentSchema>;
@@ -180,6 +183,7 @@ export type ToolArgumentFilterField = z.infer<typeof toolArgumentFilterFieldSche
export type ModelParameterFilterField = z.infer<typeof modelParameterFilterFieldSchema>;
export type ActionMetadataFilterField = z.infer<typeof actionMetadataFilterFieldSchema>;
export type FilterPiiAction = z.infer<typeof filterPiiActionSchema>;
+export type FilterPiiCategory = z.infer<typeof filterPiiCategorySchema>;
export const userSubmittedMessageFieldPathSchema = z
.object({
@@ -279,6 +283,7 @@ export const filterPiiCustomPatternSchema = z
id: z.string().min(1).max(MAX_PII_PATTERN_ID_LENGTH),
label: z.string().min(1).max(MAX_PII_PATTERN_LABEL_LENGTH),
regex: filterPiiRegexSchema,
+ category: filterPiiCategorySchema.optional(),
})
.strict();
@@ -288,6 +293,8 @@ function createPiiFilterSchema<Field extends z.ZodTypeAny>(fieldSchema: Field) {
return z
.object({
action: filterPiiActionSchema.optional(),
+ maxCharacters: z.number().int().positive().max(262_144).optional(),
+ maxMatches: z.number().int().positive().max(4_096).optional(),
fields: z.array(fieldSchema).min(1).max(MAX_PII_PATTERNS_PER_SOURCE).optional(),
starterPatterns: z
.array(filterPiiStarterPatternSchema)