Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active
TypeScript aianthropicartifactsawsazurechatgptchatgpt-cloneclaudeclonedeepseekgeminigooglegpt-5librechatmcpo1openairesponses-apivisionwebui
🌫️ feat: Add Bounded PII Text Transformation (#16295)
lia-by-librechat[bot]7d5cbc9ebf49
diff --git a/librechat.example.yaml b/librechat.example.yamlindex e407c300dd..3f25483564 100644--- a/librechat.example.yaml+++ b/librechat.example.yaml@@ -1390,7 +1390,11 @@ endpoints:# messages:# unattributedAssistantContent: model_output # `model_output` (default) or `inspect`# pii:-# action: audit # `block` (default) or `audit`+# action: audit # `block` (default), `audit`, or `redact` (reserved for transformation-aware ingress; current routes still block)+# # `redact` requires the later encrypted owner-view and canonical-message integration.+# # maxCharacters: 65536 # bounded total inspected text per transformation session+# # maxMatches: 256 # bounded matches per transformation session+# # Custom patterns may set category: email, phone, name, credential, or custom.# fields: [name, text, summary, quote, answer, decision_response, decision_reason, content_part, attachment_reference, assembled_context]# starterPatterns: [sk_prefix, bearer_header, api_key_header]# customPatterns:diff --git a/packages/api/src/protection/detectors/pattern.spec.ts b/packages/api/src/protection/detectors/pattern.spec.tsindex 992eb6e593..cc84a1a6ad 100644--- a/packages/api/src/protection/detectors/pattern.spec.ts+++ b/packages/api/src/protection/detectors/pattern.spec.ts@@ -72,6 +72,32 @@ describe('pattern content inspector', () => {).toMatchObject({ ruleId: 'bearer_header' });});+ it.each([+ { config: { starterPatterns: ['bearer_header'] }, text: 'Bearer contract-token' },+ {+ config: {+ starterPatterns: [],+ customPatterns: [{ id: 'custom', label: 'Custom', regex: 'ORG-[A-Z]+' }],+ },+ text: 'ORG-SECRET',+ },+ ])('enforces a zero remaining allowance when locating $text', ({ config, text }) => {+ const inspector = createPatternContentInspector(config, { linearTime: true });++ expect(inspector.locate('clean text', 0)).toEqual([]);+ expect(inspector.locate('', 0)).toEqual([]);+ expect(() => inspector.locate(text, 0)).toThrow(PatternConfigurationError);+ expect(inspector.locate(text, 1)).toHaveLength(1);+ });++ it.each([-1, 0.5, Number.NaN, Number.POSITIVE_INFINITY, Number.MAX_SAFE_INTEGER + 1])(+ 'rejects invalid remaining match allowance %s before inspecting',+ (allowance) => {+ const inspector = createPatternContentInspector({}, { linearTime: true });+ expect(() => inspector.locate('clean text', allowance)).toThrow(PatternConfigurationError);+ },+ );+it('memoizes a compiled set by config identity and memory limit', () => {const compile = jest.spyOn(RE2Set.prototype, 'compile');const config: MessageFilterPiiConfig = {diff --git a/packages/api/src/protection/detectors/pattern.ts b/packages/api/src/protection/detectors/pattern.tsindex 373028a3a2..6456b931be 100644--- a/packages/api/src/protection/detectors/pattern.ts+++ b/packages/api/src/protection/detectors/pattern.ts@@ -1,4 +1,4 @@-import { RE2Set } from 're2js';+import { RE2JS, RE2Set } from 're2js';import { logger } from '@librechat/data-schemas';import {MAX_PII_CUSTOM_REGEX_CHARACTERS,@@ -9,23 +9,25 @@ import {MAX_PII_PATTERNS_PER_SOURCE,getPiiRegexProgramSize,} from 'librechat-data-provider';-import type { MessageFilterPiiConfig, FilterPiiCustomPatternConfig } from 'librechat-data-provider';+import type {+ FilterPiiCategory,+ MessageFilterPiiConfig,+ FilterPiiCustomPatternConfig,+} from 'librechat-data-provider';import type { ProtectionFinding, TextContentFragment } from '../types';-interface TestablePattern {- test(input: string): boolean;-}-interface CompiledPattern {readonly id: string;readonly label: string;- readonly pattern: TestablePattern;+ readonly pattern: RegExp;+ readonly category: FilterPiiCategory;}interface PreparedCustomPattern {readonly id: string;readonly label: string;readonly regex: string;+ readonly category: FilterPiiCategory;}interface SnapshotPatternContentInspectorConfig {@@ -67,10 +69,18 @@ export interface PatternContentInspectorPreflightCost {readonly regexes: readonly string[];}+export interface PatternTextMatch {+ readonly start: number;+ readonly end: number;+ readonly category: FilterPiiCategory;+}+export interface PatternContentInspector {readonly active: boolean;inspectFragment(fragment: TextContentFragment): ProtectionFinding | null;inspect(fragments: Iterable<TextContentFragment>): ProtectionFinding | null;+ /** Internal match locations, never safe to return as public finding metadata. */+ locate(text: string, maxMatches: number): readonly PatternTextMatch[];}export interface PatternContentInspectorConfig {@@ -86,9 +96,24 @@ export interface PatternContentInspectorOptions {}const STARTER_PATTERNS: readonly CompiledPattern[] = [- { id: 'sk_prefix', label: 'sk- prefix token', pattern: /\b(sk-)[a-zA-Z0-9_-]+/ },- { id: 'bearer_header', label: 'Bearer token', pattern: /\b(Bearer )[^\s"']+/i },- { id: 'api_key_header', label: 'api-key header', pattern: /\b(api-key:?\s+)[^\s"']+/i },+ {+ id: 'sk_prefix',+ label: 'sk- prefix token',+ pattern: /\b(sk-)[a-zA-Z0-9_-]+/,+ category: 'credential',+ },+ {+ id: 'bearer_header',+ label: 'Bearer token',+ pattern: /\b(Bearer )[^\s"']+/i,+ category: 'credential',+ },+ {+ id: 'api_key_header',+ label: 'api-key header',+ pattern: /\b(api-key:?\s+)[^\s"']+/i,+ category: 'credential',+ },];const STARTER_BY_ID = new Map(STARTER_PATTERNS.map((pattern) => [pattern.id, pattern]));@@ -201,7 +226,25 @@ function readCustomPattern(candidate: unknown, index: number): FilterPiiCustomPaif (typeof regex !== 'string' || regex.length === 0 || regex.length > MAX_PII_PATTERN_LENGTH) {throw configurationError(`customPatterns[${index}].regex is invalid`);}- return { id, label, regex };+ let category: unknown;+ try {+ category = (candidate as FilterPiiCustomPatternConfig).category;+ } catch {+ throw configurationError(`customPatterns[${index}] could not be read safely`);+ }+ if (+ category != null &&+ (typeof category !== 'string' ||+ !['email', 'phone', 'name', 'credential', 'custom'].includes(category))+ ) {+ throw configurationError(`customPatterns[${index}].category is invalid`);+ }+ return {+ id,+ label,+ regex,+ ...(category == null ? {} : { category: category as FilterPiiCategory }),+ };}function snapshotConfig(@@ -240,7 +283,7 @@ function snapshotConfig(throw configurationError('customPatterns could not be read safely');}const pattern = readCustomPattern(patternCandidate, index);- custom.push(pattern);+ custom.push({ ...pattern, category: pattern.category ?? 'custom' });regexes.push(pattern.regex);regexCharacters += pattern.regex.length;if (regexCharacters > MAX_PII_CUSTOM_REGEX_CHARACTERS) {@@ -357,10 +400,17 @@ function createSequentialInspector(patterns: readonly CompiledPattern[]): Patterfunction createInspector(active: boolean,inspectFragment: (fragment: TextContentFragment) => ProtectionFinding | null,+ locate?: (text: string, maxMatches: number) => readonly PatternTextMatch[],): PatternContentInspector {return {active,inspectFragment,+ locate(text, maxMatches) {+ if (locate == null) {+ throw configurationError('redaction requires a linear-time pattern inspector');+ }+ return locate(text, maxMatches);+ },inspect(fragments) {for (const fragment of fragments) {const finding = inspectFragment(fragment);@@ -416,7 +466,44 @@ function createLinearInspector(return pattern == null ? null : findingFor(pattern, fragment);};- return createInspector(prepared.cost.active, inspectFragment);+ const compiled = new Map<string, RE2JS>();+ const locate = (text: string, maxMatches: number): readonly PatternTextMatch[] => {+ if (!Number.isSafeInteger(maxMatches) || maxMatches < 0) {+ throw configurationError('redaction match limit must be a non-negative safe integer');+ }+ const matches: PatternTextMatch[] = [];+ const selected = customSet == null ? [] : customSet.match(text);+ const candidates: Array<readonly [string, FilterPiiCategory]> = prepared.starter.map(+ (pattern) => [+ `${pattern.pattern.ignoreCase ? '(?i)' : ''}${pattern.pattern.source}`,+ pattern.category,+ ],+ );+ for (const index of selected) {+ const pattern = prepared.custom[index];+ if (pattern != null) {+ candidates.push([pattern.regex, pattern.category]);+ }+ }+ for (const [expression, category] of candidates) {+ let regex = compiled.get(expression);+ if (regex == null) {+ regex = RE2JS.compile(expression);+ compiled.set(expression, regex);+ }+ const matcher = regex.matcher(text);+ while (matcher.find()) {+ const start = matcher.start();+ const end = matcher.end();+ if (end <= start || matches.length >= maxMatches) {+ throw configurationError('redaction match limit or empty match encountered');+ }+ matches.push({ start, end, category });+ }+ }+ return matches;+ };+ return createInspector(prepared.cost.active, inspectFragment, locate);}function cacheLinearInspector(@@ -480,6 +567,7 @@ export function createPatternContentInspector(id: pattern.id,label: pattern.label,pattern: new RegExp(pattern.regex),+ category: pattern.category,}));const inspector = createSequentialInspector([...prepared.starter, ...custom]);if (options.cacheResult !== false) {diff --git a/packages/api/src/protection/index.ts b/packages/api/src/protection/index.tsindex 18e325633a..264f49b381 100644--- a/packages/api/src/protection/index.ts+++ b/packages/api/src/protection/index.ts@@ -1,5 +1,6 @@export * from './types';export * from './runtime';+export * from './transform';export * from './title';export * from './legacy';export * from './provenance';diff --git a/packages/api/src/protection/transform.spec.ts b/packages/api/src/protection/transform.spec.tsnew file mode 100644index 0000000000..88022c2981--- /dev/null+++ b/packages/api/src/protection/transform.spec.ts@@ -0,0 +1,227 @@+import type { FiltersConfig } from 'librechat-data-provider';+import type { TextContentFragment } from './types';+import { createPiiTextTransformer, PiiTransformationError } from './transform';+import { inspectContent } from './runtime';++jest.mock('@librechat/data-schemas', () => ({+ logger: { warn: jest.fn(), error: jest.fn(), info: jest.fn(), debug: jest.fn() },+}));++const pii: NonNullable<NonNullable<FiltersConfig['messages']>['pii']> = {+ action: 'redact' as const,+ starterPatterns: [],+ fields: ['text'],+ customPatterns: [+ {+ id: 'email',+ label: 'Email address',+ regex: '(?i)[a-z]+@[a-z]+\\.[a-z]+',+ category: 'email' as const,+ },+ ],+};++function fragment(text: string, extras: Partial<TextContentFragment> = {}): TextContentFragment {+ return {+ id: 'chat.text',+ path: '/text',+ text,+ source: 'message',+ field: 'text',+ format: 'plain',+ treatment: 'replaceable',+ provenance: 'user',+ ...extras,+ } as TextContentFragment;+}++describe('opt-in PII text transformation', () => {+ it('leaves the existing inspection path blocking until a caller integrates transformation', () => {+ const filters: FiltersConfig = { messages: { pii } };+ expect(inspectContent([fragment('[email protected]')], { filters })).toMatchObject({+ detectorId: 'pii-pattern',+ ruleId: 'email',+ });+ });++ it('uses typed placeholders and returns no matched values in metadata', () => {+ const session = createPiiTextTransformer(pii).createSession();+ const first = session.transform(fragment('😀 [email protected], [email protected]'));+ const second = session.transform(fragment('Next: [email protected] and [email protected]'));++ expect(first).toEqual({+ version: 1,+ content: '😀 [EMAIL_1], [EMAIL_1]',+ replacements: 2,+ categories: [{ category: 'EMAIL', count: 2 }],+ });+ expect(second.content).toBe('Next: [EMAIL_2] and [EMAIL_1]');+ expect(JSON.stringify(first.categories)).not.toContain('Alice');+ expect(JSON.stringify(second.categories)).not.toContain('Bob');+ const newTurn = createPiiTextTransformer(pii).createSession();+ expect(newTurn.transform(fragment('[email protected]')).content).toBe('[EMAIL_1]');+ });++ it('never aliases a user-supplied placeholder to another private value', () => {+ const session = createPiiTextTransformer(pii).createSession();+ expect(session.transform(fragment('[EMAIL_1] [email protected]')).content).toBe(+ '[EMAIL_1] [EMAIL_2]',+ );+ expect(() => session.transform(fragment('A forged [EMAIL_2] marker'))).toThrow(+ PiiTransformationError,+ );+ });++ it('reserves literal markers across selected fragments without sharing session state', () => {+ const transformer = createPiiTextTransformer(pii);+ const session = transformer.createSession();+ const independent = transformer.createSession();++ expect(session.transform(fragment('[EMAIL_1] [EMAIL_3]'))).toEqual({+ version: 1,+ content: '[EMAIL_1] [EMAIL_3]',+ replacements: 0,+ categories: [],+ });+ expect(session.transform(fragment('[email protected]')).content).toBe('[EMAIL_2]');+ expect(independent.transform(fragment('[email protected]')).content).toBe('[EMAIL_1]');+ expect(session.transform(fragment('[email protected] [email protected]')).content).toBe(+ '[EMAIL_4] [EMAIL_2]',+ );+ expect(session.transform(fragment('[EMAIL_1]'))).toMatchObject({ replacements: 0 });+ expect(() => session.transform(fragment('[EMAIL_4]'))).toThrow(PiiTransformationError);+ });++ it('still charges selected literal markers against the character budget', () => {+ const session = createPiiTextTransformer({ ...pii, maxCharacters: 18 }).createSession();+ expect(session.transform(fragment('[EMAIL_1]')).replacements).toBe(0);+ expect(session.transform(fragment('[EMAIL_2]')).replacements).toBe(0);+ expect(() => session.transform(fragment('[EMAIL_3]'))).toThrow(PiiTransformationError);+ });++ it('does not charge an excluded field against the selected character budget', () => {+ const text = '[email protected]';+ const session = createPiiTextTransformer({+ ...pii,+ maxCharacters: text.length,+ }).createSession();+ const excluded = fragment(text.repeat(100), { field: 'summary' });++ expect(session.transform(excluded)).toEqual({+ version: 1,+ content: excluded.text,+ replacements: 0,+ categories: [],+ });+ expect(session.transform(fragment(text)).content).toBe('[EMAIL_1]');+ });++ it('neither validates nor reserves excluded markers before or after selected fragments', () => {+ const session = createPiiTextTransformer(pii).createSession();+ const excluded = fragment('[EMAIL_1] [EMAIL_2]', { field: 'summary' });++ expect(session.transform(excluded).content).toBe(excluded.text);+ expect(session.transform(fragment('[email protected]')).content).toBe('[EMAIL_1]');+ expect(session.transform(excluded).content).toBe(excluded.text);+ expect(session.transform(fragment('[email protected]')).content).toBe('[EMAIL_2]');+ });++ it('accepts clean and empty fragments at the match limit but rejects one more match', () => {+ const session = createPiiTextTransformer({ ...pii, maxMatches: 1 }).createSession();++ expect(session.transform(fragment('[email protected]')).replacements).toBe(1);+ for (const text of ['No private details here.', '']) {+ expect(session.transform(fragment(text))).toEqual({+ version: 1,+ content: text,+ replacements: 0,+ categories: [],+ });+ }+ expect(() => session.transform(fragment('[email protected]'))).toThrow(PiiTransformationError);+ });++ it('redacts selected built-in credential headers without leaking the value', () => {+ const session = createPiiTextTransformer({+ action: 'redact',+ starterPatterns: ['bearer_header'],+ }).createSession();+ const result = session.transform(fragment('Authorization: bEaReR contract-token'));+ expect(result.content).toBe('Authorization: [CREDENTIAL_1]');+ expect(JSON.stringify(result.categories)).not.toContain('contract-token');+ });++ it('merges overlapping patterns so partial sensitive values never survive', () => {+ const config = {+ action: 'redact' as const,+ starterPatterns: [],+ customPatterns: [+ { id: 'early', label: 'early', regex: 'ORG-SECRET', category: 'credential' as const },+ { id: 'later', label: 'later', regex: 'SECRET-DATA', category: 'custom' as const },+ ],+ };+ const result = createPiiTextTransformer(config)+ .createSession()+ .transform(fragment('prefix ORG-SECRET-DATA suffix'));+ expect(result.content).toBe('prefix [CREDENTIAL_1] suffix');+ expect(result.replacements).toBe(1);+ expect(result.content).not.toContain('SECRET');+ });++ it('respects configured fields without attempting to replace other fields', () => {+ const session = createPiiTextTransformer(pii).createSession();+ expect(session.transform(fragment('[email protected]', { field: 'summary' })).content).toBe(+ '[email protected]',+ );+ expect(session.transform(fragment('[email protected]')).content).toBe('[EMAIL_1]');+ });++ it('fails closed for non-replaceable matches and structured or URI content', () => {+ for (const extras of [+ { treatment: 'inspect_only' as const },+ { format: 'json' as const },+ { format: 'uri' as const },+ ]) {+ expect(() =>+ createPiiTextTransformer(pii)+ .createSession()+ .transform(fragment('[email protected]', extras)),+ ).toThrow(PiiTransformationError);+ }+ });++ it('bounds characters and matches across all fragments in one session', () => {+ const config = {+ ...pii,+ maxCharacters: 50,+ maxMatches: 2,+ };+ const session = createPiiTextTransformer(config).createSession();+ expect(session.transform(fragment('[email protected] [email protected]')).replacements).toBe(2);+ expect(() => session.transform(fragment('[email protected]'))).toThrow(PiiTransformationError);+ expect(() =>+ createPiiTextTransformer({ ...config, maxCharacters: 4 })+ .createSession()+ .transform(fragment('[email protected]')),+ ).toThrow(PiiTransformationError);+ });++ it('rejects empty regex matches, invalid limits and missing redact action without echoing input', () => {+ expect(() => createPiiTextTransformer({ ...pii, maxMatches: -1 })).toThrow();+ expect(() => createPiiTextTransformer({ ...pii, maxMatches: 0 })).toThrow();+ expect(() => createPiiTextTransformer({ ...pii, action: 'block' })).toThrow();+ const session = createPiiTextTransformer({+ action: 'redact',+ starterPatterns: [],+ customPatterns: [{ id: 'empty', label: 'empty', regex: 'a*' }],+ }).createSession();+ let failure: Error | undefined;+ try {+ session.transform(fragment('secret value'));+ } catch (error) {+ failure = error as Error;+ }+ expect(failure).toBeInstanceOf(PiiTransformationError);+ expect(failure?.message).not.toContain('secret value');+ });+});diff --git a/packages/api/src/protection/transform.ts b/packages/api/src/protection/transform.tsnew file mode 100644index 0000000000..ddfd9695ae--- /dev/null+++ b/packages/api/src/protection/transform.ts@@ -0,0 +1,184 @@+import {+ DEFAULT_PII_REDACTION_MAX_CHARACTERS,+ DEFAULT_PII_REDACTION_MAX_MATCHES,+ MAX_PII_PATTERNS_PER_SOURCE,+} from 'librechat-data-provider';+import type { FilterPiiAction, FilterPiiCategory } from 'librechat-data-provider';+import type { PatternContentInspectorConfig, PatternTextMatch } from './detectors/pattern';+import type { TextContentFragment } from './types';+import { createPatternContentInspector, PatternConfigurationError } from './detectors/pattern';++export type PiiPlaceholderType = Uppercase<FilterPiiCategory>;++export interface PiiRedactionConfig extends PatternContentInspectorConfig {+ readonly action?: FilterPiiAction;+ readonly fields?: readonly string[];+ readonly maxCharacters?: number;+ readonly maxMatches?: number;+}++export interface PiiRedactionMetadata {+ readonly category: PiiPlaceholderType;+ readonly count: number;+}++export interface PiiTransformationResult {+ readonly version: 1;+ readonly content: string;+ readonly replacements: number;+ readonly categories: readonly PiiRedactionMetadata[];+}++export class PiiTransformationError extends Error {+ constructor(reason: 'limit' | 'unreplaceable' | 'inspection') {+ super(`PII transformation failed: ${reason}`);+ this.name = 'PiiTransformationError';+ }+}++export interface PiiTransformationSession {+ /** Applies this rule to selected text fields; the caller must still enforce all other policies. */+ transform(fragment: TextContentFragment): PiiTransformationResult;+}++function readLimit(value: number | undefined, fallback: number, maximum: number): number {+ if (value == null) {+ return fallback;+ }+ if (!Number.isSafeInteger(value) || value <= 0 || value > maximum) {+ throw new PatternConfigurationError('Invalid PII transformation limit');+ }+ return value;+}++function mergedMatches(matches: readonly PatternTextMatch[]): PatternTextMatch[] {+ const sorted = [...matches].sort(+ (left, right) =>+ left.start - right.start ||+ right.end - left.end ||+ left.category.localeCompare(right.category),+ );+ const merged: PatternTextMatch[] = [];+ for (const match of sorted) {+ const previous = merged[merged.length - 1];+ if (previous == null || match.start >= previous.end) {+ merged.push(match);+ } else if (match.end > previous.end) {+ merged[merged.length - 1] = { ...previous, end: match.end };+ }+ }+ return merged;+}++/**+ * Explicit, opt-in transformation. Legacy inspectors still reject `redact` findings+ * until the authenticated submission and storage pipeline consumes this API.+ * A session owns only ephemeral, bounded mappings; never log or persist it.+ */+export function createPiiTextTransformer(config: PiiRedactionConfig): {+ createSession(): PiiTransformationSession;+} {+ if (config.action !== 'redact') {+ throw new PatternConfigurationError('PII transformation requires the redact action');+ }+ const maxCharacters = readLimit(+ config.maxCharacters,+ DEFAULT_PII_REDACTION_MAX_CHARACTERS,+ 262_144,+ );+ const maxMatches = readLimit(config.maxMatches, DEFAULT_PII_REDACTION_MAX_MATCHES, 4_096);+ let fields: ReadonlySet<string> | undefined;+ if (config.fields != null) {+ try {+ if (!Array.isArray(config.fields) || config.fields.length > MAX_PII_PATTERNS_PER_SOURCE) {+ throw new PatternConfigurationError('Invalid PII transformation fields');+ }+ fields = new Set(config.fields);+ if (fields.size === 0 || [...fields].some((field) => typeof field !== 'string')) {+ throw new PatternConfigurationError('Invalid PII transformation fields');+ }+ } catch {+ throw new PatternConfigurationError('Invalid PII transformation fields');+ }+ }+ const inspector = createPatternContentInspector(config, { linearTime: true });++ return {+ createSession() {+ const placeholders = new Map<string, string>();+ const issued = new Set<string>();+ const reserved = new Set<string>();+ const nextByCategory = new Map<PiiPlaceholderType, number>();+ let remainingCharacters = maxCharacters;+ let remainingMatches = maxMatches;+ return {+ transform(fragment) {+ let matches: readonly PatternTextMatch[];+ try {+ if (fields != null && !fields.has(fragment.field)) {+ return { version: 1, content: fragment.text, replacements: 0, categories: [] };+ }+ if (typeof fragment.text !== 'string' || fragment.text.length > remainingCharacters) {+ throw new PiiTransformationError('limit');+ }+ remainingCharacters -= fragment.text.length;+ for (const marker of fragment.text.matchAll(+ /\[(?:EMAIL|PHONE|NAME|CREDENTIAL|CUSTOM)_[1-9]\d*\]/g,+ )) {+ reserved.add(marker[0]);+ if (issued.has(marker[0])) {+ throw new PiiTransformationError('inspection');+ }+ }+ matches = inspector.locate(fragment.text, remainingMatches);+ } catch (error) {+ if (error instanceof PiiTransformationError) {+ throw error;+ }+ throw new PiiTransformationError('inspection');+ }+ if (matches.length === 0) {+ return { version: 1, content: fragment.text, replacements: 0, categories: [] };+ }+ if (+ fragment.treatment !== 'replaceable' ||+ !['plain', 'markdown'].includes(fragment.format)+ ) {+ throw new PiiTransformationError('unreplaceable');+ }+ const merged = mergedMatches(matches);+ remainingMatches -= matches.length;+ const categories = new Map<PiiPlaceholderType, number>();+ const parts: string[] = [];+ let offset = 0;+ for (const { start, end, category } of merged) {+ const type = category.toUpperCase() as PiiPlaceholderType;+ const value = fragment.text.slice(start, end);+ const key = JSON.stringify([type, value]);+ let placeholder = placeholders.get(key);+ if (placeholder == null) {+ let next = nextByCategory.get(type) ?? 0;+ do {+ next++;+ placeholder = `[${type}_${next}]`;+ } while (reserved.has(placeholder) || issued.has(placeholder));+ nextByCategory.set(type, next);+ placeholders.set(key, placeholder);+ issued.add(placeholder);+ }+ categories.set(type, (categories.get(type) ?? 0) + 1);+ parts.push(fragment.text.slice(offset, start), placeholder);+ offset = end;+ }+ parts.push(fragment.text.slice(offset));+ return {+ version: 1,+ content: parts.join(''),+ replacements: merged.length,+ categories: [...categories].map(([category, count]) => ({ category, count })),+ };+ },+ };+ },+ };+}diff --git a/packages/data-provider/src/filters.spec.ts b/packages/data-provider/src/filters.spec.tsindex c92df1ea8a..eaae6af784 100644--- a/packages/data-provider/src/filters.spec.ts+++ b/packages/data-provider/src/filters.spec.ts@@ -57,7 +57,7 @@ describe('filtersConfigSchema', () => {).toBe(false);});- it('accepts omitted, audit, and block actions while rejecting unsupported actions', () => {+ it('accepts opt-in redact, audit, and block actions while rejecting unsupported actions', () => {expect(filtersConfigSchema.parse({ messages: { pii: {} } })).toEqual({messages: { pii: {} },});@@ -71,8 +71,44 @@ describe('filtersConfigSchema', () => {prompts: { pii: { action: 'block' } },});expect(filtersConfigSchema.safeParse({ messages: { pii: { action: 'redact' } } }).success).toBe(+ true,+ );+ expect(+ filtersConfigSchema.safeParse({ messages: { pii: { action: 'forward' } } }).success,+ ).toBe(false);+ });++ it('bounds opt-in transformation controls and validates placeholder categories', () => {+ expect(filtersConfigSchema.parse({ messages: { pii: { action: 'redact' } } })).toEqual({+ messages: { pii: { action: 'redact' } },+ });+ expect(filtersConfigSchema.safeParse({ messages: { pii: { maxCharacters: 0 } } }).success).toBe(false,);+ expect(+ filtersConfigSchema.safeParse({ messages: { pii: { maxMatches: 4_097 } } }).success,+ ).toBe(false);+ expect(+ filtersConfigSchema.safeParse({+ messages: {+ pii: {+ action: 'redact',+ customPatterns: [{ id: 'email', label: 'Email', regex: 'a@b', category: 'email' }],+ },+ },+ }).success,+ ).toBe(true);+ expect(+ filtersConfigSchema.safeParse({+ messages: {+ pii: {+ customPatterns: [+ { id: 'email', label: 'Email', regex: 'a@b', category: 'secret-value' },+ ],+ },+ },+ }).success,+ ).toBe(false);});it('combines active-pattern and selected-field checks', () => {diff --git a/packages/data-provider/src/filters.ts b/packages/data-provider/src/filters.tsindex 9d010c727e..4368777d19 100644--- a/packages/data-provider/src/filters.ts+++ b/packages/data-provider/src/filters.ts@@ -162,7 +162,10 @@ export const fileFilterFieldSchema = z.enum(FILE_FILTER_FIELDS);export const toolArgumentFilterFieldSchema = z.enum(TOOL_ARGUMENT_FILTER_FIELDS);export const modelParameterFilterFieldSchema = z.enum(MODEL_PARAMETER_FILTER_FIELDS);export const filterPiiStarterPatternSchema = z.enum(FILTER_PII_STARTER_PATTERNS);-export const filterPiiActionSchema = z.enum(['block', 'audit']);+export const filterPiiActionSchema = z.enum(['block', 'audit', 'redact']);+export const filterPiiCategorySchema = z.enum(['email', 'phone', 'name', 'credential', 'custom']);+export const DEFAULT_PII_REDACTION_MAX_CHARACTERS = 65_536;+export const DEFAULT_PII_REDACTION_MAX_MATCHES = 256;export const actionMetadataFilterFieldSchema = z.enum(ACTION_METADATA_FILTER_FIELDS);export const unattributedAssistantContentSchema = z.enum(['model_output', 'inspect']);export type UnattributedAssistantContent = z.infer<typeof unattributedAssistantContentSchema>;@@ -180,6 +183,7 @@ export type ToolArgumentFilterField = z.infer<typeof toolArgumentFilterFieldScheexport type ModelParameterFilterField = z.infer<typeof modelParameterFilterFieldSchema>;export type ActionMetadataFilterField = z.infer<typeof actionMetadataFilterFieldSchema>;export type FilterPiiAction = z.infer<typeof filterPiiActionSchema>;+export type FilterPiiCategory = z.infer<typeof filterPiiCategorySchema>;export const userSubmittedMessageFieldPathSchema = z.object({@@ -279,6 +283,7 @@ export const filterPiiCustomPatternSchema = zid: z.string().min(1).max(MAX_PII_PATTERN_ID_LENGTH),label: z.string().min(1).max(MAX_PII_PATTERN_LABEL_LENGTH),regex: filterPiiRegexSchema,+ category: filterPiiCategorySchema.optional(),}).strict();@@ -288,6 +293,8 @@ function createPiiFilterSchema<Field extends z.ZodTypeAny>(fieldSchema: Field) {return z.object({action: filterPiiActionSchema.optional(),+ maxCharacters: z.number().int().positive().max(262_144).optional(),+ maxMatches: z.number().int().positive().max(4_096).optional(),fields: z.array(fieldSchema).min(1).max(MAX_PII_PATTERNS_PER_SOURCE).optional(),starterPatterns: z.array(filterPiiStarterPatternSchema)