Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(host): redact JSON tool output by string ranges so it stays parseable

Alfredo Lopez4d8c5e94d0cb
diff --git a/mcp-host/package-lock.json b/mcp-host/package-lock.json
index 5084bf3fe..9a07b3343 100644
--- a/mcp-host/package-lock.json
+++ b/mcp-host/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-mcp-host",
- "version": "0.1.491",
+ "version": "0.1.492",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-mcp-host",
- "version": "0.1.491",
+ "version": "0.1.492",
"license": "SEE LICENSE IN LICENSE",
"dependencies": {
"@anthropic-ai/sdk": "^0.32.0",
diff --git a/mcp-host/package.json b/mcp-host/package.json
index b4e1ff97d..894f4eb5d 100644
--- a/mcp-host/package.json
+++ b/mcp-host/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-mcp-host",
- "version": "0.1.491",
+ "version": "0.1.492",
"description": "MCP Host - reads Host CRD and provides LLM access via OpenAI/Claude",
"main": "dist/main.js",
"engines": {
diff --git a/mcp-host/src/core/extensions/__tests__/prePrune.attachmentPageCollapse.test.ts b/mcp-host/src/core/extensions/__tests__/prePrune.attachmentPageCollapse.test.ts
index a068fa97a..522f344da 100644
--- a/mcp-host/src/core/extensions/__tests__/prePrune.attachmentPageCollapse.test.ts
+++ b/mcp-host/src/core/extensions/__tests__/prePrune.attachmentPageCollapse.test.ts
@@ -421,6 +421,64 @@ describe('C17 prePrune — real attachment_read producer (M1, L14)', () => {
})
}
+ it('collapses an env-dump page whose PWD= match runs across its newline escapes', async () => {
+ const head = 'PWD=/app\nHOME=/root\nPORT=5432'
+ const fileText = `${head}\nUSER=app\n`
+ const tool = readerFor(Buffer.from(fileText))
+ const registry: ToolRegistry = {
+ get: name => (name === tool.name() ? tool : null),
+ listDefinitions: () => [],
+ register: () => undefined,
+ }
+ const result = await executeSingleTool(
+ {
+ id: 'tc_env_dump',
+ name: 'clerum__attachment_read',
+ arguments: { attachmentId: ATTACHMENT_ID, maxBytes: Buffer.byteLength(head) },
+ },
+ {
+ toolRegistry: registry,
+ toolOutputProcessor: new DefaultToolOutputProcessor(safety),
+ safety,
+ events: new SimpleEventEmitter(),
+ toolTimeout: 1000,
+ progressReporter: undefined,
+ toolProgressInterval: 0,
+ measureToolMessage: message =>
+ Math.max(Math.ceil(Buffer.byteLength(message.content ?? '', 'utf8') / 4) + 4, 1),
+ }
+ )
+
+ expect(result.is_error).toBe(false)
+ expect(result.content.startsWith(OPEN)).toBe(true)
+ expect(result.content.endsWith(CLOSE)).toBe(true)
+ const inner = result.content.slice(OPEN.length, -CLOSE.length)
+ expect(parsesAsJson(inner)).toBe(true)
+ expect(JSON.parse(inner).nextOffset).toBe(Buffer.byteLength(head))
+ expect(inner).not.toContain('HOME=/root')
+
+ const messages: ChatMessage[] = [
+ userMsg('Review the attached config'),
+ assistantToolCall('tc_env_dump', 'clerum__attachment_read', { attachmentId: ATTACHMENT_ID }),
+ { role: 'tool', tool_call_id: 'tc_env_dump', name: result.name, content: result.content },
+ userMsg('next turn'),
+ ]
+ const pruned = prePrune(messages, OPTIONS, PRESSURE_ON)
+ expect(pruned.passesApplied).toContain('attachment_page_collapse')
+ const stub = JSON.parse(pruned.messages[2].content.slice(OPEN.length, -CLOSE.length))
+ expect(stub).toEqual({
+ attachmentId: ATTACHMENT_ID,
+ referenceId: expect.any(String),
+ kind: 'text',
+ byteRange: { offset: 0, length: Buffer.byteLength(head) },
+ truncated: true,
+ limit: 'max_bytes',
+ nextOffset: Buffer.byteLength(head),
+ text: stubMarker(),
+ })
+ expect(() => validateToolLinkages(pruned.messages)).not.toThrow()
+ })
+
it('collapses a page whose text ends inside password=<value> and keeps its paging fields', async () => {
const head = 'DB_HOST=db.internal\npassword=supersecret99'
const fileText = `${head}\nPORT=5432\n`
diff --git a/mcp-host/src/core/safety/__tests__/safety.test.ts b/mcp-host/src/core/safety/__tests__/safety.test.ts
index 84b7a0940..bf1331e04 100644
--- a/mcp-host/src/core/safety/__tests__/safety.test.ts
+++ b/mcp-host/src/core/safety/__tests__/safety.test.ts
@@ -165,11 +165,13 @@ describe('BasicSafety', () => {
})
})
- it('sanitizes keys with the same rules on the leaf-wise path', () => {
+ it('redacts keys too, and every string a match runs into', () => {
+ // The text pass reads `password=supersecret99":"v"` as one credential, so
+ // both the key and the value it runs into are redacted.
const output = '{"password=supersecret99":"v","n":"password=abcdefghij"}'
const result = safety.sanitizeOutput('mcp__config', output)
- expect(result.content).toBe('{"[REDACTED]":"v","n":"[REDACTED]"}')
- expect(JSON.parse(result.content)).toEqual({ '[REDACTED]': 'v', n: '[REDACTED]' })
+ expect(result.content).toBe('{"[REDACTED]":"[REDACTED]","n":"[REDACTED]"}')
+ expect(JSON.parse(result.content)).toEqual({ '[REDACTED]': '[REDACTED]', n: '[REDACTED]' })
expect(result.content).not.toContain('supersecret99')
expect(result.content).not.toContain('abcdefghij')
})
@@ -190,26 +192,139 @@ describe('BasicSafety', () => {
})
})
- it('keeps the text-pass redaction when a match spans JSON syntax the leaves cannot see', () => {
- // The text pass reads `password:":"secret12345"}` as one credential; no
- // single key or value matches, so the leaf pass would leak the value.
+ it('redacts a key and the value a match spans across JSON syntax', () => {
+ // The text pass reads `password:":"secret12345"}` as one credential; the
+ // match is clipped to both strings it covers.
const output = '{"password:":"secret12345"}'
- const textPass = safety.sanitizeFreeformContent(output, { secretWarning: WARNING })
const result = safety.sanitizeOutput('clerum__attachment_read', output)
expect(result.was_modified).toBe(true)
- expect(result.content).toBe(textPass.content)
+ expect(result.warnings).toEqual([WARNING])
+ expect(result.content).toBe('{"[REDACTED]":"[REDACTED]"}')
expect(result.content).not.toContain('secret12345')
})
- it('keeps the text-pass redaction when redacted keys would collide', () => {
+ it('keeps both members when redacted keys become equal', () => {
const output = '{"pwd=aaaaaaaa1":"x","pwd=bbbbbbbb2":"y"}'
- const textPass = safety.sanitizeFreeformContent(output, { secretWarning: WARNING })
const result = safety.sanitizeOutput('mcp__config', output)
- expect(textPass.was_modified).toBe(true)
- expect(result.content).toBe(textPass.content)
+ // Witness: the output parses and still holds two members.
+ expect(result.content).toBe('{"[REDACTED]":"[REDACTED]","[REDACTED]":"[REDACTED]"}')
+ expect(result.content.match(/":"/g)).toHaveLength(2)
expect(result.content).not.toContain('aaaaaaaa1')
expect(result.content).not.toContain('bbbbbbbb2')
})
+
+ it('redacts an env dump written with raw \\n escapes and keeps the paging fields', () => {
+ const output = JSON.stringify({
+ attachmentId: 'att_1',
+ text: 'PWD=/app\nHOME=/root\nPORT=5432',
+ nextOffset: 31,
+ })
+ // Witness: the text pass alone breaks this output.
+ const textPass = safety.sanitizeFreeformContent(output, { secretWarning: WARNING })
+ expect(() => JSON.parse(textPass.content)).toThrow()
+
+ const result = safety.sanitizeOutput('clerum__attachment_read', output)
+ expect(result.was_modified).toBe(true)
+ expect(result.content).toBe('{"attachmentId":"att_1","text":"[REDACTED]","nextOffset":31}')
+ expect(JSON.parse(result.content)).toEqual({
+ attachmentId: 'att_1',
+ text: '[REDACTED]',
+ nextOffset: 31,
+ })
+ expect(result.content).not.toContain('HOME=/root')
+ })
+
+ it('redacts as much as the text pass when a separator is written as an escape', () => {
+ const token = 'SyntheticBearer0123456789'
+ const output = `{"a":"password=ab\\u002ccdefghijk","b":"Bearer ${token}"}`
+ expect(JSON.parse(output)).toEqual({ a: 'password=ab,cdefghijk', b: `Bearer ${token}` })
+
+ const result = safety.sanitizeOutput('mcp__config', output)
+ expect(result.content).toBe('{"a":"[REDACTED]","b":"[REDACTED]"}')
+ expect(JSON.parse(result.content)).toEqual({ a: '[REDACTED]', b: '[REDACTED]' })
+ expect(result.content).not.toContain('ab')
+ expect(result.content).not.toContain('cdefghijk')
+ expect(result.content).not.toContain(token)
+ })
+
+ it('keeps every byte outside the redacted string identical', () => {
+ const output = [
+ '{',
+ ' "big": 12345678901234567890,',
+ ' "one": 1.0,',
+ ' "huge": 1e400,',
+ ' "neg": -0,',
+ ' "dup": "first",',
+ ' "dup": "second",',
+ ' "text": "line\\npassword=supersecret99"',
+ '}',
+ ].join('\n')
+ const textPass = safety.sanitizeFreeformContent(output, { secretWarning: WARNING })
+ expect(() => JSON.parse(textPass.content)).toThrow()
+
+ const result = safety.sanitizeOutput('mcp__config', output)
+ expect(result.content).toBe(output.replace('password=supersecret99', '[REDACTED]'))
+ expect(JSON.parse(result.content).text).toBe('line\n[REDACTED]')
+ expect(result.content).not.toContain('supersecret99')
+ })
+
+ it('redacts a value that only a match spanning a key exposes', () => {
+ const output = '{"a":"password=supersecret99","password:":"secret12345"}'
+ const result = safety.sanitizeOutput('mcp__config', output)
+ expect(result.content).toBe('{"a":"[REDACTED]","[REDACTED]":"[REDACTED]"}')
+ expect(JSON.parse(result.content)).toEqual({ a: '[REDACTED]', '[REDACTED]': '[REDACTED]' })
+ expect(result.content).not.toContain('supersecret99')
+ expect(result.content).not.toContain('secret12345')
+ })
+
+ it('keeps the text-pass result when a match covers a number', () => {
+ const output = '{"note":"x","pwd:":12345678}'
+ const textPass = safety.sanitizeFreeformContent(output, { secretWarning: WARNING })
+ const result = safety.sanitizeOutput('mcp__config', output)
+ expect(result.was_modified).toBe(true)
+ expect(result.content).toBe(textPass.content)
+ expect(result.content).toContain('[REDACTED]')
+ expect(result.content).not.toContain('12345678')
+ })
+
+ it('redacts inside deeply nested JSON without overflowing the stack', () => {
+ const depth = 2_500
+ const wrap = (inner: string) => '{"a":['.repeat(depth) + inner + ']}'.repeat(depth)
+ const output = wrap('"password=supersecret99"')
+ let result: ReturnType<typeof safety.sanitizeOutput> | undefined
+ expect(() => {
+ result = safety.sanitizeOutput('mcp__config', output)
+ }).not.toThrow()
+ expect(result!.content).toBe(wrap('"[REDACTED]"'))
+ expect(() => JSON.parse(result!.content)).not.toThrow()
+ expect(result!.content).not.toContain('supersecret99')
+ })
+ })
+
+ describe('configured secrets written as JSON escapes', () => {
+ const SECRET = 'q"uo\\te-secret'
+ const ESCAPED = 'q\\"uo\\\\te-secret'
+ const s = new BasicSafety(() => [{ name: 'DB_PASSWORD', value: SECRET }])
+
+ it('redacts the escaped form when the text pass keeps the JSON valid', () => {
+ const output = JSON.stringify({ env: `DB=${SECRET}`, ok: true })
+ expect(output).toContain(ESCAPED)
+ const result = s.sanitizeOutput('shell_exec', output)
+ expect(result.content).toBe('{"env":"DB=[REDACTED:DB_PASSWORD]","ok":true}')
+ expect(JSON.parse(result.content).ok).toBe(true)
+ expect(result.content).not.toContain(SECRET)
+ expect(result.content).not.toContain(ESCAPED)
+ })
+
+ it('redacts the escaped form on the JSON-preserving path', () => {
+ const output = JSON.stringify({ text: 'line\npassword=supersecret99', env: `DB=${SECRET}` })
+ const result = s.sanitizeOutput('clerum__attachment_read', output)
+ expect(result.content).toBe('{"text":"line\\n[REDACTED]","env":"DB=[REDACTED:DB_PASSWORD]"}')
+ expect(JSON.parse(result.content).env).toBe('DB=[REDACTED:DB_PASSWORD]')
+ expect(result.content).not.toContain(SECRET)
+ expect(result.content).not.toContain(ESCAPED)
+ expect(result.content).not.toContain('supersecret99')
+ })
})
describe('tool_output tag escaping in the wrapper (M2)', () => {
@@ -256,6 +371,54 @@ describe('BasicSafety', () => {
const wrapped = safety.wrapForLlm('search', '<tool_outputs> </tool_output_x>', false)
expect(wrapped).toBe(`${OPEN}<tool_outputs> </tool_output_x>${CLOSE}`)
})
+
+ it.each([
+ ['whitespace after the slash', '</ tool_output>', '&lt;/ tool_output&gt;'],
+ ['whitespace after the bracket', '< tool_output>', '&lt; tool_output&gt;'],
+ ['whitespace around the slash', '< / tool_output >', '&lt; / tool_output &gt;'],
+ ['an unclosed forged opening tag', '<tool_output name="x"', '&lt;tool_output name="x"'],
+ ])('escapes a tag with %s', (_label, tag, escaped) => {
+ const wrapped = safety.wrapForLlm('search', `before ${tag} after`, false)
+ expect(wrapped.startsWith(OPEN)).toBe(true)
+ expect(wrapped.endsWith(CLOSE)).toBe(true)
+ expect(wrapped.slice(OPEN.length, -CLOSE.length)).toBe(`before ${escaped} after`)
+ })
+
+ it('escapes many unclosed tag starts in linear time', () => {
+ const count = 40_000
+ const content = '<tool_output'.repeat(count)
+ const started = performance.now()
+ const wrapped = safety.wrapForLlm('search', content, false)
+ const elapsedMs = performance.now() - started
+ expect(wrapped.match(/&lt;tool_output/g)).toHaveLength(count)
+ expect(elapsedMs).toBeLessThan(2_000)
+ }, 60_000)
+ })
+
+ describe('tool_output tag filtering in assistant responses', () => {
+ it.each([
+ ['whitespace after the slash', '</ tool_output>'],
+ ['whitespace after the bracket', '< tool_output x="y">'],
+ ['upper-case closing tag with a space', '</TOOL_OUTPUT >'],
+ ])('filters a tag with %s', (_label, tag) => {
+ const result = safety.sanitizeAssistantResponse(`before ${tag} after`)
+ expect(result.content).toBe('before [filtered] after')
+ expect(result.warnings).toEqual([
+ 'Potential tool_output tag filtered from assistant response',
+ ])
+ })
+
+ it('scans many unclosed tag starts in linear time and keeps the text', () => {
+ const count = 40_000
+ const content = '<tool_output'.repeat(count)
+ const started = performance.now()
+ const result = safety.sanitizeAssistantResponse(content)
+ const elapsedMs = performance.now() - started
+ // Witness: the filter ran over the whole text and removed nothing.
+ expect(result.content).toBe(content)
+ expect(result.content.match(/<tool_output/g)).toHaveLength(count)
+ expect(elapsedMs).toBeLessThan(2_000)
+ }, 60_000)
})
describe('ConfigStore secret-value redaction', () => {
diff --git a/mcp-host/src/core/safety/safety.ts b/mcp-host/src/core/safety/safety.ts
index 135565f6e..f19990d69 100644
--- a/mcp-host/src/core/safety/safety.ts
+++ b/mcp-host/src/core/safety/safety.ts
@@ -24,8 +24,12 @@ import { SanitizedOutput, ValidationResult } from '../types'
export type SecretEntriesProvider = () => Array<{ name: string; value: string }>
// Any opening or closing `tool_output` tag, whatever its attributes, spacing or
-// case: the model must not read a forged wrapper boundary inside the content.
-const TOOL_OUTPUT_TAG_PATTERN = /<\/?tool_output\b[^>]*>/gi
+// case, closed or not: the model must not read a forged wrapper boundary inside
+// the content. `[^<>]*` stops at the next `<`, so each match scans only up to
+// the next tag start and the whole pass stays linear; the optional `>` escapes
+// an unclosed forged tag too. The replacement only escapes `<` and `>`, so no
+// text is lost.
+const TOOL_OUTPUT_TAG_PATTERN = /<(?:\s*\/)?\s*tool_output\b[^<>]*>?/gi
function wrapToolOutput(toolName: string, content: string, wasSanitized: boolean): string {
const escaped = content.replace(TOOL_OUTPUT_TAG_PATTERN, tag =>
@@ -43,6 +47,66 @@ function parsesAsJson(text: string): boolean {
}
}
+type RedactionRange = { start: number; end: number; replacement: string }
+
+/**
+ * String tokens of a valid JSON text, found in one iterative scan. `tokenOf[i]`
+ * is the index of the token whose inner span (between its quotes, escapes still
+ * encoded) holds code unit `i`, or -1 outside every string. `escapeOffset[i]`
+ * is the distance from code unit `i` back to the backslash that opens the
+ * escape sequence holding it, or 0 when `i` is not inside one past its start.
+ */
+function scanJsonStrings(text: string): {
+ tokens: Array<{ start: number; end: number }>
+ tokenOf: Int32Array
+ escapeOffset: Uint8Array
+} {
+ const tokens: Array<{ start: number; end: number }> = []
+ const tokenOf = new Int32Array(text.length).fill(-1)
+ const escapeOffset = new Uint8Array(text.length)
+ let i = 0
+ while (i < text.length) {
+ if (text.charCodeAt(i) !== 0x22) {
+ i++
+ continue
+ }
+ const start = i + 1
+ let j = start
+ while (j < text.length && text.charCodeAt(j) !== 0x22) {
+ if (text.charCodeAt(j) === 0x5c) {
+ const length = text.charCodeAt(j + 1) === 0x75 ? 6 : 2
+ for (let k = 1; k < length; k++) escapeOffset[j + k] = k
+ j += length
+ } else {
+ j++
+ }
+ }
+ if (j >= text.length) throw new Error('scanJsonStrings: unterminated string in valid JSON')
+ tokenOf.fill(tokens.length, start, j)
+ tokens.push({ start, end: j })
+ i = j + 1
+ }
+ return { tokens, tokenOf, escapeOffset }
+}
+
+// JSON punctuation and whitespace: the only bytes outside a string that a
+// redaction range may cover while the output can still be redacted in place.
+function isJsonStructural(code: number): boolean {
+ return (
+ code === 0x22 || // "
+ code === 0x7b || // {
+ code === 0x7d || // }
+ code === 0x5b || // [
+ code === 0x5d || // ]
+ code === 0x3a || // :
+ code === 0x2c || // ,
+ code === 0x20 ||
+ code === 0x09 ||
+ code === 0x0a ||
+ code === 0x0d
+ )
+}
+
export class BasicSafety implements Safety {
/**
* Optional callback that returns ConfigStore-managed secret values. When
@@ -78,13 +142,18 @@ export class BasicSafety implements Safety {
/https:\/\/hooks\.slack\.com\/services\/[A-Za-z0-9\/]+/g,
]
+ private static readonly INJECTION_REPLACEMENT = '[filtered]'
+ private static readonly SECRET_REPLACEMENT = '[REDACTED]'
+
private static readonly ASSISTANT_RESPONSE_FILTER_PATTERNS: Array<{
pattern: RegExp
replacement: string
warning: string
}> = [
{
- pattern: /<\/?tool_output\b[^>]*>/gi,
+ // Closed tags only, because this replacement removes text. `[^<>]*`
+ // keeps the scan linear (see TOOL_OUTPUT_TAG_PATTERN).
+ pattern: /<(?:\s*\/)?\s*tool_output\b[^<>]*>/gi,
replacement: '[filtered]',
warning: 'Potential tool_output tag filtered from assistant response',
},
@@ -297,19 +366,27 @@ export class BasicSafety implements Safety {
}
/**
- * Tool-output redaction that keeps JSON output parseable. The patterns run
- * over the raw text, so a match can swallow JSON syntax: the password value
- * class `[^\s,;]{8,}` eats a closing `"}` when the value ends the last
- * string of an object, as an attachment page cut inside `password=…` does.
- * Only when the original output was valid JSON and the text pass made it
- * invalid is the redaction redone leaf by leaf: every key and string value
- * goes through the same rules and the result is re-serialized. Every other
- * output keeps the text-pass result byte for byte.
+ * Tool-output redaction that keeps JSON output parseable. The text pass
+ * (`sanitizeFreeformContent`) runs over the raw text, so a match can swallow
+ * JSON syntax: the password value class `[^\s,;]{8,}` eats a closing `"}`,
+ * and a raw `\n` escape is not whitespace to it. The text-pass result is
+ * returned unchanged when it did not modify the output, when it still
+ * parses, or when the original output does not parse.
*
- * The text-pass result is kept as well when the leaf pass cannot reproduce
- * it: a match that spans JSON syntax (a key ending in `password:` followed
- * by its value) is invisible to any single leaf, and two keys that redact to
- * the same text cannot both be kept. Redaction wins over parseability there.
+ * Otherwise the redaction is redone in place on the original text:
+ * 1. One iterative scan records the inner span of every string token, keys
+ * and values alike, with escapes still encoded.
+ * 2. Every injection pattern, secret pattern and configured secret form is
+ * matched independently against the original text.
+ * 3. If a match covers any byte outside every string other than JSON
+ * punctuation or whitespace (a number, `true`, `false`, `null`), or
+ * touches no string at all, the text-pass result is returned.
+ * 4. Each match is clipped to every string it covers and widened to whole
+ * escape sequences. Pieces that overlap or touch inside one string are
+ * merged (`[REDACTED]` when their replacements differ) and spliced in.
+ * Every byte outside the replaced pieces stays as it was.
+ * 5. If that result does not parse, the text-pass result is returned.
+ * The warnings are always the text pass's.
*/
private sanitizeToolOutputContent(toolName: string, content: string): SanitizedOutput {
const options = { secretWarning: `Potential secret detected in ${toolName} output` }
@@ -317,35 +394,109 @@ export class BasicSafety implements Safety {
if (!textPass.was_modified || parsesAsJson(textPass.content) || !parsesAsJson(content)) {
return textPass
}
- const leaves = { warnings: new Set<string>(), modified: false, keyCollision: false }
- const redacted = JSON.stringify(this.sanitizeJsonLeaves(JSON.parse(content), options, leaves))
- const residual = this.sanitizeFreeformContent(redacted, options)
- if (!leaves.modified || leaves.keyCollision || residual.was_modified) return textPass
- return { content: redacted, was_modified: true, warnings: [...leaves.warnings] }
+ const { tokens, tokenOf, escapeOffset } = scanJsonStrings(content)
+ const pieces: RedactionRange[] = []
+ for (const range of this.redactionRanges(content)) {
+ let touchedString = false
+ let i = range.start
+ while (i < range.end) {
+ const token = tokenOf[i]
+ if (token === -1) {
+ if (!isJsonStructural(content.charCodeAt(i))) return textPass
+ i++
+ continue
+ }
+ const tokenEnd = tokens[token].end
+ let start = i
+ let end = Math.min(range.end, tokenEnd)
+ start -= escapeOffset[start]
+ if (end < tokenEnd && escapeOffset[end] > 0) {
+ const escapeStart = end - escapeOffset[end]
+ end = escapeStart + (content.charCodeAt(escapeStart + 1) === 0x75 ? 6 : 2)
+ }
+ pieces.push({ start, end, replacement: range.replacement })
+ touchedString = true
+ i = tokenEnd
+ }
+ if (!touchedString) return textPass
+ }
+
+ // A piece ends at most at its string's closing quote and the next string
+ // starts at least two code units later, so merging sorted pieces that
+ // overlap or touch only ever joins pieces of the same string.
+ pieces.sort((a, b) => a.start - b.start)
+ const merged: RedactionRange[] = []
+ for (const piece of pieces) {
+ const last = merged[merged.length - 1]
+ if (last && piece.start <= last.end) {
+ last.end = Math.max(last.end, piece.end)
+ if (last.replacement !== piece.replacement) {
+ last.replacement = BasicSafety.SECRET_REPLACEMENT
+ }
+ } else {
+ merged.push({ ...piece })
+ }
+ }
+ let redacted = ''
+ let cursor = 0
+ for (const piece of merged) {
+ redacted += content.slice(cursor, piece.start) + piece.replacement
+ cursor = piece.end
+ }
+ redacted += content.slice(cursor)
+ if (!parsesAsJson(redacted)) return textPass
+ return { content: redacted, was_modified: true, warnings: textPass.warnings }
}
- private sanitizeJsonLeaves(
- value: unknown,
- options: { secretWarning: string },
- leaves: { warnings: Set<string>; modified: boolean; keyCollision: boolean }
- ): unknown {
- if (typeof value === 'string') {
- const result = this.sanitizeFreeformContent(value, options)
- for (const warning of result.warnings) leaves.warnings.add(warning)
- if (result.was_modified) leaves.modified = true
- return result.content
+ /**
+ * Every match of the redaction rules `sanitizeFreeformContent` applies to
+ * tool output, each rule matched independently against `content`.
+ */
+ private redactionRanges(content: string): RedactionRange[] {
+ const ranges: RedactionRange[] = []
+ const addMatches = (pattern: RegExp, replacement: string) => {
+ for (const match of content.matchAll(pattern)) {
+ if (match[0].length === 0) continue
+ ranges.push({ start: match.index, end: match.index + match[0].length, replacement })
+ }
+ }
+ for (const pattern of BasicSafety.INJECTION_PATTERNS) {
+ addMatches(pattern, BasicSafety.INJECTION_REPLACEMENT)
}
- if (Array.isArray(value))
- return value.map(item => this.sanitizeJsonLeaves(item, options, leaves))
- if (value === null || typeof value !== 'object') return value
- // A null-prototype record keeps a parsed `__proto__` key as an own property.
- const out: Record<string, unknown> = Object.create(null)
- for (const [key, item] of Object.entries(value)) {
- const sanitizedKey = this.sanitizeJsonLeaves(key, options, leaves) as string
- if (Object.prototype.hasOwnProperty.call(out, sanitizedKey)) leaves.keyCollision = true
- out[sanitizedKey] = this.sanitizeJsonLeaves(item, options, leaves)
+ for (const pattern of BasicSafety.SECRET_PATTERNS) {
+ addMatches(pattern, BasicSafety.SECRET_REPLACEMENT)
+ }
+ for (const secret of this.configuredSecretForms()) {
+ const replacement = `[REDACTED:${secret.name}]`
+ for (const form of secret.forms) {
+ let at = content.indexOf(form)
+ while (at !== -1) {
+ ranges.push({ start: at, end: at + form.length, replacement })
+ at = content.indexOf(form, at + form.length)
+ }
+ }
}
- return out
+ return ranges
+ }
+
+ /**
+ * The ConfigStore secret values to redact by literal match, each with every
+ * form it takes in output: the plain value and, when different, its JSON
+ * string encoding (a value holding `"`, `\` or a control character appears
+ * escaped inside raw JSON). The encoded form, never shorter, comes first so
+ * the plain pass cannot split it.
+ */
+ private configuredSecretForms(): Array<{ name: string; forms: string[] }> {
+ const secrets: Array<{ name: string; forms: string[] }> = []
+ for (const entry of this.secretEntriesProvider?.() ?? []) {
+ if (!entry.value || entry.value.length < 4) continue // ignore trivially-short values
+ const encoded = JSON.stringify(entry.value).slice(1, -1)
+ secrets.push({
+ name: entry.name,
+ forms: encoded === entry.value ? [entry.value] : [encoded, entry.value],
+ })
+ }
+ return secrets
}
/**
@@ -367,7 +518,7 @@ export class BasicSafety implements Safety {
for (const pattern of BasicSafety.INJECTION_PATTERNS) {
const before = sanitized
- sanitized = sanitized.replace(pattern, '[filtered]')
+ sanitized = sanitized.replace(pattern, BasicSafety.INJECTION_REPLACEMENT)
if (sanitized !== before) {
warnings.push('Potential prompt injection pattern filtered')
}
@@ -383,23 +534,29 @@ export class BasicSafety implements Safety {
for (const pattern of BasicSafety.SECRET_PATTERNS) {
const before = sanitized
- sanitized = sanitized.replace(pattern, '[REDACTED]')
+ sanitized = sanitized.replace(pattern, BasicSafety.SECRET_REPLACEMENT)
if (sanitized !== before) {
warnings.push(options.secretWarning)
}
}
// Defense-in-depth: redact ConfigStore-managed secret values by literal
- // substring match. Catches values that don't fit the regex shapes
- // above — operator-supplied integration tokens, the LLM key, etc.
- // Length-descending traversal ensures a longer secret containing a
- // shorter one is masked before the shorter pass would erase its anchor.
- const entries = this.secretEntriesProvider?.() ?? []
- for (const entry of entries) {
- if (!entry.value || entry.value.length < 4) continue // ignore trivially-short values
- if (!sanitized.includes(entry.value)) continue
- sanitized = sanitized.split(entry.value).join(`[REDACTED:${entry.name}]`)
- warnings.push(`ConfigStore secret value redacted (${entry.name})`)
+ // substring match, in plain and JSON-encoded form. Catches values that
+ // don't fit the regex shapes above — operator-supplied integration
+ // tokens, the LLM key, etc. Entries are traversed in provider order;
+ // ConfigStore sorts them by descending length, so a longer secret
+ // containing a shorter one is masked before the shorter pass would erase
+ // its anchor.
+ for (const secret of this.configuredSecretForms()) {
+ const before = sanitized
+ for (const form of secret.forms) {
+ if (sanitized.includes(form)) {
+ sanitized = sanitized.split(form).join(`[REDACTED:${secret.name}]`)
+ }
+ }
+ if (sanitized !== before) {
+ warnings.push(`ConfigStore secret value redacted (${secret.name})`)
+ }
}
return {