Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

perf(host): track redaction replacements as segments

Alfredo Lopez0ef36f16c589
diff --git a/mcp-host/package-lock.json b/mcp-host/package-lock.json
index f9d86951e..92694a93a 100644
--- a/mcp-host/package-lock.json
+++ b/mcp-host/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-mcp-host",
- "version": "0.1.498",
+ "version": "0.1.499",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-mcp-host",
- "version": "0.1.498",
+ "version": "0.1.499",
"license": "SEE LICENSE IN LICENSE",
"dependencies": {
"@anthropic-ai/sdk": "^0.32.0",
diff --git a/mcp-host/package.json b/mcp-host/package.json
index a7e1eacfc..6c77eba38 100644
--- a/mcp-host/package.json
+++ b/mcp-host/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-mcp-host",
- "version": "0.1.498",
+ "version": "0.1.499",
"description": "MCP Host - reads Host CRD and provides LLM access via OpenAI/Claude",
"main": "dist/main.js",
"engines": {
diff --git a/mcp-host/src/core/safety/__tests__/safety.test.ts b/mcp-host/src/core/safety/__tests__/safety.test.ts
index c61ffff48..a86ca6bd6 100644
--- a/mcp-host/src/core/safety/__tests__/safety.test.ts
+++ b/mcp-host/src/core/safety/__tests__/safety.test.ts
@@ -658,6 +658,76 @@ describe('BasicSafety', () => {
}, 60_000)
})
+ describe('a later rule matching inside an earlier replacement', () => {
+ const freeform = (entries: Array<{ name: string; value: string }>, content: string) =>
+ new BasicSafety(() => entries).sanitizeFreeformContent(content, { secretWarning: 'w' })
+ const warned = (...names: string[]) =>
+ names.map(name => `ConfigStore secret value redacted (${name})`)
+
+ it('cuts the replacement in three without losing a part', () => {
+ // `REDA` lies strictly inside `[REDACTED:A]`, so the replacement is
+ // split into a part before the match, the match and a part after it.
+ expect(
+ freeform(
+ [
+ { name: 'A', value: 'abcd' },
+ { name: 'B', value: 'REDA' },
+ ],
+ 'abcd'
+ )
+ ).toEqual({ content: '[REDACTED]', was_modified: true, warnings: warned('A', 'B') })
+ })
+
+ it('maps the part before the match to the replaced range', () => {
+ // B cuts `[REDACTED:A]x` at `ED:A`; C then matches `DACT` in the part
+ // before B's match, which stands for `ghij` only, so `x` stays.
+ expect(
+ freeform(
+ [
+ { name: 'A', value: 'ghij' },
+ { name: 'B', value: 'ED:A' },
+ { name: 'C', value: 'DACT' },
+ ],
+ 'ghijx'
+ )
+ ).toEqual({ content: '[REDACTED]x', was_modified: true, warnings: warned('A', 'B', 'C') })
+ })
+
+ it('maps the part after the match to the replaced range', () => {
+ // B cuts `[REDACTED:A]` at `DACT`; C then matches `D:A]` in the part
+ // after B's match, which stands for `efgh`, so one marker remains.
+ expect(
+ freeform(
+ [
+ { name: 'A', value: 'efgh' },
+ { name: 'B', value: 'DACT' },
+ { name: 'C', value: 'D:A]' },
+ ],
+ 'efgh'
+ )
+ ).toEqual({ content: '[REDACTED]', was_modified: true, warnings: warned('A', 'B', 'C') })
+ })
+
+ it('treats adjacent replacements as covering the text they span', () => {
+ // `abcdef` no longer appears after A, so B is matched on the original
+ // text; C and A together cover it, so B adds nothing and is not reported.
+ expect(
+ freeform(
+ [
+ { name: 'A', value: 'efgh' },
+ { name: 'B', value: 'abcdef' },
+ { name: 'C', value: 'abcd' },
+ ],
+ 'abcdefghij'
+ )
+ ).toEqual({
+ content: '[REDACTED:C][REDACTED:A]ij',
+ was_modified: true,
+ warnings: warned('A', 'C'),
+ })
+ })
+ })
+
describe('redaction patterns that can match nothing', () => {
it('refuses a filter that matches an empty string', () => {
const filter = (pattern: RegExp) => ({
diff --git a/mcp-host/src/core/safety/safety.ts b/mcp-host/src/core/safety/safety.ts
index 6c7822b66..7a7df9b54 100644
--- a/mcp-host/src/core/safety/safety.ts
+++ b/mcp-host/src/core/safety/safety.ts
@@ -96,55 +96,158 @@ function scanJsonStrings(text: string): {
* range of the original text behind each of its code units. An original code
* unit stands for itself; a replacement's code units stand for the union of
* the ranges it replaced, so a later match that covers part of an earlier
- * replacement covers everything that replacement stands for. Both range ends
- * are non-decreasing along the copy, so a match `[a, b)` stands for
- * `[origStart[a], origEnd[b - 1])`.
+ * replacement covers everything that replacement stands for, and a match
+ * `[a, b)` stands for the range from code unit `a`'s start to code unit
+ * `b - 1`'s end.
+ *
+ * Only the replacements are stored, as segments sorted along the copy: code
+ * units `[textStart, textEnd)` stand for `[origStart, origEnd)`. The code
+ * units between two segments are original and consecutive, and the first one
+ * after a segment stands for that segment's `origEnd`, so a replacement costs
+ * time in the number of segments, not in the length of the text.
*/
class RedactionReplay {
text: string
readonly ranges: RedactionRange[] = []
- private origStart: Int32Array
- private origEnd: Int32Array
+ private count = 0
+ private textStart = new Int32Array(0)
+ private textEnd = new Int32Array(0)
+ private origStart = new Int32Array(0)
+ private origEnd = new Int32Array(0)
constructor(content: string) {
this.text = content
- this.origStart = new Int32Array(content.length)
- this.origEnd = new Int32Array(content.length)
- for (let i = 0; i < content.length; i++) {
- this.origStart[i] = i
- this.origEnd[i] = i + 1
- }
}
- /** Replaces sorted, non-overlapping, non-empty `matches` of `text`. */
+ /**
+ * Replaces sorted, non-overlapping, non-empty `matches` of `text` with a
+ * non-empty `replacement`.
+ */
replace(matches: Array<[number, number]>, replacement: string): void {
if (matches.length === 0) return
- let length = this.text.length
- for (const [a, b] of matches) length += replacement.length - (b - a)
- const nextStart = new Int32Array(length)
- const nextEnd = new Int32Array(length)
+ if (replacement.length === 0) throw new Error('RedactionReplay: empty replacement')
+ // Each match adds its own segment, and one more when it lies strictly
+ // inside a segment, which it cuts into a part before and a part after it.
+ // Typed arrays drop writes past their end, so the count is checked below.
+ const capacity = this.count + 2 * matches.length
+ const textStart = new Int32Array(capacity)
+ const textEnd = new Int32Array(capacity)
+ const origStart = new Int32Array(capacity)
+ const origEnd = new Int32Array(capacity)
+ let count = 0
const parts: string[] = []
let cursor = 0
- let at = 0
+ let shift = 0
+ let k = 0
+ // The end of the last segment before the current position, in the
+ // current copy and in the original: an original code unit `p` after it
+ // stands for `prevOrigEnd + (p - prevTextEnd)`.
+ let prevTextEnd = 0
+ let prevOrigEnd = 0
for (const [a, b] of matches) {
- nextStart.set(this.origStart.subarray(cursor, a), at)
- nextEnd.set(this.origEnd.subarray(cursor, a), at)
- at += a - cursor
- const start = this.origStart[a]
- const end = this.origEnd[b - 1]
+ while (k < this.count && this.textEnd[k] <= a) {
+ textStart[count] = this.textStart[k] + shift
+ textEnd[count] = this.textEnd[k] + shift
+ origStart[count] = this.origStart[k]
+ origEnd[count] = this.origEnd[k]
+ count++
+ prevTextEnd = this.textEnd[k]
+ prevOrigEnd = this.origEnd[k]
+ k++
+ }
+ let start: number
+ if (k < this.count && this.textStart[k] <= a) {
+ start = this.origStart[k]
+ if (this.textStart[k] < a) {
+ // Keep the part of the segment before the match.
+ textStart[count] = this.textStart[k] + shift
+ textEnd[count] = a + shift
+ origStart[count] = this.origStart[k]
+ origEnd[count] = this.origEnd[k]
+ count++
+ }
+ } else {
+ start = prevOrigEnd + (a - prevTextEnd)
+ }
+ let last = -1
+ while (k < this.count && this.textStart[k] < b) {
+ last = k
+ k++
+ }
+ let end: number
+ if (last !== -1 && this.textEnd[last] > b) {
+ end = this.origEnd[last]
+ // Keep the part of the segment after the match for the next one.
+ k = last
+ this.textStart[last] = b
+ } else {
+ if (last !== -1) {
+ prevTextEnd = this.textEnd[last]
+ prevOrigEnd = this.origEnd[last]
+ }
+ end = prevOrigEnd + (b - prevTextEnd)
+ }
this.ranges.push({ start, end, replacement })
- nextStart.fill(start, at, at + replacement.length)
- nextEnd.fill(end, at, at + replacement.length)
- at += replacement.length
+ textStart[count] = a + shift
+ textEnd[count] = a + shift + replacement.length
+ origStart[count] = start
+ origEnd[count] = end
+ count++
parts.push(this.text.slice(cursor, a), replacement)
cursor = b
+ shift += replacement.length - (b - a)
+ }
+ for (; k < this.count; k++) {
+ textStart[count] = this.textStart[k] + shift
+ textEnd[count] = this.textEnd[k] + shift
+ origStart[count] = this.origStart[k]
+ origEnd[count] = this.origEnd[k]
+ count++
}
- nextStart.set(this.origStart.subarray(cursor), at)
- nextEnd.set(this.origEnd.subarray(cursor), at)
+ if (count > capacity) throw new Error('RedactionReplay: segment capacity exceeded')
parts.push(this.text.slice(cursor))
this.text = parts.join('')
- this.origStart = nextStart
- this.origEnd = nextEnd
+ this.count = count
+ this.textStart = textStart
+ this.textEnd = textEnd
+ this.origStart = origStart
+ this.origEnd = origEnd
+ }
+}
+
+/**
+ * Whether every code unit of each queried range lies in some range of
+ * `ranges`. Ranges that overlap or touch are joined first, so a queried range
+ * is covered exactly when one joined range holds it.
+ */
+function coverageOf(ranges: RedactionRange[]): (start: number, end: number) => boolean {
+ const sorted = [...ranges].sort((x, y) => x.start - y.start)
+ const starts: number[] = []
+ const ends: number[] = []
+ for (const range of sorted) {
+ const last = ends.length - 1
+ if (last >= 0 && range.start <= ends[last]) {
+ if (range.end > ends[last]) ends[last] = range.end
+ } else {
+ starts.push(range.start)
+ ends.push(range.end)
+ }
+ }
+ return (start, end) => {
+ // The last joined range that starts at or before `start`.
+ let low = 0
+ let high = starts.length - 1
+ let found = -1
+ while (low <= high) {
+ const middle = (low + high) >> 1
+ if (starts[middle] <= start) {
+ found = middle
+ low = middle + 1
+ } else {
+ high = middle - 1
+ }
+ }
+ return found !== -1 && ends[found] >= end
}
}
@@ -706,9 +809,13 @@ export class BasicSafety implements Safety {
let found = false
const original: Array<[number, number]> = []
for (const find of rule.find) {
+ // Until a rule replaces something, the copy is `content`, so the
+ // matches on `content` are the replay's own, which it covers whole.
+ const untouched = replay.ranges.length === 0
const matches = find(replay.text)
if (matches.length > 0) found = true
replay.replace(matches, rule.replacement)
+ if (untouched) continue
// A loop, not `push(...)`: spreading one argument per match overflows
// the call stack at about 120k matches.
for (const match of find(content)) original.push(match)
@@ -719,26 +826,14 @@ export class BasicSafety implements Safety {
// A match on `content` that the replay already covers adds nothing, and
// dropping it keeps the result equal to the in-order replacement.
- // `uncoveredBefore[i]` counts the code units before `i` no replay range
- // covers.
- const depth = new Int32Array(content.length + 1)
- for (const range of replay.ranges) {
- depth[range.start]++
- depth[range.end]--
- }
- const uncoveredBefore = new Int32Array(content.length + 1)
- let covering = 0
- for (let i = 0; i < content.length; i++) {
- covering += depth[i]
- uncoveredBefore[i + 1] = uncoveredBefore[i] + (covering === 0 ? 1 : 0)
- }
+ const covered = coverageOf(replay.ranges)
const ranges = [...replay.ranges]
const warnings: string[] = []
rules.forEach((rule, index) => {
let found = replayed[index]
for (const [start, end] of direct[index]) {
- if (uncoveredBefore[end] - uncoveredBefore[start] === 0) continue
+ if (covered(start, end)) continue
ranges.push({ start, end, replacement: rule.replacement })
found = true
}