Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
7.6 KB
Raw Blame
/**
 * secret-scan-stream.ts: the windowed scan reports exactly what scanText
 * reports on the whole text (pattern, 1-based line, fingerprint), whatever the
 * window seams and chunk boundaries cut: single-line values and private keys
 * swept across both seams of a window, a line longer than a window split
 * mid-line, multi-byte characters split across chunks, invalid UTF-8 and a
 * BOM. Windows are shrunk to the smallest the module allows (the windowChars
 * test seam) so a sweep stays small; the overlap is the production one.
 * Secrets are assembled at runtime from split literals.
 */
import { describe, test, expect } from 'bun:test';
import { scanText, type SecretFinding } from '../src/core/secret-scan.ts';
import { scanTextStream, STREAM_SCAN_OVERLAP_CHARS as O, streamScanWork } from '../src/core/secret-scan-stream.ts';

let seed = 7;
const rnd = (): number => (seed = (seed * 1103515245 + 12345) % 2147483648) / 2147483648;
const ALNUM = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789';
const rs = (n: number, cs = ALNUM): string => Array.from({ length: n }, () => cs[Math.floor(rnd() * cs.length)]).join('');
const BEGIN = ['-----BEGIN', ' RSA PRIVATE KEY', '-----'].join('');
const END = ['-----END', ' RSA PRIVATE KEY', '-----'].join('');
const pem = (): string => [BEGIN, ...Array.from({ length: 200 }, () => rs(64, ALNUM + '+/')), END].join('\n');
const SECRETS: Array<() => string> = [
  () => '\nkey: ' + 'sk-' + rs(40) + '\n',
  () => ' Bearer ' + 'eyJ' + rs(4000) + '.' + rs(4000) + '.' + rs(4000) + ' ',
  () => '\n' + pem() + '\n',
  () => ' ' + 'sk-ant-' + rs(30) + ' ',
];
const W = 3 * O;

async function* chunks(buf: Uint8Array, size: () => number): AsyncGenerator<Uint8Array> {
  for (let i = 0; i < buf.length; ) {
    const n = size();
    yield buf.subarray(i, i + n);
    i += n;
  }
}
const keys = (fs: SecretFinding[]): string[] => fs.map((f) => `${f.pattern}@${f.line}#${f.fingerprint}`).sort();

async function expectSameAsWhole(buf: Buffer, windowChars = W, size = () => 1 + Math.floor(rnd() * 70_000)): Promise<SecretFinding[]> {
  const whole = scanText(buf.toString('utf-8'));
  const streamed = await scanTextStream(chunks(buf, size), {}, windowChars);
  expect(keys(streamed)).toEqual(keys(whole));
  return streamed;
}

describe('scanTextStream', () => {
  test('values and private keys swept across both seams of a window match the whole-text scan', async () => {
    const lines = Array.from({ length: 4000 }, () => rs(80, ALNUM + ' ')).join('\n');
    const long = rs(5 * O, ALNUM + ' ');
    let found = 0;
    for (const filler of [lines, long]) {
      for (const make of SECRETS) {
        for (let at = W - O - 12_000; at < W + 12_000; at += 3001) {
          found += (await expectSameAsWhole(Buffer.from(filler.slice(0, at) + make() + filler.slice(at)))).length;
        }
      }
    }
    expect(found).toBeGreaterThanOrEqual(2 * SECRETS.length * 8);
  });

  test('a line longer than a window is split mid-line and still yields every value on it', async () => {
    const parts: string[] = [];
    for (let i = 0; i < 40; i++) parts.push(rs(9000 + Math.floor(rnd() * 9000), ALNUM + ' '), ' ' + 'gh' + 'p_' + rs(36) + ' ');
    const found = await expectSameAsWhole(Buffer.from('head\n' + parts.join('') + '\ntail\n'));
    expect(found).toHaveLength(40);
    expect(found.every((f) => f.line === 2)).toBe(true);
  });

  test('multi-byte characters split across chunks, a surrogate pair at a mid-line split, invalid UTF-8 and a BOM', async () => {
    const emoji = '\u{1F600}'.repeat(2 * O);
    const text = '\uFEFF' + 'é日本語 ' + 'sk-ant-' + rs(30) + '\n' + emoji + ' ' + 'sk-' + rs(40) + ' ' + emoji + '\n' + 'é'.repeat(O) + ' ' + 'sk-' + rs(40) + '\n';
    expect(text.charCodeAt(O + 1) & 0xfc00).toBe(0xdc00); // the first mid-line split point is a low surrogate
    expect(await expectSameAsWhole(Buffer.from(text), 2 * O + 1, () => 7)).toHaveLength(3);
    const invalid = Buffer.concat([Buffer.from('a\xff\xfe\n'), Buffer.from([0xe6, 0x97, 0x0a, 0xc3]), Buffer.from(' sk-' + rs(40) + '\n'), Buffer.alloc(3 * O, 0x61)]);
    expect(await expectSameAsWhole(invalid, W, () => 3)).toHaveLength(1);
  });

  test('work counters bound the window and cover every byte', async () => {
    const buf = Buffer.from(Array.from({ length: 20_000 }, () => rs(99, ALNUM + ' ')).join('\n'));
    for (const k of Object.keys(streamScanWork) as Array<keyof typeof streamScanWork>) streamScanWork[k] = 0;
    await scanTextStream(chunks(buf, () => 65_536), {}, W);
    expect(streamScanWork.bytes).toBe(buf.length);
    expect(streamScanWork.chars).toBeGreaterThanOrEqual(buf.length);
    expect(streamScanWork.largestWindow).toBe(W);
    expect(streamScanWork.windows).toBeGreaterThanOrEqual(Math.ceil(buf.length / W));
    expect(streamScanWork.windows).toBeLessThanOrEqual(Math.ceil(buf.length / (W - O - 100)) + 1);
  });

test('every pattern slid across a mid-line cut at every offset matches the whole-text scan', async () => {
    const W2 = 2 * O;
    const cut = O; // a newline-free window is cut mid-line at exactly the overlap before its end
    const filler = rs(W2 + 6000, ALNUM.slice(0, 26) + ' ');
    const b64 = (s: string): string => Buffer.from(s).toString('base64');
    const hex = (n: number): string => rs(n, '0123456789abcdef');
    const cases: Array<{ s: string; highEntropy?: boolean }> = [
      { s: 'Bearer ' + rs(40) }, { s: 'BEARER ' + rs(40) }, { s: 'bearer\t\t' + rs(24) },
      { s: 'Authorization: Basic ' + b64('alice:' + rs(10)) }, { s: 'authorization:basic  ' + b64('u:p') },
      { s: 'Basic ' + b64('alice:' + rs(20)) },
      { s: 'postgres' + '://alice:' + rs(16) + '@db.example:5432/app' }, { s: 'https' + '://alice:' + rs(16) + '@host.example/x' },
      { s: 'eyJ' + rs(20) + '.' + rs(20) + '.' + rs(20) },
      { s: 'sk-ant-' + rs(30) }, { s: 'sk-proj-' + rs(30) }, { s: 'sk-' + rs(40) }, { s: 'pa-' + rs(30) },
      { s: 'github_pat_' + rs(30) }, { s: 'gh' + 'p_' + rs(36) }, { s: 'glpat-' + rs(24) }, { s: 'xox' + 'b-' + rs(20) },
      { s: 'AKIA' + rs(16, 'ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789') }, { s: 'AIza' + rs(35) },
      { s: 'sk_' + 'live_' + rs(24) }, { s: 'whsec_' + rs(30) }, { s: 'SG.' + rs(20) + '.' + rs(20) },
      { s: 'AC' + hex(32) }, { s: 'sb_secret_' + rs(24) }, { s: 'sbp_' + hex(40) }, { s: 'npm_' + rs(36) },
      { s: 'hf_' + rs(34) }, { s: 'dop_v1_' + hex(64) }, { s: 'gbrain_at_' + hex(64) },
      { s: 'Bearer ' + 'sk-ant-' + rs(30) }, { s: 'Bearer ' + rs(12) + 'gh' + 'p_' + rs(36) },
      { s: 'api_key = "' + rs(18) + '7Q"', highEntropy: true }, { s: 'SMTP_PASSWORD=' + rs(14) + '9z', highEntropy: true },
      { s: 'token:   ' + rs(16) + '3x', highEntropy: true },
    ];
    let probes = 0;
    let found = 0;
    for (const { s, highEntropy } of cases) {
      const opts = highEntropy ? { highEntropy: true } : {};
      for (let into = -2; into <= Math.min(s.length, 32); into++) {
        const at = cut - into;
        const text = filler.slice(0, at - 1) + ' ' + s + ' ' + filler.slice(at + s.length + 1, W2 + 6000);
        const buf = Buffer.from(text);
        const whole = scanText(text, opts);
        const streamed = await scanTextStream(chunks(buf, () => 1 + Math.floor(rnd() * 70_000)), opts, W2);
        expect({ s: s.slice(0, 12), into, got: keys(streamed) }).toEqual({ s: s.slice(0, 12), into, got: keys(whole) });
        expect(whole.length).toBeGreaterThan(0);
        probes++;
        found += streamed.length;
      }
    }
    expect(probes).toBeGreaterThan(cases.length * 20);
    expect(found).toBeGreaterThanOrEqual(probes);
  });

  test('a window under twice the overlap is refused', async () => {
    await expect(scanTextStream(chunks(Buffer.from('x'), () => 1), {}, 2 * O - 1)).rejects.toThrow(RangeError);
  });
});