Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
5.3 KB
Raw Blame
/**
 * Windowed secret scan of a byte stream too large to hold as one string: the
 * pre-push gate's staged blobs over PUSH_MAX_SCAN_BYTES (workspace-push.ts).
 *
 * The bytes are decoded with a streaming UTF-8 decoder (a character split
 * across chunks is joined, an invalid sequence becomes U+FFFD as
 * `Buffer.toString('utf-8')` makes it, a BOM is kept) and scanned by
 * `scanTextSpans` in windows of at most STREAM_SCAN_WINDOW_CHARS UTF-16
 * units. Each next window starts at a line start at least
 * STREAM_SCAN_OVERLAP_CHARS before the previous window's end; a line too long
 * for that is split mid-line at exactly the overlap (never inside a surrogate
 * pair). The overlap is longer than any claim of the push gate's patterns: a
 * private key (PEM_BODY_MAX_CHARS 16384 of body plus fences, RFC 1421 header
 * lines and indentation, under 18 Ki) and the bounded line values (jwt
 * 3 x 4096 + 2, basic_auth 2048, URL userinfo 384, assignment values 4096).
 *
 * OWNERSHIP. A claim belongs to the window holding the start of the match that
 * made it, anchor included (`from`: the `Bearer ` or `Authorization: Basic `
 * keyword, an assignment key, or the boundary character or escape before a value), not
 * to the window holding the value's first character: a mid-line cut between
 * an anchor and its value leaves the next window a value with no anchor, which
 * no pattern claims. A window keeps a claim whose match starts before the next
 * window's start (the whole match lies inside this window, since it is shorter
 * than the overlap); the next window sees every match that starts at or after
 * its start. The last window keeps everything. A match longer than the overlap
 * (an anchor whose whitespace run before the token is that long) is the
 * accepted miss. A claim lying wholly inside a span kept by
 * an earlier window is the seam's partial view of it (the body of a key cut at
 * the window start, a suffix of a split line) and is dropped, which also drops
 * an exact duplicate (same offset and pattern). Line numbers are 1-based over
 * the whole stream. What a seam can add is a finding the whole-text scan would
 * not make (a cut read as a boundary): that blocks the push, never leaks.
 * An unbounded vendor value longer than the overlap is still reported, cut
 * at the window end.
 */
import { scanTextSpans, type ScanOpts, type SecretFinding } from './secret-scan.ts';

/** Largest text window one scan holds (UTF-16 units). */
export const STREAM_SCAN_WINDOW_CHARS = 8 * 1024 ** 2;

/** Text every window repeats from the end of the one before it (see the header). */
export const STREAM_SCAN_OVERLAP_CHARS = 64 * 1024;

/**
 * @internal Work counters (test/helpers/work-counts.ts): windows scanned,
 * bytes read from the stream, text units scanned (overlap included) and the
 * largest window, so a test pins boundedness without timing.
 */
export const streamScanWork = { windows: 0, bytes: 0, chars: 0, largestWindow: 0 };

/**
 * Scan `chunks` as one UTF-8 text; the findings `scanText` would report on
 * the whole text, plus any a seam adds (header). `windowChars` is a test
 * seam; it must be at least twice the overlap.
 */
export async function scanTextStream(
  chunks: AsyncIterable<Uint8Array>, opts: ScanOpts = {}, windowChars = STREAM_SCAN_WINDOW_CHARS,
): Promise<SecretFinding[]> {
  if (windowChars < 2 * STREAM_SCAN_OVERLAP_CHARS) throw new RangeError(`a scan window must be at least ${2 * STREAM_SCAN_OVERLAP_CHARS} chars`);
  const decoder = new TextDecoder('utf-8', { ignoreBOM: true });
  const findings: SecretFinding[] = [];
  let carried: Array<{ start: number; end: number }> = [];
  let pending = '';
  let base = 0;
  let baseLine = 1;
  const drain = (final: boolean): void => {
    while (pending.length >= windowChars || (final && pending.length > 0)) {
      const last = final && pending.length <= windowChars;
      const text = last ? pending : pending.slice(0, windowChars);
      let next = text.length;
      if (!last) {
        const preferred = text.length - STREAM_SCAN_OVERLAP_CHARS;
        const lineStart = text.lastIndexOf('\n', preferred - 1) + 1;
        next = lineStart > preferred / 2 ? lineStart : preferred - ((text.charCodeAt(preferred) & 0xfc00) === 0xdc00 ? 1 : 0);
      }
      streamScanWork.windows++;
      streamScanWork.chars += text.length;
      if (text.length > streamScanWork.largestWindow) streamScanWork.largestWindow = text.length;
      const kept: Array<{ start: number; end: number }> = [];
      for (const claim of scanTextSpans(text, opts)) {
        if (claim.from >= next) continue;
        const start = base + claim.start;
        const end = base + claim.end;
        if (carried.some((c) => c.start <= start && end <= c.end)) continue;
        kept.push({ start, end });
        if (claim.finding) findings.push({ ...claim.finding, line: baseLine + claim.finding.line - 1 });
      }
      if (last) {
        pending = '';
        return;
      }
      for (let i = text.indexOf('\n'); i !== -1 && i < next; i = text.indexOf('\n', i + 1)) baseLine++;
      base += next;
      carried = [...carried, ...kept].filter((c) => c.end > base);
      pending = pending.slice(next);
    }
  };
  for await (const chunk of chunks) {
    streamScanWork.bytes += chunk.length;
    pending += decoder.decode(chunk, { stream: true });
    drain(false);
  }
  pending += decoder.decode();
  drain(true);
  return findings;
}