Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
9.3 KB
Raw Blame
/**
 * workspace-push over a staged file larger than PUSH_MAX_SCAN_BYTES (25 MiB):
 * the pre-push secret gate streams the blob and scans it in bounded windows
 * (secret-scan-stream.ts) instead of refusing it as unscannable, against a
 * real local bare remote as in workspace-push.serial.test.ts.
 *
 * The file is 314,573 lines of 99 characters plus a newline (100 bytes a
 * line), so every window seam sits at a known line: window 0 ends at
 * character STREAM_SCAN_WINDOW_CHARS, and each next window starts at the line
 * start at or before its predecessor's end minus STREAM_SCAN_OVERLAP_CHARS.
 * Secrets fill whole 99-character lines (a private key's fences and
 * 99-character base64 body lines included), so placing one shifts nothing.
 * Every secret is assembled at runtime from split literals.
 */
import { describe, test, expect, beforeEach, afterEach } from 'bun:test';
import { mkdtempSync, rmSync, writeFileSync, truncateSync } from 'fs';
import { join } from 'path';
import { tmpdir } from 'os';
import { execFileSync } from 'child_process';
import { randomBytes } from 'crypto';
import { workspacePush, PUSH_MAX_SCAN_BYTES, PUSH_STREAM_SCAN_MAX_BYTES } from '../src/core/workspace-push.ts';
import { STREAM_SCAN_OVERLAP_CHARS, STREAM_SCAN_WINDOW_CHARS, streamScanWork } from '../src/core/secret-scan-stream.ts';

const T = 180_000; // explicit per-test timeout — bun ignores bunfig.toml's key
const LINE = 100;
const LINES = 314_573;
const SIZE = LINE * LINES;
const FILLER = 'lorem ipsum dolor sit amet consectetur adipiscing elit sed do eiusmod tempor incididunt ut labore'.padEnd(99, ' ').slice(0, 99);
const OPENAI = 'sk-' + 'A1b2C3d4E5f6G7h8I9j0K1l2M3n4P5q6R7s8';
const GITHUB = 'gh' + 'p_' + 'Z9y8X7w6V5u4T3s2R1q0P9o8N7m6L5k4J3i2';
const BEGIN = ['-----BEGIN', ' RSA PRIVATE KEY', '-----'].join('');
const END = ['-----END', ' RSA PRIVATE KEY', '-----'].join('');

// #2943: env: process.env is REQUIRED, or spawned git misses the HOME swap.
function git(cwd: string, ...args: string[]): string {
  return execFileSync('git', ['-C', cwd, '-c', 'protocol.file.allow=always', ...args], {
    stdio: ['ignore', 'pipe', 'pipe'], encoding: 'utf-8', env: process.env,
  }).trim();
}

let root: string;
let work: string;
let bare: string;
let saved: Record<string, string | undefined> = {};

beforeEach(() => {
  root = mkdtempSync(join(tmpdir(), 'wsp-large-'));
  saved = {
    HOME: process.env.HOME,
    GBRAIN_HOME: process.env.GBRAIN_HOME,
    GBRAIN_GIT_ALLOW_FILE_TRANSPORT: process.env.GBRAIN_GIT_ALLOW_FILE_TRANSPORT,
  };
  process.env.HOME = mkdtempSync(join(root, 'home-'));
  process.env.GBRAIN_HOME = process.env.HOME;
  process.env.GBRAIN_GIT_ALLOW_FILE_TRANSPORT = '1';
  bare = mkdtempSync(join(root, 'origin-')) + '.git';
  execFileSync('git', ['init', '-q', '--bare', '-b', 'main', bare], { stdio: 'ignore', env: process.env });
  work = mkdtempSync(join(root, 'work-'));
  execFileSync('git', ['-c', 'protocol.file.allow=always', 'clone', '-q', bare, work], { stdio: 'ignore', env: process.env });
  git(work, 'config', 'user.email', '[email protected]');
  git(work, 'config', 'user.name', 'tester');
  writeFileSync(join(work, 'README.md'), 'init\n');
  git(work, 'add', 'README.md');
  git(work, 'commit', '-qm', 'init');
  git(work, 'push', '-q', 'origin', 'main');
});
afterEach(() => {
  for (const [k, v] of Object.entries(saved)) {
    if (v === undefined) delete process.env[k];
    else process.env[k] = v;
  }
  rmSync(root, { recursive: true, force: true });
});

function push() {
  for (const k of Object.keys(streamScanWork) as Array<keyof typeof streamScanWork>) streamScanWork[k] = 0;
  return workspacePush({ dir: work, branch: 'main', allowUnverifiedRemote: true });
}

/** The ~30 MiB file, with edits (0-based line index to a line of at most 99 chars) in place. */
function writeBig(edits: Map<number, string> = new Map()): void {
  const lines = new Array<string>(LINES).fill(FILLER);
  for (const [i, text] of edits) {
    expect(text.length).toBeLessThanOrEqual(99);
    lines[i] = text.padEnd(99, ' ');
  }
  writeFileSync(join(work, 'big.md'), lines.join('\n') + '\n');
}

/** A private key: header on line first, bodyLines base64 lines, footer after them. */
function placeKey(edits: Map<number, string>, first: number, bodyLines: number): void {
  edits.set(first, ' '.repeat(99 - BEGIN.length) + BEGIN);
  for (let i = 1; i <= bodyLines; i++) edits.set(first + i, randomBytes(75).toString('base64').slice(0, 99));
  edits.set(first + bodyLines + 1, END);
}

// Window 0 is [0, W); window k + 1 starts STEP after window k.
const W = STREAM_SCAN_WINDOW_CHARS;
const STEP = Math.floor((W - STREAM_SCAN_OVERLAP_CHARS) / LINE) * LINE;

describe('pre-push secret scan over the 25 MiB cap', () => {
  test('a clean ~30 MiB staged file is scanned in bounded windows and pushed', async () => {
    expect(SIZE).toBeGreaterThan(PUSH_MAX_SCAN_BYTES);
    writeBig();
    const r = await push();
    expect(r.status).toBe('pushed');
    expect(r.ok).toBe(true);
    expect(Number(git(bare, 'cat-file', '-s', 'refs/heads/main:big.md'))).toBe(SIZE);
    expect(streamScanWork.bytes).toBe(SIZE);
    expect(streamScanWork.chars).toBeGreaterThanOrEqual(SIZE);
    expect(streamScanWork.largestWindow).toBeLessThanOrEqual(STREAM_SCAN_WINDOW_CHARS);
    expect(streamScanWork.windows).toBe(Math.ceil((SIZE - W) / STEP) + 1);
  }, T);

  test('secrets across window seams and near the end block with their file and line, once each', async () => {
    const edits = new Map<number, string>();
    const acrossEnd = Math.floor(W / LINE); // the line window 0 cuts (at column W % 100)
    edits.set(acrossEnd, OPENAI + ' token');
    const keyAcrossStart = STEP / LINE - 10; // header before window 1's start, footer after it
    placeKey(edits, keyAcrossStart, 19);
    const keyAcrossEnd = Math.floor((STEP + W) / LINE) - 10; // straddles window 1's end
    placeKey(edits, keyAcrossEnd, 19);
    const nearEnd = LINES - 3;
    edits.set(nearEnd, 'deploy ' + GITHUB);
    expect(acrossEnd * LINE).toBeLessThan(W);
    expect(acrossEnd * LINE + OPENAI.length).toBeGreaterThan(W);
    writeBig(edits);
    const before = git(bare, 'rev-parse', 'refs/heads/main');
    const r = await push();
    expect(r.status).toBe('blocked_secrets');
    expect(r.ok).toBe(false);
    const got = (r.findings ?? []).map((f) => ({ file: f.file, line: f.line, pattern: f.pattern })).sort((a, b) => a.line - b.line);
    expect(got).toEqual([
      { file: 'big.md', line: keyAcrossStart + 1, pattern: 'private_key_pem' },
      { file: 'big.md', line: acrossEnd + 1, pattern: 'openai' },
      { file: 'big.md', line: keyAcrossEnd + 1, pattern: 'private_key_pem' },
      { file: 'big.md', line: nearEnd + 1, pattern: 'github_token' },
    ]);
    expect(JSON.stringify(r).includes(OPENAI)).toBe(false);
    expect(git(bare, 'rev-parse', 'refs/heads/main')).toBe(before); // nothing pushed
    expect(git(work, 'diff', '--cached', '--name-only')).toBe(''); // index restored
    expect(streamScanWork.largestWindow).toBeLessThanOrEqual(STREAM_SCAN_WINDOW_CHARS);
    expect(streamScanWork.bytes).toBe(SIZE);
  }, T);

  test('a newline-free stretch cut mid-line between an anchor and its value still blocks: Bearer and Basic at every seam', async () => {
    // One line: every window is cut mid-line at exactly the overlap before its end.
    const cut = (k: number): number => k * (W - STREAM_SCAN_OVERLAP_CHARS);
    const token = 'Q7' + randomBytes(30).toString('hex');
    const basic = Buffer.from('alice:' + randomBytes(9).toString('hex')).toString('base64');
    const placed: Array<{ at: number; text: string }> = [
      { at: cut(1) - ' Bearer '.length, text: ' Bearer ' + token + ' ' },
      { at: cut(2) - ' Bearer '.length + 4, text: ' Bearer ' + token.split('').reverse().join('') + ' ' },
      { at: cut(3) - ' Authorization: Basic '.length, text: ' Authorization: Basic ' + basic + ' ' },
    ];
    const size = 30 * 1024 ** 2;
    expect(cut(3) + STREAM_SCAN_OVERLAP_CHARS).toBeLessThan(size);
    const body = Buffer.alloc(size, 'a');
    for (const p of placed) body.write(p.text, p.at, 'latin1');
    writeFileSync(join(work, 'big.md'), body);
    const before = git(bare, 'rev-parse', 'refs/heads/main');
    const r = await push();
    expect(r.status).toBe('blocked_secrets');
    expect((r.findings ?? []).map((f) => `${f.file}:${f.line}:${f.pattern}`).sort()).toEqual(['big.md:1:basic_auth', 'big.md:1:bearer', 'big.md:1:bearer']);
    expect(JSON.stringify(r).includes(token)).toBe(false);
    expect(git(bare, 'rev-parse', 'refs/heads/main')).toBe(before);
    expect(streamScanWork.windows).toBe(4);
    expect(streamScanWork.largestWindow).toBeLessThanOrEqual(STREAM_SCAN_WINDOW_CHARS);
  }, T);

  test('a staged blob over the streamed-scan ceiling still blocks as unscannable', async () => {
    const huge = join(work, 'huge.bin');
    writeFileSync(huge, '');
    truncateSync(huge, PUSH_STREAM_SCAN_MAX_BYTES + 1); // sparse: no disk blocks, still that many bytes for git
    const before = git(bare, 'rev-parse', 'refs/heads/main');
    const r = await push();
    expect(r.status).toBe('blocked_unscannable');
    expect(r.unscannable?.some((u) => u.startsWith('huge.bin') && u.includes('scan ceiling'))).toBe(true);
    expect(r.reason).toContain('scan ceiling');
    expect(streamScanWork.bytes).toBe(0); // refused by its recorded size, never read
    expect(git(bare, 'rev-parse', 'refs/heads/main')).toBe(before);
    expect(git(work, 'diff', '--cached', '--name-only')).toBe('');
  }, T);
});