Skip to content

armgit / cindy

Public Imported from GitHub
0

Consider it done. The open-source AI agent that works out of the box · 想到,就能做到。开源、开箱即用的 AI Agent。

16.4 KB
Raw Blame
#!/usr/bin/env node
/**
 * 从 desktop log 校验 codex-proxy E2E 不变量。
 *
 * 用法:
 *   node apps/desktop/scripts/assert-codex-proxy-e2e.mjs \
 *     --log-dir apps/desktop/logs \
 *     --expect <thread-id>:<unique-sentinel> \
 *     --plain-thread <ordinary-codex-thread-id> \
 *     --body-dump apps/desktop/logs/codex-proxy-dumps
 *
 * 主要输入是 createMakerLogger('codex-proxy') 输出的 desktop log NDJSON 记录。
 * 可选的 --body-dump 路径只用于本地校验 transformed request dump,例如验证
 * sentinel 是否存在、是否重复。它可以是 JSON 文件目录、单个 JSON 文件,或包含
 * 以下任一结构的 NDJSON:
 *   { "threadId": "...", "body": { "instructions": "...", "input": [...] } }
 * or
 *   { "selectedThreadId": "...", "instructions": "...", "input": [...] }
 */

import fs from 'node:fs';
import path from 'node:path';
import process from 'node:process';
import { pathToFileURL } from 'node:url';

function parseArgs(argv) {
  const out = {
    logDir: path.resolve('apps/desktop/logs'),
    threads: [],
    minThreads: 1,
    minHitsPerThread: 1,
    sentinel: '',
    bodyDump: '',
    expectations: [],
    plainThreads: [],
    forbidDeveloper: [],
    forbidBodyField: ['output_config'],
  };
  for (let i = 0; i < argv.length; i += 1) {
    const arg = argv[i];
    const next = () => {
      i += 1;
      if (i >= argv.length) throw new Error(`${arg} requires a value`);
      return argv[i];
    };
    switch (arg) {
      case '--log-dir':
        out.logDir = path.resolve(next());
        break;
      case '--thread':
        out.threads.push(...next().split(',').map((s) => s.trim()).filter(Boolean));
        break;
      case '--min-threads':
        out.minThreads = Number(next());
        break;
      case '--min-hits-per-thread':
        out.minHitsPerThread = Number(next());
        break;
      case '--sentinel':
        out.sentinel = next();
        break;
      case '--expect':
        out.expectations.push(parseExpectation(next()));
        break;
      case '--plain-thread':
        out.plainThreads.push(...next().split(',').map((s) => s.trim()).filter(Boolean));
        break;
      case '--forbid-developer':
        out.forbidDeveloper.push(next());
        break;
      case '--forbid-body-field':
        out.forbidBodyField.push(next());
        break;
      case '--body-dump':
        out.bodyDump = path.resolve(next());
        break;
      case '--help':
      case '-h':
        printHelp();
        process.exit(0);
        break;
      default:
        throw new Error(`Unknown argument: ${arg}`);
    }
  }
  if (out.expectations.length > 0) {
    out.minThreads = Math.max(out.minThreads, out.expectations.length);
  }
  if (!Number.isFinite(out.minThreads) || out.minThreads < 1) {
    throw new Error('--min-threads must be a positive number');
  }
  if (!Number.isFinite(out.minHitsPerThread) || out.minHitsPerThread < 1) {
    throw new Error('--min-hits-per-thread must be a positive number');
  }
  return out;
}

function parseExpectation(raw) {
  const colon = raw.indexOf(':');
  const equal = raw.indexOf('=');
  const sep = colon >= 0 ? colon : equal;
  if (sep <= 0 || sep >= raw.length - 1) {
    throw new Error('--expect must be formatted as <threadId>:<sentinel>');
  }
  return {
    threadId: raw.slice(0, sep).trim(),
    sentinel: raw.slice(sep + 1),
  };
}

function printHelp() {
  process.stdout.write(`Assert codex-proxy E2E logs.

Options:
  --log-dir <dir>             Desktop logs dir. Default: apps/desktop/logs
  --thread <id[,id...]>       Restrict assertions to these thread ids.
  --expect <id:sentinel>      Assert a transformed body for this thread contains only its sentinel.
                              Can be repeated for multi-session no-cross-contamination checks.
  --plain-thread <id[,id...]> Assert this ordinary codex thread has no orca worker contract in instructions.
  --min-threads <n>           Require at least n distinct selectedThreadId values. Default: 1
  --min-hits-per-thread <n>   Require at least n injection hits per selected thread. Default: 1
  --sentinel <text>           Back-compat single-sentinel assertion for selected transformed body dumps.
  --forbid-developer <text>   Extra marker that must not appear in input[] developer messages.
  --forbid-body-field <name>  Field name that must not appear anywhere in transformed bodies.
                              Default: output_config. Can be repeated.
  --body-dump <path>          Optional transformed request body dump file or directory.
`);
}

const warnings = [];

function warn(message) {
  warnings.push(message);
}

function walkFiles(root) {
  const files = [];
  const visit = (dir) => {
    let entries;
    try {
      entries = fs.readdirSync(dir, { withFileTypes: true });
    } catch (err) {
      warn(`cannot read directory ${dir}: ${err instanceof Error ? err.message : String(err)}`);
      return;
    }
    for (const entry of entries) {
      const full = path.join(dir, entry.name);
      if (entry.isDirectory()) visit(full);
      else if (entry.isFile()) files.push(full);
    }
  };
  visit(root);
  return files;
}

function logFiles(logDir) {
  return walkFiles(logDir).filter((file) => {
    const base = path.basename(file);
    return (base.startsWith('agent-') || /^\d{4}-\d{2}-\d{2}\.ndjson$/.test(base)) &&
      base.endsWith('.ndjson');
  });
}

function parseValue(raw) {
  const text = raw.trim();
  if (text === 'true') return true;
  if (text === 'false') return false;
  if (text === 'null') return null;
  if (/^-?\d+(?:\.\d+)?$/.test(text)) return Number(text);
  return text;
}

function parseInjectionFields(msg) {
  if (!msg.includes('event') || !msg.includes('codex_proxy_injection')) return null;
  const fields = {};
  for (const line of msg.split('\n')) {
    const match = /^\s*([A-Za-z0-9_]+)\s+:\s*([\s\S]*)$/.exec(line);
    if (!match) continue;
    fields[match[1]] = parseValue(match[2]);
  }
  return fields.event === 'codex_proxy_injection' ? fields : null;
}

function readInjectionEvents(logDir) {
  const events = [];
  for (const file of logFiles(logDir)) {
    const lines = fs.readFileSync(file, 'utf8').split(/\r?\n/).filter(Boolean);
    for (const line of lines) {
      let record;
      try {
        record = JSON.parse(line);
      } catch {
        continue;
      }
      const scope = typeof record.scope === 'string' ? record.scope : '';
      const msg = typeof record.msg === 'string' ? record.msg : '';
      if (!/^codex-proxy(\/|$)/.test(scope)) continue;
      const fields = parseInjectionFields(msg);
      if (!fields) continue;
      events.push({
        file,
        ts: typeof record.ts === 'number' ? record.ts : 0,
        seq: typeof record.seq === 'number' ? record.seq : 0,
        ...fields,
      });
    }
  }
  events.sort((a, b) => (a.ts - b.ts) || (a.seq - b.seq));
  return events;
}

function countOccurrences(haystack, needle) {
  if (!needle) return 0;
  let count = 0;
  let offset = 0;
  while (true) {
    const idx = haystack.indexOf(needle, offset);
    if (idx < 0) return count;
    count += 1;
    offset = idx + needle.length;
  }
}

function readBodyDump(fileOrDir) {
  if (!fileOrDir) return [];
  let stat;
  try {
    stat = fs.statSync(fileOrDir);
  } catch (err) {
    if (err && typeof err === 'object' && err.code === 'ENOENT') {
      throw new Error(`body dump path not found: ${fileOrDir}`);
    }
    throw new Error(`body dump path is not readable: ${fileOrDir}`);
  }
  if (stat.isDirectory()) {
    return walkFiles(fileOrDir)
      .filter((file) => file.endsWith('.json') || file.endsWith('.ndjson'))
      .sort()
      .flatMap((file) => readBodyDumpFile(file));
  }
  return readBodyDumpFile(fileOrDir);
}

function readBodyDumpFile(file) {
  let raw;
  try {
    raw = fs.readFileSync(file, 'utf8').trim();
  } catch (err) {
    warn(`cannot read body dump file ${file}: ${err instanceof Error ? err.message : String(err)}`);
    return [];
  }
  if (!raw) return [];

  const records = [];
  if (file.endsWith('.json')) {
    let parsed;
    try {
      parsed = JSON.parse(raw);
    } catch {
      warn(`cannot parse JSON body dump file ${file}; skipping`);
      return [];
    }
    records.push(...(Array.isArray(parsed) ? parsed : [parsed]));
  } else {
    for (const line of raw.split(/\r?\n/).filter(Boolean)) {
      try {
        records.push(JSON.parse(line));
      } catch {
        warn(`cannot parse NDJSON body dump line in ${file}; skipping`);
      }
    }
  }

  return records.map((parsed, index) => normalizeBodyDumpRecord(parsed, file, index));
}

function normalizeBodyDumpRecord(parsed, file, index) {
  const body = parsed.body && typeof parsed.body === 'object' ? parsed.body : parsed;
  const threadId = parsed.threadId ?? parsed.selectedThreadId ?? body.threadId ?? body.selectedThreadId;
  return {
    index,
    file,
    threadId: typeof threadId === 'string' ? threadId : '',
    instructions: typeof body.instructions === 'string' ? body.instructions : '',
    input: Array.isArray(body.input) ? body.input : [],
    body,
  };
}

function containsField(value, fieldName) {
  if (value == null || typeof value !== 'object') return false;
  if (Array.isArray(value)) return value.some((item) => containsField(item, fieldName));
  if (Object.hasOwn(value, fieldName)) return true;
  return Object.values(value).some((item) => containsField(item, fieldName));
}

function fail(message) {
  throw new Error(message);
}

function selectedThreadIds(opts) {
  const ids = new Set(opts.threads);
  for (const expectation of opts.expectations) ids.add(expectation.threadId);
  for (const threadId of opts.plainThreads) ids.add(threadId);
  return [...ids];
}

function assertInjections(events, opts) {
  if (events.length === 0) fail(`No codex_proxy_injection events found under ${opts.logDir}`);
  const threadIds = selectedThreadIds(opts);
  const selected = threadIds.length > 0
    ? events.filter((event) => threadIds.includes(String(event.selectedThreadId ?? '')))
    : events;
  if (selected.length === 0) {
    fail(`No codex_proxy_injection events matched threads: ${threadIds.join(', ')}`);
  }

  const first = selected[0];
  if (first.selectedHeaderName !== 'thread-id') {
    fail(`First selected request used ${first.selectedHeaderName}; expected thread-id`);
  }
  if (first.registryHit !== true) {
    fail('First selected request did not hit registry');
  }
  if (first.appended !== true) {
    fail(`First selected request appended=${first.appended}; expected true`);
  }

  for (const event of selected) {
    if (event.registryHit !== true) {
      fail(`Registry miss for thread ${event.selectedThreadId ?? '<missing>'} in ${event.file}`);
    }
    if (event.appended !== true && event.alreadyPresent !== true) {
      fail(`Injection did not append or report alreadyPresent for thread ${event.selectedThreadId ?? '<missing>'} in ${event.file}`);
    }
  }

  const byThread = new Map();
  for (const event of selected) {
    const threadId = String(event.selectedThreadId ?? '');
    if (!threadId) fail(`Injection event missing selectedThreadId in ${event.file}`);
    byThread.set(threadId, (byThread.get(threadId) ?? 0) + 1);
  }
  if (byThread.size < opts.minThreads) {
    fail(`Expected at least ${opts.minThreads} distinct threads, got ${byThread.size}`);
  }
  for (const [threadId, count] of byThread) {
    if (count < opts.minHitsPerThread) {
      fail(`Thread ${threadId} has ${count} injection hit(s), expected at least ${opts.minHitsPerThread}`);
    }
  }
  return { selected, byThread };
}

function textContent(value) {
  if (value == null) return '';
  if (typeof value === 'string') return value;
  if (typeof value === 'number' || typeof value === 'boolean') return String(value);
  if (Array.isArray(value)) return value.map(textContent).join('\n');
  if (typeof value === 'object') return Object.values(value).map(textContent).join('\n');
  return '';
}

function developerMessages(input) {
  return input.filter((item) => item && typeof item === 'object' && item.role === 'developer');
}

function bodyLabel(body) {
  return `${body.file ?? '<memory>'}#${body.index} thread=${body.threadId || '<unknown>'}`;
}

function assertSentinelBodies(opts) {
  const hasExpectations = opts.expectations.length > 0;
  if (!opts.sentinel && !hasExpectations && opts.forbidDeveloper.length === 0 && opts.plainThreads.length === 0) {
    return { bodyCount: 0 };
  }
  if (!opts.bodyDump) {
    fail('sentinel / body assertions require --body-dump so transformed instructions can be inspected');
  }
  const bodies = readBodyDump(opts.bodyDump);
  const threadIds = selectedThreadIds(opts);
  const selected = threadIds.length > 0
    ? bodies.filter((body) => threadIds.includes(body.threadId))
    : bodies;
  if (selected.length === 0) {
    fail(`No transformed body dump records matched sentinel assertion (${opts.bodyDump})`);
  }

  for (const body of selected) {
    for (const fieldName of opts.forbidBodyField) {
      if (fieldName && containsField(body.body, fieldName)) {
        fail(`Forbidden body field "${fieldName}" appears in ${bodyLabel(body)}`);
      }
    }
  }

  const expectedSentinels = hasExpectations
    ? opts.expectations.map((expectation) => expectation.sentinel)
    : opts.sentinel ? [opts.sentinel] : [];
  const developerForbidden = [...new Set([...expectedSentinels, ...opts.forbidDeveloper])].filter(Boolean);

  for (const body of selected) {
    for (const message of developerMessages(body.input)) {
      const text = textContent(message);
      for (const marker of developerForbidden) {
        if (text.includes(marker)) {
          fail(`Developer input message contains forbidden marker "${marker}" in ${bodyLabel(body)}`);
        }
      }
    }
  }

  if (hasExpectations) {
    for (const expectation of opts.expectations) {
      const matching = selected.filter((body) => body.threadId === expectation.threadId);
      if (matching.length === 0) {
        fail(`No transformed body dump records matched expected thread ${expectation.threadId}`);
      }
      for (const body of matching) {
        const ownCount = countOccurrences(body.instructions, expectation.sentinel);
        if (ownCount !== 1) {
          fail(`Sentinel occurrence count for ${bodyLabel(body)}: ${ownCount}, expected 1`);
        }
        for (const other of opts.expectations) {
          if (other.threadId === expectation.threadId) continue;
          const otherCount = countOccurrences(body.instructions, other.sentinel);
          if (otherCount !== 0) {
            fail(`Cross-thread sentinel "${other.sentinel}" appeared ${otherCount} time(s) in ${bodyLabel(body)}`);
          }
        }
      }
    }
  } else if (opts.sentinel) {
    for (const body of selected) {
      const count = countOccurrences(body.instructions, opts.sentinel);
      if (count !== 1) {
        fail(`Sentinel occurrence count for ${bodyLabel(body)}: ${count}, expected 1`);
      }
    }
  }

  const plainMarkers = ['send_to_lead', 'worker_id', 'workerId'];
  for (const threadId of opts.plainThreads) {
    const matching = selected.filter((body) => body.threadId === threadId);
    if (matching.length === 0) fail(`No transformed body dump records matched plain thread ${threadId}`);
    for (const body of matching) {
      for (const marker of plainMarkers) {
        if (body.instructions.includes(marker)) {
          fail(`Plain codex thread ${threadId} instructions contain orca marker "${marker}" in ${bodyLabel(body)}`);
        }
      }
    }
  }
  return { bodyCount: selected.length };
}

export function runAssertions(opts) {
  const events = readInjectionEvents(opts.logDir);
  const injectionSummary = assertInjections(events, opts);
  const bodySummary = assertSentinelBodies(opts);
  return {
    events: injectionSummary.selected.length,
    threads: injectionSummary.byThread.size,
    bodies: bodySummary.bodyCount,
  };
}

if (import.meta.url === pathToFileURL(process.argv[1] ?? '').href) {
  try {
    const opts = parseArgs(process.argv.slice(2));
    const summary = runAssertions(opts);
    if (warnings.length > 0) {
      process.stderr.write(`${warnings.map((message) => `[warn] ${message}`).join('\n')}\n`);
    }
    process.stdout.write(
      `codex-proxy E2E assertions passed: events=${summary.events}, threads=${summary.threads}, bodies=${summary.bodies}\n`,
    );
  } catch (err) {
    if (warnings.length > 0) {
      process.stderr.write(`${warnings.map((message) => `[warn] ${message}`).join('\n')}\n`);
    }
    process.stderr.write(`codex-proxy E2E assertions failed: ${err instanceof Error ? err.message : String(err)}\n`);
    process.exitCode = 1;
  }
}