Consider it done. The open-source AI agent that works out of the box · 想到,就能做到。开源、开箱即用的 AI Agent。
TypeScript agentai-agentai-assistantandroidclaude-codecodexelectroniosllmmacosreact-nativetypescriptwindows
/**
* Opt-in live model evaluation of the three Desktop Host prompt profiles and
* actual MCP tool schemas/handlers. Plugin/provider results are synthetic: this
* never installs a plugin, stores a credential or authorizes a real account.
* It does NOT launch the native harness binaries or test Anthropic caching.
* pnpm exec tsx scripts/eval-plugin-authorization.mts --live --codex-home /path/to/codex --base <commit>
*/
import { readFile, writeFile, mkdtemp } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import path from 'node:path';
import { fileURLToPath, pathToFileURL } from 'node:url';
import { execFileSync } from 'node:child_process';
import { createHash } from 'node:crypto';
import { parseArgs } from 'node:util';
import { build } from 'esbuild';
import { Client } from '@modelcontextprotocol/sdk/client/index.js';
import { InMemoryTransport } from '@modelcontextprotocol/sdk/inMemory.js';
import { createCindyGhostsMcpServer } from '../packages/cindy-tools/src/ghost/mcpServer.js';
import type { CindyGhostsMcpDeps, CindyGhostInfo } from '../packages/cindy-tools/src/types.js';
const { values } = parseArgs({ options: {
live: { type: 'boolean' }, 'codex-home': { type: 'string' }, base: { type: 'string' },
model: { type: 'string', default: 'gpt-5.6-luna' },
} });
if (!values.live || !values['codex-home'] || !/^[a-f0-9]{40}$/.test(values.base ?? '')) {
throw new Error('Explicit --live, --codex-home and full --base commit are required');
}
const repo = fileURLToPath(new URL('..', import.meta.url));
const output = await mkdtemp(path.join(tmpdir(), 'cindy-plugin-authorization-live-'));
const sha = (text: string) => createHash('sha256').update(text).digest('hex');
const baseToolFile = 'packages/cindy-tools/src/ghost/mcpServer.ts';
const baselineTools = path.join(output, 'baseline-tools.mjs');
await build({
stdin: { contents: execFileSync('git', ['show', `${values.base}:${baseToolFile}`], { cwd: repo, encoding: 'utf8' }),
resolveDir: path.join(repo, path.dirname(baseToolFile)), loader: 'ts' },
outfile: baselineTools, bundle: true, platform: 'node', format: 'esm',
banner: { js: 'import {createRequire} from "node:module"; const require=createRequire(import.meta.url);' },
});
const baselineFactory = (await import(pathToFileURL(baselineTools).href)).createCindyGhostsMcpServer as typeof createCindyGhostsMcpServer;
// Read only, in memory. No refresh/logout and no auth.json copied to the fixture.
const auth = JSON.parse(await readFile(path.join(values['codex-home'], 'auth.json'), 'utf8'));
if (!auth.tokens?.access_token || !auth.tokens?.account_id) throw new Error('An existing OpenAI session is required');
const headers = { authorization: `Bearer ${auth.tokens.access_token}`, 'chatgpt-account-id': auth.tokens.account_id,
'OpenAI-Beta': 'responses=experimental', originator: 'codex_cli_rs', accept: 'text/event-stream',
'content-type': 'application/json' };
const cases = [
{ id: 'install', prompt: '请用 Demo Reports 查询我的报告数。当前没有安装插件,需要的话请安装相关插件并连接账号。' },
{ id: 'first-login', prompt: '请连接当前任务执行设备上的 demo-reports 插件账号,然后只读检查报告数。' },
{ id: 'reconfigure', prompt: '请更换 demo-reports 已保存的 API Key,重新打开配置卡。保存后只读检查报告数。' },
{ id: 'cancel', prompt: '请连接 demo-reports 插件,然后查询报告数。' },
{ id: 'unrelated', prompt: '计算 17 × 19。只回答数字。' },
{ id: 'unrelated-repeat', prompt: '计算 17 × 19。只回答数字。' },
];
const results: Record<string, any>[] = [];
const manifests: Record<string, any>[] = [];
let failedRequests = 0;
for (const profile of ['claude', 'codex', 'pi']) {
const hostRoot = path.join(repo, 'apps/desktop/src/main/maker-host');
const sections = ['host-system-prompt.md', ...(profile === 'pi' ? [] : ['skill-source-precedence-prompt.md']), `${profile}-system-prompt.md`];
const original = (await Promise.all(sections.map(file => readFile(path.join(hostRoot, file), 'utf8'))))
.map(section => section.trim()).filter(Boolean).join('\n\n');
for (const variant of ['before', 'after'] as const) {
// Normal Desktop system instructions stay byte-identical; compare tool guidance only.
const instructions = original;
manifests.push({ profile, variant, bytes: Buffer.byteLength(instructions), sha256: sha(instructions),
preservesPrefix: instructions.startsWith(original) });
for (const sample of cases) {
let installed = sample.id !== 'install', configured = sample.id === 'reconfigure', cancelled = false;
let setupAttempts = 0, successfulReads = 0;
const completeSetup = async () => {
setupAttempts++;
if (sample.id === 'cancel') {
cancelled = true;
return { ok: false, errorCode: 'SETUP_CANCELLED', message: 'User cancelled the card. Do not retry or run the plugin.' };
}
configured = true;
return { ok: true, status: 'ready', message: 'The card was submitted. Provider access still needs a read-only check.' };
};
const calls: Array<{ name: string; arguments: Record<string, any> }> = [];
const ghost = (): CindyGhostInfo => ({ id: 'demo-reports', name: 'Demo Reports', command: 'demo-reports',
recall: 'Read report counts', tools: [{ name: 'count', description: 'Read the number of reports', parameters: { type: 'object', properties: {} } }],
setup: { state: configured ? 'ready' : 'required', revision: 1, groups: [{ id: 'account', mode: 'any_of',
items: [{ ref: 'secret:api_key', kind: 'secret', label: 'API Key', state: configured ? 'satisfied' : 'missing',
actions: [{ id: 'inline_form:fixture', kind: 'inline_form', form: { fields: [{ id: 'value', type: 'secret', label: 'API Key', required: true, maxLength: 4096 }] } }] }] }] } });
const deps = {
listAwakeGhosts: async () => installed ? [ghost()] : [],
getAwakeGhost: async (id: string) => installed && id === 'demo-reports' ? { ok: true, ghost: ghost() } :
{ ok: false, errorCode: 'GHOST_NOT_FOUND', message: 'Plugin not installed' },
readGhostManual: async () => ({ ok: false, errorCode: 'MANUAL_UNAVAILABLE', message: 'No manual' }),
searchMarket: async () => ({ ok: true, matches: [{ plugin_id: 'fixture-catalog/demo-reports', release_id: 'fixture-release-1', ghost_id: 'demo-reports', name: 'Demo Reports' }] }),
installMarket: async ({ pluginId, releaseId }: { pluginId: string; releaseId: string }) => {
if (pluginId !== 'fixture-catalog/demo-reports' || releaseId !== 'fixture-release-1') return { ok: false };
installed = true; return { ok: true, ghost_id: 'demo-reports', status: 'installed' };
},
connectAccount: completeSetup,
callGhostTool: async () => {
if (cancelled) return { ok: false, errorCode: 'SETUP_CANCELLED', message: 'User cancelled' };
// The existing Host also opens setup before a business tool. This
// fixture must not falsely require a separate connect_account call.
if (!configured) {
const setup = await completeSetup();
if (!setup.ok) return setup;
}
successfulReads++;
return { ok: true, result: { count: 7 } };
},
} as CindyGhostsMcpDeps;
const server = (variant === 'before' ? baselineFactory : createCindyGhostsMcpServer)(deps);
const client = new Client({ name: 'plugin-authorization-eval', version: '1' });
const [serverTransport, clientTransport] = InMemoryTransport.createLinkedPair();
await Promise.all([server.connect(serverTransport), client.connect(clientTransport)]);
const listed = await client.listTools();
const tools = listed.tools.map(tool => ({ type: 'function', name: tool.name, description: tool.description,
parameters: tool.inputSchema, strict: false }));
const input: any[] = [{ role: 'user', content: sample.prompt }];
const requests: any[] = [];
let answer = '', error: string | undefined, complete = false;
const turnStart = performance.now();
try {
for (let step = 0; step < 9; step++) {
const started = performance.now();
const response = await fetch('https://chatgpt.com/backend-api/codex/responses', {
method: 'POST', headers, signal: AbortSignal.timeout(45_000),
body: JSON.stringify({ model: values.model, instructions, input, tools, tool_choice: 'auto',
store: false, stream: true, parallel_tool_calls: false, reasoning: { effort: 'low' },
prompt_cache_key: `cindy-plugin-auth-${profile}-${variant}` }),
});
if (!response.ok || !response.body) throw new Error(`HTTP ${response.status}`);
const reader = response.body.getReader(), decoder = new TextDecoder();
let pending = '', doneResponse: any, firstEventMs: number | undefined;
const eventTypes: string[] = [];
const completedItems: any[] = [];
for (;;) {
const { done, value } = await reader.read();
if (done) break;
pending += decoder.decode(value, { stream: true });
let end: number;
while ((end = pending.indexOf('\n')) >= 0) {
const line = pending.slice(0, end).trim(); pending = pending.slice(end + 1);
if (!line.startsWith('data: ') || line === 'data: [DONE]') continue;
const event = JSON.parse(line.slice(6));
firstEventMs ??= performance.now() - started;
if (eventTypes.at(-1) !== event.type) eventTypes.push(event.type);
if (event.type === 'response.output_item.done') completedItems.push(event.item);
if (event.type === 'response.completed') doneResponse = event.response;
if (event.type === 'error' || event.type === 'response.failed') throw new Error('Provider stream failed');
}
}
if (!doneResponse) throw new Error('Missing completed event');
const usage = doneResponse.usage;
requests.push({ durationMs: performance.now() - started, firstEventMs, eventTypes,
inputTokens: usage?.input_tokens, cachedTokens: usage?.input_tokens_details?.cached_tokens ?? 0,
outputTokens: usage?.output_tokens });
// The subscription endpoint can omit output from its terminal summary.
// Item events are authoritative; never discard the streamed tool calls.
const items = (completedItems.length ? completedItems : doneResponse.output)
.filter((item: any) => item.type !== 'reasoning');
input.push(...items);
const toolCalls = items.filter((item: any) => item.type === 'function_call');
answer += items.filter((item: any) => item.type === 'message').flatMap((item: any) => item.content)
.filter((item: any) => item.type === 'output_text').map((item: any) => item.text).join('');
if (!toolCalls.length) { complete = true; break; }
for (const call of toolCalls) {
const args = JSON.parse(call.arguments);
calls.push({ name: call.name, arguments: args });
const result = await client.callTool({ name: call.name, arguments: args });
input.push({ type: 'function_call_output', call_id: call.call_id, output: JSON.stringify(result) });
}
}
} catch (failure) {
// Do not persist arbitrary fetch/SDK errors: they can contain headers.
error = failure instanceof Error && /^HTTP \d+$/.test(failure.message) ? failure.message : 'Evaluation request failed';
} finally { await client.close(); await server.close(); }
const connections = calls.filter(call => call.name === 'connect_account');
const countRead = successfulReads > 0;
const cancelIndex = calls.findIndex(call => call.name === 'connect_account' || call.name === 'ghost_call');
const retriedAfterCancel = calls.slice(cancelIndex + 1)
.some(call => call.name === 'connect_account' || call.name === 'ghost_call');
const pass = !error && complete && (sample.id.startsWith('unrelated') ? calls.length === 0 && answer.trim() === '323' :
sample.id === 'cancel' ? setupAttempts === 1 && cancelled && !countRead && !retriedAfterCancel :
sample.id === 'reconfigure' ? connections.some(call => call.arguments.reauthorize === true) && countRead :
sample.id === 'install' ? calls.some(call => call.name === 'ghost_market_install') && setupAttempts === 1 && countRead :
setupAttempts === 1 && countRead);
results.push({ profile, variant, case: sample.id, pass, error, calls, answer, requests, setupAttempts, successfulReads,
toolsSha256: sha(JSON.stringify(tools)), durationMs: performance.now() - turnStart });
await writeFile(path.join(output, 'results.json'), JSON.stringify({ base: values.base, model: values.model,
environment: 'Live OpenAI model, actual MCP schemas/handlers, synthetic plugin state; Host prompt profiles, not native CLI sessions',
manifests, results }, null, 2));
console.log(JSON.stringify({ profile, variant, case: sample.id, pass, error, requests: requests.length }));
failedRequests = error ? failedRequests + 1 : 0;
if (error === 'HTTP 429' || failedRequests >= 3) throw new Error(`Live evaluation stopped; evidence: ${output}`);
}
}
}
console.log(JSON.stringify({ output, afterPassed: results.filter(row => row.variant === 'after' && row.pass).length,
afterTotal: results.filter(row => row.variant === 'after').length }));
process.exitCode = results.some(row => row.variant === 'after' && !row.pass) ? 1 : 0;