Skip to content

armgit / cindy

Public Imported from GitHub
0

Consider it done. The open-source AI agent that works out of the box · 想到,就能做到。开源、开箱即用的 AI Agent。

test(codex): clean native speed resources on failure

Chrisf89d9e256c37
diff --git a/packages/maker-core/src/agents/codex/token-speed.native.test.ts b/packages/maker-core/src/agents/codex/token-speed.native.test.ts
index 1980706e2c..43b0b2b9c5 100644
--- a/packages/maker-core/src/agents/codex/token-speed.native.test.ts
+++ b/packages/maker-core/src/agents/codex/token-speed.native.test.ts
@@ -14,229 +14,252 @@ it.skipIf(!process.env.CINDY_CODEX_TEST_BINARY)(
'measures real native tool turns and resets on a new user send',
async () => {
const root = await mkdtemp(path.join(tmpdir(), 'cindy-speed-native-'));
- const home = path.join(root, 'codex');
- const workdir = path.join(root, 'work');
- await mkdir(home);
- await mkdir(workdir);
- const warnings: unknown[] = [];
- const logger: Logger = {
- trace() {},
- debug() {},
- info() {},
- warn(...a: unknown[]) {
- warnings.push(a);
- },
- error(...a: unknown[]) {
- warnings.push(a);
- },
- fatal() {},
- child() {
- return logger;
- },
- };
- let requests = 0;
- const server = createServer(async (req, res) => {
- for await (const _ of req) {
- /* drain local request */
- }
- if (!req.url?.endsWith('/responses')) {
- res.writeHead(404).end();
- return;
- }
- const n = ++requests;
- const item =
- n === 1
- ? {
- type: 'function_call',
- id: 'fc-1',
- call_id: 'tool-1',
- name: 'exec_command',
- arguments: JSON.stringify({
- cmd: 'node -e "setTimeout(()=>{},1000)"',
- yield_time_ms: 1000,
- }),
- status: 'completed',
- }
- : {
- type: 'message',
- id: `msg-${n}`,
- role: 'assistant',
- status: 'completed',
- content: [
- {
- type: 'output_text',
- text: 'Speed validation complete.',
- annotations: [],
- },
- ],
- };
- res.writeHead(200, {
- 'content-type': 'text/event-stream',
- connection: 'close',
- });
- const send = (event: unknown) =>
- res.write(`data: ${JSON.stringify(event)}\n\n`);
- send({
- type: 'response.created',
- response: { id: `r-${n}`, status: 'in_progress', output: [] },
- });
- // real-wait: integration tier measures native wall-clock generation against a local SSE server.
- await new Promise((r) => setTimeout(r, 1100));
- send({ type: 'response.output_item.done', output_index: 0, item });
- send({
- type: 'response.completed',
- response: {
- id: `r-${n}`,
- status: 'completed',
- output: [item],
- usage: { input_tokens: 10, output_tokens: 100, total_tokens: 110 },
- },
- });
- res.end();
- });
- await new Promise<void>((r) => server.listen(0, '127.0.0.1', r));
- const address = server.address();
- assert(address && typeof address !== 'string');
- const endpoint = `http://127.0.0.1:${address.port}/v1`;
- const agent = new CodexAgent({
- binaryPath: process.env.CINDY_CODEX_TEST_BINARY!,
- logger,
- runtimeConfig: {},
- prepareCodexExtraSpawnConfig: async () => ({
- extraArgs: [
- '-c',
- 'model_provider="fixture"',
- '-c',
- `model_providers.fixture={name="Local speed validation",base_url="${endpoint}",wire_api="responses",requires_openai_auth=false,request_max_retries=0,supports_websockets=false}`,
- '-c',
- 'check_for_update_on_startup=false',
- '-c',
- 'analytics.enabled=false',
- '-c',
- 'features.code_mode=false',
- // Fresh profile: keep the adapter-disabled MCP transport syntactically valid.
- '-c',
- 'mcp_servers.cindy_memory={command="false",enabled=false}',
- ],
- extraEnv: {},
- }),
- auth: {
- getState: async () => ({ authenticated: true }),
- triggerLogin: async () => ({ authenticated: true }),
- logout: async () => {},
- getAuthEnv: async () => ({
- HOME: root,
- USERPROFILE: root,
- CODEX_HOME: home,
- OPENAI_API_KEY: '',
- CODEX_API_KEY: '',
- NO_PROXY: '127.0.0.1,localhost',
- }),
- },
- });
- let handle: AgentSessionHandle | undefined;
- const events: AgentEvent[] = [];
try {
- handle = await agent.startSession({
- sessionId: 'speed-native',
- model: 'gpt-6-astra',
- workingDir: workdir,
- permissionMode: 'bypassPermissions',
- makerMemoryEnabled: false,
- });
- let doneResolve: () => void = () => {};
- void (async () => {
- for await (const e of handle!.events()) {
- if (
- e.type === 'status' ||
- e.type === 'done' ||
- e.type === 'error' ||
- e.type === 'tool_use' ||
- e.type === 'tool_result'
- )
- events.push(e);
- if (e.type === 'done') doneResolve();
+ const home = path.join(root, 'codex');
+ const workdir = path.join(root, 'work');
+ await mkdir(home);
+ await mkdir(workdir);
+ const warnings: unknown[] = [];
+ const logger: Logger = {
+ trace() {},
+ debug() {},
+ info() {},
+ warn(...a: unknown[]) {
+ warnings.push(a);
+ },
+ error(...a: unknown[]) {
+ warnings.push(a);
+ },
+ fatal() {},
+ child() {
+ return logger;
+ },
+ };
+ let requests = 0;
+ const server = createServer(async (req, res) => {
+ for await (const _ of req) {
+ /* drain local request */
}
- })();
- const run = async () => {
- const done = new Promise<void>((r) => (doneResolve = r));
- await handle!.send(
- { type: 'user', content: 'Run the tool and finish.' },
- { throwOnStartFailure: true },
- );
- let timer: ReturnType<typeof setTimeout> | undefined;
+ if (!req.url?.endsWith('/responses')) {
+ res.writeHead(404).end();
+ return;
+ }
+ const n = ++requests;
+ const item =
+ n === 1
+ ? {
+ type: 'function_call',
+ id: 'fc-1',
+ call_id: 'tool-1',
+ name: 'exec_command',
+ arguments: JSON.stringify({
+ cmd: 'node -e "setTimeout(()=>{},1000)"',
+ yield_time_ms: 1000,
+ }),
+ status: 'completed',
+ }
+ : {
+ type: 'message',
+ id: `msg-${n}`,
+ role: 'assistant',
+ status: 'completed',
+ content: [
+ {
+ type: 'output_text',
+ text: 'Speed validation complete.',
+ annotations: [],
+ },
+ ],
+ };
+ res.writeHead(200, {
+ 'content-type': 'text/event-stream',
+ connection: 'close',
+ });
+ const send = (event: unknown) =>
+ res.write(`data: ${JSON.stringify(event)}\n\n`);
+ send({
+ type: 'response.created',
+ response: { id: `r-${n}`, status: 'in_progress', output: [] },
+ });
+ // real-wait: integration tier measures native wall-clock generation against a local SSE server.
+ await new Promise((r) => setTimeout(r, 1100));
+ send({ type: 'response.output_item.done', output_index: 0, item });
+ send({
+ type: 'response.completed',
+ response: {
+ id: `r-${n}`,
+ status: 'completed',
+ output: [item],
+ usage: { input_tokens: 10, output_tokens: 100, total_tokens: 110 },
+ },
+ });
+ res.end();
+ });
+ try {
+ await new Promise<void>((resolve, reject) => {
+ server.once('error', reject);
+ server.listen(0, '127.0.0.1', () => {
+ server.off('error', reject);
+ resolve();
+ });
+ });
+ const address = server.address();
+ assert(address && typeof address !== 'string');
+ const endpoint = `http://127.0.0.1:${address.port}/v1`;
+ const agent = new CodexAgent({
+ binaryPath: process.env.CINDY_CODEX_TEST_BINARY!,
+ logger,
+ runtimeConfig: {},
+ prepareCodexExtraSpawnConfig: async () => ({
+ extraArgs: [
+ '-c',
+ 'model_provider="fixture"',
+ '-c',
+ `model_providers.fixture={name="Local speed validation",base_url="${endpoint}",wire_api="responses",requires_openai_auth=false,request_max_retries=0,supports_websockets=false}`,
+ '-c',
+ 'check_for_update_on_startup=false',
+ '-c',
+ 'analytics.enabled=false',
+ '-c',
+ 'features.code_mode=false',
+ // Fresh profile: keep the adapter-disabled MCP transport syntactically valid.
+ '-c',
+ 'mcp_servers.cindy_memory={command="false",enabled=false}',
+ ],
+ extraEnv: {},
+ }),
+ auth: {
+ getState: async () => ({ authenticated: true }),
+ triggerLogin: async () => ({ authenticated: true }),
+ logout: async () => {},
+ getAuthEnv: async () => ({
+ HOME: root,
+ USERPROFILE: root,
+ CODEX_HOME: home,
+ OPENAI_API_KEY: '',
+ CODEX_API_KEY: '',
+ NO_PROXY: '127.0.0.1,localhost',
+ }),
+ },
+ });
try {
- await Promise.race([
- done,
- new Promise((_, reject) => {
- // real-wait: failure deadline for the native subprocess; successful runs cancel it.
- timer = setTimeout(
- () => reject(new Error('native turn timeout')),
- 20000,
+ let handle: AgentSessionHandle | undefined;
+ const events: AgentEvent[] = [];
+ try {
+ handle = await agent.startSession({
+ sessionId: 'speed-native',
+ model: 'gpt-6-astra',
+ workingDir: workdir,
+ permissionMode: 'bypassPermissions',
+ makerMemoryEnabled: false,
+ });
+ let doneResolve: () => void = () => {};
+ void (async () => {
+ for await (const e of handle!.events()) {
+ if (
+ e.type === 'status' ||
+ e.type === 'done' ||
+ e.type === 'error' ||
+ e.type === 'tool_use' ||
+ e.type === 'tool_result'
+ )
+ events.push(e);
+ if (e.type === 'done') doneResolve();
+ }
+ })();
+ const run = async () => {
+ const done = new Promise<void>((r) => (doneResolve = r));
+ await handle!.send(
+ { type: 'user', content: 'Run the tool and finish.' },
+ { throwOnStartFailure: true },
);
- }),
- ]);
+ let timer: ReturnType<typeof setTimeout> | undefined;
+ try {
+ await Promise.race([
+ done,
+ new Promise((_, reject) => {
+ // real-wait: failure deadline for the native subprocess; successful runs cancel it.
+ timer = setTimeout(
+ () => reject(new Error('native turn timeout')),
+ 20000,
+ );
+ }),
+ ]);
+ } finally {
+ clearTimeout(timer);
+ }
+ };
+ const startedAt = Date.now();
+ await run();
+ const firstWallMs = Date.now() - startedAt;
+ const readings = () =>
+ events.flatMap((e) => {
+ const speed =
+ e.type === 'status'
+ ? (e.data as UsageSnapshot).tokenSpeed
+ : undefined;
+ return speed ? [speed] : [];
+ });
+ const first = readings().at(-1);
+ assert(
+ first && first.tokens === 200,
+ JSON.stringify({ first, events, warnings }),
+ );
+ assert(
+ first.elapsedMs >= 2000 && first.elapsedMs < 5000,
+ JSON.stringify(first),
+ );
+ assert(
+ firstWallMs - first.elapsedMs >= 800,
+ 'tool wait must be excluded from generation time',
+ );
+ assert.equal(events.filter((e) => e.type === 'tool_use').length, 1);
+ await run();
+ const second = readings().at(-1);
+ assert(second);
+ assert.equal(second.tokens, 100);
+ const evidence = {
+ requests,
+ first,
+ second,
+ tools: events
+ .filter((e) => e.type === 'tool_use')
+ .map((e) => e.data),
+ statuses: events
+ .filter((e) => e.type === 'status')
+ .map((e) => e.data),
+ errors: events.filter((e) => e.type === 'error'),
+ warnings,
+ };
+ if (process.env.CINDY_SPEED_EVIDENCE)
+ await writeFile(
+ process.env.CINDY_SPEED_EVIDENCE,
+ JSON.stringify(evidence, null, 2),
+ );
+ console.log(
+ JSON.stringify({
+ requests,
+ first,
+ second,
+ toolCount: evidence.tools.length,
+ errors: evidence.errors,
+ }),
+ );
+ } finally {
+ await handle?.close();
+ }
} finally {
- clearTimeout(timer);
+ await agent.dispose();
}
- };
- const startedAt = Date.now();
- await run();
- const firstWallMs = Date.now() - startedAt;
- const readings = () =>
- events.flatMap((e) => {
- const speed =
- e.type === 'status'
- ? (e.data as UsageSnapshot).tokenSpeed
- : undefined;
- return speed ? [speed] : [];
- });
- const first = readings().at(-1);
- assert(
- first && first.tokens === 200,
- JSON.stringify({ first, events, warnings }),
- );
- assert(
- first.elapsedMs >= 2000 && first.elapsedMs < 5000,
- JSON.stringify(first),
- );
- assert(
- firstWallMs - first.elapsedMs >= 800,
- 'tool wait must be excluded from generation time',
- );
- assert.equal(events.filter((e) => e.type === 'tool_use').length, 1);
- await run();
- const second = readings().at(-1);
- assert(second);
- assert.equal(second.tokens, 100);
- const evidence = {
- requests,
- first,
- second,
- tools: events.filter((e) => e.type === 'tool_use').map((e) => e.data),
- statuses: events.filter((e) => e.type === 'status').map((e) => e.data),
- errors: events.filter((e) => e.type === 'error'),
- warnings,
- };
- if (process.env.CINDY_SPEED_EVIDENCE)
- await writeFile(
- process.env.CINDY_SPEED_EVIDENCE,
- JSON.stringify(evidence, null, 2),
- );
- console.log(
- JSON.stringify({
- requests,
- first,
- second,
- toolCount: evidence.tools.length,
- errors: evidence.errors,
- }),
- );
+ } finally {
+ // Independent scopes ensure an earlier cleanup failure cannot skip this one.
+ try {
+ server.closeAllConnections();
+ } finally {
+ await new Promise<void>((r) => server.close(() => r()));
+ }
+ }
} finally {
- await handle?.close();
- await agent.dispose();
- server.closeAllConnections();
- await new Promise<void>((r) => server.close(() => r()));
await rm(root, { recursive: true, force: true });
}
},