Skip to content

armgit / cindy

Public Imported from GitHub
0

Consider it done. The open-source AI agent that works out of the box · 想到,就能做到。开源、开箱即用的 AI Agent。

fix(codex): exclude unobserved tool output from speed calibration

zqchris346e61be564a
diff --git a/docs/dev-rules/response-speed.md b/docs/dev-rules/response-speed.md
index f5c2136975..87dc8a1bc9 100644
--- a/docs/dev-rules/response-speed.md
+++ b/docs/dev-rules/response-speed.md
@@ -18,6 +18,9 @@ Claude 显示「流内等待」,SDK 暴露事件前的请求时间不可测。
生成计时从内容开始,到响应结束或工具/审批暂停;最后一个工具/审批结束后重新显示等待,下一次内容才恢复;并行内容已恢复时不被工具结束打断。
Codex 在 `agentMessage` 完成时关闭观测响应,迟到 usage/turn 完成只校准计数,不延长生成时间。
+Codex 工具参数、patch 和 plan 只有成品 item,没有可配对的参数生成流;含这些输出的轮次
+不能用整轮 usage 放大后续回答的采样。保留已观测内容的估算曲线与完成平均,真实账单总量
+照常保存,但省略该轮历史 TPS。下一轮重新判断;纯流式回复仍可校准。
Pi 重试关闭失败片段,下一请求重新等待;每轮重新清空测量。零 usage 是有效零;缺少 usage
则保留估算。仅结束事件有 output、没有可观测流,或检测到挂起/时钟异常时不提供 TPS。
diff --git a/packages/maker-core/src/agents/codex/index.test.ts b/packages/maker-core/src/agents/codex/index.test.ts
index cb359ed738..06a3a196c6 100644
--- a/packages/maker-core/src/agents/codex/index.test.ts
+++ b/packages/maker-core/src/agents/codex/index.test.ts
@@ -18454,8 +18454,14 @@ describe('CodexAgent MCP thread context hooks', () => {
now = 25_000;
handlers.turnCompleted!({ threadId: 'start-thread-id', turn: { id: 'speed-turn', status: 'completed', durationMs: 24_000 } });
await waitForExpectation(() => expect(events.some((event) => event.type === 'done')).toBe(true));
- expect(latestSpeed()).toMatchObject({ phase: 'complete', outputTokens: 60, durationMs: observedMs, averageRate: 60_000 / observedMs, estimated: false });
- expect(events.find((event) => event.type === 'done')?.data.usage).toMatchObject({ completionTokens: 60, durationMs: observedMs });
+ // Tool arguments are not a measured stream: the whole-turn 60 output
+ // tokens must not inflate these two/three observed character estimates.
+ const observedTokens = mode === 'parallel-output' ? 3 : 2;
+ expect(latestSpeed()).toMatchObject({ phase: 'complete', outputTokens: observedTokens,
+ durationMs: observedMs, averageRate: observedTokens * 1_000 / observedMs, estimated: true });
+ const usage = events.find((event) => event.type === 'done')?.data.usage;
+ expect(usage).toMatchObject({ completionTokens: 60 });
+ expect(usage?.durationMs).toBeUndefined();
now = 26_000;
handlers.turnStarted!({ threadId: 'start-thread-id', turn: { id: 'next-speed-turn' } });
now = 26_100;
@@ -18467,6 +18473,72 @@ describe('CodexAgent MCP thread context hooks', () => {
}
});
+ it.each(['commandExecution', 'mcpToolCall', 'fileChange', 'plan'])(
+ 'keeps tool-first %s turns approximate and restores calibration on the next text-only turn',
+ async (type) => {
+ let now = 1_000;
+ const nowSpy = vi.spyOn(Date, 'now').mockImplementation(() => now);
+ const agent = new CodexAgent(createDeps());
+ const host = installFakeHost(agent);
+ const handle = await agent.startSession({ sessionId: `session-tool-first-${type}`, model: 'gpt-5.4', workingDir: '/repo' });
+ const handlers = host.getThreadHandlers()!;
+ const events: AgentEvent[] = [];
+ void (async () => { for await (const event of handle.events()) events.push(event); })();
+ const latestSpeed = () => events.findLast(event => event.type === 'status')?.data.responseSpeed;
+ try {
+ handlers.turnStarted!({ threadId: 'start-thread-id', turn: { id: 'tool-first' } });
+ const item = type === 'commandExecution'
+ ? { id: 'first-tool', type, command: 'pwd', cwd: '/repo', status: 'inProgress' }
+ : type === 'mcpToolCall'
+ ? { id: 'first-tool', type, server: 'test', tool: 'lookup', arguments: { query: 'where' }, status: 'inProgress' }
+ : type === 'fileChange'
+ ? { id: 'first-tool', type, changes: [{ path: '/repo/example', kind: { type: 'update' }, diff: '+answer' }], status: 'completed' }
+ : { id: 'first-tool', type, text: 'Look up the answer' };
+ const execution = type === 'commandExecution' || type === 'mcpToolCall';
+ now = 2_000;
+ if (execution) {
+ handlers.itemStarted!({ threadId: 'start-thread-id', turnId: 'tool-first', item } as never);
+ await waitForExpectation(() => expect(latestSpeed()).toMatchObject({ phase: 'paused', toolActive: true, firstResponseMs: 1_000, durationMs: 0 }));
+ now = 8_000;
+ }
+ handlers.itemCompleted!({ threadId: 'start-thread-id', turnId: 'tool-first', item: { ...item, status: 'completed' } } as never);
+ if (execution) await waitForExpectation(() => expect(latestSpeed()).toMatchObject({ phase: 'waiting', toolActive: false }));
+
+ const finishAnswer = async (turnId: string, start: number, cumulativeOutput: number, lastOutput: number) => {
+ now = start;
+ handlers.agentMessageDelta!({ threadId: 'start-thread-id', turnId, itemId: `${turnId}-answer`, delta: 'a'.repeat(40) });
+ now = start + 1_000;
+ handlers.agentMessageDelta!({ threadId: 'start-thread-id', turnId, itemId: `${turnId}-answer`, delta: 'b'.repeat(40) });
+ await waitForExpectation(() => expect(latestSpeed()).toMatchObject({ phase: 'generating', recentRate: 10 }));
+ now = start + 2_000;
+ handlers.itemCompleted!({ threadId: 'start-thread-id', turnId, item: { id: `${turnId}-answer`, type: 'agentMessage', text: 'a'.repeat(40) + 'b'.repeat(40) } } as never);
+ now = start + 3_000;
+ handlers.tokenUsageUpdated!({ threadId: 'start-thread-id', turnId, tokenUsage: {
+ total: { inputTokens: 20, cachedInputTokens: 0, outputTokens: cumulativeOutput, reasoningOutputTokens: 0, totalTokens: cumulativeOutput + 20 },
+ last: { inputTokens: 10, cachedInputTokens: 0, outputTokens: lastOutput, reasoningOutputTokens: 0, totalTokens: lastOutput + 10 },
+ } } as never);
+ handlers.turnCompleted!({ threadId: 'start-thread-id', turn: { id: turnId, status: 'completed' } });
+ await waitForExpectation(() => expect(latestSpeed()?.phase).toBe('complete'));
+ return events.findLast(event => event.type === 'done')?.data.usage;
+ };
+ const mixedUsage = await finishAnswer('tool-first', 10_000, 150, 150);
+ expect(latestSpeed()).toMatchObject({ estimated: true, outputTokens: 20, durationMs: 2_000, averageRate: 10 });
+ expect(latestSpeed()?.samples.every((sample: { rate: number }) => sample.rate <= 10)).toBe(true);
+ expect(mixedUsage).toMatchObject({ completionTokens: 150 });
+ expect(mixedUsage?.durationMs).toBeUndefined();
+
+ now = 14_000;
+ handlers.turnStarted!({ threadId: 'start-thread-id', turn: { id: 'text-only' } });
+ const textUsage = await finishAnswer('text-only', 15_000, 190, 40);
+ expect(latestSpeed()).toMatchObject({ estimated: false, outputTokens: 40, durationMs: 2_000, averageRate: 20 });
+ expect(textUsage).toMatchObject({ completionTokens: 40, durationMs: 2_000 });
+ } finally {
+ await handle.close();
+ nowSpy.mockRestore();
+ }
+ },
+ );
+
it('starts response-first generation timing at turn/started and excludes an approval wait', async () => {
let now = 1_000;
const nowSpy = vi.spyOn(Date, 'now').mockImplementation(() => now);
diff --git a/packages/maker-core/src/agents/codex/index.ts b/packages/maker-core/src/agents/codex/index.ts
index 37741bf0d4..5de6519956 100644
--- a/packages/maker-core/src/agents/codex/index.ts
+++ b/packages/maker-core/src/agents/codex/index.ts
@@ -10726,6 +10726,7 @@ assertRouteCurrent();
text?: unknown;
} | null | undefined;
if (!candidate || candidate.type !== 'plan') return false;
+ translatorRt.responseSpeedHasUnobservedOutput = true;
if (typeof candidate.text === 'string') proposedPlanText = candidate.text;
return true;
}
@@ -10897,8 +10898,14 @@ assertRouteCurrent();
const realTurnUsage = usageTracker.getTurnUsage();
const realTurnUsageSegments = usageTracker.getTurnUsageSegments();
finalizeCodexGenerationTurn(translatorRt, turn.id);
- translatorRt.responseSpeed.finish(realTurnUsageSegments.length > 0 ? realTurnUsage.output : undefined);
- if (!translatorRt.generationTimingReliable) translatorRt.responseSpeed.invalidate();
+ const unmeasuredOutputOnly = translatorRt.responseSpeedHasUnobservedOutput
+ && translatorRt.responseSpeed.snapshot().outputTokens === 0;
+ // With no sampled units, retain the real total but expose no rate. Mixed
+ // turns retain only their observed estimates, never a whole-turn scale.
+ translatorRt.responseSpeed.finish(realTurnUsageSegments.length > 0
+ && (!translatorRt.responseSpeedHasUnobservedOutput || unmeasuredOutputOnly)
+ ? realTurnUsage.output : undefined);
+ if (!translatorRt.generationTimingReliable || unmeasuredOutputOnly) translatorRt.responseSpeed.invalidate();
const speedSnapshot = translatorRt.responseSpeed.snapshot();
const generationDurationMs = calibratedResponseDuration(speedSnapshot, realTurnUsage.output);
const codexDoneUsage = {
diff --git a/packages/maker-core/src/agents/codex/translator.ts b/packages/maker-core/src/agents/codex/translator.ts
index a24f5a203e..7a22a7cd0e 100644
--- a/packages/maker-core/src/agents/codex/translator.ts
+++ b/packages/maker-core/src/agents/codex/translator.ts
@@ -82,6 +82,8 @@ export interface CodexRuntimeState {
itemTextLen: Map<string, number>;
/** Current model-active interval start; null while tools/approvals own the turn. */
responseSpeed: ResponseSpeedTracker;
+ /** Tool inputs arrive after generation, so whole-turn usage cannot calibrate only observed text. */
+ responseSpeedHasUnobservedOutput: boolean;
generationStartedAt: number | null;
/** Tool/approval boundaries currently owning the turn. Generation resumes after all finish. */
generationPendingToolIds: Set<string>;
@@ -133,6 +135,7 @@ export function newCodexRuntimeState(): CodexRuntimeState {
reasoningTextLen: new Map(),
itemTextLen: new Map(),
responseSpeed: new ResponseSpeedTracker(),
+ responseSpeedHasUnobservedOutput: false,
generationStartedAt: null,
generationPendingToolIds: new Set(),
generationDurationMs: 0,
@@ -193,6 +196,7 @@ export function resetCodexGenerationTiming(rt: CodexRuntimeState): void {
rt.generationOutputDurationMs = 0;
rt.generationTurnId = null;
rt.generationTimingReliable = true;
+ rt.responseSpeedHasUnobservedOutput = false;
}
function closeCodexGenerationInterval(rt: CodexRuntimeState, endedAt: number): void {
@@ -404,6 +408,13 @@ export function observeCodexItemTiming(
rt.responseSpeed.content();
}
noteCodexGenerationBoundary(rt, phase, item, notification);
+ // These inputs have no argument delta/start pair. Their output tokens may be
+ // included in turn usage, but their generation time is not in our samples.
+ // Retain measured streaming estimates without scaling them by that total.
+ if ((CODEX_GENERATION_PAUSE_ITEM_TYPES.has(itemType) && itemType !== 'contextCompaction')
+ || itemType === 'fileChange' || itemType === 'plan') {
+ rt.responseSpeedHasUnobservedOutput = true;
+ }
}
export function translateItemNotification(