Skip to content

Make Codex and Claude Code subscriptions go further: a local model router that gives routine agent work to economical models and keeps your strongest model for the hard parts.

feat: compare competence by execution group and model configuration (#47)

澄潭8c04ea2411da
diff --git a/apps/desktop/src/features/PlanQuality.tsx b/apps/desktop/src/features/PlanQuality.tsx
index 931edd39..a4c00155 100644
--- a/apps/desktop/src/features/PlanQuality.tsx
+++ b/apps/desktop/src/features/PlanQuality.tsx
@@ -1,57 +1,22 @@
-import React, { useEffect, useRef, useState } from 'react';
+import React, { useEffect, useMemo, useRef, useState } from 'react';
import { UiIcon } from '../ui';
import { safeDiagnosticCode } from '../error-code';
import { observationRead } from './observation-client';
import { qualityExecutionModels } from './plan-quality-models';
+import { PlanQualityStage, type PlanQualitySample } from './PlanQualityStage';
+import { qualityBranchLabel, qualityModelRows, qualityNativeModelName, qualityReasoningLabel, type PlanQualityModel, type QualityExecution, type QualityModelRow, type QualitySummary } from './plan-quality-state';
-type Assessment = {
- trigger_request_id: string;
- target_from_ordinal: number;
- target_through_ordinal: number;
- score: number;
- partial: boolean;
- reason?: string | null;
- evidence_available: boolean;
-};
+export type { PlanQualityModel } from './plan-quality-state';
+export type { PlanQualitySample } from './PlanQualityStage';
-export type PlanQualitySample = {
- segment_id: string;
- session_id: string;
- plan_id: string;
- plan_revision: number;
- selected_branch_id: string;
- executed_branch_id?: string | null;
- model_configuration_id?: string | null;
- attribution: 'single' | 'mixed' | 'unknown';
- first_turn_ordinal: number;
- last_observed_turn_ordinal: number;
- first_at_ms: number;
- last_at_ms: number;
- history_partial: boolean;
- first_request_id?: string | null;
- last_request_id?: string | null;
- execution_evidence_available: boolean;
- assessment?: Assessment | null;
-};
-
-export type PlanQualityModel = {
- model_configuration_id: string;
- display_name: string;
-};
-
-type Page = { samples: PlanQualitySample[]; next_cursor?: string | null };
-type ScoreFilter = 'all' | 'low' | 'high';
+type Page = { samples: PlanQualitySample[]; summary: QualitySummary; next_cursor?: string | null };
+type ScoreFilter = 'all' | 'low' | 'high' | 'unrated';
type Period = 'all' | 'seven_days';
-type VersionScope = 'current' | 'retained';
+const EMPTY_SUMMARY: QualitySummary = { models: [], scored_stage_count: 0, unrated_stage_count: 0, session_count: 0, available_revisions: [] };
export function PlanQuality({
- planId,
- sessionId,
- planRevision,
- currentModels = [],
- language,
- compact = false,
- onOpenEvidence,
+ planId, sessionId, planRevision, currentModels = [], language, compact = false,
+ active = true, refreshVersion = 0, onOpenEvidence,
}: {
planId?: string | null;
sessionId?: string | null;
@@ -59,52 +24,53 @@ export function PlanQuality({
currentModels?: readonly PlanQualityModel[];
language: 'zh' | 'en';
compact?: boolean;
+ active?: boolean;
+ refreshVersion?: number;
onOpenEvidence?: (sessionId: string, requestId: string) => void;
}) {
const text = (zh: string, en: string) => language === 'zh' ? zh : en;
const [samples, setSamples] = useState<PlanQualitySample[]>([]);
+ const [summary, setSummary] = useState<QualitySummary>(EMPTY_SUMMARY);
const [executionModels, setExecutionModels] = useState<Record<string, string>>({});
const [cursor, setCursor] = useState<string | null>(null);
const [score, setScore] = useState<ScoreFilter>('all');
const [period, setPeriod] = useState<Period>(compact ? 'all' : 'seven_days');
- const [versionScope, setVersionScope] = useState<VersionScope>('current');
+ const [version, setVersion] = useState('current');
+ const [selected, setSelected] = useState<{ key: string; execution: QualityExecution | null } | null>(null);
+ const [reload, setReload] = useState(0);
const [busy, setBusy] = useState(false);
const [loaded, setLoaded] = useState(false);
const [error, setError] = useState('');
const generation = useRef(0);
- const queryWindow = useRef<{ from?: number; to: number } | null>(null);
+ // A detail filter or next page must retain exactly the same time window.
+ const window = useMemo(() => {
+ const now = Date.now();
+ return { from: period === 'seven_days' ? now - 7 * 86_400_000 : undefined, to: now + 1 };
+ }, [planId, sessionId, planRevision, period, version, refreshVersion, reload, active]);
+ const revision = version === 'current' ? planRevision || null : version === 'all' ? null : Number(version);
async function load(next: string | null = null) {
- if (!planId && !sessionId) return;
+ if (!active || (!planId && !sessionId)) return;
const epoch = ++generation.current;
setBusy(true); setError('');
- if (!next || !queryWindow.current) {
- const now = Date.now();
- queryWindow.current = {
- from: period === 'seven_days' ? now - 7 * 86_400_000
- : undefined,
- to: now + 1,
- };
- }
- const window = queryWindow.current;
try {
const page = await observationRead<Page>('plan_quality', {
- plan_id: planId || null,
- session_id: sessionId || null,
- plan_revision: planRevision && versionScope === 'current' ? planRevision : null,
- model_configuration_id: null,
- from_ms: window.from,
- to_ms: window.to,
+ plan_id: planId || null, session_id: sessionId || null, plan_revision: revision,
+ execution: selected?.execution ?? null,
+ from_ms: window.from, to_ms: window.to,
score_gt: score === 'high' ? 0.5 : null,
score_lt: score === 'low' ? 0.5 : null,
- limit: 20,
- cursor: next,
+ unrated_only: score === 'unrated', limit: 20, cursor: next,
});
if (epoch !== generation.current) return;
- setSamples(current => next ? [...current, ...page.samples] : page.samples);
- setCursor(page.next_cursor ?? null);
+ setSummary(page.summary);
+ const stages = selected && !selected.execution ? [] : page.samples;
+ setSamples(current => next ? [...current, ...stages] : stages);
+ setCursor(selected && !selected.execution ? null : page.next_cursor ?? null);
setLoaded(true);
- const models = await qualityExecutionModels(page.samples, async (session, request) => {
+ // Historical records can lack the native-name projection. Recover only
+ // from an exact request in the same session, never a neighbouring request.
+ const models = await qualityExecutionModels(stages.filter(sample => !sample.native_model), async (session, request) => {
const result = await observationRead<{ requests: { session_id: string; request_id: string; final_native_model: string | null }[] }>('timeline', {
session_id: session, request_id: request, from_ms: 0, to_ms: window.to,
only_model_switch: false, limit: 1, cursor: null,
@@ -120,58 +86,72 @@ export function PlanQuality({
}
useEffect(() => {
+ setSummary(EMPTY_SUMMARY);
+ }, [planId, sessionId, revision, window]);
+
+ useEffect(() => {
+ if (!active) { generation.current++; setBusy(false); return; }
setSamples([]); setExecutionModels({}); setCursor(null); setLoaded(false);
- queryWindow.current = null;
void load();
return () => { generation.current++; };
- }, [planId, sessionId, planRevision, score, period, versionScope]);
+ }, [planId, sessionId, revision, score, selected?.key, window, active]);
- const scoreText = (sample: PlanQualitySample) => sample.assessment
- ? `${sample.assessment.score.toFixed(2)} / 1`
- : text('未评分', 'Not rated');
- const branchText = (sample: PlanQualitySample) => {
- const branch = sample.executed_branch_id ?? sample.selected_branch_id;
- const label = branch === 'smart_saving_simple' ? text('省钱分支', 'Economy branch')
- : branch === 'smart_saving_complex' ? text('主力分支', 'Primary branch') : text('其他分支', 'Other branch');
- return sample.executed_branch_id ? label : `${label} · ${text('实际分支未记录', 'Execution branch not recorded')}`;
- };
- const currentModelNames = new Map(currentModels.map(model => [model.model_configuration_id, model.display_name]));
+ const rows = qualityModelRows(revision === planRevision ? currentModels : [], summary.models);
+ const groups = [...new Set(rows.map(row => row.branch))];
+ const selectedRow = rows.find(row => row.key === selected?.key);
+ const rowName = (row: QualityModelRow) => row.summary?.execution.attribution === 'mixed' ? text('多个执行模型', 'Multiple execution models')
+ : row.summary?.execution.attribution === 'unknown' ? text('执行模型未确认', 'Execution model unknown')
+ : row.configured?.display_name ?? qualityNativeModelName(row.summary?.native_model ?? text('模型名称不可用', 'Model name unavailable'));
+ const modelNames = new Map(currentModels.map(model => [model.model_configuration_id, model.display_name]));
+ const sampleName = (sample: PlanQualitySample) => sample.attribution === 'mixed' ? text('多个执行模型', 'Multiple execution models')
+ : sample.attribution === 'unknown' ? text('执行模型未确认', 'Execution model unknown')
+ : qualityNativeModelName(sample.native_model ?? executionModels[sample.segment_id]
+ ?? (sample.model_configuration_id ? modelNames.get(sample.model_configuration_id) : null)
+ ?? text('模型名称不可用', 'Model name unavailable'));
+ const resetDetail = () => { setSelected(null); setScore('all'); };
+ const stageCount = summary.scored_stage_count + summary.unrated_stage_count;
+ const filter = <label><span className="sr-only">{text('阶段评分筛选', 'Stage score filter')}</span><select className="select quality-score-filter" value={score} onChange={event => setScore(event.target.value as ScoreFilter)}>
+ <option value="all">{text('全部阶段', 'All stages')}</option>
+ <option value="low">{text('胜任度低于 0.5', 'Competence below 0.5')}</option>
+ <option value="high">{text('胜任度高于 0.5', 'Competence above 0.5')}</option>
+ <option value="unrated">{text('待评分', 'Unrated')}</option>
+ </select></label>;
- return <div className={`plan-quality${compact ? ' compact' : ''}`}>
- {!compact && currentModels.length > 0 && <div className="quality-model-scope">
- <span className="quality-model-scope-label">{text('当前生效版本模型', 'Models in the active revision')}</span>
- <div className="quality-model-list">{currentModels.map(model => <span className="badge no-dot" key={model.model_configuration_id} title={model.model_configuration_id}>{model.display_name}</span>)}</div>
- </div>}
- {!compact && <div className="quality-filters">
- <label><span className="sr-only">{text('评分筛选', 'Score filter')}</span><select className="select" value={score} onChange={event => setScore(event.target.value as ScoreFilter)}><option value="all">{text('全部评分', 'All scores')}</option><option value="low">{text('胜任度低于 0.5', 'Competence below 0.5')}</option><option value="high">{text('胜任度高于 0.5', 'Competence above 0.5')}</option></select></label>
- <label><span className="sr-only">{text('时间范围', 'Time range')}</span><select className="select" value={period} onChange={event => setPeriod(event.target.value as Period)}><option value="seven_days">{text('最近 7 天', 'Last 7 days')}</option><option value="all">{text('全部保留期', 'All retained')}</option></select></label>
- {planRevision && <label><span className="sr-only">{text('计划版本', 'Plan version')}</span><select className="select" value={versionScope} onChange={event => setVersionScope(event.target.value as VersionScope)}><option value="current">{text(`当前版本 r${planRevision}`, `Current r${planRevision}`)}</option><option value="retained">{text('全部保留版本', 'All retained versions')}</option></select></label>}
- </div>}
- {error && <div className="callout bad" role="alert" data-error-code={error}><UiIcon name="warning" /><span>{text('运行表现暂时无法读取。', 'Runtime performance is temporarily unavailable.')}</span><button className="btn" type="button" onClick={() => void load()}>{text('重试', 'Retry')}</button></div>}
+ return <div className={'plan-quality' + (compact ? ' compact' : '')}>
+ <div className="quality-toolbar">
+ <div className="quality-scope-meta">{text(stageCount + ' 个执行阶段 · ' + summary.scored_stage_count + ' 已评分 · ' + summary.unrated_stage_count + ' 待评分', stageCount + ' stages · ' + summary.scored_stage_count + ' scored · ' + summary.unrated_stage_count + ' unrated')}</div>
+ <div className="quality-filters">
+ {!compact && <label><span className="sr-only">{text('时间范围', 'Time range')}</span><select className="select" value={period} onChange={event => { resetDetail(); setPeriod(event.target.value as Period); }}>
+ <option value="seven_days">{text('最近 7 天', 'Last 7 days')}</option><option value="all">{text('全部保留期', 'All retained')}</option>
+ </select></label>}
+ {!compact && planRevision && <label><span className="sr-only">{text('计划版本', 'Plan version')}</span><select className="select" value={version} onChange={event => { resetDetail(); setVersion(event.target.value); }}>
+ <option value="current">{text('当前生效 r' + planRevision, 'Active r' + planRevision)}</option>
+ {summary.available_revisions.filter(value => value !== planRevision).map(value => <option key={value} value={String(value)}>{text('历史版本 r' + value, 'Revision r' + value)}</option>)}
+ <option value="all">{text('全部保留版本', 'All retained revisions')}</option>
+ </select></label>}
+ <button className="btn btn-quiet quality-refresh" type="button" disabled={busy} onClick={() => setReload(value => value + 1)}><UiIcon name="refresh" />{text('刷新', 'Refresh')}</button>
+ </div>
+ </div>
+ {error && <div className="callout bad" role="alert" data-error-code={error}><UiIcon name="warning" /><span>{text('模型表现暂时无法读取。', 'Model performance is temporarily unavailable.')}</span><button className="btn" type="button" onClick={() => setReload(value => value + 1)}>{text('重试', 'Retry')}</button></div>}
{!loaded && busy && <div className="oc-status-row" role="status"><span className="oc-spinner" /><p>{text('正在读取模型表现…', 'Reading model performance…')}</p></div>}
- {loaded && !busy && !error && samples.length === 0 && <p className="oc-meta">{text('当前范围还没有模型阶段记录。', 'No model-stage records are available for this scope yet.')}</p>}
- <div className="quality-list">{samples.map(sample => {
- const executionRequestId = sample.execution_evidence_available ? sample.last_request_id ?? sample.first_request_id : null;
- const feedbackRequestId = sample.assessment?.evidence_available ? sample.assessment.trigger_request_id : null;
- const modelName = sample.attribution === 'mixed' ? text('多个执行模型', 'Multiple execution models')
- : sample.attribution === 'unknown' ? text('执行模型未确认', 'Execution model unknown')
- : executionModels[sample.segment_id]
- ?? (sample.model_configuration_id ? currentModelNames.get(sample.model_configuration_id) : null)
- ?? (busy ? text('正在读取模型名称…', 'Reading model name…') : text('模型名称不可用', 'Model name unavailable'));
- return <article className="quality-row" key={sample.segment_id}>
- <div className="quality-row-main"><strong title={sample.model_configuration_id ?? undefined}>{modelName}</strong><span title={sample.executed_branch_id ?? sample.selected_branch_id}>{branchText(sample)} · {text(`轮次 ${sample.first_turn_ordinal}–${sample.last_observed_turn_ordinal}`, `turn ${sample.first_turn_ordinal}–${sample.last_observed_turn_ordinal}`)}</span><span>{new Date(sample.last_at_ms).toLocaleString(language === 'zh' ? 'zh-CN' : 'en')} · {text(`计划版本 r${sample.plan_revision}`, `plan r${sample.plan_revision}`)}</span></div>
- <div className="quality-score"><strong>{scoreText(sample)}</strong><span>{text('胜任度', 'Competence')}</span></div>
- <div className="quality-flags">
- {sample.history_partial && <span className="badge warn no-dot" title={text('该阶段的执行历史存在记录缺口;这不等于 Agent 收到的回答不完整。', 'The recorded execution history has gaps; this does not establish that the Agent received an incomplete answer.')}>{text('历史记录不完整', 'Incomplete history')}</span>}
- {sample.assessment?.partial && <span className="badge warn no-dot" title={text('评分依据仅覆盖该阶段的部分证据。', 'The assessment is based on only part of the evidence for this stage.')}>{text('评分证据不完整', 'Partial assessment evidence')}</span>}
- {sample.attribution !== 'single' && <span className="badge warn no-dot">{sample.attribution === 'mixed' ? text('无法归因于单一模型', 'Not attributable to one model') : text('模型归属未确认', 'Model attribution unknown')}</span>}
- </div>
- {!sample.assessment && <p className="quality-coverage">{text('尚未产生该阶段的胜任度评分;后续评估触发后才会显示。', 'No competence assessment has been recorded for this stage. A score appears after a subsequent assessment is triggered.')}</p>}
- {sample.assessment && <p className="quality-coverage">{text(`评分覆盖轮次 ${sample.assessment.target_from_ordinal}–${sample.assessment.target_through_ordinal}`, `Assessed turns ${sample.assessment.target_from_ordinal}–${sample.assessment.target_through_ordinal}`)}</p>}
- {sample.assessment?.reason && <p className="quality-reason">{sample.assessment.reason}</p>}
- <div className="quality-evidence-actions"><button className="btn btn-quiet" type="button" disabled={!executionRequestId || !onOpenEvidence} onClick={() => executionRequestId && onOpenEvidence?.(sample.session_id, executionRequestId)}>{executionRequestId ? text('执行证据', 'Execution evidence') : text('执行证据不可用', 'Execution unavailable')}</button>{sample.assessment && <button className="btn btn-quiet" type="button" disabled={!feedbackRequestId || !onOpenEvidence} onClick={() => feedbackRequestId && onOpenEvidence?.(sample.session_id, feedbackRequestId)}>{feedbackRequestId ? text('评分触发反馈', 'Assessment feedback') : text('反馈不可用', 'Feedback unavailable')}</button>}</div>
- </article>;
- })}</div>
- {cursor && <button className="btn" type="button" disabled={busy} onClick={() => void load(cursor)}>{text('加载更多阶段', 'Load more stages')}</button>}
+ {!compact && <div className="quality-model-scope">{groups.map(branch => <section className="quality-group" key={branch} data-execution-group={branch}>
+ <header className="quality-group-heading"><h4>{qualityBranchLabel(branch, language)}</h4><span>{text(rows.filter(row => row.branch === branch).length + ' 个模型配置', rows.filter(row => row.branch === branch).length + (rows.filter(row => row.branch === branch).length === 1 ? ' model configuration' : ' model configurations'))}</span></header>
+ <div className="quality-model-columns" aria-hidden="true"><span>{text('模型 / 思考设置', 'Model / Reasoning')}</span><span>{text('平均阶段胜任度', 'Average stage competence')}</span><span>{text('阶段样本', 'Stage samples')}</span><span /></div>
+ {rows.filter(row => row.branch === branch).map(row => <div className={'quality-model-row' + (selected?.key === row.key ? ' selected' : '')} key={row.key} data-model-configuration={row.configured?.model_configuration_id ?? row.summary?.execution.model_configuration_id ?? ''}>
+ <div className="quality-model-name"><strong>{rowName(row)}</strong><span>{qualityReasoningLabel(row.summary ? row.summary.reasoning_profile_id : row.configured?.reasoning_profile_id, language)}{version === 'all' && row.summary && ' · r' + row.summary.execution.plan_revision}</span></div>
+ <div className="quality-model-average">{row.summary?.average_score != null ? <><strong>{row.summary.average_score.toFixed(2)} <small>/ 1</small></strong><div className="quality-score-bar" aria-hidden="true"><span style={{ width: (row.summary.average_score * 100) + '%' }} /></div></> : <span>{row.summary ? text('待评分', 'Unrated') : text('暂无记录', 'No records')}</span>}</div>
+ <div className="quality-model-counts"><span>{text((row.summary?.scored_stage_count ?? 0) + ' 已评分', (row.summary?.scored_stage_count ?? 0) + ' scored')}</span><span>{text((row.summary?.unrated_stage_count ?? 0) + ' 待评分', (row.summary?.unrated_stage_count ?? 0) + ' unrated')}</span></div>
+ <button type="button" className="btn btn-quiet quality-view-stages" aria-expanded={selected?.key === row.key} onClick={() => { setScore('all'); setSelected(selected?.key === row.key ? null : { key: row.key, execution: row.summary?.execution ?? null }); }}>{selected?.key === row.key ? text('收起阶段', 'Hide stages') : text('查看阶段', 'View stages')}<UiIcon name="chevronRight" /></button>
+ </div>)}
+ </section>)}</div>}
+ {!compact && loaded && groups.length > 0 && <p className="quality-average-note">{text('平均值使用当前范围内各已评分阶段的最新原始评分,待评分阶段不计入平均。', 'Averages use the latest raw assessment of every scored stage in this scope. Unrated stages are excluded.')}</p>}
+ {!compact && loaded && !error && groups.length === 0 && <p className="oc-meta">{text('当前范围还没有模型阶段记录。', 'No model-stage records are available for this scope yet.')}</p>}
+ {(compact || selected) && <section className="quality-stage-section">
+ {!compact && <div className="quality-stage-heading"><div><h4>{selectedRow ? rowName(selectedRow) : text('执行阶段', 'Execution stages')}</h4><span>{text('按实际执行模型与推理配置查看', 'Stages for the actual execution and reasoning configuration')}</span></div>{filter}</div>}
+ {compact && stageCount > 20 && <div className="quality-filters">{filter}</div>}
+ {loaded && !busy && !error && samples.length === 0 && <p className="oc-meta">{text('没有符合筛选条件的执行阶段。', 'No execution stages match this filter.')}</p>}
+ <div className="quality-list">{samples.map(sample => <PlanQualityStage key={sample.segment_id} sample={sample} modelName={sampleName(sample)} language={language} onOpenEvidence={onOpenEvidence} />)}</div>
+ {cursor && <button className="btn quality-more" type="button" disabled={busy} onClick={() => void load(cursor)}>{text('加载更多阶段', 'Load more stages')}</button>}
+ </section>}
</div>;
}
diff --git a/apps/desktop/src/features/PlanQualityStage.tsx b/apps/desktop/src/features/PlanQualityStage.tsx
new file mode 100644
index 00000000..335589ff
--- /dev/null
+++ b/apps/desktop/src/features/PlanQualityStage.tsx
@@ -0,0 +1,81 @@
+import React from 'react';
+import { Disclosure, UiIcon } from '../ui';
+import { qualityBranchLabel, qualityReasoningLabel } from './plan-quality-state';
+
+type Assessment = {
+ trigger_request_id: string;
+ assessed_at_ms: number;
+ target_from_ordinal: number;
+ target_through_ordinal: number;
+ score: number;
+ partial: boolean;
+ reason?: string | null;
+ evidence_available: boolean;
+};
+
+export type PlanQualitySample = {
+ segment_id: string;
+ session_id: string;
+ plan_id: string;
+ plan_revision: number;
+ selected_branch_id: string;
+ executed_branch_id?: string | null;
+ model_configuration_id?: string | null;
+ profile_digest?: string | null;
+ native_model?: string | null;
+ reasoning_profile_id?: string | null;
+ attribution: 'single' | 'mixed' | 'unknown';
+ first_turn_ordinal: number;
+ last_observed_turn_ordinal: number;
+ first_at_ms: number;
+ last_at_ms: number;
+ history_partial: boolean;
+ first_request_id?: string | null;
+ last_request_id?: string | null;
+ execution_evidence_available: boolean;
+ assessment?: Assessment | null;
+};
+
+export function PlanQualityStage({ sample, modelName, language, onOpenEvidence }: {
+ sample: PlanQualitySample;
+ modelName: string;
+ language: 'zh' | 'en';
+ onOpenEvidence?: (sessionId: string, requestId: string) => void;
+}) {
+ const text = (zh: string, en: string) => language === 'zh' ? zh : en;
+ const assessment = sample.assessment;
+ const executionRequestId = sample.execution_evidence_available ? sample.last_request_id ?? sample.first_request_id : null;
+ const triggerRequestId = assessment?.evidence_available ? assessment.trigger_request_id : null;
+ const branch = sample.executed_branch_id ?? 'unrecorded';
+ return <article className="quality-row" data-stage-id={sample.segment_id}>
+ <div className="quality-row-header">
+ <div className="quality-row-main">
+ <strong>{modelName}</strong>
+ <span>{qualityReasoningLabel(sample.reasoning_profile_id, language)}</span>
+ <span>{qualityBranchLabel(branch, language)} · {text('轮次 ' + sample.first_turn_ordinal + '–' + sample.last_observed_turn_ordinal, 'Turns ' + sample.first_turn_ordinal + '–' + sample.last_observed_turn_ordinal)}</span>
+ </div>
+ <div className="quality-score"><strong>{assessment ? assessment.score.toFixed(2) + ' / 1' : text('未评分', 'Unrated')}</strong><span>{text('阶段胜任度', 'Stage competence')}</span></div>
+ </div>
+ <div className="quality-stage-meta">{new Date(sample.last_at_ms).toLocaleString(language === 'zh' ? 'zh-CN' : 'en')} · {text('计划版本 r' + sample.plan_revision, 'Plan r' + sample.plan_revision)}</div>
+ <Disclosure className="quality-assessment-details" label={assessment ? text('评分详情', 'Assessment details') : text('阶段详情', 'Stage details')} language={language}>
+ {!assessment && <p className="quality-coverage">{text('尚未产生该阶段的胜任度评分;后续评估触发后才会显示。', 'No competence assessment has been recorded for this stage. A score appears after a subsequent assessment is triggered.')}</p>}
+ {assessment && <>
+ <p className="quality-coverage">{text('评分覆盖轮次 ' + assessment.target_from_ordinal + '–' + assessment.target_through_ordinal, 'Assessed turns ' + assessment.target_from_ordinal + '–' + assessment.target_through_ordinal)}</p>
+ <p className="quality-coverage">{text('原始评分', 'Raw score')} · {assessment.score}{assessment.assessed_at_ms != null && <> · {new Date(assessment.assessed_at_ms).toLocaleString(language === 'zh' ? 'zh-CN' : 'en')}</>}</p>
+ {assessment.reason && <p className="quality-reason">{assessment.reason}</p>}
+ </>}
+ <div className="quality-flags">
+ {sample.history_partial && <span className="badge warn no-dot">{text('历史记录不完整', 'Incomplete history')}</span>}
+ {assessment?.partial && <span className="badge warn no-dot">{text('评分证据不完整', 'Partial assessment evidence')}</span>}
+ {sample.attribution !== 'single' && <span className="badge warn no-dot">{sample.attribution === 'mixed' ? text('无法归因于单一模型', 'Not attributable to one model') : text('模型归属未确认', 'Model attribution unknown')}</span>}
+ </div>
+ <p className="quality-coverage">{text('阶段评分用于观察该阶段的执行表现。', 'The score describes execution of this stage.')}</p>
+ <div className="quality-diagnostic-identity">
+ {sample.executed_branch_id ?? sample.selected_branch_id} · {sample.model_configuration_id ?? 'unknown'} · {sample.profile_digest ?? 'unknown'}
+ {sample.native_model && sample.native_model !== modelName && <div>{text('原始模型标识', 'Original model ID')} · {sample.native_model}</div>}
+ </div>
+ {assessment && <button className="btn btn-quiet" type="button" disabled={!triggerRequestId || !onOpenEvidence} onClick={() => triggerRequestId && onOpenEvidence?.(sample.session_id, triggerRequestId)}>{triggerRequestId ? text('打开评分触发请求', 'Open assessment-trigger request') : text('评分触发请求不可用', 'Assessment-trigger request unavailable')}</button>}
+ </Disclosure>
+ <div className="quality-evidence-actions"><button className="btn btn-quiet" type="button" disabled={!executionRequestId || !onOpenEvidence} onClick={() => executionRequestId && onOpenEvidence?.(sample.session_id, executionRequestId)}><UiIcon name="activity" />{executionRequestId ? text('执行证据', 'Execution evidence') : text('执行证据不可用', 'Execution unavailable')}</button></div>
+ </article>;
+}
diff --git a/apps/desktop/src/features/Sessions.tsx b/apps/desktop/src/features/Sessions.tsx
index fcb5aec9..e97f1982 100644
--- a/apps/desktop/src/features/Sessions.tsx
+++ b/apps/desktop/src/features/Sessions.tsx
@@ -64,6 +64,7 @@ export function Sessions({ initialSession = null, initialRequest = null, languag
const [facts, setFacts] = useState<SafeFact[]>([]);
const [factsRequest, setFactsRequest] = useState<string | null>(null);
const [factsOpen, setFactsOpen] = useState(false);
+ const [qualityOpen, setQualityOpen] = useState(false);
const [factsError, setFactsError] = useState('');
const [factCursor, setFactCursor] = useState<string | null>(null);
const [factPartial, setFactPartial] = useState(false);
@@ -277,7 +278,7 @@ export function Sessions({ initialSession = null, initialRequest = null, languag
<div className="oc-model-back"><button className="btn btn-quiet" type="button" onClick={() => { timelineGeneration.current++; factGeneration.current++; setSession(null); setRequest(null); setFactsOpen(false); if (!listLoaded) void load(); }}><UiIcon name="arrowLeft" />{text('返回会话列表', 'Back to sessions')}</button></div>
<header className="transcript-head"><div><h2><ContentExcerpt session={session} language={language} fallback={text('会话正文', 'Conversation')} limit={56} revision={refreshVersion + reload} onModel={model => setSessionModels(current => current[session] === model ? current : { ...current, [session]: model })} onRoute={route => setSessionRoutes(current => current[session] === route ? current : { ...current, [session]: route })} /></h2><p>{[agentName(selectedSummary?.agent_id), sessionRoutes[session], selectedSummary ? sessionTimeLabel(selectedSummary.last_request_at_ms, language) : undefined].filter(Boolean).join(' · ')}</p></div><div className="transcript-actions"><button className="btn" type="button" disabled={!request} onClick={() => factsOpen ? setFactsOpen(false) : openFacts()}><UiIcon name="activity" />{factsOpen ? text('收起运行记录', 'Hide run record') : text('运行记录', 'Run record')}</button>{!factsOpen && <button className="icon-btn" type="button" aria-label={text('清理此会话', 'Clear this session')} disabled={busy} onClick={() => { setCleanupError(''); setCleanupOpen(true); }}><UiIcon name="trash" /></button>}</div></header>
{timelineError && <div className="callout bad session-detail-feedback" role="alert" data-error-code={timelineError}><UiIcon name="warning" /><div><strong>{text('会话详情没有刷新', 'The session details were not refreshed')}</strong><p>{requests.length ? text('已加载的请求仍然保留。', 'Previously loaded requests are still available.') : text('暂时无法读取这条会话。', 'This session is temporarily unavailable.')}</p><button className="btn" type="button" onClick={() => void timeline(session)}>{text('重试', 'Retry')}</button></div></div>}
- <Disclosure className="native-details session-model-performance" label={text('模型表现', 'Model performance')} language={language}><PlanQuality sessionId={session} language={language} compact onOpenEvidence={(targetSession, targetRequest) => void timeline(targetSession, null, undefined, targetRequest)} /></Disclosure>
+ <Disclosure className="native-details session-model-performance" label={text('模型表现', 'Model performance')} language={language} onOpenChange={setQualityOpen}><PlanQuality key={session} sessionId={session} language={language} compact active={active && qualityOpen} refreshVersion={refreshVersion + reload} onOpenEvidence={(targetSession, targetRequest) => void timeline(targetSession, null, undefined, targetRequest)} /></Disclosure>
{requests.length > 1 && <Disclosure className="native-details request-timeline" label={text(`请求记录(${requests.length})`, `Requests (${requests.length})`)} language={language}><nav className="native-list" aria-label={text('请求时间线', 'Request timeline')}>{requests.map((item, index) => <button className={`list-row${request === item.request_id ? ' active' : ''}`} key={item.request_id} aria-current={request === item.request_id ? 'true' : undefined} onClick={() => { setRequest(item.request_id); setReadingRequests([item.request_id]); setAnchor(null); }}><span className="row-main"><span className="row-title">{text('请求', 'Request')} {index + 1}</span><span className="row-meta">{item.routing_context?.state === 'recorded' && item.routing_context.display_name ? item.routing_context.display_name : item.final_native_model ?? text('路由未记录', 'Route not recorded')}{item.within_request_fallback === true ? text(' · 请求内回退', ' · Request fallback') : item.between_turn_model_change === true ? text(' · 跨轮切换', ' · Turn change') : ''}</span></span></button>)}{requestCursor && <button className="btn" disabled={busy} onClick={() => void timeline(session, requestCursor)}>{text('加载更多请求', 'Load more requests')}</button>}</nav></Disclosure>}
{displayedRequestIds.length ? displayedRequestIds.map((id, index) => <div key={id} className="request-chapter">{index > 0 && <h3>{text('后续请求', 'Following request')} {requests.findIndex(item => item.request_id === id) + 1}</h3>}<RequestBody request={id} summary={requests.find(item => item.request_id === id)} attemptModels={id === request ? attemptModels : []} anchor={id === request ? anchor : null} language={language} onOpenFacts={openFacts} onContentState={reportRequestContentState} onNextRequest={index === displayedRequestIds.length - 1 && nextRequestId ? () => { setReadingRequests(current => [...current, nextRequestId]); setRequest(nextRequestId); setAnchor(null); } : undefined} onMoreRequests={index === displayedRequestIds.length - 1 && !nextRequestId && requestCursor ? () => void timeline(session, requestCursor) : undefined} /></div>) : !busy && <div className="empty-state transcript-empty"><div><p>{text('此会话尚未取得可显示的请求。', 'No displayable request was retrieved for this session.')}</p></div></div>}
</article> : <div className="empty-state session-no-selection"><div><span className="empty-icon"><UiIcon name="sessions" /></span><h3>{text('选择一个会话', 'Select a session')}</h3></div></div>}
diff --git a/apps/desktop/src/features/plan-quality-state.ts b/apps/desktop/src/features/plan-quality-state.ts
new file mode 100644
index 00000000..6146c3fc
--- /dev/null
+++ b/apps/desktop/src/features/plan-quality-state.ts
@@ -0,0 +1,109 @@
+export type QualityExecution = {
+ plan_revision: number;
+ selected_branch_id: string | null;
+ executed_branch_id: string | null;
+ model_configuration_id: string | null;
+ profile_digest: string | null;
+ attribution: 'single' | 'mixed' | 'unknown';
+};
+
+export type QualityModelSummary = {
+ execution: QualityExecution;
+ native_model: string | null;
+ reasoning_profile_id: string | null;
+ scored_stage_count: number;
+ unrated_stage_count: number;
+ average_score: number | null;
+};
+
+export type QualitySummary = {
+ models: QualityModelSummary[];
+ scored_stage_count: number;
+ unrated_stage_count: number;
+ session_count: number;
+ available_revisions: number[];
+};
+
+export type PlanQualityModel = {
+ model_configuration_id: string;
+ display_name: string;
+ branch_id: string;
+ reasoning_profile_id: string | null;
+};
+
+export type QualityModelRow = {
+ key: string;
+ branch: string;
+ configured?: PlanQualityModel;
+ summary?: QualityModelSummary;
+};
+
+export function qualityExecutionKey(execution: QualityExecution): string {
+ return JSON.stringify([
+ execution.plan_revision, execution.selected_branch_id, execution.executed_branch_id,
+ execution.model_configuration_id, execution.profile_digest, execution.attribution,
+ ]);
+}
+
+/** Preserve published candidate order; never merge distinct execution profiles
+ * or sort models by score. Historical/unattributable rows retain their own key.
+ */
+export function qualityModelRows(configured: readonly PlanQualityModel[], summaries: readonly QualityModelSummary[]): QualityModelRow[] {
+ const remaining = new Set(summaries);
+ const rows: QualityModelRow[] = [];
+ const branchOf = (s: QualityModelSummary) => s.execution.attribution === 'single'
+ ? s.execution.executed_branch_id ?? 'unrecorded'
+ : 'unattributed';
+ for (const model of configured) {
+ const matches = summaries.filter(s => remaining.has(s)
+ && s.execution.attribution === 'single'
+ && s.execution.executed_branch_id === model.branch_id
+ && s.execution.model_configuration_id === model.model_configuration_id
+ && (s.reasoning_profile_id === model.reasoning_profile_id
+ || (!s.reasoning_profile_id && configured.filter(c => c.branch_id === model.branch_id
+ && c.model_configuration_id === model.model_configuration_id).length === 1)));
+ if (!matches.length) rows.push({
+ key: JSON.stringify(['configured', model.branch_id, model.model_configuration_id, model.reasoning_profile_id]),
+ branch: model.branch_id, configured: model,
+ });
+ for (const summary of matches) {
+ remaining.delete(summary);
+ rows.push({ key: qualityExecutionKey(summary.execution), branch: branchOf(summary), configured: model, summary });
+ }
+ }
+ for (const summary of summaries) {
+ if (remaining.has(summary)) rows.push({ key: qualityExecutionKey(summary.execution), branch: branchOf(summary), summary });
+ }
+ return rows;
+}
+
+export function qualityBranchLabel(branch: string, language: 'zh' | 'en'): string {
+ const labels: Record<string, [string, string]> = {
+ smart_saving_simple: ['日常执行 · 省钱分组', 'Routine work · Economy group'],
+ smart_saving_complex: ['复杂推理 · 主力分组', 'Complex work · Primary group'],
+ fixed: ['固定候选', 'Fixed candidates'],
+ fixed_model: ['固定候选', 'Fixed candidates'],
+ free: ['免费候选', 'Free candidates'],
+ free_first: ['免费候选', 'Free candidates'],
+ primary: ['主力分组', 'Primary group'],
+ unrecorded: ['执行分组未记录', 'Execution group unrecorded'],
+ unattributed: ['执行身份未确认', 'Unattributed execution'],
+ };
+ return labels[branch]?.[language === 'zh' ? 0 : 1] ?? (language === 'zh' ? '其他执行分组' : 'Other execution group');
+}
+
+export function qualityReasoningLabel(profile: string | null | undefined, language: 'zh' | 'en'): string {
+ if (profile && /^budget-[0-9]+$/.test(profile)) return profile.slice(7) + ' tokens';
+ // Native authored profile names are readable; a digest/path is diagnostic only.
+ if (profile && profile.length <= 32 && /^[a-zA-Z][a-zA-Z0-9 _-]*$/.test(profile)) return profile;
+ return language === 'zh' ? '推理配置未记录' : 'Reasoning configuration unrecorded';
+}
+
+/** CPA uses a routing prefix to pin the current Codex account. It is transport
+ * identity, not part of the model's human-facing name. Preserve other namespaces.
+ */
+export function qualityNativeModelName(nativeModel: string): string {
+ const prefix = 'hiroute-codex-current/';
+ return nativeModel.startsWith(prefix) && nativeModel.length > prefix.length
+ ? nativeModel.slice(prefix.length) : nativeModel;
+}
diff --git a/apps/desktop/src/occami/styles/pages.css b/apps/desktop/src/occami/styles/pages.css
index 63ffd12a..2837099b 100644
--- a/apps/desktop/src/occami/styles/pages.css
+++ b/apps/desktop/src/occami/styles/pages.css
@@ -370,27 +370,57 @@ button.list-row { width: 100%; border-top: 0; border-right: 0; border-left: 0; c
.field-actions { display: flex; align-items: center; flex-wrap: wrap; gap: 9px; }
.classifier-secret-create { margin-top: 10px; align-items: end; }
.classifier-secret-create .field-actions { min-height: 40px; }
-.quality-model-scope { display: flex; align-items: center; flex-wrap: wrap; gap: 8px 12px; margin-bottom: 10px; }
-.quality-model-scope-label { color: var(--text-muted); font-size: calc(11px * var(--hr-text-scale)); }
-.quality-model-list { display: flex; flex-wrap: wrap; gap: 6px; }
-.quality-filters { display: flex; align-items: center; flex-wrap: wrap; gap: 8px; margin-bottom: 12px; }
-.quality-list { display: grid; gap: 8px; }
-.quality-row { display: grid; grid-template-columns: minmax(0, 1fr) auto auto; align-items: center; gap: 8px 14px; padding: 12px; border: 1px solid var(--border); border-radius: 10px; background: var(--bg-soft); }
+/* Competence follows the published group / model configuration / stage hierarchy. */
+.quality-view-tabs { margin: 20px 0 0; width: fit-content; }
+.quality-toolbar, .quality-group-heading, .quality-stage-heading { display: flex; align-items: center; justify-content: space-between; flex-wrap: wrap; gap: 12px; }
+.quality-toolbar { margin-bottom: 18px; }
+.quality-scope-meta, .quality-group-heading > span, .quality-stage-heading span, .quality-average-note { color: var(--text-muted); font-size: calc(11px * var(--hr-text-scale)); }
+.quality-filters { display: flex; align-items: center; flex-wrap: wrap; gap: 8px; }
+.quality-model-scope { display: grid; gap: 20px; }
+.quality-group { overflow: hidden; border: 1px solid var(--border); border-radius: 12px; }
+.quality-group-heading { padding: 16px 18px; background: var(--bg-soft); }
+.quality-group-heading h4, .quality-stage-heading h4 { margin: 0; font-size: calc(13px * var(--hr-text-scale)); }
+.quality-model-columns, .quality-model-row { display: grid; grid-template-columns: minmax(150px, 1.5fr) minmax(140px, 1fr) minmax(92px, .65fr) auto; align-items: center; gap: 16px; padding: 14px 18px; }
+.quality-model-columns { padding-top: 10px; padding-bottom: 10px; color: var(--text-muted); font-size: calc(10px * var(--hr-text-scale)); border-bottom: 1px solid var(--border); }
+.quality-model-columns > :last-child { width: 110px; }
+.quality-model-row + .quality-model-row { border-top: 1px solid var(--border); }
+.quality-model-row.selected { background: var(--accent-soft); }
+.quality-model-name { min-width: 0; }
+.quality-model-name strong, .quality-model-name span, .quality-model-counts span { display: block; }
+.quality-model-name strong { font-size: calc(13px * var(--hr-text-scale)); overflow-wrap: anywhere; }
+.quality-model-name span, .quality-model-counts span { margin-top: 4px; font-size: calc(11px * var(--hr-text-scale)); color: var(--text-muted); }
+.quality-model-average { max-width: 190px; }
+.quality-model-average > strong { font-size: calc(19px * var(--hr-text-scale)); }
+.quality-model-average small { color: var(--text-muted); font-size: calc(11px * var(--hr-text-scale)); font-weight: 400; }
+.quality-model-average > span { color: var(--text-muted); font-size: calc(12px * var(--hr-text-scale)); }
+.quality-score-bar { height: 4px; border-radius: 4px; background: var(--border); margin-top: 8px; overflow: hidden; }
+.quality-score-bar > span { display: block; height: 100%; background: var(--accent); border-radius: inherit; }
+.quality-view-stages[aria-expanded='true'] .icon { transform: rotate(90deg); }
+.quality-average-note { margin: 12px 0 0; line-height: 1.6; }
+.quality-stage-section { margin-top: 24px; }
+.plan-quality.compact .quality-stage-section { margin-top: 0; }
+.quality-stage-heading { margin-bottom: 14px; }
+.quality-stage-heading span { display: block; margin-top: 5px; }
+.quality-list { display: grid; gap: 12px; }
+.quality-row { padding: 16px 18px; border: 1px solid var(--border); border-radius: 12px; background: var(--bg-soft); }
+.quality-row-header { display: flex; align-items: flex-start; justify-content: space-between; gap: 16px; }
.quality-row-main { min-width: 0; }
.quality-row-main strong, .quality-row-main span { display: block; }
-.quality-row-main strong { overflow: hidden; font-size: calc(12px * var(--hr-text-scale)); text-overflow: ellipsis; white-space: nowrap; }
-.quality-row-main span { margin-top: 3px; color: var(--text-muted); font-size: calc(10px * var(--hr-text-scale)); }
-.quality-score { min-width: 66px; text-align: right; }
+.quality-row-main strong { overflow-wrap: anywhere; font-size: calc(13px * var(--hr-text-scale)); }
+.quality-row-main span, .quality-stage-meta { margin-top: 5px; color: var(--text-muted); font-size: calc(11px * var(--hr-text-scale)); }
+.quality-score { flex: none; text-align: right; }
.quality-score strong, .quality-score span { display: block; }
-.quality-score strong { font-size: calc(16px * var(--hr-text-scale)); }
-.quality-score span { color: var(--text-muted); font-size: calc(9px * var(--hr-text-scale)); }
-.quality-flags { display: flex; flex-wrap: wrap; justify-content: flex-end; gap: 5px; }
-.quality-coverage { grid-column: 1 / -2; margin: 0; color: var(--text-muted); font-size: calc(10px * var(--hr-text-scale)); }
-.quality-reason { grid-column: 1 / -2; overflow-wrap: anywhere; margin: 0; color: var(--text-soft); font-size: calc(11px * var(--hr-text-scale)); }
-.quality-evidence-actions { grid-column: -2 / -1; display: flex; flex-wrap: wrap; justify-content: flex-end; gap: 6px; }
-.plan-quality.compact .quality-row { grid-template-columns: minmax(0, 1fr) auto; }
-.plan-quality.compact .quality-flags { grid-column: 1; justify-content: flex-start; }
-@media(max-width:760px){.quality-row{grid-template-columns:1fr auto}.quality-flags{grid-column:1;justify-content:flex-start}.quality-evidence-actions{grid-column:1/-1;justify-content:flex-start}.quality-coverage,.quality-reason{grid-column:1/-1}}
+.quality-score strong { font-size: calc(20px * var(--hr-text-scale)); }
+.quality-score span { margin-top: 3px; color: var(--text-muted); font-size: calc(10px * var(--hr-text-scale)); }
+.quality-stage-meta { margin-top: 12px; }
+.quality-assessment-details { margin-top: 12px; }
+.quality-flags { display: flex; flex-wrap: wrap; gap: 5px; }
+.quality-coverage { margin: 8px 0; color: var(--text-muted); font-size: calc(11px * var(--hr-text-scale)); }
+.quality-reason { overflow-wrap: anywhere; margin: 10px 0; color: var(--text-soft); font-size: calc(12px * var(--hr-text-scale)); line-height: 1.6; }
+.quality-diagnostic-identity { overflow-wrap: anywhere; margin: 8px 0; font-size: calc(10px * var(--hr-text-scale)); color: var(--text-muted); }
+.quality-evidence-actions { display: flex; flex-wrap: wrap; justify-content: flex-end; gap: 6px; margin-top: 8px; }
+.quality-more { margin-top: 14px; }
+@media(max-width:900px){.quality-model-columns{display:none}.quality-model-row{grid-template-columns:minmax(120px,1fr) auto}.quality-model-average{grid-column:2;grid-row:1}.quality-model-counts{grid-column:1}.quality-view-stages{grid-column:2;grid-row:2}}
.option-panel { display: grid; gap: 9px; }
.option-row { display: flex; align-items: center; justify-content: space-between; gap: 20px; padding: 13px; border: 1px solid var(--border); border-radius: 10px; background: var(--bg-soft); }
diff --git a/apps/desktop/src/plan-editor.tsx b/apps/desktop/src/plan-editor.tsx
index c2821212..154d8d55 100644
--- a/apps/desktop/src/plan-editor.tsx
+++ b/apps/desktop/src/plan-editor.tsx
@@ -112,17 +112,30 @@ function activePlanSelections(plan: Plan): Selection[] {
}
function activePlanQualityModels(plan: Plan, candidates: Candidate[]): PlanQualityModel[] {
const byBinding = new Map(candidates.map(candidate => [candidate.binding_id, candidate]));
- const seen = new Set<string>();
+ const strategy = plan.desired.strategy;
+ const groups: [string, Selection[]][] = plan.desired.mode === 'smart_saving'
+ ? [['smart_saving_simple', strategy.economy ?? []], ['smart_saving_complex', strategy.primary ?? []]]
+ : plan.desired.mode === 'free_first'
+ ? [['free', strategy.candidates ?? []], ['smart_saving_complex', strategy.primary_fallback ? strategy.primary ?? [] : []]]
+ : [['fixed', strategy.candidates ?? []]];
const models: PlanQualityModel[] = [];
- for (const selection of activePlanSelections(plan)) {
- const candidate = byBinding.get(selection.binding_id);
- if (!candidate || seen.has(candidate.model_configuration_id)) continue;
- seen.add(candidate.model_configuration_id);
- models.push({ model_configuration_id: candidate.model_configuration_id, display_name: candidate.display_name });
+ for (const [branch_id, selections] of groups) {
+ for (const selection of selections) {
+ const candidate = byBinding.get(selection.binding_id);
+ if (!candidate) continue;
+ const reasoning = selection.reasoning;
+ const reasoning_profile_id = reasoning?.kind === 'profile' ? reasoning.profile
+ : reasoning?.kind === 'toggle' ? reasoning.enabled ? 'enabled' : 'disabled'
+ : reasoning?.kind === 'budget' ? 'budget-' + reasoning.tokens
+ : candidate.reasoning.kind === 'fixed' ? candidate.reasoning.profile : null;
+ if (!models.some(model => model.branch_id === branch_id && model.model_configuration_id === candidate.model_configuration_id && model.reasoning_profile_id === reasoning_profile_id)) {
+ models.push({ model_configuration_id: candidate.model_configuration_id, display_name: candidate.display_name, branch_id, reasoning_profile_id });
+ }
+ }
}
return models;
}
-export const PlanEditor = forwardRef<PlanEditorHandle, { plan?: Plan; draft?: Draft; creating?: boolean; initialBindingId?: string; editingMemory?: PlanEditorMemory; language: 'zh' | 'en'; active?: boolean; usedBy?: { id: string; name: string; brand: 'codex' | 'claude-code' | 'qoder' | 'pi' | 'agent'; isDefault: boolean }[]; onOpenAgent?: (agentId: string) => void; onOpenSession?: (sessionId: string, requestId: string) => void; onDone: () => Promise<void>; onClose: () => void; onDirty?: (dirty: boolean) => void; onEdit?: () => void; onBusyChange?: (busy: boolean) => void; onOperation?: (operation: PlanOperation | null) => void; onPersisted?: (editor: PersistedEditor<Plan, Draft> | null, action: 'save_draft' | 'publish', identity: PersistenceIdentity, operation: PlanOperation | null) => void }>(function PlanEditor({ plan, draft, creating = false, initialBindingId, editingMemory, language, active = true, usedBy = [], onOpenAgent, onOpenSession, onDone, onClose, onDirty, onEdit, onBusyChange, onOperation, onPersisted }, ref) {
+export const PlanEditor = forwardRef<PlanEditorHandle, { plan?: Plan; draft?: Draft; creating?: boolean; initialBindingId?: string; editingMemory?: PlanEditorMemory; language: 'zh' | 'en'; active?: boolean; refreshVersion?: number; usedBy?: { id: string; name: string; brand: 'codex' | 'claude-code' | 'qoder' | 'pi' | 'agent'; isDefault: boolean }[]; onOpenAgent?: (agentId: string) => void; onOpenSession?: (sessionId: string, requestId: string) => void; onDone: () => Promise<void>; onClose: () => void; onDirty?: (dirty: boolean) => void; onEdit?: () => void; onBusyChange?: (busy: boolean) => void; onOperation?: (operation: PlanOperation | null) => void; onPersisted?: (editor: PersistedEditor<Plan, Draft> | null, action: 'save_draft' | 'publish', identity: PersistenceIdentity, operation: PlanOperation | null) => void }>(function PlanEditor({ plan, draft, creating = false, initialBindingId, editingMemory, language, active = true, refreshVersion = 0, usedBy = [], onOpenAgent, onOpenSession, onDone, onClose, onDirty, onEdit, onBusyChange, onOperation, onPersisted }, ref) {
const en = language === 'en', text = (zh: string, eng: string) => en ? eng : zh;
const [editor, setEditor] = useState<Editor>(() => structuredClone(draft?.editor ?? (plan ? reopen(plan) : emptyEditor())));
const [draftId] = useState(() => draft?.draft_id ?? 'draft/' + crypto.randomUUID());
@@ -130,6 +143,7 @@ export const PlanEditor = forwardRef<PlanEditorHandle, { plan?: Plan; draft?: Dr
const [options, setOptions] = useState<Options | null>(null), [busy, setBusy] = useState(false), [error, setError] = useState('');
const [baseline, setBaseline] = useState(() => JSON.stringify(editor));
const [notice, setNotice] = useState('');
+ const [view, setView] = useState<'configuration' | 'performance'>('configuration');
const [keywordInput, setKeywordInput] = useState('');
const [invalidFields, setInvalidFields] = useState(false);
const [validationIssue, setValidationIssue] = useState<ValidationIssue | null>(null);
@@ -472,7 +486,11 @@ export const PlanEditor = forwardRef<PlanEditorHandle, { plan?: Plan; draft?: Dr
const editorTone = dirty ? 'warn' : plan && base.expected_draft_revision === null ? 'good' : 'info';
return <section className="plan-editor" aria-label={text('路由编辑器', 'Plan editor')}>{!creating && <header className="editor-header"><div className="editor-title"><div className="title-with-status"><h2>{editor.display_name || text('新建智能路由', 'New smart routing')}</h2><span className={`badge ${editorTone} no-dot`}>{editorState}</span></div></div><div className="editor-actions"><button className="btn" type="button" disabled={busy || (!dirty && Boolean(plan || draft))} onClick={() => void act('save_draft')}>{text('保存草稿', 'Save draft')}</button><button type="button" className="btn btn-primary" disabled={busy || (!dirty && !!plan && base.expected_draft_revision === null)} onClick={() => void act('publish')}><UiIcon name="upload" />{plan ? text('发布更改', 'Publish changes') : text('启用', 'Enable')}</button></div></header>}
{notice && <div className="callout" role="status"><UiIcon name="info" /><span>{notice}</span></div>}{error && <div role="alert" className="callout bad" data-error-code={diagnostic}><UiIcon name="warning" /><span>{error}</span></div>}
- <fieldset disabled={busy}><section className="editor-section"><div className="editor-section-heading"><div><h3>{text('这份智能路由用来做什么', 'What this routing is for')}</h3><p>{text('Agent 根据用途选择适合任务的路由。', 'Your Agent uses this description to choose a route.')}</p></div></div><div className="plan-identity-fields"><label><span className="sr-only">{text('名称', 'Name')}</span><input className="input" autoFocus={creating} required placeholder={text('名称,例如:代码实现', 'Name, e.g. Code implementation')} aria-invalid={invalidFields && !editor.display_name.trim()} aria-describedby={invalidFields && !editor.display_name.trim() ? "route-name-error" : undefined} value={editor.display_name} maxLength={128} onChange={e => update({ display_name: e.target.value })} />{invalidFields && !editor.display_name.trim() && <span id="route-name-error" className="oc-inline-error">{text('请填写路由名称', 'Enter a route name')}</span>}</label><label><span className="sr-only">{text('使用场景', 'Purpose')}</span><input className="input" required placeholder={text('使用场景:适合做什么,期望交付什么', 'Purpose: tasks and expected results')} maxLength={512} aria-invalid={invalidFields && !editor.purpose.trim()} aria-describedby={invalidFields && !editor.purpose.trim() ? "route-purpose-error" : undefined} value={editor.purpose} onChange={e => update({ purpose: e.target.value })} />{invalidFields && !editor.purpose.trim() && <span id="route-purpose-error" className="oc-inline-error">{text('请填写使用场景', 'Enter a purpose')}</span>}</label>
+ {plan && <div className="segmented quality-view-tabs" role="tablist" aria-label={text('智能路由视图', 'Smart routing views')}>
+ <button className={'segment' + (view === 'configuration' ? ' active' : '')} type="button" role="tab" aria-selected={view === 'configuration'} onClick={() => setView('configuration')}>{text('路由配置', 'Routing configuration')}</button>
+ <button className={'segment' + (view === 'performance' ? ' active' : '')} type="button" role="tab" aria-selected={view === 'performance'} onClick={() => setView('performance')}>{text('模型表现', 'Model performance')}</button>
+ </div>}
+ <fieldset disabled={busy}><div hidden={!!plan && view !== 'configuration'}><section className="editor-section"><div className="editor-section-heading"><div><h3>{text('这份智能路由用来做什么', 'What this routing is for')}</h3><p>{text('Agent 根据用途选择适合任务的路由。', 'Your Agent uses this description to choose a route.')}</p></div></div><div className="plan-identity-fields"><label><span className="sr-only">{text('名称', 'Name')}</span><input className="input" autoFocus={creating} required placeholder={text('名称,例如:代码实现', 'Name, e.g. Code implementation')} aria-invalid={invalidFields && !editor.display_name.trim()} aria-describedby={invalidFields && !editor.display_name.trim() ? "route-name-error" : undefined} value={editor.display_name} maxLength={128} onChange={e => update({ display_name: e.target.value })} />{invalidFields && !editor.display_name.trim() && <span id="route-name-error" className="oc-inline-error">{text('请填写路由名称', 'Enter a route name')}</span>}</label><label><span className="sr-only">{text('使用场景', 'Purpose')}</span><input className="input" required placeholder={text('使用场景:适合做什么,期望交付什么', 'Purpose: tasks and expected results')} maxLength={512} aria-invalid={invalidFields && !editor.purpose.trim()} aria-describedby={invalidFields && !editor.purpose.trim() ? "route-purpose-error" : undefined} value={editor.purpose} onChange={e => update({ purpose: e.target.value })} />{invalidFields && !editor.purpose.trim() && <span id="route-purpose-error" className="oc-inline-error">{text('请填写使用场景', 'Enter a purpose')}</span>}</label>
</div><div className="field plan-alias"><label><span className="field-label">{text('接入模型名', 'Connection model name')}</span><input className="input plan-alias-field" readOnly={!!plan} value={editor.custom_alias ?? options?.suggested_alias ?? ''} placeholder="hiroute-…" maxLength={64} aria-invalid={validationIssue?.field === 'alias'} onChange={e => update({ custom_alias: e.target.value })} /></label><span className="field-help">{text('在 Agent 中使用此名称调用这条智能路由。创建后保持稳定。', 'Use this name in an Agent to call the smart route. It remains stable after creation.')}</span>{validationIssue?.field === 'alias' && <span className="oc-inline-error">{validationIssue.message}</span>}{!plan && editor.custom_alias !== undefined && <button className="btn" type="button" onClick={() => update({ custom_alias: undefined })}>{text('恢复自动名称', 'Use automatic name')}</button>}</div></section>
<section className="editor-section"><div className="editor-section-heading"><div><h3>{text('怎样使用模型', 'How to use models')}</h3></div></div>{optionsError && <div className="callout warn route-local-error" role="alert"><UiIcon name="warning" /><span>{optionsError}</span><button className="btn" type="button" onClick={() => setOptionsRetry(value => value + 1)}>{text('重试', 'Retry')}</button></div>}<div className="mode-switcher plan-mode-switcher">{(['fixed_model', 'smart_saving', 'free_first'] as Mode[]).map((m, i) => <button className={`mode-card${editor.mode === m ? ' active' : ''}`} type="button" aria-pressed={editor.mode === m} key={m} onClick={() => changeMode(m)}><strong>{text(['固定模型', '智能省钱', '免费优先'][i], ['Fixed model', 'Smart saving', 'Free first'][i])}</strong><span>{text(['按固定顺序依次尝试候选模型', '简单任务用省钱组合,复杂任务用主力组合', '先用免费模型,可选择主力兜底'][i], ['Try candidate models in a fixed order', 'Economy for simple tasks; primary for complex work', 'Use free models first, with optional primary fallback'][i])}</span></button>)}</div></section>
{editor.mode === 'fixed_model' && <section className="editor-section"><div className="editor-section-heading"><div><h3>{text('固定模型顺序', 'Fixed model order')}</h3><p>{text('HiRoute 按此顺序尝试;不满足能力或暂时不可用的模型会被跳过。', 'HiRoute tries this order, skipping models that cannot satisfy the request or are temporarily unavailable.')}</p></div></div>{group(text('候选模型', 'Candidate models'), text('发布后保持此顺序', 'Keep this order after publication'), editor.candidates, candidates => update({ candidates }), { groupId: 'fixed' })}</section>}
@@ -506,9 +524,9 @@ export const PlanEditor = forwardRef<PlanEditorHandle, { plan?: Plan; draft?: Dr
{plan && <section className="editor-section"><div className="editor-section-heading"><div><h3>{text('在哪里使用', 'Where it is used')}</h3><p>{text('在 Agent 页面调整模型路由。', 'Adjust model routing from the Agent page.')}</p></div></div>{usedBy.length ? <div className="agent-card-list">{usedBy.map(agent => <button className="agent-card" type="button" key={agent.id} onClick={() => onOpenAgent?.(agent.id)} disabled={!onOpenAgent}><BrandIcon kind={agent.brand} label={`${agent.name} logo`} /><div className="agent-main"><strong>{agent.name}</strong><span>{agent.isDefault ? text('主 Agent 默认使用', 'Default for main agent') : text('用于模型路由', 'Used for model routing')}</span></div><UiIcon name="chevronRight" /></button>)}</div> : <span className="muted plan-unused">{text('尚未被任何 Agent 用作模型路由', 'Not used by any agent for model routing yet')}</span>}</section>}
- {plan && <section className="editor-section"><div className="editor-section-heading"><div><h3>{text('运行表现', 'Runtime performance')}</h3><p>{text('默认比较当前生效版本所选模型在时间范围内的阶段胜任度;未评分与部分证据会明确标出。', 'Compare stage competence over time for models selected by the active revision. Unrated and partial evidence remain explicit.')}</p></div></div><PlanQuality planId={plan.agent_plan_id} planRevision={plan.agent_plan_revision} currentModels={activePlanQualityModels(plan, options?.candidates ?? [])} language={language} onOpenEvidence={onOpenSession} /></section>}
-
{(base.expected_draft_revision !== null || plan) && <div className="editor-actions editor-footer">{base.expected_draft_revision !== null && <button className="btn" type="button" onClick={() => void act('discard_draft')}>{text('丢弃草稿', 'Discard draft')}</button>}{plan && <Disclosure className="native-details" label={text('更多操作', 'More actions')} language={language}><button className="btn" type="button" onClick={() => void act(plan.head.status === 'disabled' ? 'enable' : 'disable')}>{plan.head.status === 'disabled' ? text('恢复调用', 'Enable') : text('停用路由', 'Disable')}</button><button className="btn btn-danger" type="button" onClick={() => void act('delete')}>{text('删除路由', 'Delete')}</button></Disclosure>}</div>}
+ </div>
+ {plan && <section className="editor-section quality-performance-panel" hidden={view !== 'performance'} role="tabpanel"><div className="editor-section-heading"><div><h3>{text('模型表现', 'Model performance')}</h3><p>{text('观察已发布路由中各模型的阶段胜任度,再查看具体执行阶段与证据。', 'Compare stage competence for models in the published route, then inspect execution stages and evidence.')}</p></div><span className="badge no-dot">{text('生效版本 r' + plan.agent_plan_revision, 'Active r' + plan.agent_plan_revision)}</span></div><PlanQuality planId={plan.agent_plan_id} planRevision={plan.agent_plan_revision} currentModels={activePlanQualityModels(plan, options?.candidates ?? [])} language={language} active={active && view === 'performance'} refreshVersion={refreshVersion} onOpenEvidence={onOpenSession} /></section>}
</fieldset>
<ClassifierProtocolDialog open={classifierProtocolOpen} language={language} onClose={() => setClassifierProtocolOpen(false)} />
{effort && <ReasoningDialog
diff --git a/apps/desktop/src/product/DesktopApp.tsx b/apps/desktop/src/product/DesktopApp.tsx
index ab2188d3..ad0efbf3 100644
--- a/apps/desktop/src/product/DesktopApp.tsx
+++ b/apps/desktop/src/product/DesktopApp.tsx
@@ -632,6 +632,7 @@ export function DesktopApp() {
key={routingIntent.key}
language={language}
active={page === 'routing'}
+ refreshVersion={refreshVersion}
snapshot={home.desktopSnapshot}
agentSnapshot={home.agentSnapshot}
operation={operation}
diff --git a/apps/desktop/src/product/RoutingPage.tsx b/apps/desktop/src/product/RoutingPage.tsx
index 15bdb08f..5b341f9b 100644
--- a/apps/desktop/src/product/RoutingPage.tsx
+++ b/apps/desktop/src/product/RoutingPage.tsx
@@ -12,9 +12,9 @@ import { routingEditorEntries } from '../routing-editor-entries';
export type RoutingEditorIntent = { key: string; plan?: Plan; draft?: Draft; staleDraft?: boolean; initialBindingId?: string; editingMemory?: PlanEditorMemory };
-export function RoutingPage({ language, active = true, snapshot, agentSnapshot, operation, loading, busy, initialEditor = null, notice, onRefresh, onOperation, onOpenAgent, onOpenSession }: {
+export function RoutingPage({ language, active = true, refreshVersion = 0, snapshot, agentSnapshot, operation, loading, busy, initialEditor = null, notice, onRefresh, onOperation, onOpenAgent, onOpenSession }: {
language: 'zh' | 'en'; snapshot: DesktopSnapshot | null; agentSnapshot?: AgentSnapshot | null; loading: boolean; busy: boolean;
- active?: boolean;
+ active?: boolean; refreshVersion?: number;
operation?: DesktopOperation | null;
initialEditor?: RoutingEditorIntent | null; notice?: string; onRefresh(): Promise<void>;
onOperation(operation: DesktopOperation | null): void; onOpenAgent?(agentId: string): void; onOpenSession?(sessionId: string, requestId: string): void;
@@ -143,7 +143,7 @@ export function RoutingPage({ language, active = true, snapshot, agentSnapshot,
};
const editorView = editor ? <fieldset className="detail-fieldset" disabled={!mutable || busy || localBusy || editorBusy}>
{editor.staleDraft && <div className="callout warn" role="status"><UiIcon name="warning" /><div><strong>{text('这是基于旧版本的草稿', 'This draft is based on an older version')}</strong><p>{text('当前生效路由已单独显示在列表中。保留此草稿供查看;如需继续,请先读取当前配置再重新编辑。', 'The active route is listed separately. This draft is retained for inspection; reload the current route before editing further.')}</p></div></div>}
- <PlanEditor ref={planEditor} key={`${editor.key}:${editor.draft?.revision ?? editor.plan?.head.head_revision ?? 'new'}`} creating={creating} plan={editor.plan} draft={editor.draft} initialBindingId={editor.initialBindingId} editingMemory={editor.editingMemory} language={language} active={active} usedBy={usedBy} onOpenAgent={onOpenAgent} onOpenSession={onOpenSession} onDirty={next => { setDirty(next); if (next) setPersistedNotice(''); }} onEdit={() => setPendingPersistence(null)} onBusyChange={setEditorBusy} onOperation={onOperation} onPersisted={(next, action, identity, submitted) => { setPendingPersistence(next ? null : { action, identity, operationId: submitted?.operation_id ?? null }); if (next) { returnEditor.current = null; setEditor(current => ({ ...next, editingMemory: current?.editingMemory ?? {} })); setDirty(false); setPersistedNotice(action === 'save_draft' ? text('草稿已保存,可以继续编辑或发布。', 'Draft saved. Continue editing or publish.') : text('更改已发布,新请求将使用当前路由。', 'Changes published. New requests will use this routing.')); } }} onDone={onRefresh} onClose={closeEditor} />
+ <PlanEditor ref={planEditor} key={`${editor.key}:${editor.draft?.revision ?? editor.plan?.head.head_revision ?? 'new'}`} creating={creating} plan={editor.plan} draft={editor.draft} initialBindingId={editor.initialBindingId} editingMemory={editor.editingMemory} language={language} active={active} refreshVersion={refreshVersion} usedBy={usedBy} onOpenAgent={onOpenAgent} onOpenSession={onOpenSession} onDirty={next => { setDirty(next); if (next) setPersistedNotice(''); }} onEdit={() => setPendingPersistence(null)} onBusyChange={setEditorBusy} onOperation={onOperation} onPersisted={(next, action, identity, submitted) => { setPendingPersistence(next ? null : { action, identity, operationId: submitted?.operation_id ?? null }); if (next) { returnEditor.current = null; setEditor(current => ({ ...next, editingMemory: current?.editingMemory ?? {} })); setDirty(false); setPersistedNotice(action === 'save_draft' ? text('草稿已保存,可以继续编辑或发布。', 'Draft saved. Continue editing or publish.') : text('更改已发布,新请求将使用当前路由。', 'Changes published. New requests will use this routing.')); } }} onDone={onRefresh} onClose={closeEditor} />
</fieldset> : null;
return <ProductPage
title={creating ? text('新建智能路由', 'New smart routing') : text('智能路由', 'Smart routing')}
diff --git a/apps/desktop/tests/plan-quality-state.test.mjs b/apps/desktop/tests/plan-quality-state.test.mjs
new file mode 100644
index 00000000..18a7dbba
--- /dev/null
+++ b/apps/desktop/tests/plan-quality-state.test.mjs
@@ -0,0 +1,51 @@
+import { test } from 'node:test';
+import assert from 'node:assert/strict';
+import { qualityExecutionKey, qualityModelRows, qualityNativeModelName, qualityReasoningLabel } from '../src/features/plan-quality-state.ts';
+
+const configured = (branch, model, profile = 'high') => ({ branch_id: branch, model_configuration_id: model, reasoning_profile_id: profile, display_name: model + ' readable' });
+const summary = (branch, model, profile = 'high', digest = profile) => ({
+ execution: { plan_revision: 1, selected_branch_id: null, executed_branch_id: branch, model_configuration_id: model, profile_digest: digest, attribution: 'single' },
+ reasoning_profile_id: profile, native_model: model, scored_stage_count: 1, unrated_stage_count: 0, average_score: .8,
+});
+
+test('published order survives a higher score and the same model in other groups or reasoning profiles', () => {
+ const models = [configured('economy', 'a', 'low'), configured('economy', 'b'), configured('primary', 'a'), configured('primary', 'a', 'medium')];
+ const stats = [summary('primary', 'a', 'medium'), summary('primary', 'a'), summary('economy', 'b'), { ...summary('economy', 'a', 'low'), average_score: 0 }];
+ const rows = qualityModelRows(models, stats);
+ assert.deepEqual(rows.map(row => [row.branch, row.configured.model_configuration_id, row.summary.reasoning_profile_id]), [
+ ['economy', 'a', 'low'], ['economy', 'b', 'high'], ['primary', 'a', 'high'], ['primary', 'a', 'medium'],
+ ]);
+ assert.equal(rows[0].summary.average_score, 0);
+ assert.equal(new Set(rows.map(row => row.key)).size, 4);
+});
+
+test('distinct execution digests remain separate and unused models acquire no invented score', () => {
+ const rows = qualityModelRows([configured('primary', 'a'), configured('primary', 'unused')], [summary('primary', 'a', 'high', 'digest/old'), summary('primary', 'a', 'high', 'digest/new')]);
+ assert.equal(rows.length, 3);
+ assert.notEqual(rows[0].key, rows[1].key);
+ assert.equal(rows[2].configured.model_configuration_id, 'unused');
+ assert.equal(rows[2].summary, undefined);
+});
+
+test('missing reasoning evidence cannot assign a historical stage to one of multiple configured profiles', () => {
+ const unknown = summary('primary', 'a', null, 'digest/unknown');
+ const rows = qualityModelRows([configured('primary', 'a', 'low'), configured('primary', 'a', 'high')], [unknown]);
+ assert.equal(rows.length, 3);
+ assert.equal(rows[2].configured, undefined);
+ assert.equal(rows[2].summary, unknown);
+ const mixed = { ...unknown, execution: { ...unknown.execution, attribution: 'mixed' } };
+ assert.equal(qualityModelRows([], [mixed])[0].branch, 'unattributed');
+ assert.notEqual(qualityExecutionKey(mixed.execution), qualityExecutionKey(unknown.execution));
+});
+
+test('reasoning controls use readable labels and keep opaque identities out of the overview', () => {
+ assert.equal(qualityReasoningLabel('budget-16000', 'zh'), '16000 tokens');
+ assert.equal(qualityReasoningLabel('high', 'en'), 'high');
+ assert.equal(qualityReasoningLabel('xhigh', 'zh'), 'xhigh');
+ assert.equal(qualityReasoningLabel('medium', 'zh'), 'medium');
+ assert.equal(qualityReasoningLabel('sha256:1234', 'zh'), '推理配置未记录');
+ assert.equal(qualityNativeModelName('hiroute-codex-current/gpt-6-astra'), 'gpt-6-astra');
+ assert.equal(qualityNativeModelName('Qwen/Qwen3.8-Flash'), 'Qwen/Qwen3.8-Flash');
+ assert.equal(qualityNativeModelName('other-provider/gpt-6-astra'), 'other-provider/gpt-6-astra');
+ assert.equal(qualityNativeModelName('hiroute-codex-current/'), 'hiroute-codex-current/');
+});
diff --git a/apps/desktop/tests/v3/browser/product-fixtures.ts b/apps/desktop/tests/v3/browser/product-fixtures.ts
index 2c3f7cac..f791a9f1 100644
--- a/apps/desktop/tests/v3/browser/product-fixtures.ts
+++ b/apps/desktop/tests/v3/browser/product-fixtures.ts
@@ -646,6 +646,7 @@ export function mockProductInvoke(command: string, payload: Record<string, any>
const intent = payload?.request?.intent ?? {};
const view = intent.view;
const query = intent.query ?? {};
+ if (view === 'plan_quality') return { samples: [], summary: { models: [], scored_stage_count: 0, unrated_stage_count: 0, session_count: 0, available_revisions: [] }, next_cursor: null };
if (view === 'sessions') {
const rows = query.only_model_switch
? sessionRows.filter(row => row.fallback_request_count > 0)
diff --git a/apps/desktop/tests/v3/browser/product-shell-scenarios.mjs b/apps/desktop/tests/v3/browser/product-shell-scenarios.mjs
index 77dede82..74c54517 100644
--- a/apps/desktop/tests/v3/browser/product-shell-scenarios.mjs
+++ b/apps/desktop/tests/v3/browser/product-shell-scenarios.mjs
@@ -122,7 +122,8 @@ const scenarios = [
}),
scenario('desktop.routing.reactivation-models', ['routing-editor'], 'Returning to an existing route reloads saved model choices and scopes quality to active models', async () => {
await fresh(); await routing();
- await until(() => document.querySelector('.quality-model-scope'), 'active model scope');
+ await click('模型表现');
+ await until(() => document.querySelector('.quality-model-scope')?.textContent.includes('Qwen'), 'active model scope');
assert(document.querySelector('.quality-model-scope').textContent.includes('Qwen'), 'Active route models are absent');
assert(!document.querySelector('.quality-model-filter'), 'Quality asks users to type internal model IDs');
const before = calls('compute_management_snapshot').length;
diff --git a/apps/desktop/tests/v3/browser/session-window-scenarios.mjs b/apps/desktop/tests/v3/browser/session-window-scenarios.mjs
index 90d8875d..3976643a 100644
--- a/apps/desktop/tests/v3/browser/session-window-scenarios.mjs
+++ b/apps/desktop/tests/v3/browser/session-window-scenarios.mjs
@@ -43,7 +43,148 @@ function seedUserText(control, request, textValue) {
control.texts[contentId] = textValue;
}
+const emptySummary = { models: [], scored_stage_count: 0, unrated_stage_count: 0, session_count: 0, available_revisions: [] };
+const openQuality = () => { const disclosure = document.querySelector('.session-model-performance summary'); if (disclosure && !disclosure.parentElement.open) disclosure.click(); };
+
const scenarios = [
+ ['switching from a filtered 21-stage session resets the filter for a two-stage session', async () => {
+ const largeSession = 'observation-session/low-21';
+ const smallSession = 'observation-session/high-2';
+ const at = Date.now() - 60_000;
+ const stages = (session, count) => Array.from({ length: count }, (_, index) => ({
+ segment_id: `${session}/stage/${index}`, session_id: session, plan_id: 'plan/quality', plan_revision: 1,
+ selected_branch_id: 'smart_saving_simple', executed_branch_id: 'smart_saving_simple', attribution: 'single',
+ native_model: 'qwen3.8-flash', reasoning_profile_id: 'low',
+ first_turn_ordinal: index + 1, last_observed_turn_ordinal: index + 1, first_at_ms: at, last_at_ms: at,
+ history_partial: false, execution_evidence_available: false,
+ assessment: { score: session === largeSession && index === 0 ? .4 : .9, partial: false, target_from_ordinal: index + 1, target_through_ordinal: index + 1, evidence_available: false },
+ }));
+ await fresh(control => {
+ control.requests[largeSession] = [control.makeRequest(largeSession, 0, at + 1)];
+ control.requests[smallSession] = [control.makeRequest(smallSession, 0, at)];
+ control.views.plan_quality = query => {
+ const all = stages(query.session_id, query.session_id === largeSession ? 21 : 2);
+ const matched = query.score_lt == null ? all : all.filter(stage => stage.assessment.score < query.score_lt);
+ return { samples: matched.slice(0, 20), summary: { ...emptySummary, scored_stage_count: all.length }, next_cursor: matched.length > 20 ? 'page/two' : null };
+ };
+ });
+ await until(() => rows().length === 2, 'two sessions');
+ rows()[0].click(); openQuality();
+ await until(() => document.querySelectorAll('.quality-row').length === 20 && document.querySelector('.quality-score-filter'), 'large session first page');
+ const filter = document.querySelector('.quality-score-filter');
+ filter.value = 'low'; filter.dispatchEvent(new Event('change', { bubbles: true }));
+ await until(() => document.querySelectorAll('.quality-row').length === 1, 'low stage filter');
+ const beforeRefresh = reads('plan_quality').length;
+ document.querySelector('.quality-refresh').click();
+ await until(() => reads('plan_quality').length > beforeRefresh && document.querySelectorAll('.quality-row').length === 1, 'same-session filtered refresh');
+ assert(queryOf(reads('plan_quality').at(-1)).score_lt === .5, 'Same-session refresh cleared the score filter');
+ rows()[1].click();
+ await until(() => reads('plan_quality').some(call => queryOf(call).session_id === smallSession) && !document.querySelector('.quality-refresh').disabled, 'small session loaded');
+ const smallReads = reads('plan_quality').filter(call => queryOf(call).session_id === smallSession);
+ assert(smallReads.every(call => queryOf(call).score_lt == null), 'The new session inherited the previous low-score filter');
+ assert(document.querySelectorAll('.quality-row').length === 2, 'The new session hid its two high-score stages');
+ assert([...document.querySelectorAll('.quality-score strong')].every(element => element.textContent === '0.90 / 1'), 'The new session displayed stages from the previous session');
+ assert(!document.querySelector('.quality-more'), 'The new session inherited the previous pagination');
+ }],
+ ['plan competence stays full-scope while exact model stages are filtered and paged', async () => {
+ const identity = { plan_revision: 1, selected_branch_id: null, executed_branch_id: 'smart_saving_simple', model_configuration_id: 'model/qwen', profile_digest: 'sha256/profile-low', attribution: 'single' };
+ const stats = {
+ models: [{ execution: identity, native_model: 'qwen3.8-flash', reasoning_profile_id: 'low', scored_stage_count: 25, unrated_stage_count: 1, average_score: .492 }],
+ scored_stage_count: 25, unrated_stage_count: 1, session_count: 2, available_revisions: [2, 1],
+ };
+ const stage = index => ({
+ segment_id: 'bulk/' + index, session_id: 'session/bulk', plan_id: 'plan/quality', plan_revision: 1,
+ selected_branch_id: 'smart_saving_simple', executed_branch_id: 'smart_saving_simple',
+ model_configuration_id: 'model/qwen', profile_digest: identity.profile_digest,
+ native_model: 'qwen3.8-flash', reasoning_profile_id: 'low', attribution: 'single',
+ first_turn_ordinal: index + 1, last_observed_turn_ordinal: index + 1,
+ first_at_ms: Date.now() - 60000, last_at_ms: Date.now() - 60000,
+ history_partial: false, execution_evidence_available: false,
+ assessment: index === 25 ? null : { score: index === 0 ? 0 : .5, target_from_ordinal: index + 1, target_through_ordinal: index + 1, partial: false, evidence_available: false },
+ });
+ await fresh(control => {
+ seedSimple(control);
+ control.qualityModels = [
+ { model_configuration_id: 'model/qwen', display_name: 'Qwen Flash', reasoning_profile_id: 'low', branch_id: 'smart_saving_simple' },
+ { model_configuration_id: 'model/unused', display_name: 'Unused model', reasoning_profile_id: 'high', branch_id: 'smart_saving_complex' },
+ ];
+ control.views.plan_quality = query => ({
+ summary: stats,
+ samples: query.unrated_only ? [stage(25)] : query.score_lt ? [stage(0)]
+ : Array.from({ length: query.cursor ? 6 : 20 }, (_, index) => stage(index + (query.cursor ? 20 : 0))),
+ next_cursor: query.unrated_only || query.score_lt || query.cursor ? null : 'page/two',
+ });
+ });
+ c().showQualityPlan();
+ await until(() => document.querySelector('.quality-model-average strong')?.textContent.includes('0.49'), 'full-scope mean');
+ const modelRows = [...document.querySelectorAll('.quality-model-row')];
+ assert(modelRows[0].innerText.includes('25 已评分') && modelRows[0].innerText.includes('1 待评分'), 'Full-scope sample counts disappeared');
+ assert(modelRows[1].innerText.includes('暂无记录') && !modelRows[1].innerText.includes('0.00'), 'Unused candidate was assigned a zero score');
+ assert(!document.querySelector('.quality-row'), 'The overview starts with a long stage list');
+ modelRows[0].querySelector('button').click();
+ await until(() => document.querySelectorAll('.quality-row').length === 20, 'first stage page');
+ const first = queryOf(reads('plan_quality').at(-1));
+ assert(JSON.stringify(first.execution) === JSON.stringify(identity), 'Drill-down did not send the complete actual execution identity');
+ button('加载更多阶段').click();
+ await until(() => document.querySelectorAll('.quality-row').length === 26, 'second stage page');
+ const second = queryOf(reads('plan_quality').at(-1));
+ assert(first.from_ms === second.from_ms && first.to_ms === second.to_ms, 'Stage pagination changed the summary window');
+ const select = document.querySelector('.quality-score-filter');
+ select.value = 'low'; select.dispatchEvent(new Event('change', { bubbles: true }));
+ await until(() => document.querySelectorAll('.quality-row').length === 1 && document.querySelector('.quality-score strong')?.textContent === '0.00 / 1', 'zero score remains scored');
+ assert(document.querySelector('.quality-model-average strong').textContent.includes('0.49'), 'Low-score filtering recomputed the mean');
+ select.value = 'unrated'; select.dispatchEvent(new Event('change', { bubbles: true }));
+ await until(() => document.querySelector('.quality-score strong')?.textContent === '未评分', 'unrated stage filter');
+ assert(document.querySelector('.quality-model-average strong').textContent.includes('0.49'), 'Unrated filtering erased the mean');
+ document.querySelector('[data-model-configuration="model/unused"] .quality-view-stages').click();
+ await until(() => document.querySelector('.quality-stage-section') && document.querySelectorAll('.quality-row').length === 0, 'unused candidate has no execution stages');
+ const beforeUnusedRefresh = reads('plan_quality').length;
+ button('刷新').click();
+ await until(() => reads('plan_quality').length > beforeUnusedRefresh && document.querySelector('.quality-model-average strong')?.textContent.includes('0.49'), 'refresh of unused candidate retains full-scope summary');
+ assert(!document.querySelector('.quality-row') && !document.querySelector('.quality-more'), 'Unused candidate displayed another model’s stages or pagination');
+ assert(document.querySelector('[data-model-configuration="model/qwen"]').innerText.includes('25 已评分'), 'Unused candidate refresh erased the other model’s counts');
+ const beforeUnusedReentry = reads('plan_quality').length;
+ c().showHome(); await tick(); c().showQualityPlan();
+ await until(() => reads('plan_quality').length > beforeUnusedReentry && document.querySelector('.quality-model-average strong')?.textContent.includes('0.49'), 're-entry with unused candidate retains full-scope summary');
+ assert(!document.querySelector('.quality-row'), 'Unused candidate re-entry displayed another model’s stages');
+ const version = [...document.querySelectorAll('select')].find(element => element.querySelector('option[value="2"]'));
+ assert(version, 'Available retained revision cannot be selected');
+ version.value = '2'; version.dispatchEvent(new Event('change', { bubbles: true }));
+ await until(() => queryOf(reads('plan_quality').at(-1)).plan_revision === 2, 'exact retained revision');
+ assert(!queryOf(reads('plan_quality').at(-1)).execution && !queryOf(reads('plan_quality').at(-1)).unrated_only, 'Revision change retained an incompatible detail filter');
+ }],
+ ['quality refresh and re-entry pick up assessments and reject stale results', async () => {
+ let score = null;
+ const page = () => ({
+ samples: [{
+ segment_id: 'stage/refresh', session_id: 'observation-session/simple', plan_id: 'plan/quality', plan_revision: 1,
+ selected_branch_id: 'smart_saving_complex', executed_branch_id: 'smart_saving_complex', attribution: 'single',
+ native_model: 'gpt-6-astra', reasoning_profile_id: 'high',
+ first_turn_ordinal: 1, last_observed_turn_ordinal: 1, first_at_ms: Date.now() - 60000, last_at_ms: Date.now() - 60000,
+ history_partial: false, execution_evidence_available: false,
+ assessment: score == null ? null : { score, partial: false, target_from_ordinal: 1, target_through_ordinal: 1, evidence_available: false },
+ }],
+ summary: { ...emptySummary, scored_stage_count: score == null ? 0 : 1, unrated_stage_count: score == null ? 1 : 0 },
+ next_cursor: null,
+ });
+ await fresh(control => { seedSimple(control); control.views.plan_quality = page; });
+ await until(() => rows().length === 1, 'refresh session');
+ rows()[0].click(); openQuality();
+ await until(() => document.querySelector('.quality-score strong')?.textContent === '未评分', 'initial unrated stage');
+ const initial = queryOf(reads('plan_quality').at(-1));
+ score = .78; c().bumpRefresh();
+ await until(() => document.querySelector('.quality-score strong')?.textContent === '0.78 / 1', 'new assessment after refresh');
+ assert(queryOf(reads('plan_quality').at(-1)).to_ms > initial.to_ms, 'Quality refresh reused a stale upper bound');
+ const slow = deferred(); const before = reads('plan_quality').length;
+ c().views.plan_quality = () => slow.promise; c().bumpRefresh();
+ await until(() => reads('plan_quality').length > before, 'slow assessment query');
+ c().views.plan_quality = page; score = .9; c().bumpRefresh();
+ await until(() => document.querySelector('.quality-score strong')?.textContent === '0.90 / 1', 'latest assessment query');
+ score = .1; slow.resolve(page()); await tick();
+ assert(document.querySelector('.quality-score strong')?.textContent === '0.90 / 1', 'An older assessment overwrote a newer result');
+ c().showHome(); await tick(); score = .95; c().showSessions();
+ await until(() => document.querySelector('.quality-score strong')?.textContent === '0.95 / 1', 'fresh assessment after page re-entry');
+ }],
['quality stages show their executed models and distinguish evidence coverage', async () => {
await fresh(control => {
const session = 'observation-session/quality';
@@ -61,18 +202,21 @@ const scenarios = [
first_request_id: request.request_id, last_request_id: request.request_id,
history_partial: index === 0, execution_evidence_available: true,
assessment: index ? null : { trigger_request_id: strong.request_id, target_from_ordinal: 1, target_through_ordinal: 1, score: 0.485, partial: true, evidence_available: true },
- })), next_cursor: null });
+ })), summary: { ...emptySummary, scored_stage_count: 1, unrated_stage_count: 1 }, next_cursor: null });
});
await until(() => rows().length === 1, 'quality session');
rows()[0].click();
+ openQuality();
await until(() => document.querySelectorAll('.quality-row').length === 2, 'quality stages');
- const disclosure = document.querySelector('.session-model-performance summary');
- if (disclosure && !disclosure.parentElement.open) disclosure.click();
await until(() => document.querySelectorAll('.quality-row-main strong')[1]?.textContent === 'gpt-6-astra', 'recorded model names');
const names = [...document.querySelectorAll('.quality-row-main strong')].map(element => element.textContent);
assert(names.join('|') === 'qwen3.8-flash|gpt-6-astra', `Wrong historical names: ${names}`);
+ const overview = document.querySelector('.plan-quality').innerText;
+ assert(!overview.includes('评分证据不完整') && !overview.includes('历史记录不完整'), 'Diagnostics crowd the overview');
+ document.querySelectorAll('.quality-assessment-details summary').forEach(element => element.click());
+ await tick();
const quality = document.querySelector('.plan-quality').innerText;
- assert(!quality.includes('model/runtime-fallback/') && !quality.includes('smart_saving_'), 'Internal IDs appear as primary labels');
+ assert(names.every(name => !name.includes('model/runtime-fallback/')), 'Internal IDs appear as primary labels');
assert(quality.includes('历史记录不完整') && quality.includes('评分证据不完整'), 'Distinct evidence gaps are not explained');
assert(quality.includes('尚未产生该阶段的胜任度评分'), 'Unrated stage lacks an explanation');
assert(quality.includes('评分覆盖轮次 1–1'), 'Assessment coverage disappeared');
@@ -89,14 +233,15 @@ const scenarios = [
first_turn_ordinal: 1, last_observed_turn_ordinal: 1, first_at_ms: Date.now() - 60000, last_at_ms: Date.now() - 60000,
history_partial: false, execution_evidence_available: false,
assessment: { trigger_request_id: 'request/missing', target_from_ordinal: 1, target_through_ordinal: 1, score: 0.6, partial: false, evidence_available: false },
- }], next_cursor: null });
+ }], summary: { ...emptySummary, scored_stage_count: 1 }, next_cursor: null });
});
await until(() => rows().length === 1, 'missing-evidence session');
rows()[0].click();
+ openQuality();
await until(() => document.querySelector('.quality-row-main strong')?.textContent === '模型名称不可用', 'readable missing model');
const quality = document.querySelector('.plan-quality');
assert(quality.textContent.includes('0.60 / 1'), 'Missing name removed an available score');
- assert(!quality.textContent.includes('opaque-missing'), 'Opaque ID leaked into the primary content');
+ assert(!quality.innerText.includes('opaque-missing'), 'Opaque ID leaked into the primary content');
assert(quality.querySelector('.quality-evidence-actions button').disabled, 'Missing execution evidence is actionable');
}],
['a session that starts after mount appears on an explicit re-query', async () => {
diff --git a/apps/desktop/tests/v3/browser/session-window.tsx b/apps/desktop/tests/v3/browser/session-window.tsx
index 6448563d..928e0d6f 100644
--- a/apps/desktop/tests/v3/browser/session-window.tsx
+++ b/apps/desktop/tests/v3/browser/session-window.tsx
@@ -1,6 +1,7 @@
import React, { useState } from 'react';
import { createRoot } from 'react-dom/client';
import { mockIPC } from '@tauri-apps/api/mocks';
+import { PlanQuality, type PlanQualityModel } from '../../../src/features/PlanQuality';
import { Sessions } from '../../../src/features/Sessions';
import { Home, HomeNavigation } from '../../../src/features/home';
import type { HomeReads } from '../../../src/features/home/types';
@@ -41,6 +42,8 @@ const control = {
bumpRefresh: () => {},
showHome: () => {},
showSessions: () => {},
+ showQualityPlan: () => {},
+ qualityModels: [] as PlanQualityModel[],
reset: () => {},
};
@@ -100,7 +103,7 @@ function observationRead(view: string, query: Record<string, any>) {
case 'catalog': return { contents: control.catalog[query.request_id] ?? [], transcript_roots: [], roots_partial: false, next_cursor: null };
case 'content': return { state: 'complete', chunks: [{ text: control.texts[query.content_id] ?? '', original_byte_offset: 0 }], next_cursor: null };
case 'facts': return { facts: [], projection_partial: false, next_cursor: null };
- case 'plan_quality': return { samples: [], next_cursor: null };
+ case 'plan_quality': return { samples: [], summary: { models: [], scored_stage_count: 0, unrated_stage_count: 0, session_count: 0, available_revisions: [] }, next_cursor: null };
case 'home_value': return {
pending_requests: 0,
provisional_requests: 0,
@@ -150,10 +153,12 @@ const navItems = [
function Harness() {
const [generation, setGeneration] = useState(0);
const [refresh, setRefresh] = useState(0);
+ const [qualityVisible, setQualityVisible] = useState(false);
const [homeVisible, setHomeVisible] = useState(false);
control.bumpRefresh = () => setRefresh(value => value + 1);
control.showHome = () => setHomeVisible(true);
- control.showSessions = () => setHomeVisible(false);
+ control.showSessions = () => { setHomeVisible(false); setQualityVisible(false); };
+ control.showQualityPlan = () => { setHomeVisible(false); setQualityVisible(true); };
control.reset = () => {
control.requests = {};
control.catalog = {};
@@ -162,6 +167,8 @@ function Harness() {
control.views = {};
control.handlers = {};
control.commands = [];
+ control.qualityModels = [];
+ setQualityVisible(false);
setHomeVisible(false);
setRefresh(0);
setGeneration(value => value + 1);
@@ -189,7 +196,8 @@ function Harness() {
<main className="main"><div>
<p className="sr-only">组件测试:所有 IPC 为 mock;不连接 Tauri、daemon 或真实数据。</p>
{homeVisible && <Home language="zh" reads={homeReads} onAction={() => {}} />}
- <div hidden={homeVisible}><Sessions key={generation} active={!homeVisible} language="zh" refreshVersion={refresh} onOpenAgents={() => {}} /></div>
+ {qualityVisible && <PlanQuality key={'quality-' + generation} planId="plan/quality" planRevision={1} currentModels={control.qualityModels} active={!homeVisible} refreshVersion={refresh} language="zh" />}
+ <div hidden={homeVisible || qualityVisible}><Sessions key={generation} active={!homeVisible && !qualityVisible} language="zh" refreshVersion={refresh} onOpenAgents={() => {}} /></div>
</div></main>
</div>
</PresentationRoot>;
diff --git a/apps/desktop/tests/v3/visual/session-window.mjs b/apps/desktop/tests/v3/visual/session-window.mjs
index 0c4aaab4..81affc11 100644
--- a/apps/desktop/tests/v3/visual/session-window.mjs
+++ b/apps/desktop/tests/v3/visual/session-window.mjs
@@ -56,10 +56,16 @@ let vite;
let chrome;
try {
vite = spawn(path.join(desktopRoot, 'node_modules/.bin/vite'), ['tests/v3/browser', '--host', '127.0.0.1', '--port', String(httpPort), '--strictPort'], { cwd: desktopRoot, detached: true, stdio: ['ignore', 'pipe', 'pipe'] });
+ vite.stdout.resume();
+ vite.stderr.on('data', data => process.stderr.write(data));
await waitFor(`http://127.0.0.1:${httpPort}/`);
const chromeExecutable = process.env.CHROME_BIN || (process.platform === 'darwin' ? '/Applications/Google Chrome.app/Contents/MacOS/Google Chrome' : 'google-chrome');
chrome = spawn(chromeExecutable, ['--headless=new', '--disable-gpu', `--remote-debugging-port=${cdpPort}`, '--remote-allow-origins=*', `--user-data-dir=${profile}`, '--window-size=1280,900', 'about:blank'], { detached: true, stdio: ['ignore', 'pipe', 'pipe'] });
- await waitFor(`http://127.0.0.1:${cdpPort}/json/version`);
+ chrome.stdout.resume();
+ let chromeErrors = '';
+ chrome.stderr.on('data', data => { chromeErrors = (chromeErrors + data).slice(-8000); });
+ try { await waitFor(`http://127.0.0.1:${cdpPort}/json/version`); }
+ catch (error) { throw new Error(`${error.message}\n${chromeErrors}`); }
const runner = spawn(process.execPath, [path.join(here, 'session-window-active.mjs'), String(cdpPort), `http://127.0.0.1:${httpPort}/`, outputRoot], { cwd: desktopRoot, stdio: 'inherit' });
const outcome = await waitForExit(runner);
if (outcome.code !== 0) process.exitCode = outcome.code ?? 1;
diff --git a/crates/cli/src/observation.rs b/crates/cli/src/observation.rs
index 8978c0b1..8e90192a 100644
--- a/crates/cli/src/observation.rs
+++ b/crates/cli/src/observation.rs
@@ -21,6 +21,8 @@ pub(super) fn plan_quality_samples(options: &[String]) -> Result<Value, ErrorCod
segment_id: None,
plan_revision: None,
model_configuration_id: None,
+ execution: None,
+ unrated_only: false,
from_ms: None,
to_ms: None,
score_gt: None,
diff --git a/crates/domain/src/observation/plan_quality.rs b/crates/domain/src/observation/plan_quality.rs
index 37548c3f..b631873e 100644
--- a/crates/domain/src/observation/plan_quality.rs
+++ b/crates/domain/src/observation/plan_quality.rs
@@ -17,6 +17,10 @@ pub struct PlanQualitySamplesQuery {
#[serde(default, skip_serializing_if = "Option::is_none")]
pub model_configuration_id: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
+ pub execution: Option<PlanQualityExecutionIdentity>,
+ #[serde(default)]
+ pub unrated_only: bool,
+ #[serde(default, skip_serializing_if = "Option::is_none")]
pub from_ms: Option<i64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub to_ms: Option<i64>,
@@ -60,6 +64,10 @@ pub struct PlanQualitySample {
pub model_configuration_id: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub profile_digest: Option<String>,
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub native_model: Option<String>,
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub reasoning_profile_id: Option<String>,
pub attribution: AgentTurnAttributionV1,
pub first_turn_id: String,
pub first_turn_ordinal: u64,
@@ -85,6 +93,42 @@ pub struct PlanQualitySample {
#[serde(deny_unknown_fields)]
pub struct PlanQualitySamplesPage {
pub samples: Vec<PlanQualitySample>,
+ pub summary: PlanQualitySummary,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub next_cursor: Option<String>,
}
+
+/// Full authorized scope, independent of the stage page and detail filters.
+#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
+#[serde(deny_unknown_fields)]
+pub struct PlanQualitySummary {
+ pub models: Vec<PlanQualityModelSummary>,
+ pub scored_stage_count: u64,
+ pub unrated_stage_count: u64,
+ pub session_count: u64,
+ pub available_revisions: Vec<u64>,
+}
+
+#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
+#[serde(deny_unknown_fields)]
+pub struct PlanQualityModelSummary {
+ pub execution: PlanQualityExecutionIdentity,
+ pub native_model: Option<String>,
+ pub reasoning_profile_id: Option<String>,
+ pub scored_stage_count: u64,
+ pub unrated_stage_count: u64,
+ pub average_score: Option<f64>,
+}
+
+/// The exact aggregate key, also used for stage drill-down (including nulls).
+#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
+#[serde(deny_unknown_fields)]
+pub struct PlanQualityExecutionIdentity {
+ pub plan_revision: u64,
+ /// Present only when no executed branch was recorded; never replaces it.
+ pub selected_branch_id: Option<String>,
+ pub executed_branch_id: Option<String>,
+ pub model_configuration_id: Option<String>,
+ pub profile_digest: Option<String>,
+ pub attribution: AgentTurnAttributionV1,
+}
diff --git a/crates/observation/src/query_v2/plan_quality.rs b/crates/observation/src/query_v2/plan_quality.rs
index 7cf354f9..37d9446f 100644
--- a/crates/observation/src/query_v2/plan_quality.rs
+++ b/crates/observation/src/query_v2/plan_quality.rs
@@ -1,13 +1,60 @@
use hiroute_domain::{
- AgentTurnAttributionV1, CanonicalDigest, PlanQualityAssessment, PlanQualitySample,
- PlanQualitySamplesPage, PlanQualitySamplesQuery,
+ AgentTurnAttributionV1, CanonicalDigest, PlanQualityAssessment, PlanQualityExecutionIdentity,
+ PlanQualityModelSummary, PlanQualitySample, PlanQualitySamplesPage, PlanQualitySamplesQuery,
+ PlanQualitySummary,
};
-use rusqlite::{Connection, OpenFlags, params};
+use rusqlite::{Connection, OpenFlags, Transaction, named_params};
use serde::{Deserialize, Serialize};
use super::{ObservationReaderContext, ObservationV2Error, QueryDeadline, identifier};
use crate::LocalObservationStore;
+/// Both projections share the same scope and transaction. Detail filters never
+/// enter this CTE, so pagination and drill-down cannot change the full-scope mean.
+const SCOPE: &str = "
+ SELECT q.*,
+ COALESCE(
+ (SELECT a.model_id FROM observation_attempt_models_v2 a
+ JOIN valuation_requests_v2 v ON v.workspace_id=a.workspace_id
+ AND v.request_id=a.request_id AND v.accepted_ordinal=a.ordinal
+ JOIN logical_requests r ON r.workspace_id=a.workspace_id AND r.request_id=a.request_id
+ WHERE a.workspace_id=q.workspace_id AND a.request_id=q.last_request_id
+ AND r.session_id=q.session_id LIMIT 1),
+ (SELECT a.model_id FROM observation_attempt_models_v2 a
+ JOIN valuation_requests_v2 v ON v.workspace_id=a.workspace_id
+ AND v.request_id=a.request_id AND v.accepted_ordinal=a.ordinal
+ JOIN logical_requests r ON r.workspace_id=a.workspace_id AND r.request_id=a.request_id
+ WHERE a.workspace_id=q.workspace_id AND a.request_id=q.first_request_id
+ AND r.session_id=q.session_id LIMIT 1)
+ ) AS native_model,
+ (SELECT json_extract(p.body_json,'$.fact.reasoning_profile_id')
+ FROM execution_fact_events e
+ JOIN observation_sensitive_payloads_v2 p ON p.id='fact:'||e.envelope_digest
+ JOIN logical_requests r ON r.workspace_id=e.workspace_id AND r.request_id=e.request_id
+ WHERE e.workspace_id=q.workspace_id AND r.session_id=q.session_id
+ AND e.request_id IN(q.last_request_id,q.first_request_id)
+ AND json_extract(p.body_json,'$.fact.kind')='candidate_decision'
+ AND json_extract(p.body_json,'$.fact.model_configuration_id')=q.model_configuration_id
+ AND json_extract(p.body_json,'$.fact.profile_digest')=q.profile_digest
+ ORDER BY e.sequence DESC LIMIT 1
+ ) AS reasoning_profile_id
+ FROM plan_quality_segments q
+ JOIN sessions s ON s.workspace_id=q.workspace_id AND s.session_id=q.session_id
+ LEFT JOIN observation_run_links l ON l.workspace_id=q.workspace_id
+ AND l.request_id=q.first_request_id
+ WHERE q.workspace_id=:workspace AND q.last_at_ms>=:from AND q.last_at_ms<:to
+ AND q.last_at_ms IS NOT NULL AND q.selected_branch_id IS NOT NULL
+ AND (:plan IS NULL OR q.plan_id=:plan)
+ AND (:session IS NULL OR q.session_id=:session)
+ AND (:revision IS NULL OR q.plan_revision=:revision)
+ AND (:runs IS NULL OR (l.conflicted=0 AND
+ l.run_id IN(SELECT value FROM json_each(:runs))))
+ AND (s.tombstone_reason IS NULL OR EXISTS(
+ SELECT 1 FROM observation_tombstones t
+ WHERE t.workspace_id=q.workspace_id AND t.session_id=q.session_id
+ AND t.delete_scope='content_only'))
+";
+
#[derive(Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
struct Cursor {
@@ -76,18 +123,28 @@ impl LocalObservationStore {
.map(serde_json::to_string)
.transpose()
.map_err(|_| ObservationV2Error::Invalid)?;
+ let summary = summarize(&transaction, reader, query, from_ms, to_ms, &runs)?;
+ let execution = query
+ .execution
+ .as_ref()
+ .map(serde_json::to_string)
+ .transpose()
+ .map_err(|_| ObservationV2Error::Invalid)?;
let mut rows = {
- let mut statement = transaction.prepare(
- "SELECT q.segment_id,q.session_id,q.plan_id,q.plan_revision,
+ let sql = format!(
+ "WITH quality AS ({SCOPE})
+ SELECT q.segment_id,q.session_id,q.plan_id,q.plan_revision,
q.selected_branch_id,q.executed_branch_id,q.model_configuration_id,
q.profile_digest,q.attribution,q.first_turn_id,q.first_turn_ordinal,
q.last_observed_turn_id,q.last_observed_turn_ordinal,q.first_at_ms,
q.last_at_ms,q.history_partial,q.first_request_id,q.last_request_id,
EXISTS(SELECT 1 FROM logical_requests first_req
WHERE first_req.workspace_id=q.workspace_id
+ AND first_req.session_id=q.session_id
AND first_req.request_id=q.first_request_id)
AND EXISTS(SELECT 1 FROM logical_requests last_req
WHERE last_req.workspace_id=q.workspace_id
+ AND last_req.session_id=q.session_id
AND last_req.request_id=q.last_request_id),
CASE WHEN COALESCE(l.conflicted,0)=0 THEN json_extract(l.body_json,'$.task_id') END,
CASE WHEN COALESCE(l.conflicted,0)=0 THEN l.run_id END,
@@ -99,107 +156,49 @@ impl LocalObservationStore {
q.assessment_event_id IS NOT NULL AND EXISTS(
SELECT 1 FROM logical_requests trigger_req
WHERE trigger_req.workspace_id=q.workspace_id
- AND trigger_req.request_id=q.assessment_trigger_request_id)
- FROM plan_quality_segments q
- JOIN sessions s ON s.workspace_id=q.workspace_id AND s.session_id=q.session_id
+ AND trigger_req.session_id=q.session_id
+ AND trigger_req.request_id=q.assessment_trigger_request_id),
+ q.native_model,q.reasoning_profile_id
+ FROM quality q
LEFT JOIN observation_run_links l ON l.workspace_id=q.workspace_id
AND l.request_id=q.first_request_id
LEFT JOIN execution_fact_events e ON e.workspace_id=q.workspace_id
AND e.event_id=q.assessment_event_id
LEFT JOIN observation_sensitive_payloads_v2 p
ON p.id='fact:'||e.envelope_digest
- WHERE q.workspace_id=?1 AND q.last_at_ms>=?2 AND q.last_at_ms<?3
- AND q.last_at_ms IS NOT NULL AND q.selected_branch_id IS NOT NULL
- AND (?4 IS NULL OR q.plan_id=?4)
- AND (?5 IS NULL OR q.session_id=?5)
- AND (?6 IS NULL OR q.segment_id=?6)
- AND (?7 IS NULL OR q.plan_revision=?7)
- AND (?8 IS NULL OR q.model_configuration_id=?8)
- AND (?9 IS NULL OR q.score>?9)
- AND (?10 IS NULL OR q.score<?10)
- AND (q.last_at_ms<?11 OR
- (q.last_at_ms=?11 AND q.segment_id>?12))
- AND (?13 IS NULL OR (l.conflicted=0 AND
- l.run_id IN(SELECT value FROM json_each(?13))))
- AND (s.tombstone_reason IS NULL OR EXISTS(
- SELECT 1 FROM observation_tombstones t
- WHERE t.workspace_id=q.workspace_id AND t.session_id=q.session_id
- AND t.delete_scope='content_only'))
- ORDER BY q.last_at_ms DESC,q.segment_id
- LIMIT ?14",
- )?;
+ WHERE (:segment IS NULL OR q.segment_id=:segment)
+ AND (:model IS NULL OR q.model_configuration_id=:model)
+ AND (:unrated=0 OR q.assessment_event_id IS NULL)
+ AND (:gt IS NULL OR q.score>:gt)
+ AND (:lt IS NULL OR q.score<:lt)
+ AND (:execution IS NULL OR (
+ q.plan_revision=json_extract(:execution,'$.plan_revision')
+ AND q.executed_branch_id IS json_extract(:execution,'$.executed_branch_id')
+ AND (q.executed_branch_id IS NOT NULL OR
+ q.selected_branch_id=json_extract(:execution,'$.selected_branch_id'))
+ AND q.model_configuration_id IS json_extract(:execution,'$.model_configuration_id')
+ AND q.profile_digest IS json_extract(:execution,'$.profile_digest')
+ AND q.attribution=json_extract(:execution,'$.attribution')))
+ AND (q.last_at_ms<:last_at OR
+ (q.last_at_ms=:last_at AND q.segment_id>:last_segment))
+ ORDER BY q.last_at_ms DESC,q.segment_id LIMIT :limit"
+ );
+ let mut statement = transaction.prepare(&sql)?;
statement
.query_map(
- params![
- reader.workspace().as_str(),
- from_ms,
- to_ms,
- query.plan_id,
- query.session_id,
- query.segment_id,
- query
- .plan_revision
- .and_then(|value| i64::try_from(value).ok()),
- query.model_configuration_id,
- query.score_gt,
- query.score_lt,
- cursor.last_at_ms,
- cursor.last_segment_id,
- runs,
- u64::from(query.limit) + 1,
- ],
- |row| {
- let attribution = match row.get::<_, String>(8)?.as_str() {
- "single" => AgentTurnAttributionV1::Single,
- "mixed" => AgentTurnAttributionV1::Mixed,
- "unknown" => AgentTurnAttributionV1::Unknown,
- _ => return Err(rusqlite::Error::InvalidQuery),
- };
- let assessment_event_id: Option<String> = row.get(21)?;
- let assessment = assessment_event_id
- .map(|event_id| {
- Ok::<PlanQualityAssessment, rusqlite::Error>(
- PlanQualityAssessment {
- event_id,
- trigger_request_id: row.get(22)?,
- assessed_at_ms: row.get(23)?,
- target_from_turn_id: row.get(24)?,
- target_through_turn_id: row.get(25)?,
- target_from_ordinal: sql_u64(row.get(26)?)?,
- target_through_ordinal: sql_u64(row.get(27)?)?,
- score: row.get(28)?,
- partial: row.get(29)?,
- reason: row.get(30)?,
- evidence_available: row.get(31)?,
- },
- )
- })
- .transpose()?;
- Ok(PlanQualitySample {
- segment_id: row.get(0)?,
- session_id: row.get(1)?,
- plan_id: row.get(2)?,
- plan_revision: sql_u64(row.get(3)?)?,
- selected_branch_id: row.get(4)?,
- executed_branch_id: row.get(5)?,
- model_configuration_id: row.get(6)?,
- profile_digest: row.get(7)?,
- attribution,
- first_turn_id: row.get(9)?,
- first_turn_ordinal: sql_u64(row.get(10)?)?,
- last_observed_turn_id: row.get(11)?,
- last_observed_turn_ordinal: sql_u64(row.get(12)?)?,
- first_at_ms: row.get(13)?,
- last_at_ms: row.get(14)?,
- history_partial: row.get(15)?,
- first_request_id: row.get(16)?,
- last_request_id: row.get(17)?,
- execution_evidence_available: row.get(18)?,
- task_id: row.get(19)?,
- run_id: row.get(20)?,
- assessment,
- })
+ named_params! {
+ ":workspace": reader.workspace().as_str(),
+ ":from": from_ms, ":to": to_ms,
+ ":plan": query.plan_id, ":session": query.session_id,
+ ":revision": query.plan_revision.and_then(|v| i64::try_from(v).ok()),
+ ":runs": runs, ":segment": query.segment_id,
+ ":model": query.model_configuration_id, ":unrated": query.unrated_only,
+ ":gt": query.score_gt, ":lt": query.score_lt,
+ ":execution": execution, ":last_at": cursor.last_at_ms,
+ ":last_segment": cursor.last_segment_id,
+ ":limit": u64::from(query.limit) + 1,
},
+ sample,
)?
.collect::<Result<Vec<_>, _>>()?
};
@@ -228,16 +227,154 @@ impl LocalObservationStore {
reader.check(now_ms, false, false)?;
Ok(PlanQualitySamplesPage {
samples: rows,
+ summary,
next_cursor,
})
}
}
+fn summarize(
+ transaction: &Transaction<'_>,
+ reader: &ObservationReaderContext,
+ query: &PlanQualitySamplesQuery,
+ from_ms: i64,
+ to_ms: i64,
+ runs: &Option<String>,
+) -> Result<PlanQualitySummary, ObservationV2Error> {
+ let sql = format!(
+ "WITH quality AS ({SCOPE})
+ SELECT plan_revision,
+ CASE WHEN executed_branch_id IS NULL THEN selected_branch_id END,
+ executed_branch_id,model_configuration_id,profile_digest,attribution,
+ CASE WHEN COUNT(DISTINCT native_model)=1 THEN MAX(native_model) END,
+ CASE WHEN COUNT(DISTINCT reasoning_profile_id)=1 THEN MAX(reasoning_profile_id) END,
+ COUNT(score),COUNT(*)-COUNT(score),AVG(score)
+ FROM quality GROUP BY plan_revision,
+ CASE WHEN executed_branch_id IS NULL THEN selected_branch_id END,
+ executed_branch_id,model_configuration_id,profile_digest,attribution
+ ORDER BY plan_revision DESC,executed_branch_id,model_configuration_id,profile_digest,attribution
+ LIMIT 1001"
+ );
+ let scope_params = named_params! {
+ ":workspace": reader.workspace().as_str(), ":from": from_ms, ":to": to_ms,
+ ":plan": query.plan_id, ":session": query.session_id,
+ ":revision": query.plan_revision.and_then(|v| i64::try_from(v).ok()), ":runs": runs,
+ };
+ let models = transaction
+ .prepare(&sql)?
+ .query_map(scope_params, |row| {
+ Ok(PlanQualityModelSummary {
+ execution: PlanQualityExecutionIdentity {
+ plan_revision: sql_u64(row.get(0)?)?,
+ selected_branch_id: row.get(1)?,
+ executed_branch_id: row.get(2)?,
+ model_configuration_id: row.get(3)?,
+ profile_digest: row.get(4)?,
+ attribution: attribution(row.get::<_, String>(5)?.as_str())?,
+ },
+ native_model: row.get(6)?,
+ reasoning_profile_id: row.get(7)?,
+ scored_stage_count: sql_u64(row.get(8)?)?,
+ unrated_stage_count: sql_u64(row.get(9)?)?,
+ average_score: row.get(10)?,
+ })
+ })?
+ .collect::<Result<Vec<_>, _>>()?;
+ if models.len() > 1000 {
+ return Err(ObservationV2Error::Unavailable);
+ }
+ let session_count: i64 = transaction.query_row(
+ &format!("WITH quality AS ({SCOPE}) SELECT COUNT(DISTINCT session_id) FROM quality"),
+ scope_params,
+ |row| row.get(0),
+ )?;
+ let revisions = transaction.prepare(
+ &format!("WITH quality AS ({SCOPE}) SELECT DISTINCT plan_revision FROM quality ORDER BY plan_revision DESC"),
+ )?.query_map(named_params! {
+ ":workspace": reader.workspace().as_str(), ":from": from_ms, ":to": to_ms,
+ ":plan": query.plan_id, ":session": query.session_id,
+ ":revision": Option::<i64>::None, ":runs": runs,
+ }, |row| sql_u64(row.get(0)?))?.collect::<Result<Vec<_>, _>>()?;
+ Ok(PlanQualitySummary {
+ scored_stage_count: models.iter().map(|m| m.scored_stage_count).sum(),
+ unrated_stage_count: models.iter().map(|m| m.unrated_stage_count).sum(),
+ session_count: sql_u64(session_count)?,
+ models,
+ available_revisions: revisions,
+ })
+}
+
+fn sample(row: &rusqlite::Row<'_>) -> rusqlite::Result<PlanQualitySample> {
+ let assessment_event_id: Option<String> = row.get(21)?;
+ let assessment = assessment_event_id
+ .map(|event_id| {
+ Ok::<_, rusqlite::Error>(PlanQualityAssessment {
+ event_id,
+ trigger_request_id: row.get(22)?,
+ assessed_at_ms: row.get(23)?,
+ target_from_turn_id: row.get(24)?,
+ target_through_turn_id: row.get(25)?,
+ target_from_ordinal: sql_u64(row.get(26)?)?,
+ target_through_ordinal: sql_u64(row.get(27)?)?,
+ score: row.get(28)?,
+ partial: row.get(29)?,
+ reason: row.get(30)?,
+ evidence_available: row.get(31)?,
+ })
+ })
+ .transpose()?;
+ Ok(PlanQualitySample {
+ segment_id: row.get(0)?,
+ session_id: row.get(1)?,
+ plan_id: row.get(2)?,
+ plan_revision: sql_u64(row.get(3)?)?,
+ selected_branch_id: row.get(4)?,
+ executed_branch_id: row.get(5)?,
+ model_configuration_id: row.get(6)?,
+ profile_digest: row.get(7)?,
+ attribution: attribution(row.get::<_, String>(8)?.as_str())?,
+ first_turn_id: row.get(9)?,
+ first_turn_ordinal: sql_u64(row.get(10)?)?,
+ last_observed_turn_id: row.get(11)?,
+ last_observed_turn_ordinal: sql_u64(row.get(12)?)?,
+ first_at_ms: row.get(13)?,
+ last_at_ms: row.get(14)?,
+ history_partial: row.get(15)?,
+ first_request_id: row.get(16)?,
+ last_request_id: row.get(17)?,
+ execution_evidence_available: row.get(18)?,
+ task_id: row.get(19)?,
+ run_id: row.get(20)?,
+ assessment,
+ native_model: row.get(32)?,
+ reasoning_profile_id: row.get(33)?,
+ })
+}
+
fn validate(query: &PlanQualitySamplesQuery) -> Result<(), ObservationV2Error> {
+ let execution_invalid = query.execution.as_ref().is_some_and(|e| {
+ e.plan_revision == 0
+ || i64::try_from(e.plan_revision).is_err()
+ || e.executed_branch_id.is_some() == e.selected_branch_id.is_some()
+ || [
+ &e.selected_branch_id,
+ &e.executed_branch_id,
+ &e.model_configuration_id,
+ &e.profile_digest,
+ ]
+ .into_iter()
+ .flatten()
+ .any(|value| !identifier(value))
+ || query
+ .plan_revision
+ .is_some_and(|revision| revision != e.plan_revision)
+ });
if query.plan_id.is_none() && query.session_id.is_none()
|| query.limit == 0
|| query.limit > 200
|| query.plan_revision == Some(0)
+ || execution_invalid
+ || query.unrated_only && (query.score_gt.is_some() || query.score_lt.is_some())
|| query.from_ms.is_some_and(|value| value < 0)
|| query.to_ms.is_some_and(|value| value < 0)
|| query
@@ -271,6 +408,15 @@ fn validate(query: &PlanQualitySamplesQuery) -> Result<(), ObservationV2Error> {
}
}
+fn attribution(value: &str) -> rusqlite::Result<AgentTurnAttributionV1> {
+ match value {
+ "single" => Ok(AgentTurnAttributionV1::Single),
+ "mixed" => Ok(AgentTurnAttributionV1::Mixed),
+ "unknown" => Ok(AgentTurnAttributionV1::Unknown),
+ _ => Err(rusqlite::Error::InvalidQuery),
+ }
+}
+
fn sql_u64(value: i64) -> rusqlite::Result<u64> {
value.try_into().map_err(|error| {
rusqlite::Error::FromSqlConversionFailure(
diff --git a/crates/observation/src/store/tests/plan_quality.rs b/crates/observation/src/store/tests/plan_quality.rs
index 7d978589..d7500ff7 100644
--- a/crates/observation/src/store/tests/plan_quality.rs
+++ b/crates/observation/src/store/tests/plan_quality.rs
@@ -9,6 +9,8 @@ use crate::query_v2::ObservationReaderContext;
use super::support::{Fixture, fact_channel, offer_fact, open_store, writer};
+mod summary;
+
const PLAN: &str = "plan/codex-daily";
const MODEL: &str = "model/config-a";
@@ -130,6 +132,8 @@ fn query(plan_revision: Option<u64>) -> PlanQualitySamplesQuery {
segment_id: None,
plan_revision,
model_configuration_id: None,
+ execution: None,
+ unrated_only: false,
from_ms: Some(0),
to_ms: Some(10_000),
score_gt: None,
diff --git a/crates/observation/src/store/tests/plan_quality/summary.rs b/crates/observation/src/store/tests/plan_quality/summary.rs
new file mode 100644
index 00000000..5f20f8ec
--- /dev/null
+++ b/crates/observation/src/store/tests/plan_quality/summary.rs
@@ -0,0 +1,333 @@
+use super::*;
+
+#[test]
+fn quality_summary_resolves_retained_execution_names_and_exact_reasoning_profiles() {
+ let directory = tempfile::tempdir().unwrap();
+ let store = open_store(directory.path());
+ let fixture = Fixture::new("quality-summary-metadata");
+ let writer = writer(&store);
+ let channel = fact_channel(&fixture, 256 * 1024);
+ let mut facts = super::super::support::request_facts(
+ &fixture,
+ "attempt-quality",
+ super::super::support::finished("attempt-quality"),
+ 1_000,
+ );
+ if let ExecutionFactV1::CandidateDecision(candidate) = &mut facts[1].fact {
+ candidate.model_configuration_id = MODEL.into();
+ candidate.reasoning_profile_id = Some("high".into());
+ }
+ if let ExecutionFactV1::AttemptStarted {
+ model_configuration_id,
+ ..
+ } = &mut facts[4].fact
+ {
+ *model_configuration_id = MODEL.into();
+ }
+ let request_finished = facts.pop().unwrap().fact;
+ for envelope in facts {
+ assert!(matches!(
+ offer_fact(&writer, &channel, envelope),
+ WriterCycleOutcome::Ack(_)
+ ));
+ }
+ let mut sequence = 9;
+ accept(
+ &writer,
+ &channel,
+ &fixture,
+ &mut sequence,
+ finished(
+ &fixture,
+ "segment-metadata",
+ "agent-turn-1",
+ 1,
+ "smart_saving_complex",
+ 1_000,
+ ),
+ 1_010,
+ );
+ accept(
+ &writer,
+ &channel,
+ &fixture,
+ &mut sequence,
+ request_finished,
+ 1_011,
+ );
+ let result = store
+ .observed_plan_quality_samples(&reader(), &query(Some(17)), 5_000)
+ .unwrap();
+ assert_eq!(result.samples[0].native_model.as_deref(), Some("native-a"));
+ assert_eq!(
+ result.samples[0].reasoning_profile_id.as_deref(),
+ Some("high")
+ );
+ assert_eq!(
+ result.summary.models[0].native_model.as_deref(),
+ Some("native-a")
+ );
+ assert_eq!(
+ result.summary.models[0].reasoning_profile_id.as_deref(),
+ Some("high")
+ );
+}
+
+fn accept(
+ writer: &crate::LocalObservationWriter,
+ channel: &crate::FactChannel,
+ fixture: &Fixture,
+ sequence: &mut u64,
+ fact: ExecutionFactV1,
+ at: u64,
+) {
+ *sequence += 1;
+ assert!(matches!(
+ offer_fact(
+ writer,
+ channel,
+ fixture.fact(*sequence, fact, i64::try_from(at).unwrap())
+ ),
+ WriterCycleOutcome::Ack(_)
+ ));
+}
+
+#[test]
+fn quality_summary_uses_latest_raw_scores_over_full_scope_and_ignores_detail_filters() {
+ let directory = tempfile::tempdir().unwrap();
+ let store = open_store(directory.path());
+ let fixture = Fixture::new("quality-summary-full-scope");
+ let channel = fact_channel(&fixture, 256 * 1024);
+ let writer = writer(&store);
+ let mut sequence = 0;
+ for i in 0..26 {
+ let segment = format!("segment-{i}");
+ let turn = format!("agent-turn-{}", i + 1);
+ let at = 1_000 + i * 20;
+ accept(
+ &writer,
+ &channel,
+ &fixture,
+ &mut sequence,
+ finished(&fixture, &segment, &turn, i + 1, "smart_saving_simple", at),
+ at,
+ );
+ if i < 25 {
+ let score = if i == 0 {
+ 0.0
+ } else if i == 24 {
+ 0.4
+ } else {
+ 0.5
+ };
+ accept(
+ &writer,
+ &channel,
+ &fixture,
+ &mut sequence,
+ assessment(
+ &fixture,
+ &segment,
+ (&turn, &turn, i + 1, i + 1),
+ at + 11,
+ score,
+ ),
+ at + 11,
+ );
+ }
+ }
+ accept(
+ &writer,
+ &channel,
+ &fixture,
+ &mut sequence,
+ assessment(
+ &fixture,
+ "segment-24",
+ ("agent-turn-25", "agent-turn-25", 25, 25),
+ 2_000,
+ 0.8,
+ ),
+ 2_000,
+ );
+
+ let mut q = query(Some(17));
+ q.limit = 20;
+ let first = store
+ .observed_plan_quality_samples(&reader(), &q, 5_000)
+ .unwrap();
+ assert_eq!(first.samples.len(), 20);
+ assert!(first.next_cursor.is_some());
+ assert_eq!(first.summary.scored_stage_count, 25);
+ assert_eq!(first.summary.unrated_stage_count, 1);
+ assert_eq!(first.summary.session_count, 1);
+ assert_eq!(first.summary.available_revisions, [17]);
+ let model = &first.summary.models[0];
+ assert_eq!(model.scored_stage_count, 25);
+ assert_eq!(model.unrated_stage_count, 1);
+ assert!((model.average_score.unwrap() - 0.492).abs() < 1e-12);
+
+ q.cursor = first.next_cursor.clone();
+ let next = store
+ .observed_plan_quality_samples(&reader(), &q, 5_000)
+ .unwrap();
+ assert_eq!(next.samples.len(), 6);
+ assert_eq!(next.summary, first.summary);
+ q.cursor = None;
+ q.score_lt = Some(0.5);
+ q.execution = Some(model.execution.clone());
+ let low = store
+ .observed_plan_quality_samples(&reader(), &q, 5_000)
+ .unwrap();
+ assert_eq!(low.samples.len(), 1);
+ assert_eq!(low.samples[0].assessment.as_ref().unwrap().score, 0.0);
+ assert_eq!(low.summary, first.summary);
+ q.score_lt = None;
+ q.unrated_only = true;
+ let unrated = store
+ .observed_plan_quality_samples(&reader(), &q, 5_000)
+ .unwrap();
+ assert_eq!(unrated.samples.len(), 1);
+ assert!(unrated.samples[0].assessment.is_none());
+ assert_eq!(unrated.summary, first.summary);
+ q.score_lt = Some(0.5);
+ assert!(matches!(
+ store.observed_plan_quality_samples(&reader(), &q, 5_000),
+ Err(crate::query_v2::ObservationV2Error::Invalid)
+ ));
+}
+
+#[test]
+fn quality_summary_separates_actual_groups_profiles_and_revisions_and_exactly_drills_down() {
+ let directory = tempfile::tempdir().unwrap();
+ let store = open_store(directory.path());
+ let fixture = Fixture::new("quality-summary-identities");
+ let channel = fact_channel(&fixture, 256 * 1024);
+ let writer = writer(&store);
+ let mut sequence = 0;
+ for (i, branch, other_profile, revision) in [
+ (1, "smart_saving_simple", false, 17),
+ (2, "smart_saving_complex", false, 17),
+ (3, "smart_saving_complex", true, 17),
+ (4, "smart_saving_complex", true, 18),
+ ] {
+ let segment = format!("segment-{i}");
+ let turn = format!("agent-turn-{i}");
+ let mut fact = finished(&fixture, &segment, &turn, i, branch, 1_000 + i * 20);
+ if let ExecutionFactV1::AgentTurnFinished {
+ profile_digest,
+ plan_revision,
+ selected_branch_id,
+ ..
+ } = &mut fact
+ {
+ *plan_revision = revision;
+ if other_profile {
+ *profile_digest = Some(CanonicalDigest::of_bytes(b"profile-b").into());
+ }
+ // A fallback executes the complex group after selection of simple.
+ if i == 2 {
+ *selected_branch_id = "smart_saving_simple".into();
+ }
+ }
+ accept(
+ &writer,
+ &channel,
+ &fixture,
+ &mut sequence,
+ fact,
+ 1_000 + i * 20,
+ );
+ }
+ let all = store
+ .observed_plan_quality_samples(&reader(), &query(None), 5_000)
+ .unwrap();
+ assert_eq!(all.summary.models.len(), 4);
+ assert_eq!(all.summary.available_revisions, [18, 17]);
+ for model in &all.summary.models {
+ let mut q = query(None);
+ q.execution = Some(model.execution.clone());
+ let page = store
+ .observed_plan_quality_samples(&reader(), &q, 5_000)
+ .unwrap();
+ assert_eq!(page.samples.len(), 1);
+ assert_eq!(page.summary, all.summary);
+ assert_eq!(
+ page.samples[0].profile_digest,
+ model.execution.profile_digest
+ );
+ assert_eq!(page.samples[0].plan_revision, model.execution.plan_revision);
+ assert_eq!(
+ page.samples[0].executed_branch_id,
+ model.execution.executed_branch_id
+ );
+ }
+ let current = store
+ .observed_plan_quality_samples(&reader(), &query(Some(17)), 5_000)
+ .unwrap();
+ assert_eq!(current.summary.models.len(), 3);
+ assert_eq!(current.summary.available_revisions, [18, 17]);
+ let mut narrow = query(Some(17));
+ narrow.from_ms = Some(1_051);
+ let narrow = store
+ .observed_plan_quality_samples(&reader(), &narrow, 5_000)
+ .unwrap();
+ assert_eq!(narrow.summary.models.len(), 1);
+}
+
+#[test]
+fn quality_summary_does_not_attribute_mixed_execution_and_respects_run_authority() {
+ let directory = tempfile::tempdir().unwrap();
+ let store = open_store(directory.path());
+ let fixture = Fixture::new("quality-summary-mixed");
+ let channel = fact_channel(&fixture, 256 * 1024);
+ let writer = writer(&store);
+ let mut sequence = 0;
+ let mut fact = finished(
+ &fixture,
+ "mixed-segment",
+ "agent-turn-1",
+ 1,
+ "smart_saving_simple",
+ 1_000,
+ );
+ if let ExecutionFactV1::AgentTurnFinished {
+ executed_branch_id,
+ model_configuration_id,
+ profile_digest,
+ attribution,
+ ..
+ } = &mut fact
+ {
+ *executed_branch_id = None;
+ *model_configuration_id = None;
+ *profile_digest = None;
+ *attribution = AgentTurnAttributionV1::Mixed;
+ }
+ accept(&writer, &channel, &fixture, &mut sequence, fact, 1_000);
+ let page = store
+ .observed_plan_quality_samples(&reader(), &query(Some(17)), 5_000)
+ .unwrap();
+ assert_eq!(page.summary.models.len(), 1);
+ let mixed = &page.summary.models[0];
+ assert_eq!(mixed.execution.attribution, AgentTurnAttributionV1::Mixed);
+ assert!(mixed.execution.model_configuration_id.is_none());
+ assert!(mixed.average_score.is_none());
+ let scoped_reader = ObservationReaderContext::run_scoped(
+ hiroute_domain::WorkspaceId::default(),
+ "worker".into(),
+ 1,
+ 20_000,
+ ["other-run".into()].into_iter().collect(),
+ false,
+ false,
+ )
+ .unwrap();
+ let hidden = store
+ .observed_plan_quality_samples(&scoped_reader, &query(Some(17)), 5_000)
+ .unwrap();
+ assert!(hidden.samples.is_empty());
+ assert!(hidden.summary.models.is_empty());
+ assert_eq!(hidden.summary.session_count, 0);
+ assert!(hidden.summary.available_revisions.is_empty());
+}
diff --git a/news/2026-10-04-astra-qwen.en.md b/news/2026-10-04-astra-qwen.en.md
index e2c37509..9580abe9 100644
--- a/news/2026-10-04-astra-qwen.en.md
+++ b/news/2026-10-04-astra-qwen.en.md
@@ -118,13 +118,13 @@ This case shows execution feedback influencing model selection throughout the ta
### Inspect each stage in HiRoute
-That execution feedback is also available for inspection. A session's **Model performance** view shows the actual model, competence score and assessment coverage for each execution stage. If progress looks insufficient, execution-evidence and assessment-feedback links lead back to the relevant activity. Stages that have not received a score remain visible too.
+A session's **Model performance** view shows the actual model, reasoning setting, stage competence and execution turns. Open execution evidence to inspect the activity; expand assessment details to check coverage and open the assessment-trigger request. Stages that have not received a score remain visible too.
[![Session model performance: inspect Qwen and Astra stage scores, assessment coverage and evidence links](assets/desktop-session-competence-en.png)](assets/desktop-session-competence-en.png)
-A routing plan's **Runtime performance** view provides a broader perspective. Select a plan version and time range to see each model's average stage competence, together with counts of scored and unrated stages. Open a model's stages to inspect the underlying execution and assess which combination fits your tasks. The average includes scored stages only; unrated stages do not enter the calculation.
+A routing plan's **Model performance** tab provides a broader perspective. Select a plan version and time range to inspect model configurations within each execution group, with average stage competence and counts of scored and unrated stages. A group can contain multiple models; different reasoning settings remain separate, and rows follow the plan's candidate order. Open a model's stages to inspect the underlying execution and assess which combination fits your tasks. Averages use the latest raw assessment of each scored stage in the scope; unrated stages are excluded.
-[![Routing-plan performance: inspect model averages and stage counts within one version and time range](assets/desktop-plan-competence-en.png)](assets/desktop-plan-competence-en.png)
+[![Routing-plan model performance: inspect model configurations, averages and stage counts by execution group](assets/desktop-plan-competence-en.png)](assets/desktop-plan-competence-en.png)
## Enterprise gateway experience, lightweight local execution
diff --git a/news/2026-10-04-astra-qwen.zh-CN.md b/news/2026-10-04-astra-qwen.zh-CN.md
index c71233cd..b30320d7 100644
--- a/news/2026-10-04-astra-qwen.zh-CN.md
+++ b/news/2026-10-04-astra-qwen.zh-CN.md
@@ -118,13 +118,13 @@ Qwen 的资料整理并不差,差距主要出现在最后的判断:两次建
### 回到 HiRoute,看清每个阶段的表现
-这些执行反馈也能在 HiRoute 中回看。会话的**模型表现**按执行阶段展示实际使用的模型、胜任度和评分覆盖范围。发现某一段进展不理想,可以沿“执行证据”和“评分触发反馈”定位相关过程;尚未产生评分的阶段也会保留展示。
+会话的**模型表现**按执行阶段展示实际使用的模型、思考设置、阶段胜任度和执行轮次。点击“执行证据”回看相关过程,在“评分详情”中核对评分覆盖范围并打开评分触发请求;尚未产生评分的阶段也会保留展示。
[![会话中的模型表现:查看 Qwen 与 Astra 各执行阶段的胜任度、覆盖范围和证据入口](assets/desktop-session-competence-zh.png)](assets/desktop-session-competence-zh.png)
-路由计划的**运行表现**则提供更长时间范围的视角:选定计划版本和时间范围,查看各模型的阶段平均胜任度,以及已评分、未评分的阶段数。点击“查看阶段”可以继续核对具体执行,帮助判断怎样的模型组合更适合自己的任务。均值按已评分阶段计算,未评分阶段不计入。
+路由计划的**模型表现**提供更长时间范围的视角:选定计划版本和时间范围,按执行分组查看模型配置的阶段平均胜任度,以及已评分、未评分的阶段数。一个分组可以包含多个模型,同一模型的不同思考设置分别展示,并保持计划中的候选顺序。点击“查看阶段”继续核对具体执行,帮助判断怎样的模型组合更适合自己的任务。均值使用范围内每个已评分阶段的最新原始评分,未评分阶段不计入。
-[![路由计划中的运行表现:在同一版本和时间范围内查看各模型的阶段平均胜任度与样本数](assets/desktop-plan-competence-zh.png)](assets/desktop-plan-competence-zh.png)
+[![路由计划中的模型表现:按执行分组查看模型配置的阶段平均胜任度与样本数](assets/desktop-plan-competence-zh.png)](assets/desktop-plan-competence-zh.png)
## 企业级网关实践,轻量本地实现
diff --git a/news/ILLUSTRATIONS.md b/news/ILLUSTRATIONS.md
index d014882c..bc862cc2 100644
--- a/news/ILLUSTRATIONS.md
+++ b/news/ILLUSTRATIONS.md
@@ -96,23 +96,31 @@ as current product evidence or substitute a new score into the frozen case.
## Competence product views
-Two additional images explain the intended final observation interface:
+The four images are native captures of HiRoute Desktop on macOS:
- Session model performance ([Chinese](assets/desktop-session-competence-zh.png) /
- [English](assets/desktop-session-competence-en.png)): actual models,
- stage scores, assessment coverage and evidence links.
-- Routing-plan performance ([Chinese](assets/desktop-plan-competence-zh.png) /
- [English](assets/desktop-plan-competence-en.png)): model averages
- within one plan version and time range, with scored and unrated stage counts.
-
-These images were generated with the built-in `image_gen` tool. They are product-design
-assets, not native Desktop captures. The per-model aggregate view is planned
-functionality; the images do not establish that its implementation has passed
-acceptance. The article describes the intended final interface. Production notes are
-kept here and in the [complete prompts and data record](competence-product-views.json),
-without overlays on the images.
-
-The values follow the visible execution stages in the retained HTTPX observation:
+ [English](assets/desktop-session-competence-en.png)): actual execution models,
+ reasoning settings, stage scores and evidence actions.
+- Routing-plan model performance ([Chinese](assets/desktop-plan-competence-zh.png) /
+ [English](assets/desktop-plan-competence-en.png)): model configurations grouped
+ by execution branch, scoped averages, and scored/unrated stage counts.
+
+Desktop reads an isolated copy of the unchanged observation records retained from
+this HTTPX case. The images come from the production UI and local daemon, without
+mock IPC, generated interface text or image edits. Both languages were selected
+through the app's settings, with 100% text size in all four captures. The Chinese
+routing list translates plan names and purposes in the isolated replay catalog;
+the English list retains the source names. Plan IDs, revisions, model candidates,
+execution identities and scores are unchanged. The main competence views display
+`gpt-6-astra`; the full CPA transport ID remains in expandable stage details.
+Reasoning settings use the original tier names (`medium` and `xhigh`) in both
+languages.
+Native validation covered stage and assessment-trigger
+navigation, exact model drill-down, unrated filtering, scope changes, refresh on
+re-entry, and preserving draft settings without changing published-plan statistics.
+Capture hashes, binary hashes and the source evidence archive are recorded in
+[the capture provenance](competence-product-views.json).
+
Qwen has one scored stage at 0.485; Astra has one at 0.775 and one unrated stage.
The existing two-decimal display produces 0.48 and 0.78. The earlier 0.410 assessment
lacks an attributed visible execution stage and is excluded from this product-view
@@ -120,14 +128,10 @@ average. A prior 0.615 assessment of the later Qwen stage is not another indepen
stage. No score has been added to the final Astra stage.
Stage scores describe different stretches of work, not a controlled model ranking
-or whole-delivery acceptance. Partial-assessment and historical-gap states remain
-part of the underlying evidence; their notices belong in expanded evidence details
-in this design. Removing them from the primary view does not alter the experiment
-records. Each language edition uses matching localized interface images. The English
-versions translate the navigation, filters, labels, actions and explanatory text while
-preserving the source layout, model identities, numerical values, stage counts and
-unrated state. Their source-image hashes and exact localization prompts are retained
-in the same provenance record.
+or whole-delivery acceptance. Retained diagnostic states and raw identities are
+available in expanded stage details. The captures validate the product's observation
+interface; they do not constitute a new model-quality experiment. Each article uses
+matching localized images and the same original numerical values and stage counts.
## Local architecture
diff --git a/news/assets/desktop-plan-competence-en.png b/news/assets/desktop-plan-competence-en.png
index bf37899a..c724008f 100644
Binary files a/news/assets/desktop-plan-competence-en.png and b/news/assets/desktop-plan-competence-en.png differ
diff --git a/news/assets/desktop-plan-competence-zh.png b/news/assets/desktop-plan-competence-zh.png
index 6386f46c..6a58aac0 100644
Binary files a/news/assets/desktop-plan-competence-zh.png and b/news/assets/desktop-plan-competence-zh.png differ
diff --git a/news/assets/desktop-session-competence-en.png b/news/assets/desktop-session-competence-en.png
index 7c8b4589..80223810 100644
Binary files a/news/assets/desktop-session-competence-en.png and b/news/assets/desktop-session-competence-en.png differ
diff --git a/news/assets/desktop-session-competence-zh.png b/news/assets/desktop-session-competence-zh.png
index 7addfb63..eb77d1b7 100644
Binary files a/news/assets/desktop-session-competence-zh.png and b/news/assets/desktop-session-competence-zh.png differ
diff --git a/news/competence-product-views.json b/news/competence-product-views.json
index 02e51558..c23e7f13 100644
--- a/news/competence-product-views.json
+++ b/news/competence-product-views.json
@@ -1,68 +1,72 @@
{
- "created_at": "2026-10-05",
- "tool": "built-in image_gen",
- "status": "Product-design visuals for the intended final interface; generated images, not native experiment captures.",
- "implementation_status": "The per-model aggregate view is planned product functionality. These images do not establish implementation acceptance.",
- "presentation": "No production-note overlays. The article explains the final product view; this file records provenance.",
- "data_basis": {
- "experiment": "https://github.com/higress-group/HiRoute/releases/download/experiment-evidence-2026-10-04/evidence-2026-10-04.zip",
- "archive_member": "cases/unattended-engineering/results/2026-10-04/evidence.json",
- "scope": "Visible execution stages from the retained historical HTTPX session; stage performance is not whole-delivery acceptance or a same-task model benchmark.",
- "selection": "Use the latest valid assessment per stage with actual execution attribution. Exclude the 0.410 assessment-only record that lacks a visible attributed execution stage.",
- "qwen": {
- "raw_mean_score": 0.485,
- "display_score": "0.48",
- "scored_stages": 1,
- "unrated_stages": 0
- },
- "astra": {
- "raw_mean_score": 0.775,
- "display_score": "0.78",
- "scored_stages": 1,
- "unrated_stages": 1
- },
- "format": "Use raw scores for aggregation and the existing two-decimal display. Unrated stages are not zero and do not enter the mean.",
- "evidence_states": "Original partial-assessment and historical-gap facts remain in evidence details. Hiding primary-view diagnostic notices does not change source records."
+ "schema": "hiroute.competence-native-captures/v1",
+ "capture_date": "2026-10-05",
+ "capture": {
+ "application": "HiRoute Desktop",
+ "platform": "macOS",
+ "build_profile": "Debug",
+ "method": "tauri-pilot screenshot_native (cgwindow)",
+ "image_edits": false,
+ "desktop_binary_sha256": "3da3197ad2794b55db3cb8d6d8c56b6758163c64bee7c2adb41f741542f25159",
+ "daemon_binary_sha256": "6bd42ec8d4fa3fd524f16bf4e03ac31f85b7bdc83fd23d59f9b0cb2dfc0e0f49",
+ "source_commit": "c9268e65180dd3368d43e39dc497418f4ad2060a"
+ },
+ "data_origin": {
+ "case": "Unattended Convergence v2 mixed / HTTPX streaming JSON",
+ "plan_revision": 1,
+ "archive": "https://github.com/higress-group/HiRoute/releases/download/experiment-evidence-2026-10-04/evidence-2026-10-04.zip",
+ "member": "cases/unattended-engineering/results/2026-10-04/evidence.json",
+ "retained_request_count": 40,
+ "stage_identity_and_scores_sha256": "895d97366463e04d47e5a13307d864c203c0881b17494b1638fe613b1232fa09",
+ "display_labels": "Chinese plan names and purposes are translated in the isolated replay catalog. Plan IDs, revision numbers, candidates, execution identities and observation scores are unchanged; the English catalog retains the original names."
},
+ "visible_stage_scores": {
+ "Qwen 3.8 Flash": [
+ 0.485
+ ],
+ "GPT-6 Astra": [
+ 0.775,
+ null
+ ]
+ },
+ "calculation": "Latest raw assessment per attributed execution stage. Unrated stages are excluded from the average; detail filters and pagination do not change the aggregation scope.",
+ "evidence_limit": "Native product replay of unchanged original experiment records, not a new model-quality experiment or a controlled same-task model ranking.",
"images": [
{
- "path": "assets/desktop-session-competence-zh.png",
- "sha256": "50c8c749e5837744ea7b548992c81d5242583ec5d55d777e2d4d70e0602d14be",
- "initial_prompt": "Use case: ui-mockup. Edit the referenced HiRoute session UI illustration to follow a carefully specified implementable product design. Preserve the HiRoute H logo, authentic light-grey macOS desktop-app frame, narrow lavender left navigation, session sidebar, restrained purple controls, rounded outlined cards, flat screen-like style and crisp Chinese typography. Preserve landscape aspect ratio about 1.60, full app viewport, polished but realistic. Change the main content exactly as specified; do not invent data.\nMain sidebar selected item \"会话\", other sidebar items \"首页\", \"模型\", \"智能路由\", \"Agent\", \"设置\". Main header \"会话\". Session sidebar selection \"HTTPX JSON 流式迭代\", subline \"Astra × Qwen\". Main session title \"HTTPX JSON 流式迭代\", with a small outline \"运行记录\" button.\nSection \"模型表现\", subline \"查看各执行阶段的评分、覆盖范围与证据\". Three spacious stacked cards in THIS order:\n1. Model title \"GPT-6 Astra\"; subline \"主力分支 · 执行轮次 5\"; large score state at right \"未评分\" with caption \"胜任度\". Body line \"尚未产生该阶段的评分\". A discreet expandable text row \"证据状态:历史记录有缺口\". Bottom-right text action \"执行证据\". This card must remain visible and must not imply a zero or a final quality score.\n2. Model title \"Qwen-3.8 Flash\"; subline \"省钱分支 · 执行轮次 4\"; large score at right \"0.48 / 1\" with \"胜任度\". Body line \"评分覆盖轮次 3–4 · 原始评分 0.485\". Small amber-dot muted evidence line \"评分基于部分证据 · 历史记录有缺口\". Bottom-right two text actions \"执行证据\" and \"评分触发反馈\".\n3. Model title \"GPT-6 Astra\"; subline \"主力分支 · 执行轮次 2\"; large score at right \"0.78 / 1\" with \"胜任度\". Body line \"评分覆盖轮次 2 · 原始评分 0.775\". Small amber-dot muted evidence line \"评分基于部分证据 · 历史记录有缺口\". Same two text actions.\nDo not repeat large warning blocks: use the compact evidence-status lines described. Show clear aligned columns and ample whitespace. Beneath cards an understated info strip: \"评分对应各自阶段,不代表整单验收结果;未评分阶段保留展示。\"\nA small but comfortably readable permanent bottom caption outside the score cards reads exactly \"界面效果示意 · 非原始实验截图\". Do NOT use the phrase \"设计示意\" anywhere.\nREMOVE old invented task-role labels such as 初始研究阶段 or 上下文摘要阶段, all 0.5 threshold references, \"触发升级\" badges, the Qwen 0.41 card, and the model-relay timeline. These are deliberately not part of this general product UI. Do not show any 0.45 average, no fabricated timestamps, no internal IDs, no complete-evidence badge, no cost/success claims, no invented conversations. Keep the authentic HiRoute look. Opaque light background.",
- "edit_prompts": [
- "Use case: precise-object-edit. Make a narrow text-removal edit to this HiRoute session interface illustration. Preserve the complete app layout, logos, names, score values 0.48 / 1 and 0.78 / 1, the top Astra 未评分 state, execution/assessment ranges, all evidence buttons, sidebar selection, colors, Chinese typography, footer illustration disclosure, aspect ratio and everything unrelated.\nRemove every amber or grey evidence-status information strip inside the three stage cards. Specifically remove the entire top strip \"证据状态:历史记录有缺口\", and remove both strips \"评分基于部分证据 · 历史记录有缺口\", including their dots, disclosure triangle and tinted backgrounds. Do not replace them with any alternative warning, badge, status text, or completeness claim. Those diagnostics belong inside a future expanded evidence view, not this primary screen.\nKeep the top card sentence \"尚未产生该阶段的评分\". Keep each scored card's \"评分覆盖轮次 ...\" and raw score unchanged. Retain \"执行证据\" and \"评分触发反馈\" buttons, aligned neatly at the lower right of their cards. Rebalance only the inner whitespace as needed for a clean, calm product UI. Keep the bottom info line \"评分对应各自阶段,不代表整单验收结果;未评分阶段保留展示。\" and the visible caption \"界面效果示意 · 非原始实验截图\".\nNo words 部分证据, 历史记录有缺口, 证据不完整, or 设计示意 anywhere. Do not invent positive evidence-completeness labels. No change to any measured number, model identity, or unrated state. Opaque background.",
- "Use case: precise-object-edit. Edit this exact HiRoute session UI image with ONE narrow correction only. In the TOP model card, which is GPT-6 Astra, 执行轮次 5 and 未评分, remove the text action 评分触发反馈 and the vertical separator before it. That unrated card has no assessment feedback. Keep only the 执行证据 action, neatly aligned at the lower right of that top card. In the SECOND Qwen card and THIRD scored Astra card, keep BOTH 执行证据 and 评分触发反馈 actions exactly as they are. Preserve every other pixel as closely as possible: complete layout, score numbers 0.48 and 0.78, raw 0.485 and 0.775, all Chinese wording, card positions, sidebar, fonts, colors, original dimensions, opaque background and the footer 界面效果示意 · 非原始实验截图. Do not reintroduce any partial-evidence or history-gap labels. Do not add, remove or change anything else.",
- "Use case: precise-object-edit. Edit this exact HiRoute Desktop session interface image with ONE change only. Remove the entire bottommost purple outlined caption box containing the lock icon and the exact words 界面效果示意 · 非原始实验截图. Remove its text, icon, border, and tint, and fill that area seamlessly with the surrounding app background. Preserve the complete app frame and canvas dimensions; do not crop. Keep the info strip immediately ABOVE it about stage scores and whole-delivery acceptance unchanged. Preserve all other text, all model names and numbers, all card positions, sidebar, typography, colors, and buttons. In particular the top 未评分 Astra card must retain ONLY its 执行证据 action, with no 评分触发反馈 action; the two scored cards retain both actions. Do not add any replacement caption, disclaimer, watermark, status badge, or footer. No changes to numerical values 0.48, 0.78, 0.485, 0.775, or the 未评分 state. This is a clean product-design asset for the intended final interface; its provenance is stored separately. Opaque light background."
- ]
+ "file": "assets/desktop-session-competence-zh.png",
+ "view": "session",
+ "language": "zh",
+ "width": 1920,
+ "height": 1080,
+ "text_scale_percent": 100,
+ "sha256": "77acce3ed9554a566e88bb95492e6446e6a28e765ba83edaa71d323cf8187276"
},
{
- "path": "assets/desktop-plan-competence-zh.png",
- "sha256": "8dd492e844b3376709ce02276d84c581ad9580afdccd7a87c1c982a92aedb68c",
- "initial_prompt": "Use case: ui-mockup. Edit the referenced HiRoute smart routing UI illustration to match an implementable general-purpose observation design. Preserve the HiRoute H logo, original light-grey/white macOS app frame, left navigation highlighting \"智能路由\", inner route list, typography, fine outlined rounded cards and restrained lavender accents. Flat realistic desktop screen, landscape aspect ratio around 1.60, polished, readable Chinese. No poster effects.\nRoute list selected title \"Astra × Qwen · Smart Routing\". Right pane title same, small \"已启用\" badge, muted strategy \"智能省钱\". In this viewport focus on the route's \"运行表现\" section; remove the previous bulky three-card route-configuration strip so the observation view is spacious. The main header is \"运行表现\", subtitle \"查看所选版本与时间范围内的实际执行阶段\". Two scope dropdowns at top: \"当前生效版本 r1\" and \"最近 7 天\".\nThen two equal-size neutral-style model summary cards side by side, not a ranking:\nLeft card model \"Qwen-3.8 Flash\", subline \"省钱组合\". Metric label \"阶段平均胜任度\". Very large exact value \"0.48 / 1\". Below exactly \"已评分 1 段 · 未评分 0 段\". A small amber-dot muted line \"其中部分证据评分:1 段\". Bottom text action \"查看阶段 →\".\nRight card model \"GPT-6 Astra\", subline \"主力组合\". Metric label \"阶段平均胜任度\". Very large exact value \"0.78 / 1\". Below \"已评分 1 段 · 未评分 1 段\". Same evidence line \"其中部分证据评分:1 段\". Bottom text action \"查看阶段 →\".\nUse matching neutral/purple accent styles for both cards. NO progress bars, threshold guide, traffic-light capability verdict, leaderboard, trophy, percent sign or 0.50 line. Remove the old 0.45 value.\nBelow cards a single subdued info line \"均值按已评分阶段计算;未评分不计入,部分证据数量单独展示。\"\nBelow is section \"阶段明细\" with a dropdown \"全部评分\" visibly belonging only to this detail section. Three compact outlined rows:\n\"GPT-6 Astra\" | \"主力分支 · 执行轮次 5\" | \"未评分\" | text action \"执行证据\"\n\"Qwen-3.8 Flash\" | \"省钱分支 · 执行轮次 4\" | \"0.48 / 1\" | small \"部分证据\" badge | \"查看证据\"\n\"GPT-6 Astra\" | \"主力分支 · 执行轮次 2\" | \"0.78 / 1\" | small \"部分证据\" badge | \"查看证据\".\nRows should be visible, not clipped. A small explanatory line: \"反映该路由中的阶段表现,不代表模型综合能力或任务成功率。\"\nAt the bottom a modest, legible caption reads exactly \"界面效果示意 · 非原始实验截图\". Do NOT include the phrase \"设计示意\" anywhere, and remove its old pill.\nDo not invent samples or task-role labels, raw scores, plot trends, extra switches, timestamps or costs. The exact means come from one eligible Qwen score 0.485 formatted with the current app rule as 0.48, and one eligible Astra score 0.775 formatted as 0.78; an additional Astra stage is unrated. This is a labeled UI illustration for a not-yet-implemented read-only aggregate, not a real app screenshot. Preserve authentic HiRoute style, opaque light background.",
- "edit_prompts": [
- "Use case: precise-object-edit. Make a narrow edit to this HiRoute routing-plan interface illustration. Preserve layout, HiRoute logo, model names, 0.48 / 1 and 0.78 / 1 averages, score/sample counts, model cards, dropdowns, detail rows, action buttons, typography and sidebar.\nRemove the two amber-dot lines \"其中部分证据评分:1 段\" entirely from the model cards; do not put replacement text or badges there. Keep \"已评分 1 段 · 未评分 0 段\" on Qwen and \"已评分 1 段 · 未评分 1 段\" on Astra. Keep 查看阶段 actions and rebalance card whitespace neatly.\nIn the explanatory strip below the cards replace the entire sentence with exactly \"均值按已评分阶段计算,未评分阶段不计入。\"\nIn the lower 阶段明细 rows remove both amber \"部分证据\" badges, keeping all model names, execution ranges, numerical scores, the 未评分 state, and 执行证据/查看证据 actions unchanged. No alternative warning or completeness badge.\nPreserve the bottom explanatory sentence about not representing general model capability or task success rate, and preserve the footer \"界面效果示意 · 非原始实验截图\". No words 部分证据, 历史记录有缺口, 证据不完整, or 设计示意 anywhere in the image. Do not fabricate complete-evidence status. These details will remain accessible only when a user opens evidence in the future product implementation. No other changes; authentic flat HiRoute Desktop style, opaque background.",
- "Use case: precise-object-edit. Edit this exact HiRoute routing-plan interface image with ONE change only. Remove the small bottom-right caption 界面效果示意 · 非原始实验截图, replacing the text pixels seamlessly with the surrounding app background. Preserve the entire app frame and original canvas dimensions; do not crop. Keep the bottom-left info sentence about stage performance and general capability unchanged. Preserve all UI panels, logos, model names, scores 0.48 / 1 and 0.78 / 1, sample counts, unrated state, dropdowns, evidence links, fonts, spacing and colors exactly as closely as possible. Do not add any replacement caption, watermark, disclaimer or status. This is a clean product-design asset for the intended final interface; provenance is stored separately. Opaque light background."
- ]
+ "file": "assets/desktop-session-competence-en.png",
+ "view": "session",
+ "language": "en",
+ "width": 1920,
+ "height": 1080,
+ "text_scale_percent": 100,
+ "sha256": "d7c28392e8739d5c798a929be70766501cf14569583547170d7f2d9b1753c433"
},
{
- "path": "assets/desktop-session-competence-en.png",
- "language": "en",
- "sha256": "a91f87d8a76092ddb8ee685fe581a3b2331ed2e8ab6bcc6188734d2a5192a2db",
- "source_image": "assets/desktop-session-competence-zh.png",
- "source_sha256": "50c8c749e5837744ea7b548992c81d5242583ec5d55d777e2d4d70e0602d14be",
- "edit_prompt": "Use case: text-localization.\nAsset type: English-language version of an existing HiRoute Desktop product UI illustration.\nInput image: the single referenced image is the edit target.\nPrimary request: Translate ALL Chinese UI text to English, preserving the existing full desktop frame, layout, sidebar widths, panels, icons, lavender accents, light background, typography hierarchy and data. This is localization of the same product-design asset, not a new scene. Preserve the original approximately 1.6:1 landscape ratio and complete app frame; use crisp high-resolution readable text. Never crop. Reflow or slightly reduce text size only where English needs room, without overlapping or truncating labels.\n\nExact English UI text to use:\nTop window title: \"Sessions\".\nLeft navigation: \"Home\", \"Models\", \"Smart Routing\", \"Agent\", \"Sessions\", \"Settings\". Keep the HiRoute logo and \"AGENT ROUTING\".\nBottom-left status: \"Service running\" and \"Runs locally\".\nSession search: \"Search conversations\".\nTabs: \"All\" and \"Request fallbacks\".\nSelected session title and large main title: \"HTTPX JSON Streaming\". Session subline remains \"Astra × Qwen\".\nUpper-right button: \"Run History\".\nMain section: \"Model Performance\".\nSection subtitle: \"Stage scores, coverage and evidence\".\n\nTop card:\n\"GPT-6 Astra\"\n\"Primary branch · Execution turn 5\"\nLarge right status \"Unrated\", with smaller label \"Competence\".\nBody: \"No assessment for this stage yet.\"\nBottom-right action: ONLY \"Execution evidence\". Do not add an assessment-feedback action to this unrated card.\n\nMiddle card:\n\"Qwen-3.8 Flash\"\n\"Economy branch · Execution turn 4\"\nLarge exact score \"0.48 / 1\", smaller label \"Competence\".\nBody: \"Assessment turns 3–4 · Raw score 0.485\".\nBottom-right actions: \"Execution evidence\" | \"Assessment feedback\".\n\nBottom card:\n\"GPT-6 Astra\"\n\"Primary branch · Execution turn 2\"\nLarge exact score \"0.78 / 1\", smaller label \"Competence\".\nBody: \"Assessment turn 2 · Raw score 0.775\".\nBottom-right actions: \"Execution evidence\" | \"Assessment feedback\".\n\nBottom information strip: \"Scores describe individual stages, not whole-task acceptance. Unrated stages remain visible.\"\nConstraints: Preserve every numerical value, the three cards and their order, the top unrated state, the raw scores 0.485 and 0.775, execution/assessment turn attribution, all existing icons and controls. Use no Chinese characters anywhere. Do not add data, scores, charts, thresholds, evidence-completeness badges, footers, watermarks or captions. Preserve the clean full-frame UI design and all unrelated artwork. Opaque background.",
- "tool": "built-in image_gen",
- "change": "English text localization; original scores, stage counts and unrated state retained."
+ "file": "assets/desktop-plan-competence-zh.png",
+ "view": "plan",
+ "language": "zh",
+ "width": 1920,
+ "height": 1080,
+ "text_scale_percent": 100,
+ "sha256": "99bb1d202de501563ffe9e877b038fd83c2da9fc1ea4736b88590bc4c63474bf"
},
{
- "path": "assets/desktop-plan-competence-en.png",
+ "file": "assets/desktop-plan-competence-en.png",
+ "view": "plan",
"language": "en",
- "sha256": "db20535fe5c7b96d11ac9aa60b98a8eb6b814b1f8018cea2a6a326ba805a135d",
- "source_image": "assets/desktop-plan-competence-zh.png",
- "source_sha256": "8dd492e844b3376709ce02276d84c581ad9580afdccd7a87c1c982a92aedb68c",
- "edit_prompt": "Use case: text-localization.\nAsset type: English-language version of an existing HiRoute Desktop routing-plan performance UI illustration.\nInput image: the single referenced image is the edit target.\nPrimary request: Translate ALL Chinese UI text to English. Preserve the authentic full app frame, approximately 1.6:1 landscape ratio, sidebar and route-list columns, two equal metric cards, three detail rows, icons, logos, subdued grey/lavender palette, hierarchy and exact values. Keep a flat polished desktop UI with crisp legible English. Never crop. Allow minor text reflow or font-size adjustments to fit English labels without overlap or truncation.\n\nExact English UI text:\nTop window title and page heading: \"Smart Routing\".\nPage subtitle: \"Create reusable routing plans for your agents.\"\nTop-right button: \"+ New Route\".\nNavigation: \"Home\", \"Models\", \"Smart Routing\", \"Agent\", \"Sessions\", \"Settings\". Preserve \"HiRoute\" and \"AGENT ROUTING\".\nRoute list and main route title: \"Astra × Qwen · Smart Routing\".\nRoute list subline: \"Smart Saving · Research and implementation\".\nMain route strategy: \"Smart Saving\".\nGreen status badge: \"Enabled\".\nHeader buttons: \"Save Draft\", \"Publish Changes\".\nSection: \"Runtime Performance\".\nSubtitle: \"Execution stages for the selected version and time range.\"\nScope dropdowns: \"Active version r1\" and \"Last 7 days\".\n\nLeft metric card:\n\"Qwen-3.8 Flash\"\n\"Economy group\"\n\"Average stage competence\"\nLarge exact value: \"0.48 / 1\"\nCount: \"1 scored stage · 0 unrated\"\nAction: \"View stages →\".\n\nRight metric card:\n\"GPT-6 Astra\"\n\"Primary group\"\n\"Average stage competence\"\nLarge exact value: \"0.78 / 1\"\nCount: \"1 scored stage · 1 unrated\"\nAction: \"View stages →\".\n\nInformation strip below cards: \"Averages include scored stages only. Unrated stages are excluded.\"\nDetail section heading: \"Stage Details\".\nDetail-only dropdown: \"All scores\".\nThree rows in this exact order:\n\"GPT-6 Astra\" | \"Primary branch · Turn 5\" | \"Unrated\" | \"Execution evidence\"\n\"Qwen-3.8 Flash\" | \"Economy branch · Turn 4\" | \"0.48 / 1\" | \"View evidence\"\n\"GPT-6 Astra\" | \"Primary branch · Turn 2\" | \"0.78 / 1\" | \"View evidence\".\nBottom note: \"Stage performance in this route is not overall model capability or task success rate.\"\n\nConstraints: Do not change any score, sample count, model identity, version r1, 7-day filter, turn number or unrated state. Keep both model cards equally neutral rather than a leaderboard. Preserve original product/provider logos. No Chinese characters anywhere. Add no metrics, thresholds, charts, warning badges, completeness claims, watermarks or production-note captions. Preserve the full app frame and all unrelated artwork. Opaque light background.",
- "tool": "built-in image_gen",
- "change": "English text localization; original scores, stage counts and unrated state retained."
+ "width": 1920,
+ "height": 1080,
+ "text_scale_percent": 100,
+ "sha256": "43a09d5fa12c959ea794f0dbe049bcadabe72a2f967146039d6677d162bcd100"
}
]
}