Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
import { describe, it, expect, vi, beforeEach } from 'vitest';
// Mock fs/promises and fs before importing the module
vi.mock('fs/promises', () => ({
readFile: vi.fn(),
mkdir: vi.fn()
}));
vi.mock('fs', () => ({
existsSync: vi.fn(() => true)
}));
// Mock cos.js to avoid circular dependency
vi.mock('./cos.js', () => ({}));
vi.mock('./cosEvents.js', () => ({
cosEvents: { on: vi.fn(), emit: vi.fn() },
emitLog: vi.fn()
}));
// Mock fileUtils.js to use our mocked fs/promises
vi.mock('../lib/fileUtils.js', async (importOriginal) => {
const fsPromises = await import('fs/promises');
const fs = await import('fs');
return {
ensureDir: vi.fn(),
atomicWrite: vi.fn().mockResolvedValue(undefined),
readJSONFile: vi.fn(async (filePath, defaultValue) => {
if (!fs.existsSync(filePath)) return defaultValue;
const content = await fsPromises.readFile(filePath, 'utf-8');
if (!content || !content.trim()) return defaultValue;
return JSON.parse(content);
}),
PATHS: { cos: '/tmp/test/cos' }
};
});
import { readFile } from 'fs/promises';
import { atomicWrite } from '../lib/fileUtils.js';
import { SKIP_LEARNING_VERDICT } from '../lib/learningVerdict.js';
import { resetTaskTypeLearning, getSkippedTaskTypes, recordTaskCompletion, getRoutingAccuracy, suggestModelTier, recalculateModelTierMetrics, clearLearningCache, getTaskTypeConfidence, getConfidenceLevels, dismissRecommendation, restoreRecommendation, getDismissedRecommendations, clearDismissedRecommendations, getLearningInsights } from './taskLearning.js';
const makeLearningData = (overrides = {}) => ({
tryReadFile: vi.fn().mockResolvedValue(null),
version: 1,
lastUpdated: '2026-01-26T00:00:00.000Z',
byTaskType: {
'self-improve:ui': {
completed: 200,
succeeded: 10,
failed: 190,
totalDurationMs: 2000000,
avgDurationMs: 10000,
lastCompleted: '2026-01-25T00:00:00.000Z',
successRate: 5
},
'user-task': {
completed: 40,
succeeded: 30,
failed: 10,
totalDurationMs: 4000000,
avgDurationMs: 100000,
lastCompleted: '2026-01-26T00:00:00.000Z',
successRate: 75
}
},
byModelTier: {
'user-specified': {
completed: 240,
succeeded: 40,
failed: 200,
totalDurationMs: 6000000,
avgDurationMs: 25000
}
},
errorPatterns: {
'server-error': {
count: 190,
taskTypes: { 'self-improve:ui': 185, 'user-task': 5 },
lastOccurred: '2026-01-25T00:00:00.000Z'
},
'unknown': {
count: 10,
taskTypes: { 'self-improve:ui': 10 },
lastOccurred: '2026-01-24T00:00:00.000Z'
}
},
totals: {
completed: 240,
succeeded: 40,
failed: 200,
totalDurationMs: 6000000,
avgDurationMs: 25000
},
...overrides
});
// A correlation-quality window proving the enriched failure signal predicts
// outcomes (perfect correlation, >= MIN_CORRELATION_SAMPLES) — so `suggestModelTier`
// runs the AGGRESSIVE failure-sample bar (3) and the #2329 avoidance mechanics
// engage. Without this the #2344 gate holds avoidance back to the conservative
// bar (5) until the signal has earned it. Injected into the #2329 avoidance
// tests to isolate their intent from the new gate (which has its own tests).
const provenCorrelationWindow = () => Array.from({ length: 24 }, (_, i) => ({
taskType: 'user-task', tier: 'heavy',
predictedRisk: i % 2 === 0,
bad: i % 2 === 0,
recordedAt: '2026-01-26T00:00:00.000Z'
}));
describe('TaskLearning - resetTaskTypeLearning', () => {
let savedData;
beforeEach(() => {
vi.clearAllMocks();
clearLearningCache();
savedData = null;
atomicWrite.mockImplementation(async (_path, data) => {
savedData = data;
});
});
it('should return not-found when task type does not exist', async () => {
readFile.mockResolvedValue(JSON.stringify(makeLearningData()));
const result = await resetTaskTypeLearning('nonexistent-type');
expect(result.reset).toBe(false);
expect(result.reason).toBe('task-type-not-found');
});
it('should remove the task type from byTaskType', async () => {
readFile.mockResolvedValue(JSON.stringify(makeLearningData()));
const result = await resetTaskTypeLearning('self-improve:ui');
expect(result.reset).toBe(true);
expect(result.taskType).toBe('self-improve:ui');
expect(savedData.byTaskType['self-improve:ui']).toBeUndefined();
expect(savedData.byTaskType['user-task']).toBeDefined();
});
it('should subtract task type metrics from totals', async () => {
readFile.mockResolvedValue(JSON.stringify(makeLearningData()));
await resetTaskTypeLearning('self-improve:ui');
// Original totals: completed=240, succeeded=40, failed=200, totalDurationMs=6000000
// self-improve:ui: completed=200, succeeded=10, failed=190, totalDurationMs=2000000
// After reset: completed=40, succeeded=30, failed=10, totalDurationMs=4000000
expect(savedData.totals.completed).toBe(40);
expect(savedData.totals.succeeded).toBe(30);
expect(savedData.totals.failed).toBe(10);
expect(savedData.totals.totalDurationMs).toBe(4000000);
expect(savedData.totals.avgDurationMs).toBe(100000); // 4000000 / 40
});
it('should clean up error patterns referencing the task type', async () => {
readFile.mockResolvedValue(JSON.stringify(makeLearningData()));
await resetTaskTypeLearning('self-improve:ui');
// server-error had 190 total (185 from ui, 5 from user-task) → should now have 5
expect(savedData.errorPatterns['server-error'].count).toBe(5);
expect(savedData.errorPatterns['server-error'].taskTypes['self-improve:ui']).toBeUndefined();
expect(savedData.errorPatterns['server-error'].taskTypes['user-task']).toBe(5);
// unknown had 10 total, all from ui → should be removed entirely
expect(savedData.errorPatterns['unknown']).toBeUndefined();
});
it('should return previous metrics in result', async () => {
readFile.mockResolvedValue(JSON.stringify(makeLearningData()));
const result = await resetTaskTypeLearning('self-improve:ui');
expect(result.previousMetrics).toEqual({
completed: 200,
succeeded: 10,
failed: 190,
successRate: 5
});
});
it('should handle totals going to zero gracefully', async () => {
const data = makeLearningData({
byTaskType: {
'self-improve:ui': {
completed: 100, succeeded: 5, failed: 95,
totalDurationMs: 500000, avgDurationMs: 5000,
lastCompleted: '2026-01-25T00:00:00.000Z', successRate: 5
}
},
errorPatterns: {},
totals: {
completed: 100, succeeded: 5, failed: 95,
totalDurationMs: 500000, avgDurationMs: 5000
}
});
readFile.mockResolvedValue(JSON.stringify(data));
await resetTaskTypeLearning('self-improve:ui');
expect(savedData.totals.completed).toBe(0);
expect(savedData.totals.avgDurationMs).toBe(0);
});
it('clamps successDurationMs to >= 0 when a drifted store would go negative', async () => {
// Hand-edited / older store where the running total drifted below a single
// task type's accumulated successDurationMs. Subtracting unclamped would go
// negative and feed calculateDurationETA a negative avgDurationMs.
const data = makeLearningData({
byTaskType: {
'self-improve:ui': {
completed: 10, succeeded: 8, failed: 2,
totalDurationMs: 50000, avgDurationMs: 5000,
successDurationMs: 500000, successMaxDurationMs: 80000,
lastCompleted: '2026-01-25T00:00:00.000Z', successRate: 80
}
},
errorPatterns: {},
totals: {
completed: 10, succeeded: 8, failed: 2,
totalDurationMs: 50000, avgDurationMs: 5000,
successDurationMs: 100000, successMaxDurationMs: 80000
}
});
readFile.mockResolvedValue(JSON.stringify(data));
await resetTaskTypeLearning('self-improve:ui');
// 100000 - 500000 would be -400000; the clamp pins it at 0.
expect(savedData.totals.successDurationMs).toBe(0);
});
});
describe('TaskLearning - getSkippedTaskTypes', () => {
beforeEach(() => {
vi.clearAllMocks();
clearLearningCache();
});
it('should return task types with <30% success and 5+ attempts', async () => {
readFile.mockResolvedValue(JSON.stringify(makeLearningData()));
const skipped = await getSkippedTaskTypes();
expect(skipped).toHaveLength(1);
expect(skipped[0].taskType).toBe('self-improve:ui');
expect(skipped[0].successRate).toBe(5);
});
it('should not include task types with >= 30% success', async () => {
readFile.mockResolvedValue(JSON.stringify(makeLearningData()));
const skipped = await getSkippedTaskTypes();
const userTask = skipped.find(s => s.taskType === 'user-task');
expect(userTask).toBeUndefined();
});
it('should return empty after resetting a skipped type', async () => {
const data = makeLearningData({
byTaskType: {
'self-improve:ui': {
completed: 100, succeeded: 5, failed: 95,
totalDurationMs: 500000, avgDurationMs: 5000,
lastCompleted: '2026-01-25T00:00:00.000Z', successRate: 5
}
},
errorPatterns: {},
totals: {
completed: 100, succeeded: 5, failed: 95,
totalDurationMs: 500000, avgDurationMs: 5000
}
});
// Track what was written so subsequent reads return updated data
let currentData = JSON.stringify(data);
readFile.mockImplementation(async () => currentData);
atomicWrite.mockImplementation(async (_path, data) => {
currentData = JSON.stringify(data);
});
await resetTaskTypeLearning('self-improve:ui');
const skipped = await getSkippedTaskTypes();
expect(skipped).toHaveLength(0);
});
it('should clean up routingAccuracy data when resetting a task type', async () => {
let savedData;
atomicWrite.mockImplementation(async (_path, data) => {
savedData = data;
});
const data = makeLearningData({
routingAccuracy: {
'self-improve:ui': {
heavy: { succeeded: 2, failed: 8, lastAttempt: '2026-01-25T00:00:00.000Z' },
medium: { succeeded: 0, failed: 5, lastAttempt: '2026-01-24T00:00:00.000Z' }
},
'user-task': {
heavy: { succeeded: 10, failed: 2, lastAttempt: '2026-01-26T00:00:00.000Z' }
}
}
});
readFile.mockResolvedValue(JSON.stringify(data));
await resetTaskTypeLearning('self-improve:ui');
expect(savedData.routingAccuracy['self-improve:ui']).toBeUndefined();
expect(savedData.routingAccuracy['user-task']).toBeDefined();
});
it('should subtract from byModelTier when resetting a task type with routing data', async () => {
let savedData;
atomicWrite.mockImplementation(async (_path, data) => {
savedData = data;
});
const data = makeLearningData({
byModelTier: {
heavy: {
completed: 50,
succeeded: 20,
failed: 30,
totalDurationMs: 500000,
avgDurationMs: 10000
},
medium: {
completed: 10,
succeeded: 2,
failed: 8,
totalDurationMs: 100000,
avgDurationMs: 10000
}
},
routingAccuracy: {
'self-improve:ui': {
heavy: { succeeded: 5, failed: 15, lastAttempt: '2026-01-25T00:00:00.000Z' },
medium: { succeeded: 0, failed: 5, lastAttempt: '2026-01-24T00:00:00.000Z' }
},
'user-task': {
heavy: { succeeded: 10, failed: 2, lastAttempt: '2026-01-26T00:00:00.000Z' }
}
}
});
readFile.mockResolvedValue(JSON.stringify(data));
await resetTaskTypeLearning('self-improve:ui');
// heavy: was 50 completed (20 succeeded, 30 failed), subtract 20 (5+15) from self-improve:ui
expect(savedData.byModelTier.heavy.completed).toBe(30);
expect(savedData.byModelTier.heavy.succeeded).toBe(15);
expect(savedData.byModelTier.heavy.failed).toBe(15);
// medium: was 10 completed, subtract 5 (0+5) from self-improve:ui
expect(savedData.byModelTier.medium.completed).toBe(5);
expect(savedData.byModelTier.medium.succeeded).toBe(2);
expect(savedData.byModelTier.medium.failed).toBe(3);
// user-task routing should be untouched
expect(savedData.routingAccuracy['user-task']).toBeDefined();
});
it('should delete byModelTier entry when count reaches zero', async () => {
let savedData;
atomicWrite.mockImplementation(async (_path, data) => {
savedData = data;
});
const data = makeLearningData({
byModelTier: {
'user-specified': {
completed: 240,
succeeded: 40,
failed: 200,
totalDurationMs: 6000000,
avgDurationMs: 25000
},
medium: {
completed: 5,
succeeded: 0,
failed: 5,
totalDurationMs: 50000,
avgDurationMs: 10000
}
},
routingAccuracy: {
'self-improve:ui': {
medium: { succeeded: 0, failed: 5, lastAttempt: '2026-01-24T00:00:00.000Z' }
}
}
});
readFile.mockResolvedValue(JSON.stringify(data));
await resetTaskTypeLearning('self-improve:ui');
// medium tier had 5 completed, all from self-improve:ui → should be deleted
expect(savedData.byModelTier.medium).toBeUndefined();
// user-specified should be untouched (no routing data for it)
expect(savedData.byModelTier['user-specified']).toBeDefined();
});
});
describe('TaskLearning - recordTaskCompletion routing accuracy', () => {
let savedData;
beforeEach(() => {
vi.clearAllMocks();
clearLearningCache();
savedData = null;
atomicWrite.mockImplementation(async (_path, data) => {
savedData = data;
});
});
it('should record routing accuracy for successful task', async () => {
readFile.mockResolvedValue(JSON.stringify(makeLearningData()));
const agent = {
metadata: { modelTier: 'heavy', taskDescription: 'Fix some UI bugs' },
result: { success: true, duration: 60000 }
};
const task = { description: 'Fix some UI bugs', taskType: 'user', metadata: {} };
await recordTaskCompletion(agent, task);
expect(savedData.routingAccuracy).toBeDefined();
expect(savedData.routingAccuracy['user-task']).toBeDefined();
expect(savedData.routingAccuracy['user-task']['heavy']).toBeDefined();
expect(savedData.routingAccuracy['user-task']['heavy'].succeeded).toBe(1);
expect(savedData.routingAccuracy['user-task']['heavy'].failed).toBe(0);
});
it('should record routing accuracy for failed task', async () => {
readFile.mockResolvedValue(JSON.stringify(makeLearningData()));
const agent = {
metadata: { modelTier: 'light', taskDescription: 'Fix UI' },
result: { success: false, duration: 30000 }
};
const task = { description: 'Fix UI', taskType: 'user', metadata: {} };
await recordTaskCompletion(agent, task);
expect(savedData.routingAccuracy['user-task']['light'].succeeded).toBe(0);
expect(savedData.routingAccuracy['user-task']['light'].failed).toBe(1);
expect(savedData.routingAccuracy['user-task']['light'].lastAttempt).toBeDefined();
});
it('should accumulate routing accuracy across multiple completions', async () => {
let currentData = JSON.stringify(makeLearningData());
readFile.mockImplementation(async () => currentData);
atomicWrite.mockImplementation(async (_path, data) => {
currentData = JSON.stringify(data);
savedData = data;
});
const makeAgent = (tier, success) => ({
metadata: { modelTier: tier, taskDescription: 'Test task' },
result: { success, duration: 30000 }
});
const task = { description: 'Test task', taskType: 'user', metadata: {} };
await recordTaskCompletion(makeAgent('medium', true), task);
await recordTaskCompletion(makeAgent('medium', true), task);
await recordTaskCompletion(makeAgent('medium', false), task);
expect(savedData.routingAccuracy['user-task']['medium'].succeeded).toBe(2);
expect(savedData.routingAccuracy['user-task']['medium'].failed).toBe(1);
});
it('counts a clean exit that MISSED its declared criterion as a tier failure (#2344)', async () => {
readFile.mockResolvedValue(JSON.stringify(makeLearningData()));
// Runner exit-code success, but the declared success criterion was not met
// (e.g. agent finished clean yet committed nothing). The learning aggregates
// must treat the validation verdict as authoritative — not the exit code.
const agent = {
metadata: { modelTier: 'heavy', taskDescription: 'Ship the fix' },
result: { success: true, validationPassed: false, duration: 60000 }
};
const task = { description: 'Ship the fix', taskType: 'internal', metadata: {} };
await recordTaskCompletion(agent, task);
const routing = savedData.routingAccuracy['internal-task']['heavy'];
expect(routing.succeeded).toBe(0);
expect(routing.failed).toBe(1);
expect(savedData.byModelTier['heavy'].succeeded).toBe(0);
expect(savedData.byModelTier['heavy'].failed).toBe(1);
expect(savedData.byTaskType['internal-task'].succeeded).toBe(0);
expect(savedData.byTaskType['internal-task'].failed).toBe(1);
});
it('counts a non-zero exit that MET its declared criterion as a tier success (#2344)', async () => {
readFile.mockResolvedValue(JSON.stringify(makeLearningData()));
// Runner reported failure, but a task-referencing commit was found — the
// criterion was met, so the tier gets credit.
const agent = {
metadata: { modelTier: 'medium', taskDescription: 'Ship the fix' },
result: { success: false, validationPassed: true, duration: 60000 }
};
const task = { description: 'Ship the fix', taskType: 'internal', metadata: {} };
await recordTaskCompletion(agent, task);
const routing = savedData.routingAccuracy['internal-task']['medium'];
expect(routing.succeeded).toBe(1);
expect(routing.failed).toBe(0);
});
});
/**
* The "don't record this run" channel (#4107). A programmatic-I/O output hook
* that aborted BEFORE evaluating the agent's output (`no-app` / `app-not-found`,
* e.g. the app was deleted mid-run) stamps SKIP_LEARNING_VERDICT rather than the
* undeclared `null` — which used to fall back to the exit code and bank a free
* success for the task type.
*/
describe('TaskLearning - skip-recording verdict (#4107)', () => {
let savedData;
beforeEach(() => {
vi.clearAllMocks();
clearLearningCache();
savedData = null;
atomicWrite.mockImplementation(async (_path, data) => {
savedData = data;
});
readFile.mockResolvedValue(JSON.stringify(makeLearningData()));
});
const skipAgent = () => ({
metadata: { modelTier: 'heavy', taskDescription: 'Study the app' },
result: { success: true, validationPassed: SKIP_LEARNING_VERDICT, duration: 60000 }
});
const task = { description: 'Study the app', taskType: 'internal', metadata: {} };
it('writes NOTHING at all for a skipped run', async () => {
const returned = await recordTaskCompletion(skipAgent(), task);
// The whole point: no persist, so no aggregate, window, or total moves.
expect(atomicWrite).not.toHaveBeenCalled();
expect(savedData).toBeNull();
expect(returned).toBeNull();
});
it('does not bank the exit code as a success for the task type', async () => {
// The exact overcount #4107 fixes: exit 0 + a hook that never looked at the
// output used to come out the other side as a recorded success.
await recordTaskCompletion(skipAgent(), task);
expect(savedData).toBeNull();
// ...while an otherwise-identical run with the UNDECLARED sentinel still
// records, so the skip can't have swallowed the ordinary null path.
const undeclared = { ...skipAgent(), result: { success: true, validationPassed: null, duration: 60000 } };
await recordTaskCompletion(undeclared, task);
expect(atomicWrite).toHaveBeenCalledTimes(1);
expect(savedData.byTaskType['internal-task'].succeeded).toBe(1);
expect(savedData.byTaskType['internal-task'].failed).toBe(0);
});
it('does not blame the model either — a failing exit is skipped the same way', async () => {
// `false` would have poisoned the #2329 failure-signature window with a
// non-failure, so an aborted hook must not record a failure either.
const failed = {
...skipAgent(),
result: {
success: false,
validationPassed: SKIP_LEARNING_VERDICT,
duration: 60000,
errorAnalysis: { category: 'unknown', message: 'boom', origin: 'output-scan' }
}
};
await recordTaskCompletion(failed, task);
expect(atomicWrite).not.toHaveBeenCalled();
expect(savedData).toBeNull();
});
it('still records a genuine criterion MISS as a failure (#2344 unchanged)', async () => {
const missed = { ...skipAgent(), result: { success: true, validationPassed: false, duration: 60000 } };
await recordTaskCompletion(missed, task);
expect(atomicWrite).toHaveBeenCalledTimes(1);
expect(savedData.byTaskType['internal-task'].succeeded).toBe(0);
expect(savedData.byTaskType['internal-task'].failed).toBe(1);
expect(savedData.byModelTier['heavy'].failed).toBe(1);
});
});
describe('TaskLearning - environmental failures (#2618)', () => {
let savedData;
beforeEach(() => {
vi.clearAllMocks();
clearLearningCache();
savedData = null;
atomicWrite.mockImplementation(async (_path, data) => {
savedData = data;
});
});
const makeFailedAgent = (category, tier = 'heavy') => ({
metadata: { modelTier: tier, taskDescription: 'Fix UI' },
result: {
success: false,
duration: 30000,
errorAnalysis: { category, message: `provider says ${category}` }
}
});
const task = { description: 'Fix UI', taskType: 'user', metadata: {} };
it('keeps a rate-limit failure out of every success-rate aggregate and counts it separately', async () => {
const baseline = makeLearningData();
readFile.mockResolvedValue(JSON.stringify(baseline));
await recordTaskCompletion(makeFailedAgent('rate-limit'), task);
// byTaskType untouched — same counters/rate as the baseline, no recency sample.
expect(savedData.byTaskType['user-task']).toMatchObject({
completed: baseline.byTaskType['user-task'].completed,
succeeded: baseline.byTaskType['user-task'].succeeded,
failed: baseline.byTaskType['user-task'].failed,
successRate: baseline.byTaskType['user-task'].successRate
});
expect(savedData.byTaskType['user-task'].recentOutcomes).toBeUndefined();
// byModelTier untouched — the heavy tier bucket is not even initialized.
expect(savedData.byModelTier['heavy']).toBeUndefined();
// routingAccuracy untouched.
expect(savedData.routingAccuracy?.['user-task']).toBeUndefined();
// totals untouched — the overall rate is an aggregate too.
expect(savedData.totals).toMatchObject({
completed: baseline.totals.completed,
succeeded: baseline.totals.succeeded,
failed: baseline.totals.failed
});
// Correlation window untouched — an outage grades no routing prediction.
expect(savedData.correlationWindow ?? []).toEqual([]);
// ...but the event stays visible in the separate environmental record.
expect(savedData.environmentalFailures['rate-limit']).toMatchObject({
count: 1,
taskTypes: { 'user-task': 1 }
});
expect(savedData.environmentalFailures['rate-limit'].lastOccurred).toEqual(expect.any(String));
});
it('still records errorPatterns and failureSignatures for environmental categories (diagnostics)', async () => {
readFile.mockResolvedValue(JSON.stringify(makeLearningData()));
await recordTaskCompletion(makeFailedAgent('billing-error'), task);
expect(savedData.errorPatterns['billing-error']).toMatchObject({
count: 1,
taskTypes: { 'user-task': 1 }
});
expect(savedData.failureSignatures['billing-error'].count).toBe(1);
expect(savedData.failureSignatures['billing-error'].recent[0]).toMatchObject({
taskType: 'user-task',
messageSnippet: 'provider says billing-error'
});
});
it('never creates a byTaskType bucket for a type whose only history is an outage', async () => {
readFile.mockResolvedValue(JSON.stringify(makeLearningData()));
const freshTask = { description: '[auto-fix] investigate the crash', taskType: 'internal', metadata: {} };
await recordTaskCompletion(makeFailedAgent('startup-failure'), freshTask);
expect(savedData.byTaskType['auto-fix']).toBeUndefined();
expect(savedData.environmentalFailures['startup-failure'].taskTypes['auto-fix']).toBe(1);
});
it('accumulates environmental counts across completions', async () => {
let currentData = JSON.stringify(makeLearningData());
readFile.mockImplementation(async () => currentData);
atomicWrite.mockImplementation(async (_path, data) => {
currentData = JSON.stringify(data);
savedData = data;
});
await recordTaskCompletion(makeFailedAgent('rate-limit'), task);
await recordTaskCompletion(makeFailedAgent('rate-limit'), task);
await recordTaskCompletion(makeFailedAgent('auth'), task);
expect(savedData.environmentalFailures['rate-limit'].count).toBe(2);
expect(savedData.environmentalFailures['auth'].count).toBe(1);
expect(savedData.totals.failed).toBe(makeLearningData().totals.failed);
});
it('leaves a timeout failure counting against the aggregates exactly as today', async () => {
const baseline = makeLearningData();
readFile.mockResolvedValue(JSON.stringify(baseline));
await recordTaskCompletion(makeFailedAgent('timeout', 'light'), task);
// timeout is deliberately NOT environmental — it can be genuine task/model signal.
expect(savedData.byTaskType['user-task'].failed).toBe(baseline.byTaskType['user-task'].failed + 1);
expect(savedData.byModelTier['light'].failed).toBe(1);
expect(savedData.routingAccuracy['user-task']['light'].failed).toBe(1);
expect(savedData.totals.failed).toBe(baseline.totals.failed + 1);
expect(savedData.environmentalFailures).toBeUndefined();
});
it('loads a learning.json without the environmentalFailures key cleanly (back-compat)', async () => {
// makeLearningData has no environmentalFailures key — the additive key is
// initialized on first environmental recording, and a plain success write
// round-trips the file without fabricating one.
readFile.mockResolvedValue(JSON.stringify(makeLearningData()));
const agent = {
metadata: { modelTier: 'heavy', taskDescription: 'Fix UI' },
result: { success: true, duration: 30000 }
};
await recordTaskCompletion(agent, task);
expect(savedData.environmentalFailures).toBeUndefined();
expect(savedData.byTaskType['user-task'].succeeded)
.toBe(makeLearningData().byTaskType['user-task'].succeeded + 1);
});
it('surfaces environmentalFailures via getLearningInsights, sorted by count', async () => {
readFile.mockResolvedValue(JSON.stringify(makeLearningData({
environmentalFailures: {
'auth': { count: 2, lastOccurred: '2026-07-01T00:00:00.000Z', taskTypes: { 'user-task': 2 } },
'rate-limit': { count: 7, lastOccurred: '2026-07-02T00:00:00.000Z', taskTypes: { 'user-task': 4, 'auto-fix': 3 } }
}
})));
const insights = await getLearningInsights();
expect(insights.insights.environmentalFailures).toEqual([
{ category: 'rate-limit', count: 7, lastOccurred: '2026-07-02T00:00:00.000Z', affectedTypes: ['user-task', 'auto-fix'] },
{ category: 'auth', count: 2, lastOccurred: '2026-07-01T00:00:00.000Z', affectedTypes: ['user-task'] }
]);
});
it('surfaces an empty environmentalFailures list for a learning.json that predates the key', async () => {
readFile.mockResolvedValue(JSON.stringify(makeLearningData()));
const insights = await getLearningInsights();
expect(insights.insights.environmentalFailures).toEqual([]);
});
});
describe('TaskLearning - getRoutingAccuracy', () => {
beforeEach(() => {
vi.clearAllMocks();
clearLearningCache();
});
it('should return routing accuracy matrix with misroutes', async () => {
const data = makeLearningData({
routingAccuracy: {
'self-improve:ui': {
light: { succeeded: 1, failed: 9, lastAttempt: '2026-01-25T00:00:00.000Z' },
heavy: { succeeded: 8, failed: 2, lastAttempt: '2026-01-26T00:00:00.000Z' }
},
'user-task': {
medium: { succeeded: 15, failed: 3, lastAttempt: '2026-01-26T00:00:00.000Z' }
}
}
});
readFile.mockResolvedValue(JSON.stringify(data));
const result = await getRoutingAccuracy();
expect(result.matrix).toHaveLength(2);
expect(result.totalMisroutes).toBe(1); // self-improve:ui on light (10% success, 10 attempts)
// Check misroutes
expect(result.misroutes).toHaveLength(1);
expect(result.misroutes[0].taskType).toBe('self-improve:ui');
expect(result.misroutes[0].tier).toBe('light');
expect(result.misroutes[0].successRate).toBe(10);
// Check tier overview
expect(result.tierOverview.length).toBeGreaterThan(0);
});
it('should return empty results when no routing data exists', async () => {
readFile.mockResolvedValue(JSON.stringify(makeLearningData()));
const result = await getRoutingAccuracy();
expect(result.matrix).toHaveLength(0);
expect(result.misroutes).toHaveLength(0);
expect(result.totalMisroutes).toBe(0);
});
it('should sort matrix tiers by success rate descending', async () => {
const data = makeLearningData({
routingAccuracy: {
'user-task': {
light: { succeeded: 1, failed: 4, lastAttempt: '2026-01-25T00:00:00.000Z' },
medium: { succeeded: 3, failed: 2, lastAttempt: '2026-01-25T00:00:00.000Z' },
heavy: { succeeded: 9, failed: 1, lastAttempt: '2026-01-26T00:00:00.000Z' }
}
}
});
readFile.mockResolvedValue(JSON.stringify(data));
const result = await getRoutingAccuracy();
const tiers = result.matrix[0].tiers;
expect(tiers[0].tier).toBe('heavy'); // 90%
expect(tiers[1].tier).toBe('medium'); // 60%
expect(tiers[2].tier).toBe('light'); // 20%
});
it('reports the learned tier advice per task type, and omits it where history says nothing', async () => {
const data = makeLearningData({
byTaskType: {
'self-improve:ui': {
completed: 20, succeeded: 8, failed: 12,
totalDurationMs: 2000000, avgDurationMs: 100000,
successRate: 40
}
},
routingAccuracy: {
'self-improve:ui': {
light: { succeeded: 1, failed: 9, lastAttempt: '2026-01-25T00:00:00.000Z' },
heavy: { succeeded: 7, failed: 3, lastAttempt: '2026-01-26T00:00:00.000Z' }
},
'user-task': {
medium: { succeeded: 2, failed: 0, lastAttempt: '2026-01-26T00:00:00.000Z' }
}
}
});
readFile.mockResolvedValue(JSON.stringify(data));
const result = await getRoutingAccuracy();
const byType = Object.fromEntries(result.matrix.map((entry) => [entry.taskType, entry]));
expect(byType['self-improve:ui'].suggestion.avoidTiers).toContain('light');
expect(byType['user-task'].suggestion).toBeUndefined();
});
it('advises only pinnable tiers: retired thinking-level names map to their tier, user-specified never appears', async () => {
const outcome = { completed: 15, succeeded: 12, failed: 3, totalDurationMs: 1500000, avgDurationMs: 100000, successRate: 80 };
const data = makeLearningData({
byTaskType: { 'legacy-high': outcome, 'pinned-only': outcome },
routingAccuracy: {
'legacy-high': { high: { succeeded: 9, failed: 1, lastAttempt: '2026-01-26T00:00:00.000Z' } },
'pinned-only': { 'user-specified': { succeeded: 9, failed: 1, lastAttempt: '2026-01-26T00:00:00.000Z' } }
}
});
readFile.mockResolvedValue(JSON.stringify(data));
const result = await getRoutingAccuracy();
const byType = Object.fromEntries(result.matrix.map((entry) => [entry.taskType, entry]));
expect(byType['legacy-high'].suggestion.tier).toBe('heavy');
expect(byType['pinned-only'].suggestion).toBeUndefined();
// History is kept verbatim — only the advice is normalized.
expect(byType['legacy-high'].tiers[0].tier).toBe('high');
});
});
describe('TaskLearning - suggestModelTier with routing signals', () => {
beforeEach(() => {
vi.clearAllMocks();
clearLearningCache();
});
it('should suggest avoiding failing tiers', async () => {
const data = makeLearningData({
byTaskType: {
'self-improve:ui': {
completed: 20, succeeded: 8, failed: 12,
totalDurationMs: 2000000, avgDurationMs: 100000,
successRate: 40
}
},
routingAccuracy: {
'self-improve:ui': {
light: { succeeded: 1, failed: 9, lastAttempt: '2026-01-25T00:00:00.000Z' },
heavy: { succeeded: 7, failed: 3, lastAttempt: '2026-01-26T00:00:00.000Z' }
}
}
});
readFile.mockResolvedValue(JSON.stringify(data));
const result = await suggestModelTier('self-improve:ui');
expect(result).not.toBeNull();
expect(result.avoidTiers).toContain('light');
});
it('should suggest best performing tier when available', async () => {
const data = makeLearningData({
byTaskType: {
'user-task': {
completed: 15, succeeded: 12, failed: 3,
totalDurationMs: 1500000, avgDurationMs: 100000,
successRate: 80
}
},
routingAccuracy: {
'user-task': {
medium: { succeeded: 9, failed: 1, lastAttempt: '2026-01-26T00:00:00.000Z' },
heavy: { succeeded: 3, failed: 2, lastAttempt: '2026-01-25T00:00:00.000Z' }
}
}
});
readFile.mockResolvedValue(JSON.stringify(data));
const result = await suggestModelTier('user-task');
expect(result).not.toBeNull();
expect(result.suggested).toBe('medium'); // 90% success vs heavy at 60%
});
it('should return null when insufficient data', async () => {
readFile.mockResolvedValue(JSON.stringify(makeLearningData({
byTaskType: {
'new-task': { completed: 2, succeeded: 1, failed: 1, successRate: 50 }
}
})));
const result = await suggestModelTier('new-task');
expect(result).toBeNull();
});
it('steers off a freshly-failing tier even below the completions threshold (#2329)', async () => {
// Only 4 completions — too few for a routingAccuracy-based tier suggestion —
// but 3 recent failures on 'heavy' meet the failure-signal bar, so routing
// still emits an avoidance-only suggestion instead of null.
const data = makeLearningData({
byTaskType: {
'fresh-task': {
completed: 4, succeeded: 1, failed: 3,
totalDurationMs: 400000, avgDurationMs: 100000, successRate: 25
}
},
routingAccuracy: {},
correlationWindow: provenCorrelationWindow(),
failureSignatures: {
'test-failure': {
count: 3,
recent: [
{ modelTier: 'heavy', provider: 'claude', model: 'opus', taskType: 'fresh-task' },
{ modelTier: 'heavy', provider: 'claude', model: 'opus', taskType: 'fresh-task' },
{ modelTier: 'heavy', provider: 'claude', model: 'opus', taskType: 'fresh-task' }
]
}
}
});
readFile.mockResolvedValue(JSON.stringify(data));
const result = await suggestModelTier('fresh-task');
expect(result).not.toBeNull();
expect(result.suggested).toBeNull();
expect(result.avoidTiers).toContain('heavy');
expect(result.failureSignal).toMatchObject({ tier: 'heavy', failures: 3 });
});
it('still returns null below the completions threshold when there is no failure signal (#2329)', async () => {
readFile.mockResolvedValue(JSON.stringify(makeLearningData({
byTaskType: {
'quiet-task': { completed: 3, succeeded: 2, failed: 1, successRate: 66 }
},
routingAccuracy: {},
failureSignatures: {}
})));
expect(await suggestModelTier('quiet-task')).toBeNull();
});
it('folds enriched failure signatures into avoidTiers even when routingAccuracy is quiet (#2329)', async () => {
// Overall success is healthy (75%) and routingAccuracy carries no failing
// tier — pre-#2329 this returned null. The failure signatures show three
// recent 'heavy'-tier failures for this task type, so routing now emits an
// avoidance-only suggestion that steers selection off 'heavy'.
const data = makeLearningData({
byTaskType: {
'sig-task': {
completed: 12, succeeded: 9, failed: 3,
totalDurationMs: 1200000, avgDurationMs: 100000, successRate: 75
}
},
routingAccuracy: {},
correlationWindow: provenCorrelationWindow(),
failureSignatures: {
'test-failure': {
count: 3,
recent: [
{ modelTier: 'heavy', provider: 'claude', model: 'opus', taskType: 'sig-task' },
{ modelTier: 'heavy', provider: 'claude', model: 'opus', taskType: 'sig-task' },
{ modelTier: 'heavy', provider: 'codex', model: 'gpt', taskType: 'sig-task' }
]
}
}
});
readFile.mockResolvedValue(JSON.stringify(data));
const result = await suggestModelTier('sig-task');
expect(result).not.toBeNull();
expect(result.suggested).toBeNull();
expect(result.avoidTiers).toContain('heavy');
expect(result.failureSignal).toMatchObject({ tier: 'heavy', failures: 3, provider: 'claude' });
});
it('drops to avoidance-only when the tier it would pick is itself failure-flagged (#2329)', async () => {
// routingAccuracy: heavy is failing (<40%) and 'medium' is the only 60-79%
// "successful" alternative — but the failure signatures show 3 recent medium
// failures too. Routing must NOT then suggest medium; it drops to avoidance-
// only (suggested: null) so selection steers off both heavy and medium.
const data = makeLearningData({
byTaskType: {
'flap-task': {
completed: 20, succeeded: 9, failed: 11,
totalDurationMs: 2000000, avgDurationMs: 100000, successRate: 45
}
},
routingAccuracy: {
'flap-task': {
heavy: { succeeded: 1, failed: 9, lastAttempt: '2026-01-26T00:00:00.000Z' }, // 10%
medium: { succeeded: 7, failed: 3, lastAttempt: '2026-01-26T00:00:00.000Z' } // 70% (unproven)
}
},
correlationWindow: provenCorrelationWindow(),
failureSignatures: {
'test-failure': {
count: 3,
recent: [
{ modelTier: 'medium', provider: 'claude', model: 'sonnet', taskType: 'flap-task' },
{ modelTier: 'medium', provider: 'claude', model: 'sonnet', taskType: 'flap-task' },
{ modelTier: 'medium', provider: 'claude', model: 'sonnet', taskType: 'flap-task' }
]
}
}
});
readFile.mockResolvedValue(JSON.stringify(data));
const result = await suggestModelTier('flap-task');
expect(result.suggested).toBeNull();
expect(result.avoidTiers).toEqual(expect.arrayContaining(['heavy', 'medium']));
expect(result.failureSignal).toMatchObject({ tier: 'medium', provider: 'claude', model: 'sonnet' });
});
it('holds avoidance back to the conservative bar until correlation is proven (#2344)', async () => {
// 3 recent 'heavy' failures — enough for the AGGRESSIVE bar (3) — but the
// correlation window is empty (unproven), so the #2344 gate applies the
// conservative bar (5) and does NOT steer off 'heavy' yet.
const fixture = (correlationWindow) => makeLearningData({
byTaskType: {
'gate-task': {
completed: 12, succeeded: 9, failed: 3,
totalDurationMs: 1200000, avgDurationMs: 100000, successRate: 75
}
},
routingAccuracy: {},
correlationWindow,
failureSignatures: {
'test-failure': {
count: 3,
recent: [
{ modelTier: 'heavy', provider: 'claude', model: 'opus', taskType: 'gate-task' },
{ modelTier: 'heavy', provider: 'claude', model: 'opus', taskType: 'gate-task' },
{ modelTier: 'heavy', provider: 'claude', model: 'opus', taskType: 'gate-task' }
]
}
}
});
// Unproven correlation (empty window) → no avoidance despite 3 failures.
readFile.mockResolvedValue(JSON.stringify(fixture([])));
clearLearningCache();
expect(await suggestModelTier('gate-task')).toBeNull();
// Proven correlation (>0.8, >= min samples) → the same 3 failures now steer.
readFile.mockResolvedValue(JSON.stringify(fixture(provenCorrelationWindow())));
clearLearningCache();
const proven = await suggestModelTier('gate-task');
expect(proven).not.toBeNull();
expect(proven.avoidTiers).toContain('heavy');
});
it('does not let failure signatures condemn a proven tier (#2329)', async () => {
// 'heavy' clears the high-success threshold in routingAccuracy, so the
// proven-tier suggestion stands and its absolute failures never add it to
// avoidTiers.
const data = makeLearningData({
byTaskType: {
'proven-task': {
completed: 20, succeeded: 18, failed: 2,
totalDurationMs: 2000000, avgDurationMs: 100000, successRate: 90
}
},
routingAccuracy: {
'proven-task': {
heavy: { succeeded: 18, failed: 2, lastAttempt: '2026-01-26T00:00:00.000Z' } // 90%
}
},
failureSignatures: {
'test-failure': {
count: 3,
recent: [
{ modelTier: 'heavy', provider: 'claude', model: 'opus', taskType: 'proven-task' },
{ modelTier: 'heavy', provider: 'claude', model: 'opus', taskType: 'proven-task' },
{ modelTier: 'heavy', provider: 'claude', model: 'opus', taskType: 'proven-task' }
]
}
}
});
readFile.mockResolvedValue(JSON.stringify(data));
const result = await suggestModelTier('proven-task');
expect(result.suggested).toBe('heavy');
expect(result.avoidTiers).not.toContain('heavy');
});
it('should prefer the lightest tier when several clear the high-success threshold', async () => {
// heavy has the marginally-higher success rate, but light also clears 80% —
// we should route to light to avoid spending a heavier model for the same outcome.
const data = makeLearningData({
byTaskType: {
'docs-task': {
completed: 20, succeeded: 17, failed: 3,
totalDurationMs: 2000000, avgDurationMs: 100000,
successRate: 85
}
},
routingAccuracy: {
'docs-task': {
light: { succeeded: 9, failed: 2, lastAttempt: '2026-01-26T00:00:00.000Z' }, // 82%
heavy: { succeeded: 8, failed: 1, lastAttempt: '2026-01-25T00:00:00.000Z' } // 89%
}
}
});
readFile.mockResolvedValue(JSON.stringify(data));
const result = await suggestModelTier('docs-task');
expect(result).not.toBeNull();
expect(result.suggested).toBe('light');
expect(result.reason).toMatch(/lightest of 2 proven tiers/);
});
it('should rank routable thinking-level tier names by real cost, not as default-weight', async () => {
// routingAccuracy keys can be thinking-level names (high/xhigh), not just
// light/default/medium/heavy. xhigh (opus) is heavier than high (provider-
// heavy). Both clear 80%, so we must pick high — an earlier bug ranked every
// unmapped name at default-weight, which would have let xhigh win the tie.
const data = makeLearningData({
byTaskType: {
'reason-task': {
completed: 20, succeeded: 18, failed: 2,
totalDurationMs: 2000000, avgDurationMs: 100000,
successRate: 90
}
},
routingAccuracy: {
'reason-task': {
high: { succeeded: 9, failed: 2, lastAttempt: '2026-01-26T00:00:00.000Z' }, // 82%
xhigh: { succeeded: 10, failed: 1, lastAttempt: '2026-01-25T00:00:00.000Z' } // 91%
}
}
});
readFile.mockResolvedValue(JSON.stringify(data));
const result = await suggestModelTier('reason-task');
expect(result).not.toBeNull();
expect(result.suggested).toBe('high');
});
it('should not suggest a non-routable local-preferred tier (minimal/low) — picks the lightest tier the runtime can run', async () => {
// minimal/low resolve to a local model, but the learning→selection path
// doesn't switch providers for them, so suggesting minimal would make the
// selector fall through to the provider default. Even though minimal is the
// cheapest by cost, it must be excluded so the proven `light` tier wins.
const data = makeLearningData({
byTaskType: {
'local-task': {
completed: 20, succeeded: 18, failed: 2,
totalDurationMs: 2000000, avgDurationMs: 100000,
successRate: 90
}
},
routingAccuracy: {
'local-task': {
minimal: { succeeded: 10, failed: 1, lastAttempt: '2026-01-26T00:00:00.000Z' }, // 91%
light: { succeeded: 9, failed: 2, lastAttempt: '2026-01-25T00:00:00.000Z' } // 82%
}
}
});
readFile.mockResolvedValue(JSON.stringify(data));
const result = await suggestModelTier('local-task');
expect(result).not.toBeNull();
expect(result.suggested).toBe('light');
});
it('should not prefer an unknown tier name as the cheapest', async () => {
// user-specified (and any future/unknown tier name) must rank as heaviest,
// so it never wins the "lightest proven tier" tie over a known-light tier.
const data = makeLearningData({
byTaskType: {
'pref-task': {
completed: 20, succeeded: 18, failed: 2,
totalDurationMs: 2000000, avgDurationMs: 100000,
successRate: 90
}
},
routingAccuracy: {
'pref-task': {
'user-specified': { succeeded: 10, failed: 1, lastAttempt: '2026-01-26T00:00:00.000Z' }, // 91%
light: { succeeded: 9, failed: 2, lastAttempt: '2026-01-25T00:00:00.000Z' } // 82%
}
}
});
readFile.mockResolvedValue(JSON.stringify(data));
const result = await suggestModelTier('pref-task');
expect(result).not.toBeNull();
expect(result.suggested).toBe('light');
});
});
describe('TaskLearning - recalculateModelTierMetrics', () => {
let savedData;
beforeEach(() => {
vi.clearAllMocks();
clearLearningCache();
savedData = null;
atomicWrite.mockImplementation(async (_path, data) => {
savedData = data;
});
});
it('should rebuild byModelTier from routingAccuracy data', async () => {
const data = makeLearningData({
byModelTier: {
heavy: {
completed: 650, succeeded: 3, failed: 647,
totalDurationMs: 19500000, avgDurationMs: 30000
},
medium: {
completed: 200, succeeded: 150, failed: 50,
totalDurationMs: 40000000, avgDurationMs: 200000
}
},
byTaskType: {
'user-task': {
completed: 40, succeeded: 30, failed: 10,
totalDurationMs: 4000000, avgDurationMs: 100000,
successRate: 75
},
'internal-task': {
completed: 20, succeeded: 18, failed: 2,
totalDurationMs: 2000000, avgDurationMs: 100000,
successRate: 90
}
},
routingAccuracy: {
'user-task': {
medium: { succeeded: 25, failed: 5, lastAttempt: '2026-01-26T00:00:00.000Z' },
low: { succeeded: 8, failed: 0, lastAttempt: '2026-01-26T00:00:00.000Z' }
},
'internal-task': {
medium: { succeeded: 15, failed: 1, lastAttempt: '2026-01-26T00:00:00.000Z' }
}
}
});
readFile.mockResolvedValue(JSON.stringify(data));
const result = await recalculateModelTierMetrics();
expect(result.recalculated).toBe(true);
expect(result.changes.length).toBeGreaterThan(0);
// heavy tier should be gone (not in routingAccuracy)
expect(savedData.byModelTier.heavy).toBeUndefined();
// medium tier should reflect routing accuracy: 25+15=40 succeeded, 5+1=6 failed
expect(savedData.byModelTier.medium.completed).toBe(46);
expect(savedData.byModelTier.medium.succeeded).toBe(40);
expect(savedData.byModelTier.medium.failed).toBe(6);
// low tier: 8 succeeded, 0 failed
expect(savedData.byModelTier.low.completed).toBe(8);
expect(savedData.byModelTier.low.succeeded).toBe(8);
expect(savedData.byModelTier.low.failed).toBe(0);
});
it('should not save when nothing changes', async () => {
const data = makeLearningData({
byModelTier: {
medium: { completed: 3, succeeded: 2, failed: 1, totalDurationMs: 300000, avgDurationMs: 100000 }
},
routingAccuracy: {
'user-task': {
medium: { succeeded: 2, failed: 1, lastAttempt: '2026-01-26T00:00:00.000Z' }
}
}
});
readFile.mockResolvedValue(JSON.stringify(data));
const result = await recalculateModelTierMetrics();
expect(result.recalculated).toBe(false);
expect(atomicWrite).not.toHaveBeenCalled();
});
it('should handle empty routingAccuracy', async () => {
const data = makeLearningData({
byModelTier: {
heavy: { completed: 100, succeeded: 1, failed: 99, totalDurationMs: 100000, avgDurationMs: 1000 }
},
routingAccuracy: {}
});
readFile.mockResolvedValue(JSON.stringify(data));
const result = await recalculateModelTierMetrics();
expect(result.recalculated).toBe(true);
// All tiers should be cleared since no routing accuracy data exists
expect(savedData.byModelTier).toEqual({});
});
it('should estimate durations from task type averages', async () => {
const data = makeLearningData({
byModelTier: {},
byTaskType: {
'user-task': {
completed: 10, succeeded: 8, failed: 2,
totalDurationMs: 1000000, avgDurationMs: 100000,
successRate: 80
}
},
routingAccuracy: {
'user-task': {
medium: { succeeded: 5, failed: 1, lastAttempt: '2026-01-26T00:00:00.000Z' }
}
}
});
readFile.mockResolvedValue(JSON.stringify(data));
await recalculateModelTierMetrics();
// 6 agents × 100000ms avg = 600000ms total, 600000/6 = 100000ms avg
expect(savedData.byModelTier.medium.totalDurationMs).toBe(600000);
expect(savedData.byModelTier.medium.avgDurationMs).toBe(100000);
});
});
describe('TaskLearning - getTaskTypeConfidence', () => {
beforeEach(() => {
vi.clearAllMocks();
clearLearningCache();
});
it('should return "new" tier for unknown task type with no data', async () => {
const data = makeLearningData({ byTaskType: {} });
readFile.mockResolvedValue(JSON.stringify(data));
const result = await getTaskTypeConfidence('unknown-type');
expect(result.tier).toBe('new');
expect(result.autoApprove).toBe(true);
expect(result.completed).toBe(0);
expect(result.successRate).toBeNull();
});
it('should return "new" tier when completed < minSamples (default 5)', async () => {
const data = makeLearningData({
byTaskType: { 'sparse-type': { completed: 3, succeeded: 3, failed: 0, successRate: 100 } }
});
readFile.mockResolvedValue(JSON.stringify(data));
const result = await getTaskTypeConfidence('sparse-type');
expect(result.tier).toBe('new');
expect(result.autoApprove).toBe(true);
});
it('should return "high" tier when successRate >= 80 and enough samples', async () => {
const data = makeLearningData({
byTaskType: { 'reliable-type': { completed: 20, succeeded: 18, failed: 2, successRate: 90 } }
});
readFile.mockResolvedValue(JSON.stringify(data));
const result = await getTaskTypeConfidence('reliable-type');
expect(result.tier).toBe('high');
expect(result.autoApprove).toBe(true);
expect(result.successRate).toBe(90);
});
it('should return "medium" tier when successRate is between 50 and 80', async () => {
const data = makeLearningData({
byTaskType: { 'ok-type': { completed: 10, succeeded: 6, failed: 4, successRate: 60 } }
});
readFile.mockResolvedValue(JSON.stringify(data));
const result = await getTaskTypeConfidence('ok-type');
expect(result.tier).toBe('medium');
expect(result.autoApprove).toBe(true);
});
it('should return "low" tier when successRate < 50 and enough samples', async () => {
const data = makeLearningData({
byTaskType: { 'flaky-type': { completed: 10, succeeded: 3, failed: 7, successRate: 30 } }
});
readFile.mockResolvedValue(JSON.stringify(data));
const result = await getTaskTypeConfidence('flaky-type');
expect(result.tier).toBe('low');
expect(result.autoApprove).toBe(false);
});
it('should respect threshold overrides', async () => {
const data = makeLearningData({
byTaskType: { 'borderline': { completed: 10, succeeded: 7, failed: 3, successRate: 70 } }
});
readFile.mockResolvedValue(JSON.stringify(data));
// With default thresholds (high=80), 70% is medium
const defaultResult = await getTaskTypeConfidence('borderline');
expect(defaultResult.tier).toBe('medium');
clearLearningCache();
readFile.mockResolvedValue(JSON.stringify(data));
// With lowered highThreshold=60, 70% becomes high
const overrideResult = await getTaskTypeConfidence('borderline', { highThreshold: 60 });
expect(overrideResult.tier).toBe('high');
});
it('should respect minSamples override', async () => {
const data = makeLearningData({
byTaskType: { 'small-sample': { completed: 3, succeeded: 3, failed: 0, successRate: 100 } }
});
readFile.mockResolvedValue(JSON.stringify(data));
// With minSamples=2, 3 completions is enough to classify
const result = await getTaskTypeConfidence('small-sample', { minSamples: 2 });
expect(result.tier).toBe('high');
expect(result.autoApprove).toBe(true);
});
});
describe('TaskLearning - getConfidenceLevels', () => {
beforeEach(() => {
vi.clearAllMocks();
clearLearningCache();
});
it('should group task types into correct tiers', async () => {
const data = makeLearningData({
byTaskType: {
'high-type': { completed: 10, succeeded: 9, failed: 1, successRate: 90 },
'medium-type': { completed: 10, succeeded: 6, failed: 4, successRate: 60 },
'low-type': { completed: 10, succeeded: 3, failed: 7, successRate: 30 },
'new-type': { completed: 2, succeeded: 2, failed: 0, successRate: 100 }
}
});
readFile.mockResolvedValue(JSON.stringify(data));
const result = await getConfidenceLevels();
expect(result.levels.high.map(t => t.taskType)).toContain('high-type');
expect(result.levels.medium.map(t => t.taskType)).toContain('medium-type');
expect(result.levels.low.map(t => t.taskType)).toContain('low-type');
expect(result.levels.new.map(t => t.taskType)).toContain('new-type');
});
it('should return correct summary counts', async () => {
const data = makeLearningData({
byTaskType: {
'h1': { completed: 10, succeeded: 9, failed: 1, successRate: 90 },
'h2': { completed: 10, succeeded: 8, failed: 2, successRate: 80 },
'm1': { completed: 10, succeeded: 6, failed: 4, successRate: 60 },
'l1': { completed: 10, succeeded: 3, failed: 7, successRate: 30 }
}
});
readFile.mockResolvedValue(JSON.stringify(data));
const result = await getConfidenceLevels();
expect(result.summary.high).toBe(2);
expect(result.summary.medium).toBe(1);
expect(result.summary.low).toBe(1);
expect(result.summary.total).toBe(4);
expect(result.summary.requireApproval).toBe(1);
});
it('should sort tiers by successRate descending', async () => {
const data = makeLearningData({
byTaskType: {
'a': { completed: 10, succeeded: 8, failed: 2, successRate: 80 },
'b': { completed: 10, succeeded: 9, failed: 1, successRate: 90 },
'c': { completed: 10, succeeded: 8, failed: 2, successRate: 80 }
}
});
readFile.mockResolvedValue(JSON.stringify(data));
const result = await getConfidenceLevels();
const highRates = result.levels.high.map(t => t.successRate);
expect(highRates).toEqual([...highRates].sort((a, b) => b - a));
});
it('should reflect threshold overrides in returned thresholds', async () => {
const data = makeLearningData({ byTaskType: {} });
readFile.mockResolvedValue(JSON.stringify(data));
const result = await getConfidenceLevels({ highThreshold: 90, lowThreshold: 60, minSamples: 3 });
expect(result.thresholds).toEqual({ highThreshold: 90, lowThreshold: 60, minSamples: 3 });
});
it('should return empty levels when no task types exist', async () => {
const data = makeLearningData({ byTaskType: {} });
readFile.mockResolvedValue(JSON.stringify(data));
const result = await getConfidenceLevels();
expect(result.summary.total).toBe(0);
expect(result.levels.high).toHaveLength(0);
expect(result.levels.low).toHaveLength(0);
});
});
describe('TaskLearning - recommendation dismissal', () => {
let writes;
// Route reads by file path so learning.json and dismissed-recommendations.json
// can return different content within a single test.
const setupFiles = ({ learning = {}, dismissed = {} } = {}) => {
readFile.mockImplementation(async (filePath) => {
if (String(filePath).endsWith('dismissed-recommendations.json')) {
return JSON.stringify(dismissed);
}
return JSON.stringify(learning);
});
};
beforeEach(() => {
vi.clearAllMocks();
clearLearningCache();
writes = {};
atomicWrite.mockImplementation(async (filePath, data) => {
writes[String(filePath)] = data;
});
});
it('dismissRecommendation persists id with snapshot and timestamp', async () => {
setupFiles();
const result = await dismissRecommendation('error-pattern:unknown', { kind: 'count', value: 74 });
expect(result).toEqual({ id: 'error-pattern:unknown', dismissed: true });
const saved = Object.entries(writes).find(([p]) => p.endsWith('dismissed-recommendations.json'))?.[1];
expect(saved).toBeDefined();
expect(saved['error-pattern:unknown']).toMatchObject({
snapshot: { kind: 'count', value: 74 }
});
expect(saved['error-pattern:unknown'].dismissedAt).toBeDefined();
});
it('restoreRecommendation removes the id from the dismissed map', async () => {
setupFiles({
dismissed: {
'error-pattern:unknown': { dismissedAt: '2026-05-07T00:00:00.000Z', snapshot: { kind: 'count', value: 74 } }
}
});
const result = await restoreRecommendation('error-pattern:unknown');
expect(result).toEqual({ id: 'error-pattern:unknown', restored: true });
const saved = Object.entries(writes).find(([p]) => p.endsWith('dismissed-recommendations.json'))?.[1];
expect(saved['error-pattern:unknown']).toBeUndefined();
});
it('restoreRecommendation reports false when id was not dismissed', async () => {
setupFiles({ dismissed: {} });
const result = await restoreRecommendation('error-pattern:unknown');
expect(result).toEqual({ id: 'error-pattern:unknown', restored: false });
});
it('clearDismissedRecommendations writes an empty map', async () => {
setupFiles({
dismissed: {
'a': { dismissedAt: '2026-05-07T00:00:00.000Z' },
'b': { dismissedAt: '2026-05-07T01:00:00.000Z' }
}
});
const result = await clearDismissedRecommendations();
expect(result).toEqual({ cleared: true });
const saved = Object.entries(writes).find(([p]) => p.endsWith('dismissed-recommendations.json'))?.[1];
expect(saved).toEqual({});
});
it('getDismissedRecommendations sorts by dismissedAt desc', async () => {
setupFiles({
dismissed: {
'older': { dismissedAt: '2026-05-01T00:00:00.000Z' },
'newer': { dismissedAt: '2026-05-07T00:00:00.000Z' }
}
});
const list = await getDismissedRecommendations();
expect(list.map(d => d.id)).toEqual(['newer', 'older']);
});
it('getLearningInsights filters dismissed rate-based recommendation permanently', async () => {
const learning = makeLearningData({
byTaskType: {
// 100% success rate — would generate top-perf optimization recommendation
'self-improve:accessibility': {
completed: 50, succeeded: 50, failed: 0,
totalDurationMs: 500000, avgDurationMs: 10000,
lastCompleted: '2026-05-07T00:00:00.000Z', successRate: 100
}
},
errorPatterns: {},
byModelTier: {},
totals: { completed: 50, succeeded: 50, failed: 0, totalDurationMs: 500000, avgDurationMs: 10000 }
});
setupFiles({
learning,
dismissed: {
'top-perf:self-improve:accessibility': {
dismissedAt: '2026-05-07T00:00:00.000Z',
snapshot: { kind: 'rate', value: 100 }
}
}
});
const insights = await getLearningInsights();
const ids = (insights.recommendations || []).map(r => r.id);
expect(ids).not.toContain('top-perf:self-improve:accessibility');
});
it('getLearningInsights re-surfaces a count-based dismissal when the count grows past threshold', async () => {
const learning = makeLearningData({
byTaskType: {},
errorPatterns: {
'unknown': {
count: 200, // grew significantly past the snapshot of 74
taskTypes: { 'self-improve:ui': 200 },
lastOccurred: '2026-05-08T00:00:00.000Z'
}
},
byModelTier: {},
totals: { completed: 200, succeeded: 0, failed: 200, totalDurationMs: 0, avgDurationMs: 0 }
});
setupFiles({
learning,
dismissed: {
'error-pattern:unknown': {
dismissedAt: '2026-04-01T00:00:00.000Z',
snapshot: { kind: 'count', value: 74 }
}
}
});
const insights = await getLearningInsights();
const ids = (insights.recommendations || []).map(r => r.id);
expect(ids).toContain('error-pattern:unknown');
});
it('getLearningInsights keeps a count-based dismissal suppressed when count is unchanged', async () => {
const learning = makeLearningData({
byTaskType: {},
errorPatterns: {
'unknown': {
count: 80, // only marginally higher than dismissal snapshot of 74
taskTypes: { 'self-improve:ui': 80 },
lastOccurred: '2026-05-08T00:00:00.000Z'
}
},
byModelTier: {},
totals: { completed: 80, succeeded: 0, failed: 80, totalDurationMs: 0, avgDurationMs: 0 }
});
setupFiles({
learning,
dismissed: {
'error-pattern:unknown': {
dismissedAt: '2026-04-01T00:00:00.000Z',
snapshot: { kind: 'count', value: 74 }
}
}
});
const insights = await getLearningInsights();
const ids = (insights.recommendations || []).map(r => r.id);
expect(ids).not.toContain('error-pattern:unknown');
});
it('getLearningInsights gives every recommendation a stable id', async () => {
const learning = makeLearningData({
byTaskType: {
'self-improve:accessibility': {
completed: 50, succeeded: 50, failed: 0,
totalDurationMs: 500000, avgDurationMs: 10000,
lastCompleted: '2026-05-07T00:00:00.000Z', successRate: 100
}
},
errorPatterns: {
'unknown': { count: 74, taskTypes: { 'self-improve:ui': 74 }, lastOccurred: '2026-05-07T00:00:00.000Z' }
},
byModelTier: {},
totals: { completed: 50, succeeded: 50, failed: 0, totalDurationMs: 500000, avgDurationMs: 10000 }
});
setupFiles({ learning, dismissed: {} });
const insights = await getLearningInsights();
expect(insights.recommendations.length).toBeGreaterThan(0);
for (const rec of insights.recommendations) {
expect(rec.id).toBeTruthy();
expect(typeof rec.id).toBe('string');
}
});
});