Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

63.3 KB
Raw Blame
import { describe, it, expect, vi, beforeEach } from 'vitest';

// Mock fs/promises and fs before importing the module
vi.mock('fs/promises', () => ({
  readFile: vi.fn(),
  mkdir: vi.fn()
}));

vi.mock('fs', () => ({
  existsSync: vi.fn(() => true)
}));

// Mock cos.js to avoid circular dependency
vi.mock('./cos.js', () => ({}));
vi.mock('./cosEvents.js', () => ({
  cosEvents: { on: vi.fn(), emit: vi.fn() },
  emitLog: vi.fn()
}));

// Mock fileUtils.js to use our mocked fs/promises
vi.mock('../lib/fileUtils.js', async (importOriginal) => {
  const fsPromises = await import('fs/promises');
  const fs = await import('fs');
  return {
    ensureDir: vi.fn(),
    atomicWrite: vi.fn().mockResolvedValue(undefined),
    readJSONFile: vi.fn(async (filePath, defaultValue) => {
      if (!fs.existsSync(filePath)) return defaultValue;
      const content = await fsPromises.readFile(filePath, 'utf-8');
      if (!content || !content.trim()) return defaultValue;
      return JSON.parse(content);
    }),
    PATHS: { cos: '/tmp/test/cos' }
  };
});

import { readFile } from 'fs/promises';
import { atomicWrite } from '../lib/fileUtils.js';
import { SKIP_LEARNING_VERDICT } from '../lib/learningVerdict.js';
import { resetTaskTypeLearning, getSkippedTaskTypes, recordTaskCompletion, getRoutingAccuracy, suggestModelTier, recalculateModelTierMetrics, clearLearningCache, getTaskTypeConfidence, getConfidenceLevels, dismissRecommendation, restoreRecommendation, getDismissedRecommendations, clearDismissedRecommendations, getLearningInsights } from './taskLearning.js';

const makeLearningData = (overrides = {}) => ({
tryReadFile: vi.fn().mockResolvedValue(null),
  version: 1,
  lastUpdated: '2026-01-26T00:00:00.000Z',
  byTaskType: {
    'self-improve:ui': {
      completed: 200,
      succeeded: 10,
      failed: 190,
      totalDurationMs: 2000000,
      avgDurationMs: 10000,
      lastCompleted: '2026-01-25T00:00:00.000Z',
      successRate: 5
    },
    'user-task': {
      completed: 40,
      succeeded: 30,
      failed: 10,
      totalDurationMs: 4000000,
      avgDurationMs: 100000,
      lastCompleted: '2026-01-26T00:00:00.000Z',
      successRate: 75
    }
  },
  byModelTier: {
    'user-specified': {
      completed: 240,
      succeeded: 40,
      failed: 200,
      totalDurationMs: 6000000,
      avgDurationMs: 25000
    }
  },
  errorPatterns: {
    'server-error': {
      count: 190,
      taskTypes: { 'self-improve:ui': 185, 'user-task': 5 },
      lastOccurred: '2026-01-25T00:00:00.000Z'
    },
    'unknown': {
      count: 10,
      taskTypes: { 'self-improve:ui': 10 },
      lastOccurred: '2026-01-24T00:00:00.000Z'
    }
  },
  totals: {
    completed: 240,
    succeeded: 40,
    failed: 200,
    totalDurationMs: 6000000,
    avgDurationMs: 25000
  },
  ...overrides
});

// A correlation-quality window proving the enriched failure signal predicts
// outcomes (perfect correlation, >= MIN_CORRELATION_SAMPLES) — so `suggestModelTier`
// runs the AGGRESSIVE failure-sample bar (3) and the #2329 avoidance mechanics
// engage. Without this the #2344 gate holds avoidance back to the conservative
// bar (5) until the signal has earned it. Injected into the #2329 avoidance
// tests to isolate their intent from the new gate (which has its own tests).
const provenCorrelationWindow = () => Array.from({ length: 24 }, (_, i) => ({
  taskType: 'user-task', tier: 'heavy',
  predictedRisk: i % 2 === 0,
  bad: i % 2 === 0,
  recordedAt: '2026-01-26T00:00:00.000Z'
}));

describe('TaskLearning - resetTaskTypeLearning', () => {
  let savedData;

  beforeEach(() => {
    vi.clearAllMocks();
    clearLearningCache();
    savedData = null;
    atomicWrite.mockImplementation(async (_path, data) => {
      savedData = data;
    });
  });

  it('should return not-found when task type does not exist', async () => {
    readFile.mockResolvedValue(JSON.stringify(makeLearningData()));

    const result = await resetTaskTypeLearning('nonexistent-type');

    expect(result.reset).toBe(false);
    expect(result.reason).toBe('task-type-not-found');
  });

  it('should remove the task type from byTaskType', async () => {
    readFile.mockResolvedValue(JSON.stringify(makeLearningData()));

    const result = await resetTaskTypeLearning('self-improve:ui');

    expect(result.reset).toBe(true);
    expect(result.taskType).toBe('self-improve:ui');
    expect(savedData.byTaskType['self-improve:ui']).toBeUndefined();
    expect(savedData.byTaskType['user-task']).toBeDefined();
  });

  it('should subtract task type metrics from totals', async () => {
    readFile.mockResolvedValue(JSON.stringify(makeLearningData()));

    await resetTaskTypeLearning('self-improve:ui');

    // Original totals: completed=240, succeeded=40, failed=200, totalDurationMs=6000000
    // self-improve:ui: completed=200, succeeded=10, failed=190, totalDurationMs=2000000
    // After reset: completed=40, succeeded=30, failed=10, totalDurationMs=4000000
    expect(savedData.totals.completed).toBe(40);
    expect(savedData.totals.succeeded).toBe(30);
    expect(savedData.totals.failed).toBe(10);
    expect(savedData.totals.totalDurationMs).toBe(4000000);
    expect(savedData.totals.avgDurationMs).toBe(100000); // 4000000 / 40
  });

  it('should clean up error patterns referencing the task type', async () => {
    readFile.mockResolvedValue(JSON.stringify(makeLearningData()));

    await resetTaskTypeLearning('self-improve:ui');

    // server-error had 190 total (185 from ui, 5 from user-task) → should now have 5
    expect(savedData.errorPatterns['server-error'].count).toBe(5);
    expect(savedData.errorPatterns['server-error'].taskTypes['self-improve:ui']).toBeUndefined();
    expect(savedData.errorPatterns['server-error'].taskTypes['user-task']).toBe(5);

    // unknown had 10 total, all from ui → should be removed entirely
    expect(savedData.errorPatterns['unknown']).toBeUndefined();
  });

  it('should return previous metrics in result', async () => {
    readFile.mockResolvedValue(JSON.stringify(makeLearningData()));

    const result = await resetTaskTypeLearning('self-improve:ui');

    expect(result.previousMetrics).toEqual({
      completed: 200,
      succeeded: 10,
      failed: 190,
      successRate: 5
    });
  });

  it('should handle totals going to zero gracefully', async () => {
    const data = makeLearningData({
      byTaskType: {
        'self-improve:ui': {
          completed: 100, succeeded: 5, failed: 95,
          totalDurationMs: 500000, avgDurationMs: 5000,
          lastCompleted: '2026-01-25T00:00:00.000Z', successRate: 5
        }
      },
      errorPatterns: {},
      totals: {
        completed: 100, succeeded: 5, failed: 95,
        totalDurationMs: 500000, avgDurationMs: 5000
      }
    });
    readFile.mockResolvedValue(JSON.stringify(data));

    await resetTaskTypeLearning('self-improve:ui');

    expect(savedData.totals.completed).toBe(0);
    expect(savedData.totals.avgDurationMs).toBe(0);
  });

  it('clamps successDurationMs to >= 0 when a drifted store would go negative', async () => {
    // Hand-edited / older store where the running total drifted below a single
    // task type's accumulated successDurationMs. Subtracting unclamped would go
    // negative and feed calculateDurationETA a negative avgDurationMs.
    const data = makeLearningData({
      byTaskType: {
        'self-improve:ui': {
          completed: 10, succeeded: 8, failed: 2,
          totalDurationMs: 50000, avgDurationMs: 5000,
          successDurationMs: 500000, successMaxDurationMs: 80000,
          lastCompleted: '2026-01-25T00:00:00.000Z', successRate: 80
        }
      },
      errorPatterns: {},
      totals: {
        completed: 10, succeeded: 8, failed: 2,
        totalDurationMs: 50000, avgDurationMs: 5000,
        successDurationMs: 100000, successMaxDurationMs: 80000
      }
    });
    readFile.mockResolvedValue(JSON.stringify(data));

    await resetTaskTypeLearning('self-improve:ui');

    // 100000 - 500000 would be -400000; the clamp pins it at 0.
    expect(savedData.totals.successDurationMs).toBe(0);
  });
});

describe('TaskLearning - getSkippedTaskTypes', () => {
  beforeEach(() => {
    vi.clearAllMocks();
    clearLearningCache();
  });

  it('should return task types with <30% success and 5+ attempts', async () => {
    readFile.mockResolvedValue(JSON.stringify(makeLearningData()));

    const skipped = await getSkippedTaskTypes();

    expect(skipped).toHaveLength(1);
    expect(skipped[0].taskType).toBe('self-improve:ui');
    expect(skipped[0].successRate).toBe(5);
  });

  it('should not include task types with >= 30% success', async () => {
    readFile.mockResolvedValue(JSON.stringify(makeLearningData()));

    const skipped = await getSkippedTaskTypes();

    const userTask = skipped.find(s => s.taskType === 'user-task');
    expect(userTask).toBeUndefined();
  });

  it('should return empty after resetting a skipped type', async () => {
    const data = makeLearningData({
      byTaskType: {
        'self-improve:ui': {
          completed: 100, succeeded: 5, failed: 95,
          totalDurationMs: 500000, avgDurationMs: 5000,
          lastCompleted: '2026-01-25T00:00:00.000Z', successRate: 5
        }
      },
      errorPatterns: {},
      totals: {
        completed: 100, succeeded: 5, failed: 95,
        totalDurationMs: 500000, avgDurationMs: 5000
      }
    });

    // Track what was written so subsequent reads return updated data
    let currentData = JSON.stringify(data);
    readFile.mockImplementation(async () => currentData);
    atomicWrite.mockImplementation(async (_path, data) => {
      currentData = JSON.stringify(data);
    });

    await resetTaskTypeLearning('self-improve:ui');
    const skipped = await getSkippedTaskTypes();

    expect(skipped).toHaveLength(0);
  });

  it('should clean up routingAccuracy data when resetting a task type', async () => {
    let savedData;
    atomicWrite.mockImplementation(async (_path, data) => {
      savedData = data;
    });

    const data = makeLearningData({
      routingAccuracy: {
        'self-improve:ui': {
          heavy: { succeeded: 2, failed: 8, lastAttempt: '2026-01-25T00:00:00.000Z' },
          medium: { succeeded: 0, failed: 5, lastAttempt: '2026-01-24T00:00:00.000Z' }
        },
        'user-task': {
          heavy: { succeeded: 10, failed: 2, lastAttempt: '2026-01-26T00:00:00.000Z' }
        }
      }
    });
    readFile.mockResolvedValue(JSON.stringify(data));

    await resetTaskTypeLearning('self-improve:ui');

    expect(savedData.routingAccuracy['self-improve:ui']).toBeUndefined();
    expect(savedData.routingAccuracy['user-task']).toBeDefined();
  });

  it('should subtract from byModelTier when resetting a task type with routing data', async () => {
    let savedData;
    atomicWrite.mockImplementation(async (_path, data) => {
      savedData = data;
    });

    const data = makeLearningData({
      byModelTier: {
        heavy: {
          completed: 50,
          succeeded: 20,
          failed: 30,
          totalDurationMs: 500000,
          avgDurationMs: 10000
        },
        medium: {
          completed: 10,
          succeeded: 2,
          failed: 8,
          totalDurationMs: 100000,
          avgDurationMs: 10000
        }
      },
      routingAccuracy: {
        'self-improve:ui': {
          heavy: { succeeded: 5, failed: 15, lastAttempt: '2026-01-25T00:00:00.000Z' },
          medium: { succeeded: 0, failed: 5, lastAttempt: '2026-01-24T00:00:00.000Z' }
        },
        'user-task': {
          heavy: { succeeded: 10, failed: 2, lastAttempt: '2026-01-26T00:00:00.000Z' }
        }
      }
    });
    readFile.mockResolvedValue(JSON.stringify(data));

    await resetTaskTypeLearning('self-improve:ui');

    // heavy: was 50 completed (20 succeeded, 30 failed), subtract 20 (5+15) from self-improve:ui
    expect(savedData.byModelTier.heavy.completed).toBe(30);
    expect(savedData.byModelTier.heavy.succeeded).toBe(15);
    expect(savedData.byModelTier.heavy.failed).toBe(15);
    // medium: was 10 completed, subtract 5 (0+5) from self-improve:ui
    expect(savedData.byModelTier.medium.completed).toBe(5);
    expect(savedData.byModelTier.medium.succeeded).toBe(2);
    expect(savedData.byModelTier.medium.failed).toBe(3);
    // user-task routing should be untouched
    expect(savedData.routingAccuracy['user-task']).toBeDefined();
  });

  it('should delete byModelTier entry when count reaches zero', async () => {
    let savedData;
    atomicWrite.mockImplementation(async (_path, data) => {
      savedData = data;
    });

    const data = makeLearningData({
      byModelTier: {
        'user-specified': {
          completed: 240,
          succeeded: 40,
          failed: 200,
          totalDurationMs: 6000000,
          avgDurationMs: 25000
        },
        medium: {
          completed: 5,
          succeeded: 0,
          failed: 5,
          totalDurationMs: 50000,
          avgDurationMs: 10000
        }
      },
      routingAccuracy: {
        'self-improve:ui': {
          medium: { succeeded: 0, failed: 5, lastAttempt: '2026-01-24T00:00:00.000Z' }
        }
      }
    });
    readFile.mockResolvedValue(JSON.stringify(data));

    await resetTaskTypeLearning('self-improve:ui');

    // medium tier had 5 completed, all from self-improve:ui → should be deleted
    expect(savedData.byModelTier.medium).toBeUndefined();
    // user-specified should be untouched (no routing data for it)
    expect(savedData.byModelTier['user-specified']).toBeDefined();
  });
});

describe('TaskLearning - recordTaskCompletion routing accuracy', () => {
  let savedData;

  beforeEach(() => {
    vi.clearAllMocks();
    clearLearningCache();
    savedData = null;
    atomicWrite.mockImplementation(async (_path, data) => {
      savedData = data;
    });
  });

  it('should record routing accuracy for successful task', async () => {
    readFile.mockResolvedValue(JSON.stringify(makeLearningData()));

    const agent = {
      metadata: { modelTier: 'heavy', taskDescription: 'Fix some UI bugs' },
      result: { success: true, duration: 60000 }
    };
    const task = { description: 'Fix some UI bugs', taskType: 'user', metadata: {} };

    await recordTaskCompletion(agent, task);

    expect(savedData.routingAccuracy).toBeDefined();
    expect(savedData.routingAccuracy['user-task']).toBeDefined();
    expect(savedData.routingAccuracy['user-task']['heavy']).toBeDefined();
    expect(savedData.routingAccuracy['user-task']['heavy'].succeeded).toBe(1);
    expect(savedData.routingAccuracy['user-task']['heavy'].failed).toBe(0);
  });

  it('should record routing accuracy for failed task', async () => {
    readFile.mockResolvedValue(JSON.stringify(makeLearningData()));

    const agent = {
      metadata: { modelTier: 'light', taskDescription: 'Fix UI' },
      result: { success: false, duration: 30000 }
    };
    const task = { description: 'Fix UI', taskType: 'user', metadata: {} };

    await recordTaskCompletion(agent, task);

    expect(savedData.routingAccuracy['user-task']['light'].succeeded).toBe(0);
    expect(savedData.routingAccuracy['user-task']['light'].failed).toBe(1);
    expect(savedData.routingAccuracy['user-task']['light'].lastAttempt).toBeDefined();
  });

  it('should accumulate routing accuracy across multiple completions', async () => {
    let currentData = JSON.stringify(makeLearningData());
    readFile.mockImplementation(async () => currentData);
    atomicWrite.mockImplementation(async (_path, data) => {
      currentData = JSON.stringify(data);
      savedData = data;
    });

    const makeAgent = (tier, success) => ({
      metadata: { modelTier: tier, taskDescription: 'Test task' },
      result: { success, duration: 30000 }
    });
    const task = { description: 'Test task', taskType: 'user', metadata: {} };

    await recordTaskCompletion(makeAgent('medium', true), task);
    await recordTaskCompletion(makeAgent('medium', true), task);
    await recordTaskCompletion(makeAgent('medium', false), task);

    expect(savedData.routingAccuracy['user-task']['medium'].succeeded).toBe(2);
    expect(savedData.routingAccuracy['user-task']['medium'].failed).toBe(1);
  });

  it('counts a clean exit that MISSED its declared criterion as a tier failure (#2344)', async () => {
    readFile.mockResolvedValue(JSON.stringify(makeLearningData()));

    // Runner exit-code success, but the declared success criterion was not met
    // (e.g. agent finished clean yet committed nothing). The learning aggregates
    // must treat the validation verdict as authoritative — not the exit code.
    const agent = {
      metadata: { modelTier: 'heavy', taskDescription: 'Ship the fix' },
      result: { success: true, validationPassed: false, duration: 60000 }
    };
    const task = { description: 'Ship the fix', taskType: 'internal', metadata: {} };

    await recordTaskCompletion(agent, task);

    const routing = savedData.routingAccuracy['internal-task']['heavy'];
    expect(routing.succeeded).toBe(0);
    expect(routing.failed).toBe(1);
    expect(savedData.byModelTier['heavy'].succeeded).toBe(0);
    expect(savedData.byModelTier['heavy'].failed).toBe(1);
    expect(savedData.byTaskType['internal-task'].succeeded).toBe(0);
    expect(savedData.byTaskType['internal-task'].failed).toBe(1);
  });

  it('counts a non-zero exit that MET its declared criterion as a tier success (#2344)', async () => {
    readFile.mockResolvedValue(JSON.stringify(makeLearningData()));

    // Runner reported failure, but a task-referencing commit was found — the
    // criterion was met, so the tier gets credit.
    const agent = {
      metadata: { modelTier: 'medium', taskDescription: 'Ship the fix' },
      result: { success: false, validationPassed: true, duration: 60000 }
    };
    const task = { description: 'Ship the fix', taskType: 'internal', metadata: {} };

    await recordTaskCompletion(agent, task);

    const routing = savedData.routingAccuracy['internal-task']['medium'];
    expect(routing.succeeded).toBe(1);
    expect(routing.failed).toBe(0);
  });
});

/**
 * The "don't record this run" channel (#4107). A programmatic-I/O output hook
 * that aborted BEFORE evaluating the agent's output (`no-app` / `app-not-found`,
 * e.g. the app was deleted mid-run) stamps SKIP_LEARNING_VERDICT rather than the
 * undeclared `null` — which used to fall back to the exit code and bank a free
 * success for the task type.
 */
describe('TaskLearning - skip-recording verdict (#4107)', () => {
  let savedData;

  beforeEach(() => {
    vi.clearAllMocks();
    clearLearningCache();
    savedData = null;
    atomicWrite.mockImplementation(async (_path, data) => {
      savedData = data;
    });
    readFile.mockResolvedValue(JSON.stringify(makeLearningData()));
  });

  const skipAgent = () => ({
    metadata: { modelTier: 'heavy', taskDescription: 'Study the app' },
    result: { success: true, validationPassed: SKIP_LEARNING_VERDICT, duration: 60000 }
  });
  const task = { description: 'Study the app', taskType: 'internal', metadata: {} };

  it('writes NOTHING at all for a skipped run', async () => {
    const returned = await recordTaskCompletion(skipAgent(), task);

    // The whole point: no persist, so no aggregate, window, or total moves.
    expect(atomicWrite).not.toHaveBeenCalled();
    expect(savedData).toBeNull();
    expect(returned).toBeNull();
  });

  it('does not bank the exit code as a success for the task type', async () => {
    // The exact overcount #4107 fixes: exit 0 + a hook that never looked at the
    // output used to come out the other side as a recorded success.
    await recordTaskCompletion(skipAgent(), task);
    expect(savedData).toBeNull();

    // ...while an otherwise-identical run with the UNDECLARED sentinel still
    // records, so the skip can't have swallowed the ordinary null path.
    const undeclared = { ...skipAgent(), result: { success: true, validationPassed: null, duration: 60000 } };
    await recordTaskCompletion(undeclared, task);
    expect(atomicWrite).toHaveBeenCalledTimes(1);
    expect(savedData.byTaskType['internal-task'].succeeded).toBe(1);
    expect(savedData.byTaskType['internal-task'].failed).toBe(0);
  });

  it('does not blame the model either — a failing exit is skipped the same way', async () => {
    // `false` would have poisoned the #2329 failure-signature window with a
    // non-failure, so an aborted hook must not record a failure either.
    const failed = {
      ...skipAgent(),
      result: {
        success: false,
        validationPassed: SKIP_LEARNING_VERDICT,
        duration: 60000,
        errorAnalysis: { category: 'unknown', message: 'boom', origin: 'output-scan' }
      }
    };
    await recordTaskCompletion(failed, task);

    expect(atomicWrite).not.toHaveBeenCalled();
    expect(savedData).toBeNull();
  });

  it('still records a genuine criterion MISS as a failure (#2344 unchanged)', async () => {
    const missed = { ...skipAgent(), result: { success: true, validationPassed: false, duration: 60000 } };
    await recordTaskCompletion(missed, task);

    expect(atomicWrite).toHaveBeenCalledTimes(1);
    expect(savedData.byTaskType['internal-task'].succeeded).toBe(0);
    expect(savedData.byTaskType['internal-task'].failed).toBe(1);
    expect(savedData.byModelTier['heavy'].failed).toBe(1);
  });
});

describe('TaskLearning - environmental failures (#2618)', () => {
  let savedData;

  beforeEach(() => {
    vi.clearAllMocks();
    clearLearningCache();
    savedData = null;
    atomicWrite.mockImplementation(async (_path, data) => {
      savedData = data;
    });
  });

  const makeFailedAgent = (category, tier = 'heavy') => ({
    metadata: { modelTier: tier, taskDescription: 'Fix UI' },
    result: {
      success: false,
      duration: 30000,
      errorAnalysis: { category, message: `provider says ${category}` }
    }
  });
  const task = { description: 'Fix UI', taskType: 'user', metadata: {} };

  it('keeps a rate-limit failure out of every success-rate aggregate and counts it separately', async () => {
    const baseline = makeLearningData();
    readFile.mockResolvedValue(JSON.stringify(baseline));

    await recordTaskCompletion(makeFailedAgent('rate-limit'), task);

    // byTaskType untouched — same counters/rate as the baseline, no recency sample.
    expect(savedData.byTaskType['user-task']).toMatchObject({
      completed: baseline.byTaskType['user-task'].completed,
      succeeded: baseline.byTaskType['user-task'].succeeded,
      failed: baseline.byTaskType['user-task'].failed,
      successRate: baseline.byTaskType['user-task'].successRate
    });
    expect(savedData.byTaskType['user-task'].recentOutcomes).toBeUndefined();
    // byModelTier untouched — the heavy tier bucket is not even initialized.
    expect(savedData.byModelTier['heavy']).toBeUndefined();
    // routingAccuracy untouched.
    expect(savedData.routingAccuracy?.['user-task']).toBeUndefined();
    // totals untouched — the overall rate is an aggregate too.
    expect(savedData.totals).toMatchObject({
      completed: baseline.totals.completed,
      succeeded: baseline.totals.succeeded,
      failed: baseline.totals.failed
    });
    // Correlation window untouched — an outage grades no routing prediction.
    expect(savedData.correlationWindow ?? []).toEqual([]);

    // ...but the event stays visible in the separate environmental record.
    expect(savedData.environmentalFailures['rate-limit']).toMatchObject({
      count: 1,
      taskTypes: { 'user-task': 1 }
    });
    expect(savedData.environmentalFailures['rate-limit'].lastOccurred).toEqual(expect.any(String));
  });

  it('still records errorPatterns and failureSignatures for environmental categories (diagnostics)', async () => {
    readFile.mockResolvedValue(JSON.stringify(makeLearningData()));

    await recordTaskCompletion(makeFailedAgent('billing-error'), task);

    expect(savedData.errorPatterns['billing-error']).toMatchObject({
      count: 1,
      taskTypes: { 'user-task': 1 }
    });
    expect(savedData.failureSignatures['billing-error'].count).toBe(1);
    expect(savedData.failureSignatures['billing-error'].recent[0]).toMatchObject({
      taskType: 'user-task',
      messageSnippet: 'provider says billing-error'
    });
  });

  it('never creates a byTaskType bucket for a type whose only history is an outage', async () => {
    readFile.mockResolvedValue(JSON.stringify(makeLearningData()));

    const freshTask = { description: '[auto-fix] investigate the crash', taskType: 'internal', metadata: {} };
    await recordTaskCompletion(makeFailedAgent('startup-failure'), freshTask);

    expect(savedData.byTaskType['auto-fix']).toBeUndefined();
    expect(savedData.environmentalFailures['startup-failure'].taskTypes['auto-fix']).toBe(1);
  });

  it('accumulates environmental counts across completions', async () => {
    let currentData = JSON.stringify(makeLearningData());
    readFile.mockImplementation(async () => currentData);
    atomicWrite.mockImplementation(async (_path, data) => {
      currentData = JSON.stringify(data);
      savedData = data;
    });

    await recordTaskCompletion(makeFailedAgent('rate-limit'), task);
    await recordTaskCompletion(makeFailedAgent('rate-limit'), task);
    await recordTaskCompletion(makeFailedAgent('auth'), task);

    expect(savedData.environmentalFailures['rate-limit'].count).toBe(2);
    expect(savedData.environmentalFailures['auth'].count).toBe(1);
    expect(savedData.totals.failed).toBe(makeLearningData().totals.failed);
  });

  it('leaves a timeout failure counting against the aggregates exactly as today', async () => {
    const baseline = makeLearningData();
    readFile.mockResolvedValue(JSON.stringify(baseline));

    await recordTaskCompletion(makeFailedAgent('timeout', 'light'), task);

    // timeout is deliberately NOT environmental — it can be genuine task/model signal.
    expect(savedData.byTaskType['user-task'].failed).toBe(baseline.byTaskType['user-task'].failed + 1);
    expect(savedData.byModelTier['light'].failed).toBe(1);
    expect(savedData.routingAccuracy['user-task']['light'].failed).toBe(1);
    expect(savedData.totals.failed).toBe(baseline.totals.failed + 1);
    expect(savedData.environmentalFailures).toBeUndefined();
  });

  it('loads a learning.json without the environmentalFailures key cleanly (back-compat)', async () => {
    // makeLearningData has no environmentalFailures key — the additive key is
    // initialized on first environmental recording, and a plain success write
    // round-trips the file without fabricating one.
    readFile.mockResolvedValue(JSON.stringify(makeLearningData()));

    const agent = {
      metadata: { modelTier: 'heavy', taskDescription: 'Fix UI' },
      result: { success: true, duration: 30000 }
    };
    await recordTaskCompletion(agent, task);

    expect(savedData.environmentalFailures).toBeUndefined();
    expect(savedData.byTaskType['user-task'].succeeded)
      .toBe(makeLearningData().byTaskType['user-task'].succeeded + 1);
  });

  it('surfaces environmentalFailures via getLearningInsights, sorted by count', async () => {
    readFile.mockResolvedValue(JSON.stringify(makeLearningData({
      environmentalFailures: {
        'auth': { count: 2, lastOccurred: '2026-07-01T00:00:00.000Z', taskTypes: { 'user-task': 2 } },
        'rate-limit': { count: 7, lastOccurred: '2026-07-02T00:00:00.000Z', taskTypes: { 'user-task': 4, 'auto-fix': 3 } }
      }
    })));

    const insights = await getLearningInsights();

    expect(insights.insights.environmentalFailures).toEqual([
      { category: 'rate-limit', count: 7, lastOccurred: '2026-07-02T00:00:00.000Z', affectedTypes: ['user-task', 'auto-fix'] },
      { category: 'auth', count: 2, lastOccurred: '2026-07-01T00:00:00.000Z', affectedTypes: ['user-task'] }
    ]);
  });

  it('surfaces an empty environmentalFailures list for a learning.json that predates the key', async () => {
    readFile.mockResolvedValue(JSON.stringify(makeLearningData()));
    const insights = await getLearningInsights();
    expect(insights.insights.environmentalFailures).toEqual([]);
  });
});

describe('TaskLearning - getRoutingAccuracy', () => {
  beforeEach(() => {
    vi.clearAllMocks();
    clearLearningCache();
  });

  it('should return routing accuracy matrix with misroutes', async () => {
    const data = makeLearningData({
      routingAccuracy: {
        'self-improve:ui': {
          light: { succeeded: 1, failed: 9, lastAttempt: '2026-01-25T00:00:00.000Z' },
          heavy: { succeeded: 8, failed: 2, lastAttempt: '2026-01-26T00:00:00.000Z' }
        },
        'user-task': {
          medium: { succeeded: 15, failed: 3, lastAttempt: '2026-01-26T00:00:00.000Z' }
        }
      }
    });
    readFile.mockResolvedValue(JSON.stringify(data));

    const result = await getRoutingAccuracy();

    expect(result.matrix).toHaveLength(2);
    expect(result.totalMisroutes).toBe(1); // self-improve:ui on light (10% success, 10 attempts)

    // Check misroutes
    expect(result.misroutes).toHaveLength(1);
    expect(result.misroutes[0].taskType).toBe('self-improve:ui');
    expect(result.misroutes[0].tier).toBe('light');
    expect(result.misroutes[0].successRate).toBe(10);

    // Check tier overview
    expect(result.tierOverview.length).toBeGreaterThan(0);
  });

  it('should return empty results when no routing data exists', async () => {
    readFile.mockResolvedValue(JSON.stringify(makeLearningData()));

    const result = await getRoutingAccuracy();

    expect(result.matrix).toHaveLength(0);
    expect(result.misroutes).toHaveLength(0);
    expect(result.totalMisroutes).toBe(0);
  });

  it('should sort matrix tiers by success rate descending', async () => {
    const data = makeLearningData({
      routingAccuracy: {
        'user-task': {
          light: { succeeded: 1, failed: 4, lastAttempt: '2026-01-25T00:00:00.000Z' },
          medium: { succeeded: 3, failed: 2, lastAttempt: '2026-01-25T00:00:00.000Z' },
          heavy: { succeeded: 9, failed: 1, lastAttempt: '2026-01-26T00:00:00.000Z' }
        }
      }
    });
    readFile.mockResolvedValue(JSON.stringify(data));

    const result = await getRoutingAccuracy();

    const tiers = result.matrix[0].tiers;
    expect(tiers[0].tier).toBe('heavy');   // 90%
    expect(tiers[1].tier).toBe('medium');  // 60%
    expect(tiers[2].tier).toBe('light');   // 20%
  });

  it('reports the learned tier advice per task type, and omits it where history says nothing', async () => {
    const data = makeLearningData({
      byTaskType: {
        'self-improve:ui': {
          completed: 20, succeeded: 8, failed: 12,
          totalDurationMs: 2000000, avgDurationMs: 100000,
          successRate: 40
        }
      },
      routingAccuracy: {
        'self-improve:ui': {
          light: { succeeded: 1, failed: 9, lastAttempt: '2026-01-25T00:00:00.000Z' },
          heavy: { succeeded: 7, failed: 3, lastAttempt: '2026-01-26T00:00:00.000Z' }
        },
        'user-task': {
          medium: { succeeded: 2, failed: 0, lastAttempt: '2026-01-26T00:00:00.000Z' }
        }
      }
    });
    readFile.mockResolvedValue(JSON.stringify(data));

    const result = await getRoutingAccuracy();
    const byType = Object.fromEntries(result.matrix.map((entry) => [entry.taskType, entry]));

    expect(byType['self-improve:ui'].suggestion.avoidTiers).toContain('light');
    expect(byType['user-task'].suggestion).toBeUndefined();
  });

  it('advises only pinnable tiers: retired thinking-level names map to their tier, user-specified never appears', async () => {
    const outcome = { completed: 15, succeeded: 12, failed: 3, totalDurationMs: 1500000, avgDurationMs: 100000, successRate: 80 };
    const data = makeLearningData({
      byTaskType: { 'legacy-high': outcome, 'pinned-only': outcome },
      routingAccuracy: {
        'legacy-high': { high: { succeeded: 9, failed: 1, lastAttempt: '2026-01-26T00:00:00.000Z' } },
        'pinned-only': { 'user-specified': { succeeded: 9, failed: 1, lastAttempt: '2026-01-26T00:00:00.000Z' } }
      }
    });
    readFile.mockResolvedValue(JSON.stringify(data));

    const result = await getRoutingAccuracy();
    const byType = Object.fromEntries(result.matrix.map((entry) => [entry.taskType, entry]));

    expect(byType['legacy-high'].suggestion.tier).toBe('heavy');
    expect(byType['pinned-only'].suggestion).toBeUndefined();
    // History is kept verbatim — only the advice is normalized.
    expect(byType['legacy-high'].tiers[0].tier).toBe('high');
  });
});

describe('TaskLearning - suggestModelTier with routing signals', () => {
  beforeEach(() => {
    vi.clearAllMocks();
    clearLearningCache();
  });

  it('should suggest avoiding failing tiers', async () => {
    const data = makeLearningData({
      byTaskType: {
        'self-improve:ui': {
          completed: 20, succeeded: 8, failed: 12,
          totalDurationMs: 2000000, avgDurationMs: 100000,
          successRate: 40
        }
      },
      routingAccuracy: {
        'self-improve:ui': {
          light: { succeeded: 1, failed: 9, lastAttempt: '2026-01-25T00:00:00.000Z' },
          heavy: { succeeded: 7, failed: 3, lastAttempt: '2026-01-26T00:00:00.000Z' }
        }
      }
    });
    readFile.mockResolvedValue(JSON.stringify(data));

    const result = await suggestModelTier('self-improve:ui');

    expect(result).not.toBeNull();
    expect(result.avoidTiers).toContain('light');
  });

  it('should suggest best performing tier when available', async () => {
    const data = makeLearningData({
      byTaskType: {
        'user-task': {
          completed: 15, succeeded: 12, failed: 3,
          totalDurationMs: 1500000, avgDurationMs: 100000,
          successRate: 80
        }
      },
      routingAccuracy: {
        'user-task': {
          medium: { succeeded: 9, failed: 1, lastAttempt: '2026-01-26T00:00:00.000Z' },
          heavy: { succeeded: 3, failed: 2, lastAttempt: '2026-01-25T00:00:00.000Z' }
        }
      }
    });
    readFile.mockResolvedValue(JSON.stringify(data));

    const result = await suggestModelTier('user-task');

    expect(result).not.toBeNull();
    expect(result.suggested).toBe('medium'); // 90% success vs heavy at 60%
  });

  it('should return null when insufficient data', async () => {
    readFile.mockResolvedValue(JSON.stringify(makeLearningData({
      byTaskType: {
        'new-task': { completed: 2, succeeded: 1, failed: 1, successRate: 50 }
      }
    })));

    const result = await suggestModelTier('new-task');
    expect(result).toBeNull();
  });

  it('steers off a freshly-failing tier even below the completions threshold (#2329)', async () => {
    // Only 4 completions — too few for a routingAccuracy-based tier suggestion —
    // but 3 recent failures on 'heavy' meet the failure-signal bar, so routing
    // still emits an avoidance-only suggestion instead of null.
    const data = makeLearningData({
      byTaskType: {
        'fresh-task': {
          completed: 4, succeeded: 1, failed: 3,
          totalDurationMs: 400000, avgDurationMs: 100000, successRate: 25
        }
      },
      routingAccuracy: {},
      correlationWindow: provenCorrelationWindow(),
      failureSignatures: {
        'test-failure': {
          count: 3,
          recent: [
            { modelTier: 'heavy', provider: 'claude', model: 'opus', taskType: 'fresh-task' },
            { modelTier: 'heavy', provider: 'claude', model: 'opus', taskType: 'fresh-task' },
            { modelTier: 'heavy', provider: 'claude', model: 'opus', taskType: 'fresh-task' }
          ]
        }
      }
    });
    readFile.mockResolvedValue(JSON.stringify(data));

    const result = await suggestModelTier('fresh-task');
    expect(result).not.toBeNull();
    expect(result.suggested).toBeNull();
    expect(result.avoidTiers).toContain('heavy');
    expect(result.failureSignal).toMatchObject({ tier: 'heavy', failures: 3 });
  });

  it('still returns null below the completions threshold when there is no failure signal (#2329)', async () => {
    readFile.mockResolvedValue(JSON.stringify(makeLearningData({
      byTaskType: {
        'quiet-task': { completed: 3, succeeded: 2, failed: 1, successRate: 66 }
      },
      routingAccuracy: {},
      failureSignatures: {}
    })));
    expect(await suggestModelTier('quiet-task')).toBeNull();
  });

  it('folds enriched failure signatures into avoidTiers even when routingAccuracy is quiet (#2329)', async () => {
    // Overall success is healthy (75%) and routingAccuracy carries no failing
    // tier — pre-#2329 this returned null. The failure signatures show three
    // recent 'heavy'-tier failures for this task type, so routing now emits an
    // avoidance-only suggestion that steers selection off 'heavy'.
    const data = makeLearningData({
      byTaskType: {
        'sig-task': {
          completed: 12, succeeded: 9, failed: 3,
          totalDurationMs: 1200000, avgDurationMs: 100000, successRate: 75
        }
      },
      routingAccuracy: {},
      correlationWindow: provenCorrelationWindow(),
      failureSignatures: {
        'test-failure': {
          count: 3,
          recent: [
            { modelTier: 'heavy', provider: 'claude', model: 'opus', taskType: 'sig-task' },
            { modelTier: 'heavy', provider: 'claude', model: 'opus', taskType: 'sig-task' },
            { modelTier: 'heavy', provider: 'codex', model: 'gpt', taskType: 'sig-task' }
          ]
        }
      }
    });
    readFile.mockResolvedValue(JSON.stringify(data));

    const result = await suggestModelTier('sig-task');
    expect(result).not.toBeNull();
    expect(result.suggested).toBeNull();
    expect(result.avoidTiers).toContain('heavy');
    expect(result.failureSignal).toMatchObject({ tier: 'heavy', failures: 3, provider: 'claude' });
  });

  it('drops to avoidance-only when the tier it would pick is itself failure-flagged (#2329)', async () => {
    // routingAccuracy: heavy is failing (<40%) and 'medium' is the only 60-79%
    // "successful" alternative — but the failure signatures show 3 recent medium
    // failures too. Routing must NOT then suggest medium; it drops to avoidance-
    // only (suggested: null) so selection steers off both heavy and medium.
    const data = makeLearningData({
      byTaskType: {
        'flap-task': {
          completed: 20, succeeded: 9, failed: 11,
          totalDurationMs: 2000000, avgDurationMs: 100000, successRate: 45
        }
      },
      routingAccuracy: {
        'flap-task': {
          heavy: { succeeded: 1, failed: 9, lastAttempt: '2026-01-26T00:00:00.000Z' }, // 10%
          medium: { succeeded: 7, failed: 3, lastAttempt: '2026-01-26T00:00:00.000Z' } // 70% (unproven)
        }
      },
      correlationWindow: provenCorrelationWindow(),
      failureSignatures: {
        'test-failure': {
          count: 3,
          recent: [
            { modelTier: 'medium', provider: 'claude', model: 'sonnet', taskType: 'flap-task' },
            { modelTier: 'medium', provider: 'claude', model: 'sonnet', taskType: 'flap-task' },
            { modelTier: 'medium', provider: 'claude', model: 'sonnet', taskType: 'flap-task' }
          ]
        }
      }
    });
    readFile.mockResolvedValue(JSON.stringify(data));

    const result = await suggestModelTier('flap-task');
    expect(result.suggested).toBeNull();
    expect(result.avoidTiers).toEqual(expect.arrayContaining(['heavy', 'medium']));
    expect(result.failureSignal).toMatchObject({ tier: 'medium', provider: 'claude', model: 'sonnet' });
  });

  it('holds avoidance back to the conservative bar until correlation is proven (#2344)', async () => {
    // 3 recent 'heavy' failures — enough for the AGGRESSIVE bar (3) — but the
    // correlation window is empty (unproven), so the #2344 gate applies the
    // conservative bar (5) and does NOT steer off 'heavy' yet.
    const fixture = (correlationWindow) => makeLearningData({
      byTaskType: {
        'gate-task': {
          completed: 12, succeeded: 9, failed: 3,
          totalDurationMs: 1200000, avgDurationMs: 100000, successRate: 75
        }
      },
      routingAccuracy: {},
      correlationWindow,
      failureSignatures: {
        'test-failure': {
          count: 3,
          recent: [
            { modelTier: 'heavy', provider: 'claude', model: 'opus', taskType: 'gate-task' },
            { modelTier: 'heavy', provider: 'claude', model: 'opus', taskType: 'gate-task' },
            { modelTier: 'heavy', provider: 'claude', model: 'opus', taskType: 'gate-task' }
          ]
        }
      }
    });

    // Unproven correlation (empty window) → no avoidance despite 3 failures.
    readFile.mockResolvedValue(JSON.stringify(fixture([])));
    clearLearningCache();
    expect(await suggestModelTier('gate-task')).toBeNull();

    // Proven correlation (>0.8, >= min samples) → the same 3 failures now steer.
    readFile.mockResolvedValue(JSON.stringify(fixture(provenCorrelationWindow())));
    clearLearningCache();
    const proven = await suggestModelTier('gate-task');
    expect(proven).not.toBeNull();
    expect(proven.avoidTiers).toContain('heavy');
  });

  it('does not let failure signatures condemn a proven tier (#2329)', async () => {
    // 'heavy' clears the high-success threshold in routingAccuracy, so the
    // proven-tier suggestion stands and its absolute failures never add it to
    // avoidTiers.
    const data = makeLearningData({
      byTaskType: {
        'proven-task': {
          completed: 20, succeeded: 18, failed: 2,
          totalDurationMs: 2000000, avgDurationMs: 100000, successRate: 90
        }
      },
      routingAccuracy: {
        'proven-task': {
          heavy: { succeeded: 18, failed: 2, lastAttempt: '2026-01-26T00:00:00.000Z' } // 90%
        }
      },
      failureSignatures: {
        'test-failure': {
          count: 3,
          recent: [
            { modelTier: 'heavy', provider: 'claude', model: 'opus', taskType: 'proven-task' },
            { modelTier: 'heavy', provider: 'claude', model: 'opus', taskType: 'proven-task' },
            { modelTier: 'heavy', provider: 'claude', model: 'opus', taskType: 'proven-task' }
          ]
        }
      }
    });
    readFile.mockResolvedValue(JSON.stringify(data));

    const result = await suggestModelTier('proven-task');
    expect(result.suggested).toBe('heavy');
    expect(result.avoidTiers).not.toContain('heavy');
  });

  it('should prefer the lightest tier when several clear the high-success threshold', async () => {
    // heavy has the marginally-higher success rate, but light also clears 80% —
    // we should route to light to avoid spending a heavier model for the same outcome.
    const data = makeLearningData({
      byTaskType: {
        'docs-task': {
          completed: 20, succeeded: 17, failed: 3,
          totalDurationMs: 2000000, avgDurationMs: 100000,
          successRate: 85
        }
      },
      routingAccuracy: {
        'docs-task': {
          light: { succeeded: 9, failed: 2, lastAttempt: '2026-01-26T00:00:00.000Z' }, // 82%
          heavy: { succeeded: 8, failed: 1, lastAttempt: '2026-01-25T00:00:00.000Z' }  // 89%
        }
      }
    });
    readFile.mockResolvedValue(JSON.stringify(data));

    const result = await suggestModelTier('docs-task');

    expect(result).not.toBeNull();
    expect(result.suggested).toBe('light');
    expect(result.reason).toMatch(/lightest of 2 proven tiers/);
  });

  it('should rank routable thinking-level tier names by real cost, not as default-weight', async () => {
    // routingAccuracy keys can be thinking-level names (high/xhigh), not just
    // light/default/medium/heavy. xhigh (opus) is heavier than high (provider-
    // heavy). Both clear 80%, so we must pick high — an earlier bug ranked every
    // unmapped name at default-weight, which would have let xhigh win the tie.
    const data = makeLearningData({
      byTaskType: {
        'reason-task': {
          completed: 20, succeeded: 18, failed: 2,
          totalDurationMs: 2000000, avgDurationMs: 100000,
          successRate: 90
        }
      },
      routingAccuracy: {
        'reason-task': {
          high: { succeeded: 9, failed: 2, lastAttempt: '2026-01-26T00:00:00.000Z' },  // 82%
          xhigh: { succeeded: 10, failed: 1, lastAttempt: '2026-01-25T00:00:00.000Z' }  // 91%
        }
      }
    });
    readFile.mockResolvedValue(JSON.stringify(data));

    const result = await suggestModelTier('reason-task');

    expect(result).not.toBeNull();
    expect(result.suggested).toBe('high');
  });

  it('should not suggest a non-routable local-preferred tier (minimal/low) — picks the lightest tier the runtime can run', async () => {
    // minimal/low resolve to a local model, but the learning→selection path
    // doesn't switch providers for them, so suggesting minimal would make the
    // selector fall through to the provider default. Even though minimal is the
    // cheapest by cost, it must be excluded so the proven `light` tier wins.
    const data = makeLearningData({
      byTaskType: {
        'local-task': {
          completed: 20, succeeded: 18, failed: 2,
          totalDurationMs: 2000000, avgDurationMs: 100000,
          successRate: 90
        }
      },
      routingAccuracy: {
        'local-task': {
          minimal: { succeeded: 10, failed: 1, lastAttempt: '2026-01-26T00:00:00.000Z' }, // 91%
          light: { succeeded: 9, failed: 2, lastAttempt: '2026-01-25T00:00:00.000Z' }      // 82%
        }
      }
    });
    readFile.mockResolvedValue(JSON.stringify(data));

    const result = await suggestModelTier('local-task');

    expect(result).not.toBeNull();
    expect(result.suggested).toBe('light');
  });

  it('should not prefer an unknown tier name as the cheapest', async () => {
    // user-specified (and any future/unknown tier name) must rank as heaviest,
    // so it never wins the "lightest proven tier" tie over a known-light tier.
    const data = makeLearningData({
      byTaskType: {
        'pref-task': {
          completed: 20, succeeded: 18, failed: 2,
          totalDurationMs: 2000000, avgDurationMs: 100000,
          successRate: 90
        }
      },
      routingAccuracy: {
        'pref-task': {
          'user-specified': { succeeded: 10, failed: 1, lastAttempt: '2026-01-26T00:00:00.000Z' }, // 91%
          light: { succeeded: 9, failed: 2, lastAttempt: '2026-01-25T00:00:00.000Z' }              // 82%
        }
      }
    });
    readFile.mockResolvedValue(JSON.stringify(data));

    const result = await suggestModelTier('pref-task');

    expect(result).not.toBeNull();
    expect(result.suggested).toBe('light');
  });
});

describe('TaskLearning - recalculateModelTierMetrics', () => {
  let savedData;

  beforeEach(() => {
    vi.clearAllMocks();
    clearLearningCache();
    savedData = null;
    atomicWrite.mockImplementation(async (_path, data) => {
      savedData = data;
    });
  });

  it('should rebuild byModelTier from routingAccuracy data', async () => {
    const data = makeLearningData({
      byModelTier: {
        heavy: {
          completed: 650, succeeded: 3, failed: 647,
          totalDurationMs: 19500000, avgDurationMs: 30000
        },
        medium: {
          completed: 200, succeeded: 150, failed: 50,
          totalDurationMs: 40000000, avgDurationMs: 200000
        }
      },
      byTaskType: {
        'user-task': {
          completed: 40, succeeded: 30, failed: 10,
          totalDurationMs: 4000000, avgDurationMs: 100000,
          successRate: 75
        },
        'internal-task': {
          completed: 20, succeeded: 18, failed: 2,
          totalDurationMs: 2000000, avgDurationMs: 100000,
          successRate: 90
        }
      },
      routingAccuracy: {
        'user-task': {
          medium: { succeeded: 25, failed: 5, lastAttempt: '2026-01-26T00:00:00.000Z' },
          low: { succeeded: 8, failed: 0, lastAttempt: '2026-01-26T00:00:00.000Z' }
        },
        'internal-task': {
          medium: { succeeded: 15, failed: 1, lastAttempt: '2026-01-26T00:00:00.000Z' }
        }
      }
    });
    readFile.mockResolvedValue(JSON.stringify(data));

    const result = await recalculateModelTierMetrics();

    expect(result.recalculated).toBe(true);
    expect(result.changes.length).toBeGreaterThan(0);

    // heavy tier should be gone (not in routingAccuracy)
    expect(savedData.byModelTier.heavy).toBeUndefined();

    // medium tier should reflect routing accuracy: 25+15=40 succeeded, 5+1=6 failed
    expect(savedData.byModelTier.medium.completed).toBe(46);
    expect(savedData.byModelTier.medium.succeeded).toBe(40);
    expect(savedData.byModelTier.medium.failed).toBe(6);

    // low tier: 8 succeeded, 0 failed
    expect(savedData.byModelTier.low.completed).toBe(8);
    expect(savedData.byModelTier.low.succeeded).toBe(8);
    expect(savedData.byModelTier.low.failed).toBe(0);
  });

  it('should not save when nothing changes', async () => {
    const data = makeLearningData({
      byModelTier: {
        medium: { completed: 3, succeeded: 2, failed: 1, totalDurationMs: 300000, avgDurationMs: 100000 }
      },
      routingAccuracy: {
        'user-task': {
          medium: { succeeded: 2, failed: 1, lastAttempt: '2026-01-26T00:00:00.000Z' }
        }
      }
    });
    readFile.mockResolvedValue(JSON.stringify(data));

    const result = await recalculateModelTierMetrics();

    expect(result.recalculated).toBe(false);
    expect(atomicWrite).not.toHaveBeenCalled();
  });

  it('should handle empty routingAccuracy', async () => {
    const data = makeLearningData({
      byModelTier: {
        heavy: { completed: 100, succeeded: 1, failed: 99, totalDurationMs: 100000, avgDurationMs: 1000 }
      },
      routingAccuracy: {}
    });
    readFile.mockResolvedValue(JSON.stringify(data));

    const result = await recalculateModelTierMetrics();

    expect(result.recalculated).toBe(true);
    // All tiers should be cleared since no routing accuracy data exists
    expect(savedData.byModelTier).toEqual({});
  });

  it('should estimate durations from task type averages', async () => {
    const data = makeLearningData({
      byModelTier: {},
      byTaskType: {
        'user-task': {
          completed: 10, succeeded: 8, failed: 2,
          totalDurationMs: 1000000, avgDurationMs: 100000,
          successRate: 80
        }
      },
      routingAccuracy: {
        'user-task': {
          medium: { succeeded: 5, failed: 1, lastAttempt: '2026-01-26T00:00:00.000Z' }
        }
      }
    });
    readFile.mockResolvedValue(JSON.stringify(data));

    await recalculateModelTierMetrics();

    // 6 agents × 100000ms avg = 600000ms total, 600000/6 = 100000ms avg
    expect(savedData.byModelTier.medium.totalDurationMs).toBe(600000);
    expect(savedData.byModelTier.medium.avgDurationMs).toBe(100000);
  });
});

describe('TaskLearning - getTaskTypeConfidence', () => {
  beforeEach(() => {
    vi.clearAllMocks();
    clearLearningCache();
  });

  it('should return "new" tier for unknown task type with no data', async () => {
    const data = makeLearningData({ byTaskType: {} });
    readFile.mockResolvedValue(JSON.stringify(data));

    const result = await getTaskTypeConfidence('unknown-type');
    expect(result.tier).toBe('new');
    expect(result.autoApprove).toBe(true);
    expect(result.completed).toBe(0);
    expect(result.successRate).toBeNull();
  });

  it('should return "new" tier when completed < minSamples (default 5)', async () => {
    const data = makeLearningData({
      byTaskType: { 'sparse-type': { completed: 3, succeeded: 3, failed: 0, successRate: 100 } }
    });
    readFile.mockResolvedValue(JSON.stringify(data));

    const result = await getTaskTypeConfidence('sparse-type');
    expect(result.tier).toBe('new');
    expect(result.autoApprove).toBe(true);
  });

  it('should return "high" tier when successRate >= 80 and enough samples', async () => {
    const data = makeLearningData({
      byTaskType: { 'reliable-type': { completed: 20, succeeded: 18, failed: 2, successRate: 90 } }
    });
    readFile.mockResolvedValue(JSON.stringify(data));

    const result = await getTaskTypeConfidence('reliable-type');
    expect(result.tier).toBe('high');
    expect(result.autoApprove).toBe(true);
    expect(result.successRate).toBe(90);
  });

  it('should return "medium" tier when successRate is between 50 and 80', async () => {
    const data = makeLearningData({
      byTaskType: { 'ok-type': { completed: 10, succeeded: 6, failed: 4, successRate: 60 } }
    });
    readFile.mockResolvedValue(JSON.stringify(data));

    const result = await getTaskTypeConfidence('ok-type');
    expect(result.tier).toBe('medium');
    expect(result.autoApprove).toBe(true);
  });

  it('should return "low" tier when successRate < 50 and enough samples', async () => {
    const data = makeLearningData({
      byTaskType: { 'flaky-type': { completed: 10, succeeded: 3, failed: 7, successRate: 30 } }
    });
    readFile.mockResolvedValue(JSON.stringify(data));

    const result = await getTaskTypeConfidence('flaky-type');
    expect(result.tier).toBe('low');
    expect(result.autoApprove).toBe(false);
  });

  it('should respect threshold overrides', async () => {
    const data = makeLearningData({
      byTaskType: { 'borderline': { completed: 10, succeeded: 7, failed: 3, successRate: 70 } }
    });
    readFile.mockResolvedValue(JSON.stringify(data));

    // With default thresholds (high=80), 70% is medium
    const defaultResult = await getTaskTypeConfidence('borderline');
    expect(defaultResult.tier).toBe('medium');

    clearLearningCache();
    readFile.mockResolvedValue(JSON.stringify(data));

    // With lowered highThreshold=60, 70% becomes high
    const overrideResult = await getTaskTypeConfidence('borderline', { highThreshold: 60 });
    expect(overrideResult.tier).toBe('high');
  });

  it('should respect minSamples override', async () => {
    const data = makeLearningData({
      byTaskType: { 'small-sample': { completed: 3, succeeded: 3, failed: 0, successRate: 100 } }
    });
    readFile.mockResolvedValue(JSON.stringify(data));

    // With minSamples=2, 3 completions is enough to classify
    const result = await getTaskTypeConfidence('small-sample', { minSamples: 2 });
    expect(result.tier).toBe('high');
    expect(result.autoApprove).toBe(true);
  });
});

describe('TaskLearning - getConfidenceLevels', () => {
  beforeEach(() => {
    vi.clearAllMocks();
    clearLearningCache();
  });

  it('should group task types into correct tiers', async () => {
    const data = makeLearningData({
      byTaskType: {
        'high-type':   { completed: 10, succeeded: 9, failed: 1, successRate: 90 },
        'medium-type': { completed: 10, succeeded: 6, failed: 4, successRate: 60 },
        'low-type':    { completed: 10, succeeded: 3, failed: 7, successRate: 30 },
        'new-type':    { completed: 2,  succeeded: 2, failed: 0, successRate: 100 }
      }
    });
    readFile.mockResolvedValue(JSON.stringify(data));

    const result = await getConfidenceLevels();
    expect(result.levels.high.map(t => t.taskType)).toContain('high-type');
    expect(result.levels.medium.map(t => t.taskType)).toContain('medium-type');
    expect(result.levels.low.map(t => t.taskType)).toContain('low-type');
    expect(result.levels.new.map(t => t.taskType)).toContain('new-type');
  });

  it('should return correct summary counts', async () => {
    const data = makeLearningData({
      byTaskType: {
        'h1': { completed: 10, succeeded: 9, failed: 1, successRate: 90 },
        'h2': { completed: 10, succeeded: 8, failed: 2, successRate: 80 },
        'm1': { completed: 10, succeeded: 6, failed: 4, successRate: 60 },
        'l1': { completed: 10, succeeded: 3, failed: 7, successRate: 30 }
      }
    });
    readFile.mockResolvedValue(JSON.stringify(data));

    const result = await getConfidenceLevels();
    expect(result.summary.high).toBe(2);
    expect(result.summary.medium).toBe(1);
    expect(result.summary.low).toBe(1);
    expect(result.summary.total).toBe(4);
    expect(result.summary.requireApproval).toBe(1);
  });

  it('should sort tiers by successRate descending', async () => {
    const data = makeLearningData({
      byTaskType: {
        'a': { completed: 10, succeeded: 8, failed: 2, successRate: 80 },
        'b': { completed: 10, succeeded: 9, failed: 1, successRate: 90 },
        'c': { completed: 10, succeeded: 8, failed: 2, successRate: 80 }
      }
    });
    readFile.mockResolvedValue(JSON.stringify(data));

    const result = await getConfidenceLevels();
    const highRates = result.levels.high.map(t => t.successRate);
    expect(highRates).toEqual([...highRates].sort((a, b) => b - a));
  });

  it('should reflect threshold overrides in returned thresholds', async () => {
    const data = makeLearningData({ byTaskType: {} });
    readFile.mockResolvedValue(JSON.stringify(data));

    const result = await getConfidenceLevels({ highThreshold: 90, lowThreshold: 60, minSamples: 3 });
    expect(result.thresholds).toEqual({ highThreshold: 90, lowThreshold: 60, minSamples: 3 });
  });

  it('should return empty levels when no task types exist', async () => {
    const data = makeLearningData({ byTaskType: {} });
    readFile.mockResolvedValue(JSON.stringify(data));

    const result = await getConfidenceLevels();
    expect(result.summary.total).toBe(0);
    expect(result.levels.high).toHaveLength(0);
    expect(result.levels.low).toHaveLength(0);
  });
});

describe('TaskLearning - recommendation dismissal', () => {
  let writes;

  // Route reads by file path so learning.json and dismissed-recommendations.json
  // can return different content within a single test.
  const setupFiles = ({ learning = {}, dismissed = {} } = {}) => {
    readFile.mockImplementation(async (filePath) => {
      if (String(filePath).endsWith('dismissed-recommendations.json')) {
        return JSON.stringify(dismissed);
      }
      return JSON.stringify(learning);
    });
  };

  beforeEach(() => {
    vi.clearAllMocks();
    clearLearningCache();
    writes = {};
    atomicWrite.mockImplementation(async (filePath, data) => {
      writes[String(filePath)] = data;
    });
  });

  it('dismissRecommendation persists id with snapshot and timestamp', async () => {
    setupFiles();

    const result = await dismissRecommendation('error-pattern:unknown', { kind: 'count', value: 74 });

    expect(result).toEqual({ id: 'error-pattern:unknown', dismissed: true });
    const saved = Object.entries(writes).find(([p]) => p.endsWith('dismissed-recommendations.json'))?.[1];
    expect(saved).toBeDefined();
    expect(saved['error-pattern:unknown']).toMatchObject({
      snapshot: { kind: 'count', value: 74 }
    });
    expect(saved['error-pattern:unknown'].dismissedAt).toBeDefined();
  });

  it('restoreRecommendation removes the id from the dismissed map', async () => {
    setupFiles({
      dismissed: {
        'error-pattern:unknown': { dismissedAt: '2026-05-07T00:00:00.000Z', snapshot: { kind: 'count', value: 74 } }
      }
    });

    const result = await restoreRecommendation('error-pattern:unknown');

    expect(result).toEqual({ id: 'error-pattern:unknown', restored: true });
    const saved = Object.entries(writes).find(([p]) => p.endsWith('dismissed-recommendations.json'))?.[1];
    expect(saved['error-pattern:unknown']).toBeUndefined();
  });

  it('restoreRecommendation reports false when id was not dismissed', async () => {
    setupFiles({ dismissed: {} });

    const result = await restoreRecommendation('error-pattern:unknown');

    expect(result).toEqual({ id: 'error-pattern:unknown', restored: false });
  });

  it('clearDismissedRecommendations writes an empty map', async () => {
    setupFiles({
      dismissed: {
        'a': { dismissedAt: '2026-05-07T00:00:00.000Z' },
        'b': { dismissedAt: '2026-05-07T01:00:00.000Z' }
      }
    });

    const result = await clearDismissedRecommendations();

    expect(result).toEqual({ cleared: true });
    const saved = Object.entries(writes).find(([p]) => p.endsWith('dismissed-recommendations.json'))?.[1];
    expect(saved).toEqual({});
  });

  it('getDismissedRecommendations sorts by dismissedAt desc', async () => {
    setupFiles({
      dismissed: {
        'older': { dismissedAt: '2026-05-01T00:00:00.000Z' },
        'newer': { dismissedAt: '2026-05-07T00:00:00.000Z' }
      }
    });

    const list = await getDismissedRecommendations();
    expect(list.map(d => d.id)).toEqual(['newer', 'older']);
  });

  it('getLearningInsights filters dismissed rate-based recommendation permanently', async () => {
    const learning = makeLearningData({
      byTaskType: {
        // 100% success rate — would generate top-perf optimization recommendation
        'self-improve:accessibility': {
          completed: 50, succeeded: 50, failed: 0,
          totalDurationMs: 500000, avgDurationMs: 10000,
          lastCompleted: '2026-05-07T00:00:00.000Z', successRate: 100
        }
      },
      errorPatterns: {},
      byModelTier: {},
      totals: { completed: 50, succeeded: 50, failed: 0, totalDurationMs: 500000, avgDurationMs: 10000 }
    });
    setupFiles({
      learning,
      dismissed: {
        'top-perf:self-improve:accessibility': {
          dismissedAt: '2026-05-07T00:00:00.000Z',
          snapshot: { kind: 'rate', value: 100 }
        }
      }
    });

    const insights = await getLearningInsights();
    const ids = (insights.recommendations || []).map(r => r.id);
    expect(ids).not.toContain('top-perf:self-improve:accessibility');
  });

  it('getLearningInsights re-surfaces a count-based dismissal when the count grows past threshold', async () => {
    const learning = makeLearningData({
      byTaskType: {},
      errorPatterns: {
        'unknown': {
          count: 200, // grew significantly past the snapshot of 74
          taskTypes: { 'self-improve:ui': 200 },
          lastOccurred: '2026-05-08T00:00:00.000Z'
        }
      },
      byModelTier: {},
      totals: { completed: 200, succeeded: 0, failed: 200, totalDurationMs: 0, avgDurationMs: 0 }
    });
    setupFiles({
      learning,
      dismissed: {
        'error-pattern:unknown': {
          dismissedAt: '2026-04-01T00:00:00.000Z',
          snapshot: { kind: 'count', value: 74 }
        }
      }
    });

    const insights = await getLearningInsights();
    const ids = (insights.recommendations || []).map(r => r.id);
    expect(ids).toContain('error-pattern:unknown');
  });

  it('getLearningInsights keeps a count-based dismissal suppressed when count is unchanged', async () => {
    const learning = makeLearningData({
      byTaskType: {},
      errorPatterns: {
        'unknown': {
          count: 80, // only marginally higher than dismissal snapshot of 74
          taskTypes: { 'self-improve:ui': 80 },
          lastOccurred: '2026-05-08T00:00:00.000Z'
        }
      },
      byModelTier: {},
      totals: { completed: 80, succeeded: 0, failed: 80, totalDurationMs: 0, avgDurationMs: 0 }
    });
    setupFiles({
      learning,
      dismissed: {
        'error-pattern:unknown': {
          dismissedAt: '2026-04-01T00:00:00.000Z',
          snapshot: { kind: 'count', value: 74 }
        }
      }
    });

    const insights = await getLearningInsights();
    const ids = (insights.recommendations || []).map(r => r.id);
    expect(ids).not.toContain('error-pattern:unknown');
  });

  it('getLearningInsights gives every recommendation a stable id', async () => {
    const learning = makeLearningData({
      byTaskType: {
        'self-improve:accessibility': {
          completed: 50, succeeded: 50, failed: 0,
          totalDurationMs: 500000, avgDurationMs: 10000,
          lastCompleted: '2026-05-07T00:00:00.000Z', successRate: 100
        }
      },
      errorPatterns: {
        'unknown': { count: 74, taskTypes: { 'self-improve:ui': 74 }, lastOccurred: '2026-05-07T00:00:00.000Z' }
      },
      byModelTier: {},
      totals: { completed: 50, succeeded: 50, failed: 0, totalDurationMs: 500000, avgDurationMs: 10000 }
    });
    setupFiles({ learning, dismissed: {} });

    const insights = await getLearningInsights();
    expect(insights.recommendations.length).toBeGreaterThan(0);
    for (const rec of insights.recommendations) {
      expect(rec.id).toBeTruthy();
      expect(typeof rec.id).toBe('string');
    }
  });
});