Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
/**
* Task Learning — heuristic routing & scheduling decisions
*
* The "read" side of the learning data that drives runtime decisions:
* priority multipliers, advisory model-tier suggestions (reported in the CoS
* Learning view, never applied at dispatch — #8149), routing-accuracy matrices,
* adaptive cooldowns, skip/rehabilitation gating, and per-task-type
* confidence tiers. None of these mutate the store except the
* rehabilitation path, which delegates the actual reset to the metrics
* module.
*/
import { loadLearningData, emitLog, isSandboxedTaskType, computeEffectiveSuccessRate, computeWindowedStats, isSkipCandidate, DEFAULT_WINDOW_MAX_AGE_MS, ENVIRONMENTAL_ERROR_CATEGORIES } from './store.js';
import { resetTaskTypeLearning } from './reset.js';
import { computeCorrelationQuality, isCorrelationProven } from './correlationQuality.js';
/**
* Relative resource cost of each tier name that can land in
* `routingAccuracy`, lightest → heaviest. Used to break ties between tiers
* that both clear the high-success threshold: prefer the cheapest tier that
* still works well.
*
* The keys here are whatever `selectModelForTask` recorded as `tier`
* (agentModelSelection.js → agentLifecycle.js `modelTier`). Today that is a
* literal tier (`light`/`default`/`medium`/`heavy`/`ultra`) or
* `user-specified`, but learning data recorded before #8149 also carries the
* retired thinking-level names (`off`/`minimal`/`low`/`medium`/`high`/
* `xhigh`, where `minimal`/`low` were local-preferred and therefore the
* cheapest, and `high`/`xhigh` mapped to the provider's heavy model). An unknown name
* (e.g. `user-specified`, or a future tier) must NOT be treated as cheap, or
* the "prefer lightest" logic would pick it over a known-light tier — so the
* fallback ranks unknowns as heaviest.
*/
const TIER_WEIGHT = {
minimal: 0, // local-small (thinking level)
low: 1, // local-medium (thinking level)
light: 2, // provider light model (e.g. haiku)
off: 3, // no extended thinking → provider default
default: 3, // provider default model
medium: 4, // provider default + standard cloud thinking
high: 5, // provider-heavy (thinking level)
heavy: 6, // provider heavy model
xhigh: 7 // opus / heaviest (thinking level)
};
const HEAVIEST_WEIGHT = Math.max(...Object.values(TIER_WEIGHT)) + 1;
const tierWeight = (tier) => TIER_WEIGHT[tier] ?? HEAVIEST_WEIGHT;
/**
* Tier names the learning→selection path can't actually route to. The
* local-preferred thinking levels (`minimal`/`low`) resolve to a local model
* (LM Studio/Ollama), but `selectModelForTask` doesn't switch the active
* provider for them (the `localPreferred` flag is unwired) — so a task can't
* be routed there from a learned suggestion. Excluding them from suggestion
* candidates is what keeps "prefer the lightest tier" honest: recommending a
* tier nothing can run would make the selector fall through to the provider
* default instead of the lightest tier it CAN run (e.g. `light`). They stay in
* TIER_WEIGHT (the cost ordering is still accurate) — they're just not offered.
*/
// Ultra is an explicit capability request, never an automatic learned upgrade.
const NON_ROUTABLE_LEARNED_TIERS = new Set(['minimal', 'low', 'ultra']);
/**
* True for a learned tier the selection path can't actually route to
* (`minimal`/`low`). Exported so the correlation window (metrics.js) can skip
* these the same way `deriveFailureSignalAvoidance` does — recording a
* never-flaggable tier as "predicted safe" would skew the global gauge (#2344).
*/
export const isNonRoutableLearnedTier = (tier) => NON_ROUTABLE_LEARNED_TIERS.has(tier);
/**
* A recorded tier name → the tier a user can actually pin today, or null.
* Learning data from before #8149 carries the retired thinking-level names;
* pinning one (`high`) would be read as a literal model id, so advice maps them
* onto the tier they resolved to (`off` → the provider default, `high`/`xhigh`
* → heavy). `user-specified` and the local-only levels name no pinnable tier,
* so they never appear as advice — their history stays in the matrix.
*/
const ADVISORY_TIERS = Object.freeze({
light: 'light', medium: 'medium', heavy: 'heavy', ultra: 'ultra', default: 'default',
off: 'default', high: 'heavy', xhigh: 'heavy',
});
const advisoryTier = (tier) => (Object.hasOwn(ADVISORY_TIERS, tier) ? ADVISORY_TIERS[tier] : null);
/** Minimum success rate (%) for a tier to count as "proven" for a task type. */
const HIGH_SUCCESS_THRESHOLD = 80;
/**
* Minimum recent failure samples on a single tier (scoped to a task type) before
* the enriched failure signatures can steer routing away from it. Matches the
* routingAccuracy "≥3 attempts" bar so a one-off failure never condemns a tier.
* This is the AGGRESSIVE bar, used once the correlation-quality window has proven
* the signal predicts outcomes well (>0.8, issue #2344).
*/
const MIN_FAILURE_SAMPLES = 3;
/**
* Conservative failure-sample bar used until the correlation-quality window
* proves the enriched signal actually predicts bad outcomes (issue #2344). While
* the signal is unproven, require MORE recent failures before steering off a
* tier, so the system doesn't over-correct on a signal that hasn't earned it.
*/
const CONSERVATIVE_MIN_FAILURE_SAMPLES = 5;
/** NUL separator joining provider+model into one tally key (safe: neither contains NUL). */
const PAIR_SEP = '\x00';
/** Key with the highest count in a `{ key: count }` tally (null when empty). */
function topKey(counts) {
let best = null;
let bestCount = -1;
for (const [key, count] of Object.entries(counts)) {
if (count > bestCount) { best = key; bestCount = count; }
}
return best;
}
/**
* Consume the enriched failure signatures (issue #2329) to derive a
* recency-weighted tier avoidance for a task type. Pure — no I/O.
*
* `routingAccuracy` tracks per-tier success/failure counts but carries no
* provider/model attribution and is all-time (slow to react). The
* `failureSignatures.recent[]` samples (recorded by #2332) DO carry
* provider/model/tier attribution and are recency-bounded. This cross-references
* the two: a tier is steered away from only when it BOTH accumulates ≥
* MIN_FAILURE_SAMPLES recent failures for this task type AND is *unproven* in
* routingAccuracy (success rate < HIGH_SUCCESS_THRESHOLD, or no success data).
*
* Proven tiers (≥80% success) keep their slot despite a few absolute failures —
* failureSignatures records failures ONLY, so raw counts can never condemn a
* tier on their own (a tier with 100 successes + 3 failures is fine). The
* cross-check is what keeps this from starving a good tier.
*
* `minFailureSamples` (issue #2344) is the failure-sample bar a tier must clear
* before it is steered away from. Callers pass the conservative bar until the
* correlation-quality window proves the signal predicts outcomes (>0.8); default
* stays at the aggressive `MIN_FAILURE_SAMPLES` for back-compat with direct
* callers/tests.
*
* `maxAgeMs` (issue #2619) windows the recent samples by age at read time,
* mirroring `computeWindowedStats`: a failure sample older than the window is
* ignored, so a fixed low-frequency type stops being steered off a tier by an
* outage-era sample that never gets pushed out of the count-capped ring. An
* undated sample (unparseable `recordedAt`) is kept — the count cap still bounds
* it. Defaults to the shared 30-day window.
*
* @returns {{ avoidTiers: string[], sampleCount: number,
* dominant: { tier, failures, provider, model }|null }}
*/
export function deriveFailureSignalAvoidance(data, taskType, {
minFailureSamples = MIN_FAILURE_SAMPLES,
maxAgeMs = DEFAULT_WINDOW_MAX_AGE_MS,
now = Date.now()
} = {}) {
const signatures = data?.failureSignatures || {};
const routing = data?.routingAccuracy?.[taskType] || {};
const ageCutoff = Number.isFinite(maxAgeMs) && maxAgeMs > 0 ? now - maxAgeMs : null;
// Gather recent failure samples scoped to this task type across all
// categories — except environmental ones (issue #2618): failureSignatures
// deliberately still records outage-class samples for diagnostics, but a
// burst of rate-limit failures says nothing about the tier, so it must not
// reach avoidTiers through this secondary signal either.
const byTier = {};
let sampleCount = 0;
for (const [category, bucket] of Object.entries(signatures)) {
if (ENVIRONMENTAL_ERROR_CATEGORIES.has(category)) continue;
for (const sample of bucket?.recent || []) {
if (sample?.taskType !== taskType || !sample.modelTier) continue;
if (NON_ROUTABLE_LEARNED_TIERS.has(sample.modelTier)) continue;
// Age-window (#2619): drop a sample with a parseable timestamp older than
// the window; keep an undated one (the count cap still bounds it).
if (ageCutoff !== null) {
const ts = Date.parse(sample.recordedAt);
if (Number.isFinite(ts) && ts < ageCutoff) continue;
}
sampleCount++;
const tier = (byTier[sample.modelTier] ||= { failures: 0, pairs: {} });
tier.failures++;
// Tally provider+model as ONE pair so the reported attribution is a
// combination that actually failed together (not a top-provider crossed
// with an unrelated top-model). NUL-joined; '' encodes an absent side.
const pairKey = `${sample.provider ?? ''}${PAIR_SEP}${sample.model ?? ''}`;
tier.pairs[pairKey] = (tier.pairs[pairKey] || 0) + 1;
}
}
const avoidTiers = [];
let dominant = null;
for (const [tier, stats] of Object.entries(byTier)) {
const r = routing[tier];
const attempts = r ? (r.succeeded + r.failed) : 0;
// null = no success data (not "0% success") — an unproven tier by default.
const successRate = attempts > 0 ? Math.round((r.succeeded / attempts) * 100) : null;
const proven = successRate !== null && successRate >= HIGH_SUCCESS_THRESHOLD;
if (proven || stats.failures < minFailureSamples) continue;
avoidTiers.push(tier);
// `dominant` is the AVOIDED tier with the most recent failures — the tier we
// actually steer away from — so the attribution surfaced in logs/reasons can
// never name a proven tier that isn't in avoidTiers. null when nothing avoided.
if (!dominant || stats.failures > dominant.failures) {
const [provider, model] = (topKey(stats.pairs) || PAIR_SEP).split(PAIR_SEP);
dominant = { tier, failures: stats.failures, provider: provider || null, model: model || null };
}
}
return { avoidTiers, sampleCount, dominant };
}
/** Union two tier lists, preserving order and dropping duplicates. */
function mergeAvoidTiers(...lists) {
return [...new Set(lists.flat().filter(Boolean))];
}
/**
* Tiers with at least one RECENT failure sample for this task type, read from
* the enriched `failureSignatures.recent[]` rings (issue #2617). Pure.
*
* The task-type `recentOutcomes` ring carries no tier identity, so a healthy
* task-wide recent window alone cannot prove that a specific tier's failure
* evidence is stale — the recent successes may all have run on a DIFFERENT
* tier. The failure samples DO carry `modelTier` + `recordedAt`, so this is
* the tier-scoped recency signal: a failing tier that appears here failed
* recently and must keep being steered away from; one that doesn't has only
* stale (pre-window) failure evidence.
*/
function tiersWithRecentFailures(data, taskType, { now = Date.now(), maxAgeMs = DEFAULT_WINDOW_MAX_AGE_MS } = {}) {
const tiers = new Set();
for (const [category, bucket] of Object.entries(data?.failureSignatures || {})) {
// Environmental categories can't mark a tier as recently-failing (#2618).
if (ENVIRONMENTAL_ERROR_CATEGORIES.has(category)) continue;
for (const sample of bucket?.recent || []) {
if (sample?.taskType !== taskType || !sample.modelTier) continue;
const t = Date.parse(sample.recordedAt);
if (Number.isFinite(t) && now - t <= maxAgeMs) tiers.add(sample.modelTier);
}
}
return tiers;
}
/**
* Get suggested priority boost for a task type based on historical success
* Returns a multiplier: >1 for boost, <1 for demotion
*
* Reads the EFFECTIVE success rate (issue #2617): the recency-windowed rate
* when the outcome ring has enough samples, else the lifetime rate — so a type
* that recovered from a since-fixed failure burst stops being demoted.
*/
export async function getTaskTypePriorityMultiplier(taskType) {
const data = await loadLearningData();
const metrics = data.byTaskType[taskType];
if (!metrics || metrics.completed < 3) {
return 1.0; // Not enough data, use default priority
}
const { successRate } = computeEffectiveSuccessRate(metrics);
// High success rate = boost priority
if (successRate >= 90) return 1.2;
if (successRate >= 75) return 1.1;
// Low success rate = demote priority slightly (but not too much, as we want to retry)
if (successRate !== null && successRate < 50) return 0.9;
return 1.0;
}
/**
* Suggest model tier based on historical performance for a task type
* Enhanced with negative signal awareness: avoids tiers that consistently fail
* and prefers tiers with proven success for the task type.
*
* ADVISORY ONLY (#8149): surfaced per task type by `getRoutingAccuracy` for the
* CoS Learning view. Dispatch never reads it — a tier the user did not ask for
* silently replaced a task's configured default model (#8148).
*/
export async function suggestModelTier(taskType) {
// Sandboxed fallback (issue #2333): the `external/untyped` bucket aggregates
// heterogeneous work, so its tier success rates are meaningless — never let it
// drive a model-tier suggestion.
if (isSandboxedTaskType(taskType)) return null;
return suggestModelTierFrom(await loadLearningData(), taskType);
}
// `suggestModelTier` over an already-loaded snapshot, so a caller walking every
// task type (getRoutingAccuracy) loads — and clones — the store once, not per row.
function suggestModelTierFrom(data, taskType) {
if (isSandboxedTaskType(taskType)) return null;
// Auto-adjustment aggressiveness gate (issue #2344): the enriched failure
// signal steers routing more aggressively only once the correlation-quality
// window proves it actually predicts bad outcomes (>0.8). Until then, require a
// higher failure-sample bar so the system doesn't over-correct on an unproven
// signal. Computed synchronously from the already-loaded window — no extra I/O.
const correlationQuality = computeCorrelationQuality(data.correlationWindow);
const aggressive = isCorrelationProven(correlationQuality);
const minFailureSamples = aggressive ? MIN_FAILURE_SAMPLES : CONSERVATIVE_MIN_FAILURE_SAMPLES;
// Recency-weighted, provider-attributed avoidance from the enriched failure
// signatures (#2329) — folded into every suggestion below so a tier that's
// freshly degrading is steered away from before its all-time routingAccuracy
// rate crosses the hard misroute line. Computed BEFORE the completions guard:
// the signal has its own failure-sample bar, so recent failures on a tier can
// steer selection even before routingAccuracy has enough data to suggest a
// tier at all.
const failureAvoidance = deriveFailureSignalAvoidance(data, taskType, { minFailureSamples });
const withFailureSignal = (suggestion, existingAvoid = []) => {
const avoidTiers = mergeAvoidTiers(existingAvoid, failureAvoidance.avoidTiers);
// If the tier we were about to suggest is itself failure-flagged (a 60-79%
// "successful" tier, or the generic `heavy` fallback), don't route to it —
// drop to avoidance-only (suggested: null) so selection steers off it and
// picks the best non-avoided tier instead of the tier the signal condemned.
const suggested = avoidTiers.includes(suggestion.suggested) ? null : suggestion.suggested;
return {
...suggestion,
suggested,
avoidTiers,
...(failureAvoidance.dominant ? { failureSignal: failureAvoidance.dominant } : {})
};
};
// Avoidance-only suggestion (no positive tier pick) built purely from the
// failure signal — used both below the completions threshold and as the final
// fallback when routingAccuracy is quiet.
const avoidanceOnlySuggestion = () => {
const { dominant } = failureAvoidance;
return withFailureSignal({
suggested: null,
reason: `${taskType} shows ${dominant.failures} recent failure(s) on ${dominant.tier} tier`
+ (dominant.provider ? ` via ${dominant.provider}${dominant.model ? `/${dominant.model}` : ''}` : '')
+ ' — avoiding it'
});
};
const metrics = data.byTaskType[taskType];
if (!metrics || metrics.completed < 5) {
// Not enough completions for a routingAccuracy-based tier suggestion, but a
// strong recency signal can still steer selection off a freshly-failing tier.
return failureAvoidance.avoidTiers.length > 0 ? avoidanceOnlySuggestion() : null;
}
// Effective (windowed-when-available) task-type rate (issue #2617). Used both
// to gate the all-time failing-tier branch below and in the low-success
// fallback: routingAccuracy's per-tier counters have no recency window of
// their own, so after a since-fixed failure burst a tier reads "failing"
// forever — when the type's recent window is healthy, that all-time evidence
// is stale and must not steer routing (to `heavy`, or away from the very tier
// that is currently succeeding).
const effective = computeEffectiveSuccessRate(metrics);
const recentlyHealthy = effective.source === 'windowed' && effective.successRate >= 60;
// Check routing accuracy for tier-specific signals
const routingData = data.routingAccuracy?.[taskType];
if (routingData) {
// Find tiers with enough data and their success rates
const tierResults = Object.entries(routingData)
.filter(([tier, r]) => (r.succeeded + r.failed) >= 3 && !NON_ROUTABLE_LEARNED_TIERS.has(tier))
.map(([tier, r]) => {
const total = r.succeeded + r.failed;
return { tier, successRate: Math.round((r.succeeded / total) * 100), total };
})
.sort((a, b) => b.successRate - a.successRate);
// Among tiers that clear the high-success threshold, prefer the LIGHTEST
// (cheapest) one — there's no reason to spend a heavier model when a
// lighter tier already succeeds reliably. tierResults is sorted by success
// rate, so without this a heavy tier at 85% would beat a light tier at 82%
// and silently over-allocate compute for the same outcome.
const provenTiers = tierResults.filter(t => t.successRate >= HIGH_SUCCESS_THRESHOLD);
if (provenTiers.length > 0) {
const lightest = provenTiers.reduce((a, b) =>
tierWeight(b.tier) < tierWeight(a.tier) ? b : a);
const reason = provenTiers.length > 1
? `${taskType} succeeds with ${lightest.tier} tier (${lightest.successRate}%) — using lightest of ${provenTiers.length} proven tiers`
: `${taskType} has ${lightest.successRate}% success with ${lightest.tier} tier`;
return withFailureSignal(
{ suggested: lightest.tier, reason },
tierResults.filter(t => t.successRate < 40).map(t => t.tier)
);
}
// If current default tier is failing, find a better one. Recency gate
// (issue #2617): skip this branch only when the type's recent window is
// healthy AND none of the "failing" tiers has a recent failure on record
// (tiersWithRecentFailures) — then the all-time tier rates are stale
// evidence from a since-fixed burst, and acting on them would route a
// currently-working type to `heavy` (and steer selection off the tier it
// is presently succeeding on) indefinitely. A failing tier with RECENT
// failure samples keeps its evidence: the task-wide recovery may have
// happened on a different tier, and the ring carries no tier identity.
const failingTiers = tierResults.filter(t => t.successRate < 40);
const recentFailTiers = tiersWithRecentFailures(data, taskType);
const failingEvidenceStale = recentlyHealthy && !failingTiers.some(t => recentFailTiers.has(t.tier));
if (failingTiers.length > 0 && !failingEvidenceStale) {
const successfulTier = tierResults.find(t => t.successRate >= 60);
return withFailureSignal(
{
suggested: successfulTier?.tier || 'heavy',
reason: `${taskType} fails with ${failingTiers.map(t => t.tier).join(', ')} (${failingTiers.map(t => `${t.successRate}%`).join(', ')})`
},
failingTiers.map(t => t.tier)
);
}
}
// Fallback: if the overall EFFECTIVE success rate is low (issue #2617:
// windowed when the ring has enough samples, else lifetime), suggest a
// heavier model. On the lifetime rate alone, a type that recovered from a
// since-fixed failure burst would be routed to `heavy` forever.
if (effective.successRate !== null && effective.successRate < 60) {
return withFailureSignal({
suggested: 'heavy',
reason: `${taskType} has ${effective.successRate}% success rate - heavier model may help`
});
}
// No tier-level or overall signal, but the enriched failure signatures flagged
// a currently-degrading tier — emit an avoidance-only suggestion so selection
// steers off it (agentModelSelection picks the best non-avoided tier) even
// though the all-time routingAccuracy rate hasn't crossed the misroute line.
if (failureAvoidance.avoidTiers.length > 0) {
return avoidanceOnlySuggestion();
}
return null; // Current selection is working fine
}
/**
* Get routing accuracy metrics showing which model tiers succeed/fail for each task type
* Returns a matrix suitable for display in the Learning tab UI
*/
export async function getRoutingAccuracy() {
const data = await loadLearningData();
const routingData = data.routingAccuracy || {};
const matrix = [];
const tierSummary = {};
for (const [taskType, tiers] of Object.entries(routingData)) {
const taskEntry = { taskType, tiers: [] };
for (const [tier, counts] of Object.entries(tiers)) {
const total = counts.succeeded + counts.failed;
if (total === 0) continue;
const successRate = Math.round((counts.succeeded / total) * 100);
taskEntry.tiers.push({
tier,
succeeded: counts.succeeded,
failed: counts.failed,
total,
successRate,
lastAttempt: counts.lastAttempt
});
// Aggregate tier summary
if (!tierSummary[tier]) {
tierSummary[tier] = { succeeded: 0, failed: 0, taskTypes: 0, misroutes: 0 };
}
tierSummary[tier].succeeded += counts.succeeded;
tierSummary[tier].failed += counts.failed;
tierSummary[tier].taskTypes++;
if (successRate < 40 && total >= 3) {
tierSummary[tier].misroutes++;
}
}
// Sort tiers by success rate descending
taskEntry.tiers.sort((a, b) => b.successRate - a.successRate);
if (taskEntry.tiers.length > 0) {
matrix.push(taskEntry);
}
}
// Calculate tier-level success rates
const tierOverview = Object.entries(tierSummary).map(([tier, s]) => {
const total = s.succeeded + s.failed;
return {
tier,
successRate: total > 0 ? Math.round((s.succeeded / total) * 100) : 0,
total,
taskTypes: s.taskTypes,
misroutes: s.misroutes
};
}).sort((a, b) => b.successRate - a.successRate);
// Identify misroutes: task+tier combos with <40% success and 3+ attempts
const misroutes = [];
for (const entry of matrix) {
for (const tier of entry.tiers) {
if (tier.successRate < 40 && tier.total >= 3) {
misroutes.push({
taskType: entry.taskType,
tier: tier.tier,
successRate: tier.successRate,
failed: tier.failed,
total: tier.total
});
}
}
}
misroutes.sort((a, b) => a.successRate - b.successRate);
// Advice, not routing: the tier each task type's history recommends, for the
// user to pin on the task or schedule if they agree.
for (const entry of matrix) {
const suggestion = suggestModelTierFrom(data, entry.taskType);
const tier = advisoryTier(suggestion?.suggested);
const avoidTiers = [...new Set((suggestion?.avoidTiers || []).map(advisoryTier).filter(Boolean))];
if (tier || avoidTiers.length > 0) {
entry.suggestion = { tier, avoidTiers, reason: suggestion.reason || null };
}
}
return { matrix, tierOverview, misroutes, totalMisroutes: misroutes.length };
}
/**
* Get a performance summary for logging during task evaluation
* Provides insights about how different task types are performing
*/
export async function getPerformanceSummary({ sinceByTaskType = {}, since = null } = {}) {
const data = await loadLearningData();
const summary = {
totalCompleted: data.totals.completed,
overallSuccessRate: data.totals.completed > 0
? Math.round((data.totals.succeeded / data.totals.completed) * 100)
: 0,
avgDurationMin: Math.round(data.totals.avgDurationMs / 60000),
topPerformers: [],
needsAttention: [],
skipped: []
};
// Analyze each task type on the EFFECTIVE rate (issue #2617) so this
// summary — logged during evaluation and served to the Learning UI — agrees
// with the actual cooldown/skip decisions instead of resurrecting the
// "punished forever" lifetime view they just stopped using.
for (const [taskType, metrics] of Object.entries(data.byTaskType)) {
if (metrics.completed < 3) continue;
// Alert acknowledgements reset only the evidence window, never learning
// history or scheduler decisions. Undated samples cannot prove a new issue.
const cutoff = Math.max(Date.parse(since) || 0, Date.parse(sinceByTaskType[taskType]) || 0);
const alertMetrics = cutoff ? { ...metrics, recentOutcomes: (metrics.recentOutcomes || [])
.filter(outcome => Date.parse(outcome.t) > cutoff) } : metrics;
const { successRate, source: rateSource, windowedCompleted } = computeEffectiveSuccessRate(alertMetrics);
if (cutoff && rateSource !== 'windowed') continue;
const entry = {
taskType,
successRate,
// Evidence pairing (issue #2617): a windowed rate must travel with its
// own sample count — "0% success across 200 tasks" (lifetime completed
// next to a 6-sample windowed rate) materially overstates the evidence.
rateSource,
windowedCompleted,
completed: metrics.completed,
avgDurationMin: Math.round(metrics.avgDurationMs / 60000)
};
if (successRate >= 80) {
summary.topPerformers.push(entry);
} else if (successRate !== null && successRate < 50 && metrics.completed >= 5) {
summary.needsAttention.push(entry);
// Also mark as skipped if very low (same predicate as getSkippedTaskTypes)
if (isSkipCandidate(alertMetrics)) {
summary.skipped.push(entry);
}
}
}
// Sort by success rate
summary.topPerformers.sort((a, b) => b.successRate - a.successRate);
summary.needsAttention.sort((a, b) => a.successRate - b.successRate);
return summary;
}
/**
* Get adaptive cooldown multiplier for a task type based on historical performance
*
* This allows the CoS to work more efficiently:
* - High success rate tasks: Reduced cooldown (can work on similar tasks sooner)
* - Low success rate tasks: Increased cooldown (give time for fixes/investigation)
* - Very low success rate: Skip this task type (needs review)
*
* @param {string} taskType - The task type (e.g., 'self-improve:ui-bugs')
* @returns {Object} Cooldown adjustment info
*/
export async function getAdaptiveCooldownMultiplier(taskType) {
// Sandboxed fallback (issue #2333): never skip or throttle the heterogeneous
// `external/untyped` bucket — a poor aggregate success rate here would create
// a routing blind spot that silently drops ALL unclassified work. Keep it at
// the default cooldown and always eligible.
if (isSandboxedTaskType(taskType)) {
return { multiplier: 1.0, reason: 'sandboxed-untyped', skip: false, successRate: null, completed: 0 };
}
const data = await loadLearningData();
const metrics = data.byTaskType[taskType];
// Not enough data - use default cooldown
if (!metrics || metrics.completed < 3) {
return {
multiplier: 1.0,
reason: 'insufficient-data',
skip: false,
successRate: null,
completed: metrics?.completed || 0
};
}
// Effective rate (issue #2617): windowed when the outcome ring has enough
// samples, else lifetime — so a since-fixed failure burst stops inflating the
// cooldown (or holding `skip: true`) once the type succeeds again. All the
// completed-count thresholds below stay on the LIFETIME counter, unchanged.
const { successRate, source: rateSource } = computeEffectiveSuccessRate(metrics);
// Very high success (90%+): Reduce cooldown by 30% - this task type works well
if (successRate >= 90) {
return {
multiplier: 0.7,
reason: 'high-success',
skip: false,
successRate,
rateSource,
completed: metrics.completed,
recommendation: `Task type has ${successRate}% success rate - reduced cooldown`
};
}
// Good success (75-89%): Slight reduction (15%)
if (successRate >= 75) {
return {
multiplier: 0.85,
reason: 'good-success',
skip: false,
successRate,
rateSource,
completed: metrics.completed
};
}
// Moderate success (50-74%): Default cooldown
if (successRate >= 50) {
return {
multiplier: 1.0,
reason: 'moderate-success',
skip: false,
successRate,
rateSource,
completed: metrics.completed
};
}
// Low success (30-49%): Increase cooldown by 50%
if (successRate >= 30) {
return {
multiplier: 1.5,
reason: 'low-success',
skip: false,
successRate,
rateSource,
completed: metrics.completed,
recommendation: `Task type has only ${successRate}% success rate - increased cooldown`
};
}
// Very low success (<30%) with significant attempts: Skip this task type
if (metrics.completed >= 5) {
return {
multiplier: 0, // Effectively infinite cooldown
reason: 'skip-failing',
skip: true,
successRate,
rateSource,
completed: metrics.completed,
recommendation: `Task type has ${successRate}% success rate after ${metrics.completed} attempts - skipping until reviewed`
};
}
// Very low success but few attempts: Double cooldown and keep trying
return {
multiplier: 2.0,
reason: 'very-low-success',
skip: false,
successRate,
rateSource,
completed: metrics.completed,
recommendation: `Task type has ${successRate}% success rate - doubled cooldown for retry`
};
}
/**
* True when a skip is LIFETIME-driven (the stored all-time rate itself is
* below the skip bar), as opposed to a windowed-driven skip caused by a recent
* failure burst on an otherwise-healthy type. Rehabilitation is a DESTRUCTIVE
* reset (`resetTaskTypeLearning` deletes the whole bucket — duration/ETA
* stats, routing accuracy, error patterns), which only makes sense when the
* lifetime data itself is what's poisoned; a windowed-driven skip self-heals
* as the window rolls off, so resetting would wipe hundreds of good runs of
* learning for a type that was never structurally broken.
*/
function isLifetimeDrivenSkip(metrics) {
return Number.isFinite(metrics?.successRate) && metrics.successRate < 30;
}
// Recovery-based rehabilitation bars (issue #2619). The idle path only ever
// resets a type that has gone QUIET; a type that was fixed but still runs
// regularly never goes idle, so its poisoned lifetime/duration/failure-signature
// data would never be cleared. These bars catch that case: a poor-lifetime type
// whose RECENT window is clean has demonstrably recovered.
const RECOVERY_MIN_WINDOW_SAMPLES = 5;
const RECOVERY_MIN_WINDOW_SUCCESS_RATE = HIGH_SUCCESS_THRESHOLD; // 80%
// "Poor lifetime" bar — a lifetime rate at/above this would already earn
// auto-approval on its own, so there is nothing stale to rehabilitate.
const RECOVERY_MAX_LIFETIME_SUCCESS_RATE = 50;
/**
* True when a task type has RECOVERED (issue #2619): its lifetime success rate
* is still poor (< RECOVERY_MAX_LIFETIME_SUCCESS_RATE — would deny auto-approval
* on lifetime alone) but its recency window is clean (≥ RECOVERY_MIN_WINDOW_SAMPLES
* outcomes at ≥ RECOVERY_MIN_WINDOW_SUCCESS_RATE% success). A recovery reset
* clears the stale lifetime/duration/failure-signature data so the type stops
* being penalized by history it has already outgrown — and, unlike the idle
* path, it fires even for a type that still runs regularly. Pure.
*/
function hasRecoveredWindow(metrics, { now = Date.now() } = {}) {
if (!Number.isFinite(metrics?.successRate) || metrics.successRate >= RECOVERY_MAX_LIFETIME_SUCCESS_RATE) {
return false;
}
const { windowedCompleted, windowedSuccessRate } = computeWindowedStats(metrics?.recentOutcomes, { now });
return windowedSuccessRate !== null
&& windowedCompleted >= RECOVERY_MIN_WINDOW_SAMPLES
&& windowedSuccessRate >= RECOVERY_MIN_WINDOW_SUCCESS_RATE;
}
/**
* Get all task types that should be skipped due to poor performance
* Useful for filtering out problematic task types in evaluateTasks
*/
export async function getSkippedTaskTypes() {
const data = await loadLearningData();
const skipped = [];
for (const [taskType, metrics] of Object.entries(data.byTaskType)) {
// Sandboxed fallback is never globally skipped (issue #2333).
if (isSandboxedTaskType(taskType)) continue;
// Skip if: completed >= 5 AND effective success rate < 30%
if (isSkipCandidate(metrics)) {
const { successRate, source: rateSource, windowedCompleted } = computeEffectiveSuccessRate(metrics);
skipped.push({
taskType,
successRate,
// Evidence pairing (issue #2617): a windowed rate ships with its own
// sample count so consumers never render it beside the lifetime total.
rateSource,
windowedCompleted,
completed: metrics.completed,
lastCompleted: metrics.lastCompleted
});
}
}
return skipped;
}
/**
* Check if a specific task type should be skipped
*/
export async function shouldSkipTaskType(taskType) {
const result = await getAdaptiveCooldownMultiplier(taskType);
return result.skip;
}
/**
* Check if any skipped task types are eligible for automatic rehabilitation
* Task types that have been skipped for a grace period get a "fresh start" opportunity
*
* Auto-rehabilitation rules:
* - Task must have been skipped (success rate < 30% with 5+ attempts)
* - Must have been at least rehabilitationGracePeriodMs since last completion
* - Reset the task type's learning data to give it a fresh chance
*
* This allows CoS to automatically retry previously-failing task types
* after enough time has passed for fixes to be applied.
*
* @param {number} gracePeriodMs - Minimum time since last attempt (default: 7 days)
* @returns {Object} Summary of rehabilitated task types
*/
export async function checkAndRehabilitateSkippedTasks(gracePeriodMs = 7 * 24 * 60 * 60 * 1000) {
const data = await loadLearningData();
const rehabilitated = [];
const now = Date.now();
for (const [taskType, metrics] of Object.entries(data.byTaskType)) {
// Sandboxed fallback is never skipped, so never rehabilitated either (#2333).
if (isSandboxedTaskType(taskType)) continue;
const lastCompletedTime = metrics.lastCompleted
? new Date(metrics.lastCompleted).getTime()
: 0;
const timeSinceLastAttempt = now - lastCompletedTime;
const daysSinceLastAttempt = Math.round(timeSinceLastAttempt / (24 * 60 * 60 * 1000));
// IDLE path (existing): a LIFETIME-driven skip (effective rate < 30% with 5+
// attempts, issue #2617) that has sat untouched for the grace period gets a
// fresh start. Only a lifetime-driven skip qualifies — a windowed-driven skip
// (recent burst on a healthy lifetime record) self-heals as the window rolls,
// so the destructive reset would wipe good learning data.
const idleEligible = isSkipCandidate(metrics)
&& isLifetimeDrivenSkip(metrics)
&& timeSinceLastAttempt >= gracePeriodMs;
// RECOVERY path (issue #2619): a poor-lifetime type whose recent window is
// clean is rehabilitated regardless of idle time — a fixed type that still
// runs never goes idle, so the idle gate alone would never clear its stale
// lifetime data. Evaluated only when the idle path didn't already fire.
const recoveryEligible = !idleEligible && hasRecoveredWindow(metrics, { now });
if (!idleEligible && !recoveryEligible) continue;
const trigger = idleEligible ? 'idle' : 'recovery';
emitLog('info', `Auto-rehabilitating ${taskType} (${trigger}: was ${metrics.successRate}% lifetime success${idleEligible ? `, ${daysSinceLastAttempt} days since last attempt` : ', recent window recovered'})`, {
taskType,
trigger,
previousSuccessRate: metrics.successRate,
previousAttempts: metrics.completed,
daysSinceLastAttempt
}, '📚 TaskLearning');
// Reset this task type's data
await resetTaskTypeLearning(taskType);
rehabilitated.push({
taskType,
trigger,
previousSuccessRate: metrics.successRate,
previousAttempts: metrics.completed,
daysSinceLastAttempt
});
}
if (rehabilitated.length > 0) {
emitLog('success', `Auto-rehabilitated ${rehabilitated.length} skipped task type(s)`, {
rehabilitated: rehabilitated.map(r => r.taskType)
}, '📚 TaskLearning');
}
return { rehabilitated, count: rehabilitated.length };
}
/**
* Get all skipped task types with their rehabilitation eligibility status
* Useful for UI display and debugging
* @param {number} gracePeriodMs - Grace period for rehabilitation eligibility
* @returns {Array} List of skipped task types with status info
*/
export async function getSkippedTaskTypesWithStatus(gracePeriodMs = 7 * 24 * 60 * 60 * 1000) {
const data = await loadLearningData();
const skipped = [];
const now = Date.now();
for (const [taskType, metrics] of Object.entries(data.byTaskType)) {
// Sandboxed fallback is never skipped (#2333) — exclude from the skip status list.
if (isSandboxedTaskType(taskType)) continue;
// Only include task types that would be skipped (effective rate, issue #2617)
if (!isSkipCandidate(metrics)) {
continue;
}
const lastCompletedTime = metrics.lastCompleted
? new Date(metrics.lastCompleted).getTime()
: 0;
const timeSinceLastAttempt = now - lastCompletedTime;
// Mirrors checkAndRehabilitateSkippedTasks: only a lifetime-driven skip is
// ever rehabilitation-eligible (a windowed-driven skip self-heals instead).
const eligibleForRehabilitation = timeSinceLastAttempt >= gracePeriodMs && isLifetimeDrivenSkip(metrics);
const timeUntilEligible = timeSinceLastAttempt >= gracePeriodMs
? 0
: gracePeriodMs - timeSinceLastAttempt;
const { successRate, source: rateSource, windowedCompleted } = computeEffectiveSuccessRate(metrics);
skipped.push({
taskType,
successRate,
rateSource,
windowedCompleted,
completed: metrics.completed,
lastCompleted: metrics.lastCompleted,
daysSinceLastAttempt: Math.round(timeSinceLastAttempt / (24 * 60 * 60 * 1000)),
eligibleForRehabilitation,
daysUntilEligible: Math.ceil(timeUntilEligible / (24 * 60 * 60 * 1000))
});
}
return skipped;
}
/**
* Pure classifier — returns { tier, autoApprove, successRate, rateSource } for
* a given metrics object. Shared by getTaskTypeConfidence() and
* getConfidenceLevels() so the auto-approve gate and the Learning tab display
* always agree. Classifies on the EFFECTIVE success rate (issue #2617):
* windowed when the outcome ring has enough samples, else lifetime — so a type
* with a poor lifetime rate but a healthy recent window regains auto-approval.
* The returned `successRate` is the rate the tier was classified on.
*/
function classifyConfidenceTier(metrics, { highThreshold = 80, lowThreshold = 50, minSamples = 5 } = {}) {
const completed = metrics?.completed ?? 0;
const effective = computeEffectiveSuccessRate(metrics);
// Compare on a 0-defaulted value, but RETURN the raw nullable rate — the
// null sentinel ("no rate recorded") must survive to callers (repo
// absent-vs-empty rule), not collapse into a fabricated 0%.
const rate = effective.successRate ?? 0;
const result = { successRate: effective.successRate, rateSource: effective.source, windowedCompleted: effective.windowedCompleted };
if (completed < minSamples) return { tier: 'new', autoApprove: true, ...result };
if (rate >= highThreshold) return { tier: 'high', autoApprove: true, ...result };
if (rate >= lowThreshold) return { tier: 'medium', autoApprove: true, ...result };
return { tier: 'low', autoApprove: false, ...result };
}
/**
* Calculate confidence tier for a specific task type based on learning data.
*
* @param {string} taskType - The task type to evaluate
* @param {Object} [thresholds] - Override default thresholds
* @returns {Promise<Object>} Confidence assessment
*/
export async function getTaskTypeConfidence(taskType, thresholds = {}) {
const data = await loadLearningData();
const metrics = data.byTaskType[taskType];
const { tier, autoApprove, successRate, rateSource, windowedCompleted } = classifyConfidenceTier(metrics, thresholds);
// Reasons quote the EFFECTIVE rate the tier was classified on (issue #2617),
// annotated when it came from the recency window — and paired with the
// WINDOW's sample count, not the lifetime one, so "100% recent success
// across 15 recent runs" never overstates its evidence as 200 lifetime runs.
const rateLabel = rateSource === 'windowed' ? `${successRate ?? 0}% recent success` : `${successRate ?? 0}% success`;
const attemptsLabel = rateSource === 'windowed' ? `${windowedCompleted} recent runs` : `${metrics?.completed} runs`;
const reasons = {
new: `Fewer than ${thresholds.minSamples ?? 5} completions — auto-approve by default`,
high: `${rateLabel} across ${attemptsLabel} — high confidence`,
medium: `${rateLabel} — acceptable confidence`,
low: `${rateLabel} after ${attemptsLabel} — requires approval`
};
return {
taskType,
tier,
autoApprove,
// Nullable: preserves the "no rate recorded" sentinel (absent bucket OR a
// bucket without a stored rate and a thin window) instead of a fake 0.
successRate,
rateSource,
windowedCompleted,
completed: metrics?.completed ?? 0,
reason: reasons[tier]
};
}
/**
* Get confidence levels for all tracked task types.
* Returns a summary suitable for display in the Learning tab UI.
*
* @param {Object} [thresholds] - Override default thresholds
* @returns {Promise<Object>} All task types grouped by confidence tier
*/
export async function getConfidenceLevels(thresholds = {}) {
const data = await loadLearningData();
const levels = { high: [], medium: [], low: [], new: [] };
for (const [taskType, metrics] of Object.entries(data.byTaskType)) {
const { tier, autoApprove, successRate, rateSource, windowedCompleted } = classifyConfidenceTier(metrics, thresholds);
levels[tier].push({
taskType,
successRate: successRate ?? 0, // display list keeps the pre-existing 0 default
rateSource,
windowedCompleted,
completed: metrics.completed || 0,
autoApprove,
lastCompleted: metrics.lastCompleted
});
}
for (const tier of Object.values(levels)) {
tier.sort((a, b) => b.successRate - a.successRate);
}
const { highThreshold = 80, lowThreshold = 50, minSamples = 5 } = thresholds;
return {
levels,
thresholds: { highThreshold, lowThreshold, minSamples },
summary: {
high: levels.high.length,
medium: levels.medium.length,
low: levels.low.length,
new: levels.new.length,
total: Object.values(levels).reduce((sum, arr) => sum + arr.length, 0),
requireApproval: levels.low.length
}
};
}