Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
feat: single ordered provider fallback list under Models > Fallbacks
[._.]/ Adam Eivycd567c540a78
diff --git a/client/src/components/models/ProviderFallbacks.jsx b/client/src/components/models/ProviderFallbacks.jsxnew file mode 100644index 0000000000..2f004ebdc9--- /dev/null+++ b/client/src/components/models/ProviderFallbacks.jsx@@ -0,0 +1,120 @@+import { useCallback, useEffect, useMemo, useState } from 'react';+import { ArrowDown, ArrowUp, ListOrdered, Plus, Save, Trash2 } from 'lucide-react';+import toast from '../ui/Toast';+import * as api from '../../services/api';+import PageSkeleton from '../ui/PageSkeleton';+import ProviderModelSelector from '../ProviderModelSelector';++/**+ * Fallbacks — ONE ordered list of provider presets tried in turn when a run+ * fails or its provider is unavailable, whichever preset the run started on.+ * The step after the one that failed runs next; past the last step the list+ * wraps to the first. A preset may appear more than once with different models+ * or effort (limits and availability differ per model). An empty list keeps the+ * legacy per-preset fallback behavior.+ */++export default function ProviderFallbacks() {+ const [providers, setProviders] = useState([]);+ const [entries, setEntries] = useState(null);+ const [dirty, setDirty] = useState(false);+ const [saving, setSaving] = useState(false);++ useEffect(() => {+ Promise.all([api.getProviders(), api.getProviderFallbacks()])+ .then(([provs, chain]) => {+ setProviders((provs.providers || []).filter((p) => p.enabled !== false));+ setEntries(chain.entries || []);+ })+ .catch(() => setEntries([]));+ }, []);++ const byId = useMemo(() => Object.fromEntries(providers.map((p) => [p.id, p])), [providers]);++ const update = useCallback((next) => { setEntries(next); setDirty(true); }, []);+ const patch = (i, changes) => update(entries.map((e, at) => (at === i ? { ...e, ...changes } : e)));+ const move = (i, delta) => {+ const next = [...entries];+ const [item] = next.splice(i, 1);+ next.splice(i + delta, 0, item);+ update(next);+ };+ const add = () => {+ if (!providers.length) return;+ update([...entries, { providerId: providers[0].id }]);+ };++ const save = async () => {+ setSaving(true);+ const result = await api.saveProviderFallbacks(entries).catch(() => null);+ setSaving(false);+ if (!result) return;+ setEntries(result.entries);+ setDirty(false);+ toast.success('Fallback list saved');+ };++ if (!entries) return <PageSkeleton header="none" label="Loading fallbacks" cards={2} sidebar={false} />;++ return (+ <div className="mx-auto w-full max-w-4xl space-y-4">+ <p className="text-sm text-gray-400">+ When a run fails or its provider is unavailable, PortOS tries the next preset in this list after the one that+ failed, wrapping from the last back to the first. The same preset can appear several times with different+ models. With an empty list, each preset's own fallback setting applies.+ </p>++ {entries.length === 0 && (+ <div className="rounded border border-dashed border-port-border p-6 text-center text-sm text-gray-400">+ <ListOrdered size={20} aria-hidden="true" className="mx-auto mb-2" />+ No fallback list yet.+ </div>+ )}++ <ol className="space-y-2">+ {entries.map((entry, i) => {+ const provider = byId[entry.providerId];+ return (+ <li key={`${i}-${entry.providerId}`} className="flex flex-wrap items-end gap-2 rounded border border-port-border bg-port-card p-3">+ <span className="w-6 pb-2 text-sm text-gray-400">{i + 1}.</span>+ <div className="min-w-0 flex-1">+ <ProviderModelSelector+ providers={providers}+ selectedProviderId={entry.providerId}+ selectedModel={entry.model || ''}+ availableModels={provider?.models || []}+ onProviderChange={(id) => patch(i, { providerId: id, model: undefined, effort: undefined })}+ onModelChange={(model) => patch(i, { model: model || undefined, effort: undefined })}+ effort={entry.effort || ''}+ onEffortChange={(effort) => patch(i, { effort: effort || undefined })}+ emptyModelOption="Preset default"+ alwaysShowModel+ label="Preset"+ />+ </div>+ <div className="flex gap-1">+ <button type="button" aria-label={`Move step ${i + 1} up`} disabled={i === 0} onClick={() => move(i, -1)}+ className="rounded border border-port-border p-2 text-gray-300 hover:text-white disabled:opacity-30"><ArrowUp size={14} aria-hidden="true" /></button>+ <button type="button" aria-label={`Move step ${i + 1} down`} disabled={i === entries.length - 1} onClick={() => move(i, 1)}+ className="rounded border border-port-border p-2 text-gray-300 hover:text-white disabled:opacity-30"><ArrowDown size={14} aria-hidden="true" /></button>+ <button type="button" aria-label={`Remove step ${i + 1}`} onClick={() => update(entries.filter((_, at) => at !== i))}+ className="rounded border border-port-border p-2 text-port-error hover:bg-port-error/10"><Trash2 size={14} aria-hidden="true" /></button>+ </div>+ </li>+ );+ })}+ </ol>++ <div className="flex flex-wrap gap-2">+ <button type="button" onClick={add} disabled={!providers.length}+ className="inline-flex items-center gap-1.5 rounded border border-port-border px-3 py-2 text-sm text-gray-200 hover:text-white disabled:opacity-50">+ <Plus size={14} aria-hidden="true" /> Add step+ </button>+ <button type="button" onClick={save} disabled={!dirty || saving}+ className="inline-flex items-center gap-1.5 rounded bg-port-accent px-3 py-2 text-sm text-white disabled:opacity-50">+ <Save size={14} aria-hidden="true" /> {saving ? 'Saving…' : 'Save'}+ </button>+ </div>+ </div>+ );+}diff --git a/client/src/lib/navPresentation.js b/client/src/lib/navPresentation.jsindex 8456bef9c9..04e5011a11 100644--- a/client/src/lib/navPresentation.js+++ b/client/src/lib/navPresentation.js@@ -100,6 +100,7 @@ import { AudioLines,UserRound,Video,ListTodo,+ ListOrdered,Plug,TerminalSquare,} from 'lucide-react';@@ -235,6 +236,7 @@ export const NAV_PRESENTATION = {'/models/decision-classifiers': { icon: Scale },'/models/decision-classifiers/jev': { icon: Scale },'/models/embeddings': { icon: Braces },+ '/models/fallbacks': { icon: ListOrdered },'/models/llms': { icon: Cpu },'/models/llms/abuse': { icon: ShieldAlert },'/models/llms-runtimes': { icon: Server },diff --git a/client/src/pages/Models.jsx b/client/src/pages/Models.jsxindex cc4c0ecd41..95c7b08de9 100644--- a/client/src/pages/Models.jsx+++ b/client/src/pages/Models.jsx@@ -61,6 +61,7 @@ const TAB_CONTENT = {'code-reviewers': CodeReviewersTab,'decision-classifiers': lazyWithReload(() => import('../components/models/DecisionClassifiers')),embeddings: EmbeddingsTab,+ fallbacks: lazyWithReload(() => import('../components/models/ProviderFallbacks')),llms: LocalLlmTab,'llms-runtimes': LocalLlmRuntimesView,loras: Loras,diff --git a/client/src/pages/Models.test.jsx b/client/src/pages/Models.test.jsxindex 479838c8d2..12ed9400ea 100644--- a/client/src/pages/Models.test.jsx+++ b/client/src/pages/Models.test.jsx@@ -28,6 +28,7 @@ vi.mock('../components/settings/LocalLlmTab', () => ({}));vi.mock('../components/settings/LocalLlmRuntimesView.jsx', () => ({ default: ({ view }) => <div data-testid="runtime-panel" data-runtime={view || ''}>runtimes panel</div> }));vi.mock('../components/settings/EmbeddingsTab', () => ({ default: () => <div>embeddings panel</div> }));+vi.mock('../components/models/ProviderFallbacks', () => ({ default: () => <div>fallbacks panel</div> }));vi.mock('../components/models/Image3dRuntimes', () => ({ default: () => <div>3d runtimes panel</div> }));vi.mock('../components/models/ModelStatusTab', () => ({ default: () => <div>status panel</div> }));vi.mock('../components/models/SubscriptionsTab', () => ({ default: () => <div>subscriptions panel</div> }));@@ -51,6 +52,7 @@ const PANEL_MARKER = {'code-reviewers': 'code reviewers panel','decision-classifiers': 'classifiers panel',embeddings: 'embeddings panel',+ fallbacks: 'fallbacks panel',llms: 'llms panel',abuse: 'llms panel',jev: 'llms panel',diff --git a/client/src/services/apiProviders.js b/client/src/services/apiProviders.jsindex 089a511f98..8981996f67 100644--- a/client/src/services/apiProviders.js+++ b/client/src/services/apiProviders.js@@ -251,3 +251,11 @@ export const getFleetLlmHostUsage = (options) => request('/providers/fleet-host/export const stopFleetLlmHost = (options) => request('/providers/fleet-host/stop', { method: 'POST', ...options });export const getFleetPeerHosts = (options) => request('/providers/fleet-peer-hosts', options);export const revealFleetPeerHostKey = (peerId, options) => request(`/providers/fleet-peer-hosts/${encodeURIComponent(peerId)}/key`, { method: 'POST', ...options });++// The single ordered provider fallback list (Models > Fallbacks).+export const getProviderFallbacks = (options) => request('/provider-fallbacks', options);+export const saveProviderFallbacks = (entries, options) => request('/provider-fallbacks', {+ method: 'PUT',+ body: JSON.stringify({ entries }),+ ...options+});diff --git a/docs/API.md b/docs/API.mdindex fb6ce52bcf..807f07c0d1 100644--- a/docs/API.md+++ b/docs/API.md@@ -837,6 +837,7 @@ Every mounted API prefix (see `server/index.js` for the authoritative list). Dom| `/api/standardize` | App PM2 standardizer || `/api/stacker-news`, `/api/x` | Social integrations || `/api/model-personality` | LLM personality tests |+| `/api/provider-fallbacks` | GET/PUT the single ordered provider fallback list `{ entries: [{ providerId, model?, effort? }] }` (Models > Fallbacks); the step after the failed one runs next, wrapping to the first || `/api/providers/comparison` | Selectable model inventory, sourced PortOS composite, and public evidence; POST `/import` merges researched observations (see [MODEL-COMPARISON.md](./MODEL-COMPARISON.md)) || `/api/models/performance/task-benchmark` | Machine-local PortOS task benchmark history and configured-provider inventory; `POST /discover` and `POST /run` perform explicit user-requested actions || `/api/browser` | Managed Chromium |diff --git a/server/index.js b/server/index.jsindex b5e98423c9..aee03b4154 100644--- a/server/index.js+++ b/server/index.js@@ -1,4 +1,5 @@import maintenanceRoutes from './routes/maintenance.js';+import providerFallbackRoutes from './routes/providerFallbacks.js';import './services/databaseBootFence.js';import express from 'express';import { Server } from 'socket.io';@@ -324,6 +325,7 @@ app.use('/api/detect', detectRoutes);app.use('/api/scaffold', scaffoldRoutes);// AI Toolkit routes with PortOS extensions+app.use('/api/provider-fallbacks', providerFallbackRoutes);app.use('/api/providers/comparison', createModelComparisonRoutes());app.use('/api/models/performance/task-benchmark', createModelPerformanceBenchmarkRoutes(aiToolkit.services.providers));app.use('/api/providers', createPortOSProviderRoutes(aiToolkit));diff --git a/server/lib/README.md b/server/lib/README.mdindex e1fe52e70f..089c2f4b67 100644--- a/server/lib/README.md+++ b/server/lib/README.md@@ -286,6 +286,7 @@ The barrel `server/lib/index.js` is a machine-checkable enumeration of every pub| `providerModelAliases.js` | HAND-AUTHORED canonical→executable model aliases for one provider route (#6369) — the correction surface for a `modelMap` a refresh could only fill with what it could VERIFY. Observed aliases (`ai_route_bindings.model_map`) and user overrides (`model_alias_overrides`) are stored apart and merged on READ by `effectiveModelAliases` (`{ ...observed, ...overrides }`), which is what lets a hand-authored alias outlast the next refresh; `applyModelAliasPatch` is the three-valued edit (absent preserves, a string sets, `null` is the only thing that removes); `staleModelAliases` reports — never repairs — an override naming a spelling the record no longer lists, and makes no claim at all about a route listing no models; `connectionCatalogModels` is the union an alias write re-derives the shared catalog from, so a correction actually reaches the model menus; `modelAliasRevision` is the stale-edit fingerprint, reusing `routeSettingsRevision` because `ai_route_bindings` has no revision column. Pure: no I/O. || `providerModePairing.js` | May an existing record gain the OTHER execution mode of its harness (#7501) — `tuiModeAddition(provider, providers, executionModes?)` is the ONE verdict behind both the `canAddTuiMode` flag `GET /providers` decorates each record with (so the card offers "Add interactive mode") and the refusal `POST /providers/:id/modes/tui` answers with, so a card can never advertise an action the server rejects. Refuses a non-CLI record (the CLI id is the stem, so minting it from a TUI record would be a rename), a record storing no command, and a KNOWN harness declaring no TUI mode; an UNKNOWN harness passes with empty argv rather than a guess, while a known one starts from its recipe’s proven interactive line. Asks whether the harness already HAS its interactive mode (the record’s `providerModeGroups` group — the shipped `grok-cli`/`grok-tui` pair leaves `grok-cli-tui` unclaimed, so an id-only check would mint a third record beside it), and separately 409s on a taken `<id>-tui` rather than suffixing around it. Pure: no I/O. || `providerPresets.js` | PRESETS (#7565, epic #7561) — a stored `data/providers.json` record read as a (harness, method, service) tuple plus the user's own defaults and overrides. A record carrying `harnessId` + `method` + `serviceId` is a DERIVED preset (`isDerivedPreset` in `providerGraphRecords.js`, `presetKind`): `materializeDerivedPreset({record, harness, instance, catalog, bootstrap})` re-derives every connection-owned value (program, endpoint, credential, backend markers, inline bootstrap, `models` = plan-filtered catalog ∩ `catalogNarrowing`) through `materializeRoute` while keeping every per-preset field (name, argv with a binding's suffix preserved, timeouts, pins, effort, fallback, generation params, consent flags, unknown fields) and the record's own OpenCode inline config when it already reaches the service; `refusedDerivedEdits(updates, derived, ownedEnvNames)` names the connection-owned keys a direct edit would move (`DERIVED_PRESET_OWNED_KEYS`, plus env vars by name); `derivedPresetDrift` / `derivedPresetPatch` compare a record with its derivation; `presetDerivable(record)` is the editor's "offer the button" test; `matchBootstrapApp` / `bootstrapInputFor` map an inline `credentialBootstrap` to a configured app and back; `planPresetBackfill({graph, providers, bootstraps, env})` is the boot-time stamp of every routed legacy record whose re-derivation is a FIXPOINT — additive keys only, every refusal reported with a reason. Pure; the store-backed half is `services/providerPresets.js`. `impliedCatalogNarrowing` is the narrowing that keeps a record on its current list once derived, and `rederivePreset` re-derives one derived preset from an instance (derived record + patch), shared by the backfill, `rematerializeDerivedPresets` and the service fold. |+| `providerFallbackChainCache.js` | Import-free process cache of the user's single ordered provider fallback list (`peekFallbackChain` sync read for routing, `setFallbackChainCache`, `normalizeFallbackChain`). Filled by `services/providerFallbackChain.js` from settings `providerFallbackChain`; read by the toolkit's `getFallbackProvider`. || `providerRef.js` | Provider REFERENCE grammar (#7564, epic #7561) — the two spellings a `{ providerId, model, effort }` selection may name a provider by: a PRESET record id (`PRESET_ID_RE`, the slug `createProvider` mints) or a COMPOSITE `<harness>.<cli|tui|api>@<service-slug>[+<bootstrap-slug>]` (`COMPOSITE_ID_RE`; the bootstrap suffix only with a process method). `parseProviderRef(id)` → `{kind:'preset'}` / `{kind:'composite', harnessId, method, serviceSlug, bootstrapSlug}` / `null`, `isCompositeProviderId` / `isPresetProviderId`, `formatCompositeId(parts)` (throws on parts that would not parse back), `MAX_PROVIDER_REF_LENGTH`, and the two validation messages (`PROVIDER_REF_MESSAGE`, `PRESET_ONLY_MESSAGE`). The zod forms — `providerRefSchema` / `providerRefFieldSchema` (either grammar) and `presetProviderIdSchema` (preset-only, for `PUT /providers/active`, a record's `fallbackProvider`, an app's `taskTypeOverrides` pin) — live in `zodCompat.js` so `cosValidation.js` can share them. Pure and dependency-free; mirrored in `aiToolkit/internal/providerRef.js` (the toolkit may not import out) behind `providerRef.parity.test.js`. Resolution is host-side: `services/compositeProviders.js`. || `providerRouteRecipes.js` | Minting a FRESH executable route from a connection and a harness command recipe (#6369) — the reverse of `providerConnections.js`, which reads an existing record. `buildRouteRecord({harnessId, mode, providerId, name, connection})` materializes a connection's endpoint, credentials and backend marker into a DISABLED `data/providers.json` record through `PROVIDER_HARNESSES[].recipe`; `mintRouteIds` allocates readable ids keeping the `<stem>` / `<stem>-tui` pairing `providerModeGroups` recognizes; `bindingBlocker` is the pre-write refusal (no recipe, unsupported mode, a transport the backend does not declare, a credential the program requires); `CREATABLE_CONNECTION_KINDS` / `connectionKindLabel` / `connectionKindMarkers` bound the backends a route can honestly describe (`slotstream` and `vendor` deliberately excluded); `bindingProtocol` names the wire a binding needs. `materializeRoute({harness, method, serviceInstance, bootstrap?, selection?, overrides?})` (#7562) generalizes `buildRouteRecord` from a connection to a SERVICE instance: picks the harness's first compatible binding, writes base URL / credential / static env / backend markers / inline OpenCode config / Pi `--provider` per that binding plus the recipe's mode argv, stamps `harnessId` / `method` / `serviceId` / `servicePlan`, and — for a non-`api` method with a bootstrap app — the `credentialBootstrap` object `composeBootstrapSpawn` turns into `<bootstrap> run <harness> -- <args>`; throws typed errors (`HARNESS_SERVICE_INCOMPATIBLE`, `SERVICE_ENDPOINT_REQUIRED`, `SERVICE_CREDENTIAL_REQUIRED`, `SERVICE_CREDENTIAL_BOOTSTRAP_REQUIRED`) rather than minting a route that cannot run. `materializeRouteOutcome(input)` is the same body as a VERDICT (`{ record }` or `{ error }` — the typed 400 without the throw) for the composite resolver (#7564), which reads a refusal as the reason to show beside a saved selection. `routeDescribesService(profile, instance)` is the round-trip contract: fed back through `providerConnectionProfile`, a materialized record classifies onto the service it was built for. Pure: no I/O, no spawn, no provider call.| `providerRouteSettings.js` | The MODE-OVERRIDE half of a provider route (#6369) — the route-owned fields a human tunes per execution mode (`args`, `timeout`, `effort` and the `defaultModel`/`lightModel`/`mediumModel`/`heavyModel`/`ultraModel` pins), mirroring what the toolkit's `sharedModeUpdates` already declares mode-specific. `ROUTE_SETTING_FIELDS` / `ROUTE_SETTING_KEYS` / `routeSettingKeys(mode)` name the set (`args` reaches cli/tui only); `routeSettingsFor(provider)` reads it off a record with absent and cleared collapsing to `null`; `routeSettingsSchema` is the wire shape `GET /api/providers/management` publishes; `unsupportedRouteSettings` rejects a key the route's mode does not accept; and `routeSettingsRevision` is the stale-edit fingerprint — a hash of the values as they are in providers.json rather than a graph row revision, so an edit made through `/ai/edit/:providerId` is caught too. Connection-owned values and execution consent (`enabled`) are deliberately absent. Pure: no I/O.diff --git a/server/lib/aiToolkit/index.js b/server/lib/aiToolkit/index.jsindex b846bb1797..be344cd3db 100644--- a/server/lib/aiToolkit/index.js+++ b/server/lib/aiToolkit/index.js@@ -65,6 +65,8 @@ export function createAIToolkit(config = {}) {// Host gate for "can this provider run at all?" — see providerStatus.js.// Unset standalone, so the toolkit routes on enabled + availability alone.prerequisitesMet = null,+ // Host ordered fallback list — see providerStatus.js. Unset standalone.+ getFallbackChain = null,// Host persistence hook fired after every providers.json write — see// createProviderService(). PortOS keeps its provider connection graph// reconciled through it; unset standalone.@@ -93,6 +95,7 @@ export function createAIToolkit(config = {}) {statusFile,defaultFallbackPriority,prerequisitesMet,+ getFallbackChain,onStatusChange: (eventData) => {io?.emit('provider:status:changed', eventData);}diff --git a/server/lib/aiToolkit/providerStatus.js b/server/lib/aiToolkit/providerStatus.jsindex 9eb6817318..c5425d662a 100644--- a/server/lib/aiToolkit/providerStatus.js+++ b/server/lib/aiToolkit/providerStatus.js@@ -198,9 +198,27 @@ export function createProviderStatusService(config = {}) {// PATH probe) and this directory stays self-contained. It MUST be// synchronous and MUST default to "yes": an unknown answer has to route// exactly as it did before, never take a working provider out of the chain.- prerequisitesMet = null+ prerequisitesMet = null,+ // Host-supplied `() => [{ providerId, model?, effort? }]` — the user's ONE+ // ordered fallback list. When it is non-empty it replaces the per-provider+ // `fallbackProvider` and the system priority list: whichever entry the run+ // started on, the next entry (wrapping past the end to the first) is tried.+ // Sync by contract (a cache read); a throwing or non-array answer reads as+ // "no chain" so the legacy tiers still route.+ getFallbackChain = null} = config;+ function readFallbackChain() {+ if (typeof getFallbackChain !== 'function') return [];+ try {+ const chain = getFallbackChain();+ return Array.isArray(chain) ? chain.filter(e => e && typeof e.providerId === 'string' && e.providerId) : [];+ } catch (err) {+ console.error(`❌ Fallback chain read failed: ${err.message}`);+ return [];+ }+ }+const STATUS_PATH = join(dataDir, statusFile);const events = new EventEmitter();@@ -566,6 +584,26 @@ export function createProviderStatusService(config = {}) {if (accepted) return accepted;}+ const chain = readFallbackChain();+ if (chain.length > 0) {+ // Rotate the list so it starts just after the entry the run began on.+ // Match on provider AND model when the caller names the model, so two+ // entries of one provider (different models, different limits) are+ // distinct steps; an unlisted primary starts at the top.+ const primaryModel = requestCapabilities?.primaryModel ?? null;+ let at = primaryModel+ ? chain.findIndex(e => e.providerId === primaryProviderId && e.model === primaryModel)+ : -1;+ if (at < 0) at = chain.findIndex(e => e.providerId === primaryProviderId);+ const ordered = at < 0 ? chain : [...chain.slice(at + 1), ...chain.slice(0, at)];+ for (const entry of ordered) {+ const accepted = acceptCandidate(providers[entry.providerId], 'chain', entry.model || null);+ if (accepted) return { ...accepted, effort: entry.effort || null };+ }+ if (capabilityRejections.length > 0) throw noEligibleFallbackError(capabilityRejections);+ return null;+ }+const primaryProvider = providers[primaryProviderId];// Guard against `fallbackProvider === self` — a misconfigured provider// would otherwise loop back to itself and silently retry the samediff --git a/server/lib/aiToolkit/providerStatus.test.js b/server/lib/aiToolkit/providerStatus.test.jsindex 4faee0a9f5..83533a4140 100644--- a/server/lib/aiToolkit/providerStatus.test.js+++ b/server/lib/aiToolkit/providerStatus.test.js@@ -996,3 +996,49 @@ describe('Provider Status Service', () => {});});});++describe('getFallbackProvider — global ordered fallback list', () => {+ const providers = Object.fromEntries(['a', 'b', 'c'].map((id) => [id, { id, enabled: true, models: ['m1', 'm2'], fallbackProvider: 'c' }]));+ let dir;+ let chain;+ let svc;++ beforeEach(async () => {+ dir = await mkdtemp(join(tmpdir(), 'portos-fallback-chain-'));+ chain = [{ providerId: 'a' }, { providerId: 'b', model: 'm1', effort: 'max' }, { providerId: 'c' }];+ svc = createProviderStatusService({ dataDir: dir, statusFile: 's.json', getFallbackChain: () => chain });+ await svc.init();+ });+ afterEach(() => rm(dir, { recursive: true, force: true }));++ it('continues after the failed entry and carries the step model and effort', () => {+ expect(svc.getFallbackProvider('a', providers)).toMatchObject({ source: 'chain', model: 'm1', effort: 'max', provider: { id: 'b' } });+ });++ it('wraps past the last entry to the first instead of using per-provider links', () => {+ expect(svc.getFallbackProvider('c', providers).provider.id).toBe('a');+ });++ it('starts at the top for a provider that is not listed', () => {+ expect(svc.getFallbackProvider('z', providers).provider.id).toBe('a');+ });++ it('skips unavailable entries and returns null when none can run', async () => {+ await svc.markUnavailable('b', { reason: 'usage-limit', waitTimeMs: 60000 });+ expect(svc.getFallbackProvider('a', providers).provider.id).toBe('c');+ await svc.markUnavailable('c', { reason: 'usage-limit', waitTimeMs: 60000 });+ await svc.markUnavailable('a', { reason: 'usage-limit', waitTimeMs: 60000 });+ expect(svc.getFallbackProvider('a', providers)).toBeNull();+ });++ it('lets one preset repeat with different models, matched by the primary model', () => {+ chain = [{ providerId: 'a', model: 'm1' }, { providerId: 'a', model: 'm2' }, { providerId: 'b' }];+ expect(svc.getFallbackProvider('a', providers, null, null, { primaryModel: 'm1' })).toMatchObject({ provider: { id: 'a' }, model: 'm2' });+ expect(svc.getFallbackProvider('a', providers, null, null, { primaryModel: 'm2' }).provider.id).toBe('b');+ });++ it('falls back to the legacy tiers when the list is empty', () => {+ chain = [];+ expect(svc.getFallbackProvider('a', providers)).toMatchObject({ source: 'provider', provider: { id: 'c' } });+ });+});diff --git a/server/lib/index.js b/server/lib/index.jsindex 65da717b9e..6ea152a6a7 100644--- a/server/lib/index.js+++ b/server/lib/index.js@@ -576,6 +576,7 @@ export * from './providerServiceMerge.js';export * from './providerModelAliases.js';export * from './providerModePairing.js';export * from './providerPresets.js';+export * from './providerFallbackChainCache.js';export * from './providerRef.js';export * from './providerRouteRecipes.js';export * from './serviceDefinitions.js';diff --git a/server/lib/navManifest.js b/server/lib/navManifest.jsindex 8dbf73fe4c..a67d4cad62 100644--- a/server/lib/navManifest.js+++ b/server/lib/navManifest.js@@ -332,6 +332,7 @@ const RAW_NAV_COMMANDS = [// prefix: they are opaque and stored in palette history, so renaming them// would orphan those entries — only the path, label and section move.{ id: 'nav.models.comparison', path: '/models/comparison', label: 'Comparison', section: 'Models', tabId: 'comparison', navGroup: 'Evaluate', aliases: ['model-comparison', 'model-cost', 'model-intelligence'], keywords: ['benchmark', 'effort', 'cost', 'quota', 'compare', 'intelligence'] },+ { id: 'nav.models.fallbacks', path: '/models/fallbacks', label: 'Fallbacks', section: 'Models', tabId: 'fallbacks', navGroup: 'Connect', aliases: ['fallbacks', 'provider-fallbacks', 'fallback-list', 'fallback-order'], keywords: ['failover', 'preset order', 'retry', 'next provider', 'usage limit', 'ordered list', 'wrap'] },{ id: 'nav.models.3d', path: '/models/3d', label: '3D', section: 'Models', tabId: '3d', navGroup: 'Library', aliases: ['3d-runtimes', 'image-to-3d-runtimes', 'trellis-install', 'pixal3d-install'], keywords: ['trellis', 'pixal3d', 'install', 'repair', 'runtime', 'mesh', 'image to 3d', 'on-device'] },{ id: 'nav.models.decision-classifiers', path: '/models/decision-classifiers', label: 'Decision Classifiers', section: 'Models', tabId: 'decision-classifiers', navGroup: 'Policies', aliases: ['decision-classifiers', 'classifiers'], keywords: ['jev', 'laya', 'mlx', 'decision systems', 'experiments'] },{ id: 'nav.models.laya-mlx', path: '/models/decision-classifiers/laya-mlx', label: 'Laya-MLX', section: 'Models', aliases: ['laya', 'laya-mlx', 'layla'], keywords: ['classifier', 'apple silicon', 'typed decisions'] },diff --git a/server/lib/navManifest.test.js b/server/lib/navManifest.test.jsindex d9e8af743d..b445831487 100644--- a/server/lib/navManifest.test.js+++ b/server/lib/navManifest.test.js@@ -165,7 +165,7 @@ describe('nav contract — generated section child navigation', () => {it('keeps Models groups and destinations in the design order', () => {expect(getSectionNavGroups('Models').map(({ label, tabs }) => [label, tabs.map((tab) => tab.label)])).toEqual([- ['Connect', ['AI Services', 'Harnesses', 'Providers', 'Runtimes']],+ ['Connect', ['AI Services', 'Fallbacks', 'Harnesses', 'Providers', 'Runtimes']],['Evaluate', ['Comparison', 'Performance', 'Playground']],['Library', ['3D', 'Embeddings', 'LoRAs', 'Media', 'Model Library', 'Training']],['Operate', ['Quota Burn', 'Status', 'Subscriptions', 'Usage']],diff --git a/server/lib/providerFallbackChainCache.js b/server/lib/providerFallbackChainCache.jsnew file mode 100644index 0000000000..7169213100--- /dev/null+++ b/server/lib/providerFallbackChainCache.js@@ -0,0 +1,24 @@+/**+ * Process cache of the user's ordered provider fallback list. Kept free of+ * imports so routing code (promptRunner, agent resolution, bootstrap) can read+ * it synchronously without pulling in settings or validation. The settings+ * binding that fills it lives in `services/providerFallbackChain.js`.+ */++let chain = [];++/** Normalize to `[{ providerId, model?, effort? }]`, dropping malformed rows. */+export const normalizeFallbackChain = (entries) => (Array.isArray(entries) ? entries : [])+ .filter((e) => e && typeof e.providerId === 'string' && e.providerId)+ .map(({ providerId, model, effort }) => ({+ providerId,+ ...(model ? { model } : {}),+ ...(effort ? { effort } : {}),+ }));++export const peekFallbackChain = () => chain;++export const setFallbackChainCache = (entries) => {+ chain = normalizeFallbackChain(entries);+ return chain;+};diff --git a/server/lib/validation.js b/server/lib/validation.jsindex 4be72ba8ec..4e53b17bc1 100644--- a/server/lib/validation.js+++ b/server/lib/validation.js@@ -2847,3 +2847,15 @@ export const voicePiperFetchBodySchema = z.object({ voice: nonEmptyString });export const writersRoomSceneImageBodySchema = z.object({sceneId: nullishString, filename: nullishString, jobId: nullishString, prompt: nullishString,});++// One step of the global provider fallback list. `providerId` is a preset or+// composite reference; `model` / `effort` are optional pins for that step, so+// one preset may appear several times with different models or effort.+export const providerFallbackEntrySchema = z.object({+ providerId: providerRefSchema,+ model: z.preprocess((v) => (v === '' ? undefined : v), z.string().trim().min(1).max(200).optional()),+ effort: z.preprocess((v) => (v === '' ? undefined : v), z.string().trim().min(1).max(40).optional()),+});+export const providerFallbackChainBodySchema = z.object({+ entries: z.array(providerFallbackEntrySchema).max(50),+});diff --git a/server/routes/providerFallbacks.js b/server/routes/providerFallbacks.jsnew file mode 100644index 0000000000..069d5aaf1b--- /dev/null+++ b/server/routes/providerFallbacks.js@@ -0,0 +1,17 @@+import { Router } from 'express';+import { asyncHandler } from '../lib/errorHandler.js';+import { validateRequest, providerFallbackChainBodySchema } from '../lib/validation.js';+import { getFallbackChain, saveFallbackChain } from '../services/providerFallbackChain.js';++const router = Router();++router.get('/', asyncHandler(async (req, res) => {+ res.json({ entries: await getFallbackChain() });+}));++router.put('/', asyncHandler(async (req, res) => {+ const body = validateRequest(providerFallbackChainBodySchema, req.body ?? {});+ res.json({ entries: await saveFallbackChain(body) });+}));++export default router;diff --git a/server/routes/providerFallbacks.test.js b/server/routes/providerFallbacks.test.jsnew file mode 100644index 0000000000..0c5364a4e1--- /dev/null+++ b/server/routes/providerFallbacks.test.js@@ -0,0 +1,32 @@+import { describe, it, expect, vi, beforeEach } from 'vitest';+import express from 'express';+import { request } from '../lib/testHelper.js';+import { errorMiddleware } from '../lib/errorHandler.js';++const mocks = vi.hoisted(() => ({ getFallbackChain: vi.fn(), saveFallbackChain: vi.fn() }));+vi.mock('../services/providerFallbackChain.js', () => mocks);++const routes = (await import('./providerFallbacks.js')).default;+const app = express();+app.use(express.json());+app.use('/api/provider-fallbacks', routes);+app.use(errorMiddleware);++beforeEach(() => vi.clearAllMocks());++describe('provider fallback list routes', () => {+ it('saves an ordered list that repeats a preset with different models', async () => {+ const entries = [{ providerId: 'codex', model: 'm1', effort: 'max' }, { providerId: 'codex', model: 'm2' }, { providerId: 'agy' }];+ mocks.saveFallbackChain.mockResolvedValue(entries);+ const res = await request(app).put('/api/provider-fallbacks').send({ entries });+ expect(res.status).toBe(200);+ expect(mocks.saveFallbackChain).toHaveBeenCalledWith({ entries });+ expect(res.body.entries).toEqual(entries);+ });++ it('rejects a malformed provider reference before it reaches the store', async () => {+ const res = await request(app).put('/api/provider-fallbacks').send({ entries: [{ providerId: 'Not A Preset!' }] });+ expect(res.status).toBe(400);+ expect(mocks.saveFallbackChain).not.toHaveBeenCalled();+ });+});diff --git a/server/services/agentProviderResolution.js b/server/services/agentProviderResolution.jsindex 22bc9eb027..2505587b28 100644--- a/server/services/agentProviderResolution.js+++ b/server/services/agentProviderResolution.js@@ -1,6 +1,7 @@import { isForgeMaintenanceTask, hasCurrentForgeMaintenanceEvidence, LEGACY_FORGE_MAINTENANCE_REASON } from '../lib/forgeMaintenanceTasks.js';import { isProcessProvider } from '../lib/providerTypes.js';import { isCompositeProviderId } from '../lib/providerRef.js';+import { peekFallbackChain } from '../lib/providerFallbackChainCache.js';import { isPrivateSecurityTask, PRIVATE_SECURITY_DELIVERY } from '../lib/privateSecurityPolicy.js';import { supportsPublicReviewProvider } from '../lib/providerVendors.js';/**@@ -235,6 +236,10 @@ async function resolveOrdinaryProviderAndModel(task) {// The caller's mode policy travels WITH the fallback request, so an// ineligible candidate is skipped during selection rather than picked and// then rejected below — which used to burn the one retry the cascade had.+ const chainComposites = peekFallbackChain().map((e) => e.providerId).filter(isCompositeProviderId);+ if (chainComposites.length) {+ await (await import('./compositeProviders.js')).withCompositeCandidates(providersMap, chainComposites);+ }const fallbackResult = await getFallbackProvider(provider.id, providersMap, taskFallbackId, taskFallbackModel, {allowedModes: allowedModesFor(AGENT_CALLER_POLICY),});diff --git a/server/services/bootstrap.js b/server/services/bootstrap.jsindex bed6a0a3f4..e6aa7b337f 100644--- a/server/services/bootstrap.js+++ b/server/services/bootstrap.js@@ -180,6 +180,8 @@ import { outcomesStore as liOutcomesStore } from './layeredIntelligenceOutcomes.import * as gameStore from './games/store.js';import * as fableLoomStore from './fableLoom/store.js';import { prerequisitesMetForRouting } from './providerPrerequisites.js';+import { peekFallbackChain } from '../lib/providerFallbackChainCache.js';+import { initFallbackChain } from './providerFallbackChain.js';import { localCachedModelIds } from './localCachedModels.js';import { stopCodexAppServer } from './codexAppServer.js';import { logFailureWithStack as logBootstrapFailure } from '../lib/failureLogging.js';@@ -323,6 +325,9 @@ export const bootstrapServices = async ({ io, dataDir, dataReferenceDir, serverD// of dying at spawn time. Sync by contract: it reads the runtime probe's// cache and never blocks.prerequisitesMet: prerequisitesMetForRouting,+ // The user's single ordered fallback list (Models > Fallbacks); a cache+ // read, so it is sync as the toolkit requires.+ getFallbackChain: peekFallbackChain,// MTPLX and Slotstream each serve one checkpoint per process and report// only that one, so a refresh could never surface a newly downloaded// checkpoint — and for Slotstream it pruned the record's other shipped ids.@@ -351,7 +356,11 @@ export const bootstrapServices = async ({ io, dataDir, dataReferenceDir, serverD// Warm the providers file at startup so the codex-sentinel migration runs// before any inbound request can hit the providers cache. A pure READ of// on-disk provider config — it dispatches nothing to a provider.- warmProviders: (aiToolkit) => aiToolkit.services.providers.getAllProviders(),+ warmProviders: async (aiToolkit) => {+ // Prime the sync fallback-list cache from settings.json (disk only).+ await initFallbackChain().catch((err) => logBootstrapFailure('⚠️ Failed to load provider fallback list', err));+ return aiToolkit.services.providers.getAllProviders();+ },// Ensure the provider paired with the active local-LLM backend (LLM_BACKEND in// .env, chosen at setup time) is enabled, so a fresh install can use Ollama /diff --git a/server/services/promptRunner.js b/server/services/promptRunner.jsindex c4de382dd1..eff890daff 100644--- a/server/services/promptRunner.js+++ b/server/services/promptRunner.js@@ -31,6 +31,7 @@ import { isVisionCapableCodexTuiProvider } from '../lib/codex.js';import { createRun, executeApiRun, executeCliRun, extractBakedModel, hasModelFlag, stopRun, patchRunMetadata, finalizeRunRecord } from './runner.js';import { getActiveProvider, getProviderById, getAllProviders } from './providers.js';import { isCompositeProviderId } from '../lib/providerRef.js';+import { peekFallbackChain } from '../lib/providerFallbackChainCache.js';// `./tuiPromptRunner.js` (which drags node-pty in through `./shell.js`) and// `./providerExecutionReadiness.js` are imported lazily on the branches that// actually execute a run — see their call sites below. promptRunner.js is reached@@ -1099,6 +1100,9 @@ async function runPromptThroughProviderAdmitted(rawArgs) {...args,provider: fallback,model: picked.model ?? undefined,+ // A step in the global fallback list may pin its own effort; the+ // primary's effort belongs to the primary's model, so it is not inherited.+ ...(picked.effort ? { effort: picked.effort } : {}),runId: undefined, // fresh runId so the failed primary's record stays intact});} catch (fallbackError) {@@ -1212,6 +1216,10 @@ async function pickFallbackProvider(failed, requestCapabilities) {if (isCompositeProviderId(failed.id)) {await (await import('./compositeProviders.js')).withCompositeCandidates(providersMap, [failed.id]);}+ const chainComposites = peekFallbackChain().map((e) => e.providerId).filter(isCompositeProviderId);+ if (chainComposites.length) {+ await (await import('./compositeProviders.js')).withCompositeCandidates(providersMap, chainComposites);+ }const picked = providerStatus.getFallbackProvider(failed.id, providersMap, null, null, requestCapabilities);if (!picked?.provider) return null;@@ -1220,7 +1228,7 @@ async function pickFallbackProvider(failed, requestCapabilities) {// without its sibling's apiKey and fail auth (NVIDIA NIM 401s "Header of type// `authorization` was missing") despite a stored key. Attach it from the same// map (synchronous — no re-read to go stale).- return { provider: attachGatewaySiblingKey(picked.provider, providersMap), model: picked.model ?? null };+ return { provider: attachGatewaySiblingKey(picked.provider, providersMap), model: picked.model ?? null, effort: picked.effort ?? null };}/**diff --git a/server/services/providerFallbackChain.js b/server/services/providerFallbackChain.jsnew file mode 100644index 0000000000..6c0e757b7c--- /dev/null+++ b/server/services/providerFallbackChain.js@@ -0,0 +1,35 @@+import { getSettings, settingsEvents, updateSettings } from './settings.js';+import { normalizeFallbackChain, setFallbackChainCache } from '../lib/providerFallbackChainCache.js';++/**+ * The ONE ordered provider fallback list (settings key `providerFallbackChain`).+ *+ * Whichever entry a run started on, the toolkit's `getFallbackProvider` tries+ * the entries after it in order and wraps past the end to the first, so the+ * list replaces the old per-provider `fallbackProvider` links (which could form+ * cycles). An entry names a preset/composite id plus an optional model and+ * effort; the same preset may repeat with different models.+ *+ * The toolkit reads it synchronously, so a module cache is kept coherent from+ * the `settings:updated` event and primed once at boot (disk only, no LLM call).+ */++settingsEvents.on('settings:updated', (settings) => {+ setFallbackChainCache(settings?.providerFallbackChain);+});++export async function initFallbackChain() {+ const cache = setFallbackChainCache((await getSettings()).providerFallbackChain);+ if (cache.length) console.log(`🔗 Provider fallback list loaded (${cache.length} entries)`);+ return cache;+}++export const getFallbackChain = async () => normalizeFallbackChain((await getSettings()).providerFallbackChain);++// `entries` arrive validated by the route (providerFallbackChainBodySchema).+export async function saveFallbackChain({ entries }) {+ const next = normalizeFallbackChain(entries);+ await updateSettings({ providerFallbackChain: next });+ setFallbackChainCache(next);+ return next;+}