Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

feat: single ordered provider fallback list under Models > Fallbacks

[._.]/ Adam Eivycd567c540a78
diff --git a/client/src/components/models/ProviderFallbacks.jsx b/client/src/components/models/ProviderFallbacks.jsx
new file mode 100644
index 0000000000..2f004ebdc9
--- /dev/null
+++ b/client/src/components/models/ProviderFallbacks.jsx
@@ -0,0 +1,120 @@
+import { useCallback, useEffect, useMemo, useState } from 'react';
+import { ArrowDown, ArrowUp, ListOrdered, Plus, Save, Trash2 } from 'lucide-react';
+import toast from '../ui/Toast';
+import * as api from '../../services/api';
+import PageSkeleton from '../ui/PageSkeleton';
+import ProviderModelSelector from '../ProviderModelSelector';
+
+/**
+ * Fallbacks — ONE ordered list of provider presets tried in turn when a run
+ * fails or its provider is unavailable, whichever preset the run started on.
+ * The step after the one that failed runs next; past the last step the list
+ * wraps to the first. A preset may appear more than once with different models
+ * or effort (limits and availability differ per model). An empty list keeps the
+ * legacy per-preset fallback behavior.
+ */
+
+export default function ProviderFallbacks() {
+ const [providers, setProviders] = useState([]);
+ const [entries, setEntries] = useState(null);
+ const [dirty, setDirty] = useState(false);
+ const [saving, setSaving] = useState(false);
+
+ useEffect(() => {
+ Promise.all([api.getProviders(), api.getProviderFallbacks()])
+ .then(([provs, chain]) => {
+ setProviders((provs.providers || []).filter((p) => p.enabled !== false));
+ setEntries(chain.entries || []);
+ })
+ .catch(() => setEntries([]));
+ }, []);
+
+ const byId = useMemo(() => Object.fromEntries(providers.map((p) => [p.id, p])), [providers]);
+
+ const update = useCallback((next) => { setEntries(next); setDirty(true); }, []);
+ const patch = (i, changes) => update(entries.map((e, at) => (at === i ? { ...e, ...changes } : e)));
+ const move = (i, delta) => {
+ const next = [...entries];
+ const [item] = next.splice(i, 1);
+ next.splice(i + delta, 0, item);
+ update(next);
+ };
+ const add = () => {
+ if (!providers.length) return;
+ update([...entries, { providerId: providers[0].id }]);
+ };
+
+ const save = async () => {
+ setSaving(true);
+ const result = await api.saveProviderFallbacks(entries).catch(() => null);
+ setSaving(false);
+ if (!result) return;
+ setEntries(result.entries);
+ setDirty(false);
+ toast.success('Fallback list saved');
+ };
+
+ if (!entries) return <PageSkeleton header="none" label="Loading fallbacks" cards={2} sidebar={false} />;
+
+ return (
+ <div className="mx-auto w-full max-w-4xl space-y-4">
+ <p className="text-sm text-gray-400">
+ When a run fails or its provider is unavailable, PortOS tries the next preset in this list after the one that
+ failed, wrapping from the last back to the first. The same preset can appear several times with different
+ models. With an empty list, each preset&apos;s own fallback setting applies.
+ </p>
+
+ {entries.length === 0 && (
+ <div className="rounded border border-dashed border-port-border p-6 text-center text-sm text-gray-400">
+ <ListOrdered size={20} aria-hidden="true" className="mx-auto mb-2" />
+ No fallback list yet.
+ </div>
+ )}
+
+ <ol className="space-y-2">
+ {entries.map((entry, i) => {
+ const provider = byId[entry.providerId];
+ return (
+ <li key={`${i}-${entry.providerId}`} className="flex flex-wrap items-end gap-2 rounded border border-port-border bg-port-card p-3">
+ <span className="w-6 pb-2 text-sm text-gray-400">{i + 1}.</span>
+ <div className="min-w-0 flex-1">
+ <ProviderModelSelector
+ providers={providers}
+ selectedProviderId={entry.providerId}
+ selectedModel={entry.model || ''}
+ availableModels={provider?.models || []}
+ onProviderChange={(id) => patch(i, { providerId: id, model: undefined, effort: undefined })}
+ onModelChange={(model) => patch(i, { model: model || undefined, effort: undefined })}
+ effort={entry.effort || ''}
+ onEffortChange={(effort) => patch(i, { effort: effort || undefined })}
+ emptyModelOption="Preset default"
+ alwaysShowModel
+ label="Preset"
+ />
+ </div>
+ <div className="flex gap-1">
+ <button type="button" aria-label={`Move step ${i + 1} up`} disabled={i === 0} onClick={() => move(i, -1)}
+ className="rounded border border-port-border p-2 text-gray-300 hover:text-white disabled:opacity-30"><ArrowUp size={14} aria-hidden="true" /></button>
+ <button type="button" aria-label={`Move step ${i + 1} down`} disabled={i === entries.length - 1} onClick={() => move(i, 1)}
+ className="rounded border border-port-border p-2 text-gray-300 hover:text-white disabled:opacity-30"><ArrowDown size={14} aria-hidden="true" /></button>
+ <button type="button" aria-label={`Remove step ${i + 1}`} onClick={() => update(entries.filter((_, at) => at !== i))}
+ className="rounded border border-port-border p-2 text-port-error hover:bg-port-error/10"><Trash2 size={14} aria-hidden="true" /></button>
+ </div>
+ </li>
+ );
+ })}
+ </ol>
+
+ <div className="flex flex-wrap gap-2">
+ <button type="button" onClick={add} disabled={!providers.length}
+ className="inline-flex items-center gap-1.5 rounded border border-port-border px-3 py-2 text-sm text-gray-200 hover:text-white disabled:opacity-50">
+ <Plus size={14} aria-hidden="true" /> Add step
+ </button>
+ <button type="button" onClick={save} disabled={!dirty || saving}
+ className="inline-flex items-center gap-1.5 rounded bg-port-accent px-3 py-2 text-sm text-white disabled:opacity-50">
+ <Save size={14} aria-hidden="true" /> {saving ? 'Saving…' : 'Save'}
+ </button>
+ </div>
+ </div>
+ );
+}
diff --git a/client/src/lib/navPresentation.js b/client/src/lib/navPresentation.js
index 8456bef9c9..04e5011a11 100644
--- a/client/src/lib/navPresentation.js
+++ b/client/src/lib/navPresentation.js
@@ -100,6 +100,7 @@ import { AudioLines,
UserRound,
Video,
ListTodo,
+ ListOrdered,
Plug,
TerminalSquare,
} from 'lucide-react';
@@ -235,6 +236,7 @@ export const NAV_PRESENTATION = {
'/models/decision-classifiers': { icon: Scale },
'/models/decision-classifiers/jev': { icon: Scale },
'/models/embeddings': { icon: Braces },
+ '/models/fallbacks': { icon: ListOrdered },
'/models/llms': { icon: Cpu },
'/models/llms/abuse': { icon: ShieldAlert },
'/models/llms-runtimes': { icon: Server },
diff --git a/client/src/pages/Models.jsx b/client/src/pages/Models.jsx
index cc4c0ecd41..95c7b08de9 100644
--- a/client/src/pages/Models.jsx
+++ b/client/src/pages/Models.jsx
@@ -61,6 +61,7 @@ const TAB_CONTENT = {
'code-reviewers': CodeReviewersTab,
'decision-classifiers': lazyWithReload(() => import('../components/models/DecisionClassifiers')),
embeddings: EmbeddingsTab,
+ fallbacks: lazyWithReload(() => import('../components/models/ProviderFallbacks')),
llms: LocalLlmTab,
'llms-runtimes': LocalLlmRuntimesView,
loras: Loras,
diff --git a/client/src/pages/Models.test.jsx b/client/src/pages/Models.test.jsx
index 479838c8d2..12ed9400ea 100644
--- a/client/src/pages/Models.test.jsx
+++ b/client/src/pages/Models.test.jsx
@@ -28,6 +28,7 @@ vi.mock('../components/settings/LocalLlmTab', () => ({
}));
vi.mock('../components/settings/LocalLlmRuntimesView.jsx', () => ({ default: ({ view }) => <div data-testid="runtime-panel" data-runtime={view || ''}>runtimes panel</div> }));
vi.mock('../components/settings/EmbeddingsTab', () => ({ default: () => <div>embeddings panel</div> }));
+vi.mock('../components/models/ProviderFallbacks', () => ({ default: () => <div>fallbacks panel</div> }));
vi.mock('../components/models/Image3dRuntimes', () => ({ default: () => <div>3d runtimes panel</div> }));
vi.mock('../components/models/ModelStatusTab', () => ({ default: () => <div>status panel</div> }));
vi.mock('../components/models/SubscriptionsTab', () => ({ default: () => <div>subscriptions panel</div> }));
@@ -51,6 +52,7 @@ const PANEL_MARKER = {
'code-reviewers': 'code reviewers panel',
'decision-classifiers': 'classifiers panel',
embeddings: 'embeddings panel',
+ fallbacks: 'fallbacks panel',
llms: 'llms panel',
abuse: 'llms panel',
jev: 'llms panel',
diff --git a/client/src/services/apiProviders.js b/client/src/services/apiProviders.js
index 089a511f98..8981996f67 100644
--- a/client/src/services/apiProviders.js
+++ b/client/src/services/apiProviders.js
@@ -251,3 +251,11 @@ export const getFleetLlmHostUsage = (options) => request('/providers/fleet-host/
export const stopFleetLlmHost = (options) => request('/providers/fleet-host/stop', { method: 'POST', ...options });
export const getFleetPeerHosts = (options) => request('/providers/fleet-peer-hosts', options);
export const revealFleetPeerHostKey = (peerId, options) => request(`/providers/fleet-peer-hosts/${encodeURIComponent(peerId)}/key`, { method: 'POST', ...options });
+
+// The single ordered provider fallback list (Models > Fallbacks).
+export const getProviderFallbacks = (options) => request('/provider-fallbacks', options);
+export const saveProviderFallbacks = (entries, options) => request('/provider-fallbacks', {
+ method: 'PUT',
+ body: JSON.stringify({ entries }),
+ ...options
+});
diff --git a/docs/API.md b/docs/API.md
index fb6ce52bcf..807f07c0d1 100644
--- a/docs/API.md
+++ b/docs/API.md
@@ -837,6 +837,7 @@ Every mounted API prefix (see `server/index.js` for the authoritative list). Dom
| `/api/standardize` | App PM2 standardizer |
| `/api/stacker-news`, `/api/x` | Social integrations |
| `/api/model-personality` | LLM personality tests |
+| `/api/provider-fallbacks` | GET/PUT the single ordered provider fallback list `{ entries: [{ providerId, model?, effort? }] }` (Models > Fallbacks); the step after the failed one runs next, wrapping to the first |
| `/api/providers/comparison` | Selectable model inventory, sourced PortOS composite, and public evidence; POST `/import` merges researched observations (see [MODEL-COMPARISON.md](./MODEL-COMPARISON.md)) |
| `/api/models/performance/task-benchmark` | Machine-local PortOS task benchmark history and configured-provider inventory; `POST /discover` and `POST /run` perform explicit user-requested actions |
| `/api/browser` | Managed Chromium |
diff --git a/server/index.js b/server/index.js
index b5e98423c9..aee03b4154 100644
--- a/server/index.js
+++ b/server/index.js
@@ -1,4 +1,5 @@
import maintenanceRoutes from './routes/maintenance.js';
+import providerFallbackRoutes from './routes/providerFallbacks.js';
import './services/databaseBootFence.js';
import express from 'express';
import { Server } from 'socket.io';
@@ -324,6 +325,7 @@ app.use('/api/detect', detectRoutes);
app.use('/api/scaffold', scaffoldRoutes);
// AI Toolkit routes with PortOS extensions
+app.use('/api/provider-fallbacks', providerFallbackRoutes);
app.use('/api/providers/comparison', createModelComparisonRoutes());
app.use('/api/models/performance/task-benchmark', createModelPerformanceBenchmarkRoutes(aiToolkit.services.providers));
app.use('/api/providers', createPortOSProviderRoutes(aiToolkit));
diff --git a/server/lib/README.md b/server/lib/README.md
index e1fe52e70f..089c2f4b67 100644
--- a/server/lib/README.md
+++ b/server/lib/README.md
@@ -286,6 +286,7 @@ The barrel `server/lib/index.js` is a machine-checkable enumeration of every pub
| `providerModelAliases.js` | HAND-AUTHORED canonical→executable model aliases for one provider route (#6369) — the correction surface for a `modelMap` a refresh could only fill with what it could VERIFY. Observed aliases (`ai_route_bindings.model_map`) and user overrides (`model_alias_overrides`) are stored apart and merged on READ by `effectiveModelAliases` (`{ ...observed, ...overrides }`), which is what lets a hand-authored alias outlast the next refresh; `applyModelAliasPatch` is the three-valued edit (absent preserves, a string sets, `null` is the only thing that removes); `staleModelAliases` reports — never repairs — an override naming a spelling the record no longer lists, and makes no claim at all about a route listing no models; `connectionCatalogModels` is the union an alias write re-derives the shared catalog from, so a correction actually reaches the model menus; `modelAliasRevision` is the stale-edit fingerprint, reusing `routeSettingsRevision` because `ai_route_bindings` has no revision column. Pure: no I/O. |
| `providerModePairing.js` | May an existing record gain the OTHER execution mode of its harness (#7501) — `tuiModeAddition(provider, providers, executionModes?)` is the ONE verdict behind both the `canAddTuiMode` flag `GET /providers` decorates each record with (so the card offers "Add interactive mode") and the refusal `POST /providers/:id/modes/tui` answers with, so a card can never advertise an action the server rejects. Refuses a non-CLI record (the CLI id is the stem, so minting it from a TUI record would be a rename), a record storing no command, and a KNOWN harness declaring no TUI mode; an UNKNOWN harness passes with empty argv rather than a guess, while a known one starts from its recipe’s proven interactive line. Asks whether the harness already HAS its interactive mode (the record’s `providerModeGroups` group — the shipped `grok-cli`/`grok-tui` pair leaves `grok-cli-tui` unclaimed, so an id-only check would mint a third record beside it), and separately 409s on a taken `<id>-tui` rather than suffixing around it. Pure: no I/O. |
| `providerPresets.js` | PRESETS (#7565, epic #7561) — a stored `data/providers.json` record read as a (harness, method, service) tuple plus the user's own defaults and overrides. A record carrying `harnessId` + `method` + `serviceId` is a DERIVED preset (`isDerivedPreset` in `providerGraphRecords.js`, `presetKind`): `materializeDerivedPreset({record, harness, instance, catalog, bootstrap})` re-derives every connection-owned value (program, endpoint, credential, backend markers, inline bootstrap, `models` = plan-filtered catalog ∩ `catalogNarrowing`) through `materializeRoute` while keeping every per-preset field (name, argv with a binding's suffix preserved, timeouts, pins, effort, fallback, generation params, consent flags, unknown fields) and the record's own OpenCode inline config when it already reaches the service; `refusedDerivedEdits(updates, derived, ownedEnvNames)` names the connection-owned keys a direct edit would move (`DERIVED_PRESET_OWNED_KEYS`, plus env vars by name); `derivedPresetDrift` / `derivedPresetPatch` compare a record with its derivation; `presetDerivable(record)` is the editor's "offer the button" test; `matchBootstrapApp` / `bootstrapInputFor` map an inline `credentialBootstrap` to a configured app and back; `planPresetBackfill({graph, providers, bootstraps, env})` is the boot-time stamp of every routed legacy record whose re-derivation is a FIXPOINT — additive keys only, every refusal reported with a reason. Pure; the store-backed half is `services/providerPresets.js`. `impliedCatalogNarrowing` is the narrowing that keeps a record on its current list once derived, and `rederivePreset` re-derives one derived preset from an instance (derived record + patch), shared by the backfill, `rematerializeDerivedPresets` and the service fold. |
+| `providerFallbackChainCache.js` | Import-free process cache of the user's single ordered provider fallback list (`peekFallbackChain` sync read for routing, `setFallbackChainCache`, `normalizeFallbackChain`). Filled by `services/providerFallbackChain.js` from settings `providerFallbackChain`; read by the toolkit's `getFallbackProvider`. |
| `providerRef.js` | Provider REFERENCE grammar (#7564, epic #7561) — the two spellings a `{ providerId, model, effort }` selection may name a provider by: a PRESET record id (`PRESET_ID_RE`, the slug `createProvider` mints) or a COMPOSITE `<harness>.<cli|tui|api>@<service-slug>[+<bootstrap-slug>]` (`COMPOSITE_ID_RE`; the bootstrap suffix only with a process method). `parseProviderRef(id)` → `{kind:'preset'}` / `{kind:'composite', harnessId, method, serviceSlug, bootstrapSlug}` / `null`, `isCompositeProviderId` / `isPresetProviderId`, `formatCompositeId(parts)` (throws on parts that would not parse back), `MAX_PROVIDER_REF_LENGTH`, and the two validation messages (`PROVIDER_REF_MESSAGE`, `PRESET_ONLY_MESSAGE`). The zod forms — `providerRefSchema` / `providerRefFieldSchema` (either grammar) and `presetProviderIdSchema` (preset-only, for `PUT /providers/active`, a record's `fallbackProvider`, an app's `taskTypeOverrides` pin) — live in `zodCompat.js` so `cosValidation.js` can share them. Pure and dependency-free; mirrored in `aiToolkit/internal/providerRef.js` (the toolkit may not import out) behind `providerRef.parity.test.js`. Resolution is host-side: `services/compositeProviders.js`. |
| `providerRouteRecipes.js` | Minting a FRESH executable route from a connection and a harness command recipe (#6369) — the reverse of `providerConnections.js`, which reads an existing record. `buildRouteRecord({harnessId, mode, providerId, name, connection})` materializes a connection's endpoint, credentials and backend marker into a DISABLED `data/providers.json` record through `PROVIDER_HARNESSES[].recipe`; `mintRouteIds` allocates readable ids keeping the `<stem>` / `<stem>-tui` pairing `providerModeGroups` recognizes; `bindingBlocker` is the pre-write refusal (no recipe, unsupported mode, a transport the backend does not declare, a credential the program requires); `CREATABLE_CONNECTION_KINDS` / `connectionKindLabel` / `connectionKindMarkers` bound the backends a route can honestly describe (`slotstream` and `vendor` deliberately excluded); `bindingProtocol` names the wire a binding needs. `materializeRoute({harness, method, serviceInstance, bootstrap?, selection?, overrides?})` (#7562) generalizes `buildRouteRecord` from a connection to a SERVICE instance: picks the harness's first compatible binding, writes base URL / credential / static env / backend markers / inline OpenCode config / Pi `--provider` per that binding plus the recipe's mode argv, stamps `harnessId` / `method` / `serviceId` / `servicePlan`, and — for a non-`api` method with a bootstrap app — the `credentialBootstrap` object `composeBootstrapSpawn` turns into `<bootstrap> run <harness> -- <args>`; throws typed errors (`HARNESS_SERVICE_INCOMPATIBLE`, `SERVICE_ENDPOINT_REQUIRED`, `SERVICE_CREDENTIAL_REQUIRED`, `SERVICE_CREDENTIAL_BOOTSTRAP_REQUIRED`) rather than minting a route that cannot run. `materializeRouteOutcome(input)` is the same body as a VERDICT (`{ record }` or `{ error }` — the typed 400 without the throw) for the composite resolver (#7564), which reads a refusal as the reason to show beside a saved selection. `routeDescribesService(profile, instance)` is the round-trip contract: fed back through `providerConnectionProfile`, a materialized record classifies onto the service it was built for. Pure: no I/O, no spawn, no provider call.
| `providerRouteSettings.js` | The MODE-OVERRIDE half of a provider route (#6369) — the route-owned fields a human tunes per execution mode (`args`, `timeout`, `effort` and the `defaultModel`/`lightModel`/`mediumModel`/`heavyModel`/`ultraModel` pins), mirroring what the toolkit's `sharedModeUpdates` already declares mode-specific. `ROUTE_SETTING_FIELDS` / `ROUTE_SETTING_KEYS` / `routeSettingKeys(mode)` name the set (`args` reaches cli/tui only); `routeSettingsFor(provider)` reads it off a record with absent and cleared collapsing to `null`; `routeSettingsSchema` is the wire shape `GET /api/providers/management` publishes; `unsupportedRouteSettings` rejects a key the route's mode does not accept; and `routeSettingsRevision` is the stale-edit fingerprint — a hash of the values as they are in providers.json rather than a graph row revision, so an edit made through `/ai/edit/:providerId` is caught too. Connection-owned values and execution consent (`enabled`) are deliberately absent. Pure: no I/O.
diff --git a/server/lib/aiToolkit/index.js b/server/lib/aiToolkit/index.js
index b846bb1797..be344cd3db 100644
--- a/server/lib/aiToolkit/index.js
+++ b/server/lib/aiToolkit/index.js
@@ -65,6 +65,8 @@ export function createAIToolkit(config = {}) {
// Host gate for "can this provider run at all?" — see providerStatus.js.
// Unset standalone, so the toolkit routes on enabled + availability alone.
prerequisitesMet = null,
+ // Host ordered fallback list — see providerStatus.js. Unset standalone.
+ getFallbackChain = null,
// Host persistence hook fired after every providers.json write — see
// createProviderService(). PortOS keeps its provider connection graph
// reconciled through it; unset standalone.
@@ -93,6 +95,7 @@ export function createAIToolkit(config = {}) {
statusFile,
defaultFallbackPriority,
prerequisitesMet,
+ getFallbackChain,
onStatusChange: (eventData) => {
io?.emit('provider:status:changed', eventData);
}
diff --git a/server/lib/aiToolkit/providerStatus.js b/server/lib/aiToolkit/providerStatus.js
index 9eb6817318..c5425d662a 100644
--- a/server/lib/aiToolkit/providerStatus.js
+++ b/server/lib/aiToolkit/providerStatus.js
@@ -198,9 +198,27 @@ export function createProviderStatusService(config = {}) {
// PATH probe) and this directory stays self-contained. It MUST be
// synchronous and MUST default to "yes": an unknown answer has to route
// exactly as it did before, never take a working provider out of the chain.
- prerequisitesMet = null
+ prerequisitesMet = null,
+ // Host-supplied `() => [{ providerId, model?, effort? }]` — the user's ONE
+ // ordered fallback list. When it is non-empty it replaces the per-provider
+ // `fallbackProvider` and the system priority list: whichever entry the run
+ // started on, the next entry (wrapping past the end to the first) is tried.
+ // Sync by contract (a cache read); a throwing or non-array answer reads as
+ // "no chain" so the legacy tiers still route.
+ getFallbackChain = null
} = config;
+ function readFallbackChain() {
+ if (typeof getFallbackChain !== 'function') return [];
+ try {
+ const chain = getFallbackChain();
+ return Array.isArray(chain) ? chain.filter(e => e && typeof e.providerId === 'string' && e.providerId) : [];
+ } catch (err) {
+ console.error(`❌ Fallback chain read failed: ${err.message}`);
+ return [];
+ }
+ }
+
const STATUS_PATH = join(dataDir, statusFile);
const events = new EventEmitter();
@@ -566,6 +584,26 @@ export function createProviderStatusService(config = {}) {
if (accepted) return accepted;
}
+ const chain = readFallbackChain();
+ if (chain.length > 0) {
+ // Rotate the list so it starts just after the entry the run began on.
+ // Match on provider AND model when the caller names the model, so two
+ // entries of one provider (different models, different limits) are
+ // distinct steps; an unlisted primary starts at the top.
+ const primaryModel = requestCapabilities?.primaryModel ?? null;
+ let at = primaryModel
+ ? chain.findIndex(e => e.providerId === primaryProviderId && e.model === primaryModel)
+ : -1;
+ if (at < 0) at = chain.findIndex(e => e.providerId === primaryProviderId);
+ const ordered = at < 0 ? chain : [...chain.slice(at + 1), ...chain.slice(0, at)];
+ for (const entry of ordered) {
+ const accepted = acceptCandidate(providers[entry.providerId], 'chain', entry.model || null);
+ if (accepted) return { ...accepted, effort: entry.effort || null };
+ }
+ if (capabilityRejections.length > 0) throw noEligibleFallbackError(capabilityRejections);
+ return null;
+ }
+
const primaryProvider = providers[primaryProviderId];
// Guard against `fallbackProvider === self` — a misconfigured provider
// would otherwise loop back to itself and silently retry the same
diff --git a/server/lib/aiToolkit/providerStatus.test.js b/server/lib/aiToolkit/providerStatus.test.js
index 4faee0a9f5..83533a4140 100644
--- a/server/lib/aiToolkit/providerStatus.test.js
+++ b/server/lib/aiToolkit/providerStatus.test.js
@@ -996,3 +996,49 @@ describe('Provider Status Service', () => {
});
});
});
+
+describe('getFallbackProvider — global ordered fallback list', () => {
+ const providers = Object.fromEntries(['a', 'b', 'c'].map((id) => [id, { id, enabled: true, models: ['m1', 'm2'], fallbackProvider: 'c' }]));
+ let dir;
+ let chain;
+ let svc;
+
+ beforeEach(async () => {
+ dir = await mkdtemp(join(tmpdir(), 'portos-fallback-chain-'));
+ chain = [{ providerId: 'a' }, { providerId: 'b', model: 'm1', effort: 'max' }, { providerId: 'c' }];
+ svc = createProviderStatusService({ dataDir: dir, statusFile: 's.json', getFallbackChain: () => chain });
+ await svc.init();
+ });
+ afterEach(() => rm(dir, { recursive: true, force: true }));
+
+ it('continues after the failed entry and carries the step model and effort', () => {
+ expect(svc.getFallbackProvider('a', providers)).toMatchObject({ source: 'chain', model: 'm1', effort: 'max', provider: { id: 'b' } });
+ });
+
+ it('wraps past the last entry to the first instead of using per-provider links', () => {
+ expect(svc.getFallbackProvider('c', providers).provider.id).toBe('a');
+ });
+
+ it('starts at the top for a provider that is not listed', () => {
+ expect(svc.getFallbackProvider('z', providers).provider.id).toBe('a');
+ });
+
+ it('skips unavailable entries and returns null when none can run', async () => {
+ await svc.markUnavailable('b', { reason: 'usage-limit', waitTimeMs: 60000 });
+ expect(svc.getFallbackProvider('a', providers).provider.id).toBe('c');
+ await svc.markUnavailable('c', { reason: 'usage-limit', waitTimeMs: 60000 });
+ await svc.markUnavailable('a', { reason: 'usage-limit', waitTimeMs: 60000 });
+ expect(svc.getFallbackProvider('a', providers)).toBeNull();
+ });
+
+ it('lets one preset repeat with different models, matched by the primary model', () => {
+ chain = [{ providerId: 'a', model: 'm1' }, { providerId: 'a', model: 'm2' }, { providerId: 'b' }];
+ expect(svc.getFallbackProvider('a', providers, null, null, { primaryModel: 'm1' })).toMatchObject({ provider: { id: 'a' }, model: 'm2' });
+ expect(svc.getFallbackProvider('a', providers, null, null, { primaryModel: 'm2' }).provider.id).toBe('b');
+ });
+
+ it('falls back to the legacy tiers when the list is empty', () => {
+ chain = [];
+ expect(svc.getFallbackProvider('a', providers)).toMatchObject({ source: 'provider', provider: { id: 'c' } });
+ });
+});
diff --git a/server/lib/index.js b/server/lib/index.js
index 65da717b9e..6ea152a6a7 100644
--- a/server/lib/index.js
+++ b/server/lib/index.js
@@ -576,6 +576,7 @@ export * from './providerServiceMerge.js';
export * from './providerModelAliases.js';
export * from './providerModePairing.js';
export * from './providerPresets.js';
+export * from './providerFallbackChainCache.js';
export * from './providerRef.js';
export * from './providerRouteRecipes.js';
export * from './serviceDefinitions.js';
diff --git a/server/lib/navManifest.js b/server/lib/navManifest.js
index 8dbf73fe4c..a67d4cad62 100644
--- a/server/lib/navManifest.js
+++ b/server/lib/navManifest.js
@@ -332,6 +332,7 @@ const RAW_NAV_COMMANDS = [
// prefix: they are opaque and stored in palette history, so renaming them
// would orphan those entries — only the path, label and section move.
{ id: 'nav.models.comparison', path: '/models/comparison', label: 'Comparison', section: 'Models', tabId: 'comparison', navGroup: 'Evaluate', aliases: ['model-comparison', 'model-cost', 'model-intelligence'], keywords: ['benchmark', 'effort', 'cost', 'quota', 'compare', 'intelligence'] },
+ { id: 'nav.models.fallbacks', path: '/models/fallbacks', label: 'Fallbacks', section: 'Models', tabId: 'fallbacks', navGroup: 'Connect', aliases: ['fallbacks', 'provider-fallbacks', 'fallback-list', 'fallback-order'], keywords: ['failover', 'preset order', 'retry', 'next provider', 'usage limit', 'ordered list', 'wrap'] },
{ id: 'nav.models.3d', path: '/models/3d', label: '3D', section: 'Models', tabId: '3d', navGroup: 'Library', aliases: ['3d-runtimes', 'image-to-3d-runtimes', 'trellis-install', 'pixal3d-install'], keywords: ['trellis', 'pixal3d', 'install', 'repair', 'runtime', 'mesh', 'image to 3d', 'on-device'] },
{ id: 'nav.models.decision-classifiers', path: '/models/decision-classifiers', label: 'Decision Classifiers', section: 'Models', tabId: 'decision-classifiers', navGroup: 'Policies', aliases: ['decision-classifiers', 'classifiers'], keywords: ['jev', 'laya', 'mlx', 'decision systems', 'experiments'] },
{ id: 'nav.models.laya-mlx', path: '/models/decision-classifiers/laya-mlx', label: 'Laya-MLX', section: 'Models', aliases: ['laya', 'laya-mlx', 'layla'], keywords: ['classifier', 'apple silicon', 'typed decisions'] },
diff --git a/server/lib/navManifest.test.js b/server/lib/navManifest.test.js
index d9e8af743d..b445831487 100644
--- a/server/lib/navManifest.test.js
+++ b/server/lib/navManifest.test.js
@@ -165,7 +165,7 @@ describe('nav contract — generated section child navigation', () => {
it('keeps Models groups and destinations in the design order', () => {
expect(getSectionNavGroups('Models').map(({ label, tabs }) => [label, tabs.map((tab) => tab.label)])).toEqual([
- ['Connect', ['AI Services', 'Harnesses', 'Providers', 'Runtimes']],
+ ['Connect', ['AI Services', 'Fallbacks', 'Harnesses', 'Providers', 'Runtimes']],
['Evaluate', ['Comparison', 'Performance', 'Playground']],
['Library', ['3D', 'Embeddings', 'LoRAs', 'Media', 'Model Library', 'Training']],
['Operate', ['Quota Burn', 'Status', 'Subscriptions', 'Usage']],
diff --git a/server/lib/providerFallbackChainCache.js b/server/lib/providerFallbackChainCache.js
new file mode 100644
index 0000000000..7169213100
--- /dev/null
+++ b/server/lib/providerFallbackChainCache.js
@@ -0,0 +1,24 @@
+/**
+ * Process cache of the user's ordered provider fallback list. Kept free of
+ * imports so routing code (promptRunner, agent resolution, bootstrap) can read
+ * it synchronously without pulling in settings or validation. The settings
+ * binding that fills it lives in `services/providerFallbackChain.js`.
+ */
+
+let chain = [];
+
+/** Normalize to `[{ providerId, model?, effort? }]`, dropping malformed rows. */
+export const normalizeFallbackChain = (entries) => (Array.isArray(entries) ? entries : [])
+ .filter((e) => e && typeof e.providerId === 'string' && e.providerId)
+ .map(({ providerId, model, effort }) => ({
+ providerId,
+ ...(model ? { model } : {}),
+ ...(effort ? { effort } : {}),
+ }));
+
+export const peekFallbackChain = () => chain;
+
+export const setFallbackChainCache = (entries) => {
+ chain = normalizeFallbackChain(entries);
+ return chain;
+};
diff --git a/server/lib/validation.js b/server/lib/validation.js
index 4be72ba8ec..4e53b17bc1 100644
--- a/server/lib/validation.js
+++ b/server/lib/validation.js
@@ -2847,3 +2847,15 @@ export const voicePiperFetchBodySchema = z.object({ voice: nonEmptyString });
export const writersRoomSceneImageBodySchema = z.object({
sceneId: nullishString, filename: nullishString, jobId: nullishString, prompt: nullishString,
});
+
+// One step of the global provider fallback list. `providerId` is a preset or
+// composite reference; `model` / `effort` are optional pins for that step, so
+// one preset may appear several times with different models or effort.
+export const providerFallbackEntrySchema = z.object({
+ providerId: providerRefSchema,
+ model: z.preprocess((v) => (v === '' ? undefined : v), z.string().trim().min(1).max(200).optional()),
+ effort: z.preprocess((v) => (v === '' ? undefined : v), z.string().trim().min(1).max(40).optional()),
+});
+export const providerFallbackChainBodySchema = z.object({
+ entries: z.array(providerFallbackEntrySchema).max(50),
+});
diff --git a/server/routes/providerFallbacks.js b/server/routes/providerFallbacks.js
new file mode 100644
index 0000000000..069d5aaf1b
--- /dev/null
+++ b/server/routes/providerFallbacks.js
@@ -0,0 +1,17 @@
+import { Router } from 'express';
+import { asyncHandler } from '../lib/errorHandler.js';
+import { validateRequest, providerFallbackChainBodySchema } from '../lib/validation.js';
+import { getFallbackChain, saveFallbackChain } from '../services/providerFallbackChain.js';
+
+const router = Router();
+
+router.get('/', asyncHandler(async (req, res) => {
+ res.json({ entries: await getFallbackChain() });
+}));
+
+router.put('/', asyncHandler(async (req, res) => {
+ const body = validateRequest(providerFallbackChainBodySchema, req.body ?? {});
+ res.json({ entries: await saveFallbackChain(body) });
+}));
+
+export default router;
diff --git a/server/routes/providerFallbacks.test.js b/server/routes/providerFallbacks.test.js
new file mode 100644
index 0000000000..0c5364a4e1
--- /dev/null
+++ b/server/routes/providerFallbacks.test.js
@@ -0,0 +1,32 @@
+import { describe, it, expect, vi, beforeEach } from 'vitest';
+import express from 'express';
+import { request } from '../lib/testHelper.js';
+import { errorMiddleware } from '../lib/errorHandler.js';
+
+const mocks = vi.hoisted(() => ({ getFallbackChain: vi.fn(), saveFallbackChain: vi.fn() }));
+vi.mock('../services/providerFallbackChain.js', () => mocks);
+
+const routes = (await import('./providerFallbacks.js')).default;
+const app = express();
+app.use(express.json());
+app.use('/api/provider-fallbacks', routes);
+app.use(errorMiddleware);
+
+beforeEach(() => vi.clearAllMocks());
+
+describe('provider fallback list routes', () => {
+ it('saves an ordered list that repeats a preset with different models', async () => {
+ const entries = [{ providerId: 'codex', model: 'm1', effort: 'max' }, { providerId: 'codex', model: 'm2' }, { providerId: 'agy' }];
+ mocks.saveFallbackChain.mockResolvedValue(entries);
+ const res = await request(app).put('/api/provider-fallbacks').send({ entries });
+ expect(res.status).toBe(200);
+ expect(mocks.saveFallbackChain).toHaveBeenCalledWith({ entries });
+ expect(res.body.entries).toEqual(entries);
+ });
+
+ it('rejects a malformed provider reference before it reaches the store', async () => {
+ const res = await request(app).put('/api/provider-fallbacks').send({ entries: [{ providerId: 'Not A Preset!' }] });
+ expect(res.status).toBe(400);
+ expect(mocks.saveFallbackChain).not.toHaveBeenCalled();
+ });
+});
diff --git a/server/services/agentProviderResolution.js b/server/services/agentProviderResolution.js
index 22bc9eb027..2505587b28 100644
--- a/server/services/agentProviderResolution.js
+++ b/server/services/agentProviderResolution.js
@@ -1,6 +1,7 @@
import { isForgeMaintenanceTask, hasCurrentForgeMaintenanceEvidence, LEGACY_FORGE_MAINTENANCE_REASON } from '../lib/forgeMaintenanceTasks.js';
import { isProcessProvider } from '../lib/providerTypes.js';
import { isCompositeProviderId } from '../lib/providerRef.js';
+import { peekFallbackChain } from '../lib/providerFallbackChainCache.js';
import { isPrivateSecurityTask, PRIVATE_SECURITY_DELIVERY } from '../lib/privateSecurityPolicy.js';
import { supportsPublicReviewProvider } from '../lib/providerVendors.js';
/**
@@ -235,6 +236,10 @@ async function resolveOrdinaryProviderAndModel(task) {
// The caller's mode policy travels WITH the fallback request, so an
// ineligible candidate is skipped during selection rather than picked and
// then rejected below — which used to burn the one retry the cascade had.
+ const chainComposites = peekFallbackChain().map((e) => e.providerId).filter(isCompositeProviderId);
+ if (chainComposites.length) {
+ await (await import('./compositeProviders.js')).withCompositeCandidates(providersMap, chainComposites);
+ }
const fallbackResult = await getFallbackProvider(provider.id, providersMap, taskFallbackId, taskFallbackModel, {
allowedModes: allowedModesFor(AGENT_CALLER_POLICY),
});
diff --git a/server/services/bootstrap.js b/server/services/bootstrap.js
index bed6a0a3f4..e6aa7b337f 100644
--- a/server/services/bootstrap.js
+++ b/server/services/bootstrap.js
@@ -180,6 +180,8 @@ import { outcomesStore as liOutcomesStore } from './layeredIntelligenceOutcomes.
import * as gameStore from './games/store.js';
import * as fableLoomStore from './fableLoom/store.js';
import { prerequisitesMetForRouting } from './providerPrerequisites.js';
+import { peekFallbackChain } from '../lib/providerFallbackChainCache.js';
+import { initFallbackChain } from './providerFallbackChain.js';
import { localCachedModelIds } from './localCachedModels.js';
import { stopCodexAppServer } from './codexAppServer.js';
import { logFailureWithStack as logBootstrapFailure } from '../lib/failureLogging.js';
@@ -323,6 +325,9 @@ export const bootstrapServices = async ({ io, dataDir, dataReferenceDir, serverD
// of dying at spawn time. Sync by contract: it reads the runtime probe's
// cache and never blocks.
prerequisitesMet: prerequisitesMetForRouting,
+ // The user's single ordered fallback list (Models > Fallbacks); a cache
+ // read, so it is sync as the toolkit requires.
+ getFallbackChain: peekFallbackChain,
// MTPLX and Slotstream each serve one checkpoint per process and report
// only that one, so a refresh could never surface a newly downloaded
// checkpoint — and for Slotstream it pruned the record's other shipped ids.
@@ -351,7 +356,11 @@ export const bootstrapServices = async ({ io, dataDir, dataReferenceDir, serverD
// Warm the providers file at startup so the codex-sentinel migration runs
// before any inbound request can hit the providers cache. A pure READ of
// on-disk provider config — it dispatches nothing to a provider.
- warmProviders: (aiToolkit) => aiToolkit.services.providers.getAllProviders(),
+ warmProviders: async (aiToolkit) => {
+ // Prime the sync fallback-list cache from settings.json (disk only).
+ await initFallbackChain().catch((err) => logBootstrapFailure('⚠️ Failed to load provider fallback list', err));
+ return aiToolkit.services.providers.getAllProviders();
+ },
// Ensure the provider paired with the active local-LLM backend (LLM_BACKEND in
// .env, chosen at setup time) is enabled, so a fresh install can use Ollama /
diff --git a/server/services/promptRunner.js b/server/services/promptRunner.js
index c4de382dd1..eff890daff 100644
--- a/server/services/promptRunner.js
+++ b/server/services/promptRunner.js
@@ -31,6 +31,7 @@ import { isVisionCapableCodexTuiProvider } from '../lib/codex.js';
import { createRun, executeApiRun, executeCliRun, extractBakedModel, hasModelFlag, stopRun, patchRunMetadata, finalizeRunRecord } from './runner.js';
import { getActiveProvider, getProviderById, getAllProviders } from './providers.js';
import { isCompositeProviderId } from '../lib/providerRef.js';
+import { peekFallbackChain } from '../lib/providerFallbackChainCache.js';
// `./tuiPromptRunner.js` (which drags node-pty in through `./shell.js`) and
// `./providerExecutionReadiness.js` are imported lazily on the branches that
// actually execute a run — see their call sites below. promptRunner.js is reached
@@ -1099,6 +1100,9 @@ async function runPromptThroughProviderAdmitted(rawArgs) {
...args,
provider: fallback,
model: picked.model ?? undefined,
+ // A step in the global fallback list may pin its own effort; the
+ // primary's effort belongs to the primary's model, so it is not inherited.
+ ...(picked.effort ? { effort: picked.effort } : {}),
runId: undefined, // fresh runId so the failed primary's record stays intact
});
} catch (fallbackError) {
@@ -1212,6 +1216,10 @@ async function pickFallbackProvider(failed, requestCapabilities) {
if (isCompositeProviderId(failed.id)) {
await (await import('./compositeProviders.js')).withCompositeCandidates(providersMap, [failed.id]);
}
+ const chainComposites = peekFallbackChain().map((e) => e.providerId).filter(isCompositeProviderId);
+ if (chainComposites.length) {
+ await (await import('./compositeProviders.js')).withCompositeCandidates(providersMap, chainComposites);
+ }
const picked = providerStatus.getFallbackProvider(failed.id, providersMap, null, null, requestCapabilities);
if (!picked?.provider) return null;
@@ -1220,7 +1228,7 @@ async function pickFallbackProvider(failed, requestCapabilities) {
// without its sibling's apiKey and fail auth (NVIDIA NIM 401s "Header of type
// `authorization` was missing") despite a stored key. Attach it from the same
// map (synchronous — no re-read to go stale).
- return { provider: attachGatewaySiblingKey(picked.provider, providersMap), model: picked.model ?? null };
+ return { provider: attachGatewaySiblingKey(picked.provider, providersMap), model: picked.model ?? null, effort: picked.effort ?? null };
}
/**
diff --git a/server/services/providerFallbackChain.js b/server/services/providerFallbackChain.js
new file mode 100644
index 0000000000..6c0e757b7c
--- /dev/null
+++ b/server/services/providerFallbackChain.js
@@ -0,0 +1,35 @@
+import { getSettings, settingsEvents, updateSettings } from './settings.js';
+import { normalizeFallbackChain, setFallbackChainCache } from '../lib/providerFallbackChainCache.js';
+
+/**
+ * The ONE ordered provider fallback list (settings key `providerFallbackChain`).
+ *
+ * Whichever entry a run started on, the toolkit's `getFallbackProvider` tries
+ * the entries after it in order and wraps past the end to the first, so the
+ * list replaces the old per-provider `fallbackProvider` links (which could form
+ * cycles). An entry names a preset/composite id plus an optional model and
+ * effort; the same preset may repeat with different models.
+ *
+ * The toolkit reads it synchronously, so a module cache is kept coherent from
+ * the `settings:updated` event and primed once at boot (disk only, no LLM call).
+ */
+
+settingsEvents.on('settings:updated', (settings) => {
+ setFallbackChainCache(settings?.providerFallbackChain);
+});
+
+export async function initFallbackChain() {
+ const cache = setFallbackChainCache((await getSettings()).providerFallbackChain);
+ if (cache.length) console.log(`🔗 Provider fallback list loaded (${cache.length} entries)`);
+ return cache;
+}
+
+export const getFallbackChain = async () => normalizeFallbackChain((await getSettings()).providerFallbackChain);
+
+// `entries` arrive validated by the route (providerFallbackChainBodySchema).
+export async function saveFallbackChain({ entries }) {
+ const next = normalizeFallbackChain(entries);
+ await updateSettings({ providerFallbackChain: next });
+ setFallbackChainCache(next);
+ return next;
+}