Skip to content

armgit / avibe

Public Imported from GitHub
0

Make your computer a home for your AI partner. Avibe is the local-first Agent OS: Vibey, its built-in AI partner, works out of the box, and Claude Code, Codex & OpenCode run unchanged beside it — reach them from your browser, phone, or any chat app.

fix(model-hub): let a successful source test clear its cooldown (#2210)

Call White45698f56fe81
diff --git a/core/handlers/model_hub/service.py b/core/handlers/model_hub/service.py
index ddab41e11..a357caabc 100644
--- a/core/handlers/model_hub/service.py
+++ b/core/handlers/model_hub/service.py
@@ -1216,15 +1216,17 @@ class ModelHubService:
return False
return True
- def _reserve_settlement_generation(self, source_id: str) -> int:
+ def _mint_settlement_generation(self) -> int:
self._next_settlement_generation += 1
- self._latest_source_attempt_generation[source_id] = (
- self._next_settlement_generation
- )
+ return self._next_settlement_generation
+
+ def _reserve_settlement_generation(self, source_id: str) -> int:
+ generation = self._mint_settlement_generation()
+ self._latest_source_attempt_generation[source_id] = generation
source = next((item for item in self.store.load().sources if item.id == source_id), None)
if source is not None:
self._source_attempt_identities[source_id] = source_identity(source)
- return self._next_settlement_generation
+ return generation
def _settlement_current(self, source: ModelHubSourceConfig, generation: int | None) -> bool:
return (
@@ -5873,9 +5875,10 @@ class ModelHubService:
outcome = None
source = None
admitted_at = None
+ attempt_generation = PRE_ATTEMPT_SETTLEMENT_GENERATION
async def invoke_selected() -> None:
- nonlocal source, handle, outcome, admitted_at
+ nonlocal source, handle, outcome, admitted_at, attempt_generation
# Bound local waiting too, but never call it a model failure before
# the adapter has actually admitted this Source/model invocation.
async with self._mutation_lock:
@@ -5892,8 +5895,12 @@ class ModelHubService:
self._mutation_lock.release()
def admitted() -> None:
- nonlocal admitted_at
+ nonlocal admitted_at, attempt_generation
admitted_at = time.monotonic()
+ # Ordered at admission but not reserved: a failed test must
+ # not displace another attempt's verdict, while a successful
+ # one supersedes attempts admitted before it.
+ attempt_generation = self._mint_settlement_generation()
release_owner()
try:
@@ -5926,6 +5933,11 @@ class ModelHubService:
async def settle_attempt() -> None:
try:
if source is not None and outcome is not None:
+ # The explicit test is the retry a cooldown waits for.
+ if self._verified_recovery_outcome(outcome):
+ await self._record_recovery_success(
+ source.id, attempt_generation, backend="system", model_id=model_id,
+ )
await self._verify_successful_source(
source.id, source.credential_ref, source.verification_pending, outcome,
)
@@ -5969,8 +5981,9 @@ class ModelHubService:
raise cancelled
assert source is not None and outcome is not None
succeeded = outcome.kind is RawOutcomeKind.SUCCESS
- # Classify for display only. A selected model failure cannot block other
- # models on this Source, refresh credentials, or modify route state.
+ # Classify failures for display only. A selected model failure cannot
+ # block other models on this Source, refresh credentials, or modify route
+ # state.
decision = classify_outcome(outcome)
if decision.action == "refresh":
decision = ResolutionDecision("fallback", reason="credential_revoked")
@@ -6948,7 +6961,7 @@ class ModelHubService:
self.recovery.annotations(self.store.load())
async def _record_recovery_success(
- self, source_id: str, generation: int | None, *, backend: BackendName, model_id: str,
+ self, source_id: str, generation: int | None, *, backend: EventAgent, model_id: str,
) -> None:
async with self._mutation_lock:
config = self.store.load()
@@ -6959,6 +6972,8 @@ class ModelHubService:
or source.state.status in {"needs_action", "error"}
):
return
+ # A current success outranks every attempt admitted before it.
+ self._latest_source_attempt_generation[source_id] = generation
recovered = self.recovery.succeeded(source)
if recovered and source.state.status == "cooldown":
previous = self._clone_config(config)
@@ -6971,7 +6986,7 @@ class ModelHubService:
logger.warning("Could not persist Model Hub recovered state")
if recovered:
self._record_event(
- agent=cast(EventAgent, backend), kind="recover", model_id=model_id,
+ agent=backend, kind="recover", model_id=model_id,
reason="recovery", to_source=source.id, to_label=source.display_name, now=self.now(),
)
diff --git a/tests/test_model_hub_source_probe.py b/tests/test_model_hub_source_probe.py
index 49e4c4b2f..5dd72e830 100644
--- a/tests/test_model_hub_source_probe.py
+++ b/tests/test_model_hub_source_probe.py
@@ -8,7 +8,9 @@ from unittest.mock import AsyncMock
import pytest
+from config.v2_config import ModelHubSourceStateConfig
from core.handlers.model_hub.adapter import DiscoveredModel, RawOutcomeKind, SOURCE_PROTOCOLS, SourceBinding
+from core.handlers.model_hub.classification import ResolutionDecision
from core.handlers.model_hub.service import ModelHubError
from core.handlers.model_hub import service as service_module
from tests.test_model_hub_api import FakeAdapter, FakeInvokeHandle, _service
@@ -91,6 +93,51 @@ def test_probe_invokes_selected_source_even_without_agent_routes_and_isolates_fa
asyncio.run(scenario())
[email protected]("case", ["success", "unverified_success", "failure", "newer_attempt", "older_attempt"])
+def test_successful_test_lifts_cooldown_only_for_its_own_current_attempt(tmp_path, case):
+ service, store, adapter = _service(tmp_path)
+
+ async def scenario():
+ source = (await service.create_source(_draft()))["source"]
+ store.config.sources[0].state = ModelHubSourceStateConfig(
+ status="cooldown", retry_at="2026-07-23T02:59:00+00:00",
+ detail_key="models.source.cooldown.server_error",
+ )
+ # Inference admitted before the test and still in flight.
+ older = service._reserve_settlement_generation(source["id"])
+ original = adapter.invoke
+
+ async def invoke(*args, **kwargs):
+ handle = await original(*args, **kwargs)
+ if case == "newer_attempt":
+ # Inference admitted while the test was in flight owns the verdict.
+ service._reserve_settlement_generation(source["id"])
+ outcome = await handle.outcome()
+ if case == "failure":
+ outcome = replace(outcome, kind=RawOutcomeKind.HTTP_ERROR, http_status=500)
+ return FakeInvokeHandle(replace(outcome, recovery_verified=case != "unverified_success"))
+
+ adapter.invoke = AsyncMock(side_effect=invoke)
+ await service.probe_source(source["id"], {"model": source["models"][0]["id"]})
+ if case == "older_attempt":
+ # The earlier attempt fails after the later test already succeeded.
+ await service._settle_fallback_source(
+ store.config.sources[0], ResolutionDecision("fallback", reason="server_error"),
+ backend="claude", model_id=source["models"][0]["id"], settlement_generation=older,
+ )
+ state = store.config.sources[0].state
+ recovered = [event for event in service.events.list() if event["kind"] == "recover"]
+ if case in {"success", "older_attempt"}:
+ assert state == ModelHubSourceStateConfig(status="standby")
+ assert [(event["agent"], event["to_source"]) for event in recovered] == [("system", source["id"])]
+ else:
+ assert state.status == "cooldown"
+ assert state.detail_key == "models.source.cooldown.server_error"
+ assert recovered == []
+
+ asyncio.run(scenario())
+
+
@pytest.mark.parametrize("change", ["delete", "replace", "same_handle"])
def test_test_settlement_cannot_verify_a_different_credential_identity(tmp_path, change):
class ConcurrentAdapter(FakeAdapter):
diff --git a/ui/src/components/settings/models/SourceDetailPanel.test.tsx b/ui/src/components/settings/models/SourceDetailPanel.test.tsx
index f0593edf7..56f57dcb3 100644
--- a/ui/src/components/settings/models/SourceDetailPanel.test.tsx
+++ b/ui/src/components/settings/models/SourceDetailPanel.test.tsx
@@ -1045,6 +1045,26 @@ describe('SourceDetailPanel', () => {
expect(explanation.textContent).not.toMatch(/test|测试/i);
});
+ it.each([
+ { kind: 'api_key', status: 'cooldown', offered: true },
+ { kind: 'api_key', status: 'standby', offered: false },
+ { kind: 'subscription', status: 'cooldown', offered: false },
+ ] as const)('offers an immediate test beside a $kind $status state: $offered', async ({ kind, status, offered }) => {
+ const locale = i18n.cloneInstance({ lng: 'zh' });
+ render(<ToastProvider><I18nextProvider i18n={locale}>
+ <CommittedPanel source={{ ...source, kind, state: status === 'cooldown'
+ ? { status, retry_at: null, detail_key: 'models.source.cooldown.server_error' }
+ : { status, retry_at: null, detail_key: null } }} trackMutation={immediateTrack} onReauth={noReauth} />
+ </I18nextProvider></ToastProvider>);
+ const testNow = screen.queryByRole('button', { name: locale.t('settings.models.sourceTest.testNow') });
+ expect(Boolean(testNow)).toBe(offered);
+ if (!testNow) return;
+ expect(testNow.closest('.model-hub-source-line')?.textContent).toContain(locale.t('settings.models.upstream.state.unavailableDue'));
+ await userEvent.click(testNow);
+ const dialog = await screen.findByRole('dialog');
+ expect(dialog.textContent).toContain(locale.t('settings.models.sourceTest.cooldownHint'));
+ });
+
it('sends a manual-model removal before showing any guarded-change confirm', async () => {
const remove = vi.spyOn(modelsApi, 'deleteCustomModel').mockResolvedValueOnce(source);
renderPanel();
diff --git a/ui/src/components/settings/models/SourceDetailPanel.tsx b/ui/src/components/settings/models/SourceDetailPanel.tsx
index 25896ca0a..203d287ef 100644
--- a/ui/src/components/settings/models/SourceDetailPanel.tsx
+++ b/ui/src/components/settings/models/SourceDetailPanel.tsx
@@ -667,6 +667,7 @@ export const SourceDetailPanel: React.FC<{
const lastFetched = source.last_discovered_at
? formatRelativeTime(source.last_discovered_at, t)
: t('settings.models.sourceDetail.metadata.neverFetched');
+ const testable = source.kind === 'api_key' && source.supply_channel === 'hub';
const usageSummary = adoptedBackends.length > 0
? t('settings.models.sourceDetail.usageSummary', { count: source.models.length, backends: adoptedBackends.join(i18n.language.startsWith('zh') ? '、' : ', ') })
: t('settings.models.gateway.modelCount', { count: source.models.length });
@@ -680,6 +681,8 @@ export const SourceDetailPanel: React.FC<{
<div className="model-hub-source-line flex min-w-0 flex-wrap items-center gap-x-[7px]">
<h2 ref={headingRef} tabIndex={-1} className="model-hub-source-title truncate font-bold text-foreground">{source.display_name}</h2>
{state.key && <span className={cn('model-hub-source-state model-hub-pill flex items-center gap-1.5 border', state.textClass)}><span className={cn('size-[5px] shrink-0 rounded-full', state.dotClass)} />{t(state.key, state.values)}{state.hint && <ModelHubInfoHint label={t(state.hint.labelKey) as string} content={t(state.hint.bodyKey)} className="size-[13px]" />}</span>}
+ {/* A cooldown waits for the next request; a passing test is that request. */}
+ {testable && source.state.status === 'cooldown' && <button type="button" disabled={busy} onClick={() => setTesting(true)} className="model-hub-source-state font-semibold text-foreground underline underline-offset-2 disabled:opacity-50">{t('settings.models.sourceTest.testNow')}</button>}
</div>
<p className="model-hub-source-summary truncate">{usageSummary}</p>
</div>
@@ -711,7 +714,7 @@ export const SourceDetailPanel: React.FC<{
{repairDestination === 'reauth_dialog' && <Button size="xs" className="model-hub-source-action" data-repair-kind={repair} data-repair-destination={repairDestination} disabled={busy} onClick={() => setConfirmingReauth(true)}><LogIn />{t(REPAIR_LABEL_KEY.reauth)}</Button>}
{repairDestination === 'replace_key_dialog' && <Button size="xs" className="model-hub-source-action" data-repair-kind={repair} data-repair-destination={repairDestination} disabled={busy} onClick={() => setReplacingKey(true)}>{t(REPAIR_LABEL_KEY.replace_key)}</Button>}
{source.supply_channel === 'hub' && <Button variant="outline" size="xs" className="model-hub-source-action" data-repair-kind={repairDestination === 'refetch_button' ? repair : undefined} data-repair-destination={repairDestination === 'refetch_button' ? repairDestination : undefined} disabled={busy} onClick={() => void refetch()} aria-busy={refetching}>{refetching ? <Loader2 className="animate-spin" /> : <RefreshCw />}{t('settings.models.sourceDetail.action.refetch')}</Button>}
- {source.kind === 'api_key' && source.supply_channel === 'hub' && <Button variant="outline" size="xs" className="model-hub-source-action" disabled={busy} onClick={() => setTesting(true)}><FlaskConical />{t('settings.models.sourceTest.open')}</Button>}
+ {testable && <Button variant="outline" size="xs" className="model-hub-source-action" disabled={busy} onClick={() => setTesting(true)}><FlaskConical />{t('settings.models.sourceTest.open')}</Button>}
{source.kind === 'api_key' && <Button size="xs" className="model-hub-source-action" disabled={busy || manualDraft !== null} onClick={() => setManualDraft({ modelId: '', failed: false, retryRead: false })}><Plus />{t('settings.models.sourceDetail.action.addModel')}</Button>}
<SourceManageMenu source={source} busy={busy || manageStage.kind !== 'idle'} onEdit={beginEdit} onDelete={beginDelete} />
</div>
diff --git a/ui/src/components/settings/models/SourceTestDialog.test.tsx b/ui/src/components/settings/models/SourceTestDialog.test.tsx
index 3b6221dc7..c061f6ace 100644
--- a/ui/src/components/settings/models/SourceTestDialog.test.tsx
+++ b/ui/src/components/settings/models/SourceTestDialog.test.tsx
@@ -69,6 +69,19 @@ describe('SourceTestDialog', () => {
expect(refresh).not.toHaveBeenCalled();
});
+ it.each([
+ ['lifted', 'standby', false],
+ ['kept', 'cooldown', true],
+ ] as const)('reports a passing test that %s the cooldown', async (_case, landed, warned) => {
+ const cooling: Source = { ...source, state: { status: 'cooldown', retry_at: null, detail_key: 'models.source.cooldown.server_error' } };
+ vi.mocked(settlement.unread).mockResolvedValue(landing([{ ...cooling, state: { ...cooling.state, status: landed } }]));
+ vi.spyOn(modelsApi, 'probeSource').mockResolvedValue(answer());
+ render(view(cooling));
+ await userEvent.click(screen.getByRole('button', { name: 'Run test' }));
+ const reported = await screen.findByText(/responded successfully/);
+ expect(reported.textContent?.includes('still marked unavailable')).toBe(warned);
+ });
+
it('keeps the result when success clears pending verification', async () => {
vi.spyOn(modelsApi, 'probeSource').mockResolvedValue(answer());
const rendered = render(view());
diff --git a/ui/src/components/settings/models/SourceTestDialog.tsx b/ui/src/components/settings/models/SourceTestDialog.tsx
index 1140c2482..0fdd05034 100644
--- a/ui/src/components/settings/models/SourceTestDialog.tsx
+++ b/ui/src/components/settings/models/SourceTestDialog.tsx
@@ -13,7 +13,7 @@ import { serverText } from './serverCopy';
import { selectTestModel } from './testModelPreference';
import type { Source, SourceProbeResult } from './types';
-type TestStatus = { kind: 'result'; result: SourceProbeResult } | { kind: 'unconfirmed' } | null;
+type TestStatus = { kind: 'result'; result: SourceProbeResult; stillCooling: boolean } | { kind: 'unconfirmed' } | null;
export const SourceTestDialog: React.FC<{
source: Source;
@@ -73,8 +73,11 @@ export const SourceTestDialog: React.FC<{
&& current.base_url === latest.base_url && current.protocol === latest.protocol
&& (!current.verification_pending || current.verification_pending === latest.verification_pending)
&& current.models.some((model) => model.id === selected && !model.retired);
+ // Only a verified reply lifts a cooldown, so the reconciled state, not
+ // `reachable`, says whether this test restored the provider.
continuation.settle(ticket, () => setStatus(matches
- ? { kind: 'result', result: answer } : { kind: 'unconfirmed' }));
+ ? { kind: 'result', result: answer, stillCooling: current.state.status === 'cooldown' }
+ : { kind: 'unconfirmed' }));
});
} catch {
continuation.settle(ticket, () => setStatus({ kind: 'unconfirmed' }));
@@ -84,11 +87,15 @@ export const SourceTestDialog: React.FC<{
}
};
const result = status?.kind === 'result' ? status.result : null;
+ const stillCooling = status?.kind === 'result' && status.stillCooling;
return (
<Dialog open onOpenChange={(open) => { if (!open) onClose(); }}>
<DialogContent closeLabel={t('common.close')}>
<DialogTitle className="min-w-0 break-words pr-6">{t('settings.models.sourceTest.title', { source: source.display_name })}</DialogTitle>
- <DialogDescription>{t('settings.models.sourceTest.hint')}</DialogDescription>
+ <DialogDescription>
+ {t('settings.models.sourceTest.hint')}
+ {source.state.status === 'cooldown' && ` ${t('settings.models.sourceTest.cooldownHint')}`}
+ </DialogDescription>
<Field label={t('settings.models.sourceTest.model')}>
{(id) => <Combobox id={id} ariaLabel={t('settings.models.sourceTest.model')} options={options}
value={selected} disabled={busy || options.length === 0} allowCustomValue={false}
@@ -102,6 +109,7 @@ export const SourceTestDialog: React.FC<{
model: result.model_id,
reason: serverText(t, result.error, 'settings.models.sourceTest.unknown'),
})}
+ {result.reachable && stillCooling && ` ${t('settings.models.sourceTest.stillUnavailable')}`}
</p>}
{status?.kind === 'unconfirmed' && <p role="alert" className="text-sm text-destructive-ink">{t('settings.models.sourceTest.requestFailed')}</p>}
<DialogFooter>
diff --git a/ui/src/i18n/en.json b/ui/src/i18n/en.json
index e3bd3b145..e0604bc52 100644
--- a/ui/src/i18n/en.json
+++ b/ui/src/i18n/en.json
@@ -2045,8 +2045,11 @@
},
"sourceTest": {
"open": "Test",
+ "testNow": "Test now",
"title": "Test {{source}}",
"hint": "Send one short request using this provider and the selected model. This can incur a small charge. The result applies only to this model.",
+ "cooldownHint": "A successful test makes this provider available again right away.",
+ "stillUnavailable": "The provider is still marked unavailable.",
"model": "Test model",
"empty": "No models available. Close this dialog to fetch the list or add a model ID manually.",
"run": "Run test",
diff --git a/ui/src/i18n/zh.json b/ui/src/i18n/zh.json
index a107a8238..4bae19617 100644
--- a/ui/src/i18n/zh.json
+++ b/ui/src/i18n/zh.json
@@ -2045,8 +2045,11 @@
},
"sourceTest": {
"open": "测试",
+ "testNow": "立即测试",
"title": "测试 {{source}}",
"hint": "使用此供应商和选中的模型发送一次简短请求,可能产生少量费用。结果仅代表这个模型。",
+ "cooldownHint": "测试成功会立即恢复这个供应商。",
+ "stillUnavailable": "供应商仍标记为暂不可用。",
"model": "测试模型",
"empty": "暂无模型。关闭此窗口后,可拉取列表或手动添加模型 ID。",
"run": "开始测试",