Skip to content

7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.

TypeScript

fix(router): retire models the provider killed, and stop looping on 400 "not allowed" (#647)

Tashfeencf102e2f5f6c
diff --git a/client/src/components/model-table.tsx b/client/src/components/model-table.tsx
index 00cb5649..a42a4d21 100644
--- a/client/src/components/model-table.tsx
+++ b/client/src/components/model-table.tsx
@@ -160,6 +160,14 @@ export function RowContent({
{t('models.tools')}
</span>
)}
+ {row.retiredUpstream && (
+ <span
+ title={row.retiredReason ?? undefined}
+ className="text-[10px] rounded-full px-1.5 py-0.5 bg-rose-600/15 text-rose-700 dark:bg-rose-400/15 dark:text-rose-400"
+ >
+ {t('models.retired')}
+ </span>
+ )}
{(row.penalty ?? 0) > 0 && (
<span className="text-[10px] text-amber-600 dark:text-amber-400">{t('models.penalty', { value: row.penalty })}</span>
)}
diff --git a/client/src/i18n/locales/am.json b/client/src/i18n/locales/am.json
index cde2ac39..ac7b2922 100644
--- a/client/src/i18n/locales/am.json
+++ b/client/src/i18n/locales/am.json
@@ -122,6 +122,7 @@
"visionTitle": "የምስል ግቤትን ይቀበላል",
"tools": "መሣሪያዎች",
"toolsTitle": "የተዋቀሩ የመሳሪያ ጥሪዎችን ያወጣል፣ ስለዚህ ለመሳሪያ ተሸካሚ ጥያቄዎች ብቁ ነው",
+ "retired": "ተቋርጧል",
"penalty": "- {value} ቅጣት",
"obs": "{count} obs",
"tokPerMonth": "{count} ቶክ / ሞ",
diff --git a/client/src/i18n/locales/ar.json b/client/src/i18n/locales/ar.json
index dc52c4c7..bd3b690f 100644
--- a/client/src/i18n/locales/ar.json
+++ b/client/src/i18n/locales/ar.json
@@ -122,6 +122,7 @@
"visionTitle": "يقبل إدخال الصورة",
"tools": "أدوات",
"toolsTitle": "يصدر استدعاءات أداة منظمة، لذا فهو مؤهل لطلبات حاملة الأداة",
+ "retired": "متوقف",
"penalty": "−{value} عقوبة",
"obs": "{count} ملاحظة",
"tokPerMonth": "{count} توك/مو",
diff --git a/client/src/i18n/locales/az.json b/client/src/i18n/locales/az.json
index ab85d7dc..5c4eb844 100644
--- a/client/src/i18n/locales/az.json
+++ b/client/src/i18n/locales/az.json
@@ -122,6 +122,7 @@
"visionTitle": "Şəkil girişini qəbul edir",
"tools": "Alətlər",
"toolsTitle": "Strukturlaşdırılmış alət çağırışları yayır, buna görə də alət daşıma sorğuları üçün uyğundur",
+ "retired": "Dayandırılıb",
"penalty": "−{value} penalti",
"obs": "{count}",
"tokPerMonth": "{count} tok/mo",
diff --git a/client/src/i18n/locales/bg.json b/client/src/i18n/locales/bg.json
index d0a0b11d..8935e307 100644
--- a/client/src/i18n/locales/bg.json
+++ b/client/src/i18n/locales/bg.json
@@ -122,6 +122,7 @@
"visionTitle": "Приема въвеждане на изображения",
"tools": "Инструменти",
"toolsTitle": "Изпуска структурирани повиквания на инструменти, така че е допустим за заявки, носещи инструменти,",
+ "retired": "Изтеглен",
"penalty": "−{value} наказание",
"obs": "{count} OBS",
"tokPerMonth": "{count} ток/мо",
diff --git a/client/src/i18n/locales/bn.json b/client/src/i18n/locales/bn.json
index 1ce3c993..1a797690 100644
--- a/client/src/i18n/locales/bn.json
+++ b/client/src/i18n/locales/bn.json
@@ -122,6 +122,7 @@
"visionTitle": "ইমেজ ইনপুট গ্রহণ করে",
"tools": "টুলস",
"toolsTitle": "স্ট্রাকচার্ড টুল কল নির্গত করে, তাই এটি টুল-বেয়ারিং অনুরোধের জন্য যোগ্য",
+ "retired": "প্রত্যাহৃত",
"penalty": "−{value} পেনাল্টি",
"obs": "{count} obs",
"tokPerMonth": "{count} টোক/মাস",
diff --git a/client/src/i18n/locales/cs.json b/client/src/i18n/locales/cs.json
index 1e9d745b..6e9810fa 100644
--- a/client/src/i18n/locales/cs.json
+++ b/client/src/i18n/locales/cs.json
@@ -122,6 +122,7 @@
"visionTitle": "Přijímá vstup obrazu",
"tools": "Nástroje",
"toolsTitle": "Vysílá strukturované volání nástrojů, takže je způsobilý pro požadavky na nosiče nástroje",
+ "retired": "Vyřazeno",
"penalty": "−{value} trest",
"obs": "{count} OBS",
"tokPerMonth": "{count} tok/měsíc",
diff --git a/client/src/i18n/locales/da.json b/client/src/i18n/locales/da.json
index 58e808dd..ee207dc1 100644
--- a/client/src/i18n/locales/da.json
+++ b/client/src/i18n/locales/da.json
@@ -122,6 +122,7 @@
"visionTitle": "Accepterer billedinput",
"tools": "Værktøjer",
"toolsTitle": "Udsender strukturerede værktøjskald, så den er berettiget til værktøjsbærende anmodninger",
+ "retired": "Udgået",
"penalty": "−{value} straf",
"obs": "{count} observationer",
"tokPerMonth": "{count} tok/mo",
diff --git a/client/src/i18n/locales/de.json b/client/src/i18n/locales/de.json
index e1c1246b..ef860778 100644
--- a/client/src/i18n/locales/de.json
+++ b/client/src/i18n/locales/de.json
@@ -122,6 +122,7 @@
"visionTitle": "Akzeptiert Bildeingaben",
"tools": "Werkzeuge",
"toolsTitle": "Gibt strukturierte Werkzeugaufrufe aus und ist daher für werkzeugtragende Anforderungen geeignet",
+ "retired": "Eingestellt",
"penalty": "−{value} Strafe",
"obs": "{count} obs",
"tokPerMonth": "{count} Tok/Mo",
diff --git a/client/src/i18n/locales/el.json b/client/src/i18n/locales/el.json
index 71eadbe1..6f20d44d 100644
--- a/client/src/i18n/locales/el.json
+++ b/client/src/i18n/locales/el.json
@@ -122,6 +122,7 @@
"visionTitle": "Δέχεται εισαγωγή εικόνας",
"tools": "Εργαλεία",
"toolsTitle": "Εκπέμπει δομημένες κλήσεις εργαλείων, ώστε να είναι επιλέξιμο για αιτήματα που φέρουν εργαλεία",
+ "retired": "Αποσύρθηκε",
"penalty": "−{value} ποινή",
"obs": "{count} obs",
"tokPerMonth": "{count} tok/μήνα",
diff --git a/client/src/i18n/locales/en.json b/client/src/i18n/locales/en.json
index 99acfc40..eacc81b7 100644
--- a/client/src/i18n/locales/en.json
+++ b/client/src/i18n/locales/en.json
@@ -122,6 +122,7 @@
"visionTitle": "Accepts image input",
"tools": "Tools",
"toolsTitle": "Emits structured tool calls, so it is eligible for tool-bearing requests",
+ "retired": "Retired",
"penalty": "−{value} penalty",
"obs": "{count} obs",
"tokPerMonth": "{count} tok/mo",
diff --git a/client/src/i18n/locales/es.json b/client/src/i18n/locales/es.json
index 7f68f4a6..5c0f87e1 100644
--- a/client/src/i18n/locales/es.json
+++ b/client/src/i18n/locales/es.json
@@ -122,6 +122,7 @@
"visionTitle": "Acepta imágenes como entrada",
"tools": "Herramientas",
"toolsTitle": "Emite llamadas a herramientas estructuradas, por lo que es apto para solicitudes con herramientas",
+ "retired": "Retirado",
"penalty": "penalización de −{value}",
"obs": "{count} obs",
"tokPerMonth": "{count} tok/mes",
diff --git a/client/src/i18n/locales/fa.json b/client/src/i18n/locales/fa.json
index 2c759400..7f82bc9d 100644
--- a/client/src/i18n/locales/fa.json
+++ b/client/src/i18n/locales/fa.json
@@ -122,6 +122,7 @@
"visionTitle": "ورودی تصویر را می پذیرد",
"tools": "ابزار",
"toolsTitle": "فراخوانی ابزار ساختاریافته را منتشر می کند، بنابراین برای درخواست های حاوی ابزار واجد شرایط است",
+ "retired": "متوقف",
"penalty": "- جریمه {value}",
"obs": "{count} obs",
"tokPerMonth": "{count} توک/ماه",
diff --git a/client/src/i18n/locales/fi.json b/client/src/i18n/locales/fi.json
index 5f9cdb16..efedfcb3 100644
--- a/client/src/i18n/locales/fi.json
+++ b/client/src/i18n/locales/fi.json
@@ -122,6 +122,7 @@
"visionTitle": "Hyväksyy kuvasyötteen",
"tools": "Työkalut",
"toolsTitle": "Lähettää rakenteellisia työkalukutsuja, joten se on kelpoinen työkalun kantamiseen liittyviin pyyntöihin",
+ "retired": "Poistettu",
"penalty": "−{value} rangaistus",
"obs": "{count} Obs",
"tokPerMonth": "{count} tok/mo",
diff --git a/client/src/i18n/locales/fr.json b/client/src/i18n/locales/fr.json
index ddd1e4da..7278f149 100644
--- a/client/src/i18n/locales/fr.json
+++ b/client/src/i18n/locales/fr.json
@@ -122,6 +122,7 @@
"visionTitle": "Accepte les images en entrée",
"tools": "Outils",
"toolsTitle": "Émet des appels d'outils structurés, il est donc éligible aux requêtes avec outils",
+ "retired": "Retiré",
"penalty": "pénalité de −{value}",
"obs": "{count} obs",
"tokPerMonth": "{count} tok/mois",
diff --git a/client/src/i18n/locales/gu.json b/client/src/i18n/locales/gu.json
index c291dc3b..1c206f40 100644
--- a/client/src/i18n/locales/gu.json
+++ b/client/src/i18n/locales/gu.json
@@ -122,6 +122,7 @@
"visionTitle": "ઇમેજ ઇનપુટ સ્વીકારે છે",
"tools": "સાધનો",
"toolsTitle": "સ્ટ્રક્ચર્ડ ટૂલ કૉલ્સ બહાર કાઢે છે, તેથી તે ટૂલ-બેરિંગ વિનંતીઓ માટે પાત્ર છે",
+ "retired": "બંધ કરાયું",
"penalty": "−{value} દંડ",
"obs": "{count} obs",
"tokPerMonth": "{count} tok/mo",
diff --git a/client/src/i18n/locales/ha.json b/client/src/i18n/locales/ha.json
index 2cd87708..0e92ab2c 100644
--- a/client/src/i18n/locales/ha.json
+++ b/client/src/i18n/locales/ha.json
@@ -122,6 +122,7 @@
"visionTitle": "Yana karɓar shigarwar hoto",
"tools": "Kayan aiki",
"toolsTitle": "Yana fitar da tsarin kiran kayan aiki, don haka ya cancanci buƙatun ɗaukar kayan aiki",
+ "retired": "An dakatar",
"penalty": "-{value} hukunci",
"obs": "Saukewa: {count}",
"tokPerMonth": "{count} tok/mo",
diff --git a/client/src/i18n/locales/he.json b/client/src/i18n/locales/he.json
index 7e920dd7..57cad875 100644
--- a/client/src/i18n/locales/he.json
+++ b/client/src/i18n/locales/he.json
@@ -122,6 +122,7 @@
"visionTitle": "מקבל קלט תמונה",
"tools": "כלים",
"toolsTitle": "משדר קריאות לכלים מובנים, ולכן הוא זכאי לבקשות עם כלים",
+ "retired": "הוסר",
"penalty": "−{value} עונש",
"obs": "{count} obs",
"tokPerMonth": "{count} טוק/מו",
diff --git a/client/src/i18n/locales/hi.json b/client/src/i18n/locales/hi.json
index 23d42ef6..c37f260e 100644
--- a/client/src/i18n/locales/hi.json
+++ b/client/src/i18n/locales/hi.json
@@ -122,6 +122,7 @@
"visionTitle": "छवि इनपुट स्वीकार करता है",
"tools": "उपकरण",
"toolsTitle": "संरचित टूल कॉल उत्सर्जित करता है, इसलिए यह टूल-बेयरिंग अनुरोधों के लिए पात्र है",
+ "retired": "बंद",
"penalty": "−{value} जुर्माना",
"obs": "{count} अवलोकन",
"tokPerMonth": "{count} tok/mo",
diff --git a/client/src/i18n/locales/hr.json b/client/src/i18n/locales/hr.json
index 71f25ad7..e4de72f2 100644
--- a/client/src/i18n/locales/hr.json
+++ b/client/src/i18n/locales/hr.json
@@ -122,6 +122,7 @@
"visionTitle": "Prihvaća ulaz slike",
"tools": "Alati",
"toolsTitle": "Emitira strukturirane pozive alata, pa je prihvatljiv za zahtjeve s nosačem alata",
+ "retired": "Povučen",
"penalty": "−{value} kazna",
"obs": "{count} OBS",
"tokPerMonth": "{count} tok/mo",
diff --git a/client/src/i18n/locales/hu.json b/client/src/i18n/locales/hu.json
index d1423aea..258142c5 100644
--- a/client/src/i18n/locales/hu.json
+++ b/client/src/i18n/locales/hu.json
@@ -122,6 +122,7 @@
"visionTitle": "Képbemenetet fogad",
"tools": "Eszközök",
"toolsTitle": "Strukturált eszközhívásokat bocsát ki, így jogosult szerszámtartó kérésekre",
+ "retired": "Kivonva",
"penalty": "−{value} büntető",
"obs": "{count} obs",
"tokPerMonth": "{count} tok/mo",
diff --git a/client/src/i18n/locales/id.json b/client/src/i18n/locales/id.json
index d9f136d9..7029a8c3 100644
--- a/client/src/i18n/locales/id.json
+++ b/client/src/i18n/locales/id.json
@@ -122,6 +122,7 @@
"visionTitle": "Menerima masukan gambar",
"tools": "Alat",
"toolsTitle": "Memancarkan panggilan alat terstruktur, sehingga memenuhi syarat untuk permintaan bantalan alat",
+ "retired": "Dihentikan",
"penalty": "−Penalti {value}",
"obs": "{count} obs",
"tokPerMonth": "{count} tok/bln",
diff --git a/client/src/i18n/locales/ig.json b/client/src/i18n/locales/ig.json
index ab032f19..934786f4 100644
--- a/client/src/i18n/locales/ig.json
+++ b/client/src/i18n/locales/ig.json
@@ -122,6 +122,7 @@
"visionTitle": "Nabata ntinye onyonyo",
"tools": "Ngwa",
"toolsTitle": "Ọ na-ewepụta oku akụrụngwa ahaziri ahazi, yabụ na ọ tozuru maka arịrịọ ibu ngwá ọrụ",
+ "retired": "Akwụsịla",
"penalty": "-{value} ntaramahụhụ",
"obs": "{count} obs",
"tokPerMonth": "{count} tok/mo",
diff --git a/client/src/i18n/locales/it.json b/client/src/i18n/locales/it.json
index a020d240..825dfe6c 100644
--- a/client/src/i18n/locales/it.json
+++ b/client/src/i18n/locales/it.json
@@ -122,6 +122,7 @@
"visionTitle": "Accetta input di immagini",
"tools": "Strumenti",
"toolsTitle": "Emette chiamate a strumenti strutturate, quindi è idoneo per richieste con strumenti",
+ "retired": "Ritirato",
"penalty": "−{value} penalità",
"obs": "{count} oss",
"tokPerMonth": "{count} tok/mese",
diff --git a/client/src/i18n/locales/ja.json b/client/src/i18n/locales/ja.json
index 815f45ea..5cda7dc9 100644
--- a/client/src/i18n/locales/ja.json
+++ b/client/src/i18n/locales/ja.json
@@ -122,6 +122,7 @@
"visionTitle": "画像入力を受け付ける",
"tools": "ツール",
"toolsTitle": "構造化されたツール呼び出しを発行するため、ツールを含むリクエストに適格です",
+ "retired": "提供終了",
"penalty": "−{value}ペナルティ",
"obs": "{count} オブス",
"tokPerMonth": "{count} トク/月",
diff --git a/client/src/i18n/locales/ka.json b/client/src/i18n/locales/ka.json
index 834bf129..c7d33207 100644
--- a/client/src/i18n/locales/ka.json
+++ b/client/src/i18n/locales/ka.json
@@ -122,6 +122,7 @@
"visionTitle": "იღებს სურათის შეყვანას",
"tools": "ხელსაწყოები",
"toolsTitle": "ავრცელებს სტრუქტურირებულ ხელსაწყოების ზარებს, ამიტომ ის დასაშვებია ხელსაწყოების შემცველი მოთხოვნებისთვის",
+ "retired": "შეწყვეტილია",
"penalty": "−{value} ჯარიმა",
"obs": "{count} obs",
"tokPerMonth": "{count} ტოკ/თვე",
diff --git a/client/src/i18n/locales/km.json b/client/src/i18n/locales/km.json
index 058667b8..df2f8958 100644
--- a/client/src/i18n/locales/km.json
+++ b/client/src/i18n/locales/km.json
@@ -122,6 +122,7 @@
"visionTitle": "ទទួលបញ្ចូលរូបភាព",
"tools": "ឧបករណ៍",
"toolsTitle": "បញ្ចេញការហៅឧបករណ៍ដែលមានរចនាសម្ព័ន្ធ ដូច្នេះវាមានសិទ្ធិទទួលសំណើដែលមានឧបករណ៍",
+ "retired": "បញ្ឈប់",
"penalty": "−{value} ពិន័យ",
"obs": "{count}",
"tokPerMonth": "{count} តុក/ខែ",
diff --git a/client/src/i18n/locales/kn.json b/client/src/i18n/locales/kn.json
index 1d81e1d1..30045a57 100644
--- a/client/src/i18n/locales/kn.json
+++ b/client/src/i18n/locales/kn.json
@@ -122,6 +122,7 @@
"visionTitle": "ಚಿತ್ರದ ಇನ್‌ಪುಟ್ ಅನ್ನು ಸ್ವೀಕರಿಸುತ್ತದೆ",
"tools": "ಪರಿಕರಗಳು",
"toolsTitle": "ರಚನಾತ್ಮಕ ಟೂಲ್ ಕರೆಗಳನ್ನು ಹೊರಸೂಸುತ್ತದೆ, ಆದ್ದರಿಂದ ಇದು ಟೂಲ್-ಬೇರಿಂಗ್ ವಿನಂತಿಗಳಿಗೆ ಅರ್ಹವಾಗಿದೆ",
+ "retired": "ನಿಲ್ಲಿಸಲಾಗಿದೆ",
"penalty": "−{value} ಪೆನಾಲ್ಟಿ",
"obs": "{count} obs",
"tokPerMonth": "{count} tok/mo",
diff --git a/client/src/i18n/locales/ko.json b/client/src/i18n/locales/ko.json
index e1dd7c1e..d1ccec1d 100644
--- a/client/src/i18n/locales/ko.json
+++ b/client/src/i18n/locales/ko.json
@@ -122,6 +122,7 @@
"visionTitle": "이미지 입력을 허용합니다",
"tools": "도구",
"toolsTitle": "구조화된 도구 호출을 내보내므로 도구 베어링 요청에 적합합니다.",
+ "retired": "지원 종료",
"penalty": "−{value} 페널티",
"obs": "{count} 관측",
"tokPerMonth": "{count} 톡/월",
diff --git a/client/src/i18n/locales/lt.json b/client/src/i18n/locales/lt.json
index d5b2e63b..c8c5e286 100644
--- a/client/src/i18n/locales/lt.json
+++ b/client/src/i18n/locales/lt.json
@@ -122,6 +122,7 @@
"visionTitle": "Priima vaizdo įvestį",
"tools": "Įrankiai",
"toolsTitle": "Skleidžia struktūrizuotus įrankių kvietimus, todėl yra tinkamas įrankių nešimo užklausoms",
+ "retired": "Pašalintas",
"penalty": "−{value} bauda",
"obs": "{count} obs",
"tokPerMonth": "{count} tok/mo",
diff --git a/client/src/i18n/locales/ml.json b/client/src/i18n/locales/ml.json
index 5c890149..c499ad51 100644
--- a/client/src/i18n/locales/ml.json
+++ b/client/src/i18n/locales/ml.json
@@ -122,6 +122,7 @@
"visionTitle": "ഇമേജ് ഇൻപുട്ട് സ്വീകരിക്കുന്നു",
"tools": "ഉപകരണങ്ങൾ",
"toolsTitle": "ഘടനാപരമായ ടൂൾ കോളുകൾ പുറപ്പെടുവിക്കുന്നു, അതിനാൽ ടൂൾ-ബെയറിംഗ് അഭ്യർത്ഥനകൾക്ക് ഇത് യോഗ്യമാണ്",
+ "retired": "നിർത്തി",
"penalty": "−{value} പിഴ",
"obs": "{count} ഒബ്സ്",
"tokPerMonth": "{count} ടോക്ക്/മാസം",
diff --git a/client/src/i18n/locales/mr.json b/client/src/i18n/locales/mr.json
index 693403f2..049b5209 100644
--- a/client/src/i18n/locales/mr.json
+++ b/client/src/i18n/locales/mr.json
@@ -122,6 +122,7 @@
"visionTitle": "प्रतिमा इनपुट स्वीकारते",
"tools": "साधने",
"toolsTitle": "संरचित टूल कॉल सोडते, त्यामुळे ते टूल-बेअरिंग विनंत्यांना पात्र आहे",
+ "retired": "बंद",
"penalty": "−{value} दंड",
"obs": "{count} obs",
"tokPerMonth": "{count} tok/mo",
diff --git a/client/src/i18n/locales/ms.json b/client/src/i18n/locales/ms.json
index ab112e03..1c7cf5ff 100644
--- a/client/src/i18n/locales/ms.json
+++ b/client/src/i18n/locales/ms.json
@@ -122,6 +122,7 @@
"visionTitle": "Menerima input imej",
"tools": "Alat",
"toolsTitle": "Memancarkan panggilan alat berstruktur, jadi ia layak untuk permintaan yang membawa alat",
+ "retired": "Dihentikan",
"penalty": "Penalti −{value}",
"obs": "{count} obs",
"tokPerMonth": "{count} Tok/Bulan",
diff --git a/client/src/i18n/locales/my.json b/client/src/i18n/locales/my.json
index b6168b99..c6131746 100644
--- a/client/src/i18n/locales/my.json
+++ b/client/src/i18n/locales/my.json
@@ -122,6 +122,7 @@
"visionTitle": "ပုံရိပ်ထည့်သွင်းမှုကို လက်ခံသည်",
"tools": "ကိရိယာများ",
"toolsTitle": "ဖွဲ့စည်းထားသော ကိရိယာခေါ်ဆိုမှုများ ထုတ်လွှင့်သဖြင့် ကိရိယာပါသော တောင်းဆိုမှုများအတွက် အရည်အချင်းပြည့်မီသည်။",
+ "retired": "ရပ်ဆိုင်းပြီး",
"penalty": "−{value} ပြစ်ဒဏ်",
"obs": "{count}",
"tokPerMonth": "{count} tok/mo",
diff --git a/client/src/i18n/locales/ne.json b/client/src/i18n/locales/ne.json
index b19065f8..0995f134 100644
--- a/client/src/i18n/locales/ne.json
+++ b/client/src/i18n/locales/ne.json
@@ -122,6 +122,7 @@
"visionTitle": "छवि आगत स्वीकार गर्दछ",
"tools": "उपकरणहरू",
"toolsTitle": "संरचित उपकरण कलहरू उत्सर्जन गर्दछ, त्यसैले यो उपकरण-असर अनुरोधहरूको लागि योग्य छ",
+ "retired": "बन्द",
"penalty": "−{value} पेनाल्टी",
"obs": "{count} obs",
"tokPerMonth": "{count} टोक/मो",
diff --git a/client/src/i18n/locales/nl.json b/client/src/i18n/locales/nl.json
index 995b4a71..cc70204e 100644
--- a/client/src/i18n/locales/nl.json
+++ b/client/src/i18n/locales/nl.json
@@ -122,6 +122,7 @@
"visionTitle": "Accepteert beeldinvoer",
"tools": "Gereedschappen",
"toolsTitle": "Zendt gestructureerde tooloproepen uit, zodat het in aanmerking komt voor tool-bearing verzoeken",
+ "retired": "Uitgefaseerd",
"penalty": "−{value} straf",
"obs": "{count} obs",
"tokPerMonth": "{count} tok/mo",
diff --git a/client/src/i18n/locales/no.json b/client/src/i18n/locales/no.json
index f5fc4fef..9cd23e38 100644
--- a/client/src/i18n/locales/no.json
+++ b/client/src/i18n/locales/no.json
@@ -122,6 +122,7 @@
"visionTitle": "Godtar bildeinput",
"tools": "Verktøy",
"toolsTitle": "Sender ut strukturerte verktøykall, slik at den er kvalifisert for verktøybærende forespørsler",
+ "retired": "Utgått",
"penalty": "−{value} straff",
"obs": "{count} observasjoner",
"tokPerMonth": "{count} tok/mo",
diff --git a/client/src/i18n/locales/or.json b/client/src/i18n/locales/or.json
index fce4c10f..bb6a253d 100644
--- a/client/src/i18n/locales/or.json
+++ b/client/src/i18n/locales/or.json
@@ -122,6 +122,7 @@
"visionTitle": "ଫଟୋ ଇନପୁଟ୍ ସ୍ଵୀକାର କରେ",
"tools": "ଉପକରଣ",
"toolsTitle": "ସଂରଚନା ଟୁଲ୍ କଲ୍ ନିର୍ଗତ କରେ, ତେଣୁ ଏହା ଟୁଲ୍-ବିୟରିଂ ଅନୁରୋଧ ପାଇଁ ଯୋଗ୍ୟ ଅଟେ",
+ "retired": "ବନ୍ଦ",
"penalty": "−{value} ପେନାଲ୍ଟି",
"obs": "{count} ଓବିଏସ୍",
"tokPerMonth": "{count} tok/mo",
diff --git a/client/src/i18n/locales/pa.json b/client/src/i18n/locales/pa.json
index 3ce8b87f..9278d4de 100644
--- a/client/src/i18n/locales/pa.json
+++ b/client/src/i18n/locales/pa.json
@@ -122,6 +122,7 @@
"visionTitle": "ਚਿੱਤਰ ਇੰਪੁੱਟ ਸਵੀਕਾਰ ਕਰਦਾ ਹੈ",
"tools": "ਸੰਦ",
"toolsTitle": "ਸਟ੍ਰਕਚਰਡ ਟੂਲ ਕਾਲਾਂ ਨੂੰ ਛੱਡਦਾ ਹੈ, ਇਸਲਈ ਇਹ ਟੂਲ-ਬੇਅਰਿੰਗ ਬੇਨਤੀਆਂ ਲਈ ਯੋਗ ਹੈ",
+ "retired": "ਬੰਦ",
"penalty": "−{value} ਜੁਰਮਾਨਾ",
"obs": "{count} obs",
"tokPerMonth": "{count} tok/mo",
diff --git a/client/src/i18n/locales/pl.json b/client/src/i18n/locales/pl.json
index 7f115f9f..7e85288b 100644
--- a/client/src/i18n/locales/pl.json
+++ b/client/src/i18n/locales/pl.json
@@ -122,6 +122,7 @@
"visionTitle": "Akceptuje wejście obrazu",
"tools": "Narzędzia",
"toolsTitle": "Emituje uporządkowane wywołania narzędzi, więc kwalifikuje się do żądań łożysk narzędziowych",
+ "retired": "Wycofany",
"penalty": "−{value} kara",
"obs": "{count} obs",
"tokPerMonth": "{count} tok/mies",
diff --git a/client/src/i18n/locales/pt-BR.json b/client/src/i18n/locales/pt-BR.json
index e61d255c..2a856f57 100644
--- a/client/src/i18n/locales/pt-BR.json
+++ b/client/src/i18n/locales/pt-BR.json
@@ -122,6 +122,7 @@
"visionTitle": "Aceita imagens como entrada",
"tools": "Ferramentas",
"toolsTitle": "Emite chamadas de ferramentas estruturadas, por isso é elegível para solicitações com ferramentas",
+ "retired": "Descontinuado",
"penalty": "penalidade de −{value}",
"obs": "{count} obs",
"tokPerMonth": "{count} tok/mês",
diff --git a/client/src/i18n/locales/pt-PT.json b/client/src/i18n/locales/pt-PT.json
index 55a8a2c8..859c41f6 100644
--- a/client/src/i18n/locales/pt-PT.json
+++ b/client/src/i18n/locales/pt-PT.json
@@ -122,6 +122,7 @@
"visionTitle": "Aceita entrada de imagem",
"tools": "Ferramentas",
"toolsTitle": "Emite chamadas estruturadas de ferramentas, tornando-se elegível para pedidos de suporte de ferramenta",
+ "retired": "Descontinuado",
"penalty": "−{value} penalização",
"obs": "{count} obs",
"tokPerMonth": "{count} tok/mo",
diff --git a/client/src/i18n/locales/ro.json b/client/src/i18n/locales/ro.json
index 2c2e7fcb..0d0fe304 100644
--- a/client/src/i18n/locales/ro.json
+++ b/client/src/i18n/locales/ro.json
@@ -122,6 +122,7 @@
"visionTitle": "Acceptă intrarea de imagine",
"tools": "Unelte",
"toolsTitle": "Emite apeluri structurate pentru unelte, deci este eligibil pentru cereri cu rulante de unelte",
+ "retired": "Retras",
"penalty": "−{value} penalizare",
"obs": "{count} observații",
"tokPerMonth": "{count} tok/mo",
diff --git a/client/src/i18n/locales/ru.json b/client/src/i18n/locales/ru.json
index 3b8ad01d..6ea83cb3 100644
--- a/client/src/i18n/locales/ru.json
+++ b/client/src/i18n/locales/ru.json
@@ -122,6 +122,7 @@
"visionTitle": "Принимает ввод изображения",
"tools": "Инструменты",
"toolsTitle": "Выполняет структурированные вызовы инструментов, поэтому он имеет право на запросы с инструментами.",
+ "retired": "Снят",
"penalty": "−{value} штраф",
"obs": "{count} обс",
"tokPerMonth": "{count} ток/мес",
diff --git a/client/src/i18n/locales/si.json b/client/src/i18n/locales/si.json
index 87eb6887..5d8a43eb 100644
--- a/client/src/i18n/locales/si.json
+++ b/client/src/i18n/locales/si.json
@@ -122,6 +122,7 @@
"visionTitle": "රූප ආදානය පිළිගනී",
"tools": "මෙවලම්",
"toolsTitle": "ව්‍යුහගත මෙවලම් ඇමතුම් නිකුත් කරයි, එබැවින් එය මෙවලම් දරණ ඉල්ලීම් සඳහා සුදුසුකම් ලබයි",
+ "retired": "නවතා ඇත",
"penalty": "−{value} දඩුවම",
"obs": "{count} obs",
"tokPerMonth": "{count} tok/mo",
diff --git a/client/src/i18n/locales/sk.json b/client/src/i18n/locales/sk.json
index 28d0a241..5fba552c 100644
--- a/client/src/i18n/locales/sk.json
+++ b/client/src/i18n/locales/sk.json
@@ -122,6 +122,7 @@
"visionTitle": "Prijíma vstup obrazu",
"tools": "Nástroje",
"toolsTitle": "Vysiela štruktúrované volania nástroja, takže je oprávnený na požiadavky na ložisko nástroja",
+ "retired": "Vyradené",
"penalty": "−{value} penalta",
"obs": "{count} OBS",
"tokPerMonth": "{count} tok/mo",
diff --git a/client/src/i18n/locales/sr.json b/client/src/i18n/locales/sr.json
index cfa02b24..00ce9bbd 100644
--- a/client/src/i18n/locales/sr.json
+++ b/client/src/i18n/locales/sr.json
@@ -122,6 +122,7 @@
"visionTitle": "Прихвата унос слике",
"tools": "Алати",
"toolsTitle": "Емитује структуриране позиве алата, тако да испуњава услове за захтеве који носе алате",
+ "retired": "Повучен",
"penalty": "−{value} казна",
"obs": "{count} обс",
"tokPerMonth": "{count} ток/мес",
diff --git a/client/src/i18n/locales/sv.json b/client/src/i18n/locales/sv.json
index f1f7e858..56c244fd 100644
--- a/client/src/i18n/locales/sv.json
+++ b/client/src/i18n/locales/sv.json
@@ -122,6 +122,7 @@
"visionTitle": "Accepterar bildindata",
"tools": "Verktyg",
"toolsTitle": "Skickar strukturerade verktygsanrop, så den är berättigad till verktygsbärande förfrågningar",
+ "retired": "Utgått",
"penalty": "−{value} straff",
"obs": "{count} observationer",
"tokPerMonth": "{count} tok/mo",
diff --git a/client/src/i18n/locales/sw.json b/client/src/i18n/locales/sw.json
index 4c593e0d..01698ddb 100644
--- a/client/src/i18n/locales/sw.json
+++ b/client/src/i18n/locales/sw.json
@@ -122,6 +122,7 @@
"visionTitle": "Inakubali ingizo la picha",
"tools": "Zana",
"toolsTitle": "Hutoa simu za zana zilizopangwa, kwa hivyo inastahiki maombi yenye zana",
+ "retired": "Imeondolewa",
"penalty": "−{value} adhabu",
"obs": "Sehemu za {count}",
"tokPerMonth": "{count} tok/mo",
diff --git a/client/src/i18n/locales/ta.json b/client/src/i18n/locales/ta.json
index dd77b126..5d367a5c 100644
--- a/client/src/i18n/locales/ta.json
+++ b/client/src/i18n/locales/ta.json
@@ -122,6 +122,7 @@
"visionTitle": "பட உள்ளீட்டை ஏற்கிறது",
"tools": "கருவிகள்",
"toolsTitle": "கட்டமைக்கப்பட்ட கருவி அழைப்புகளை வெளியிடுகிறது, எனவே கருவி தாங்கும் கோரிக்கைகளுக்கு இது தகுதியானது",
+ "retired": "நிறுத்தப்பட்டது",
"penalty": "−{value} அபராதம்",
"obs": "{count} obs",
"tokPerMonth": "{count} tok/mo",
diff --git a/client/src/i18n/locales/te.json b/client/src/i18n/locales/te.json
index a8092b1e..9151da57 100644
--- a/client/src/i18n/locales/te.json
+++ b/client/src/i18n/locales/te.json
@@ -122,6 +122,7 @@
"visionTitle": "చిత్రం ఇన్‌పుట్‌ని అంగీకరిస్తుంది",
"tools": "ఉపకరణాలు",
"toolsTitle": "నిర్మాణాత్మక టూల్ కాల్‌లను విడుదల చేస్తుంది, కాబట్టి ఇది టూల్-బేరింగ్ అభ్యర్థనలకు అర్హత కలిగి ఉంటుంది",
+ "retired": "నిలిపివేయబడింది",
"penalty": "−{value} పెనాల్టీ",
"obs": "{count} obs",
"tokPerMonth": "{count} టోక్/మొ",
diff --git a/client/src/i18n/locales/th.json b/client/src/i18n/locales/th.json
index 29f98803..4151357c 100644
--- a/client/src/i18n/locales/th.json
+++ b/client/src/i18n/locales/th.json
@@ -122,6 +122,7 @@
"visionTitle": "ยอมรับอินพุตรูปภาพ",
"tools": "เครื่องมือ",
"toolsTitle": "ส่งเสียงการเรียกเครื่องมือที่มีโครงสร้าง ดังนั้นจึงมีสิทธิ์ได้รับคำขอเกี่ยวกับเครื่องมือ",
+ "retired": "ยกเลิกแล้ว",
"penalty": "−{value} การลงโทษ",
"obs": "{count} obs",
"tokPerMonth": "{count} ท็อค/เดือน",
diff --git a/client/src/i18n/locales/tl.json b/client/src/i18n/locales/tl.json
index 5b548974..7b6000fd 100644
--- a/client/src/i18n/locales/tl.json
+++ b/client/src/i18n/locales/tl.json
@@ -122,6 +122,7 @@
"visionTitle": "Tumatanggap ng image input",
"tools": "Mga Kasangkapan",
"toolsTitle": "Naglalabas ng structured tool calls, kaya ito ay kwalipikado para sa mga tool-bearing request",
+ "retired": "Itinigil",
"penalty": "−{value} penalty",
"obs": "{count} obs",
"tokPerMonth": "{count} tok/buwan",
diff --git a/client/src/i18n/locales/tr.json b/client/src/i18n/locales/tr.json
index 560be136..4722474a 100644
--- a/client/src/i18n/locales/tr.json
+++ b/client/src/i18n/locales/tr.json
@@ -122,6 +122,7 @@
"visionTitle": "Görüntü girişini kabul eder",
"tools": "Araçlar",
"toolsTitle": "Yapılandırılmış araç çağrıları yayınlayarak araç taşıma isteklerine uygun hale gelir",
+ "retired": "Kaldırıldı",
"penalty": "−{value} cezası",
"obs": "{count} gözlem",
"tokPerMonth": "{count} tok/ay",
diff --git a/client/src/i18n/locales/uk.json b/client/src/i18n/locales/uk.json
index 87f9bfff..7f4daa05 100644
--- a/client/src/i18n/locales/uk.json
+++ b/client/src/i18n/locales/uk.json
@@ -122,6 +122,7 @@
"visionTitle": "Приймає введення зображення",
"tools": "Інструменти",
"toolsTitle": "Видає структуровані виклики інструментів, тому він підходить для запитів, що містять інструменти",
+ "retired": "Знято",
"penalty": "−{value} пенальті",
"obs": "{count} обс",
"tokPerMonth": "{count} ток/міс",
diff --git a/client/src/i18n/locales/ur.json b/client/src/i18n/locales/ur.json
index a6c26a40..d2c50e6b 100644
--- a/client/src/i18n/locales/ur.json
+++ b/client/src/i18n/locales/ur.json
@@ -122,6 +122,7 @@
"visionTitle": "امیج ان پٹ کو قبول کرتا ہے۔",
"tools": "اوزار",
"toolsTitle": "سٹرکچرڈ ٹول کالز کو خارج کرتا ہے، اس لیے یہ ٹول بیئرنگ کی درخواستوں کے لیے اہل ہے۔",
+ "retired": "بند",
"penalty": "−{value} جرمانہ",
"obs": "{count} obs",
"tokPerMonth": "{count} tok/mo",
diff --git a/client/src/i18n/locales/uz.json b/client/src/i18n/locales/uz.json
index 8743e1de..2632473d 100644
--- a/client/src/i18n/locales/uz.json
+++ b/client/src/i18n/locales/uz.json
@@ -122,6 +122,7 @@
"visionTitle": "Rasm kiritishni qabul qiladi",
"tools": "Asboblar",
"toolsTitle": "Strukturaviy asboblar qo'ng'iroqlarini chiqaradi, shuning uchun u asboblar so'rovlari uchun javob beradi",
+ "retired": "To'xtatilgan",
"penalty": "−{value} jarima",
"obs": "{count} obs",
"tokPerMonth": "{count} tok/oy",
diff --git a/client/src/i18n/locales/vi.json b/client/src/i18n/locales/vi.json
index ed8bcee3..44aa34eb 100644
--- a/client/src/i18n/locales/vi.json
+++ b/client/src/i18n/locales/vi.json
@@ -122,6 +122,7 @@
"visionTitle": "Chấp nhận đầu vào hình ảnh",
"tools": "Công cụ",
"toolsTitle": "Phát ra các lệnh gọi công cụ có cấu trúc, do đó nó đủ điều kiện cho các yêu cầu mang công cụ",
+ "retired": "Ngừng cung cấp",
"penalty": "−{value} phạt đền",
"obs": "{count} quan sát",
"tokPerMonth": "{count} tháng/tháng",
diff --git a/client/src/i18n/locales/yo.json b/client/src/i18n/locales/yo.json
index 3bcba6c8..61732698 100644
--- a/client/src/i18n/locales/yo.json
+++ b/client/src/i18n/locales/yo.json
@@ -122,6 +122,7 @@
"visionTitle": "Gba igbewọle aworan",
"tools": "Awọn irinṣẹ",
"toolsTitle": "Emits awọn ipe irinṣẹ ti a ṣeto, nitorinaa o yẹ fun awọn ibeere ti nso ohun elo",
+ "retired": "A ti dáwọ́ dúró",
"penalty": "-{value} ijiya",
"obs": "{count} obs",
"tokPerMonth": "{count} tok/mo",
diff --git a/client/src/i18n/locales/zh-CN.json b/client/src/i18n/locales/zh-CN.json
index 684b13ff..83e42125 100644
--- a/client/src/i18n/locales/zh-CN.json
+++ b/client/src/i18n/locales/zh-CN.json
@@ -122,6 +122,7 @@
"visionTitle": "支持图像输入",
"tools": "工具",
"toolsTitle": "支持结构化工具调用,可用于需要工具的请求",
+ "retired": "已下线",
"penalty": "−{value} 惩罚",
"obs": "{count} 次观察",
"tokPerMonth": "{count} 令牌/月",
diff --git a/client/src/i18n/locales/zh-TW.json b/client/src/i18n/locales/zh-TW.json
index 7f6ccba9..17d8c838 100644
--- a/client/src/i18n/locales/zh-TW.json
+++ b/client/src/i18n/locales/zh-TW.json
@@ -122,6 +122,7 @@
"visionTitle": "接受影像輸入",
"tools": "工具",
"toolsTitle": "會發出結構化工具呼叫,因此有資格進行工具承載請求",
+ "retired": "已下線",
"penalty": "−{value} 罰則",
"obs": "{count}觀察",
"tokPerMonth": "{count} tok/mo",
diff --git a/client/src/lib/routing.ts b/client/src/lib/routing.ts
index 8c634c7a..272b904e 100644
--- a/client/src/lib/routing.ts
+++ b/client/src/lib/routing.ts
@@ -30,6 +30,11 @@ export interface FallbackEntry {
keyId?: number | null
keyLabel?: string | null
hasOverrides?: boolean
+ // The provider reported this model as permanently gone (410 / end of life),
+ // so the gateway disabled it by itself — distinct from a switch the user
+ // flipped off. `retiredReason` is the upstream wording. See #634.
+ retiredUpstream?: boolean
+ retiredReason?: string | null
keyCount: number
// Logical-model grouping (sent by the server when unify is relevant). Absent
// for ungrouped rows; the UI falls back to a per-row "solo" group then.
diff --git a/server/src/__tests__/db/migrate/roundtrip.test.ts b/server/src/__tests__/db/migrate/roundtrip.test.ts
index feb487a0..085cd9dc 100644
--- a/server/src/__tests__/db/migrate/roundtrip.test.ts
+++ b/server/src/__tests__/db/migrate/roundtrip.test.ts
@@ -20,6 +20,7 @@ const MEDIA_MODEL_META_FILENAME = '20260726_000004_media_model_meta.ts';
const REQUEST_SERVED_MODEL_FILENAME = '20260726_000005_request_served_model.ts';
const ATTEMPT_ERROR_SUMMARY_FILENAME = '20260726_000006_attempt_error_summary.ts';
const AGENT_COMPATIBILITY_FILENAME = '20260727_000001_agent_compatibility.ts';
+const TOMBSTONE_PROVENANCE_FILENAME = '20260728_000001_tombstone_provenance.ts';
interface SchemaRow {
type: string;
@@ -86,6 +87,7 @@ describe('migration round trip', () => {
REQUEST_SERVED_MODEL_FILENAME,
ATTEMPT_ERROR_SUMMARY_FILENAME,
AGENT_COMPATIBILITY_FILENAME,
+ TOMBSTONE_PROVENANCE_FILENAME,
]);
} finally {
db.close();
diff --git a/server/src/__tests__/routes/proxy-retry.test.ts b/server/src/__tests__/routes/proxy-retry.test.ts
index 470f556f..d8f18614 100644
--- a/server/src/__tests__/routes/proxy-retry.test.ts
+++ b/server/src/__tests__/routes/proxy-retry.test.ts
@@ -1,6 +1,8 @@
import { describe, it, expect } from 'vitest';
import { isRetryableError, isPaymentRequiredError, isModelNotFoundError, isModelAccessForbiddenError } from '../../routes/proxy.js';
import { isProviderBadRequestError } from '../../lib/error-classify.js';
+import { cooldownDecisionForError } from '../../lib/fallback-loop.js';
+import { MODEL_FORBIDDEN_COOLDOWN_MS } from '../../services/ratelimit.js';
describe('isModelAccessForbiddenError (403 model-not-on-tier, drives whole-model skip — issue #256)', () => {
it('flags a 403 reaching the proxy by message or attached status', () => {
@@ -18,6 +20,59 @@ describe('isModelAccessForbiddenError (403 model-not-on-tier, drives whole-model
expect(isModelAccessForbiddenError(new Error('OpenRouter API error 404: Provider returned error'))).toBe(false);
expect(isModelAccessForbiddenError(new Error('HuggingFace Router API error 402: Payment required'))).toBe(false);
});
+
+ // Issue #618: some providers deny model access with a 400 (or 401) instead of
+ // a 403. Classified transient, those got the 90s bench — and the auto-router
+ // re-picked the same unreachable model the moment it expired, forever, with
+ // an escalating penalty. They must classify as model-inaccessible-for-this-key.
+ it('flags a 400/401 whose body says the key may not access the model (#618)', () => {
+ expect(isModelAccessForbiddenError(Object.assign(
+ new Error('API error 400: user is not allowed to access model kat-coder-pro-v2.5'),
+ { status: 400 },
+ ))).toBe(true);
+ expect(isModelAccessForbiddenError(Object.assign(
+ new Error('API error 400: action plan limited, please upgrade'),
+ { status: 400 },
+ ))).toBe(true);
+ expect(isModelAccessForbiddenError(Object.assign(
+ new Error('API error 401: you do not have access to this model'),
+ { status: 401 },
+ ))).toBe(true);
+ expect(isModelAccessForbiddenError(
+ new Error('user is not allowed to access model kat-coder-pro-v2.5'),
+ )).toBe(true);
+ });
+
+ it('still leaves ordinary 400/401 rejections alone', () => {
+ expect(isModelAccessForbiddenError(Object.assign(new Error('Bad Request'), { status: 400 }))).toBe(false);
+ expect(isModelAccessForbiddenError(Object.assign(new Error('Unauthorized'), { status: 401 }))).toBe(false);
+ expect(isModelAccessForbiddenError(new Error('401 Unauthorized'))).toBe(false);
+ expect(isModelAccessForbiddenError(new Error('400 Bad Request'))).toBe(false);
+ expect(isModelAccessForbiddenError(Object.assign(
+ new Error('Groq API error 400: Failed to call a function. Please adjust your prompt.'),
+ { status: 400 },
+ ))).toBe(false);
+ });
+});
+
+// The loop-level consequence of the classification above: a "not allowed to
+// access" 400 must take the model-forbidden bench (a day, source 'tier'), not
+// the 90s transient one that made the router re-pick it every 90 seconds.
+describe('cooldownDecisionForError on a 400 not-allowed body (#618)', () => {
+ const route = {
+ platform: 'kat', modelId: 'kat-coder-pro-v2.5', modelDbId: 1, keyId: 1,
+ rpdLimit: null, tpdLimit: null,
+ } as any;
+
+ it('benches the model like a 403 instead of the transient ladder', () => {
+ const err = Object.assign(
+ new Error('API error 400: user is not allowed to access model kat-coder-pro-v2.5'),
+ { status: 400 },
+ );
+ const decision = cooldownDecisionForError(route, err);
+ expect(decision.durationMs).toBe(MODEL_FORBIDDEN_COOLDOWN_MS);
+ expect(decision.source).toBe('tier');
+ });
});
describe('isModelNotFoundError (drives whole-model skip within a request)', () => {
diff --git a/server/src/__tests__/services/model-retirement.test.ts b/server/src/__tests__/services/model-retirement.test.ts
new file mode 100644
index 00000000..57eafb40
--- /dev/null
+++ b/server/src/__tests__/services/model-retirement.test.ts
@@ -0,0 +1,223 @@
+import { describe, it, expect, beforeAll, beforeEach } from 'vitest';
+import { initDb, getDb } from '../../db/index.js';
+import { modelRetirementSignal } from '../../lib/error-classify.js';
+import {
+ noteModelRetirementSignal,
+ resetModelRetirementObservations,
+ RETIREMENT_CONFIRMATIONS_REQUIRED,
+} from '../../services/model-retirement.js';
+import { getCatalogModelTombstone, recordCatalogModelTombstone } from '../../services/model-state.js';
+import { applyCatalog } from '../../services/catalog-sync.js';
+
+// #634: NVIDIA retires a model upstream ("has reached its end of life"), and
+// every subsequent request burns a fallback slot on the corpse because nothing
+// persisted the 404/410. These tests lock the two halves of the fix: a
+// conservative end-of-life CLASSIFIER (transient 404s must never match), and a
+// corroboration threshold before the model is actually auto-disabled.
+
+const PLATFORM = 'groq';
+const MODEL_ID = 'eol-test-model';
+
+beforeAll(() => {
+ process.env.ENCRYPTION_KEY = '0'.repeat(64);
+ initDb(':memory:');
+});
+
+function seedModel(modelId = MODEL_ID): number {
+ const db = getDb();
+ db.prepare(`
+ INSERT INTO models (platform, model_id, display_name, intelligence_rank, speed_rank, enabled, source)
+ VALUES (?, ?, 'EOL Test', 50, 50, 1, 'catalog')
+ ON CONFLICT(platform, model_id) DO UPDATE SET enabled = 1
+ `).run(PLATFORM, modelId);
+ const row = db.prepare('SELECT id FROM models WHERE platform = ? AND model_id = ?')
+ .get(PLATFORM, modelId) as { id: number };
+ db.prepare(`
+ INSERT INTO fallback_config (model_db_id, priority, enabled)
+ VALUES (?, 999, 1)
+ ON CONFLICT(model_db_id) DO UPDATE SET enabled = 1
+ `).run(row.id);
+ return row.id;
+}
+
+function isRoutable(modelDbId: number): boolean {
+ const row = getDb()
+ .prepare('SELECT enabled FROM fallback_config WHERE model_db_id = ?')
+ .get(modelDbId) as { enabled: number } | undefined;
+ return row?.enabled === 1;
+}
+
+function routeFor(modelDbId: number, modelId = MODEL_ID) {
+ return { modelDbId, platform: PLATFORM, modelId };
+}
+
+// The exact NVIDIA body from issue #634.
+const NVIDIA_EOL = Object.assign(
+ new Error(
+ "NVIDIA NIM API error 410: The model 'minimaxai/minimax-m2.7' has reached its end of life "
+ + 'on 2026-07-27T00:00:00Z and is no longer available.',
+ ),
+ { status: 410 },
+);
+
+// The transient shape from the same issue thread: a load balancer that cannot
+// find the function for the account right now. Says nothing about the model.
+const TRANSIENT_404 = Object.assign(
+ new Error(
+ "NVIDIA NIM API error 404: Function '23d4f03a-b8a6-4adb-a183-7daa083a09cc': Not found for account 'BDnmPUMI'",
+ ),
+ { status: 404 },
+);
+
+describe('modelRetirementSignal (conservative end-of-life detection — #634)', () => {
+ it('treats an explicit 410 / end-of-life body as definitive', () => {
+ expect(modelRetirementSignal(NVIDIA_EOL)).toBe('definitive');
+ expect(modelRetirementSignal(Object.assign(new Error('Gone'), { status: 410 }))).toBe('definitive');
+ expect(modelRetirementSignal(new Error('Ollama Cloud API error 410: Gone'))).toBe('definitive');
+ // The wording alone is definitive even when the provider picks a 404.
+ expect(modelRetirementSignal(Object.assign(
+ new Error('API error 404: model has reached its end of life'),
+ { status: 404 },
+ ))).toBe('definitive');
+ });
+
+ it('treats a 404 that clearly says the model is gone as probable (needs corroboration)', () => {
+ expect(modelRetirementSignal(Object.assign(
+ new Error('OpenRouter API error 404: this model is no longer available'),
+ { status: 404 },
+ ))).toBe('probable');
+ expect(modelRetirementSignal(Object.assign(
+ new Error('Groq API error 404: model has been removed'),
+ { status: 404 },
+ ))).toBe('probable');
+ });
+
+ it('never fires on transient 404s, empty not-found bodies, or unrelated failures', () => {
+ expect(modelRetirementSignal(TRANSIENT_404)).toBeNull();
+ expect(modelRetirementSignal(new Error('OpenRouter API error 404: Provider returned error'))).toBeNull();
+ expect(modelRetirementSignal(new Error('Model not found'))).toBeNull();
+ // OpenRouter says this while a model is merely unserved right now.
+ expect(modelRetirementSignal(new Error('No endpoints found for openrouter/minimax/minimax-m2.5:free'))).toBeNull();
+ expect(modelRetirementSignal(new Error('429 Too Many Requests'))).toBeNull();
+ expect(modelRetirementSignal(new Error('503 Service Unavailable'))).toBeNull();
+ // A digit run that merely contains 410 is not a 410.
+ expect(modelRetirementSignal(new Error('Groq API error 400: max_tokens 41000 exceeds 32768'))).toBeNull();
+ });
+});
+
+describe('noteModelRetirementSignal (auto-disable with a reason — #634)', () => {
+ beforeEach(() => {
+ resetModelRetirementObservations();
+ getDb().prepare("DELETE FROM catalog_model_tombstones WHERE model_id LIKE 'eol-test-%'").run();
+ });
+
+ it('auto-disables the model on a single explicit 410 end-of-life response', () => {
+ const id = seedModel();
+ expect(noteModelRetirementSignal(routeFor(id), NVIDIA_EOL, {})).toBe(true);
+ expect(isRoutable(id)).toBe(false);
+
+ const tombstone = getCatalogModelTombstone(getDb(), 'chat', PLATFORM, MODEL_ID);
+ expect(tombstone?.source).toBe('upstream_eol');
+ expect(tombstone?.reason).toContain('end of life');
+ });
+
+ it('leaves a transient 404 alone no matter how often it repeats', () => {
+ const id = seedModel();
+ for (let i = 0; i < 5; i++) {
+ expect(noteModelRetirementSignal(routeFor(id), TRANSIENT_404, {})).toBe(false);
+ }
+ expect(isRoutable(id)).toBe(true);
+ expect(getCatalogModelTombstone(getDb(), 'chat', PLATFORM, MODEL_ID)).toBeUndefined();
+ });
+
+ it('requires corroboration from distinct requests before disabling on a probable 404', () => {
+ const id = seedModel();
+ const err = Object.assign(new Error('API error 404: this model is no longer available'), { status: 404 });
+
+ expect(RETIREMENT_CONFIRMATIONS_REQUIRED).toBeGreaterThanOrEqual(2);
+ expect(noteModelRetirementSignal(routeFor(id), err, {})).toBe(false);
+ expect(isRoutable(id)).toBe(true);
+ expect(noteModelRetirementSignal(routeFor(id), err, {})).toBe(true);
+ expect(isRoutable(id)).toBe(false);
+ });
+
+ it('does not let one flaky request corroborate itself across sibling keys', () => {
+ const id = seedModel();
+ const err = Object.assign(new Error('API error 404: this model is no longer available'), { status: 404 });
+ const oneRequest = {};
+
+ for (let i = 0; i < 4; i++) {
+ expect(noteModelRetirementSignal(routeFor(id), err, oneRequest)).toBe(false);
+ }
+ expect(isRoutable(id)).toBe(true);
+ });
+
+ it('never touches user-added models (they are not catalog state)', () => {
+ const db = getDb();
+ db.prepare(`
+ INSERT INTO models (platform, model_id, display_name, intelligence_rank, speed_rank, enabled, source)
+ VALUES ('custom', 'eol-test-custom', 'Custom EOL', 50, 50, 1, 'user')
+ ON CONFLICT(platform, model_id) DO UPDATE SET enabled = 1
+ `).run();
+ const row = db.prepare("SELECT id FROM models WHERE platform = 'custom' AND model_id = 'eol-test-custom'")
+ .get() as { id: number };
+ db.prepare(`
+ INSERT INTO fallback_config (model_db_id, priority, enabled) VALUES (?, 998, 1)
+ ON CONFLICT(model_db_id) DO UPDATE SET enabled = 1
+ `).run(row.id);
+
+ expect(noteModelRetirementSignal(
+ { modelDbId: row.id, platform: 'custom', modelId: 'eol-test-custom' },
+ NVIDIA_EOL,
+ {},
+ )).toBe(false);
+ expect(isRoutable(row.id)).toBe(true);
+ });
+});
+
+describe('catalog sync vs. upstream retirement (#634)', () => {
+ function catalogWith(modelId: string, enabled: boolean) {
+ return {
+ version: '2099.01.01',
+ generatedAt: new Date().toISOString(),
+ tier: 'live' as const,
+ models: [{
+ platform: PLATFORM,
+ modelId,
+ displayName: 'EOL Test',
+ intelligenceRank: 50,
+ speedRank: 50,
+ sizeLabel: 'Medium',
+ limits: { rpm: 30, rpd: 1000, tpm: 6000, tpd: null },
+ monthlyTokenBudget: '~1M',
+ contextWindow: 8192,
+ enabled,
+ supportsVision: false,
+ supportsTools: true,
+ }],
+ quirks: [],
+ } as unknown as Parameters<typeof applyCatalog>[1];
+ }
+
+ it('re-enables an auto-retired model when a later catalog still lists it', () => {
+ resetModelRetirementObservations();
+ const id = seedModel('eol-test-relisted');
+ noteModelRetirementSignal(routeFor(id, 'eol-test-relisted'), NVIDIA_EOL, {});
+ expect(isRoutable(id)).toBe(false);
+
+ applyCatalog(getDb(), catalogWith('eol-test-relisted', true));
+
+ expect(isRoutable(id)).toBe(true);
+ expect(getCatalogModelTombstone(getDb(), 'chat', PLATFORM, 'eol-test-relisted')).toBeUndefined();
+ expect(getDb().prepare('SELECT id FROM models WHERE platform = ? AND model_id = ?')
+ .get(PLATFORM, 'eol-test-relisted')).toBeDefined();
+ });
+
+ it('still deletes models the USER tombstoned (unchanged behavior)', () => {
+ seedModel('eol-test-user-deleted');
+ recordCatalogModelTombstone(getDb(), 'chat', PLATFORM, 'eol-test-user-deleted');
+ applyCatalog(getDb(), catalogWith('eol-test-user-deleted', true));
+ expect(getDb().prepare('SELECT id FROM models WHERE platform = ? AND model_id = ?')
+ .get(PLATFORM, 'eol-test-user-deleted')).toBeUndefined();
+ });
+});
diff --git a/server/src/db/migrate/defaults.ts b/server/src/db/migrate/defaults.ts
index ae69af19..6776f0ec 100644
--- a/server/src/db/migrate/defaults.ts
+++ b/server/src/db/migrate/defaults.ts
@@ -15,6 +15,7 @@ import * as mediaModelMeta from '../migrations/20260726_000004_media_model_meta.
import * as requestServedModel from '../migrations/20260726_000005_request_served_model.js';
import * as attemptErrorSummary from '../migrations/20260726_000006_attempt_error_summary.js';
import * as agentCompatibility from '../migrations/20260727_000001_agent_compatibility.js';
+import * as tombstoneProvenance from '../migrations/20260728_000001_tombstone_provenance.js';
export interface MigrationModule {
up(db: Db): void;
@@ -42,6 +43,7 @@ export const MEDIA_MODEL_META_FILENAME = '20260726_000004_media_model_meta.ts';
export const REQUEST_SERVED_MODEL_FILENAME = '20260726_000005_request_served_model.ts';
export const ATTEMPT_ERROR_SUMMARY_FILENAME = '20260726_000006_attempt_error_summary.ts';
export const AGENT_COMPATIBILITY_FILENAME = '20260727_000001_agent_compatibility.ts';
+export const TOMBSTONE_PROVENANCE_FILENAME = '20260728_000001_tombstone_provenance.ts';
export const DEFAULT_MIGRATIONS: readonly DefaultMigration[] = [
{ filename: LEGACY_BASELINE_FILENAME, module: legacyBaseline },
@@ -60,4 +62,5 @@ export const DEFAULT_MIGRATIONS: readonly DefaultMigration[] = [
{ filename: REQUEST_SERVED_MODEL_FILENAME, module: requestServedModel },
{ filename: ATTEMPT_ERROR_SUMMARY_FILENAME, module: attemptErrorSummary },
{ filename: AGENT_COMPATIBILITY_FILENAME, module: agentCompatibility },
+ { filename: TOMBSTONE_PROVENANCE_FILENAME, module: tombstoneProvenance },
];
diff --git a/server/src/db/migrations/20260728_000001_tombstone_provenance.ts b/server/src/db/migrations/20260728_000001_tombstone_provenance.ts
new file mode 100644
index 00000000..f0966c46
--- /dev/null
+++ b/server/src/db/migrations/20260728_000001_tombstone_provenance.ts
@@ -0,0 +1,41 @@
+// Migration: record WHY a catalog model was tombstoned
+// Created: 2026-07-28
+//
+// DOWN: reversible — drops the two added columns (every existing tombstone is
+// a user deletion, which is the default the columns backfill to).
+
+import type { Db } from '../types.js';
+
+function hasColumn(db: Db, table: string, column: string): boolean {
+ const columns = db.prepare(`PRAGMA table_info(${table})`).all() as { name: string }[];
+ return columns.some((candidate) => candidate.name === column);
+}
+
+/**
+ * Tombstones used to mean exactly one thing: "the user deleted this catalog
+ * model, keep it deleted across syncs". Issue #634 adds a second, machine-made
+ * kind — "the provider retired this model upstream (410 / end of life)" — which
+ * disables the row instead of deleting it and carries the upstream wording so
+ * the dashboard can say why. `source` keeps the two apart ('user' is the
+ * backfill for every pre-existing row); `reason` is the redacted provider text.
+ */
+export function up(db: Db): void {
+ if (!hasColumn(db, 'catalog_model_tombstones', 'source')) {
+ db.prepare("ALTER TABLE catalog_model_tombstones ADD COLUMN source TEXT NOT NULL DEFAULT 'user'").run();
+ }
+ if (!hasColumn(db, 'catalog_model_tombstones', 'reason')) {
+ db.prepare('ALTER TABLE catalog_model_tombstones ADD COLUMN reason TEXT').run();
+ }
+}
+
+export function down(db: Db): void {
+ // Upstream-retirement tombstones have no meaning without their source column,
+ // and leaving them behind would silently delete live models on the next sync.
+ db.prepare("DELETE FROM catalog_model_tombstones WHERE source = 'upstream_eol'").run();
+ if (hasColumn(db, 'catalog_model_tombstones', 'reason')) {
+ db.prepare('ALTER TABLE catalog_model_tombstones DROP COLUMN reason').run();
+ }
+ if (hasColumn(db, 'catalog_model_tombstones', 'source')) {
+ db.prepare('ALTER TABLE catalog_model_tombstones DROP COLUMN source').run();
+ }
+}
diff --git a/server/src/lib/error-classify.ts b/server/src/lib/error-classify.ts
index 3893e5e0..b7bc173b 100644
--- a/server/src/lib/error-classify.ts
+++ b/server/src/lib/error-classify.ts
@@ -329,5 +329,85 @@ export function isModelNotFoundError(err: any): boolean {
export function isModelAccessForbiddenError(err: any): boolean {
if (err?.status === 403) return true;
const msg = (err?.message ?? '').toLowerCase();
- return msg.includes('403') || msg.includes('forbidden');
+ if (msg.includes('403') || msg.includes('forbidden')) return true;
+ // Not every provider spells "this key may not use this model" as a 403 (issue
+ // #618): observed live as a 400 whose body reads "user is not allowed to
+ // access model kat-coder-pro-v2.5", and as plan-gate wordings like "action
+ // plan limited". Classified transient, those took the 90s bench — and the
+ // auto-router re-picked the same permanently-unreachable model the moment it
+ // expired, failing and re-benching forever with an ever-growing penalty.
+ // Deliberately narrow: only 400/401 (or a status-less error), and only for
+ // phrasings that name access/permission to the MODEL. A bare "Bad Request",
+ // a generic "Unauthorized" (which isKeyAuthError owns), or a parameter
+ // rejection must keep their current classification.
+ const status = typeof err?.status === 'number' ? err.status : 0;
+ if (status !== 0 && status !== 400 && status !== 401) return false;
+ return MODEL_ACCESS_DENIED_PHRASES.some(phrase => msg.includes(phrase));
+}
+
+const MODEL_ACCESS_DENIED_PHRASES = [
+ 'not allowed to access',
+ 'not allowed to use',
+ 'not authorized to access',
+ 'not authorized to use',
+ 'unauthorized to access',
+ 'not permitted to access',
+ 'not permitted to use',
+ 'do not have access to',
+ 'does not have access to',
+ 'no access to model',
+ 'no access to this model',
+ 'model access denied',
+ 'access to this model is restricted',
+ // Plan/subscription gates the provider words as a limit rather than a denial.
+ 'action plan limited',
+];
+
+// ── Permanent upstream retirement (issue #634) ───────────────────────────────
+// isModelNotFoundError above rules a 404/410 model out for ONE request. That is
+// right for a load balancer hiccup and wrong for a model the provider RETIRED:
+// NVIDIA's "The model 'minimaxai/minimax-m2.7' has reached its end of life on
+// 2026-07-27T00:00:00Z and is no longer available" is true forever, yet nothing
+// persisted it, so the corpse burned a fallback slot on every later request.
+//
+// This classifier says how much a failure should be TRUSTED as permanent:
+// - 'definitive' — an explicit 410 Gone, or unmistakable end-of-life wording.
+// Acting on one response is safe.
+// - 'probable' — a 404 whose body says the model is gone in so many words.
+// Real, but a provider can also word a transient outage this way, so the
+// caller corroborates across distinct requests before acting.
+// - null — everything else, including every bare 404 ("Not found",
+// "Provider returned error", OpenRouter's "No endpoints found", NVIDIA's
+// per-account "Function '<uuid>': Not found for account '<id>'"). A
+// transient 404 must NEVER retire a healthy model.
+// Note the 410 test is deliberately stricter than isModelNotFoundError's bare
+// `msg.includes('410')`: a digit run inside a token count must not read as Gone.
+export type ModelRetirementConfidence = 'definitive' | 'probable';
+
+const END_OF_LIFE_PHRASES = [
+ 'end of life',
+ 'end-of-life',
+ 'has been retired',
+ 'has been decommissioned',
+ 'has been sunset',
+];
+
+const MODEL_GONE_PHRASES = [
+ 'no longer available',
+ 'no longer offered',
+ 'no longer supported',
+ 'has been removed',
+ 'was removed',
+ 'has been discontinued',
+ 'has been deprecated',
+ 'is deprecated',
+];
+
+export function modelRetirementSignal(err: any): ModelRetirementConfidence | null {
+ const msg = (err?.message ?? '').toLowerCase();
+ const status = typeof err?.status === 'number' ? err.status : 0;
+ const gone = status === 410 || /\berror 410\b/.test(msg) || /\b410 gone\b/.test(msg);
+ if (gone || END_OF_LIFE_PHRASES.some(phrase => msg.includes(phrase))) return 'definitive';
+ if (!isModelNotFoundError(err)) return null;
+ return MODEL_GONE_PHRASES.some(phrase => msg.includes(phrase)) ? 'probable' : null;
}
diff --git a/server/src/lib/fallback-loop.ts b/server/src/lib/fallback-loop.ts
index e9790cb8..91eb3c6d 100644
--- a/server/src/lib/fallback-loop.ts
+++ b/server/src/lib/fallback-loop.ts
@@ -45,9 +45,10 @@ import {
} from './error-classify.js';
import { sanitizeProviderErrorMessage, summarizeAttemptError } from './error-redaction.js';
import { checkKeyHealth, markKeyHealthyFromRequest } from '../services/health.js';
+import { noteModelRetirementSignal } from '../services/model-retirement.js';
import { getSetting } from '../db/index.js';
import { newBreaker, recordBreakerFailure } from './guardrails.js';
-import { newRequestTrace, runWithRequestTrace, type AttemptOutcome, type RequestTrace } from './attempt-trace.js';
+import { getRequestTrace, newRequestTrace, runWithRequestTrace, type AttemptOutcome, type RequestTrace } from './attempt-trace.js';
import { persistRequestAttempts } from './request-log.js';
// Every surface caps failover hops at the same number.
@@ -186,6 +187,12 @@ export function recordRetryableFailure(route: RouteResult, err: any, state: Fall
if (isModelNotFoundError(err) || isModelAccessForbiddenError(err) || isContextTooLargeError(err) || err?.skipModelForRequest === true) {
state.skipModels.add(route.modelDbId);
}
+ // A model-level 404/410 that says the model is GONE (not merely missing right
+ // now) outlives this request: persist it once the evidence is strong enough,
+ // or the retired model burns a fallback slot on every request forever (#634).
+ // The trace object identifies the request, so one request's failover across
+ // sibling keys counts as the single observation it is.
+ noteModelRetirementSignal(route, err, getRequestTrace());
state.skipKeys.add(`${route.platform}:${route.modelId}:${route.keyId}`);
if (err?.skipBench === true) return;
const decision = cooldownDecisionForError(route, err);
diff --git a/server/src/routes/fallback.ts b/server/src/routes/fallback.ts
index 1b6cdfef..59f236d7 100644
--- a/server/src/routes/fallback.ts
+++ b/server/src/routes/fallback.ts
@@ -72,11 +72,14 @@ fallbackRouter.get('/', (_req: Request, res: Response) => {
m.tpm_limit, m.tpd_limit, m.context_window,
m.monthly_token_budget, m.supports_vision, m.supports_tools,
m.key_id, ak.label AS key_label,
- mo.overrides_json IS NOT NULL AS has_overrides
+ mo.overrides_json IS NOT NULL AS has_overrides,
+ ts.source AS tombstone_source, ts.reason AS tombstone_reason
FROM profile_models pm
JOIN models m ON m.id = pm.model_db_id
LEFT JOIN api_keys ak ON ak.id = m.key_id
LEFT JOIN model_overrides mo ON mo.platform = m.platform AND mo.model_id = m.model_id
+ LEFT JOIN catalog_model_tombstones ts
+ ON ts.kind = 'chat' AND ts.platform = m.platform AND ts.model_id = m.model_id
WHERE pm.profile_id = ? AND m.enabled = 1
ORDER BY pm.priority ASC
`).all(activeProfileId) as any[];
@@ -89,11 +92,14 @@ fallbackRouter.get('/', (_req: Request, res: Response) => {
m.tpm_limit, m.tpd_limit, m.context_window,
m.monthly_token_budget, m.supports_vision, m.supports_tools,
m.key_id, ak.label AS key_label,
- mo.overrides_json IS NOT NULL AS has_overrides
+ mo.overrides_json IS NOT NULL AS has_overrides,
+ ts.source AS tombstone_source, ts.reason AS tombstone_reason
FROM fallback_config fc
JOIN models m ON m.id = fc.model_db_id
LEFT JOIN api_keys ak ON ak.id = m.key_id
LEFT JOIN model_overrides mo ON mo.platform = m.platform AND mo.model_id = m.model_id
+ LEFT JOIN catalog_model_tombstones ts
+ ON ts.kind = 'chat' AND ts.platform = m.platform AND ts.model_id = m.model_id
WHERE m.enabled = 1
ORDER BY fc.priority ASC
`).all() as any[];
@@ -160,6 +166,12 @@ fallbackRouter.get('/', (_req: Request, res: Response) => {
keyId: r.key_id ?? null,
keyLabel: r.key_label ?? null,
hasOverrides: Boolean(r.has_overrides),
+ // Why the switch is off: a model auto-disabled because the provider
+ // retired it upstream (410 / end of life, #634) is a different state from
+ // one the user turned off, and the dashboard says so. The reason is the
+ // provider's own (redacted) wording.
+ retiredUpstream: r.tombstone_source === 'upstream_eol',
+ retiredReason: r.tombstone_source === 'upstream_eol' ? (r.tombstone_reason ?? null) : null,
keyCount: keyCountMap.get(r.platform) ?? 0,
};
}));
diff --git a/server/src/services/catalog-sync.ts b/server/src/services/catalog-sync.ts
index db3c9587..a677b206 100644
--- a/server/src/services/catalog-sync.ts
+++ b/server/src/services/catalog-sync.ts
@@ -11,6 +11,7 @@ import {
applyModelOverrides,
deleteTombstonedCatalogModels,
isCatalogModelTombstoned,
+ reinstateUpstreamRetiredCatalogModel,
} from './model-state.js';
import { ensureAllModelsInProfiles } from './profile-models.js';
@@ -230,7 +231,9 @@ function routableContextWindow(platform: string, modelId: string, contextWindow:
* declarative config, admin adds) are never updated, never deleted, and
* never adopted — on a platform:model_id collision the user row wins and
* the catalog entry is skipped outright;
- * - catalog models the user deleted stay deleted via tombstones;
+ * - catalog models the user deleted stay deleted via tombstones, while models
+ * auto-retired from an upstream 410/end-of-life response (#634) are only
+ * disabled — a catalog that still lists them lifts the retirement;
* - models that vanished from the catalog are deleted, exactly like the
* dead-model migrations do (fallback_config row first, FK order).
*/
@@ -341,6 +344,10 @@ export function applyCatalog(db: Db, catalog: Catalog): NonNullable<SyncResult['
continue;
}
if (isCatalogModelTombstoned(db, 'chat', m.platform, m.modelId)) continue;
+ // A model auto-retired from a 410/end-of-life response (#634) is disabled,
+ // not deleted. A catalog that STILL lists it — and lists it enabled — is
+ // newer evidence than that one provider response, so lift the retirement.
+ if (m.enabled) reinstateUpstreamRetiredCatalogModel(db, m.platform, m.modelId);
inCatalog.add(`${m.platform}:${m.modelId}`);
const row = selectModel.get(m.platform, m.modelId) as
diff --git a/server/src/services/model-retirement.ts b/server/src/services/model-retirement.ts
new file mode 100644
index 00000000..d1e2dfca
--- /dev/null
+++ b/server/src/services/model-retirement.ts
@@ -0,0 +1,105 @@
+// Auto-retirement of models the upstream provider has permanently removed.
+//
+// Issue #634: NVIDIA answered every request for `minimaxai/minimax-m2.7` with
+// "410 … has reached its end of life … and is no longer available". The failure
+// classifies as a model-level skip, which rules the model out for ONE request —
+// so the next request tried it again, and the next, forever, each one paying a
+// round trip and a fallback slot for a model that is never coming back.
+//
+// The fix persists the verdict: a definitive end-of-life response disables the
+// model right away, and a merely-probable one waits for corroboration from a
+// second distinct request first, so a single flaky 404 from a load balancer can
+// never kill a healthy model. The disable is reversible in both directions —
+// the user can switch the model back on, and a catalog refresh that still lists
+// it lifts the retirement (services/catalog-sync.ts).
+
+import { getDb } from '../db/index.js';
+import { modelRetirementSignal } from '../lib/error-classify.js';
+import { summarizeAttemptError } from '../lib/error-redaction.js';
+import { isCatalogManagedModel, retireCatalogModelUpstream } from './model-state.js';
+
+/** How many DISTINCT requests must report a 'probable' signal before acting. */
+export const RETIREMENT_CONFIRMATIONS_REQUIRED = 2;
+
+/**
+ * Corroboration has to be recent. Two "no longer available" 404s a week apart
+ * are two outages; two within the hour are a retirement.
+ */
+export const RETIREMENT_OBSERVATION_WINDOW_MS = 60 * 60 * 1000;
+
+interface Observation {
+ count: number;
+ firstSeenAt: number;
+ // Identity of the request that last counted, so the same request failing over
+ // across sibling keys cannot corroborate itself.
+ lastRequest: unknown;
+}
+
+const observations = new Map<string, Observation>();
+
+/** Test seam: the counters are process-local and deliberately not persisted. */
+export function resetModelRetirementObservations(): void {
+ observations.clear();
+}
+
+export interface RetirementRoute {
+ modelDbId: number;
+ platform: string;
+ modelId: string;
+}
+
+/**
+ * Record one upstream failure against the retirement heuristic and, when the
+ * evidence is strong enough, auto-disable the model. Returns true iff this call
+ * retired it. Never throws: it runs on the proxy's failure path, where a DB
+ * hiccup must not turn a failover into a crash.
+ */
+export function noteModelRetirementSignal(
+ route: RetirementRoute,
+ err: unknown,
+ requestToken?: unknown,
+): boolean {
+ const confidence = modelRetirementSignal(err);
+ if (!confidence) return false;
+
+ const key = `${route.platform}:${route.modelId}`;
+ if (confidence === 'probable' && !confirmObservation(key, requestToken)) return false;
+
+ const reason = summarizeAttemptError((err as { message?: unknown })?.message);
+ try {
+ const db = getDb();
+ const row = db
+ .prepare('SELECT id, platform, model_id, key_id, source FROM models WHERE id = ?')
+ .get(route.modelDbId) as
+ | { id: number; platform: string; model_id: string; key_id: number | null; source: string }
+ | undefined;
+ // Only catalog-managed rows: a model the user added by hand is their state,
+ // not the catalog's, and the tombstone table has no meaning for it.
+ if (!row || !isCatalogManagedModel(row)) return false;
+ if (!retireCatalogModelUpstream(db, row.id, row.platform, row.model_id, reason)) return false;
+ observations.delete(key);
+ console.log(
+ `[ModelRetirement] ${row.platform}/${row.model_id} disabled — upstream reports it retired: ${reason}`,
+ );
+ return true;
+ } catch (dbErr: any) {
+ console.warn(`[ModelRetirement] could not record retirement for ${key}: ${dbErr?.message ?? dbErr}`);
+ return false;
+ }
+}
+
+/** True once this model has been reported gone by enough distinct requests. */
+function confirmObservation(key: string, requestToken: unknown): boolean {
+ const now = Date.now();
+ const existing = observations.get(key);
+ if (!existing || now - existing.firstSeenAt > RETIREMENT_OBSERVATION_WINDOW_MS) {
+ observations.set(key, { count: 1, firstSeenAt: now, lastRequest: requestToken });
+ return RETIREMENT_CONFIRMATIONS_REQUIRED <= 1;
+ }
+ // Same request failing over across this model's sibling keys: one request's
+ // worth of evidence, however many keys it burned.
+ if (requestToken !== undefined && existing.lastRequest === requestToken) return false;
+ existing.count++;
+ existing.lastRequest = requestToken;
+ return existing.count >= RETIREMENT_CONFIRMATIONS_REQUIRED;
+}
diff --git a/server/src/services/model-state.ts b/server/src/services/model-state.ts
index 360af803..cb2d8829 100644
--- a/server/src/services/model-state.ts
+++ b/server/src/services/model-state.ts
@@ -69,15 +69,52 @@ export function isCatalogManagedModel(row: { platform: string; key_id?: number |
return row.platform !== 'custom' && row.key_id == null;
}
+// Why a catalog model carries a tombstone:
+// 'user' — deleted in the dashboard. Stays deleted across syncs, and
+// the row is removed from `models` on the next catalog apply.
+// 'upstream_eol' — the provider reported it permanently gone (410 / end of
+// life, issue #634). The row SURVIVES and is only disabled,
+// so the dashboard can show "retired upstream" instead of
+// silently losing the model, and so a later catalog that
+// still lists it can lift the retirement.
+export type CatalogTombstoneSource = 'user' | 'upstream_eol';
+
+export interface CatalogModelTombstone {
+ source: CatalogTombstoneSource;
+ reason: string | null;
+ createdAt: string;
+}
+
+export function getCatalogModelTombstone(
+ db: Db,
+ kind: CatalogModelKind,
+ platform: string,
+ modelId: string,
+): CatalogModelTombstone | undefined {
+ const row = db
+ .prepare('SELECT source, reason, created_at FROM catalog_model_tombstones WHERE kind = ? AND platform = ? AND model_id = ?')
+ .get(kind, platform, modelId) as { source: string; reason: string | null; created_at: string } | undefined;
+ if (!row) return undefined;
+ return {
+ source: row.source === 'upstream_eol' ? 'upstream_eol' : 'user',
+ reason: row.reason ?? null,
+ createdAt: row.created_at,
+ };
+}
+
+/**
+ * True only for models the USER deleted — the "keep it deleted" contract every
+ * caller here means. An upstream-retirement tombstone deliberately does NOT
+ * count: those models stay in the catalog's write path so a refreshed catalog
+ * can reinstate them (see reinstateUpstreamRetiredCatalogModel).
+ */
export function isCatalogModelTombstoned(
db: Db,
kind: CatalogModelKind,
platform: string,
modelId: string,
): boolean {
- return !!db
- .prepare('SELECT 1 FROM catalog_model_tombstones WHERE kind = ? AND platform = ? AND model_id = ?')
- .get(kind, platform, modelId);
+ return getCatalogModelTombstone(db, kind, platform, modelId)?.source === 'user';
}
export function recordCatalogModelTombstone(
@@ -85,17 +122,71 @@ export function recordCatalogModelTombstone(
kind: CatalogModelKind,
platform: string,
modelId: string,
+ options: { source?: CatalogTombstoneSource; reason?: string | null } = {},
): void {
+ const source: CatalogTombstoneSource = options.source ?? 'user';
db.prepare(`
- INSERT INTO catalog_model_tombstones (kind, platform, model_id)
- VALUES (?, ?, ?)
- ON CONFLICT(kind, platform, model_id) DO UPDATE SET created_at = datetime('now')
- `).run(kind, platform, modelId);
- if (kind === 'chat') {
+ INSERT INTO catalog_model_tombstones (kind, platform, model_id, source, reason)
+ VALUES (?, ?, ?, ?, ?)
+ ON CONFLICT(kind, platform, model_id)
+ DO UPDATE SET created_at = datetime('now'), source = excluded.source, reason = excluded.reason
+ `).run(kind, platform, modelId, source, options.reason ?? null);
+ // A user deletion drops their local metadata edits with the row. An upstream
+ // retirement keeps the row, so it keeps the overrides too — they must survive
+ // if the model is reinstated.
+ if (kind === 'chat' && source === 'user') {
db.prepare('DELETE FROM model_overrides WHERE platform = ? AND model_id = ?').run(platform, modelId);
}
}
+/**
+ * Auto-disable a catalog model the provider reports as permanently retired
+ * (issue #634). Deliberately NOT a delete: the row stays visible in the
+ * dashboard, tagged with the upstream wording, and the user can flip it back on
+ * if they disagree. Turning off the chain entries (and the active profile's
+ * copy) is exactly what the dashboard's own switch does, so the router stops
+ * picking it while an explicitly-requested model id still resolves.
+ *
+ * Returns true when this call performed the retirement (false when it was
+ * already retired, or the user had deleted the model outright).
+ */
+export function retireCatalogModelUpstream(
+ db: Db,
+ modelDbId: number,
+ platform: string,
+ modelId: string,
+ reason: string,
+): boolean {
+ const existing = getCatalogModelTombstone(db, 'chat', platform, modelId);
+ if (existing) return false;
+ recordCatalogModelTombstone(db, 'chat', platform, modelId, { source: 'upstream_eol', reason });
+ db.prepare('UPDATE fallback_config SET enabled = 0 WHERE model_db_id = ?').run(modelDbId);
+ db.prepare('UPDATE profile_models SET enabled = 0 WHERE model_db_id = ?').run(modelDbId);
+ return true;
+}
+
+/**
+ * Lift an upstream retirement: a catalog that still lists the model — and lists
+ * it enabled — is newer and better evidence than one provider's 404. Returns
+ * true when a retirement was actually lifted.
+ */
+export function reinstateUpstreamRetiredCatalogModel(
+ db: Db,
+ platform: string,
+ modelId: string,
+): boolean {
+ if (getCatalogModelTombstone(db, 'chat', platform, modelId)?.source !== 'upstream_eol') return false;
+ clearCatalogModelTombstone(db, 'chat', platform, modelId);
+ const row = db
+ .prepare('SELECT id FROM models WHERE platform = ? AND model_id = ?')
+ .get(platform, modelId) as { id: number } | undefined;
+ if (row) {
+ db.prepare('UPDATE fallback_config SET enabled = 1 WHERE model_db_id = ?').run(row.id);
+ db.prepare('UPDATE profile_models SET enabled = 1 WHERE model_db_id = ?').run(row.id);
+ }
+ return true;
+}
+
export function clearCatalogModelTombstone(
db: Db,
kind: CatalogModelKind,
@@ -191,19 +282,23 @@ export function applyAllModelOverrides(db: Db): number {
return applied;
}
+// Only USER tombstones delete rows. An upstream-retirement tombstone disables
+// its model and keeps it (see retireCatalogModelUpstream), so deleting here
+// would throw away both the row and the reason the dashboard shows for it.
export function deleteTombstonedCatalogModels(db: Db): number {
const chatRows = db.prepare(`
SELECT m.id, m.platform, m.model_id
FROM models m
JOIN catalog_model_tombstones t
ON t.kind = 'chat' AND t.platform = m.platform AND t.model_id = m.model_id
- WHERE m.platform != 'custom' AND m.key_id IS NULL AND m.source != 'user'
+ WHERE t.source = 'user' AND m.platform != 'custom' AND m.key_id IS NULL AND m.source != 'user'
`).all() as { id: number; platform: string; model_id: string }[];
const mediaRows = db.prepare(`
SELECT mm.id
FROM media_models mm
JOIN catalog_model_tombstones t
ON t.kind = 'media' AND t.platform = mm.platform AND t.model_id = mm.model_id
+ WHERE t.source = 'user'
`).all() as { id: number }[];
const deleteChatFallback = db.prepare('DELETE FROM fallback_config WHERE model_db_id = ?');