7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.
TypeScript
fix(router): retire models the provider killed, and stop looping on 400 "not allowed" (#647)
Tashfeencf102e2f5f6c
diff --git a/client/src/components/model-table.tsx b/client/src/components/model-table.tsxindex 00cb5649..a42a4d21 100644--- a/client/src/components/model-table.tsx+++ b/client/src/components/model-table.tsx@@ -160,6 +160,14 @@ export function RowContent({{t('models.tools')}</span>)}+ {row.retiredUpstream && (+ <span+ title={row.retiredReason ?? undefined}+ className="text-[10px] rounded-full px-1.5 py-0.5 bg-rose-600/15 text-rose-700 dark:bg-rose-400/15 dark:text-rose-400"+ >+ {t('models.retired')}+ </span>+ )}{(row.penalty ?? 0) > 0 && (<span className="text-[10px] text-amber-600 dark:text-amber-400">{t('models.penalty', { value: row.penalty })}</span>)}diff --git a/client/src/i18n/locales/am.json b/client/src/i18n/locales/am.jsonindex cde2ac39..ac7b2922 100644--- a/client/src/i18n/locales/am.json+++ b/client/src/i18n/locales/am.json@@ -122,6 +122,7 @@"visionTitle": "የምስል ግቤትን ይቀበላል","tools": "መሣሪያዎች","toolsTitle": "የተዋቀሩ የመሳሪያ ጥሪዎችን ያወጣል፣ ስለዚህ ለመሳሪያ ተሸካሚ ጥያቄዎች ብቁ ነው",+ "retired": "ተቋርጧል","penalty": "- {value} ቅጣት","obs": "{count} obs","tokPerMonth": "{count} ቶክ / ሞ",diff --git a/client/src/i18n/locales/ar.json b/client/src/i18n/locales/ar.jsonindex dc52c4c7..bd3b690f 100644--- a/client/src/i18n/locales/ar.json+++ b/client/src/i18n/locales/ar.json@@ -122,6 +122,7 @@"visionTitle": "يقبل إدخال الصورة","tools": "أدوات","toolsTitle": "يصدر استدعاءات أداة منظمة، لذا فهو مؤهل لطلبات حاملة الأداة",+ "retired": "متوقف","penalty": "−{value} عقوبة","obs": "{count} ملاحظة","tokPerMonth": "{count} توك/مو",diff --git a/client/src/i18n/locales/az.json b/client/src/i18n/locales/az.jsonindex ab85d7dc..5c4eb844 100644--- a/client/src/i18n/locales/az.json+++ b/client/src/i18n/locales/az.json@@ -122,6 +122,7 @@"visionTitle": "Şəkil girişini qəbul edir","tools": "Alətlər","toolsTitle": "Strukturlaşdırılmış alət çağırışları yayır, buna görə də alət daşıma sorğuları üçün uyğundur",+ "retired": "Dayandırılıb","penalty": "−{value} penalti","obs": "{count}","tokPerMonth": "{count} tok/mo",diff --git a/client/src/i18n/locales/bg.json b/client/src/i18n/locales/bg.jsonindex d0a0b11d..8935e307 100644--- a/client/src/i18n/locales/bg.json+++ b/client/src/i18n/locales/bg.json@@ -122,6 +122,7 @@"visionTitle": "Приема въвеждане на изображения","tools": "Инструменти","toolsTitle": "Изпуска структурирани повиквания на инструменти, така че е допустим за заявки, носещи инструменти,",+ "retired": "Изтеглен","penalty": "−{value} наказание","obs": "{count} OBS","tokPerMonth": "{count} ток/мо",diff --git a/client/src/i18n/locales/bn.json b/client/src/i18n/locales/bn.jsonindex 1ce3c993..1a797690 100644--- a/client/src/i18n/locales/bn.json+++ b/client/src/i18n/locales/bn.json@@ -122,6 +122,7 @@"visionTitle": "ইমেজ ইনপুট গ্রহণ করে","tools": "টুলস","toolsTitle": "স্ট্রাকচার্ড টুল কল নির্গত করে, তাই এটি টুল-বেয়ারিং অনুরোধের জন্য যোগ্য",+ "retired": "প্রত্যাহৃত","penalty": "−{value} পেনাল্টি","obs": "{count} obs","tokPerMonth": "{count} টোক/মাস",diff --git a/client/src/i18n/locales/cs.json b/client/src/i18n/locales/cs.jsonindex 1e9d745b..6e9810fa 100644--- a/client/src/i18n/locales/cs.json+++ b/client/src/i18n/locales/cs.json@@ -122,6 +122,7 @@"visionTitle": "Přijímá vstup obrazu","tools": "Nástroje","toolsTitle": "Vysílá strukturované volání nástrojů, takže je způsobilý pro požadavky na nosiče nástroje",+ "retired": "Vyřazeno","penalty": "−{value} trest","obs": "{count} OBS","tokPerMonth": "{count} tok/měsíc",diff --git a/client/src/i18n/locales/da.json b/client/src/i18n/locales/da.jsonindex 58e808dd..ee207dc1 100644--- a/client/src/i18n/locales/da.json+++ b/client/src/i18n/locales/da.json@@ -122,6 +122,7 @@"visionTitle": "Accepterer billedinput","tools": "Værktøjer","toolsTitle": "Udsender strukturerede værktøjskald, så den er berettiget til værktøjsbærende anmodninger",+ "retired": "Udgået","penalty": "−{value} straf","obs": "{count} observationer","tokPerMonth": "{count} tok/mo",diff --git a/client/src/i18n/locales/de.json b/client/src/i18n/locales/de.jsonindex e1c1246b..ef860778 100644--- a/client/src/i18n/locales/de.json+++ b/client/src/i18n/locales/de.json@@ -122,6 +122,7 @@"visionTitle": "Akzeptiert Bildeingaben","tools": "Werkzeuge","toolsTitle": "Gibt strukturierte Werkzeugaufrufe aus und ist daher für werkzeugtragende Anforderungen geeignet",+ "retired": "Eingestellt","penalty": "−{value} Strafe","obs": "{count} obs","tokPerMonth": "{count} Tok/Mo",diff --git a/client/src/i18n/locales/el.json b/client/src/i18n/locales/el.jsonindex 71eadbe1..6f20d44d 100644--- a/client/src/i18n/locales/el.json+++ b/client/src/i18n/locales/el.json@@ -122,6 +122,7 @@"visionTitle": "Δέχεται εισαγωγή εικόνας","tools": "Εργαλεία","toolsTitle": "Εκπέμπει δομημένες κλήσεις εργαλείων, ώστε να είναι επιλέξιμο για αιτήματα που φέρουν εργαλεία",+ "retired": "Αποσύρθηκε","penalty": "−{value} ποινή","obs": "{count} obs","tokPerMonth": "{count} tok/μήνα",diff --git a/client/src/i18n/locales/en.json b/client/src/i18n/locales/en.jsonindex 99acfc40..eacc81b7 100644--- a/client/src/i18n/locales/en.json+++ b/client/src/i18n/locales/en.json@@ -122,6 +122,7 @@"visionTitle": "Accepts image input","tools": "Tools","toolsTitle": "Emits structured tool calls, so it is eligible for tool-bearing requests",+ "retired": "Retired","penalty": "−{value} penalty","obs": "{count} obs","tokPerMonth": "{count} tok/mo",diff --git a/client/src/i18n/locales/es.json b/client/src/i18n/locales/es.jsonindex 7f68f4a6..5c0f87e1 100644--- a/client/src/i18n/locales/es.json+++ b/client/src/i18n/locales/es.json@@ -122,6 +122,7 @@"visionTitle": "Acepta imágenes como entrada","tools": "Herramientas","toolsTitle": "Emite llamadas a herramientas estructuradas, por lo que es apto para solicitudes con herramientas",+ "retired": "Retirado","penalty": "penalización de −{value}","obs": "{count} obs","tokPerMonth": "{count} tok/mes",diff --git a/client/src/i18n/locales/fa.json b/client/src/i18n/locales/fa.jsonindex 2c759400..7f82bc9d 100644--- a/client/src/i18n/locales/fa.json+++ b/client/src/i18n/locales/fa.json@@ -122,6 +122,7 @@"visionTitle": "ورودی تصویر را می پذیرد","tools": "ابزار","toolsTitle": "فراخوانی ابزار ساختاریافته را منتشر می کند، بنابراین برای درخواست های حاوی ابزار واجد شرایط است",+ "retired": "متوقف","penalty": "- جریمه {value}","obs": "{count} obs","tokPerMonth": "{count} توک/ماه",diff --git a/client/src/i18n/locales/fi.json b/client/src/i18n/locales/fi.jsonindex 5f9cdb16..efedfcb3 100644--- a/client/src/i18n/locales/fi.json+++ b/client/src/i18n/locales/fi.json@@ -122,6 +122,7 @@"visionTitle": "Hyväksyy kuvasyötteen","tools": "Työkalut","toolsTitle": "Lähettää rakenteellisia työkalukutsuja, joten se on kelpoinen työkalun kantamiseen liittyviin pyyntöihin",+ "retired": "Poistettu","penalty": "−{value} rangaistus","obs": "{count} Obs","tokPerMonth": "{count} tok/mo",diff --git a/client/src/i18n/locales/fr.json b/client/src/i18n/locales/fr.jsonindex ddd1e4da..7278f149 100644--- a/client/src/i18n/locales/fr.json+++ b/client/src/i18n/locales/fr.json@@ -122,6 +122,7 @@"visionTitle": "Accepte les images en entrée","tools": "Outils","toolsTitle": "Émet des appels d'outils structurés, il est donc éligible aux requêtes avec outils",+ "retired": "Retiré","penalty": "pénalité de −{value}","obs": "{count} obs","tokPerMonth": "{count} tok/mois",diff --git a/client/src/i18n/locales/gu.json b/client/src/i18n/locales/gu.jsonindex c291dc3b..1c206f40 100644--- a/client/src/i18n/locales/gu.json+++ b/client/src/i18n/locales/gu.json@@ -122,6 +122,7 @@"visionTitle": "ઇમેજ ઇનપુટ સ્વીકારે છે","tools": "સાધનો","toolsTitle": "સ્ટ્રક્ચર્ડ ટૂલ કૉલ્સ બહાર કાઢે છે, તેથી તે ટૂલ-બેરિંગ વિનંતીઓ માટે પાત્ર છે",+ "retired": "બંધ કરાયું","penalty": "−{value} દંડ","obs": "{count} obs","tokPerMonth": "{count} tok/mo",diff --git a/client/src/i18n/locales/ha.json b/client/src/i18n/locales/ha.jsonindex 2cd87708..0e92ab2c 100644--- a/client/src/i18n/locales/ha.json+++ b/client/src/i18n/locales/ha.json@@ -122,6 +122,7 @@"visionTitle": "Yana karɓar shigarwar hoto","tools": "Kayan aiki","toolsTitle": "Yana fitar da tsarin kiran kayan aiki, don haka ya cancanci buƙatun ɗaukar kayan aiki",+ "retired": "An dakatar","penalty": "-{value} hukunci","obs": "Saukewa: {count}","tokPerMonth": "{count} tok/mo",diff --git a/client/src/i18n/locales/he.json b/client/src/i18n/locales/he.jsonindex 7e920dd7..57cad875 100644--- a/client/src/i18n/locales/he.json+++ b/client/src/i18n/locales/he.json@@ -122,6 +122,7 @@"visionTitle": "מקבל קלט תמונה","tools": "כלים","toolsTitle": "משדר קריאות לכלים מובנים, ולכן הוא זכאי לבקשות עם כלים",+ "retired": "הוסר","penalty": "−{value} עונש","obs": "{count} obs","tokPerMonth": "{count} טוק/מו",diff --git a/client/src/i18n/locales/hi.json b/client/src/i18n/locales/hi.jsonindex 23d42ef6..c37f260e 100644--- a/client/src/i18n/locales/hi.json+++ b/client/src/i18n/locales/hi.json@@ -122,6 +122,7 @@"visionTitle": "छवि इनपुट स्वीकार करता है","tools": "उपकरण","toolsTitle": "संरचित टूल कॉल उत्सर्जित करता है, इसलिए यह टूल-बेयरिंग अनुरोधों के लिए पात्र है",+ "retired": "बंद","penalty": "−{value} जुर्माना","obs": "{count} अवलोकन","tokPerMonth": "{count} tok/mo",diff --git a/client/src/i18n/locales/hr.json b/client/src/i18n/locales/hr.jsonindex 71f25ad7..e4de72f2 100644--- a/client/src/i18n/locales/hr.json+++ b/client/src/i18n/locales/hr.json@@ -122,6 +122,7 @@"visionTitle": "Prihvaća ulaz slike","tools": "Alati","toolsTitle": "Emitira strukturirane pozive alata, pa je prihvatljiv za zahtjeve s nosačem alata",+ "retired": "Povučen","penalty": "−{value} kazna","obs": "{count} OBS","tokPerMonth": "{count} tok/mo",diff --git a/client/src/i18n/locales/hu.json b/client/src/i18n/locales/hu.jsonindex d1423aea..258142c5 100644--- a/client/src/i18n/locales/hu.json+++ b/client/src/i18n/locales/hu.json@@ -122,6 +122,7 @@"visionTitle": "Képbemenetet fogad","tools": "Eszközök","toolsTitle": "Strukturált eszközhívásokat bocsát ki, így jogosult szerszámtartó kérésekre",+ "retired": "Kivonva","penalty": "−{value} büntető","obs": "{count} obs","tokPerMonth": "{count} tok/mo",diff --git a/client/src/i18n/locales/id.json b/client/src/i18n/locales/id.jsonindex d9f136d9..7029a8c3 100644--- a/client/src/i18n/locales/id.json+++ b/client/src/i18n/locales/id.json@@ -122,6 +122,7 @@"visionTitle": "Menerima masukan gambar","tools": "Alat","toolsTitle": "Memancarkan panggilan alat terstruktur, sehingga memenuhi syarat untuk permintaan bantalan alat",+ "retired": "Dihentikan","penalty": "−Penalti {value}","obs": "{count} obs","tokPerMonth": "{count} tok/bln",diff --git a/client/src/i18n/locales/ig.json b/client/src/i18n/locales/ig.jsonindex ab032f19..934786f4 100644--- a/client/src/i18n/locales/ig.json+++ b/client/src/i18n/locales/ig.json@@ -122,6 +122,7 @@"visionTitle": "Nabata ntinye onyonyo","tools": "Ngwa","toolsTitle": "Ọ na-ewepụta oku akụrụngwa ahaziri ahazi, yabụ na ọ tozuru maka arịrịọ ibu ngwá ọrụ",+ "retired": "Akwụsịla","penalty": "-{value} ntaramahụhụ","obs": "{count} obs","tokPerMonth": "{count} tok/mo",diff --git a/client/src/i18n/locales/it.json b/client/src/i18n/locales/it.jsonindex a020d240..825dfe6c 100644--- a/client/src/i18n/locales/it.json+++ b/client/src/i18n/locales/it.json@@ -122,6 +122,7 @@"visionTitle": "Accetta input di immagini","tools": "Strumenti","toolsTitle": "Emette chiamate a strumenti strutturate, quindi è idoneo per richieste con strumenti",+ "retired": "Ritirato","penalty": "−{value} penalità","obs": "{count} oss","tokPerMonth": "{count} tok/mese",diff --git a/client/src/i18n/locales/ja.json b/client/src/i18n/locales/ja.jsonindex 815f45ea..5cda7dc9 100644--- a/client/src/i18n/locales/ja.json+++ b/client/src/i18n/locales/ja.json@@ -122,6 +122,7 @@"visionTitle": "画像入力を受け付ける","tools": "ツール","toolsTitle": "構造化されたツール呼び出しを発行するため、ツールを含むリクエストに適格です",+ "retired": "提供終了","penalty": "−{value}ペナルティ","obs": "{count} オブス","tokPerMonth": "{count} トク/月",diff --git a/client/src/i18n/locales/ka.json b/client/src/i18n/locales/ka.jsonindex 834bf129..c7d33207 100644--- a/client/src/i18n/locales/ka.json+++ b/client/src/i18n/locales/ka.json@@ -122,6 +122,7 @@"visionTitle": "იღებს სურათის შეყვანას","tools": "ხელსაწყოები","toolsTitle": "ავრცელებს სტრუქტურირებულ ხელსაწყოების ზარებს, ამიტომ ის დასაშვებია ხელსაწყოების შემცველი მოთხოვნებისთვის",+ "retired": "შეწყვეტილია","penalty": "−{value} ჯარიმა","obs": "{count} obs","tokPerMonth": "{count} ტოკ/თვე",diff --git a/client/src/i18n/locales/km.json b/client/src/i18n/locales/km.jsonindex 058667b8..df2f8958 100644--- a/client/src/i18n/locales/km.json+++ b/client/src/i18n/locales/km.json@@ -122,6 +122,7 @@"visionTitle": "ទទួលបញ្ចូលរូបភាព","tools": "ឧបករណ៍","toolsTitle": "បញ្ចេញការហៅឧបករណ៍ដែលមានរចនាសម្ព័ន្ធ ដូច្នេះវាមានសិទ្ធិទទួលសំណើដែលមានឧបករណ៍",+ "retired": "បញ្ឈប់","penalty": "−{value} ពិន័យ","obs": "{count}","tokPerMonth": "{count} តុក/ខែ",diff --git a/client/src/i18n/locales/kn.json b/client/src/i18n/locales/kn.jsonindex 1d81e1d1..30045a57 100644--- a/client/src/i18n/locales/kn.json+++ b/client/src/i18n/locales/kn.json@@ -122,6 +122,7 @@"visionTitle": "ಚಿತ್ರದ ಇನ್ಪುಟ್ ಅನ್ನು ಸ್ವೀಕರಿಸುತ್ತದೆ","tools": "ಪರಿಕರಗಳು","toolsTitle": "ರಚನಾತ್ಮಕ ಟೂಲ್ ಕರೆಗಳನ್ನು ಹೊರಸೂಸುತ್ತದೆ, ಆದ್ದರಿಂದ ಇದು ಟೂಲ್-ಬೇರಿಂಗ್ ವಿನಂತಿಗಳಿಗೆ ಅರ್ಹವಾಗಿದೆ",+ "retired": "ನಿಲ್ಲಿಸಲಾಗಿದೆ","penalty": "−{value} ಪೆನಾಲ್ಟಿ","obs": "{count} obs","tokPerMonth": "{count} tok/mo",diff --git a/client/src/i18n/locales/ko.json b/client/src/i18n/locales/ko.jsonindex e1dd7c1e..d1ccec1d 100644--- a/client/src/i18n/locales/ko.json+++ b/client/src/i18n/locales/ko.json@@ -122,6 +122,7 @@"visionTitle": "이미지 입력을 허용합니다","tools": "도구","toolsTitle": "구조화된 도구 호출을 내보내므로 도구 베어링 요청에 적합합니다.",+ "retired": "지원 종료","penalty": "−{value} 페널티","obs": "{count} 관측","tokPerMonth": "{count} 톡/월",diff --git a/client/src/i18n/locales/lt.json b/client/src/i18n/locales/lt.jsonindex d5b2e63b..c8c5e286 100644--- a/client/src/i18n/locales/lt.json+++ b/client/src/i18n/locales/lt.json@@ -122,6 +122,7 @@"visionTitle": "Priima vaizdo įvestį","tools": "Įrankiai","toolsTitle": "Skleidžia struktūrizuotus įrankių kvietimus, todėl yra tinkamas įrankių nešimo užklausoms",+ "retired": "Pašalintas","penalty": "−{value} bauda","obs": "{count} obs","tokPerMonth": "{count} tok/mo",diff --git a/client/src/i18n/locales/ml.json b/client/src/i18n/locales/ml.jsonindex 5c890149..c499ad51 100644--- a/client/src/i18n/locales/ml.json+++ b/client/src/i18n/locales/ml.json@@ -122,6 +122,7 @@"visionTitle": "ഇമേജ് ഇൻപുട്ട് സ്വീകരിക്കുന്നു","tools": "ഉപകരണങ്ങൾ","toolsTitle": "ഘടനാപരമായ ടൂൾ കോളുകൾ പുറപ്പെടുവിക്കുന്നു, അതിനാൽ ടൂൾ-ബെയറിംഗ് അഭ്യർത്ഥനകൾക്ക് ഇത് യോഗ്യമാണ്",+ "retired": "നിർത്തി","penalty": "−{value} പിഴ","obs": "{count} ഒബ്സ്","tokPerMonth": "{count} ടോക്ക്/മാസം",diff --git a/client/src/i18n/locales/mr.json b/client/src/i18n/locales/mr.jsonindex 693403f2..049b5209 100644--- a/client/src/i18n/locales/mr.json+++ b/client/src/i18n/locales/mr.json@@ -122,6 +122,7 @@"visionTitle": "प्रतिमा इनपुट स्वीकारते","tools": "साधने","toolsTitle": "संरचित टूल कॉल सोडते, त्यामुळे ते टूल-बेअरिंग विनंत्यांना पात्र आहे",+ "retired": "बंद","penalty": "−{value} दंड","obs": "{count} obs","tokPerMonth": "{count} tok/mo",diff --git a/client/src/i18n/locales/ms.json b/client/src/i18n/locales/ms.jsonindex ab112e03..1c7cf5ff 100644--- a/client/src/i18n/locales/ms.json+++ b/client/src/i18n/locales/ms.json@@ -122,6 +122,7 @@"visionTitle": "Menerima input imej","tools": "Alat","toolsTitle": "Memancarkan panggilan alat berstruktur, jadi ia layak untuk permintaan yang membawa alat",+ "retired": "Dihentikan","penalty": "Penalti −{value}","obs": "{count} obs","tokPerMonth": "{count} Tok/Bulan",diff --git a/client/src/i18n/locales/my.json b/client/src/i18n/locales/my.jsonindex b6168b99..c6131746 100644--- a/client/src/i18n/locales/my.json+++ b/client/src/i18n/locales/my.json@@ -122,6 +122,7 @@"visionTitle": "ပုံရိပ်ထည့်သွင်းမှုကို လက်ခံသည်","tools": "ကိရိယာများ","toolsTitle": "ဖွဲ့စည်းထားသော ကိရိယာခေါ်ဆိုမှုများ ထုတ်လွှင့်သဖြင့် ကိရိယာပါသော တောင်းဆိုမှုများအတွက် အရည်အချင်းပြည့်မီသည်။",+ "retired": "ရပ်ဆိုင်းပြီး","penalty": "−{value} ပြစ်ဒဏ်","obs": "{count}","tokPerMonth": "{count} tok/mo",diff --git a/client/src/i18n/locales/ne.json b/client/src/i18n/locales/ne.jsonindex b19065f8..0995f134 100644--- a/client/src/i18n/locales/ne.json+++ b/client/src/i18n/locales/ne.json@@ -122,6 +122,7 @@"visionTitle": "छवि आगत स्वीकार गर्दछ","tools": "उपकरणहरू","toolsTitle": "संरचित उपकरण कलहरू उत्सर्जन गर्दछ, त्यसैले यो उपकरण-असर अनुरोधहरूको लागि योग्य छ",+ "retired": "बन्द","penalty": "−{value} पेनाल्टी","obs": "{count} obs","tokPerMonth": "{count} टोक/मो",diff --git a/client/src/i18n/locales/nl.json b/client/src/i18n/locales/nl.jsonindex 995b4a71..cc70204e 100644--- a/client/src/i18n/locales/nl.json+++ b/client/src/i18n/locales/nl.json@@ -122,6 +122,7 @@"visionTitle": "Accepteert beeldinvoer","tools": "Gereedschappen","toolsTitle": "Zendt gestructureerde tooloproepen uit, zodat het in aanmerking komt voor tool-bearing verzoeken",+ "retired": "Uitgefaseerd","penalty": "−{value} straf","obs": "{count} obs","tokPerMonth": "{count} tok/mo",diff --git a/client/src/i18n/locales/no.json b/client/src/i18n/locales/no.jsonindex f5fc4fef..9cd23e38 100644--- a/client/src/i18n/locales/no.json+++ b/client/src/i18n/locales/no.json@@ -122,6 +122,7 @@"visionTitle": "Godtar bildeinput","tools": "Verktøy","toolsTitle": "Sender ut strukturerte verktøykall, slik at den er kvalifisert for verktøybærende forespørsler",+ "retired": "Utgått","penalty": "−{value} straff","obs": "{count} observasjoner","tokPerMonth": "{count} tok/mo",diff --git a/client/src/i18n/locales/or.json b/client/src/i18n/locales/or.jsonindex fce4c10f..bb6a253d 100644--- a/client/src/i18n/locales/or.json+++ b/client/src/i18n/locales/or.json@@ -122,6 +122,7 @@"visionTitle": "ଫଟୋ ଇନପୁଟ୍ ସ୍ଵୀକାର କରେ","tools": "ଉପକରଣ","toolsTitle": "ସଂରଚନା ଟୁଲ୍ କଲ୍ ନିର୍ଗତ କରେ, ତେଣୁ ଏହା ଟୁଲ୍-ବିୟରିଂ ଅନୁରୋଧ ପାଇଁ ଯୋଗ୍ୟ ଅଟେ",+ "retired": "ବନ୍ଦ","penalty": "−{value} ପେନାଲ୍ଟି","obs": "{count} ଓବିଏସ୍","tokPerMonth": "{count} tok/mo",diff --git a/client/src/i18n/locales/pa.json b/client/src/i18n/locales/pa.jsonindex 3ce8b87f..9278d4de 100644--- a/client/src/i18n/locales/pa.json+++ b/client/src/i18n/locales/pa.json@@ -122,6 +122,7 @@"visionTitle": "ਚਿੱਤਰ ਇੰਪੁੱਟ ਸਵੀਕਾਰ ਕਰਦਾ ਹੈ","tools": "ਸੰਦ","toolsTitle": "ਸਟ੍ਰਕਚਰਡ ਟੂਲ ਕਾਲਾਂ ਨੂੰ ਛੱਡਦਾ ਹੈ, ਇਸਲਈ ਇਹ ਟੂਲ-ਬੇਅਰਿੰਗ ਬੇਨਤੀਆਂ ਲਈ ਯੋਗ ਹੈ",+ "retired": "ਬੰਦ","penalty": "−{value} ਜੁਰਮਾਨਾ","obs": "{count} obs","tokPerMonth": "{count} tok/mo",diff --git a/client/src/i18n/locales/pl.json b/client/src/i18n/locales/pl.jsonindex 7f115f9f..7e85288b 100644--- a/client/src/i18n/locales/pl.json+++ b/client/src/i18n/locales/pl.json@@ -122,6 +122,7 @@"visionTitle": "Akceptuje wejście obrazu","tools": "Narzędzia","toolsTitle": "Emituje uporządkowane wywołania narzędzi, więc kwalifikuje się do żądań łożysk narzędziowych",+ "retired": "Wycofany","penalty": "−{value} kara","obs": "{count} obs","tokPerMonth": "{count} tok/mies",diff --git a/client/src/i18n/locales/pt-BR.json b/client/src/i18n/locales/pt-BR.jsonindex e61d255c..2a856f57 100644--- a/client/src/i18n/locales/pt-BR.json+++ b/client/src/i18n/locales/pt-BR.json@@ -122,6 +122,7 @@"visionTitle": "Aceita imagens como entrada","tools": "Ferramentas","toolsTitle": "Emite chamadas de ferramentas estruturadas, por isso é elegível para solicitações com ferramentas",+ "retired": "Descontinuado","penalty": "penalidade de −{value}","obs": "{count} obs","tokPerMonth": "{count} tok/mês",diff --git a/client/src/i18n/locales/pt-PT.json b/client/src/i18n/locales/pt-PT.jsonindex 55a8a2c8..859c41f6 100644--- a/client/src/i18n/locales/pt-PT.json+++ b/client/src/i18n/locales/pt-PT.json@@ -122,6 +122,7 @@"visionTitle": "Aceita entrada de imagem","tools": "Ferramentas","toolsTitle": "Emite chamadas estruturadas de ferramentas, tornando-se elegível para pedidos de suporte de ferramenta",+ "retired": "Descontinuado","penalty": "−{value} penalização","obs": "{count} obs","tokPerMonth": "{count} tok/mo",diff --git a/client/src/i18n/locales/ro.json b/client/src/i18n/locales/ro.jsonindex 2c2e7fcb..0d0fe304 100644--- a/client/src/i18n/locales/ro.json+++ b/client/src/i18n/locales/ro.json@@ -122,6 +122,7 @@"visionTitle": "Acceptă intrarea de imagine","tools": "Unelte","toolsTitle": "Emite apeluri structurate pentru unelte, deci este eligibil pentru cereri cu rulante de unelte",+ "retired": "Retras","penalty": "−{value} penalizare","obs": "{count} observații","tokPerMonth": "{count} tok/mo",diff --git a/client/src/i18n/locales/ru.json b/client/src/i18n/locales/ru.jsonindex 3b8ad01d..6ea83cb3 100644--- a/client/src/i18n/locales/ru.json+++ b/client/src/i18n/locales/ru.json@@ -122,6 +122,7 @@"visionTitle": "Принимает ввод изображения","tools": "Инструменты","toolsTitle": "Выполняет структурированные вызовы инструментов, поэтому он имеет право на запросы с инструментами.",+ "retired": "Снят","penalty": "−{value} штраф","obs": "{count} обс","tokPerMonth": "{count} ток/мес",diff --git a/client/src/i18n/locales/si.json b/client/src/i18n/locales/si.jsonindex 87eb6887..5d8a43eb 100644--- a/client/src/i18n/locales/si.json+++ b/client/src/i18n/locales/si.json@@ -122,6 +122,7 @@"visionTitle": "රූප ආදානය පිළිගනී","tools": "මෙවලම්","toolsTitle": "ව්යුහගත මෙවලම් ඇමතුම් නිකුත් කරයි, එබැවින් එය මෙවලම් දරණ ඉල්ලීම් සඳහා සුදුසුකම් ලබයි",+ "retired": "නවතා ඇත","penalty": "−{value} දඩුවම","obs": "{count} obs","tokPerMonth": "{count} tok/mo",diff --git a/client/src/i18n/locales/sk.json b/client/src/i18n/locales/sk.jsonindex 28d0a241..5fba552c 100644--- a/client/src/i18n/locales/sk.json+++ b/client/src/i18n/locales/sk.json@@ -122,6 +122,7 @@"visionTitle": "Prijíma vstup obrazu","tools": "Nástroje","toolsTitle": "Vysiela štruktúrované volania nástroja, takže je oprávnený na požiadavky na ložisko nástroja",+ "retired": "Vyradené","penalty": "−{value} penalta","obs": "{count} OBS","tokPerMonth": "{count} tok/mo",diff --git a/client/src/i18n/locales/sr.json b/client/src/i18n/locales/sr.jsonindex cfa02b24..00ce9bbd 100644--- a/client/src/i18n/locales/sr.json+++ b/client/src/i18n/locales/sr.json@@ -122,6 +122,7 @@"visionTitle": "Прихвата унос слике","tools": "Алати","toolsTitle": "Емитује структуриране позиве алата, тако да испуњава услове за захтеве који носе алате",+ "retired": "Повучен","penalty": "−{value} казна","obs": "{count} обс","tokPerMonth": "{count} ток/мес",diff --git a/client/src/i18n/locales/sv.json b/client/src/i18n/locales/sv.jsonindex f1f7e858..56c244fd 100644--- a/client/src/i18n/locales/sv.json+++ b/client/src/i18n/locales/sv.json@@ -122,6 +122,7 @@"visionTitle": "Accepterar bildindata","tools": "Verktyg","toolsTitle": "Skickar strukturerade verktygsanrop, så den är berättigad till verktygsbärande förfrågningar",+ "retired": "Utgått","penalty": "−{value} straff","obs": "{count} observationer","tokPerMonth": "{count} tok/mo",diff --git a/client/src/i18n/locales/sw.json b/client/src/i18n/locales/sw.jsonindex 4c593e0d..01698ddb 100644--- a/client/src/i18n/locales/sw.json+++ b/client/src/i18n/locales/sw.json@@ -122,6 +122,7 @@"visionTitle": "Inakubali ingizo la picha","tools": "Zana","toolsTitle": "Hutoa simu za zana zilizopangwa, kwa hivyo inastahiki maombi yenye zana",+ "retired": "Imeondolewa","penalty": "−{value} adhabu","obs": "Sehemu za {count}","tokPerMonth": "{count} tok/mo",diff --git a/client/src/i18n/locales/ta.json b/client/src/i18n/locales/ta.jsonindex dd77b126..5d367a5c 100644--- a/client/src/i18n/locales/ta.json+++ b/client/src/i18n/locales/ta.json@@ -122,6 +122,7 @@"visionTitle": "பட உள்ளீட்டை ஏற்கிறது","tools": "கருவிகள்","toolsTitle": "கட்டமைக்கப்பட்ட கருவி அழைப்புகளை வெளியிடுகிறது, எனவே கருவி தாங்கும் கோரிக்கைகளுக்கு இது தகுதியானது",+ "retired": "நிறுத்தப்பட்டது","penalty": "−{value} அபராதம்","obs": "{count} obs","tokPerMonth": "{count} tok/mo",diff --git a/client/src/i18n/locales/te.json b/client/src/i18n/locales/te.jsonindex a8092b1e..9151da57 100644--- a/client/src/i18n/locales/te.json+++ b/client/src/i18n/locales/te.json@@ -122,6 +122,7 @@"visionTitle": "చిత్రం ఇన్పుట్ని అంగీకరిస్తుంది","tools": "ఉపకరణాలు","toolsTitle": "నిర్మాణాత్మక టూల్ కాల్లను విడుదల చేస్తుంది, కాబట్టి ఇది టూల్-బేరింగ్ అభ్యర్థనలకు అర్హత కలిగి ఉంటుంది",+ "retired": "నిలిపివేయబడింది","penalty": "−{value} పెనాల్టీ","obs": "{count} obs","tokPerMonth": "{count} టోక్/మొ",diff --git a/client/src/i18n/locales/th.json b/client/src/i18n/locales/th.jsonindex 29f98803..4151357c 100644--- a/client/src/i18n/locales/th.json+++ b/client/src/i18n/locales/th.json@@ -122,6 +122,7 @@"visionTitle": "ยอมรับอินพุตรูปภาพ","tools": "เครื่องมือ","toolsTitle": "ส่งเสียงการเรียกเครื่องมือที่มีโครงสร้าง ดังนั้นจึงมีสิทธิ์ได้รับคำขอเกี่ยวกับเครื่องมือ",+ "retired": "ยกเลิกแล้ว","penalty": "−{value} การลงโทษ","obs": "{count} obs","tokPerMonth": "{count} ท็อค/เดือน",diff --git a/client/src/i18n/locales/tl.json b/client/src/i18n/locales/tl.jsonindex 5b548974..7b6000fd 100644--- a/client/src/i18n/locales/tl.json+++ b/client/src/i18n/locales/tl.json@@ -122,6 +122,7 @@"visionTitle": "Tumatanggap ng image input","tools": "Mga Kasangkapan","toolsTitle": "Naglalabas ng structured tool calls, kaya ito ay kwalipikado para sa mga tool-bearing request",+ "retired": "Itinigil","penalty": "−{value} penalty","obs": "{count} obs","tokPerMonth": "{count} tok/buwan",diff --git a/client/src/i18n/locales/tr.json b/client/src/i18n/locales/tr.jsonindex 560be136..4722474a 100644--- a/client/src/i18n/locales/tr.json+++ b/client/src/i18n/locales/tr.json@@ -122,6 +122,7 @@"visionTitle": "Görüntü girişini kabul eder","tools": "Araçlar","toolsTitle": "Yapılandırılmış araç çağrıları yayınlayarak araç taşıma isteklerine uygun hale gelir",+ "retired": "Kaldırıldı","penalty": "−{value} cezası","obs": "{count} gözlem","tokPerMonth": "{count} tok/ay",diff --git a/client/src/i18n/locales/uk.json b/client/src/i18n/locales/uk.jsonindex 87f9bfff..7f4daa05 100644--- a/client/src/i18n/locales/uk.json+++ b/client/src/i18n/locales/uk.json@@ -122,6 +122,7 @@"visionTitle": "Приймає введення зображення","tools": "Інструменти","toolsTitle": "Видає структуровані виклики інструментів, тому він підходить для запитів, що містять інструменти",+ "retired": "Знято","penalty": "−{value} пенальті","obs": "{count} обс","tokPerMonth": "{count} ток/міс",diff --git a/client/src/i18n/locales/ur.json b/client/src/i18n/locales/ur.jsonindex a6c26a40..d2c50e6b 100644--- a/client/src/i18n/locales/ur.json+++ b/client/src/i18n/locales/ur.json@@ -122,6 +122,7 @@"visionTitle": "امیج ان پٹ کو قبول کرتا ہے۔","tools": "اوزار","toolsTitle": "سٹرکچرڈ ٹول کالز کو خارج کرتا ہے، اس لیے یہ ٹول بیئرنگ کی درخواستوں کے لیے اہل ہے۔",+ "retired": "بند","penalty": "−{value} جرمانہ","obs": "{count} obs","tokPerMonth": "{count} tok/mo",diff --git a/client/src/i18n/locales/uz.json b/client/src/i18n/locales/uz.jsonindex 8743e1de..2632473d 100644--- a/client/src/i18n/locales/uz.json+++ b/client/src/i18n/locales/uz.json@@ -122,6 +122,7 @@"visionTitle": "Rasm kiritishni qabul qiladi","tools": "Asboblar","toolsTitle": "Strukturaviy asboblar qo'ng'iroqlarini chiqaradi, shuning uchun u asboblar so'rovlari uchun javob beradi",+ "retired": "To'xtatilgan","penalty": "−{value} jarima","obs": "{count} obs","tokPerMonth": "{count} tok/oy",diff --git a/client/src/i18n/locales/vi.json b/client/src/i18n/locales/vi.jsonindex ed8bcee3..44aa34eb 100644--- a/client/src/i18n/locales/vi.json+++ b/client/src/i18n/locales/vi.json@@ -122,6 +122,7 @@"visionTitle": "Chấp nhận đầu vào hình ảnh","tools": "Công cụ","toolsTitle": "Phát ra các lệnh gọi công cụ có cấu trúc, do đó nó đủ điều kiện cho các yêu cầu mang công cụ",+ "retired": "Ngừng cung cấp","penalty": "−{value} phạt đền","obs": "{count} quan sát","tokPerMonth": "{count} tháng/tháng",diff --git a/client/src/i18n/locales/yo.json b/client/src/i18n/locales/yo.jsonindex 3bcba6c8..61732698 100644--- a/client/src/i18n/locales/yo.json+++ b/client/src/i18n/locales/yo.json@@ -122,6 +122,7 @@"visionTitle": "Gba igbewọle aworan","tools": "Awọn irinṣẹ","toolsTitle": "Emits awọn ipe irinṣẹ ti a ṣeto, nitorinaa o yẹ fun awọn ibeere ti nso ohun elo",+ "retired": "A ti dáwọ́ dúró","penalty": "-{value} ijiya","obs": "{count} obs","tokPerMonth": "{count} tok/mo",diff --git a/client/src/i18n/locales/zh-CN.json b/client/src/i18n/locales/zh-CN.jsonindex 684b13ff..83e42125 100644--- a/client/src/i18n/locales/zh-CN.json+++ b/client/src/i18n/locales/zh-CN.json@@ -122,6 +122,7 @@"visionTitle": "支持图像输入","tools": "工具","toolsTitle": "支持结构化工具调用,可用于需要工具的请求",+ "retired": "已下线","penalty": "−{value} 惩罚","obs": "{count} 次观察","tokPerMonth": "{count} 令牌/月",diff --git a/client/src/i18n/locales/zh-TW.json b/client/src/i18n/locales/zh-TW.jsonindex 7f6ccba9..17d8c838 100644--- a/client/src/i18n/locales/zh-TW.json+++ b/client/src/i18n/locales/zh-TW.json@@ -122,6 +122,7 @@"visionTitle": "接受影像輸入","tools": "工具","toolsTitle": "會發出結構化工具呼叫,因此有資格進行工具承載請求",+ "retired": "已下線","penalty": "−{value} 罰則","obs": "{count}觀察","tokPerMonth": "{count} tok/mo",diff --git a/client/src/lib/routing.ts b/client/src/lib/routing.tsindex 8c634c7a..272b904e 100644--- a/client/src/lib/routing.ts+++ b/client/src/lib/routing.ts@@ -30,6 +30,11 @@ export interface FallbackEntry {keyId?: number | nullkeyLabel?: string | nullhasOverrides?: boolean+ // The provider reported this model as permanently gone (410 / end of life),+ // so the gateway disabled it by itself — distinct from a switch the user+ // flipped off. `retiredReason` is the upstream wording. See #634.+ retiredUpstream?: boolean+ retiredReason?: string | nullkeyCount: number// Logical-model grouping (sent by the server when unify is relevant). Absent// for ungrouped rows; the UI falls back to a per-row "solo" group then.diff --git a/server/src/__tests__/db/migrate/roundtrip.test.ts b/server/src/__tests__/db/migrate/roundtrip.test.tsindex feb487a0..085cd9dc 100644--- a/server/src/__tests__/db/migrate/roundtrip.test.ts+++ b/server/src/__tests__/db/migrate/roundtrip.test.ts@@ -20,6 +20,7 @@ const MEDIA_MODEL_META_FILENAME = '20260726_000004_media_model_meta.ts';const REQUEST_SERVED_MODEL_FILENAME = '20260726_000005_request_served_model.ts';const ATTEMPT_ERROR_SUMMARY_FILENAME = '20260726_000006_attempt_error_summary.ts';const AGENT_COMPATIBILITY_FILENAME = '20260727_000001_agent_compatibility.ts';+const TOMBSTONE_PROVENANCE_FILENAME = '20260728_000001_tombstone_provenance.ts';interface SchemaRow {type: string;@@ -86,6 +87,7 @@ describe('migration round trip', () => {REQUEST_SERVED_MODEL_FILENAME,ATTEMPT_ERROR_SUMMARY_FILENAME,AGENT_COMPATIBILITY_FILENAME,+ TOMBSTONE_PROVENANCE_FILENAME,]);} finally {db.close();diff --git a/server/src/__tests__/routes/proxy-retry.test.ts b/server/src/__tests__/routes/proxy-retry.test.tsindex 470f556f..d8f18614 100644--- a/server/src/__tests__/routes/proxy-retry.test.ts+++ b/server/src/__tests__/routes/proxy-retry.test.ts@@ -1,6 +1,8 @@import { describe, it, expect } from 'vitest';import { isRetryableError, isPaymentRequiredError, isModelNotFoundError, isModelAccessForbiddenError } from '../../routes/proxy.js';import { isProviderBadRequestError } from '../../lib/error-classify.js';+import { cooldownDecisionForError } from '../../lib/fallback-loop.js';+import { MODEL_FORBIDDEN_COOLDOWN_MS } from '../../services/ratelimit.js';describe('isModelAccessForbiddenError (403 model-not-on-tier, drives whole-model skip — issue #256)', () => {it('flags a 403 reaching the proxy by message or attached status', () => {@@ -18,6 +20,59 @@ describe('isModelAccessForbiddenError (403 model-not-on-tier, drives whole-modelexpect(isModelAccessForbiddenError(new Error('OpenRouter API error 404: Provider returned error'))).toBe(false);expect(isModelAccessForbiddenError(new Error('HuggingFace Router API error 402: Payment required'))).toBe(false);});++ // Issue #618: some providers deny model access with a 400 (or 401) instead of+ // a 403. Classified transient, those got the 90s bench — and the auto-router+ // re-picked the same unreachable model the moment it expired, forever, with+ // an escalating penalty. They must classify as model-inaccessible-for-this-key.+ it('flags a 400/401 whose body says the key may not access the model (#618)', () => {+ expect(isModelAccessForbiddenError(Object.assign(+ new Error('API error 400: user is not allowed to access model kat-coder-pro-v2.5'),+ { status: 400 },+ ))).toBe(true);+ expect(isModelAccessForbiddenError(Object.assign(+ new Error('API error 400: action plan limited, please upgrade'),+ { status: 400 },+ ))).toBe(true);+ expect(isModelAccessForbiddenError(Object.assign(+ new Error('API error 401: you do not have access to this model'),+ { status: 401 },+ ))).toBe(true);+ expect(isModelAccessForbiddenError(+ new Error('user is not allowed to access model kat-coder-pro-v2.5'),+ )).toBe(true);+ });++ it('still leaves ordinary 400/401 rejections alone', () => {+ expect(isModelAccessForbiddenError(Object.assign(new Error('Bad Request'), { status: 400 }))).toBe(false);+ expect(isModelAccessForbiddenError(Object.assign(new Error('Unauthorized'), { status: 401 }))).toBe(false);+ expect(isModelAccessForbiddenError(new Error('401 Unauthorized'))).toBe(false);+ expect(isModelAccessForbiddenError(new Error('400 Bad Request'))).toBe(false);+ expect(isModelAccessForbiddenError(Object.assign(+ new Error('Groq API error 400: Failed to call a function. Please adjust your prompt.'),+ { status: 400 },+ ))).toBe(false);+ });+});++// The loop-level consequence of the classification above: a "not allowed to+// access" 400 must take the model-forbidden bench (a day, source 'tier'), not+// the 90s transient one that made the router re-pick it every 90 seconds.+describe('cooldownDecisionForError on a 400 not-allowed body (#618)', () => {+ const route = {+ platform: 'kat', modelId: 'kat-coder-pro-v2.5', modelDbId: 1, keyId: 1,+ rpdLimit: null, tpdLimit: null,+ } as any;++ it('benches the model like a 403 instead of the transient ladder', () => {+ const err = Object.assign(+ new Error('API error 400: user is not allowed to access model kat-coder-pro-v2.5'),+ { status: 400 },+ );+ const decision = cooldownDecisionForError(route, err);+ expect(decision.durationMs).toBe(MODEL_FORBIDDEN_COOLDOWN_MS);+ expect(decision.source).toBe('tier');+ });});describe('isModelNotFoundError (drives whole-model skip within a request)', () => {diff --git a/server/src/__tests__/services/model-retirement.test.ts b/server/src/__tests__/services/model-retirement.test.tsnew file mode 100644index 00000000..57eafb40--- /dev/null+++ b/server/src/__tests__/services/model-retirement.test.ts@@ -0,0 +1,223 @@+import { describe, it, expect, beforeAll, beforeEach } from 'vitest';+import { initDb, getDb } from '../../db/index.js';+import { modelRetirementSignal } from '../../lib/error-classify.js';+import {+ noteModelRetirementSignal,+ resetModelRetirementObservations,+ RETIREMENT_CONFIRMATIONS_REQUIRED,+} from '../../services/model-retirement.js';+import { getCatalogModelTombstone, recordCatalogModelTombstone } from '../../services/model-state.js';+import { applyCatalog } from '../../services/catalog-sync.js';++// #634: NVIDIA retires a model upstream ("has reached its end of life"), and+// every subsequent request burns a fallback slot on the corpse because nothing+// persisted the 404/410. These tests lock the two halves of the fix: a+// conservative end-of-life CLASSIFIER (transient 404s must never match), and a+// corroboration threshold before the model is actually auto-disabled.++const PLATFORM = 'groq';+const MODEL_ID = 'eol-test-model';++beforeAll(() => {+ process.env.ENCRYPTION_KEY = '0'.repeat(64);+ initDb(':memory:');+});++function seedModel(modelId = MODEL_ID): number {+ const db = getDb();+ db.prepare(`+ INSERT INTO models (platform, model_id, display_name, intelligence_rank, speed_rank, enabled, source)+ VALUES (?, ?, 'EOL Test', 50, 50, 1, 'catalog')+ ON CONFLICT(platform, model_id) DO UPDATE SET enabled = 1+ `).run(PLATFORM, modelId);+ const row = db.prepare('SELECT id FROM models WHERE platform = ? AND model_id = ?')+ .get(PLATFORM, modelId) as { id: number };+ db.prepare(`+ INSERT INTO fallback_config (model_db_id, priority, enabled)+ VALUES (?, 999, 1)+ ON CONFLICT(model_db_id) DO UPDATE SET enabled = 1+ `).run(row.id);+ return row.id;+}++function isRoutable(modelDbId: number): boolean {+ const row = getDb()+ .prepare('SELECT enabled FROM fallback_config WHERE model_db_id = ?')+ .get(modelDbId) as { enabled: number } | undefined;+ return row?.enabled === 1;+}++function routeFor(modelDbId: number, modelId = MODEL_ID) {+ return { modelDbId, platform: PLATFORM, modelId };+}++// The exact NVIDIA body from issue #634.+const NVIDIA_EOL = Object.assign(+ new Error(+ "NVIDIA NIM API error 410: The model 'minimaxai/minimax-m2.7' has reached its end of life "+ + 'on 2026-07-27T00:00:00Z and is no longer available.',+ ),+ { status: 410 },+);++// The transient shape from the same issue thread: a load balancer that cannot+// find the function for the account right now. Says nothing about the model.+const TRANSIENT_404 = Object.assign(+ new Error(+ "NVIDIA NIM API error 404: Function '23d4f03a-b8a6-4adb-a183-7daa083a09cc': Not found for account 'BDnmPUMI'",+ ),+ { status: 404 },+);++describe('modelRetirementSignal (conservative end-of-life detection — #634)', () => {+ it('treats an explicit 410 / end-of-life body as definitive', () => {+ expect(modelRetirementSignal(NVIDIA_EOL)).toBe('definitive');+ expect(modelRetirementSignal(Object.assign(new Error('Gone'), { status: 410 }))).toBe('definitive');+ expect(modelRetirementSignal(new Error('Ollama Cloud API error 410: Gone'))).toBe('definitive');+ // The wording alone is definitive even when the provider picks a 404.+ expect(modelRetirementSignal(Object.assign(+ new Error('API error 404: model has reached its end of life'),+ { status: 404 },+ ))).toBe('definitive');+ });++ it('treats a 404 that clearly says the model is gone as probable (needs corroboration)', () => {+ expect(modelRetirementSignal(Object.assign(+ new Error('OpenRouter API error 404: this model is no longer available'),+ { status: 404 },+ ))).toBe('probable');+ expect(modelRetirementSignal(Object.assign(+ new Error('Groq API error 404: model has been removed'),+ { status: 404 },+ ))).toBe('probable');+ });++ it('never fires on transient 404s, empty not-found bodies, or unrelated failures', () => {+ expect(modelRetirementSignal(TRANSIENT_404)).toBeNull();+ expect(modelRetirementSignal(new Error('OpenRouter API error 404: Provider returned error'))).toBeNull();+ expect(modelRetirementSignal(new Error('Model not found'))).toBeNull();+ // OpenRouter says this while a model is merely unserved right now.+ expect(modelRetirementSignal(new Error('No endpoints found for openrouter/minimax/minimax-m2.5:free'))).toBeNull();+ expect(modelRetirementSignal(new Error('429 Too Many Requests'))).toBeNull();+ expect(modelRetirementSignal(new Error('503 Service Unavailable'))).toBeNull();+ // A digit run that merely contains 410 is not a 410.+ expect(modelRetirementSignal(new Error('Groq API error 400: max_tokens 41000 exceeds 32768'))).toBeNull();+ });+});++describe('noteModelRetirementSignal (auto-disable with a reason — #634)', () => {+ beforeEach(() => {+ resetModelRetirementObservations();+ getDb().prepare("DELETE FROM catalog_model_tombstones WHERE model_id LIKE 'eol-test-%'").run();+ });++ it('auto-disables the model on a single explicit 410 end-of-life response', () => {+ const id = seedModel();+ expect(noteModelRetirementSignal(routeFor(id), NVIDIA_EOL, {})).toBe(true);+ expect(isRoutable(id)).toBe(false);++ const tombstone = getCatalogModelTombstone(getDb(), 'chat', PLATFORM, MODEL_ID);+ expect(tombstone?.source).toBe('upstream_eol');+ expect(tombstone?.reason).toContain('end of life');+ });++ it('leaves a transient 404 alone no matter how often it repeats', () => {+ const id = seedModel();+ for (let i = 0; i < 5; i++) {+ expect(noteModelRetirementSignal(routeFor(id), TRANSIENT_404, {})).toBe(false);+ }+ expect(isRoutable(id)).toBe(true);+ expect(getCatalogModelTombstone(getDb(), 'chat', PLATFORM, MODEL_ID)).toBeUndefined();+ });++ it('requires corroboration from distinct requests before disabling on a probable 404', () => {+ const id = seedModel();+ const err = Object.assign(new Error('API error 404: this model is no longer available'), { status: 404 });++ expect(RETIREMENT_CONFIRMATIONS_REQUIRED).toBeGreaterThanOrEqual(2);+ expect(noteModelRetirementSignal(routeFor(id), err, {})).toBe(false);+ expect(isRoutable(id)).toBe(true);+ expect(noteModelRetirementSignal(routeFor(id), err, {})).toBe(true);+ expect(isRoutable(id)).toBe(false);+ });++ it('does not let one flaky request corroborate itself across sibling keys', () => {+ const id = seedModel();+ const err = Object.assign(new Error('API error 404: this model is no longer available'), { status: 404 });+ const oneRequest = {};++ for (let i = 0; i < 4; i++) {+ expect(noteModelRetirementSignal(routeFor(id), err, oneRequest)).toBe(false);+ }+ expect(isRoutable(id)).toBe(true);+ });++ it('never touches user-added models (they are not catalog state)', () => {+ const db = getDb();+ db.prepare(`+ INSERT INTO models (platform, model_id, display_name, intelligence_rank, speed_rank, enabled, source)+ VALUES ('custom', 'eol-test-custom', 'Custom EOL', 50, 50, 1, 'user')+ ON CONFLICT(platform, model_id) DO UPDATE SET enabled = 1+ `).run();+ const row = db.prepare("SELECT id FROM models WHERE platform = 'custom' AND model_id = 'eol-test-custom'")+ .get() as { id: number };+ db.prepare(`+ INSERT INTO fallback_config (model_db_id, priority, enabled) VALUES (?, 998, 1)+ ON CONFLICT(model_db_id) DO UPDATE SET enabled = 1+ `).run(row.id);++ expect(noteModelRetirementSignal(+ { modelDbId: row.id, platform: 'custom', modelId: 'eol-test-custom' },+ NVIDIA_EOL,+ {},+ )).toBe(false);+ expect(isRoutable(row.id)).toBe(true);+ });+});++describe('catalog sync vs. upstream retirement (#634)', () => {+ function catalogWith(modelId: string, enabled: boolean) {+ return {+ version: '2099.01.01',+ generatedAt: new Date().toISOString(),+ tier: 'live' as const,+ models: [{+ platform: PLATFORM,+ modelId,+ displayName: 'EOL Test',+ intelligenceRank: 50,+ speedRank: 50,+ sizeLabel: 'Medium',+ limits: { rpm: 30, rpd: 1000, tpm: 6000, tpd: null },+ monthlyTokenBudget: '~1M',+ contextWindow: 8192,+ enabled,+ supportsVision: false,+ supportsTools: true,+ }],+ quirks: [],+ } as unknown as Parameters<typeof applyCatalog>[1];+ }++ it('re-enables an auto-retired model when a later catalog still lists it', () => {+ resetModelRetirementObservations();+ const id = seedModel('eol-test-relisted');+ noteModelRetirementSignal(routeFor(id, 'eol-test-relisted'), NVIDIA_EOL, {});+ expect(isRoutable(id)).toBe(false);++ applyCatalog(getDb(), catalogWith('eol-test-relisted', true));++ expect(isRoutable(id)).toBe(true);+ expect(getCatalogModelTombstone(getDb(), 'chat', PLATFORM, 'eol-test-relisted')).toBeUndefined();+ expect(getDb().prepare('SELECT id FROM models WHERE platform = ? AND model_id = ?')+ .get(PLATFORM, 'eol-test-relisted')).toBeDefined();+ });++ it('still deletes models the USER tombstoned (unchanged behavior)', () => {+ seedModel('eol-test-user-deleted');+ recordCatalogModelTombstone(getDb(), 'chat', PLATFORM, 'eol-test-user-deleted');+ applyCatalog(getDb(), catalogWith('eol-test-user-deleted', true));+ expect(getDb().prepare('SELECT id FROM models WHERE platform = ? AND model_id = ?')+ .get(PLATFORM, 'eol-test-user-deleted')).toBeUndefined();+ });+});diff --git a/server/src/db/migrate/defaults.ts b/server/src/db/migrate/defaults.tsindex ae69af19..6776f0ec 100644--- a/server/src/db/migrate/defaults.ts+++ b/server/src/db/migrate/defaults.ts@@ -15,6 +15,7 @@ import * as mediaModelMeta from '../migrations/20260726_000004_media_model_meta.import * as requestServedModel from '../migrations/20260726_000005_request_served_model.js';import * as attemptErrorSummary from '../migrations/20260726_000006_attempt_error_summary.js';import * as agentCompatibility from '../migrations/20260727_000001_agent_compatibility.js';+import * as tombstoneProvenance from '../migrations/20260728_000001_tombstone_provenance.js';export interface MigrationModule {up(db: Db): void;@@ -42,6 +43,7 @@ export const MEDIA_MODEL_META_FILENAME = '20260726_000004_media_model_meta.ts';export const REQUEST_SERVED_MODEL_FILENAME = '20260726_000005_request_served_model.ts';export const ATTEMPT_ERROR_SUMMARY_FILENAME = '20260726_000006_attempt_error_summary.ts';export const AGENT_COMPATIBILITY_FILENAME = '20260727_000001_agent_compatibility.ts';+export const TOMBSTONE_PROVENANCE_FILENAME = '20260728_000001_tombstone_provenance.ts';export const DEFAULT_MIGRATIONS: readonly DefaultMigration[] = [{ filename: LEGACY_BASELINE_FILENAME, module: legacyBaseline },@@ -60,4 +62,5 @@ export const DEFAULT_MIGRATIONS: readonly DefaultMigration[] = [{ filename: REQUEST_SERVED_MODEL_FILENAME, module: requestServedModel },{ filename: ATTEMPT_ERROR_SUMMARY_FILENAME, module: attemptErrorSummary },{ filename: AGENT_COMPATIBILITY_FILENAME, module: agentCompatibility },+ { filename: TOMBSTONE_PROVENANCE_FILENAME, module: tombstoneProvenance },];diff --git a/server/src/db/migrations/20260728_000001_tombstone_provenance.ts b/server/src/db/migrations/20260728_000001_tombstone_provenance.tsnew file mode 100644index 00000000..f0966c46--- /dev/null+++ b/server/src/db/migrations/20260728_000001_tombstone_provenance.ts@@ -0,0 +1,41 @@+// Migration: record WHY a catalog model was tombstoned+// Created: 2026-07-28+//+// DOWN: reversible — drops the two added columns (every existing tombstone is+// a user deletion, which is the default the columns backfill to).++import type { Db } from '../types.js';++function hasColumn(db: Db, table: string, column: string): boolean {+ const columns = db.prepare(`PRAGMA table_info(${table})`).all() as { name: string }[];+ return columns.some((candidate) => candidate.name === column);+}++/**+ * Tombstones used to mean exactly one thing: "the user deleted this catalog+ * model, keep it deleted across syncs". Issue #634 adds a second, machine-made+ * kind — "the provider retired this model upstream (410 / end of life)" — which+ * disables the row instead of deleting it and carries the upstream wording so+ * the dashboard can say why. `source` keeps the two apart ('user' is the+ * backfill for every pre-existing row); `reason` is the redacted provider text.+ */+export function up(db: Db): void {+ if (!hasColumn(db, 'catalog_model_tombstones', 'source')) {+ db.prepare("ALTER TABLE catalog_model_tombstones ADD COLUMN source TEXT NOT NULL DEFAULT 'user'").run();+ }+ if (!hasColumn(db, 'catalog_model_tombstones', 'reason')) {+ db.prepare('ALTER TABLE catalog_model_tombstones ADD COLUMN reason TEXT').run();+ }+}++export function down(db: Db): void {+ // Upstream-retirement tombstones have no meaning without their source column,+ // and leaving them behind would silently delete live models on the next sync.+ db.prepare("DELETE FROM catalog_model_tombstones WHERE source = 'upstream_eol'").run();+ if (hasColumn(db, 'catalog_model_tombstones', 'reason')) {+ db.prepare('ALTER TABLE catalog_model_tombstones DROP COLUMN reason').run();+ }+ if (hasColumn(db, 'catalog_model_tombstones', 'source')) {+ db.prepare('ALTER TABLE catalog_model_tombstones DROP COLUMN source').run();+ }+}diff --git a/server/src/lib/error-classify.ts b/server/src/lib/error-classify.tsindex 3893e5e0..b7bc173b 100644--- a/server/src/lib/error-classify.ts+++ b/server/src/lib/error-classify.ts@@ -329,5 +329,85 @@ export function isModelNotFoundError(err: any): boolean {export function isModelAccessForbiddenError(err: any): boolean {if (err?.status === 403) return true;const msg = (err?.message ?? '').toLowerCase();- return msg.includes('403') || msg.includes('forbidden');+ if (msg.includes('403') || msg.includes('forbidden')) return true;+ // Not every provider spells "this key may not use this model" as a 403 (issue+ // #618): observed live as a 400 whose body reads "user is not allowed to+ // access model kat-coder-pro-v2.5", and as plan-gate wordings like "action+ // plan limited". Classified transient, those took the 90s bench — and the+ // auto-router re-picked the same permanently-unreachable model the moment it+ // expired, failing and re-benching forever with an ever-growing penalty.+ // Deliberately narrow: only 400/401 (or a status-less error), and only for+ // phrasings that name access/permission to the MODEL. A bare "Bad Request",+ // a generic "Unauthorized" (which isKeyAuthError owns), or a parameter+ // rejection must keep their current classification.+ const status = typeof err?.status === 'number' ? err.status : 0;+ if (status !== 0 && status !== 400 && status !== 401) return false;+ return MODEL_ACCESS_DENIED_PHRASES.some(phrase => msg.includes(phrase));+}++const MODEL_ACCESS_DENIED_PHRASES = [+ 'not allowed to access',+ 'not allowed to use',+ 'not authorized to access',+ 'not authorized to use',+ 'unauthorized to access',+ 'not permitted to access',+ 'not permitted to use',+ 'do not have access to',+ 'does not have access to',+ 'no access to model',+ 'no access to this model',+ 'model access denied',+ 'access to this model is restricted',+ // Plan/subscription gates the provider words as a limit rather than a denial.+ 'action plan limited',+];++// ── Permanent upstream retirement (issue #634) ───────────────────────────────+// isModelNotFoundError above rules a 404/410 model out for ONE request. That is+// right for a load balancer hiccup and wrong for a model the provider RETIRED:+// NVIDIA's "The model 'minimaxai/minimax-m2.7' has reached its end of life on+// 2026-07-27T00:00:00Z and is no longer available" is true forever, yet nothing+// persisted it, so the corpse burned a fallback slot on every later request.+//+// This classifier says how much a failure should be TRUSTED as permanent:+// - 'definitive' — an explicit 410 Gone, or unmistakable end-of-life wording.+// Acting on one response is safe.+// - 'probable' — a 404 whose body says the model is gone in so many words.+// Real, but a provider can also word a transient outage this way, so the+// caller corroborates across distinct requests before acting.+// - null — everything else, including every bare 404 ("Not found",+// "Provider returned error", OpenRouter's "No endpoints found", NVIDIA's+// per-account "Function '<uuid>': Not found for account '<id>'"). A+// transient 404 must NEVER retire a healthy model.+// Note the 410 test is deliberately stricter than isModelNotFoundError's bare+// `msg.includes('410')`: a digit run inside a token count must not read as Gone.+export type ModelRetirementConfidence = 'definitive' | 'probable';++const END_OF_LIFE_PHRASES = [+ 'end of life',+ 'end-of-life',+ 'has been retired',+ 'has been decommissioned',+ 'has been sunset',+];++const MODEL_GONE_PHRASES = [+ 'no longer available',+ 'no longer offered',+ 'no longer supported',+ 'has been removed',+ 'was removed',+ 'has been discontinued',+ 'has been deprecated',+ 'is deprecated',+];++export function modelRetirementSignal(err: any): ModelRetirementConfidence | null {+ const msg = (err?.message ?? '').toLowerCase();+ const status = typeof err?.status === 'number' ? err.status : 0;+ const gone = status === 410 || /\berror 410\b/.test(msg) || /\b410 gone\b/.test(msg);+ if (gone || END_OF_LIFE_PHRASES.some(phrase => msg.includes(phrase))) return 'definitive';+ if (!isModelNotFoundError(err)) return null;+ return MODEL_GONE_PHRASES.some(phrase => msg.includes(phrase)) ? 'probable' : null;}diff --git a/server/src/lib/fallback-loop.ts b/server/src/lib/fallback-loop.tsindex e9790cb8..91eb3c6d 100644--- a/server/src/lib/fallback-loop.ts+++ b/server/src/lib/fallback-loop.ts@@ -45,9 +45,10 @@ import {} from './error-classify.js';import { sanitizeProviderErrorMessage, summarizeAttemptError } from './error-redaction.js';import { checkKeyHealth, markKeyHealthyFromRequest } from '../services/health.js';+import { noteModelRetirementSignal } from '../services/model-retirement.js';import { getSetting } from '../db/index.js';import { newBreaker, recordBreakerFailure } from './guardrails.js';-import { newRequestTrace, runWithRequestTrace, type AttemptOutcome, type RequestTrace } from './attempt-trace.js';+import { getRequestTrace, newRequestTrace, runWithRequestTrace, type AttemptOutcome, type RequestTrace } from './attempt-trace.js';import { persistRequestAttempts } from './request-log.js';// Every surface caps failover hops at the same number.@@ -186,6 +187,12 @@ export function recordRetryableFailure(route: RouteResult, err: any, state: Fallif (isModelNotFoundError(err) || isModelAccessForbiddenError(err) || isContextTooLargeError(err) || err?.skipModelForRequest === true) {state.skipModels.add(route.modelDbId);}+ // A model-level 404/410 that says the model is GONE (not merely missing right+ // now) outlives this request: persist it once the evidence is strong enough,+ // or the retired model burns a fallback slot on every request forever (#634).+ // The trace object identifies the request, so one request's failover across+ // sibling keys counts as the single observation it is.+ noteModelRetirementSignal(route, err, getRequestTrace());state.skipKeys.add(`${route.platform}:${route.modelId}:${route.keyId}`);if (err?.skipBench === true) return;const decision = cooldownDecisionForError(route, err);diff --git a/server/src/routes/fallback.ts b/server/src/routes/fallback.tsindex 1b6cdfef..59f236d7 100644--- a/server/src/routes/fallback.ts+++ b/server/src/routes/fallback.ts@@ -72,11 +72,14 @@ fallbackRouter.get('/', (_req: Request, res: Response) => {m.tpm_limit, m.tpd_limit, m.context_window,m.monthly_token_budget, m.supports_vision, m.supports_tools,m.key_id, ak.label AS key_label,- mo.overrides_json IS NOT NULL AS has_overrides+ mo.overrides_json IS NOT NULL AS has_overrides,+ ts.source AS tombstone_source, ts.reason AS tombstone_reasonFROM profile_models pmJOIN models m ON m.id = pm.model_db_idLEFT JOIN api_keys ak ON ak.id = m.key_idLEFT JOIN model_overrides mo ON mo.platform = m.platform AND mo.model_id = m.model_id+ LEFT JOIN catalog_model_tombstones ts+ ON ts.kind = 'chat' AND ts.platform = m.platform AND ts.model_id = m.model_idWHERE pm.profile_id = ? AND m.enabled = 1ORDER BY pm.priority ASC`).all(activeProfileId) as any[];@@ -89,11 +92,14 @@ fallbackRouter.get('/', (_req: Request, res: Response) => {m.tpm_limit, m.tpd_limit, m.context_window,m.monthly_token_budget, m.supports_vision, m.supports_tools,m.key_id, ak.label AS key_label,- mo.overrides_json IS NOT NULL AS has_overrides+ mo.overrides_json IS NOT NULL AS has_overrides,+ ts.source AS tombstone_source, ts.reason AS tombstone_reasonFROM fallback_config fcJOIN models m ON m.id = fc.model_db_idLEFT JOIN api_keys ak ON ak.id = m.key_idLEFT JOIN model_overrides mo ON mo.platform = m.platform AND mo.model_id = m.model_id+ LEFT JOIN catalog_model_tombstones ts+ ON ts.kind = 'chat' AND ts.platform = m.platform AND ts.model_id = m.model_idWHERE m.enabled = 1ORDER BY fc.priority ASC`).all() as any[];@@ -160,6 +166,12 @@ fallbackRouter.get('/', (_req: Request, res: Response) => {keyId: r.key_id ?? null,keyLabel: r.key_label ?? null,hasOverrides: Boolean(r.has_overrides),+ // Why the switch is off: a model auto-disabled because the provider+ // retired it upstream (410 / end of life, #634) is a different state from+ // one the user turned off, and the dashboard says so. The reason is the+ // provider's own (redacted) wording.+ retiredUpstream: r.tombstone_source === 'upstream_eol',+ retiredReason: r.tombstone_source === 'upstream_eol' ? (r.tombstone_reason ?? null) : null,keyCount: keyCountMap.get(r.platform) ?? 0,};}));diff --git a/server/src/services/catalog-sync.ts b/server/src/services/catalog-sync.tsindex db3c9587..a677b206 100644--- a/server/src/services/catalog-sync.ts+++ b/server/src/services/catalog-sync.ts@@ -11,6 +11,7 @@ import {applyModelOverrides,deleteTombstonedCatalogModels,isCatalogModelTombstoned,+ reinstateUpstreamRetiredCatalogModel,} from './model-state.js';import { ensureAllModelsInProfiles } from './profile-models.js';@@ -230,7 +231,9 @@ function routableContextWindow(platform: string, modelId: string, contextWindow:* declarative config, admin adds) are never updated, never deleted, and* never adopted — on a platform:model_id collision the user row wins and* the catalog entry is skipped outright;- * - catalog models the user deleted stay deleted via tombstones;+ * - catalog models the user deleted stay deleted via tombstones, while models+ * auto-retired from an upstream 410/end-of-life response (#634) are only+ * disabled — a catalog that still lists them lifts the retirement;* - models that vanished from the catalog are deleted, exactly like the* dead-model migrations do (fallback_config row first, FK order).*/@@ -341,6 +344,10 @@ export function applyCatalog(db: Db, catalog: Catalog): NonNullable<SyncResult['continue;}if (isCatalogModelTombstoned(db, 'chat', m.platform, m.modelId)) continue;+ // A model auto-retired from a 410/end-of-life response (#634) is disabled,+ // not deleted. A catalog that STILL lists it — and lists it enabled — is+ // newer evidence than that one provider response, so lift the retirement.+ if (m.enabled) reinstateUpstreamRetiredCatalogModel(db, m.platform, m.modelId);inCatalog.add(`${m.platform}:${m.modelId}`);const row = selectModel.get(m.platform, m.modelId) asdiff --git a/server/src/services/model-retirement.ts b/server/src/services/model-retirement.tsnew file mode 100644index 00000000..d1e2dfca--- /dev/null+++ b/server/src/services/model-retirement.ts@@ -0,0 +1,105 @@+// Auto-retirement of models the upstream provider has permanently removed.+//+// Issue #634: NVIDIA answered every request for `minimaxai/minimax-m2.7` with+// "410 … has reached its end of life … and is no longer available". The failure+// classifies as a model-level skip, which rules the model out for ONE request —+// so the next request tried it again, and the next, forever, each one paying a+// round trip and a fallback slot for a model that is never coming back.+//+// The fix persists the verdict: a definitive end-of-life response disables the+// model right away, and a merely-probable one waits for corroboration from a+// second distinct request first, so a single flaky 404 from a load balancer can+// never kill a healthy model. The disable is reversible in both directions —+// the user can switch the model back on, and a catalog refresh that still lists+// it lifts the retirement (services/catalog-sync.ts).++import { getDb } from '../db/index.js';+import { modelRetirementSignal } from '../lib/error-classify.js';+import { summarizeAttemptError } from '../lib/error-redaction.js';+import { isCatalogManagedModel, retireCatalogModelUpstream } from './model-state.js';++/** How many DISTINCT requests must report a 'probable' signal before acting. */+export const RETIREMENT_CONFIRMATIONS_REQUIRED = 2;++/**+ * Corroboration has to be recent. Two "no longer available" 404s a week apart+ * are two outages; two within the hour are a retirement.+ */+export const RETIREMENT_OBSERVATION_WINDOW_MS = 60 * 60 * 1000;++interface Observation {+ count: number;+ firstSeenAt: number;+ // Identity of the request that last counted, so the same request failing over+ // across sibling keys cannot corroborate itself.+ lastRequest: unknown;+}++const observations = new Map<string, Observation>();++/** Test seam: the counters are process-local and deliberately not persisted. */+export function resetModelRetirementObservations(): void {+ observations.clear();+}++export interface RetirementRoute {+ modelDbId: number;+ platform: string;+ modelId: string;+}++/**+ * Record one upstream failure against the retirement heuristic and, when the+ * evidence is strong enough, auto-disable the model. Returns true iff this call+ * retired it. Never throws: it runs on the proxy's failure path, where a DB+ * hiccup must not turn a failover into a crash.+ */+export function noteModelRetirementSignal(+ route: RetirementRoute,+ err: unknown,+ requestToken?: unknown,+): boolean {+ const confidence = modelRetirementSignal(err);+ if (!confidence) return false;++ const key = `${route.platform}:${route.modelId}`;+ if (confidence === 'probable' && !confirmObservation(key, requestToken)) return false;++ const reason = summarizeAttemptError((err as { message?: unknown })?.message);+ try {+ const db = getDb();+ const row = db+ .prepare('SELECT id, platform, model_id, key_id, source FROM models WHERE id = ?')+ .get(route.modelDbId) as+ | { id: number; platform: string; model_id: string; key_id: number | null; source: string }+ | undefined;+ // Only catalog-managed rows: a model the user added by hand is their state,+ // not the catalog's, and the tombstone table has no meaning for it.+ if (!row || !isCatalogManagedModel(row)) return false;+ if (!retireCatalogModelUpstream(db, row.id, row.platform, row.model_id, reason)) return false;+ observations.delete(key);+ console.log(+ `[ModelRetirement] ${row.platform}/${row.model_id} disabled — upstream reports it retired: ${reason}`,+ );+ return true;+ } catch (dbErr: any) {+ console.warn(`[ModelRetirement] could not record retirement for ${key}: ${dbErr?.message ?? dbErr}`);+ return false;+ }+}++/** True once this model has been reported gone by enough distinct requests. */+function confirmObservation(key: string, requestToken: unknown): boolean {+ const now = Date.now();+ const existing = observations.get(key);+ if (!existing || now - existing.firstSeenAt > RETIREMENT_OBSERVATION_WINDOW_MS) {+ observations.set(key, { count: 1, firstSeenAt: now, lastRequest: requestToken });+ return RETIREMENT_CONFIRMATIONS_REQUIRED <= 1;+ }+ // Same request failing over across this model's sibling keys: one request's+ // worth of evidence, however many keys it burned.+ if (requestToken !== undefined && existing.lastRequest === requestToken) return false;+ existing.count++;+ existing.lastRequest = requestToken;+ return existing.count >= RETIREMENT_CONFIRMATIONS_REQUIRED;+}diff --git a/server/src/services/model-state.ts b/server/src/services/model-state.tsindex 360af803..cb2d8829 100644--- a/server/src/services/model-state.ts+++ b/server/src/services/model-state.ts@@ -69,15 +69,52 @@ export function isCatalogManagedModel(row: { platform: string; key_id?: number |return row.platform !== 'custom' && row.key_id == null;}+// Why a catalog model carries a tombstone:+// 'user' — deleted in the dashboard. Stays deleted across syncs, and+// the row is removed from `models` on the next catalog apply.+// 'upstream_eol' — the provider reported it permanently gone (410 / end of+// life, issue #634). The row SURVIVES and is only disabled,+// so the dashboard can show "retired upstream" instead of+// silently losing the model, and so a later catalog that+// still lists it can lift the retirement.+export type CatalogTombstoneSource = 'user' | 'upstream_eol';++export interface CatalogModelTombstone {+ source: CatalogTombstoneSource;+ reason: string | null;+ createdAt: string;+}++export function getCatalogModelTombstone(+ db: Db,+ kind: CatalogModelKind,+ platform: string,+ modelId: string,+): CatalogModelTombstone | undefined {+ const row = db+ .prepare('SELECT source, reason, created_at FROM catalog_model_tombstones WHERE kind = ? AND platform = ? AND model_id = ?')+ .get(kind, platform, modelId) as { source: string; reason: string | null; created_at: string } | undefined;+ if (!row) return undefined;+ return {+ source: row.source === 'upstream_eol' ? 'upstream_eol' : 'user',+ reason: row.reason ?? null,+ createdAt: row.created_at,+ };+}++/**+ * True only for models the USER deleted — the "keep it deleted" contract every+ * caller here means. An upstream-retirement tombstone deliberately does NOT+ * count: those models stay in the catalog's write path so a refreshed catalog+ * can reinstate them (see reinstateUpstreamRetiredCatalogModel).+ */export function isCatalogModelTombstoned(db: Db,kind: CatalogModelKind,platform: string,modelId: string,): boolean {- return !!db- .prepare('SELECT 1 FROM catalog_model_tombstones WHERE kind = ? AND platform = ? AND model_id = ?')- .get(kind, platform, modelId);+ return getCatalogModelTombstone(db, kind, platform, modelId)?.source === 'user';}export function recordCatalogModelTombstone(@@ -85,17 +122,71 @@ export function recordCatalogModelTombstone(kind: CatalogModelKind,platform: string,modelId: string,+ options: { source?: CatalogTombstoneSource; reason?: string | null } = {},): void {+ const source: CatalogTombstoneSource = options.source ?? 'user';db.prepare(`- INSERT INTO catalog_model_tombstones (kind, platform, model_id)- VALUES (?, ?, ?)- ON CONFLICT(kind, platform, model_id) DO UPDATE SET created_at = datetime('now')- `).run(kind, platform, modelId);- if (kind === 'chat') {+ INSERT INTO catalog_model_tombstones (kind, platform, model_id, source, reason)+ VALUES (?, ?, ?, ?, ?)+ ON CONFLICT(kind, platform, model_id)+ DO UPDATE SET created_at = datetime('now'), source = excluded.source, reason = excluded.reason+ `).run(kind, platform, modelId, source, options.reason ?? null);+ // A user deletion drops their local metadata edits with the row. An upstream+ // retirement keeps the row, so it keeps the overrides too — they must survive+ // if the model is reinstated.+ if (kind === 'chat' && source === 'user') {db.prepare('DELETE FROM model_overrides WHERE platform = ? AND model_id = ?').run(platform, modelId);}}+/**+ * Auto-disable a catalog model the provider reports as permanently retired+ * (issue #634). Deliberately NOT a delete: the row stays visible in the+ * dashboard, tagged with the upstream wording, and the user can flip it back on+ * if they disagree. Turning off the chain entries (and the active profile's+ * copy) is exactly what the dashboard's own switch does, so the router stops+ * picking it while an explicitly-requested model id still resolves.+ *+ * Returns true when this call performed the retirement (false when it was+ * already retired, or the user had deleted the model outright).+ */+export function retireCatalogModelUpstream(+ db: Db,+ modelDbId: number,+ platform: string,+ modelId: string,+ reason: string,+): boolean {+ const existing = getCatalogModelTombstone(db, 'chat', platform, modelId);+ if (existing) return false;+ recordCatalogModelTombstone(db, 'chat', platform, modelId, { source: 'upstream_eol', reason });+ db.prepare('UPDATE fallback_config SET enabled = 0 WHERE model_db_id = ?').run(modelDbId);+ db.prepare('UPDATE profile_models SET enabled = 0 WHERE model_db_id = ?').run(modelDbId);+ return true;+}++/**+ * Lift an upstream retirement: a catalog that still lists the model — and lists+ * it enabled — is newer and better evidence than one provider's 404. Returns+ * true when a retirement was actually lifted.+ */+export function reinstateUpstreamRetiredCatalogModel(+ db: Db,+ platform: string,+ modelId: string,+): boolean {+ if (getCatalogModelTombstone(db, 'chat', platform, modelId)?.source !== 'upstream_eol') return false;+ clearCatalogModelTombstone(db, 'chat', platform, modelId);+ const row = db+ .prepare('SELECT id FROM models WHERE platform = ? AND model_id = ?')+ .get(platform, modelId) as { id: number } | undefined;+ if (row) {+ db.prepare('UPDATE fallback_config SET enabled = 1 WHERE model_db_id = ?').run(row.id);+ db.prepare('UPDATE profile_models SET enabled = 1 WHERE model_db_id = ?').run(row.id);+ }+ return true;+}+export function clearCatalogModelTombstone(db: Db,kind: CatalogModelKind,@@ -191,19 +282,23 @@ export function applyAllModelOverrides(db: Db): number {return applied;}+// Only USER tombstones delete rows. An upstream-retirement tombstone disables+// its model and keeps it (see retireCatalogModelUpstream), so deleting here+// would throw away both the row and the reason the dashboard shows for it.export function deleteTombstonedCatalogModels(db: Db): number {const chatRows = db.prepare(`SELECT m.id, m.platform, m.model_idFROM models mJOIN catalog_model_tombstones tON t.kind = 'chat' AND t.platform = m.platform AND t.model_id = m.model_id- WHERE m.platform != 'custom' AND m.key_id IS NULL AND m.source != 'user'+ WHERE t.source = 'user' AND m.platform != 'custom' AND m.key_id IS NULL AND m.source != 'user'`).all() as { id: number; platform: string; model_id: string }[];const mediaRows = db.prepare(`SELECT mm.idFROM media_models mmJOIN catalog_model_tombstones tON t.kind = 'media' AND t.platform = mm.platform AND t.model_id = mm.model_id+ WHERE t.source = 'user'`).all() as { id: number }[];const deleteChatFallback = db.prepare('DELETE FROM fallback_config WHERE model_db_id = ?');