Skip to content

7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.

TypeScript

fix(google): honor daily quota exhaustion and Pacific reset (#1343)

NBR5MTR290aff2c114f08
diff --git a/server/src/__tests__/lib/fallback-loop.test.ts b/server/src/__tests__/lib/fallback-loop.test.ts
index 696c57dd..0686087a 100644
--- a/server/src/__tests__/lib/fallback-loop.test.ts
+++ b/server/src/__tests__/lib/fallback-loop.test.ts
@@ -26,6 +26,7 @@ import {
formatAttemptTrail,
classifyAttemptError,
msUntilNextUtcMidnight,
+ msUntilNextPacificMidnight,
getFallbackTimeBudgetMs,
DEFAULT_FALLBACK_TIME_BUDGET_MS,
AUTH_FAILURE_COOLDOWN_MS,
@@ -120,6 +121,29 @@ describe('isKeyAuthError (401 = key-fatal, rotate instead of 502)', () => {
});
describe('isDailyQuotaExhaustedError + midnight benching (drift: 90s cooldown on a dead-for-the-day provider)', () => {
+ it.each([
+ ['2026-09-27T12:00:00Z', '2026-09-28T07:00:00Z'],
+ ['2026-01-27T12:00:00Z', '2026-01-28T08:00:00Z'],
+ ['2026-03-08T08:00:00Z', '2026-03-09T07:00:00Z'],
+ ['2026-11-01T07:00:00Z', '2026-11-02T08:00:00Z'],
+ ])('uses the Pacific reset from %s through DST', (now, reset) => {
+ expect(msUntilNextPacificMidnight(Date.parse(now))).toBe(Date.parse(reset) - Date.parse(now));
+ });
+
+ it('keeps a Gemini daily violation benched past a short RetryInfo delay', () => {
+ const err = Object.assign(new Error('Quota exceeded'), {
+ status: 429, dailyQuotaExhausted: true, retryAfterMs: 17_000,
+ });
+ expect(isDailyQuotaExhaustedError(err)).toBe(true);
+ const duration = cooldownForError(fakeRoute({ platform: 'google' }), err);
+ expect(Math.abs(duration - msUntilNextPacificMidnight())).toBeLessThan(1_000);
+ });
+
+ it('keeps a Gemini per-minute violation on the transient cooldown', () => {
+ const err = Object.assign(new Error('Quota exceeded'), { status: 429, dailyQuotaExhausted: false, retryAfterMs: 17_000 });
+ expect(cooldownForError(fakeRoute({ platform: 'google' }), err)).toBe(90_000);
+ });
+
it('flags real daily-allocation 429 bodies', () => {
expect(isDailyQuotaExhaustedError(new Error('Cloudflare API error 429: you have used up your daily free allocation of 10,000 neurons'))).toBe(true);
expect(isDailyQuotaExhaustedError(new Error('Rate limit exceeded: free-models-per-day'))).toBe(true);
diff --git a/server/src/__tests__/providers/google.test.ts b/server/src/__tests__/providers/google.test.ts
index 0cfa318f..04ed6bbb 100644
--- a/server/src/__tests__/providers/google.test.ts
+++ b/server/src/__tests__/providers/google.test.ts
@@ -90,6 +90,36 @@ describe('GoogleProvider', () => {
expect(await provider.validateKey('invalid-key')).toMatchObject({ valid: false });
});
+ it.each([false, true])('preserves daily quota evidence with streaming=%s (#1339)', async (stream) => {
+ vi.spyOn(global, 'fetch').mockResolvedValueOnce(new Response(JSON.stringify({ error: {
+ message: 'You exceeded your current quota.',
+ details: [
+ { '@type': 'type.googleapis.com/google.rpc.QuotaFailure', violations: [
+ { quotaId: 'GenerateRequestsPerDayPerProjectPerModel-FreeTier' },
+ { quotaId: 'GenerateRequestsPerMinutePerProjectPerModel-FreeTier' },
+ ] },
+ { '@type': 'type.googleapis.com/google.rpc.RetryInfo', retryDelay: '17s' },
+ ],
+ } }), { status: 429 }));
+ const messages = [{ role: 'user' as const, content: 'Hi' }];
+ const request = stream
+ ? provider.streamChatCompletion('test-key', messages, 'gemini-2.5-pro').next()
+ : provider.chatCompletion('test-key', messages, 'gemini-2.5-pro');
+ await expect(request).rejects.toMatchObject({ status: 429, dailyQuotaExhausted: true, retryAfterMs: 17_000 });
+ });
+
+ it.each([
+ [{ '@type': 'type.googleapis.com/google.rpc.QuotaFailure', violations: [{ quotaId: 'GenerateRequestsPerMinutePerProjectPerModel-FreeTier' }] }],
+ [{ '@type': 'type.googleapis.com/google.rpc.Help', quotaId: 'PerDay' }],
+ [{ '@type': 'type.googleapis.com/google.rpc.QuotaFailure', violations: null }],
+ ])('does not invent daily exhaustion from other details (%j)', async (...details) => {
+ vi.spyOn(global, 'fetch').mockResolvedValueOnce(new Response(JSON.stringify({ error: {
+ message: 'Quota exceeded', details,
+ } }), { status: 429 }));
+ await expect(provider.chatCompletion('test-key', [], 'gemini-2.5-pro'))
+ .rejects.toMatchObject({ dailyQuotaExhausted: false });
+ });
+
// #268: Google reports a bad key as HTTP 400 INVALID_ARGUMENT / API_KEY_INVALID,
// not 401/403. A confirmed-bad key must return false (→ auto-disable counter).
it('validateKey returns false for a genuinely invalid key (HTTP 400 API_KEY_INVALID)', async () => {
diff --git a/server/src/lib/error-classify.ts b/server/src/lib/error-classify.ts
index 8d96c70d..e1949c3e 100644
--- a/server/src/lib/error-classify.ts
+++ b/server/src/lib/error-classify.ts
@@ -353,6 +353,7 @@ export function isKeyAuthError(err: any): boolean {
// benches until the next UTC midnight instead. Requires BOTH a daily marker and
// a quota/allocation marker so an ordinary per-minute 429 never matches.
export function isDailyQuotaExhaustedError(err: any): boolean {
+ if (err?.dailyQuotaExhausted === true) return true;
const msg = (err?.message ?? '').toLowerCase();
if (!/daily|per[ -_]?day|\btoday\b/.test(msg)) return false;
return /allocation|quota|limit|exhaust|used up/.test(msg);
diff --git a/server/src/lib/fallback-loop.ts b/server/src/lib/fallback-loop.ts
index 9145a736..22dfd585 100644
--- a/server/src/lib/fallback-loop.ts
+++ b/server/src/lib/fallback-loop.ts
@@ -257,6 +257,25 @@ export function msUntilNextUtcMidnight(now = Date.now()): number {
return Math.max(next - now, 60_000);
}
+const pacificDate = new Intl.DateTimeFormat('en-US', {
+ timeZone: 'America/Los_Angeles', year: 'numeric', month: 'numeric', day: 'numeric',
+});
+const pacificHour = new Intl.DateTimeFormat('en-US', {
+ timeZone: 'America/Los_Angeles', hour: 'numeric', hourCycle: 'h23',
+});
+
+/** Gemini daily quotas reset at midnight Pacific, including DST transitions.
+ * https://ai.google.dev/gemini-api/docs/rate-limits */
+export function msUntilNextPacificMidnight(now = Date.now()): number {
+ const parts = pacificDate.formatToParts(now);
+ const part = (name: string) => Number(parts.find(p => p.type === name)!.value);
+ // 08:00 UTC on the next Pacific date is either midnight (PST) or 01:00
+ // (PDT). Read the offset on that date, not now: DST days have 23/25 hours.
+ const candidate = Date.UTC(part('year'), part('month') - 1, part('day') + 1, 8);
+ const midnight = candidate - Number(pacificHour.format(candidate)) * 3_600_000;
+ return Math.max(midnight - now, 60_000);
+}
+
/**
* The one true cooldown-duration selection after a retryable upstream failure:
* - 402 out-of-credits → a full day (PAYMENT_REQUIRED_COOLDOWN_MS)
@@ -271,6 +290,8 @@ export function msUntilNextUtcMidnight(now = Date.now()): number {
* provider Retry-After wins over the midnight heuristic: rolling daily
* windows (Groq RPD "try again in 7m12s" with a Retry-After header) reset
* well before midnight, and the provider knows its own reset time best.
+ * Gemini daily violations instead wait until midnight Pacific; a shorter
+ * RetryInfo can refer to a simultaneous per-minute violation.
* - anything else → the transient/daily escalation ladder, honoring the
* provider's Retry-After as a floor (getCooldownDurationForLimit).
*/
@@ -296,6 +317,11 @@ export function cooldownDecisionForError(route: RouteResult, err: any): Cooldown
if (isAccountSuspendedError(err)) return { durationMs: getPaymentRequiredCooldownMs(), source: 'credit' };
if (isModelAccessForbiddenError(err)) return { durationMs: getModelForbiddenCooldownMs(), source: 'tier' };
if (isDailyQuotaExhaustedError(err)) {
+ if (route.platform === 'google') {
+ // RetryInfo can describe a simultaneous per-minute violation. It cannot
+ // reopen a daily allowance before the Pacific reset (#1339).
+ return { durationMs: Math.max(msUntilNextPacificMidnight(), err?.retryAfterMs ?? 0), source: 'authoritative' };
+ }
return { durationMs: err?.retryAfterMs ?? msUntilNextUtcMidnight(), source: 'authoritative' };
}
return getCooldownDecisionForLimit(
diff --git a/server/src/providers/base.ts b/server/src/providers/base.ts
index 8126d761..6ea89f04 100644
--- a/server/src/providers/base.ts
+++ b/server/src/providers/base.ts
@@ -19,6 +19,8 @@ import { extractThinkTagsFromStream } from '../lib/think-tags.js';
export interface ProviderHttpError extends Error {
status?: number;
retryAfterMs?: number;
+ /** Explicit daily-window violation from structured upstream quota details. */
+ dailyQuotaExhausted?: boolean;
}
/** Upper bound on a provider-supplied back-off. A malformed or hostile
diff --git a/server/src/providers/google.ts b/server/src/providers/google.ts
index 45faa88a..f9905f8b 100644
--- a/server/src/providers/google.ts
+++ b/server/src/providers/google.ts
@@ -19,6 +19,22 @@ export { sanitizeForGemini } from '../lib/gemini-wire.js';
const API_BASE = 'https://generativelanguage.googleapis.com/v1beta';
+function googleHttpError(res: Response, body: any) {
+ const err = providerHttpError(res, `Google API error ${res.status}: ${body?.error?.message ?? res.statusText}`, body);
+ // Gemini's human-readable message can omit the quota window. Keep only the
+ // daily signal from QuotaFailure, not the complete upstream payload (#1339).
+ const details = body?.error?.details;
+ if (res.status === 429 && Array.isArray(details)) {
+ err.dailyQuotaExhausted = details.some(detail =>
+ detail?.['@type'] === 'type.googleapis.com/google.rpc.QuotaFailure'
+ && Array.isArray(detail.violations)
+ && detail.violations.some((violation: any) =>
+ typeof violation?.quotaId === 'string' && /PerDay/.test(violation.quotaId)),
+ );
+ }
+ return err;
+}
+
// Gemini 3 REQUIRES the `thoughtSignature` that accompanied a function call to
// be echoed back whenever that call appears in conversation history, or it
// rejects the request with 400 "Function call is missing a thought_sig". But
@@ -592,7 +608,7 @@ export class GoogleProvider extends BaseProvider {
if (!res.ok) {
const err = await res.json().catch(() => ({}));
- throw providerHttpError(res, `Google API error ${res.status}: ${(err as any).error?.message ?? res.statusText}`, err);
+ throw googleHttpError(res, err);
}
const data = await res.json() as GeminiResponse;
@@ -673,7 +689,7 @@ export class GoogleProvider extends BaseProvider {
if (!res.ok) {
const err = await res.json().catch(() => ({}));
- throw providerHttpError(res, `Google API error ${res.status}: ${(err as any).error?.message ?? res.statusText}`, err);
+ throw googleHttpError(res, err);
}
const reader = res.body?.getReader();