Skip to content

Claude Code. Any Model. The most powerful AI coding agent now speaks every language.

9.5 KB
Raw Blame
/**
 * Gemini Request Queue
 *
 * Singleton request queue for serializing Gemini API requests to prevent rate limit exhaustion.
 * Implements dynamic rate limiting based on API responses (429 errors with quotaResetDelay).
 *
 * All Gemini requests are processed sequentially through a FIFO queue with:
 * - Minimum delay between requests (default 1000ms = 60 req/min)
 * - Dynamic delay adjustment based on 429 error responses
 * - Exponential backoff for consecutive errors
 * - Automatic queue size management (max 100 requests)
 */

import { log } from "../../logger.js";
import { callAbortError } from "./call-context.js";

/**
 * Queued request with Promise callbacks
 */
interface QueuedRequest {
  fetchFn: () => Promise<Response>;
  resolve: (response: Response) => void;
  reject: (error: Error) => void;
  /** The caller's abort signal, when it passed one. */
  signal?: AbortSignal;
  /** Removes this entry from the queue on abort, while it is still waiting. */
  onAbort?: () => void;
}

/**
 * Called by the worker once it has taken an entry: the entry no longer waits in
 * the queue, so its abort listener goes, and an entry whose caller already
 * aborted is rejected instead of started. True when it was rejected.
 */
function rejectIfAborted(request: QueuedRequest): boolean {
  if (request.onAbort) request.signal?.removeEventListener("abort", request.onAbort);
  if (!request.signal?.aborted) return false;
  request.reject(callAbortError());
  return true;
}

/**
 * Queue statistics for monitoring
 */
export interface QueueStats {
  queueLength: number;
  processing: boolean;
  consecutiveErrors: number;
  currentDelayMs: number;
  totalProcessed: number;
  totalErrors: number;
}

/**
 * Singleton request queue for Gemini API
 *
 * Serializes all Gemini requests to prevent rate limit exhaustion.
 * Implements dynamic rate limiting based on API responses.
 *
 * @example
 * ```typescript
 * const queue = GeminiRequestQueue.getInstance();
 * const response = await queue.enqueue(() => fetch(url, options));
 * ```
 */
export class GeminiRequestQueue {
  private static instance: GeminiRequestQueue | null = null;
  private queue: QueuedRequest[] = [];
  private processing = false;
  private minDelayMs = 1000; // 60 requests/minute
  private lastRequestTime = 0;
  private consecutiveErrors = 0;
  private totalProcessed = 0;
  private totalErrors = 0;

  // Configuration
  private readonly baseDelayMs = 1000;
  private readonly maxDelayMs = 10000;
  private readonly maxQueueSize = 100;

  private constructor() {
    log("[GeminiQueue] Queue initialized with minDelay=1000ms, maxQueueSize=100");
  }

  /**
   * Get singleton instance
   */
  static getInstance(): GeminiRequestQueue {
    if (!GeminiRequestQueue.instance) {
      GeminiRequestQueue.instance = new GeminiRequestQueue();
    }
    return GeminiRequestQueue.instance;
  }

  /**
   * Enqueue a request to be processed
   *
   * @param fetchFn - Function that performs the fetch request
   * @returns Promise that resolves with the response
   * @throws Error if queue is full
   */
  async enqueue(
    fetchFn: () => Promise<Response>,
    opts?: { signal?: AbortSignal }
  ): Promise<Response> {
    const signal = opts?.signal;
    if (signal?.aborted) throw callAbortError();
    // Check queue size limit
    if (this.queue.length >= this.maxQueueSize) {
      log(
        `[GeminiQueue] Queue full (${this.queue.length}/${this.maxQueueSize}), rejecting request`
      );
      throw new Error("Gemini request queue full. Please retry later.");
    }

    // Create promise for this request
    return new Promise<Response>((resolve, reject) => {
      const queuedRequest: QueuedRequest = {
        fetchFn,
        resolve,
        reject,
        signal,
      };
      // An abort while the entry still WAITS removes it and rejects at once:
      // nothing of the call starts after its deadline, and it stops holding a
      // place in line. Once the worker has taken it, the worker checks again.
      if (signal) {
        queuedRequest.onAbort = () => {
          const index = this.queue.indexOf(queuedRequest);
          if (index === -1) return;
          this.queue.splice(index, 1);
          reject(callAbortError());
        };
        signal.addEventListener("abort", queuedRequest.onAbort, { once: true });
      }

      this.queue.push(queuedRequest);
      log(`[GeminiQueue] Request enqueued (queue length: ${this.queue.length})`);

      // Start processing if not already running
      if (!this.processing) {
        this.processQueue();
      }
    });
  }

  /**
   * Worker loop that processes queued requests sequentially
   */
  private async processQueue(): Promise<void> {
    if (this.processing) {
      return; // Already processing
    }

    this.processing = true;
    log("[GeminiQueue] Worker started");

    while (this.queue.length > 0) {
      const request = this.queue.shift();
      if (!request) break;
      if (rejectIfAborted(request)) continue;

      log(`[GeminiQueue] Processing request (${this.queue.length} remaining in queue)`);

      try {
        // Wait for next available slot
        await this.waitForNextSlot();
        // Aborted during the rate-limit wait: never start it.
        if (rejectIfAborted(request)) continue;

        // Execute the request
        const response = await request.fetchFn();
        this.lastRequestTime = Date.now();

        // Check for rate limit response
        if (response.status === 429) {
          this.totalErrors++;
          const errorText = await response.clone().text();
          this.handleRateLimitResponse(errorText);
          log(`[GeminiQueue] Rate limit hit (429), adjusted delay to ${this.minDelayMs}ms`);
        } else {
          // Success - reset error tracking
          this.handleSuccessResponse();
        }

        this.totalProcessed++;
        request.resolve(response);
      } catch (error) {
        // Network error or other exception
        this.totalErrors++;
        log(`[GeminiQueue] Request failed with error: ${error}`);
        request.reject(error instanceof Error ? error : new Error(String(error)));
      }
    }

    this.processing = false;
    log("[GeminiQueue] Worker stopped (queue empty)");
  }

  /**
   * Wait for the next available request slot
   * Enforces minimum delay between requests with exponential backoff for errors
   */
  private async waitForNextSlot(): Promise<void> {
    const now = Date.now();
    const timeSinceLastRequest = now - this.lastRequestTime;

    // Calculate delay with exponential backoff for consecutive errors
    let delayMs = this.minDelayMs;
    if (this.consecutiveErrors > 0) {
      // Exponential backoff: minDelayMs * (1 + consecutiveErrors * 0.5)
      const backoffMultiplier = 1 + this.consecutiveErrors * 0.5;
      delayMs = Math.min(this.minDelayMs * backoffMultiplier, this.maxDelayMs);
      log(`[GeminiQueue] Applying backoff (${this.consecutiveErrors} errors): ${delayMs}ms`);
    }

    // Wait if needed
    if (timeSinceLastRequest < delayMs) {
      const waitMs = delayMs - timeSinceLastRequest;
      log(`[GeminiQueue] Waiting ${waitMs}ms before next request`);
      await new Promise((resolve) => setTimeout(resolve, waitMs));
    }
  }

  /**
   * Handle rate limit response (429 error)
   * Parse quotaResetDelay and adjust delays accordingly
   */
  private handleRateLimitResponse(errorText: string): void {
    this.consecutiveErrors++;

    try {
      const errorData = JSON.parse(errorText);

      // Look for quotaResetDelay in error details
      // Format: "2.893149709s" or "3s"
      const quotaDetail = errorData?.error?.details?.find((d: any) => d.quotaResetDelay);
      if (quotaDetail?.quotaResetDelay) {
        const delayStr = quotaDetail.quotaResetDelay;
        const match = delayStr.match(/(\d+(?:\.\d+)?)/);
        if (match) {
          const delaySeconds = Number.parseFloat(match[1]);
          const suggestedDelayMs = Math.ceil(delaySeconds * 1000);

          // Use the larger of suggested delay or current delay
          this.minDelayMs = Math.max(suggestedDelayMs, this.minDelayMs, this.baseDelayMs);

          // Cap at maxDelayMs
          this.minDelayMs = Math.min(this.minDelayMs, this.maxDelayMs);

          log(
            `[GeminiQueue] Parsed quotaResetDelay: ${delayStr} (${suggestedDelayMs}ms), ` +
              `new minDelay: ${this.minDelayMs}ms`
          );
        }
      }
    } catch {
      // JSON parse failed, just increment error counter
      log("[GeminiQueue] Failed to parse rate limit response, using backoff");
    }

    // Apply exponential backoff
    const backoffMultiplier = 1 + this.consecutiveErrors * 0.5;
    this.minDelayMs = Math.min(this.baseDelayMs * backoffMultiplier, this.maxDelayMs);
  }

  /**
   * Handle successful response
   * Reset error counter and gradually reduce delay back to baseline
   */
  private handleSuccessResponse(): void {
    if (this.consecutiveErrors > 0) {
      log(`[GeminiQueue] Success after ${this.consecutiveErrors} errors, resetting counter`);
      this.consecutiveErrors = 0;
    }

    // Gradually reduce delay back to baseline
    if (this.minDelayMs > this.baseDelayMs) {
      this.minDelayMs = Math.max(
        this.baseDelayMs,
        this.minDelayMs * 0.9 // Reduce by 10%
      );
      log(`[GeminiQueue] Reducing delay to ${this.minDelayMs}ms`);
    }
  }

  /**
   * Get current queue statistics for monitoring
   */
  getStats(): QueueStats {
    return {
      queueLength: this.queue.length,
      processing: this.processing,
      consecutiveErrors: this.consecutiveErrors,
      currentDelayMs: this.minDelayMs,
      totalProcessed: this.totalProcessed,
      totalErrors: this.totalErrors,
    };
  }
}