Skip to content

Claude Code. Any Model. The most powerful AI coding agent now speaks every language.

16.4 KB
Raw Blame
/**
 * OpenRouter Request Queue
 *
 * Singleton request queue for serializing OpenRouter API requests to prevent rate limit exhaustion.
 * Implements dynamic rate limiting based on OpenRouter rate limit headers and 429 error responses.
 *
 * All OpenRouter requests are processed sequentially through a FIFO queue with:
 * - Minimum delay between requests (default 1000ms = 60 req/min)
 * - Dynamic delay adjustment based on rate limit headers
 * - Proactive throttling when quota is low
 * - Exponential backoff for consecutive errors
 * - Automatic queue size management (max 100 requests)
 *
 * Rate limit headers parsed:
 * - X-RateLimit-Limit-Requests: Total requests allowed
 * - X-RateLimit-Remaining-Requests: Remaining requests in current window
 * - X-RateLimit-Reset-Requests: Unix timestamp when limit resets
 * - X-RateLimit-Remaining-Tokens: Remaining tokens in current window
 * - Retry-After: Seconds to wait after 429 error
 */

import { getLogLevel, log } from "../../logger.js";
import { callAbortError } from "./call-context.js";

/**
 * Queued request with Promise callbacks
 */
interface QueuedRequest {
  fetchFn: () => Promise<Response>;
  resolve: (response: Response) => void;
  reject: (error: Error) => void;
  /** The caller's abort signal, when it passed one. */
  signal?: AbortSignal;
  /** Removes this entry from the queue on abort, while it is still waiting. */
  onAbort?: () => void;
}

/**
 * Rate limit state tracked from response headers
 */
interface RateLimitState {
  // From response headers
  limitRequests: number | null;
  limitTokens: number | null;
  remainingRequests: number | null;
  remainingTokens: number | null;
  resetTime: number | null; // Unix timestamp (seconds)

  // Internal tracking
  lastRequestTime: number;
  consecutiveErrors: number;
  currentDelayMs: number;

  // Statistics
  totalProcessed: number;
  totalErrors: number;
  total429Errors: number;
}

/**
 * Called by the worker once it has taken an entry: the entry no longer waits in
 * the queue, so its abort listener goes, and an entry whose caller already
 * aborted is rejected instead of started. True when it was rejected.
 */
function rejectIfAborted(request: QueuedRequest): boolean {
  if (request.onAbort) request.signal?.removeEventListener("abort", request.onAbort);
  if (!request.signal?.aborted) return false;
  request.reject(callAbortError());
  return true;
}

/**
 * Queue statistics for monitoring
 */
export interface QueueStats {
  queueLength: number;
  processing: boolean;
  consecutiveErrors: number;
  currentDelayMs: number;
  totalProcessed: number;
  totalErrors: number;
  total429Errors: number;
  remainingRequests: number | null;
  remainingTokens: number | null;
  resetTime: number | null;
}

/**
 * Singleton request queue for OpenRouter API
 *
 * Serializes all OpenRouter requests to prevent rate limit exhaustion.
 * Implements dynamic rate limiting based on response headers and 429 errors.
 *
 * @example
 * ```typescript
 * const queue = OpenRouterRequestQueue.getInstance();
 * const response = await queue.enqueue(() => fetch(url, options));
 * ```
 */
export class OpenRouterRequestQueue {
  private static instance: OpenRouterRequestQueue | null = null;
  private queue: QueuedRequest[] = [];
  private processing = false;

  // Rate limit state
  private rateLimitState: RateLimitState = {
    limitRequests: null,
    limitTokens: null,
    remainingRequests: null,
    remainingTokens: null,
    resetTime: null,
    lastRequestTime: 0,
    consecutiveErrors: 0,
    currentDelayMs: 1000,
    totalProcessed: 0,
    totalErrors: 0,
    total429Errors: 0,
  };

  // Configuration constants
  private readonly baseDelayMs = 1000; // 60 req/min
  private readonly maxDelayMs = 10000; // Max 10s delay
  private readonly maxQueueSize = 100;

  private constructor() {
    if (getLogLevel() === "debug") {
      log("[OpenRouterQueue] Queue initialized with baseDelay=1000ms, maxQueueSize=100");
    }
  }

  /**
   * Get singleton instance
   */
  static getInstance(): OpenRouterRequestQueue {
    if (!OpenRouterRequestQueue.instance) {
      OpenRouterRequestQueue.instance = new OpenRouterRequestQueue();
    }
    return OpenRouterRequestQueue.instance;
  }

  /**
   * Enqueue a request to be processed
   *
   * @param fetchFn - Function that performs the fetch request
   * @returns Promise that resolves with the response
   * @throws Error if queue is full
   */
  async enqueue(
    fetchFn: () => Promise<Response>,
    opts?: { signal?: AbortSignal }
  ): Promise<Response> {
    const signal = opts?.signal;
    if (signal?.aborted) throw callAbortError();
    // Check queue size limit
    if (this.queue.length >= this.maxQueueSize) {
      if (getLogLevel() === "debug") {
        log(
          `[OpenRouterQueue] Queue full (${this.queue.length}/${this.maxQueueSize}), rejecting request`
        );
      }
      throw new Error(
        `OpenRouter request queue full (${this.queue.length}/${this.maxQueueSize}). The API is rate-limited. Please wait and try again.`
      );
    }

    // Create promise for this request
    return new Promise<Response>((resolve, reject) => {
      const queuedRequest: QueuedRequest = {
        fetchFn,
        resolve,
        reject,
        signal,
      };
      // An abort while the entry still WAITS removes it and rejects at once:
      // nothing of the call starts after its deadline, and it stops holding a
      // place in line. Once the worker has taken it, the worker checks again.
      if (signal) {
        queuedRequest.onAbort = () => {
          const index = this.queue.indexOf(queuedRequest);
          if (index === -1) return;
          this.queue.splice(index, 1);
          reject(callAbortError());
        };
        signal.addEventListener("abort", queuedRequest.onAbort, { once: true });
      }

      this.queue.push(queuedRequest);
      if (getLogLevel() === "debug") {
        log(`[OpenRouterQueue] Request enqueued (queue length: ${this.queue.length})`);
      }

      // Start processing if not already running
      if (!this.processing) {
        this.processQueue();
      }
    });
  }

  /**
   * Worker loop that processes queued requests sequentially
   */
  private async processQueue(): Promise<void> {
    if (this.processing) {
      return; // Already processing
    }

    this.processing = true;
    if (getLogLevel() === "debug") {
      log("[OpenRouterQueue] Worker started");
    }

    while (this.queue.length > 0) {
      const request = this.queue.shift();
      if (!request) break;
      if (rejectIfAborted(request)) continue;

      if (getLogLevel() === "debug") {
        log(`[OpenRouterQueue] Processing request (${this.queue.length} remaining in queue)`);
      }

      try {
        // Wait for next available slot
        await this.waitForNextSlot();
        // Aborted during the rate-limit wait: never start it.
        if (rejectIfAborted(request)) continue;

        // Execute the request
        const response = await request.fetchFn();
        this.rateLimitState.lastRequestTime = Date.now();

        // Parse rate limit headers
        this.parseRateLimitHeaders(response);

        // Check for rate limit response
        if (response.status === 429) {
          this.rateLimitState.totalErrors++;
          this.rateLimitState.total429Errors++;
          await this.handleRateLimitError(response);
          if (getLogLevel() === "debug") {
            log(
              `[OpenRouterQueue] Rate limit hit (429), adjusted delay to ${this.rateLimitState.currentDelayMs}ms`
            );
          }
        } else {
          // Success - reset error tracking
          this.handleSuccessResponse();
        }

        this.rateLimitState.totalProcessed++;
        request.resolve(response);
      } catch (error) {
        // Network error or other exception
        this.rateLimitState.totalErrors++;
        this.rateLimitState.consecutiveErrors++;
        if (getLogLevel() === "debug") {
          log(`[OpenRouterQueue] Request failed with error: ${error}`);
        }
        request.reject(error instanceof Error ? error : new Error(String(error)));
      }
    }

    this.processing = false;
    if (getLogLevel() === "debug") {
      log("[OpenRouterQueue] Worker stopped (queue empty)");
    }
  }

  /**
   * Wait for the next available request slot
   * Enforces minimum delay between requests with dynamic adjustment
   */
  private async waitForNextSlot(): Promise<void> {
    const now = Date.now();
    const timeSinceLastRequest = now - this.rateLimitState.lastRequestTime;

    // Calculate delay based on current state
    const delayMs = this.calculateDelay();
    this.rateLimitState.currentDelayMs = delayMs;

    // Wait if needed
    if (timeSinceLastRequest < delayMs) {
      const waitMs = delayMs - timeSinceLastRequest;
      if (getLogLevel() === "debug") {
        log(`[OpenRouterQueue] Waiting ${waitMs}ms before next request`);
      }
      await new Promise((resolve) => setTimeout(resolve, waitMs));
    }
  }

  /**
   * Calculate dynamic delay based on rate limit state
   * Considers remaining quota, reset time, and error backoff
   */
  private calculateDelay(): number {
    let delayMs = this.baseDelayMs;

    // Factor 1: Remaining requests (proactive throttling)
    if (
      this.rateLimitState.remainingRequests !== null &&
      this.rateLimitState.limitRequests !== null &&
      this.rateLimitState.limitRequests > 0
    ) {
      const quotaPercent =
        this.rateLimitState.remainingRequests / this.rateLimitState.limitRequests;
      if (quotaPercent < 0.2) {
        // Less than 20% quota remaining - slow down significantly
        delayMs = Math.max(delayMs, 3000);
        if (getLogLevel() === "debug") {
          log(
            `[OpenRouterQueue] Low quota (${(quotaPercent * 100).toFixed(1)}%), increasing delay to ${delayMs}ms`
          );
        }
      } else if (quotaPercent < 0.5) {
        // Less than 50% quota remaining - moderate slowdown
        delayMs = Math.max(delayMs, 2000);
        if (getLogLevel() === "debug") {
          log(
            `[OpenRouterQueue] Medium quota (${(quotaPercent * 100).toFixed(1)}%), increasing delay to ${delayMs}ms`
          );
        }
      }
    }

    // Factor 2: Time until reset (spread requests evenly)
    if (this.rateLimitState.resetTime !== null && this.rateLimitState.remainingRequests !== null) {
      const now = Date.now() / 1000; // Convert to Unix timestamp
      const timeUntilReset = this.rateLimitState.resetTime - now;
      if (timeUntilReset > 0 && this.rateLimitState.remainingRequests > 0) {
        // Spread remaining requests evenly until reset
        const optimalDelay =
          (timeUntilReset * 1000) / Math.max(this.rateLimitState.remainingRequests, 1);
        delayMs = Math.max(delayMs, Math.min(optimalDelay, this.maxDelayMs));
        if (getLogLevel() === "debug") {
          log(
            `[OpenRouterQueue] Spreading ${this.rateLimitState.remainingRequests} requests ` +
              `over ${timeUntilReset.toFixed(1)}s, optimal delay: ${optimalDelay.toFixed(0)}ms`
          );
        }
      }
    }

    // Factor 3: Consecutive errors (exponential backoff)
    if (this.rateLimitState.consecutiveErrors > 0) {
      const backoffMultiplier = 1 + this.rateLimitState.consecutiveErrors * 0.5;
      delayMs = delayMs * backoffMultiplier;
      if (getLogLevel() === "debug") {
        log(
          `[OpenRouterQueue] Applying backoff (${this.rateLimitState.consecutiveErrors} errors): ${delayMs.toFixed(0)}ms`
        );
      }
    }

    // Cap at maximum
    return Math.min(delayMs, this.maxDelayMs);
  }

  /**
   * Parse rate limit headers from response
   * Updates internal rate limit state
   */
  private parseRateLimitHeaders(response: Response): void {
    // Parse request limits
    const limitRequests = response.headers.get("X-RateLimit-Limit-Requests");
    if (limitRequests) {
      this.rateLimitState.limitRequests = Number.parseInt(limitRequests, 10);
    }

    const remainingRequests = response.headers.get("X-RateLimit-Remaining-Requests");
    if (remainingRequests) {
      this.rateLimitState.remainingRequests = Number.parseInt(remainingRequests, 10);
    }

    const resetRequests = response.headers.get("X-RateLimit-Reset-Requests");
    if (resetRequests) {
      this.rateLimitState.resetTime = Number.parseFloat(resetRequests);
    }

    // Parse token limits
    const limitTokens = response.headers.get("X-RateLimit-Limit-Tokens");
    if (limitTokens) {
      this.rateLimitState.limitTokens = Number.parseInt(limitTokens, 10);
    }

    const remainingTokens = response.headers.get("X-RateLimit-Remaining-Tokens");
    if (remainingTokens) {
      this.rateLimitState.remainingTokens = Number.parseInt(remainingTokens, 10);
    }

    // Debug log headers
    if (getLogLevel() === "debug") {
      const headers = {
        limitRequests: this.rateLimitState.limitRequests,
        remainingRequests: this.rateLimitState.remainingRequests,
        resetTime: this.rateLimitState.resetTime
          ? new Date(this.rateLimitState.resetTime * 1000).toISOString()
          : null,
        limitTokens: this.rateLimitState.limitTokens,
        remainingTokens: this.rateLimitState.remainingTokens,
      };
      log(`[OpenRouterQueue] Rate limit headers: ${JSON.stringify(headers)}`);
    }
  }

  /**
   * Handle 429 rate limit error
   * Parse Retry-After header and apply exponential backoff
   */
  private async handleRateLimitError(response: Response): Promise<void> {
    this.rateLimitState.consecutiveErrors++;

    // Set remaining requests to 0 (quota exhausted)
    this.rateLimitState.remainingRequests = 0;

    // Parse Retry-After header (seconds to wait)
    const retryAfter = response.headers.get("Retry-After");
    if (retryAfter) {
      const retryAfterSeconds = Number.parseInt(retryAfter, 10);
      if (!Number.isNaN(retryAfterSeconds)) {
        const retryAfterMs = retryAfterSeconds * 1000;
        this.rateLimitState.currentDelayMs = Math.min(retryAfterMs, this.maxDelayMs);
        if (getLogLevel() === "debug") {
          log(`[OpenRouterQueue] Retry-After header: ${retryAfterSeconds}s (${retryAfterMs}ms)`);
        }
      }
    }

    // Try to parse error response body for additional info
    try {
      const errorText = await response.clone().text();
      const errorData = JSON.parse(errorText);
      if (errorData?.error?.message) {
        if (getLogLevel() === "debug") {
          log(`[OpenRouterQueue] 429 error message: ${errorData.error.message}`);
        }
      }
    } catch {
      // Ignore JSON parse errors
    }

    // Apply exponential backoff
    const backoffMultiplier = 1 + this.rateLimitState.consecutiveErrors * 0.5;
    const backoffDelay = Math.min(this.baseDelayMs * backoffMultiplier, this.maxDelayMs);
    this.rateLimitState.currentDelayMs = Math.max(this.rateLimitState.currentDelayMs, backoffDelay);

    if (getLogLevel() === "debug") {
      log(
        `[OpenRouterQueue] Applied exponential backoff: ${this.rateLimitState.currentDelayMs}ms ` +
          `(${this.rateLimitState.consecutiveErrors} consecutive errors)`
      );
    }
  }

  /**
   * Handle successful response
   * Reset error counter and gradually reduce delay back to baseline
   */
  private handleSuccessResponse(): void {
    if (this.rateLimitState.consecutiveErrors > 0) {
      if (getLogLevel() === "debug") {
        log(
          `[OpenRouterQueue] Success after ${this.rateLimitState.consecutiveErrors} errors, resetting counter`
        );
      }
      this.rateLimitState.consecutiveErrors = 0;
    }

    // Gradually reduce delay back to baseline
    if (this.rateLimitState.currentDelayMs > this.baseDelayMs) {
      this.rateLimitState.currentDelayMs = Math.max(
        this.baseDelayMs,
        this.rateLimitState.currentDelayMs * 0.9 // Reduce by 10%
      );
      if (getLogLevel() === "debug") {
        log(`[OpenRouterQueue] Reducing delay to ${this.rateLimitState.currentDelayMs}ms`);
      }
    }
  }

  /**
   * Get current queue statistics for monitoring
   */
  getStats(): QueueStats {
    return {
      queueLength: this.queue.length,
      processing: this.processing,
      consecutiveErrors: this.rateLimitState.consecutiveErrors,
      currentDelayMs: this.rateLimitState.currentDelayMs,
      totalProcessed: this.rateLimitState.totalProcessed,
      totalErrors: this.rateLimitState.totalErrors,
      total429Errors: this.rateLimitState.total429Errors,
      remainingRequests: this.rateLimitState.remainingRequests,
      remainingTokens: this.rateLimitState.remainingTokens,
      resetTime: this.rateLimitState.resetTime,
    };
  }
}