Skip to content

Claude Code. Any Model. The most powerful AI coding agent now speaks every language.

13.0 KB
Raw Blame
/**
 * Local Model Request Queue
 *
 * Singleton queue for controlling concurrency to local models (Ollama, LM Studio, vLLM, MLX, etc.)
 * to prevent GPU overload. Implements configurable parallelism with FIFO ordering.
 *
 * Unlike the OpenRouter queue which focuses on rate limiting (429 errors), this queue
 * focuses on concurrency control to prevent GPU memory exhaustion.
 *
 * All local model requests are processed through this queue with:
 * - Configurable max parallel requests (default 1 = sequential)
 * - FIFO ordering for fairness
 * - OOM error detection and retry logic
 * - Automatic queue size management (max 100 requests)
 * - Minimal delay between dispatches (100ms)
 *
 * New: Concurrency can be specified per-model using the model syntax:
 *   [email protected]:3    - Allow 3 concurrent requests
 *   [email protected]:0    - Unlimited concurrency (bypass queue)
 *
 * Environment variables:
 * - CLAUDISH_LOCAL_MAX_PARALLEL: Max concurrent requests (1-8, default: 1)
 * - CLAUDISH_LOCAL_QUEUE_ENABLED: Enable/disable queue (default: true)
 */

import { getLogLevel, log } from "../../logger.js";

/**
 * Queued request with Promise callbacks
 */
interface QueuedRequest {
  fetchFn: () => Promise<Response>;
  resolve: (response: Response) => void;
  reject: (error: Error) => void;
  providerId: string; // For debugging/stats (e.g., "ollama", "lmstudio")
  /**
   * Set when the caller can be cancelled while still QUEUED — a connection
   * retry bounded by a per-attempt clamp, or a client that went away.
   *
   * This is the only queue whose ACTIVE request may legally run for ten
   * minutes (local inference), so it is the only one where an admission wait
   * can outlive the caller's own budget. Without it an aborted attempt's entry
   * stayed in the queue and its `fetchFn` fired later, against a provider
   * nobody was waiting on.
   */
  signal?: AbortSignal;
  /** Detaches the abort listener. Always called, on every exit path. */
  detach?: () => void;
  /** True once the abort listener has rejected this entry. */
  abandoned?: boolean;
}

/** Normalise an abort reason into an Error the queue's `reject` accepts. */
function toAbortError(reason: unknown): Error {
  if (reason instanceof Error) return reason;
  const err = new Error("Request aborted while queued");
  err.name = "AbortError";
  return err;
}

/**
 * Queue statistics for monitoring
 */
export interface QueueStats {
  queueLength: number;
  activeRequests: number;
  maxParallel: number;
  totalProcessed: number;
  totalErrors: number;
  totalOOMErrors: number;
}

/**
 * Singleton request queue for local models
 *
 * Implements concurrency control to prevent GPU overload by limiting
 * the number of simultaneous requests to local models.
 *
 * Concurrency can be overridden per-model using the :N suffix in model spec:
 * - :0 = bypass queue entirely (unlimited)
 * - :N = override max parallel to N for this model
 *
 * @example
 * ```typescript
 * const queue = LocalModelQueue.getInstance();
 * const response = await queue.enqueue(() => fetch(url, options), "ollama");
 *
 * // With custom concurrency (bypasses default)
 * const response = await queue.enqueue(() => fetch(url, options), "ollama", 3);
 * ```
 */
export class LocalModelQueue {
  private static instance: LocalModelQueue | null = null;
  private queue: QueuedRequest[] = [];
  private activeRequests = 0;

  // Configuration
  private readonly defaultMaxParallel: number; // From CLAUDISH_LOCAL_MAX_PARALLEL
  private maxParallel: number; // Current effective max (can be overridden)
  private readonly maxQueueSize = 100;
  private readonly requestDelay = 100; // Small delay between dispatches (ms)

  // Statistics
  private totalProcessed = 0;
  private totalErrors = 0;
  private totalOOMErrors = 0;

  private constructor() {
    this.defaultMaxParallel = this.getMaxParallelFromEnv();
    this.maxParallel = this.defaultMaxParallel;
    if (getLogLevel() === "debug") {
      log(
        `[LocalQueue] Queue initialized with maxParallel=${this.maxParallel}, maxQueueSize=${this.maxQueueSize}`
      );
    }
  }

  /**
   * Get singleton instance
   */
  static getInstance(): LocalModelQueue {
    if (!LocalModelQueue.instance) {
      LocalModelQueue.instance = new LocalModelQueue();
    }
    return LocalModelQueue.instance;
  }

  /**
   * Check if queue is enabled via environment variable
   */
  static isEnabled(): boolean {
    const enabled = process.env.CLAUDISH_LOCAL_QUEUE_ENABLED;
    if (enabled === undefined || enabled === "") return true; // Default: enabled
    return enabled !== "false" && enabled !== "0";
  }

  /**
   * Enqueue a request to be processed
   *
   * @param fetchFn - Function that performs the fetch request
   * @param providerId - Provider identifier for debugging (e.g., "ollama", "lmstudio")
   * @param concurrencyOverride - Optional concurrency override from model spec
   *   - undefined: use default max parallel
   *   - 0: bypass queue entirely (direct execution)
   *   - N: use N as max parallel for this request
   * @returns Promise that resolves with the response
   * @throws Error if queue is full
   */
  async enqueue(
    fetchFn: () => Promise<Response>,
    providerId: string,
    concurrencyOverride?: number,
    signal?: AbortSignal
  ): Promise<Response> {
    // Already cancelled before we even got here: never occupy a slot for it.
    if (signal?.aborted) {
      throw toAbortError(signal.reason);
    }
    // Handle concurrency override
    if (concurrencyOverride !== undefined) {
      if (concurrencyOverride === 0) {
        // :0 means bypass queue entirely - execute directly
        if (getLogLevel() === "debug") {
          log(`[LocalQueue] Bypassing queue for ${providerId} (concurrency=0)`);
        }
        return fetchFn();
      }

      // Override max parallel for this session
      if (concurrencyOverride !== this.maxParallel && concurrencyOverride > 0) {
        const newMax = Math.min(concurrencyOverride, 8); // Cap at 8
        if (getLogLevel() === "debug") {
          log(
            `[LocalQueue] Overriding maxParallel: ${this.maxParallel} -> ${newMax} for ${providerId}`
          );
        }
        this.maxParallel = newMax;
      }
    }

    // Check queue size limit
    if (this.queue.length >= this.maxQueueSize) {
      if (getLogLevel() === "debug") {
        log(
          `[LocalQueue] Queue full (${this.queue.length}/${this.maxQueueSize}), rejecting request`
        );
      }
      throw new Error(
        `Local model queue full (${this.queue.length}/${this.maxQueueSize}). GPU is overloaded. Please wait for current requests to complete.`
      );
    }

    // Create promise for this request
    return new Promise<Response>((resolve, reject) => {
      const queuedRequest: QueuedRequest = {
        fetchFn,
        resolve,
        reject,
        providerId,
        signal,
      };

      if (signal) {
        const onAbort = () => {
          queuedRequest.abandoned = true;
          queuedRequest.detach?.();
          queuedRequest.detach = undefined;
          // Splice it OUT. Leaving it in and merely rejecting the promise
          // still lets processQueue invoke its fetchFn later, firing a request
          // whose caller is already gone.
          const i = this.queue.indexOf(queuedRequest);
          if (i !== -1) this.queue.splice(i, 1);
          reject(toAbortError(signal.reason));
        };
        queuedRequest.detach = () => signal.removeEventListener("abort", onAbort);
        signal.addEventListener("abort", onAbort, { once: true });
      }

      this.queue.push(queuedRequest);
      if (getLogLevel() === "debug") {
        log(
          `[LocalQueue] Request enqueued for ${providerId} (queue length: ${this.queue.length}, active: ${this.activeRequests}/${this.maxParallel})`
        );
      }

      // Start processing queue if there are available slots
      this.processQueue();
    });
  }

  /**
   * Worker loop that processes queued requests with concurrency control
   * Processes requests while:
   * 1. Queue has items
   * 2. Active requests < maxParallel
   */
  private async processQueue(): Promise<void> {
    // Process requests while queue has items AND slots available
    while (this.queue.length > 0 && this.activeRequests < this.maxParallel) {
      const request = this.queue.shift();
      if (!request) break;

      // Aborted while it waited. The listener has already rejected the caller;
      // all that is left is to not run its fetchFn.
      if (request.abandoned || request.signal?.aborted) {
        request.detach?.();
        request.detach = undefined;
        continue;
      }

      if (getLogLevel() === "debug") {
        log(
          `[LocalQueue] Processing request for ${request.providerId} (${this.queue.length} remaining in queue, ${this.activeRequests + 1}/${this.maxParallel} active)`
        );
      }

      // Execute in parallel (don't await here) to allow concurrent processing
      this.executeRequest(request).catch((err) => {
        if (getLogLevel() === "debug") {
          log(`[LocalQueue] Request execution failed: ${err}`);
        }
      });

      // Small delay between dispatches to avoid race conditions
      await this.delay(this.requestDelay);
    }
  }

  /**
   * Execute a single request with OOM error handling
   */
  private async executeRequest(request: QueuedRequest): Promise<void> {
    this.activeRequests++;
    // Past admission, the caller's signal is the fetch's business, not the
    // queue's — the fetch was handed the same signal and will end itself.
    request.detach?.();
    request.detach = undefined;

    try {
      const response = await request.fetchFn();

      // Check for OOM error (GPU out of memory)
      if (response.status === 500) {
        const errorBody = await response.clone().text();
        if (this.isOOMError(errorBody)) {
          this.totalOOMErrors++;
          if (getLogLevel() === "debug") {
            log(
              `[LocalQueue] GPU out-of-memory detected for ${request.providerId}. Consider reducing CLAUDISH_LOCAL_MAX_PARALLEL (current: ${this.maxParallel})`
            );
          }

          // Retry once after a delay
          await this.delay(2000); // 2-second delay before retry
          const retryResponse = await request.fetchFn();

          // Check retry response
          if (retryResponse.status === 500) {
            const retryErrorBody = await retryResponse.clone().text();
            if (this.isOOMError(retryErrorBody)) {
              // OOM persisted after retry - fail with helpful message
              throw new Error(
                "GPU out-of-memory error persisted after retry. Try setting CLAUDISH_LOCAL_MAX_PARALLEL=1 for sequential processing."
              );
            }
          }

          // Retry succeeded
          this.totalProcessed++;
          request.resolve(retryResponse);
          return;
        }
      }

      // Success (no OOM)
      this.totalProcessed++;
      request.resolve(response);
    } catch (error) {
      // Network error or other exception
      this.totalErrors++;
      if (getLogLevel() === "debug") {
        log(`[LocalQueue] Request failed for ${request.providerId}: ${error}`);
      }
      request.reject(error instanceof Error ? error : new Error(String(error)));
    } finally {
      this.activeRequests--;

      // Trigger next batch if queue still has items
      if (this.queue.length > 0) {
        this.processQueue();
      }
    }
  }

  /**
   * Detect GPU out-of-memory errors from response body
   * Checks for common OOM error messages from various providers
   */
  private isOOMError(errorBody: string): boolean {
    const oomPatterns = [
      "failed to allocate memory",
      "CUDA out of memory",
      "OOM",
      "out of memory",
      "memory allocation failed",
      "insufficient memory",
      "GPU memory",
    ];

    const bodyLower = errorBody.toLowerCase();
    return oomPatterns.some((pattern) => bodyLower.includes(pattern.toLowerCase()));
  }

  /**
   * Read and validate CLAUDISH_LOCAL_MAX_PARALLEL environment variable
   * Returns max parallel requests (1-8 range, default: 1)
   */
  private getMaxParallelFromEnv(): number {
    const envValue = process.env.CLAUDISH_LOCAL_MAX_PARALLEL;
    if (!envValue) return 1; // Default: sequential

    const parsed = Number.parseInt(envValue, 10);
    if (Number.isNaN(parsed) || parsed < 1) {
      log(`[LocalQueue] Invalid CLAUDISH_LOCAL_MAX_PARALLEL: ${envValue}, using default: 1`);
      return 1;
    }

    if (parsed > 8) {
      log(`[LocalQueue] CLAUDISH_LOCAL_MAX_PARALLEL too high: ${parsed}, capping at 8`);
      return 8;
    }

    return parsed;
  }

  /**
   * Utility: delay for specified milliseconds
   */
  private delay(ms: number): Promise<void> {
    return new Promise((resolve) => setTimeout(resolve, ms));
  }

  /**
   * Get current queue statistics for monitoring
   */
  getStats(): QueueStats {
    return {
      queueLength: this.queue.length,
      activeRequests: this.activeRequests,
      maxParallel: this.maxParallel,
      totalProcessed: this.totalProcessed,
      totalErrors: this.totalErrors,
      totalOOMErrors: this.totalOOMErrors,
    };
  }
}