Skip to content

Voice-to-text dictation app with local (Nvidia Parakeet/Whisper) and cloud models (BYOK). Privacy-first and available cross-platform.

32.7 KB
Raw Blame
const path = require("path");
const fs = require("fs");
const { promises: fsPromises } = require("fs");
const { app } = require("electron");
const {
  downloadFile: sharedDownloadFile,
  createDownloadSignal,
  cleanupStaleDownloads,
  checkDiskSpace,
} = require("./downloadUtils");

const modelRegistryData = require("../models/modelRegistryData.json");
const LlamaServerManager = require("./llamaServer");
const debugLogger = require("./debugLogger");
const { readGgufMetadataFromFile } = require("./ggufMetadata");
const {
  estimateTokens,
  kvBytesPerToken,
  resolveContextCeiling,
  resolveContextSize,
  BASELINE_CONTEXT_SIZE,
} = require("./llamaContextPolicy");

const MIN_FILE_SIZE = 1_000_000; // 1MB minimum for valid model files

// The context every server still STARTS at. Requests that need more grow it
// on demand, bounded by a per-machine, per-model ceiling (llamaContextPolicy).
// Keeping the starting value at the historical constant means prewarm, chat
// and every short request behave exactly as they did before #2142.
const SERVER_CONTEXT_SIZE = BASELINE_CONTEXT_SIZE;

// Slack on top of the output reservation, for tokenizer drift and llama.cpp's
// own bookkeeping.
const CONTEXT_RESERVE_TOKENS = 512;

// Context a request must leave free for its own answer. Derived in one place
// so the size the window is chosen for and the size it is checked against can
// never drift apart.
const reserveFor = (maxTokens) => maxTokens + CONTEXT_RESERVE_TOKENS;

// Only measure the prompt exactly when the estimate lands within this factor
// of the running window. Below that the answer cannot change, and dictation
// should not pay two HTTP round trips to learn nothing.
const PREFLIGHT_MEASURE_FACTOR = 2;

// The smallest answer still worth generating when the prompt has crowded the
// window. Matches the historical floor of calculateMaxTokens, so a request
// that gets trimmed to this is no smaller than the smallest one the app has
// ever made. Below it, refusing beats handing back a sentence and a half.
const MIN_OUTPUT_TOKENS = 512;

function getLocalProviders() {
  return modelRegistryData.localProviders || [];
}

class ModelError extends Error {
  constructor(message, code, details = {}) {
    super(message);
    this.name = "ModelError";
    this.code = code;
    this.details = details;
  }
}

class ModelNotFoundError extends ModelError {
  constructor(modelId) {
    super(`Model ${modelId} not found`, "MODEL_NOT_FOUND", { modelId });
  }
}

class ModelManager {
  constructor() {
    this.modelsDir = null;
    this.downloadProgress = new Map();
    this.activeDownloads = new Map();
    this.downloadReservations = new Map();
    this.activeRequests = new Map(); // Track HTTP requests for cancellation
    this.downloadLifecycleVersion = 0;
    this.serverManager = new LlamaServerManager();
    this.currentServerModelId = null;
    this._initialized = false;

    // IMPORTANT: Do NOT call app.getPath() here!
    // It can hang or fail before app.whenReady() in Electron 36+.
    // Initialization will happen on first use via ensureInitialized().
  }

  /**
   * Ensures the manager is initialized. Safe to call multiple times.
   * This must be called before any operation that requires modelsDir.
   */
  ensureInitialized() {
    if (this._initialized) return;

    // Check if app is ready before accessing app.getPath()
    if (!app.isReady()) {
      throw new Error(
        "ModelManager cannot be initialized before app.whenReady(). " +
          "This is a programming error - ensure ModelManager methods are only called after app is ready."
      );
    }

    this.modelsDir = this.getModelsDir();
    this._initialized = true;
    // Don't await - let this run in background
    this.ensureModelsDirExists();
    cleanupStaleDownloads(this.modelsDir);
  }

  getModelsDir() {
    const { getCacheRoot } = require("./modelDirUtils");
    return path.join(getCacheRoot(), "models");
  }

  async ensureModelsDirExists() {
    try {
      if (!this.modelsDir) {
        this.ensureInitialized();
      }
      await fsPromises.mkdir(this.modelsDir, { recursive: true });
    } catch (error) {
      console.error("Failed to create models directory:", error);
    }
  }

  async ensureLlamaCpp() {
    if (!this.serverManager.isAvailable()) {
      throw new ModelError(
        "llama-server binary not found. Please ensure the app is installed correctly.",
        "LLAMASERVER_NOT_FOUND"
      );
    }
    return true;
  }

  async getAllModels() {
    this.ensureInitialized();
    try {
      const modelEntries = [];

      for (const provider of getLocalProviders()) {
        for (const model of provider.models) {
          const modelPath = path.join(this.modelsDir, model.fileName);
          modelEntries.push({ model, provider, modelPath });
        }
      }

      let downloadedStates = [];

      // A download can finish between checking the final file and reading the
      // in-memory active state. Retry when that lifecycle changes so callers
      // never receive the impossible "not downloaded and not downloading" gap.
      for (let attempt = 0; attempt < 3; attempt += 1) {
        const lifecycleVersion = this.downloadLifecycleVersion;
        downloadedStates = await Promise.all(
          modelEntries.map(({ modelPath }) => this.checkModelValid(modelPath))
        );
        if (lifecycleVersion === this.downloadLifecycleVersion) break;
      }

      // Read volatile download state only after all asynchronous filesystem checks
      // finish so every model belongs to the same main-process snapshot.
      return modelEntries.map(({ model, provider, modelPath }, index) => {
        const progress = this.downloadProgress.get(model.id);
        const isDownloaded = downloadedStates[index];

        return {
          ...model,
          providerId: provider.id,
          providerName: provider.name,
          isDownloaded,
          isDownloading: this.activeDownloads.has(model.id),
          downloadProgress: progress?.progress || 0,
          downloadedSize: progress?.downloadedSize || 0,
          totalSize: progress?.totalSize || 0,
          path: isDownloaded ? modelPath : null,
        };
      });
    } catch (error) {
      console.error("[ModelManager] Error getting all models:", error);
      throw error;
    }
  }

  async getModelsWithStatus() {
    return this.getAllModels();
  }

  async isModelDownloaded(modelId) {
    this.ensureInitialized();
    const modelInfo = this.findModelById(modelId);
    if (!modelInfo) return false;

    const modelPath = path.join(this.modelsDir, modelInfo.model.fileName);
    return this.checkModelValid(modelPath);
  }

  async checkFileExists(filePath) {
    try {
      await fsPromises.access(filePath, fs.constants.F_OK);
      return true;
    } catch {
      return false;
    }
  }

  async checkModelValid(filePath) {
    try {
      const stats = await fsPromises.stat(filePath);
      return stats.size > MIN_FILE_SIZE;
    } catch {
      return false;
    }
  }

  findModelById(modelId) {
    for (const provider of getLocalProviders()) {
      const model = provider.models.find((m) => m.id === modelId);
      if (model) {
        return { model, provider };
      }
    }
    return null;
  }

  serverOptions(modelInfo) {
    return {
      contextSize: Math.min(
        SERVER_CONTEXT_SIZE,
        modelInfo.model.contextLength || SERVER_CONTEXT_SIZE
      ),
      threads: 4,
      gpuLayers: 99,
    };
  }

  async serverStartOptions(modelInfo, overrides = {}) {
    const options = { ...this.serverOptions(modelInfo), ...overrides };
    const draftPath = await this.resolveDraftPath(modelInfo.model);
    if (draftPath) options.draftModelPath = draftPath;
    return options;
  }

  // Seam so the ceiling can be exercised for any machine size in tests.
  _systemMemoryBytes() {
    return require("os").totalmem();
  }

  /**
   * The largest context this machine may give this model, and the prompt-cache
   * bound that goes with it. Memoized per model file: it reads a few MiB off
   * the front of the GGUF, so it must not sit on the hot path.
   */
  async contextCeiling(modelInfo, modelPath) {
    if (this._contextCeilingCache?.modelPath === modelPath) {
      return this._contextCeilingCache.value;
    }

    const metadata = await readGgufMetadataFromFile(modelPath);
    const sizeOf = async (filePath) => {
      if (!filePath) return 0;
      try {
        return (await fsPromises.stat(filePath)).size;
      } catch {
        return 0;
      }
    };

    const value = resolveContextCeiling({
      totalMemoryBytes: this._systemMemoryBytes(),
      weightsBytes: await sizeOf(modelPath),
      draftWeightsBytes: await sizeOf(await this.resolveDraftPath(modelInfo.model)),
      kvBytesPerToken: kvBytesPerToken(metadata),
      hasRecurrentState: metadata?.hasRecurrentState,
      // The GGUF is the file llama.cpp actually loads, so it outranks the
      // registry, which is wrong for at least the qwen2.5-7b entries.
      trainedContextTokens: metadata?.contextLength || modelInfo.model.contextLength || 0,
      // Anywhere but Apple Silicon the GPU may have its own memory that we do
      // not probe, so the system-RAM budget alone is not a safe bound. Stay
      // modest and let the GPU fallback ladder handle the rest. Intel Macs
      // count: a discrete Radeon has a few GB of its own, and the darwin start
      // path has no reduced-context rung to catch an over-sized window.
      discreteGpuUnverified: process.platform !== "darwin" || process.arch !== "arm64",
    });

    debugLogger.info("Resolved llama-server context ceiling", {
      model: modelInfo.model.id,
      architecture: metadata?.architecture ?? null,
      kvBytesPerToken: kvBytesPerToken(metadata),
      ...value,
    });

    this._contextCeilingCache = { modelPath, value };
    return value;
  }

  getReservedDownloadBytes() {
    let reserved = 0;
    for (const [modelId, bytes] of this.downloadReservations) {
      reserved += Math.max(0, bytes - (this.downloadProgress.get(modelId)?.downloadedSize || 0));
    }
    return reserved;
  }

  async downloadModel(modelId, onProgress) {
    this.ensureInitialized();
    const modelInfo = this.findModelById(modelId);
    if (!modelInfo) {
      throw new ModelNotFoundError(modelId);
    }

    const { model, provider } = modelInfo;
    const modelPath = path.join(this.modelsDir, model.fileName);

    if (await this.checkModelValid(modelPath)) {
      return modelPath;
    }

    if (this.activeDownloads.has(modelId)) {
      throw new ModelError("Model is already being downloaded", "DOWNLOAD_IN_PROGRESS", {
        modelId,
      });
    }

    this.activeDownloads.set(modelId, true);
    this.downloadLifecycleVersion += 1;
    const { signal, abort } = createDownloadSignal();
    this.activeRequests.set(modelId, { abort });

    try {
      await this.ensureModelsDirExists();

      const hasDrafter = this.modelHasDrafter(model);

      let requiredBytes = model.sizeBytes || model.sizeMb * 1_000_000 || 0;
      if (requiredBytes > 0 && hasDrafter) {
        requiredBytes += model.draftSizeBytes;
      }
      if (requiredBytes > 0) {
        this.downloadReservations.set(modelId, requiredBytes * 1.2);
        const reservedBytes = this.getReservedDownloadBytes();
        const spaceCheck = await checkDiskSpace(this.modelsDir, reservedBytes);
        if (!spaceCheck.ok) {
          throw new ModelError(
            `Not enough disk space. Need ~${Math.round(reservedBytes / 1_000_000)}MB, ` +
              `only ${Math.round(spaceCheck.availableBytes / 1_000_000)}MB available.`,
            "INSUFFICIENT_DISK_SPACE",
            { required: reservedBytes, available: spaceCheck.availableBytes }
          );
        }
      }

      const downloadUrl = this.getDownloadUrl(provider, model);

      // With a drafter, weight progress across both files by declared bytes and
      // clamp it so the bar never regresses across the phase boundary. Without a
      // drafter, keep today's single-file progress exactly.
      const combinedTotal = hasDrafter ? (model.sizeBytes || 0) + model.draftSizeBytes : 0;
      let lastCombined = 0;
      const emitCombined = (rawCombined) => {
        let combined = Math.min(rawCombined, combinedTotal);
        if (combined < lastCombined) combined = lastCombined;
        else lastCombined = combined;
        const progress = combinedTotal > 0 ? (combined / combinedTotal) * 100 : 0;
        this.downloadProgress.set(modelId, {
          modelId,
          progress,
          downloadedSize: combined,
          totalSize: combinedTotal,
        });
        if (onProgress) onProgress(progress, combined, combinedTotal);
      };

      await sharedDownloadFile(downloadUrl, modelPath, {
        signal,
        onProgress: hasDrafter
          ? (downloadedBytes) => emitCombined(downloadedBytes)
          : (downloadedBytes, totalBytes) => {
              const progress = totalBytes > 0 ? (downloadedBytes / totalBytes) * 100 : 0;
              this.downloadProgress.set(modelId, {
                modelId,
                progress,
                downloadedSize: downloadedBytes,
                totalSize: totalBytes,
              });
              if (onProgress) {
                onProgress(progress, downloadedBytes, totalBytes);
              }
            },
      });

      const stats = await fsPromises.stat(modelPath);
      if (stats.size < MIN_FILE_SIZE) {
        await fsPromises.unlink(modelPath).catch(() => {});
        throw new ModelError(
          "Downloaded file appears to be corrupted or incomplete",
          "DOWNLOAD_CORRUPTED",
          { size: stats.size, minSize: MIN_FILE_SIZE }
        );
      }

      // Drafter is opportunistic: a failure or cancel here leaves the main model
      // fully usable, so never fail the download or delete the main file.
      if (hasDrafter) {
        const draftPath = path.join(this.modelsDir, model.draftFileName);
        try {
          await sharedDownloadFile(this.getDraftDownloadUrl(provider, model), draftPath, {
            signal,
            onProgress: (downloadedBytes) => emitCombined(stats.size + downloadedBytes),
          });
          const draftStats = await fsPromises.stat(draftPath);
          if (draftStats.size < MIN_FILE_SIZE) {
            await fsPromises.unlink(draftPath).catch(() => {});
            debugLogger.warn("MTP drafter file too small, keeping model without it", { modelId });
          }
        } catch (draftError) {
          await fsPromises.unlink(draftPath).catch(() => {});
          if (draftError.isAbort) throw draftError;
          debugLogger.warn("MTP drafter download failed, keeping model without it", {
            modelId,
            error: draftError.message,
          });
        }
      }

      return modelPath;
    } catch (error) {
      if (error.isAbort) {
        throw new ModelError("Download cancelled by user", "DOWNLOAD_CANCELLED", { modelId });
      }
      if (error.isHttpError) {
        throw new ModelError(`Download failed with status ${error.statusCode}`, "DOWNLOAD_FAILED", {
          statusCode: error.statusCode,
        });
      }
      if (!(error instanceof ModelError)) {
        throw new ModelError(`Network error: ${error.message}`, "NETWORK_ERROR", {
          error: error.message,
        });
      }
      throw error;
    } finally {
      if (this.activeDownloads.delete(modelId)) {
        this.downloadLifecycleVersion += 1;
      }
      this.activeRequests.delete(modelId);
      this.downloadProgress.delete(modelId);
      this.downloadReservations.delete(modelId);
    }
  }

  getDownloadUrl(provider, model) {
    const baseUrl = provider.baseUrl || "https://huggingface.co";
    return `${baseUrl}/${model.hfRepo}/resolve/main/${model.fileName}`;
  }

  getDraftDownloadUrl(provider, model) {
    const baseUrl = provider.baseUrl || "https://huggingface.co";
    return `${baseUrl}/${model.draftHfRepo}/resolve/main/${model.draftFileName}`;
  }

  modelHasDrafter(model) {
    return Boolean(model && model.draftHfRepo && model.draftFileName && model.draftSizeBytes);
  }

  // Opportunistic MTP drafter path: only when declared and the file passes the
  // same >1MB validity gate as models. Returns null otherwise (start without MTP).
  async resolveDraftPath(model) {
    if (!this.modelHasDrafter(model)) return null;
    const draftPath = path.join(this.modelsDir, model.draftFileName);
    if (await this.checkModelValid(draftPath)) return draftPath;
    return null;
  }

  cancelDownload(modelId) {
    const entry = this.activeRequests.get(modelId);
    if (entry) {
      // Keep the guard and status visible until downloadModel's finally block
      // has finished cleaning up the writer and its temporary file.
      entry.abort();
      return true;
    }
    return false;
  }

  async deleteModel(modelId) {
    this.ensureInitialized();
    const modelInfo = this.findModelById(modelId);
    if (!modelInfo) {
      throw new ModelNotFoundError(modelId);
    }

    const modelPath = path.join(this.modelsDir, modelInfo.model.fileName);

    if (await this.checkFileExists(modelPath)) {
      await fsPromises.unlink(modelPath);
    }

    // Remove the drafter too when present, best effort (ignore ENOENT).
    if (modelInfo.model.draftFileName) {
      const draftPath = path.join(this.modelsDir, modelInfo.model.draftFileName);
      await fsPromises.unlink(draftPath).catch(() => {});
    }
  }

  async deleteAllModels() {
    this.ensureInitialized();
    try {
      if (fsPromises.rm) {
        await fsPromises.rm(this.modelsDir, { recursive: true, force: true });
      } else {
        const entries = await fsPromises
          .readdir(this.modelsDir, { withFileTypes: true })
          .catch(() => []);
        for (const entry of entries) {
          const fullPath = path.join(this.modelsDir, entry.name);
          if (entry.isDirectory()) {
            await fsPromises.rmdir(fullPath, { recursive: true }).catch(() => {});
          } else {
            await fsPromises.unlink(fullPath).catch(() => {});
          }
        }
      }
    } catch (error) {
      throw new ModelError(
        `Failed to delete models directory: ${error.message}`,
        "DELETE_ALL_ERROR",
        { error: error.message }
      );
    } finally {
      await this.ensureModelsDirExists();
    }
  }

  /**
   * Make sure the prompt actually fits the running window, growing it once if
   * it does not, trading output room for the prompt if it still does not, and
   * refusing only when no usable answer is left.
   *
   * The estimate above only picks a starting size. This is where the decision
   * becomes exact, because a wrong guess here is the bug: llama-server answers
   * an overlong prompt with a 400 whose JSON body used to reach the user.
   *
   * Returns the output allowance the request should actually be sent with.
   */
  async _ensurePromptFits({
    modelInfo,
    messages,
    options,
    estimatedTokens,
    maxTokens,
    contextFloor,
    ceilingFor,
    startServer,
  }) {
    const reserveTokens = reserveFor(maxTokens);
    const running = this.serverManager.contextSize || contextFloor;

    // Comfortably inside the window: measuring cannot change the outcome, and
    // every dictation would otherwise pay for two HTTP round trips.
    if ((estimatedTokens + reserveTokens) * PREFLIGHT_MEASURE_FACTOR <= running) return maxTokens;

    const usable = (await this.serverManager.usableContextSize()) ?? running;
    const exactTokens = await this.serverManager.countPromptTokens(messages, {
      disableThinking: options.disableThinking,
    });

    // A measurement we could not take must never fail the request; the
    // estimate already sized the window, and an overflow would still surface
    // as a typed error from llama-server itself.
    if (exactTokens === null) {
      debugLogger.warn("Could not measure prompt tokens; proceeding on the estimate", {
        model: modelInfo.model.id,
        estimatedTokens,
      });
      return maxTokens;
    }

    const neededTokens = exactTokens + reserveTokens;
    if (neededTokens <= usable) return maxTokens;

    const { ceiling: maxContext } = await ceilingFor();
    const grown = resolveContextSize({
      needed: neededTokens,
      floor: contextFloor,
      ceiling: maxContext,
    });

    if (grown > usable) await startServer(grown);

    const nowUsable =
      (await this.serverManager.usableContextSize()) ?? this.serverManager.contextSize ?? grown;
    if (neededTokens <= nowUsable) return maxTokens;

    // The window is as big as this machine allows and the prompt still does not
    // leave room for the answer we wanted. But llama-server rejects only a
    // prompt that fills the window on its own (`n_tokens >= n_ctx`) —
    // overrunning during generation just ends the reply with finish_reason
    // "length" — so a shorter answer is still a real answer, and refusing one
    // the user could have had is the regression this preflight would otherwise
    // introduce (#2142).
    //
    // No `- CONTEXT_RESERVE_TOKENS` here: that slack absorbs drift in the
    // *estimate* that sized the window, while this is an exact count of the
    // exact prompt. The MIN_OUTPUT_TOKENS floor below already keeps the prompt
    // clear of the edge by the same margin, so charging it twice would only
    // shorten the answer.
    // A caller that refuses window clips still gets no partial answer: a reply
    // that fills the trimmed allowance is refused after inference.
    const acceptsShorterOutput =
      !options.requireCompleteOutput || options.refuseClippedByWindow === true;
    const affordableOutput = nowUsable - exactTokens;
    if (acceptsShorterOutput && affordableOutput >= MIN_OUTPUT_TOKENS) {
      debugLogger.info("Trimming the output allowance to fit the context window", {
        model: modelInfo.model.id,
        requested: maxTokens,
        granted: affordableOutput,
        promptTokens: exactTokens,
        contextSize: nowUsable,
      });
      return affordableOutput;
    }

    // Either nothing usable is left, or the caller cannot accept a partial
    // answer at all (a selection edit replaces the user's own text, so half of
    // one is worse than none).
    throw new ModelError(
      `This content needs about ${neededTokens} tokens of context, but ${modelInfo.model.name} can only use ${nowUsable} on this computer.`,
      "CONTEXT_TOO_LARGE",
      {
        modelId: modelInfo.model.id,
        modelName: modelInfo.model.name,
        neededTokens,
        maxContextTokens: nowUsable,
      }
    );
  }

  async runInference(modelId, prompt, options = {}) {
    this.ensureInitialized();
    const startTime = Date.now();
    debugLogger.logReasoning("INFERENCE_START", {
      modelId,
      promptLength: prompt.length,
      options: { ...options, systemPrompt: options.systemPrompt ? "[set]" : "[not set]" },
    });

    // Ensure server is available
    if (!this.serverManager.isAvailable()) {
      debugLogger.logReasoning("INFERENCE_SERVER_NOT_AVAILABLE", {});
      throw new ModelError(
        "llama-server binary not found. Please ensure the app is installed correctly.",
        "LLAMASERVER_NOT_FOUND"
      );
    }

    const modelInfo = this.findModelById(modelId);
    if (!modelInfo) {
      debugLogger.logReasoning("INFERENCE_MODEL_NOT_FOUND", { modelId });
      throw new ModelNotFoundError(modelId);
    }

    const modelPath = path.join(this.modelsDir, modelInfo.model.fileName);
    debugLogger.logReasoning("INFERENCE_MODEL_PATH", {
      modelPath,
      modelName: modelInfo.model.name,
      providerId: modelInfo.provider.id,
    });

    if (!(await this.checkModelValid(modelPath))) {
      debugLogger.logReasoning("INFERENCE_MODEL_INVALID", { modelId, modelPath });
      throw new ModelError(
        `Model ${modelId} is not downloaded or is corrupted`,
        "MODEL_NOT_DOWNLOADED",
        { modelId }
      );
    }

    // Build messages for chat completion
    const messages = [
      { role: "system", content: options.systemPrompt || "" },
      { role: "user", content: prompt },
    ];

    const maxTokens = options.maxTokens ?? 512;
    // Whatever the prompt needs, plus room for the reply. The output reserve
    // is part of the context budget, not on top of it.
    const reserveTokens = reserveFor(maxTokens);
    const estimatedTokens = estimateTokens(options.systemPrompt || "") + estimateTokens(prompt);
    // A caller may ask for a larger floor (selection editing does); it can
    // raise the floor but never breach the ceiling.
    const contextFloor = Math.max(SERVER_CONTEXT_SIZE, options.contextSize || 0);

    let ceiling = null;
    const ceilingFor = async () => {
      if (ceiling === null) ceiling = await this.contextCeiling(modelInfo, modelPath);
      return ceiling;
    };

    let targetContextSize = contextFloor;
    // A raised floor has to clear the ceiling too, or a caller asking for a
    // window the machine cannot afford reopens #1203. The baseline floor needs
    // no check, which is what keeps the GGUF read off the short-request path.
    if (contextFloor > SERVER_CONTEXT_SIZE || estimatedTokens + reserveTokens > contextFloor) {
      const { ceiling: maxContext } = await ceilingFor();
      targetContextSize = resolveContextSize({
        needed: estimatedTokens + reserveTokens,
        floor: contextFloor,
        ceiling: maxContext,
      });
    }

    // The smallest window this machine has already refused for this model,
    // whether by failing to start or by quietly loading something smaller.
    // Retrying it would cost a second model load to learn the same thing.
    let smallestRefusedContextSize = Infinity;
    const refuse = (size) => {
      smallestRefusedContextSize = Math.min(smallestRefusedContextSize, size);
    };

    const startServer = async (contextSize) => {
      if (this.serverManager.ready && contextSize >= smallestRefusedContextSize) return;

      const { cacheRamMiB } = ceiling ?? {};
      // Only a live server already serving THIS model has a window worth
      // coming back to. contextSize outlives a crash — llamaServer clears it
      // in stop() alone, not when a process dies or a health check trips — and
      // another model's window says nothing about this one.
      const restorableContextSize =
        this.serverManager.ready &&
        this.serverManager.process &&
        this.currentServerModelId === modelId
          ? this.serverManager.contextSize
          : null;
      const startAt = async (size) =>
        this.serverManager.start(
          modelPath,
          await this.serverStartOptions(modelInfo, { contextSize: size, cacheRamMiB })
        );
      // llama.cpp's startup dump is the diagnostic, so it goes to the log here
      // and nowhere else: details crosses IPC to the renderer, which neither
      // reads it nor should carry a stderr blob and absolute model paths.
      const unavailable = (cause) => {
        debugLogger.error("llama-server could not be started", {
          model: modelId,
          contextSize,
          error: cause.message,
        });
        return new ModelError(
          `${modelInfo.model.name} could not be started on this computer.`,
          "LOCAL_SERVER_UNAVAILABLE",
          { modelId, modelName: modelInfo.model.name }
        );
      };

      try {
        await startAt(contextSize);
        // The GPU ladder can load a smaller window than it was asked for
        // without ever throwing. Recording that is what stops the preflight
        // asking for the same window again and paying a second model load to
        // be stepped down again.
        if (this.serverManager.contextSize < contextSize) refuse(contextSize);
      } catch (error) {
        refuse(contextSize);
        // start() stops the running server before it spawns, so a grow that
        // dies has already taken a working window with it. Come back at the
        // one that worked rather than leaving the user with no local model.
        if (!restorableContextSize || restorableContextSize >= contextSize) {
          throw unavailable(error);
        }

        debugLogger.warn("llama-server failed to grow; restoring the previous context", {
          model: modelId,
          attempted: contextSize,
          restoring: restorableContextSize,
          error: error.message,
        });
        try {
          await startAt(restorableContextSize);
        } catch (restoreError) {
          throw unavailable(restoreError);
        }
      }
      this.currentServerModelId = modelId;
    };

    // Start/restart the server if needed, if the model changed, or if this
    // request needs a bigger window than the running one has.
    if (
      !this.serverManager.ready ||
      this.currentServerModelId !== modelId ||
      targetContextSize > (this.serverManager.contextSize || SERVER_CONTEXT_SIZE)
    ) {
      debugLogger.logReasoning("INFERENCE_STARTING_SERVER", {
        currentModel: this.currentServerModelId,
        requestedModel: modelId,
        serverReady: this.serverManager.ready,
        contextSize: targetContextSize,
      });

      await startServer(targetContextSize);

      debugLogger.logReasoning("INFERENCE_SERVER_STARTED", {
        port: this.serverManager.port,
        model: modelId,
        contextSize: this.serverManager.contextSize,
      });
    }

    const grantedMaxTokens = await this._ensurePromptFits({
      modelInfo,
      messages,
      options,
      estimatedTokens,
      maxTokens,
      contextFloor,
      ceilingFor,
      startServer,
    });

    // The preflight shrank the reply to make room for the prompt. A reply that
    // fills that allowance was cut short by the window, not by the caller's cap,
    // and a caller that can split its material would rather be refused.
    const refuseWindowClip = options.refuseClippedByWindow === true && grantedMaxTokens < maxTokens;

    debugLogger.logReasoning("INFERENCE_SENDING_REQUEST", {
      messageCount: messages.length,
      systemPromptLength: (options.systemPrompt || "").length,
      userPromptLength: prompt.length,
    });

    try {
      const result = await this.serverManager.inference(messages, {
        temperature: options.temperature ?? 0.7,
        // The preflight may have traded output room for prompt room; sending
        // the original value would put the request back over the window.
        max_tokens: grantedMaxTokens,
        disableThinking: options.disableThinking,
        requireCompleteOutput: options.requireCompleteOutput || refuseWindowClip,
        responseFormat: options.responseFormat,
      });

      const totalTime = Date.now() - startTime;
      debugLogger.logReasoning("INFERENCE_SUCCESS", {
        totalTimeMs: totalTime,
        resultLength: result.length,
        resultPreview: result.substring(0, 200) + (result.length > 200 ? "..." : ""),
      });

      return result;
    } catch (error) {
      const totalTime = Date.now() - startTime;
      debugLogger.logReasoning("INFERENCE_FAILED", {
        totalTimeMs: totalTime,
        error: error.message,
      });
      if (refuseWindowClip && error.code === "OUTPUT_TRUNCATED") {
        throw new ModelError(
          `${modelInfo.model.name} ran out of context before the reply was complete.`,
          "CONTEXT_TOO_LARGE",
          { modelId, modelName: modelInfo.model.name, neededTokens: null, maxContextTokens: null }
        );
      }
      // A typed failure (a context overflow, say) must keep its identity, or
      // the renderer cannot translate it and the user sees raw server text.
      if (
        ["CONTEXT_TOO_LARGE", "OUTPUT_TRUNCATED", "OUTPUT_COMPLETION_UNVERIFIED"].includes(
          error.code
        )
      ) {
        throw new ModelError(error.message, error.code, {
          modelId,
          modelName: modelInfo.model.name,
          neededTokens: error.neededTokens ?? null,
          maxContextTokens: error.maxContextTokens ?? null,
        });
      }
      throw new ModelError(`Inference failed: ${error.message}`, "INFERENCE_FAILED", {
        error: error.message,
      });
    }
  }

  async stopServer() {
    await this.serverManager.stop();
    this.currentServerModelId = null;
  }

  getServerStatus() {
    return this.serverManager.getStatus();
  }

  cancelInference() {
    this.serverManager.cancelInference();
  }

  async prewarmServer(modelId) {
    if (!modelId) return false;
    this.ensureInitialized();

    const modelInfo = this.findModelById(modelId);
    if (!modelInfo) return false;

    const modelPath = path.join(this.modelsDir, modelInfo.model.fileName);
    if (!(await this.checkModelValid(modelPath))) return false;

    if (!this.serverManager.isAvailable()) return false;

    try {
      await this.serverManager.start(modelPath, await this.serverStartOptions(modelInfo));
      this.currentServerModelId = modelId;
      debugLogger.info("llama-server pre-warmed", { modelId });
      return true;
    } catch (error) {
      debugLogger.warn("Failed to pre-warm llama-server", { error: error.message });
      return false;
    }
  }
}

module.exports = {
  default: new ModelManager(),
  ModelError,
  ModelNotFoundError,
};