Skip to content

Voice-to-text dictation app with local (Nvidia Parakeet/Whisper) and cloud models (BYOK). Privacy-first and available cross-platform.

Claude/fix agent local models 7 bx6 t (#138)

Gabriel Steinf3a545aa8fce
diff --git a/env.example b/.env.example
similarity index 91%
rename from env.example
rename to .env.example
index af53161de..82c893a68 100644
--- a/env.example
+++ b/.env.example
@@ -18,5 +18,5 @@ ANTHROPIC_API_KEY=your_anthropic_api_key_here
# Get your API key from: https://makersuite.google.com/app/apikey
GEMINI_API_KEY=your_gemini_api_key_here
-# Optional: Debug mode (set to 'true' to enable verbose logging)
-DEBUG=false
\ No newline at end of file
+# Optional: Debug mode
+OPENWHISPR_LOG_LEVEL=debug
\ No newline at end of file
diff --git a/src/helpers/ModelManager.ts b/src/helpers/ModelManager.ts
index 8c4dfc3c5..e284598ff 100644
--- a/src/helpers/ModelManager.ts
+++ b/src/helpers/ModelManager.ts
@@ -7,6 +7,7 @@ import { app } from "electron";
import { modelRegistry } from "../models/ModelRegistry";
import { inferenceConfig } from "../config/InferenceConfig";
import { MODEL_CONSTRAINTS } from "../config/constants";
+import { parseLlamaCppOutput } from "../utils/llamaOutputParser";
// Error types
export class ModelError extends Error {
@@ -314,7 +315,7 @@ class ModelManager {
llamaProcess.on("close", (code) => {
if (code === 0) {
- resolve(output.trim());
+ resolve(parseLlamaCppOutput(output));
} else {
reject(new ModelError(`Inference failed: ${error}`, "INFERENCE_ERROR"));
}
diff --git a/src/helpers/modelManagerBridge.js b/src/helpers/modelManagerBridge.js
index 43bfa73f2..97528850f 100644
--- a/src/helpers/modelManagerBridge.js
+++ b/src/helpers/modelManagerBridge.js
@@ -7,6 +7,8 @@ const { pipeline } = require("stream");
const { app } = require("electron");
const modelRegistryData = require("../models/modelRegistryData.json");
+const { parseLlamaCppOutput } = require("../utils/llamaOutputParser");
+const debugLogger = require("./debugLogger");
const MIN_FILE_SIZE = 1_000_000; // 1MB minimum for valid model files
@@ -407,26 +409,56 @@ class ModelManager {
}
async ensureLlamaCpp() {
+ debugLogger.logReasoning("ENSURE_LLAMACPP_START", {});
+
const llamaCppInstaller = require("./llamaCppInstaller").default;
- if (!(await llamaCppInstaller.isInstalled())) {
+ const isInstalled = await llamaCppInstaller.isInstalled();
+ debugLogger.logReasoning("ENSURE_LLAMACPP_CHECK", { isInstalled });
+
+ if (!isInstalled) {
+ debugLogger.logReasoning("ENSURE_LLAMACPP_NOT_INSTALLED", {});
+ console.error("[LocalAI] llama.cpp is not installed");
throw new ModelError("llama.cpp is not installed", "LLAMACPP_NOT_INSTALLED");
}
this.llamaCppPath = await llamaCppInstaller.getBinaryPath();
+ debugLogger.logReasoning("ENSURE_LLAMACPP_SUCCESS", {
+ llamaCppPath: this.llamaCppPath,
+ });
return true;
}
async runInference(modelId, prompt, options = {}) {
+ const startTime = Date.now();
+ debugLogger.logReasoning("INFERENCE_START", {
+ modelId,
+ promptLength: prompt.length,
+ options: { ...options, systemPrompt: options.systemPrompt ? "[set]" : "[not set]" },
+ });
+
await this.ensureLlamaCpp();
+ debugLogger.logReasoning("INFERENCE_LLAMACPP_READY", {
+ llamaCppPath: this.llamaCppPath,
+ });
const modelInfo = this.findModelById(modelId);
if (!modelInfo) {
+ debugLogger.logReasoning("INFERENCE_MODEL_NOT_FOUND", { modelId });
+ console.error(`[LocalAI] Model not found: ${modelId}`);
throw new ModelNotFoundError(modelId);
}
const modelPath = path.join(this.modelsDir, modelInfo.model.fileName);
+ debugLogger.logReasoning("INFERENCE_MODEL_PATH", {
+ modelPath,
+ modelName: modelInfo.model.name,
+ providerId: modelInfo.provider.id,
+ });
+
if (!(await this.checkModelValid(modelPath))) {
+ debugLogger.logReasoning("INFERENCE_MODEL_INVALID", { modelId, modelPath });
+ console.error(`[LocalAI] Model invalid or not downloaded: ${modelPath}`);
throw new ModelError(
`Model ${modelId} is not downloaded or is corrupted`,
"MODEL_NOT_DOWNLOADED",
@@ -442,6 +474,12 @@ class ModelManager {
options.systemPrompt || ""
);
+ debugLogger.logReasoning("INFERENCE_PROMPT_FORMATTED", {
+ formattedPromptLength: formattedPrompt.length,
+ promptPreview:
+ formattedPrompt.substring(0, 200) + (formattedPrompt.length > 200 ? "..." : ""),
+ });
+
return new Promise((resolve, reject) => {
const args = [
"-m",
@@ -463,22 +501,58 @@ class ModelManager {
"-t",
String(options.threads || 4),
"--no-display-prompt",
+ "-no-cnv", // Disable conversation mode (model waits for stdin otherwise)
];
- const process = spawn(this.llamaCppPath, args);
+ debugLogger.logReasoning("INFERENCE_SPAWN", {
+ command: this.llamaCppPath,
+ argsCount: args.length,
+ maxTokens: options.maxTokens || 512,
+ contextSize: options.contextSize || modelInfo.model.contextLength,
+ });
+
+ const proc = spawn(this.llamaCppPath, args);
let output = "";
let error = "";
- process.stdout.on("data", (data) => {
+ debugLogger.logReasoning("INFERENCE_PROCESS_STARTED", {
+ pid: proc.pid,
+ });
+
+ proc.stdout.on("data", (data) => {
output += data.toString();
+ debugLogger.logReasoning("INFERENCE_STDOUT_CHUNK", {
+ chunkLength: data.length,
+ totalOutputLength: output.length,
+ elapsedMs: Date.now() - startTime,
+ });
});
- process.stderr.on("data", (data) => {
+ proc.stderr.on("data", (data) => {
error += data.toString();
+ debugLogger.logReasoning("INFERENCE_STDERR_CHUNK", {
+ chunkLength: data.length,
+ preview: data.toString().substring(0, 200),
+ elapsedMs: Date.now() - startTime,
+ });
});
- process.on("close", (code) => {
+ proc.on("close", (code) => {
+ const totalTime = Date.now() - startTime;
+ debugLogger.logReasoning("INFERENCE_CLOSE", {
+ exitCode: code,
+ totalTimeMs: totalTime,
+ outputLength: output.length,
+ errorLength: error.length,
+ });
+
if (code !== 0) {
+ debugLogger.logReasoning("INFERENCE_FAILED", {
+ code,
+ error: error.substring(0, 1000),
+ outputPreview: output.substring(0, 500),
+ });
+ console.error(`[LocalAI] Inference failed (${totalTime}ms): ${error.substring(0, 200)}`);
reject(
new ModelError(`Inference failed with code ${code}: ${error}`, "INFERENCE_FAILED", {
code,
@@ -486,11 +560,25 @@ class ModelManager {
})
);
} else {
- resolve(output.trim());
+ const parsed = parseLlamaCppOutput(output);
+ debugLogger.logReasoning("INFERENCE_SUCCESS", {
+ totalTimeMs: totalTime,
+ rawOutputLength: output.length,
+ parsedOutputLength: parsed.length,
+ parsedPreview: parsed.substring(0, 200) + (parsed.length > 200 ? "..." : ""),
+ });
+ resolve(parsed);
}
});
- process.on("error", (err) => {
+ proc.on("error", (err) => {
+ const totalTime = Date.now() - startTime;
+ debugLogger.logReasoning("INFERENCE_PROCESS_ERROR", {
+ error: err.message,
+ stack: err.stack,
+ totalTimeMs: totalTime,
+ });
+ console.error(`[LocalAI] Process error: ${err.message}`);
reject(
new ModelError(`Failed to start inference: ${err.message}`, "INFERENCE_START_FAILED", {
error: err.message,
diff --git a/src/utils/llamaOutputParser.js b/src/utils/llamaOutputParser.js
new file mode 100644
index 000000000..e8c93061e
--- /dev/null
+++ b/src/utils/llamaOutputParser.js
@@ -0,0 +1,73 @@
+/**
+ * Utility for parsing llama.cpp CLI output.
+ * Extracts only the generated text, filtering out diagnostic messages and system noise.
+ */
+
+// Patterns that indicate diagnostic/system output to filter out
+const DIAGNOSTIC_PATTERNS = [
+ /^llama_/i,
+ /^ggml_/i,
+ /^log_/i,
+ /^main:/i,
+ /^sampling:/i,
+ /^generate:/i,
+ /^system_info:/i,
+ /^\s*load time\s*=/i,
+ /^\s*sample time\s*=/i,
+ /^\s*prompt eval time\s*=/i,
+ /^\s*eval time\s*=/i,
+ /^\s*total time\s*=/i,
+ /^Log start$/i,
+ /^build: \d+/i,
+ /^n_threads/i,
+ /^Using .* backend/i,
+ /^\s*\d+\s+tokens?\s+/i,
+];
+
+// Special end tokens that models may output
+const END_TOKEN_PATTERNS = [/<\|im_end\|>$/, /<\|end\|>$/, /<\/s>$/, /\[end of text\]$/i];
+
+/**
+ * Parse llama.cpp output to extract only the generated text.
+ * Filters out diagnostic messages, timing stats, and other noise.
+ * @param {string} rawOutput - Raw output from llama.cpp CLI
+ * @returns {string} Cleaned output containing only generated text
+ */
+function parseLlamaCppOutput(rawOutput) {
+ if (!rawOutput) return "";
+
+ const lines = rawOutput.split("\n");
+ const filteredLines = [];
+
+ for (const line of lines) {
+ const trimmedLine = line.trim();
+
+ // Skip empty lines at the start (but keep them in the middle of content)
+ if (filteredLines.length === 0 && !trimmedLine) {
+ continue;
+ }
+
+ // Check if line matches any diagnostic pattern
+ const isDiagnostic = DIAGNOSTIC_PATTERNS.some((pattern) => pattern.test(trimmedLine));
+
+ if (!isDiagnostic) {
+ filteredLines.push(line);
+ }
+ }
+
+ // Trim trailing empty lines
+ let result = filteredLines.join("\n").trim();
+
+ // Remove common end tokens if they appear at the very end
+ for (const pattern of END_TOKEN_PATTERNS) {
+ result = result.replace(pattern, "").trim();
+ }
+
+ return result;
+}
+
+module.exports = {
+ parseLlamaCppOutput,
+ DIAGNOSTIC_PATTERNS,
+ END_TOKEN_PATTERNS,
+};