Skip to content

Voice-to-text dictation app with local (Nvidia Parakeet/Whisper) and cloud models (BYOK). Privacy-first and available cross-platform.

fix(sidecars): escalate stale-sidecar reaping to SIGKILL and verify death (#1626)

Chad Pihaa0d2bd423ed2
diff --git a/main.js b/main.js
index a34cfaab0..11104e8a8 100644
--- a/main.js
+++ b/main.js
@@ -935,7 +935,9 @@ function startAuthBridgeServer() {
// Main application startup
async function startApp() {
- reapStaleSidecars();
+ // Await so a stale sidecar is confirmed dead before new ones can spawn and
+ // contend for its port or storage lock.
+ await reapStaleSidecars();
// Phase 1: Core managers + IPC handlers before windows
initializeCoreManagers();
diff --git a/src/helpers/sidecarReaper.js b/src/helpers/sidecarReaper.js
index 7110016bd..0d978db84 100644
--- a/src/helpers/sidecarReaper.js
+++ b/src/helpers/sidecarReaper.js
@@ -10,6 +10,13 @@ const EXPECTED_BINARY_FRAGMENTS = {
diarization: ["sherpa-onnx-diarize"],
};
+// A wedged sidecar can ignore SIGTERM entirely (observed with qdrant spinning
+// at full CPU), so reaping must verify death and escalate rather than
+// fire-and-forget.
+const SIGTERM_GRACE_MS = 5000;
+const SIGKILL_GRACE_MS = 1000;
+const EXIT_POLL_INTERVAL_MS = 200;
+
function isProcessAlive(pid) {
try {
process.kill(pid, 0);
@@ -39,25 +46,61 @@ function processCommand(pid) {
}
}
-function reapStaleSidecars() {
+function signalProcess(pid, signal) {
+ try {
+ process.kill(pid, signal);
+ } catch {
+ // Already dead.
+ }
+}
+
+function sleep(ms) {
+ return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+async function waitForExit(pid, timeoutMs) {
+ const deadline = Date.now() + timeoutMs;
+ while (isProcessAlive(pid)) {
+ if (Date.now() >= deadline) return false;
+ await sleep(EXIT_POLL_INTERVAL_MS);
+ }
+ return true;
+}
+
+async function killStaleSidecar(name, pid, { sigtermGraceMs, sigkillGraceMs }) {
+ debugLogger.warn("Reaping stale sidecar", { name, pid });
+ signalProcess(pid, "SIGTERM");
+ if (await waitForExit(pid, sigtermGraceMs)) return true;
+
+ debugLogger.warn("Stale sidecar ignored SIGTERM, escalating to SIGKILL", { name, pid });
+ signalProcess(pid, "SIGKILL");
+ if (await waitForExit(pid, sigkillGraceMs)) return true;
+
+ debugLogger.error("Stale sidecar survived SIGKILL, keeping PID entry for next launch", {
+ name,
+ pid,
+ });
+ return false;
+}
+
+async function reapStaleSidecars({
+ sigtermGraceMs = SIGTERM_GRACE_MS,
+ sigkillGraceMs = SIGKILL_GRACE_MS,
+} = {}) {
const entries = sidecarPidFile.readAll();
- for (const { name, pid } of entries) {
- const fragments = EXPECTED_BINARY_FRAGMENTS[name];
- if (fragments && isProcessAlive(pid)) {
- const command = processCommand(pid);
- if (!fragments.some((fragment) => command.includes(fragment))) {
- sidecarPidFile.clear(name);
- continue;
+ await Promise.all(
+ entries.map(async ({ name, pid }) => {
+ const fragments = EXPECTED_BINARY_FRAGMENTS[name];
+ if (fragments && isProcessAlive(pid)) {
+ const command = processCommand(pid);
+ if (fragments.some((fragment) => command.includes(fragment))) {
+ const dead = await killStaleSidecar(name, pid, { sigtermGraceMs, sigkillGraceMs });
+ if (!dead) return;
+ }
}
- debugLogger.warn("Reaping stale sidecar", { name, pid });
- try {
- process.kill(pid, "SIGTERM");
- } catch {
- // Already dead.
- }
- }
- sidecarPidFile.clear(name);
- }
+ sidecarPidFile.clear(name);
+ })
+ );
}
module.exports = { reapStaleSidecars };
diff --git a/test/helpers/sidecarReaper.test.js b/test/helpers/sidecarReaper.test.js
new file mode 100644
index 000000000..bbb38a29d
--- /dev/null
+++ b/test/helpers/sidecarReaper.test.js
@@ -0,0 +1,116 @@
+const test = require("node:test");
+const assert = require("node:assert/strict");
+const { spawn } = require("child_process");
+const fs = require("fs");
+const os = require("os");
+const path = require("path");
+
+let userDataDir;
+const electronPath = require.resolve("electron");
+require.cache[electronPath] = {
+ exports: {
+ app: {
+ getPath: () => userDataDir,
+ isReady: () => false,
+ },
+ },
+};
+
+const sidecarPidFile = require("../../src/helpers/sidecarPidFile");
+const { reapStaleSidecars } = require("../../src/helpers/sidecarReaper");
+
+// Short graces keep the escalation path fast; the poll interval is 200ms so
+// anything comfortably above that works.
+const GRACES = { sigtermGraceMs: 1000, sigkillGraceMs: 1000 };
+
+// The reaper matches processes by command line, which tasklist on Windows
+// reports as the bare image name — these process-identity tests are POSIX-only.
+const posixOnly = { skip: process.platform === "win32" };
+
+function createUserDataDir(t) {
+ userDataDir = fs.mkdtempSync(path.join(os.tmpdir(), "ow-sidecar-reaper-"));
+ const dir = userDataDir;
+ t.after(() => fs.rmSync(dir, { recursive: true, force: true }));
+ return dir;
+}
+
+// The script filename must contain "qdrant" so processCommand() matches the
+// expected binary fragment for the qdrant sidecar.
+function spawnFakeSidecar(t, { scriptName, ignoreSigterm }) {
+ const handler = ignoreSigterm ? 'process.on("SIGTERM", () => {});' : "";
+ const scriptPath = path.join(userDataDir, scriptName);
+ fs.writeFileSync(scriptPath, `${handler}console.log("ready");setInterval(() => {}, 1000);`);
+ const child = spawn(process.execPath, [scriptPath], { stdio: ["ignore", "pipe", "ignore"] });
+ t.after(() => {
+ try {
+ process.kill(child.pid, "SIGKILL");
+ } catch {
+ // Already dead.
+ }
+ });
+ return new Promise((resolve) => {
+ child.stdout.once("data", () => resolve(child));
+ });
+}
+
+function isAlive(pid) {
+ try {
+ process.kill(pid, 0);
+ return true;
+ } catch {
+ return false;
+ }
+}
+
+async function waitUntilDead(pid, timeoutMs = 2000) {
+ const deadline = Date.now() + timeoutMs;
+ while (isAlive(pid) && Date.now() < deadline) {
+ await new Promise((resolve) => setTimeout(resolve, 50));
+ }
+ return !isAlive(pid);
+}
+
+test("kills a responsive stale sidecar and clears its entry", posixOnly, async (t) => {
+ createUserDataDir(t);
+ const child = await spawnFakeSidecar(t, { scriptName: "qdrant-fake.js", ignoreSigterm: false });
+ sidecarPidFile.write("qdrant", child.pid);
+
+ await reapStaleSidecars(GRACES);
+
+ assert.equal(await waitUntilDead(child.pid), true);
+ assert.deepEqual(sidecarPidFile.readAll(), []);
+});
+
+test("escalates to SIGKILL when a stale sidecar ignores SIGTERM", posixOnly, async (t) => {
+ createUserDataDir(t);
+ const child = await spawnFakeSidecar(t, { scriptName: "qdrant-stuck.js", ignoreSigterm: true });
+ sidecarPidFile.write("qdrant", child.pid);
+
+ await reapStaleSidecars(GRACES);
+
+ assert.equal(await waitUntilDead(child.pid), true);
+ assert.deepEqual(sidecarPidFile.readAll(), []);
+});
+
+test("does not kill a reused PID that is no longer the sidecar binary", posixOnly, async (t) => {
+ createUserDataDir(t);
+ const child = await spawnFakeSidecar(t, { scriptName: "other-app.js", ignoreSigterm: false });
+ sidecarPidFile.write("qdrant", child.pid);
+
+ await reapStaleSidecars(GRACES);
+
+ assert.equal(isAlive(child.pid), true);
+ assert.deepEqual(sidecarPidFile.readAll(), []);
+});
+
+test("clears entries for processes that already exited", async (t) => {
+ createUserDataDir(t);
+ const child = await spawnFakeSidecar(t, { scriptName: "qdrant-gone.js", ignoreSigterm: false });
+ sidecarPidFile.write("qdrant", child.pid);
+ process.kill(child.pid, "SIGKILL");
+ assert.equal(await waitUntilDead(child.pid), true);
+
+ await reapStaleSidecars(GRACES);
+
+ assert.deepEqual(sidecarPidFile.readAll(), []);
+});