Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(api): harness watcher restarts

delong33d1b0aa282ee
diff --git a/apps/api/src/harness.ts b/apps/api/src/harness.ts
index f61a3414f..e5ac69d26 100644
--- a/apps/api/src/harness.ts
+++ b/apps/api/src/harness.ts
@@ -8,6 +8,10 @@ import { createWriteStream } from "fs";
const childProcesses = new Set<ChildProcess>();
const stopping = new WeakSet<ChildProcess>(); // processes we're intentionally stopping
+let IS_DEV = false;
+let restartSignal: AbortController | null = null;
+let shuttingDown = false;
+
interface ProcessResult {
promise: Promise<void>;
process: ChildProcess;
@@ -18,6 +22,7 @@ interface Services {
worker?: ProcessResult;
nuqWorkers: ProcessResult[];
nuqPrefetchWorker?: ProcessResult;
+ extractWorker?: ProcessResult;
indexWorker?: ProcessResult;
command?: ProcessResult;
}
@@ -79,7 +84,7 @@ const logger = {
);
},
info(message: string) {
- console.log(message);
+ console.log(`${colors.blue}ℹ${colors.reset} ${message}`);
},
success(message: string) {
console.log(`${colors.green}✓${colors.reset} ${message}`);
@@ -87,13 +92,13 @@ const logger = {
warn(message: string) {
console.log(`${colors.yellow}!${colors.reset} ${message}`);
},
- error(message: string) {
- console.error(`${colors.red}✗${colors.reset} ${message}`);
+ error(message: string, error?: any) {
+ console.error(`${colors.red}✗${colors.reset} ${message}`, error);
},
processStart(name: string, command: string) {
const color = getProcessColor(name);
console.log(
- `${color}>${colors.reset} ${color}${colors.bold}${name}${colors.reset} ${colors.dim}${command}${colors.reset}`,
+ `${color}>${colors.reset} ${color}${colors.bold}${name.padEnd(14)}${colors.reset} ${colors.dim}${command}${colors.reset}`,
);
},
processEnd(name: string, exitCode: number | null, duration: bigint) {
@@ -104,13 +109,13 @@ const logger = {
const codeInfo =
exitCode !== 0 ? ` ${colors.red}(${exitCode})${colors.reset}` : "";
console.log(
- `${symbolColor}${symbol}${colors.reset} ${color}${colors.bold}${name}${colors.reset} ${timing}${codeInfo}`,
+ `${symbolColor}${symbol}${colors.reset} ${color}${colors.bold}${name.padEnd(14)}${colors.reset} ${timing}${codeInfo}`,
);
},
processOutput(name: string, line: string, isReduceNoise: boolean) {
const color = getProcessColor(name);
- const label = `${color}${name.padEnd(14)}${colors.reset}`;
if (!(line.includes("[nuq/metrics:") && isReduceNoise)) {
+ const label = `${color}${colors.bold}${name.padEnd(14)}${colors.reset}`;
console.log(`${label} ${line}`);
}
stream.write(`${name.padEnd(14)} ${line}\n`);
@@ -120,32 +125,60 @@ const logger = {
function waitForPort(
port: number,
host: string,
- timeoutMs = 30000,
+ options: { signal?: AbortSignal; timeoutMs?: number } = {},
): Promise<void> {
return new Promise((resolve, reject) => {
- const timeout = setTimeout(() => {
- reject(
- new Error(
- `Port ${port} did not become available within ${timeoutMs}ms`,
+ const { signal, timeoutMs = 30000 } = options;
+
+ let settled = false;
+ let retryTimer: NodeJS.Timeout | null = null;
+ let socket: net.Socket | null = null;
+
+ const done = (err?: Error) => {
+ if (settled) return;
+ settled = true;
+
+ if (retryTimer) clearTimeout(retryTimer);
+ if (socket) socket.destroy();
+ if (overallTimer) clearTimeout(overallTimer);
+
+ signal?.removeEventListener("abort", onAbort);
+ err ? reject(err) : resolve();
+ };
+
+ const abortError = () => {
+ const e = new Error("Aborted");
+ (e as any).name = "AbortError";
+ return e;
+ };
+ const onAbort = () => done(abortError());
+
+ const overallTimer = setTimeout(
+ () =>
+ done(
+ new Error(
+ `Port ${port} did not become available within ${timeoutMs}ms`,
+ ),
),
- );
- }, timeoutMs);
+ timeoutMs,
+ );
+
+ if (signal?.aborted) return done(abortError());
+ signal?.addEventListener("abort", onAbort, { once: true });
- const checkPort = () => {
- const socket = new net.Socket();
- const onError = () => {
- socket.destroy();
- setTimeout(checkPort, 250);
+ const check = () => {
+ if (settled) return;
+ socket = new net.Socket();
+ const retry = () => {
+ socket?.destroy();
+ if (!settled) retryTimer = setTimeout(check, 250);
};
- socket.once("error", onError);
- socket.setTimeout(1000);
- socket.connect(port, host, () => {
- socket.destroy();
- clearTimeout(timeout);
- resolve();
- });
+ socket.once("error", retry);
+ socket.setTimeout(1000, retry);
+ socket.connect(port, host, () => done());
};
- checkPort();
+
+ check();
});
}
@@ -235,7 +268,7 @@ function execForward(
return { promise, process: child };
}
-function terminateProcess(proc: ChildProcess): Promise<void> {
+function terminateProcess(proc: ChildProcess, force: boolean): Promise<void> {
return new Promise(resolve => {
if (!proc || proc.killed || proc.exitCode !== null) {
resolve();
@@ -244,15 +277,22 @@ function terminateProcess(proc: ChildProcess): Promise<void> {
stopping.add(proc);
+ let killTimeout: NodeJS.Timeout | null = null;
+
let resolved = false;
const cleanup = () => {
if (!resolved) {
resolved = true;
resolve();
}
+
+ if (killTimeout) {
+ clearTimeout(killTimeout);
+ killTimeout = null;
+ }
};
- proc.once("exit", cleanup);
+ proc.once("close", cleanup);
proc.once("error", cleanup);
const isWindows = process.platform === "win32";
@@ -269,9 +309,29 @@ function terminateProcess(proc: ChildProcess): Promise<void> {
} else if (proc.pid) {
try {
process.kill(-proc.pid, "SIGTERM");
- } catch {
+ } catch (e) {
proc.kill("SIGTERM");
}
+
+ // for the old workers, if they are mid-job it will wait for them to finish
+ // for dev mode for now we can just kill them (jobs will be picked up again if required)
+ if (force && IS_DEV) {
+ killTimeout = setTimeout(() => {
+ if (proc.pid) {
+ logger.warn(
+ `Process ${proc.pid} did not exit in time, forcing termination`,
+ );
+
+ try {
+ process.kill(-proc.pid, "SIGKILL");
+ } catch {
+ try {
+ proc.kill("SIGKILL");
+ } catch {}
+ }
+ }
+ }, 5000);
+ }
}
});
}
@@ -408,59 +468,105 @@ function startServices(command?: string[]): Services {
nuqWorkers,
nuqPrefetchWorker,
indexWorker,
+ extractWorker,
command: commandProcess,
};
}
-async function stopServices(services: Services) {
- const stopPromises = [
- services.api && terminateProcess(services.api.process),
- services.worker && terminateProcess(services.worker.process),
- ...services.nuqWorkers.map(w => terminateProcess(w.process)),
- services.nuqPrefetchWorker &&
- terminateProcess(services.nuqPrefetchWorker.process),
- services.indexWorker && terminateProcess(services.indexWorker.process),
- services.command && terminateProcess(services.command.process),
- ].filter(Boolean);
-
- await Promise.all(stopPromises);
+async function stopDevelopmentServices(services: Services) {
+ logger.section("Stopping services");
+
+ const processesToStop: ChildProcess[] = [
+ services.api?.process,
+ services.worker?.process,
+ ...services.nuqWorkers.map(w => w.process),
+ services.nuqPrefetchWorker?.process,
+ services.indexWorker?.process,
+ services.extractWorker?.process,
+ services.command?.process,
+ ].filter((p): p is ChildProcess => !!p);
+
+ await Promise.race([
+ await Promise.all(
+ processesToStop.map(proc => terminateProcess(proc, true)),
+ ).catch(e => {
+ logger.error("Error while stopping processes", e);
+ }),
+ ]);
}
async function runDevMode(): Promise<void> {
let currentServices: Services | null = null;
- let isFirstStart = true;
- const { TscWatchClient } = await import("tsc-watch");
+ let started = false;
+ let restarting = false;
+ let pending = false;
+ const { TscWatchClient } = await import("tsc-watch");
const watch = new TscWatchClient();
+ const restartServices = async () => {
+ if (shuttingDown) return;
+
+ pending = true;
+ restartSignal?.abort();
+
+ if (restarting) return;
+ restarting = true;
+
+ try {
+ while (pending) {
+ pending = false;
+
+ if (currentServices) {
+ await stopDevelopmentServices(currentServices);
+ currentServices = null;
+ }
+
+ if (shuttingDown) return;
+
+ currentServices = startServices();
+
+ restartSignal?.abort();
+ restartSignal = new AbortController();
+
+ try {
+ await waitForPort(Number(PORT), "localhost", {
+ signal: restartSignal?.signal,
+ });
+ } catch (e) {
+ if (e?.name !== "AbortError") throw e;
+ if (shuttingDown) return;
+
+ logger.section(
+ "Recompilation triggering during restart, trying again...",
+ );
+ continue;
+ }
+
+ logger.success("All services started");
+ started = true;
+ }
+ } finally {
+ restarting = false;
+ }
+ };
+
watch.on("started", () => {
logger.info("TypeScript compilation started");
+ pending = true;
+ restartSignal?.abort();
});
watch.on("first_success", async () => {
logger.success("Initial compilation complete");
-
- currentServices = startServices();
-
- logger.info(`Waiting for API on localhost:${PORT}`);
- await waitForPort(Number(PORT), "localhost");
- logger.success("API is ready");
-
- isFirstStart = false;
+ await restartServices();
});
watch.on("success", async () => {
- if (!isFirstStart && currentServices) {
+ if (started) {
logger.info("Recompilation complete - restarting services");
-
- await stopServices(currentServices);
- logger.info("Services stopped");
-
- currentServices = startServices();
-
- await waitForPort(Number(PORT), "localhost");
- logger.success("Services restarted");
+ await restartServices();
}
});
@@ -472,12 +578,18 @@ async function runDevMode(): Promise<void> {
watch.start("--project", ".");
await new Promise<void>(resolve => {
- process.on("SIGINT", resolve);
- process.on("SIGTERM", resolve);
+ const stop = () => {
+ process.off("SIGINT", stop);
+ process.off("SIGTERM", stop);
+ resolve();
+ };
+ process.once("SIGINT", stop);
+ process.once("SIGTERM", stop);
});
+ restartSignal?.abort();
if (currentServices) {
- await stopServices(currentServices);
+ await stopDevelopmentServices(currentServices);
}
watch.kill();
}
@@ -505,6 +617,7 @@ async function waitForTermination(services: Services): Promise<void> {
if (services.api) promises.push(services.api.promise);
if (services.worker) promises.push(services.worker.promise);
if (services.indexWorker) promises.push(services.indexWorker.promise);
+ if (services.extractWorker) promises.push(services.extractWorker.promise);
if (services.nuqPrefetchWorker)
promises.push(services.nuqPrefetchWorker.promise);
@@ -513,13 +626,16 @@ async function waitForTermination(services: Services): Promise<void> {
await Promise.race(promises);
}
-let shuttingDown = false;
async function gracefulShutdown() {
if (shuttingDown) return;
shuttingDown = true;
+ restartSignal?.abort();
logger.section("Shutting down");
- const terminationPromises = Array.from(childProcesses).map(terminateProcess);
+ const forceTerminate = IS_DEV;
+ const terminationPromises = Array.from(childProcesses).map(proc =>
+ terminateProcess(proc, forceTerminate),
+ );
await Promise.all(terminationPromises);
logger.success("All processes terminated");
}
@@ -547,13 +663,13 @@ async function main() {
}
const command = process.argv.slice(2);
- const isDev = command[0] === "--start";
+ IS_DEV = command[0] === "--start";
if (command[0] !== "--start-docker") {
await installDependencies();
}
- if (isDev) {
+ if (IS_DEV) {
await runDevMode();
} else {
await runProductionMode(command);
@@ -565,6 +681,7 @@ async function main() {
} finally {
await gracefulShutdown();
logger.info("Goodbye!");
+ process.exit(0);
}
}
diff --git a/apps/api/src/services/extract-worker.ts b/apps/api/src/services/extract-worker.ts
index 458e18b8f..36e0151f8 100644
--- a/apps/api/src/services/extract-worker.ts
+++ b/apps/api/src/services/extract-worker.ts
@@ -2,10 +2,7 @@ import "dotenv/config";
import { shutdownOtel } from "../otel";
import "./sentry";
import * as Sentry from "@sentry/node";
-import {
- getExtractQueue,
- getRedisConnection,
-} from "./queue-service";
+import { getExtractQueue, getRedisConnection } from "./queue-service";
import { Job, Queue, Worker } from "bullmq";
import { logger as _logger } from "../lib/logger";
import systemMonitor from "./system-monitor";
@@ -172,12 +169,12 @@ let isShuttingDown = false;
let isWorkerStalled = false;
process.on("SIGINT", () => {
- console.log("Received SIGTERM. Shutting down gracefully...");
+ _logger.debug("Received SIGTERM. Shutting down gracefully...");
isShuttingDown = true;
});
process.on("SIGTERM", () => {
- console.log("Received SIGTERM. Shutting down gracefully...");
+ _logger.debug("Received SIGTERM. Shutting down gracefully...");
isShuttingDown = true;
});
@@ -203,7 +200,7 @@ const workerFun = async (
while (true) {
if (isShuttingDown) {
- console.log("No longer accepting new jobs. SIGINT");
+ _logger.info("No longer accepting new jobs. SIGINT");
break;
}
const token = uuidv4();
@@ -300,17 +297,17 @@ app.listen(workerPort, () => {
process.exit(1);
});
- await Promise.all([
- workerFun(getExtractQueue(), processExtractJobInternal),
- ]);
+ await Promise.all([workerFun(getExtractQueue(), processExtractJobInternal)]);
- console.log("All workers exited. Waiting for all jobs to finish...");
+ _logger.info("All workers exited. Waiting for all jobs to finish...");
while (runningJobs.size > 0) {
await new Promise(resolve => setTimeout(resolve, 500));
}
- console.log("All jobs finished. Worker out!");
- await shutdownOtel();
- process.exit(0);
+ _logger.info("All jobs finished. Shutting down...");
+ shutdownOtel().finally(() => {
+ _logger.debug("OTEL shutdown");
+ process.exit(0);
+ });
})();
diff --git a/apps/api/src/services/indexing/index-worker.ts b/apps/api/src/services/indexing/index-worker.ts
index 55e67d0cf..efcbfdb96 100644
--- a/apps/api/src/services/indexing/index-worker.ts
+++ b/apps/api/src/services/indexing/index-worker.ts
@@ -154,6 +154,8 @@ const processPrecrawlJob = async (token: string, job: Job) => {
"precrawl.dry_run": DRY_RUN,
});
+ await new Promise(resolve => setTimeout(resolve, 20000));
+
const dateFuture = new Date();
dateFuture.setHours(dateFuture.getHours() + 1);
@@ -182,7 +184,7 @@ const processPrecrawlJob = async (token: string, job: Job) => {
"precrawl.domain_count": domains.length,
});
- logger.info(`Found ${domains.length} domains for precrawl`);
+ logger.info(`Found ${domains.length} domains for precrawl test`);
const minPriority = Math.min(...domains.map(d => d.priority));
const maxPriority = Math.max(...domains.map(d => d.priority));
@@ -732,4 +734,6 @@ const DOMAIN_FREQUENCY_INTERVAL = 10000;
clearInterval(indexRFInserterInterval);
clearInterval(omceInserterInterval);
clearInterval(domainFrequencyInterval);
+
+ logger.info("All workers shut down, exiting process");
})();
diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.ts
index 4a1ba3b69..de7e3bb34 100644
--- a/apps/api/src/services/queue-worker.ts
+++ b/apps/api/src/services/queue-worker.ts
@@ -195,12 +195,12 @@ let isShuttingDown = false;
let isWorkerStalled = false;
process.on("SIGINT", () => {
- console.log("Received SIGTERM. Shutting down gracefully...");
+ _logger.debug("Received SIGTERM. Shutting down gracefully...");
isShuttingDown = true;
});
process.on("SIGTERM", () => {
- console.log("Received SIGTERM. Shutting down gracefully...");
+ _logger.debug("Received SIGTERM. Shutting down gracefully...");
isShuttingDown = true;
});
@@ -226,7 +226,7 @@ const workerFun = async (
while (true) {
if (isShuttingDown) {
- console.log("No longer accepting new jobs. SIGINT");
+ _logger.info("No longer accepting new jobs. SIGINT");
break;
}
const token = uuidv4();
@@ -328,13 +328,15 @@ app.listen(workerPort, () => {
workerFun(getGenerateLlmsTxtQueue(), processGenerateLlmsTxtJobInternal),
]);
- console.log("All workers exited. Waiting for all jobs to finish...");
+ _logger.info("All workers exited. Waiting for all jobs to finish...");
while (runningJobs.size > 0) {
await new Promise(resolve => setTimeout(resolve, 500));
}
- console.log("All jobs finished. Worker out!");
- await shutdownOtel();
- process.exit(0);
+ _logger.info("All jobs finished. Shutting down...");
+ shutdownOtel().finally(() => {
+ _logger.debug("OTEL shutdown");
+ process.exit(0);
+ });
})();