Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(api): harness watcher restarts
delong33d1b0aa282ee
diff --git a/apps/api/src/harness.ts b/apps/api/src/harness.tsindex f61a3414f..e5ac69d26 100644--- a/apps/api/src/harness.ts+++ b/apps/api/src/harness.ts@@ -8,6 +8,10 @@ import { createWriteStream } from "fs";const childProcesses = new Set<ChildProcess>();const stopping = new WeakSet<ChildProcess>(); // processes we're intentionally stopping+let IS_DEV = false;+let restartSignal: AbortController | null = null;+let shuttingDown = false;+interface ProcessResult {promise: Promise<void>;process: ChildProcess;@@ -18,6 +22,7 @@ interface Services {worker?: ProcessResult;nuqWorkers: ProcessResult[];nuqPrefetchWorker?: ProcessResult;+ extractWorker?: ProcessResult;indexWorker?: ProcessResult;command?: ProcessResult;}@@ -79,7 +84,7 @@ const logger = {);},info(message: string) {- console.log(message);+ console.log(`${colors.blue}ℹ${colors.reset} ${message}`);},success(message: string) {console.log(`${colors.green}✓${colors.reset} ${message}`);@@ -87,13 +92,13 @@ const logger = {warn(message: string) {console.log(`${colors.yellow}!${colors.reset} ${message}`);},- error(message: string) {- console.error(`${colors.red}✗${colors.reset} ${message}`);+ error(message: string, error?: any) {+ console.error(`${colors.red}✗${colors.reset} ${message}`, error);},processStart(name: string, command: string) {const color = getProcessColor(name);console.log(- `${color}>${colors.reset} ${color}${colors.bold}${name}${colors.reset} ${colors.dim}${command}${colors.reset}`,+ `${color}>${colors.reset} ${color}${colors.bold}${name.padEnd(14)}${colors.reset} ${colors.dim}${command}${colors.reset}`,);},processEnd(name: string, exitCode: number | null, duration: bigint) {@@ -104,13 +109,13 @@ const logger = {const codeInfo =exitCode !== 0 ? ` ${colors.red}(${exitCode})${colors.reset}` : "";console.log(- `${symbolColor}${symbol}${colors.reset} ${color}${colors.bold}${name}${colors.reset} ${timing}${codeInfo}`,+ `${symbolColor}${symbol}${colors.reset} ${color}${colors.bold}${name.padEnd(14)}${colors.reset} ${timing}${codeInfo}`,);},processOutput(name: string, line: string, isReduceNoise: boolean) {const color = getProcessColor(name);- const label = `${color}${name.padEnd(14)}${colors.reset}`;if (!(line.includes("[nuq/metrics:") && isReduceNoise)) {+ const label = `${color}${colors.bold}${name.padEnd(14)}${colors.reset}`;console.log(`${label} ${line}`);}stream.write(`${name.padEnd(14)} ${line}\n`);@@ -120,32 +125,60 @@ const logger = {function waitForPort(port: number,host: string,- timeoutMs = 30000,+ options: { signal?: AbortSignal; timeoutMs?: number } = {},): Promise<void> {return new Promise((resolve, reject) => {- const timeout = setTimeout(() => {- reject(- new Error(- `Port ${port} did not become available within ${timeoutMs}ms`,+ const { signal, timeoutMs = 30000 } = options;++ let settled = false;+ let retryTimer: NodeJS.Timeout | null = null;+ let socket: net.Socket | null = null;++ const done = (err?: Error) => {+ if (settled) return;+ settled = true;++ if (retryTimer) clearTimeout(retryTimer);+ if (socket) socket.destroy();+ if (overallTimer) clearTimeout(overallTimer);++ signal?.removeEventListener("abort", onAbort);+ err ? reject(err) : resolve();+ };++ const abortError = () => {+ const e = new Error("Aborted");+ (e as any).name = "AbortError";+ return e;+ };+ const onAbort = () => done(abortError());++ const overallTimer = setTimeout(+ () =>+ done(+ new Error(+ `Port ${port} did not become available within ${timeoutMs}ms`,+ ),),- );- }, timeoutMs);+ timeoutMs,+ );++ if (signal?.aborted) return done(abortError());+ signal?.addEventListener("abort", onAbort, { once: true });- const checkPort = () => {- const socket = new net.Socket();- const onError = () => {- socket.destroy();- setTimeout(checkPort, 250);+ const check = () => {+ if (settled) return;+ socket = new net.Socket();+ const retry = () => {+ socket?.destroy();+ if (!settled) retryTimer = setTimeout(check, 250);};- socket.once("error", onError);- socket.setTimeout(1000);- socket.connect(port, host, () => {- socket.destroy();- clearTimeout(timeout);- resolve();- });+ socket.once("error", retry);+ socket.setTimeout(1000, retry);+ socket.connect(port, host, () => done());};- checkPort();++ check();});}@@ -235,7 +268,7 @@ function execForward(return { promise, process: child };}-function terminateProcess(proc: ChildProcess): Promise<void> {+function terminateProcess(proc: ChildProcess, force: boolean): Promise<void> {return new Promise(resolve => {if (!proc || proc.killed || proc.exitCode !== null) {resolve();@@ -244,15 +277,22 @@ function terminateProcess(proc: ChildProcess): Promise<void> {stopping.add(proc);+ let killTimeout: NodeJS.Timeout | null = null;+let resolved = false;const cleanup = () => {if (!resolved) {resolved = true;resolve();}++ if (killTimeout) {+ clearTimeout(killTimeout);+ killTimeout = null;+ }};- proc.once("exit", cleanup);+ proc.once("close", cleanup);proc.once("error", cleanup);const isWindows = process.platform === "win32";@@ -269,9 +309,29 @@ function terminateProcess(proc: ChildProcess): Promise<void> {} else if (proc.pid) {try {process.kill(-proc.pid, "SIGTERM");- } catch {+ } catch (e) {proc.kill("SIGTERM");}++ // for the old workers, if they are mid-job it will wait for them to finish+ // for dev mode for now we can just kill them (jobs will be picked up again if required)+ if (force && IS_DEV) {+ killTimeout = setTimeout(() => {+ if (proc.pid) {+ logger.warn(+ `Process ${proc.pid} did not exit in time, forcing termination`,+ );++ try {+ process.kill(-proc.pid, "SIGKILL");+ } catch {+ try {+ proc.kill("SIGKILL");+ } catch {}+ }+ }+ }, 5000);+ }}});}@@ -408,59 +468,105 @@ function startServices(command?: string[]): Services {nuqWorkers,nuqPrefetchWorker,indexWorker,+ extractWorker,command: commandProcess,};}-async function stopServices(services: Services) {- const stopPromises = [- services.api && terminateProcess(services.api.process),- services.worker && terminateProcess(services.worker.process),- ...services.nuqWorkers.map(w => terminateProcess(w.process)),- services.nuqPrefetchWorker &&- terminateProcess(services.nuqPrefetchWorker.process),- services.indexWorker && terminateProcess(services.indexWorker.process),- services.command && terminateProcess(services.command.process),- ].filter(Boolean);-- await Promise.all(stopPromises);+async function stopDevelopmentServices(services: Services) {+ logger.section("Stopping services");++ const processesToStop: ChildProcess[] = [+ services.api?.process,+ services.worker?.process,+ ...services.nuqWorkers.map(w => w.process),+ services.nuqPrefetchWorker?.process,+ services.indexWorker?.process,+ services.extractWorker?.process,+ services.command?.process,+ ].filter((p): p is ChildProcess => !!p);++ await Promise.race([+ await Promise.all(+ processesToStop.map(proc => terminateProcess(proc, true)),+ ).catch(e => {+ logger.error("Error while stopping processes", e);+ }),+ ]);}async function runDevMode(): Promise<void> {let currentServices: Services | null = null;- let isFirstStart = true;- const { TscWatchClient } = await import("tsc-watch");+ let started = false;+ let restarting = false;+ let pending = false;+ const { TscWatchClient } = await import("tsc-watch");const watch = new TscWatchClient();+ const restartServices = async () => {+ if (shuttingDown) return;++ pending = true;+ restartSignal?.abort();++ if (restarting) return;+ restarting = true;++ try {+ while (pending) {+ pending = false;++ if (currentServices) {+ await stopDevelopmentServices(currentServices);+ currentServices = null;+ }++ if (shuttingDown) return;++ currentServices = startServices();++ restartSignal?.abort();+ restartSignal = new AbortController();++ try {+ await waitForPort(Number(PORT), "localhost", {+ signal: restartSignal?.signal,+ });+ } catch (e) {+ if (e?.name !== "AbortError") throw e;+ if (shuttingDown) return;++ logger.section(+ "Recompilation triggering during restart, trying again...",+ );+ continue;+ }++ logger.success("All services started");+ started = true;+ }+ } finally {+ restarting = false;+ }+ };+watch.on("started", () => {logger.info("TypeScript compilation started");+ pending = true;+ restartSignal?.abort();});watch.on("first_success", async () => {logger.success("Initial compilation complete");-- currentServices = startServices();-- logger.info(`Waiting for API on localhost:${PORT}`);- await waitForPort(Number(PORT), "localhost");- logger.success("API is ready");-- isFirstStart = false;+ await restartServices();});watch.on("success", async () => {- if (!isFirstStart && currentServices) {+ if (started) {logger.info("Recompilation complete - restarting services");-- await stopServices(currentServices);- logger.info("Services stopped");-- currentServices = startServices();-- await waitForPort(Number(PORT), "localhost");- logger.success("Services restarted");+ await restartServices();}});@@ -472,12 +578,18 @@ async function runDevMode(): Promise<void> {watch.start("--project", ".");await new Promise<void>(resolve => {- process.on("SIGINT", resolve);- process.on("SIGTERM", resolve);+ const stop = () => {+ process.off("SIGINT", stop);+ process.off("SIGTERM", stop);+ resolve();+ };+ process.once("SIGINT", stop);+ process.once("SIGTERM", stop);});+ restartSignal?.abort();if (currentServices) {- await stopServices(currentServices);+ await stopDevelopmentServices(currentServices);}watch.kill();}@@ -505,6 +617,7 @@ async function waitForTermination(services: Services): Promise<void> {if (services.api) promises.push(services.api.promise);if (services.worker) promises.push(services.worker.promise);if (services.indexWorker) promises.push(services.indexWorker.promise);+ if (services.extractWorker) promises.push(services.extractWorker.promise);if (services.nuqPrefetchWorker)promises.push(services.nuqPrefetchWorker.promise);@@ -513,13 +626,16 @@ async function waitForTermination(services: Services): Promise<void> {await Promise.race(promises);}-let shuttingDown = false;async function gracefulShutdown() {if (shuttingDown) return;shuttingDown = true;+ restartSignal?.abort();logger.section("Shutting down");- const terminationPromises = Array.from(childProcesses).map(terminateProcess);+ const forceTerminate = IS_DEV;+ const terminationPromises = Array.from(childProcesses).map(proc =>+ terminateProcess(proc, forceTerminate),+ );await Promise.all(terminationPromises);logger.success("All processes terminated");}@@ -547,13 +663,13 @@ async function main() {}const command = process.argv.slice(2);- const isDev = command[0] === "--start";+ IS_DEV = command[0] === "--start";if (command[0] !== "--start-docker") {await installDependencies();}- if (isDev) {+ if (IS_DEV) {await runDevMode();} else {await runProductionMode(command);@@ -565,6 +681,7 @@ async function main() {} finally {await gracefulShutdown();logger.info("Goodbye!");+ process.exit(0);}}diff --git a/apps/api/src/services/extract-worker.ts b/apps/api/src/services/extract-worker.tsindex 458e18b8f..36e0151f8 100644--- a/apps/api/src/services/extract-worker.ts+++ b/apps/api/src/services/extract-worker.ts@@ -2,10 +2,7 @@ import "dotenv/config";import { shutdownOtel } from "../otel";import "./sentry";import * as Sentry from "@sentry/node";-import {- getExtractQueue,- getRedisConnection,-} from "./queue-service";+import { getExtractQueue, getRedisConnection } from "./queue-service";import { Job, Queue, Worker } from "bullmq";import { logger as _logger } from "../lib/logger";import systemMonitor from "./system-monitor";@@ -172,12 +169,12 @@ let isShuttingDown = false;let isWorkerStalled = false;process.on("SIGINT", () => {- console.log("Received SIGTERM. Shutting down gracefully...");+ _logger.debug("Received SIGTERM. Shutting down gracefully...");isShuttingDown = true;});process.on("SIGTERM", () => {- console.log("Received SIGTERM. Shutting down gracefully...");+ _logger.debug("Received SIGTERM. Shutting down gracefully...");isShuttingDown = true;});@@ -203,7 +200,7 @@ const workerFun = async (while (true) {if (isShuttingDown) {- console.log("No longer accepting new jobs. SIGINT");+ _logger.info("No longer accepting new jobs. SIGINT");break;}const token = uuidv4();@@ -300,17 +297,17 @@ app.listen(workerPort, () => {process.exit(1);});- await Promise.all([- workerFun(getExtractQueue(), processExtractJobInternal),- ]);+ await Promise.all([workerFun(getExtractQueue(), processExtractJobInternal)]);- console.log("All workers exited. Waiting for all jobs to finish...");+ _logger.info("All workers exited. Waiting for all jobs to finish...");while (runningJobs.size > 0) {await new Promise(resolve => setTimeout(resolve, 500));}- console.log("All jobs finished. Worker out!");- await shutdownOtel();- process.exit(0);+ _logger.info("All jobs finished. Shutting down...");+ shutdownOtel().finally(() => {+ _logger.debug("OTEL shutdown");+ process.exit(0);+ });})();diff --git a/apps/api/src/services/indexing/index-worker.ts b/apps/api/src/services/indexing/index-worker.tsindex 55e67d0cf..efcbfdb96 100644--- a/apps/api/src/services/indexing/index-worker.ts+++ b/apps/api/src/services/indexing/index-worker.ts@@ -154,6 +154,8 @@ const processPrecrawlJob = async (token: string, job: Job) => {"precrawl.dry_run": DRY_RUN,});+ await new Promise(resolve => setTimeout(resolve, 20000));+const dateFuture = new Date();dateFuture.setHours(dateFuture.getHours() + 1);@@ -182,7 +184,7 @@ const processPrecrawlJob = async (token: string, job: Job) => {"precrawl.domain_count": domains.length,});- logger.info(`Found ${domains.length} domains for precrawl`);+ logger.info(`Found ${domains.length} domains for precrawl test`);const minPriority = Math.min(...domains.map(d => d.priority));const maxPriority = Math.max(...domains.map(d => d.priority));@@ -732,4 +734,6 @@ const DOMAIN_FREQUENCY_INTERVAL = 10000;clearInterval(indexRFInserterInterval);clearInterval(omceInserterInterval);clearInterval(domainFrequencyInterval);++ logger.info("All workers shut down, exiting process");})();diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.tsindex 4a1ba3b69..de7e3bb34 100644--- a/apps/api/src/services/queue-worker.ts+++ b/apps/api/src/services/queue-worker.ts@@ -195,12 +195,12 @@ let isShuttingDown = false;let isWorkerStalled = false;process.on("SIGINT", () => {- console.log("Received SIGTERM. Shutting down gracefully...");+ _logger.debug("Received SIGTERM. Shutting down gracefully...");isShuttingDown = true;});process.on("SIGTERM", () => {- console.log("Received SIGTERM. Shutting down gracefully...");+ _logger.debug("Received SIGTERM. Shutting down gracefully...");isShuttingDown = true;});@@ -226,7 +226,7 @@ const workerFun = async (while (true) {if (isShuttingDown) {- console.log("No longer accepting new jobs. SIGINT");+ _logger.info("No longer accepting new jobs. SIGINT");break;}const token = uuidv4();@@ -328,13 +328,15 @@ app.listen(workerPort, () => {workerFun(getGenerateLlmsTxtQueue(), processGenerateLlmsTxtJobInternal),]);- console.log("All workers exited. Waiting for all jobs to finish...");+ _logger.info("All workers exited. Waiting for all jobs to finish...");while (runningJobs.size > 0) {await new Promise(resolve => setTimeout(resolve, 500));}- console.log("All jobs finished. Worker out!");- await shutdownOtel();- process.exit(0);+ _logger.info("All jobs finished. Shutting down...");+ shutdownOtel().finally(() => {+ _logger.debug("OTEL shutdown");+ process.exit(0);+ });})();