Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. π₯
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Make harness cross-platform compatible (ENG-3477) (#2110)
tom94ba7604ec5e
diff --git a/apps/api/src/harness.ts b/apps/api/src/harness.tsindex ab5f494fc..b9c9a4f76 100644--- a/apps/api/src/harness.ts+++ b/apps/api/src/harness.ts@@ -1,126 +1,322 @@import "dotenv/config";-import { exec } from "child_process";+import { type ChildProcess, spawn } from "child_process";import * as net from "net";import { basename } from "path";import { HTML_TO_MARKDOWN_PATH } from "./natives";-function waitForPort(port: number, host: string): Promise<void> {- return new Promise(resolve => {+const childProcesses = new Set<ChildProcess>();++interface ProcessResult {+ promise: Promise<void>;+ process: ChildProcess;+}++const colors = {+ reset: "\x1b[0m",+ bold: "\x1b[1m",+ dim: "\x1b[2m",+ red: "\x1b[31m",+ green: "\x1b[32m",+ yellow: "\x1b[33m",+ blue: "\x1b[34m",+ magenta: "\x1b[35m",+ cyan: "\x1b[36m",+ gray: "\x1b[90m",+ white: "\x1b[37m",+};++const processGroupColors: Record<string, string> = {+ api: colors.green,+ worker: colors.blue,+ nuq: colors.cyan,+ index: colors.magenta,+ go: colors.yellow,+ command: colors.white,+};++function getProcessGroup(name: string): string {+ let group = name;+ if (name.includes("@")) {+ group = name.split("@")[0];+ }+ if (name.includes("-")) {+ group = name.split("-")[0];+ }+ return group;+}++function getProcessColor(name: string): string {+ const group = getProcessGroup(name);+ return processGroupColors[group] || colors.gray;+}++function formatDuration(nanoseconds: bigint): string {+ const milliseconds = Number(nanoseconds) / 1e6;+ if (milliseconds < 1000) {+ return `${milliseconds.toFixed(0)}ms`;+ }+ const seconds = milliseconds / 1000;+ if (seconds < 60) {+ return `${seconds.toFixed(1)}s`;+ }+ const minutes = Math.floor(seconds / 60);+ const remainingSeconds = seconds - minutes * 60;+ return `${minutes}m ${remainingSeconds.toFixed(0)}s`;+}++const logger = {+ section(message: string) {+ console.log(+ `\n${colors.bold}${colors.blue}ββ ${message} ββ${colors.reset}\n`,+ );+ },++ info(message: string) {+ console.log(message);+ },++ success(message: string) {+ console.log(`${colors.green}β${colors.reset} ${message}`);+ },++ warn(message: string) {+ console.log(`${colors.yellow}!${colors.reset} ${message}`);+ },++ error(message: string) {+ console.error(`${colors.red}β${colors.reset} ${message}`);+ },++ processStart(name: string, command: string) {+ const color = getProcessColor(name);+ console.log(+ `${color}>${colors.reset} ${color}${colors.bold}${name}${colors.reset} ${colors.dim}${command}${colors.reset}`,+ );+ },++ processEnd(name: string, exitCode: number | null, duration: bigint) {+ const color = getProcessColor(name);+ const symbol = exitCode === 0 ? "β" : "β";+ const symbolColor = exitCode === 0 ? colors.green : colors.red;+ const timing = `${colors.dim}${formatDuration(duration)}${colors.reset}`;+ const codeInfo =+ exitCode !== 0 ? ` ${colors.red}(${exitCode})${colors.reset}` : "";+ console.log(+ `${symbolColor}${symbol}${colors.reset} ${color}${colors.bold}${name}${colors.reset} ${timing}${codeInfo}`,+ );+ },++ processOutput(name: string, line: string) {+ const color = getProcessColor(name);+ const label = `${color}${name.padEnd(12)}${colors.reset}`;+ console.log(`${label} ${line}`);+ },+};++function waitForPort(+ port: number,+ host: string,+ timeoutMs = 10000,+): Promise<void> {+ return new Promise((resolve, reject) => {+ const timeout = setTimeout(() => {+ reject(+ new Error(+ `Port ${port} did not become available within ${timeoutMs}ms`,+ ),+ );+ }, timeoutMs);+const checkPort = () => {const socket = new net.Socket();-const onError = () => {socket.destroy();- setTimeout(checkPort, 1000); // Try again in 1 second+ setTimeout(checkPort, 250);};-socket.once("error", onError);-+ socket.setTimeout(1000);socket.connect(port, host, () => {socket.destroy();+ clearTimeout(timeout);resolve();});};-checkPort();});}-if (process.argv.length < 3) {- console.error("Usage: pnpm harness <command...>");- console.error();- console.error(- "The harness ensures that the dependencies are up to date, everything is built, and the API and Worker are running, before running a command that would require the above.",- );- console.error(- "It also tears down the API and Worker after the command is run.",- );- process.exit(1);-}+function execForward(+ name: string,+ command: string | string[],+ env: Record<string, string> = {},+): ProcessResult {+ let child: ChildProcess;+ let displayCommand = "";-const command = process.argv.slice(2);+ if (typeof command === "string") {+ displayCommand = command;+ const isWindows = process.platform === "win32";+ if (isWindows) {+ child = spawn("cmd", ["/c", command], {+ env: { ...process.env, ...env },+ shell: false,+ });+ } else {+ child = spawn("sh", ["-c", command], {+ env: { ...process.env, ...env },+ shell: false,+ });+ }+ } else {+ const [cmd, ...args] = command;+ displayCommand = [cmd, ...args].join(" ");+ child = spawn(cmd, args, {+ env: { ...process.env, ...env },+ shell: false,+ });+ }-function execForward(fancyName: string, command: string): Promise<void> {- return new Promise((resolve, reject) => {- const child = exec(command);+ logger.processStart(name, displayCommand);+ childProcesses.add(child);++ const startTime = process.hrtime.bigint();+ const promise = new Promise<void>((resolve, reject) => {let stdoutBuffer = "";let stderrBuffer = "";- child.stdout?.on("data", data => {- stdoutBuffer += data;- while (stdoutBuffer.includes("\n")) {- const split = stdoutBuffer.split("\n");- const line = split[0];- stdoutBuffer = split.slice(1).join("\n");- process.stdout.write(`[${fancyName}] ${line}\n`);- }- });- child.stderr?.on("data", data => {- stderrBuffer += data;- while (stderrBuffer.includes("\n")) {- const split = stderrBuffer.split("\n");- const line = split[0];- stderrBuffer = split.slice(1).join("\n");- process.stderr.write(`[${fancyName}] ${line}\n`);++ const processOutput = (data: string, isError = false) => {+ const buffer = isError ? stderrBuffer : stdoutBuffer;+ const newBuffer =+ buffer + data.replace(/\r\n/g, "\n").replace(/\r/g, "\n");+ const lines = newBuffer.split("\n");+ const completeLines = lines.slice(0, -1);+ const remainingBuffer = lines[lines.length - 1];++ completeLines.forEach(line => {+ if (line.trim()) {+ logger.processOutput(name, line);+ }+ });++ if (isError) {+ stderrBuffer = remainingBuffer;+ } else {+ stdoutBuffer = remainingBuffer;}- });+ };++ child.stdout?.on("data", data => processOutput(data.toString(), false));+ child.stderr?.on("data", data => processOutput(data.toString(), true));+child.on("close", code => {+ childProcesses.delete(child);+ logger.processEnd(name, code, process.hrtime.bigint() - startTime);if (code !== 0) {- reject(- new Error(- `Command ${JSON.stringify(command)} failed with code ${code}`,- ),- );+ reject(new Error(`${name} failed with exit code ${code}`));} else {resolve();}});++ child.on("error", error => {+ childProcesses.delete(child);+ logger.processEnd(name, -1, process.hrtime.bigint() - startTime);+ reject(new Error(`${name} failed to start: ${error.message}`));+ });});++ return { promise, process: child };}-(async () => {- if (process.argv[2] !== "--start-docker") {- console.log("=== Installing dependencies and building all components...");- await Promise.all([- (async () => {- if (process.argv[2] !== "--start-built") {- const install = execForward("api@install", "pnpm install");- await install;-- const build = execForward("api@build", "pnpm build");- await build;- } else {- console.log("=== Skipping install and build, using built files...");- }- })(),- execForward(- "sharedLibs/crawler@build",- "cd sharedLibs/crawler && cargo build --release",- ),- execForward(- "sharedLibs/html-transformer@build",- "cd sharedLibs/html-transformer && cargo build --release",- ),- execForward(- "sharedLibs/pdf-parser@build",- "cd sharedLibs/pdf-parser && cargo build --release",- ),- (async () => {- const install = execForward(- "sharedLibs/go-html-to-md@install",- "cd sharedLibs/go-html-to-md && go mod tidy",- );- await install;-- const build = execForward(- "sharedLibs/go-html-to-md@build",- `cd sharedLibs/go-html-to-md && go build -o ${basename(HTML_TO_MARKDOWN_PATH)} -buildmode=c-shared html-to-markdown.go`,- );- await build;- })(),- ]);- }+function terminateProcess(proc: any): Promise<void> {+ return new Promise(resolve => {+ if (!proc || proc.killed || proc.exitCode !== null) {+ resolve();+ return;+ }++ let resolved = false;+ const resolveOnce = () => {+ if (!resolved) {+ resolved = true;+ resolve();+ }+ };- console.log("=== Starting API, Worker, and Index Worker...");+ proc.on("close", resolveOnce);+ proc.on("exit", resolveOnce);+ proc.on("error", resolveOnce);++ try {+ proc.kill("SIGTERM");+ } catch {+ resolveOnce();+ return;+ }++ setTimeout(() => {+ if (!proc.killed && proc.exitCode === null) {+ try {+ proc.kill("SIGKILL");+ } catch {}+ }+ resolveOnce();+ }, 5000);+ });+}++let shuttingDown = false;+async function gracefulShutdown() {+ if (shuttingDown) return;+ shuttingDown = true;++ logger.section("Shutting down");+ const terminationPromises = Array.from(childProcesses).map(terminateProcess);+ await Promise.all(terminationPromises);+ logger.success("All processes terminated");+}++async function buildDependencies() {+ logger.section("Build");++ const tasks = [+ (async () => {+ if (process.argv[2] !== "--start-built") {+ logger.info("Installing API dependencies");+ const install = execForward("api@install", "pnpm install");+ await install.promise;++ logger.info("Building API");+ const build = execForward("api@build", "pnpm build");+ await build.promise;+ } else {+ logger.warn("Skipping API install and build");+ }+ })(),++ (async () => {+ logger.info("Installing Go dependencies");+ const install = execForward(+ "go-html-to-md@install",+ "cd sharedLibs/go-html-to-md && go mod tidy",+ );+ await install.promise;++ logger.info("Building Go module");+ const build = execForward(+ "go-html-to-md@build",+ `cd sharedLibs/go-html-to-md && go build -o ${basename(HTML_TO_MARKDOWN_PATH)} -buildmode=c-shared html-to-markdown.go`,+ );+ await build.promise;+ })(),+ ];++ await Promise.all(tasks);+ logger.success("Build completed");+}++async function startServices() {+ logger.section("Starting services");const api = execForward("api",@@ -128,20 +324,27 @@ function execForward(fancyName: string, command: string): Promise<void> {? "node dist/src/index.js": "pnpm server:production:nobuild",);+const worker = execForward("worker",process.argv[2] === "--start-docker"? "node dist/src/services/queue-worker.js": "pnpm worker:production",);- const nuqWorkers = new Array(5)- .fill(0)- .map((_, i) =>- execForward(- `nuq-worker-${i}`,- `NUQ_WORKER_PORT=${3006 + i} NUQ_REDUCE_NOISE=true ${process.argv[2] === "--start-docker" ? "node dist/src/services/worker/nuq-worker.js" : "pnpm nuq-worker:production"}`,- ),- );++ const nuqWorkers = Array.from({ length: 5 }, (_, i) =>+ execForward(+ `nuq-worker-${i}`,+ process.argv[2] === "--start-docker"+ ? "node dist/src/services/worker/nuq-worker.js"+ : "pnpm nuq-worker:production",+ {+ NUQ_WORKER_PORT: String(3006 + i),+ NUQ_REDUCE_NOISE: "true",+ },+ ),+ );+const indexWorker =process.env.USE_DB_AUTHENTICATION === "true"? execForward(@@ -152,58 +355,106 @@ function execForward(fancyName: string, command: string): Promise<void> {): null;+ logger.info("Waiting for API on localhost:3002");+ await waitForPort(3002, "localhost");+ logger.success("API is ready");++ return {+ api: api.promise,+ worker: worker.promise,+ nuqWorkers: nuqWorkers.map(w => w.promise),+ indexWorker: indexWorker?.promise,+ };+}++async function runCommand(command: string[], services: any) {+ logger.section(`Running: ${command.join(" ")}`);+ const cmd = execForward("command", command);+ await Promise.race([+ cmd.promise,+ services.api,+ services.worker,+ ...services.nuqWorkers,+ ...(services.indexWorker ? [services.indexWorker] : []),+ ]);+}++async function waitForTermination(services: any) {+ logger.info("All services running. Press Ctrl+C to stop");+ await Promise.race([+ new Promise<void>(resolve => {+ process.on("SIGINT", resolve);+ process.on("SIGTERM", resolve);+ }),+ services.api,+ services.worker,+ ...services.nuqWorkers,+ ...(services.indexWorker ? [services.indexWorker] : []),+ ]);+}++function printUsage() {+ console.error(+ `${colors.bold}Usage:${colors.reset} pnpm harness <command...>\n`,+ );+ console.error(`${colors.bold}Special commands:${colors.reset}`);+ console.error(` --start Start services and wait for termination`);+ console.error(` --start-built Start services without rebuilding`);+ console.error(` --start-docker Start services (skip build completely)\n`);+ console.error(+ `The harness ensures dependencies are installed, everything is built,`,+ );+ console.error(`and services are running before executing your command.`);+}++async function main() {+ process.on("SIGINT", gracefulShutdown);+ process.on("SIGTERM", gracefulShutdown);+try {- await Promise.race([- waitForPort(3002, "localhost"),- new Promise(reject =>- setTimeout(() => reject(new Error("API did not start in time")), 10000),- ),- ]);-- if (- process.argv[2] === "--start" ||- process.argv[2] === "--start-built" ||- process.argv[2] === "--start-docker"- ) {- console.log(- "=== Everything is up and running, waiting for termination or failure...",- );- await Promise.race([- new Promise(resolve => {- process.on("SIGINT", resolve);- process.on("SIGTERM", resolve);- }),- api,- worker,- ...nuqWorkers,- ...(indexWorker ? [indexWorker] : []),- ]);+ if (process.argv.length < 3) {+ printUsage();+ process.exit(1);+ }++ const command = process.argv.slice(2);+ const isStartCommand = [+ "--start",+ "--start-built",+ "--start-docker",+ ].includes(command[0]);++ if (command[0] !== "--start-docker") {+ await buildDependencies();+ }++ const services = await startServices();++ if (isStartCommand) {+ await waitForTermination(services);} else {- console.log("=== Running command...");- const cmd = execForward("command", command.join(" "));- await Promise.race([- cmd,- api,- worker,- ...nuqWorkers,- ...(indexWorker ? [indexWorker] : []),- ]);+ await runCommand(command, services);}+ } catch (error: any) {+ logger.error("Fatal error occurred");+ console.error(error?.stack || error?.message || error);+ process.exit(1);} finally {- console.log("=== Tearing down API, Worker, and Index Worker...");- exec("pkill -f 'queue-worker.js'");- exec("pkill -f 'index.js'");- if (indexWorker) {- exec("pkill -f 'index-worker.js'");- }- exec("pkill -f 'nuq-worker.js'");- await Promise.all([- api,- worker,- ...nuqWorkers,- ...(indexWorker ? [indexWorker] : []),- ]);+ await gracefulShutdown();+ logger.info("Goodbye!");}+}- console.log("=== Goodbye!");-})();+process.on("unhandledRejection", async (reason, promise) => {+ logger.error("Unhandled rejection");+ console.error(reason);+ await gracefulShutdown();+ process.exit(1);+});++main().catch(async error => {+ logger.error("Fatal error in main");+ console.error(error?.stack || error?.message || error);+ await gracefulShutdown();+ process.exit(1);+});