Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. π₯
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Perform watching inside of harness (ENG-3514) (#2131)
tomf3129b776565
diff --git a/apps/api/package.json b/apps/api/package.jsonindex 11b35ab97..2c83dc3db 100644--- a/apps/api/package.json+++ b/apps/api/package.json@@ -4,8 +4,8 @@"description": "","main": "index.js","scripts": {- "start": "tsc-watch --onSuccess \"node dist/src/harness.js --start-built\"",- "start:production": "tsx src/harness.ts --start",+ "start": "tsc && node dist/src/harness.js --start-built",+ "dev": "tsx src/harness.ts --start","server": "tsc-watch --onSuccess \"node dist/src/index.js\"","server:production": "tsc && node dist/src/index.js","server:production:nobuild": "node dist/src/index.js",diff --git a/apps/api/src/harness.ts b/apps/api/src/harness.tsindex dea00afb4..3e6a1019a 100644--- a/apps/api/src/harness.ts+++ b/apps/api/src/harness.ts@@ -6,12 +6,21 @@ import { HTML_TO_MARKDOWN_PATH } from "./natives";import { createWriteStream } from "fs";const childProcesses = new Set<ChildProcess>();+const stopping = new WeakSet<ChildProcess>(); // processes we're intentionally stoppinginterface ProcessResult {promise: Promise<void>;process: ChildProcess;}+interface Services {+ api?: ProcessResult;+ worker?: ProcessResult;+ nuqWorkers: ProcessResult[];+ indexWorker?: ProcessResult;+ command?: ProcessResult;+}+const colors = {reset: "\x1b[0m",bold: "\x1b[1m",@@ -37,12 +46,8 @@ const processGroupColors: Record<string, string> = {function getProcessGroup(name: string): string {let group = name;- if (name.includes("@")) {- group = name.split("@")[0];- }- if (name.includes("-")) {- group = name.split("-")[0];- }+ if (name.includes("@")) group = name.split("@")[0];+ if (name.includes("-")) group = name.split("-")[0];return group;}@@ -53,13 +58,9 @@ function getProcessColor(name: string): string {function formatDuration(nanoseconds: bigint): string {const milliseconds = Number(nanoseconds) / 1e6;- if (milliseconds < 1000) {- return `${milliseconds.toFixed(0)}ms`;- }+ if (milliseconds < 1000) return `${milliseconds.toFixed(0)}ms`;const seconds = milliseconds / 1000;- if (seconds < 60) {- return `${seconds.toFixed(1)}s`;- }+ if (seconds < 60) return `${seconds.toFixed(1)}s`;const minutes = Math.floor(seconds / 60);const remainingSeconds = seconds - minutes * 60;return `${minutes}m ${remainingSeconds.toFixed(0)}s`;@@ -70,33 +71,27 @@ const stream = createWriteStream("firecrawl.log");const logger = {section(message: string) {console.log(- `\n${colors.bold}${colors.blue}ββ ${message} ββ${colors.reset}\n`,+ `\n${colors.bold}${colors.blue}ββ ${message} ββ${colors.reset}\n`,);},-info(message: string) {console.log(message);},-success(message: string) {console.log(`${colors.green}β${colors.reset} ${message}`);},-warn(message: string) {console.log(`${colors.yellow}!${colors.reset} ${message}`);},-error(message: string) {console.error(`${colors.red}β${colors.reset} ${message}`);},-processStart(name: string, command: string) {const color = getProcessColor(name);console.log(`${color}>${colors.reset} ${color}${colors.bold}${name}${colors.reset} ${colors.dim}${command}${colors.reset}`,);},-processEnd(name: string, exitCode: number | null, duration: bigint) {const color = getProcessColor(name);const symbol = exitCode === 0 ? "β" : "β";@@ -108,7 +103,6 @@ const logger = {`${symbolColor}${symbol}${colors.reset} ${color}${colors.bold}${name}${colors.reset} ${timing}${codeInfo}`,);},-processOutput(name: string, line: string) {const color = getProcessColor(name);const label = `${color}${name.padEnd(12)}${colors.reset}`;@@ -120,7 +114,7 @@ const logger = {function waitForPort(port: number,host: string,- timeoutMs = 10000,+ timeoutMs = 20000,): Promise<void> {return new Promise((resolve, reject) => {const timeout = setTimeout(() => {@@ -156,19 +150,21 @@ function execForward(): ProcessResult {let child: ChildProcess;let displayCommand = "";+ const isWindows = process.platform === "win32";if (typeof command === "string") {displayCommand = command;- const isWindows = process.platform === "win32";if (isWindows) {child = spawn("cmd", ["/c", command], {env: { ...process.env, ...env },shell: false,+ detached: false,});} else {child = spawn("sh", ["-c", command], {env: { ...process.env, ...env },shell: false,+ detached: true,});}} else {@@ -177,6 +173,7 @@ function execForward(child = spawn(cmd, args, {env: { ...process.env, ...env },shell: false,+ detached: !isWindows,});}@@ -197,16 +194,11 @@ function execForward(const remainingBuffer = lines[lines.length - 1];completeLines.forEach(line => {- if (line.trim()) {- logger.processOutput(name, line);- }+ if (line.trim()) logger.processOutput(name, line);});- if (isError) {- stderrBuffer = remainingBuffer;- } else {- stdoutBuffer = remainingBuffer;- }+ if (isError) stderrBuffer = remainingBuffer;+ else stdoutBuffer = remainingBuffer;};child.stdout?.on("data", data => processOutput(data.toString(), false));@@ -215,7 +207,8 @@ function execForward(child.on("close", code => {childProcesses.delete(child);logger.processEnd(name, code, process.hrtime.bigint() - startTime);- if (code !== 0) {+ const wasStopping = stopping.has(child);+ if (code !== 0 && !wasStopping) {reject(new Error(`${name} failed with exit code ${code}`));} else {resolve();@@ -225,67 +218,64 @@ function execForward(child.on("error", error => {childProcesses.delete(child);logger.processEnd(name, -1, process.hrtime.bigint() - startTime);- reject(new Error(`${name} failed to start: ${error.message}`));+ if (stopping.has(child)) resolve();+ else reject(new Error(`${name} failed to start: ${error.message}`));});});return { promise, process: child };}-function terminateProcess(proc: any): Promise<void> {+function terminateProcess(proc: ChildProcess): Promise<void> {return new Promise(resolve => {if (!proc || proc.killed || proc.exitCode !== null) {resolve();return;}+ stopping.add(proc);+let resolved = false;- const resolveOnce = () => {+ const cleanup = () => {if (!resolved) {resolved = true;resolve();}};- proc.on("close", resolveOnce);- proc.on("exit", resolveOnce);- proc.on("error", resolveOnce);+ proc.once("exit", cleanup);+ proc.once("error", cleanup);- try {- proc.kill("SIGTERM");- } catch {- resolveOnce();- return;- }+ const isWindows = process.platform === "win32";- setTimeout(() => {- if (!proc.killed && proc.exitCode === null) {- try {- proc.kill("SIGKILL");- } catch {}+ if (isWindows && proc.pid) {+ const killer = spawn(+ "taskkill",+ ["/pid", proc.pid.toString(), "/t", "/f"],+ {+ stdio: "ignore",+ },+ );+ killer.on("exit", cleanup);+ } else if (proc.pid) {+ try {+ process.kill(-proc.pid, "SIGTERM");+ } catch {+ proc.kill("SIGTERM");}- resolveOnce();- }, 5000);+ }});}-let shuttingDown = false;-async function gracefulShutdown() {- if (shuttingDown) return;- shuttingDown = true;-- logger.section("Shutting down");- const terminationPromises = Array.from(childProcesses).map(terminateProcess);- await Promise.all(terminationPromises);- logger.success("All processes terminated");-}--async function buildDependencies() {- logger.section("Build");+async function installDependencies() {+ logger.section("Installing dependencies");const tasks = [(async () => {- if (process.argv[2] !== "--start-built") {+ if (+ process.argv[2] !== "--start-built" &&+ process.argv[2] !== "--start-docker"+ ) {logger.info("Installing API dependencies");const install = execForward("api@install", "pnpm install");await install.promise;@@ -301,14 +291,14 @@ async function buildDependencies() {(async () => {logger.info("Installing Go dependencies");const install = execForward(- "go-html-to-md@install",+ "go@install","cd sharedLibs/go-html-to-md && go mod tidy",);await install.promise;logger.info("Building Go module");const build = execForward(- "go-html-to-md@build",+ "go@build",`cd sharedLibs/go-html-to-md && go build -o ${basename(HTML_TO_MARKDOWN_PATH)} -buildmode=c-shared html-to-markdown.go`,);await build.promise;@@ -316,10 +306,10 @@ async function buildDependencies() {];await Promise.all(tasks);- logger.success("Build completed");+ logger.success("Dependencies installed");}-async function startServices() {+function startServices(command?: string[]): Services {logger.section("Starting services");const api = execForward(@@ -357,44 +347,122 @@ async function startServices() {? "node dist/src/services/indexing/index-worker.js": "pnpm index-worker:production",)- : null;+ : undefined;++ const commandProcess =+ command && !command?.[0].startsWith("--")+ ? execForward("command", command)+ : undefined;++ return { api, worker, nuqWorkers, indexWorker, command: commandProcess };+}++async function stopServices(services: Services) {+ const stopPromises = [+ services.api && terminateProcess(services.api.process),+ services.worker && terminateProcess(services.worker.process),+ ...services.nuqWorkers.map(w => terminateProcess(w.process)),+ services.indexWorker && terminateProcess(services.indexWorker.process),+ services.command && terminateProcess(services.command.process),+ ].filter(Boolean);++ await Promise.all(stopPromises);+}++async function runDevMode(): Promise<void> {+ let currentServices: Services | null = null;+ let isFirstStart = true;++ const { TscWatchClient } = await import("tsc-watch");++ const watch = new TscWatchClient();++ watch.on("started", () => {+ logger.info("TypeScript compilation started");+ });++ watch.on("first_success", async () => {+ logger.success("Initial compilation complete");++ currentServices = startServices();++ logger.info("Waiting for API on localhost:3002");+ await waitForPort(3002, "localhost");+ logger.success("API is ready");++ isFirstStart = false;+ });++ watch.on("success", async () => {+ if (!isFirstStart && currentServices) {+ logger.info("Recompilation complete - restarting services");++ await stopServices(currentServices);+ logger.info("Services stopped");++ currentServices = startServices();++ await waitForPort(3002, "localhost");+ logger.success("Services restarted");+ }+ });++ watch.on("compile_errors", () => {+ logger.error("Compilation failed - services not restarted");+ });++ logger.section("Starting development mode");+ watch.start("--project", ".");++ await new Promise<void>(resolve => {+ process.on("SIGINT", resolve);+ process.on("SIGTERM", resolve);+ });++ if (currentServices) {+ await stopServices(currentServices);+ }+ watch.kill();+}++async function runProductionMode(command: string[]): Promise<void> {+ const services = startServices(command);logger.info("Waiting for API on localhost:3002");await waitForPort(3002, "localhost");- logger.success("API is ready");-- return {- api: api.promise,- worker: worker.promise,- nuqWorkers: nuqWorkers.map(w => w.promise),- indexWorker: indexWorker?.promise,- };-}-async function runCommand(command: string[], services: any) {- logger.section(`Running: ${command.join(" ")}`);- const cmd = execForward("command", command);- await Promise.race([- cmd.promise,- services.api,- services.worker,- ...services.nuqWorkers,- ...(services.indexWorker ? [services.indexWorker] : []),- ]);+ await waitForTermination(services);}-async function waitForTermination(services: any) {+async function waitForTermination(services: Services): Promise<void> {logger.info("All services running. Press Ctrl+C to stop");- await Promise.race([++ const promises: Promise<void>[] = [new Promise<void>(resolve => {process.on("SIGINT", resolve);process.on("SIGTERM", resolve);}),- services.api,- services.worker,- ...services.nuqWorkers,- ...(services.indexWorker ? [services.indexWorker] : []),- ]);+ ];++ if (services.command) promises.push(services.command.promise);+ if (services.api) promises.push(services.api.promise);+ if (services.worker) promises.push(services.worker.promise);+ if (services.indexWorker) promises.push(services.indexWorker.promise);++ promises.push(...services.nuqWorkers.map(w => w.promise));++ await Promise.race(promises);+}++let shuttingDown = false;+async function gracefulShutdown() {+ if (shuttingDown) return;+ shuttingDown = true;++ logger.section("Shutting down");+ const terminationPromises = Array.from(childProcesses).map(terminateProcess);+ await Promise.all(terminationPromises);+ logger.success("All processes terminated");}function printUsage() {@@ -402,13 +470,11 @@ function printUsage() {`${colors.bold}Usage:${colors.reset} pnpm harness <command...>\n`,);console.error(`${colors.bold}Special commands:${colors.reset}`);- console.error(` --start Start services and wait for termination`);- console.error(` --start-built Start services without rebuilding`);- console.error(` --start-docker Start services (skip build completely)\n`);console.error(- `The harness ensures dependencies are installed, everything is built,`,+ ` --start Start in development mode (auto-restart on changes)`,);- console.error(`and services are running before executing your command.`);+ console.error(` --start-built Start services without rebuilding`);+ console.error(` --start-docker Start services (skip install, assume built)`);}async function main() {@@ -422,22 +488,16 @@ async function main() {}const command = process.argv.slice(2);- const isStartCommand = [- "--start",- "--start-built",- "--start-docker",- ].includes(command[0]);+ const isDev = command[0] === "--start";if (command[0] !== "--start-docker") {- await buildDependencies();+ await installDependencies();}- const services = await startServices();-- if (isStartCommand) {- await waitForTermination(services);+ if (isDev) {+ await runDevMode();} else {- await runCommand(command, services);+ await runProductionMode(command);}} catch (error: any) {logger.error("Fatal error occurred");diff --git a/examples/kubernetes/firecrawl-helm/templates/deployment.yaml b/examples/kubernetes/firecrawl-helm/templates/deployment.yamlindex 20d772fb1..ba0be85fe 100644--- a/examples/kubernetes/firecrawl-helm/templates/deployment.yaml+++ b/examples/kubernetes/firecrawl-helm/templates/deployment.yaml@@ -22,7 +22,7 @@ spec:- name: apiimage: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"imagePullPolicy: {{ .Values.image.pullPolicy }}- args: [ "pnpm", "run", "start:production" ]+ args: [ "pnpm", "run", "start" ]ports:- containerPort: {{ .Values.service.api.port }}env: