Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. πŸ”₯

Perform watching inside of harness (ENG-3514) (#2131)

tomf3129b776565
diff --git a/apps/api/package.json b/apps/api/package.json
index 11b35ab97..2c83dc3db 100644
--- a/apps/api/package.json
+++ b/apps/api/package.json
@@ -4,8 +4,8 @@
"description": "",
"main": "index.js",
"scripts": {
- "start": "tsc-watch --onSuccess \"node dist/src/harness.js --start-built\"",
- "start:production": "tsx src/harness.ts --start",
+ "start": "tsc && node dist/src/harness.js --start-built",
+ "dev": "tsx src/harness.ts --start",
"server": "tsc-watch --onSuccess \"node dist/src/index.js\"",
"server:production": "tsc && node dist/src/index.js",
"server:production:nobuild": "node dist/src/index.js",
diff --git a/apps/api/src/harness.ts b/apps/api/src/harness.ts
index dea00afb4..3e6a1019a 100644
--- a/apps/api/src/harness.ts
+++ b/apps/api/src/harness.ts
@@ -6,12 +6,21 @@ import { HTML_TO_MARKDOWN_PATH } from "./natives";
import { createWriteStream } from "fs";
const childProcesses = new Set<ChildProcess>();
+const stopping = new WeakSet<ChildProcess>(); // processes we're intentionally stopping
interface ProcessResult {
promise: Promise<void>;
process: ChildProcess;
}
+interface Services {
+ api?: ProcessResult;
+ worker?: ProcessResult;
+ nuqWorkers: ProcessResult[];
+ indexWorker?: ProcessResult;
+ command?: ProcessResult;
+}
+
const colors = {
reset: "\x1b[0m",
bold: "\x1b[1m",
@@ -37,12 +46,8 @@ const processGroupColors: Record<string, string> = {
function getProcessGroup(name: string): string {
let group = name;
- if (name.includes("@")) {
- group = name.split("@")[0];
- }
- if (name.includes("-")) {
- group = name.split("-")[0];
- }
+ if (name.includes("@")) group = name.split("@")[0];
+ if (name.includes("-")) group = name.split("-")[0];
return group;
}
@@ -53,13 +58,9 @@ function getProcessColor(name: string): string {
function formatDuration(nanoseconds: bigint): string {
const milliseconds = Number(nanoseconds) / 1e6;
- if (milliseconds < 1000) {
- return `${milliseconds.toFixed(0)}ms`;
- }
+ if (milliseconds < 1000) return `${milliseconds.toFixed(0)}ms`;
const seconds = milliseconds / 1000;
- if (seconds < 60) {
- return `${seconds.toFixed(1)}s`;
- }
+ if (seconds < 60) return `${seconds.toFixed(1)}s`;
const minutes = Math.floor(seconds / 60);
const remainingSeconds = seconds - minutes * 60;
return `${minutes}m ${remainingSeconds.toFixed(0)}s`;
@@ -70,33 +71,27 @@ const stream = createWriteStream("firecrawl.log");
const logger = {
section(message: string) {
console.log(
- `\n${colors.bold}${colors.blue}━━ ${message} ━━${colors.reset}\n`,
+ `\n${colors.bold}${colors.blue}── ${message} ──${colors.reset}\n`,
);
},
-
info(message: string) {
console.log(message);
},
-
success(message: string) {
console.log(`${colors.green}βœ“${colors.reset} ${message}`);
},
-
warn(message: string) {
console.log(`${colors.yellow}!${colors.reset} ${message}`);
},
-
error(message: string) {
console.error(`${colors.red}βœ—${colors.reset} ${message}`);
},
-
processStart(name: string, command: string) {
const color = getProcessColor(name);
console.log(
`${color}>${colors.reset} ${color}${colors.bold}${name}${colors.reset} ${colors.dim}${command}${colors.reset}`,
);
},
-
processEnd(name: string, exitCode: number | null, duration: bigint) {
const color = getProcessColor(name);
const symbol = exitCode === 0 ? "●" : "βœ—";
@@ -108,7 +103,6 @@ const logger = {
`${symbolColor}${symbol}${colors.reset} ${color}${colors.bold}${name}${colors.reset} ${timing}${codeInfo}`,
);
},
-
processOutput(name: string, line: string) {
const color = getProcessColor(name);
const label = `${color}${name.padEnd(12)}${colors.reset}`;
@@ -120,7 +114,7 @@ const logger = {
function waitForPort(
port: number,
host: string,
- timeoutMs = 10000,
+ timeoutMs = 20000,
): Promise<void> {
return new Promise((resolve, reject) => {
const timeout = setTimeout(() => {
@@ -156,19 +150,21 @@ function execForward(
): ProcessResult {
let child: ChildProcess;
let displayCommand = "";
+ const isWindows = process.platform === "win32";
if (typeof command === "string") {
displayCommand = command;
- const isWindows = process.platform === "win32";
if (isWindows) {
child = spawn("cmd", ["/c", command], {
env: { ...process.env, ...env },
shell: false,
+ detached: false,
});
} else {
child = spawn("sh", ["-c", command], {
env: { ...process.env, ...env },
shell: false,
+ detached: true,
});
}
} else {
@@ -177,6 +173,7 @@ function execForward(
child = spawn(cmd, args, {
env: { ...process.env, ...env },
shell: false,
+ detached: !isWindows,
});
}
@@ -197,16 +194,11 @@ function execForward(
const remainingBuffer = lines[lines.length - 1];
completeLines.forEach(line => {
- if (line.trim()) {
- logger.processOutput(name, line);
- }
+ if (line.trim()) logger.processOutput(name, line);
});
- if (isError) {
- stderrBuffer = remainingBuffer;
- } else {
- stdoutBuffer = remainingBuffer;
- }
+ if (isError) stderrBuffer = remainingBuffer;
+ else stdoutBuffer = remainingBuffer;
};
child.stdout?.on("data", data => processOutput(data.toString(), false));
@@ -215,7 +207,8 @@ function execForward(
child.on("close", code => {
childProcesses.delete(child);
logger.processEnd(name, code, process.hrtime.bigint() - startTime);
- if (code !== 0) {
+ const wasStopping = stopping.has(child);
+ if (code !== 0 && !wasStopping) {
reject(new Error(`${name} failed with exit code ${code}`));
} else {
resolve();
@@ -225,67 +218,64 @@ function execForward(
child.on("error", error => {
childProcesses.delete(child);
logger.processEnd(name, -1, process.hrtime.bigint() - startTime);
- reject(new Error(`${name} failed to start: ${error.message}`));
+ if (stopping.has(child)) resolve();
+ else reject(new Error(`${name} failed to start: ${error.message}`));
});
});
return { promise, process: child };
}
-function terminateProcess(proc: any): Promise<void> {
+function terminateProcess(proc: ChildProcess): Promise<void> {
return new Promise(resolve => {
if (!proc || proc.killed || proc.exitCode !== null) {
resolve();
return;
}
+ stopping.add(proc);
+
let resolved = false;
- const resolveOnce = () => {
+ const cleanup = () => {
if (!resolved) {
resolved = true;
resolve();
}
};
- proc.on("close", resolveOnce);
- proc.on("exit", resolveOnce);
- proc.on("error", resolveOnce);
+ proc.once("exit", cleanup);
+ proc.once("error", cleanup);
- try {
- proc.kill("SIGTERM");
- } catch {
- resolveOnce();
- return;
- }
+ const isWindows = process.platform === "win32";
- setTimeout(() => {
- if (!proc.killed && proc.exitCode === null) {
- try {
- proc.kill("SIGKILL");
- } catch {}
+ if (isWindows && proc.pid) {
+ const killer = spawn(
+ "taskkill",
+ ["/pid", proc.pid.toString(), "/t", "/f"],
+ {
+ stdio: "ignore",
+ },
+ );
+ killer.on("exit", cleanup);
+ } else if (proc.pid) {
+ try {
+ process.kill(-proc.pid, "SIGTERM");
+ } catch {
+ proc.kill("SIGTERM");
}
- resolveOnce();
- }, 5000);
+ }
});
}
-let shuttingDown = false;
-async function gracefulShutdown() {
- if (shuttingDown) return;
- shuttingDown = true;
-
- logger.section("Shutting down");
- const terminationPromises = Array.from(childProcesses).map(terminateProcess);
- await Promise.all(terminationPromises);
- logger.success("All processes terminated");
-}
-
-async function buildDependencies() {
- logger.section("Build");
+async function installDependencies() {
+ logger.section("Installing dependencies");
const tasks = [
(async () => {
- if (process.argv[2] !== "--start-built") {
+ if (
+ process.argv[2] !== "--start-built" &&
+ process.argv[2] !== "--start-docker"
+ ) {
logger.info("Installing API dependencies");
const install = execForward("api@install", "pnpm install");
await install.promise;
@@ -301,14 +291,14 @@ async function buildDependencies() {
(async () => {
logger.info("Installing Go dependencies");
const install = execForward(
- "go-html-to-md@install",
+ "go@install",
"cd sharedLibs/go-html-to-md && go mod tidy",
);
await install.promise;
logger.info("Building Go module");
const build = execForward(
- "go-html-to-md@build",
+ "go@build",
`cd sharedLibs/go-html-to-md && go build -o ${basename(HTML_TO_MARKDOWN_PATH)} -buildmode=c-shared html-to-markdown.go`,
);
await build.promise;
@@ -316,10 +306,10 @@ async function buildDependencies() {
];
await Promise.all(tasks);
- logger.success("Build completed");
+ logger.success("Dependencies installed");
}
-async function startServices() {
+function startServices(command?: string[]): Services {
logger.section("Starting services");
const api = execForward(
@@ -357,44 +347,122 @@ async function startServices() {
? "node dist/src/services/indexing/index-worker.js"
: "pnpm index-worker:production",
)
- : null;
+ : undefined;
+
+ const commandProcess =
+ command && !command?.[0].startsWith("--")
+ ? execForward("command", command)
+ : undefined;
+
+ return { api, worker, nuqWorkers, indexWorker, command: commandProcess };
+}
+
+async function stopServices(services: Services) {
+ const stopPromises = [
+ services.api && terminateProcess(services.api.process),
+ services.worker && terminateProcess(services.worker.process),
+ ...services.nuqWorkers.map(w => terminateProcess(w.process)),
+ services.indexWorker && terminateProcess(services.indexWorker.process),
+ services.command && terminateProcess(services.command.process),
+ ].filter(Boolean);
+
+ await Promise.all(stopPromises);
+}
+
+async function runDevMode(): Promise<void> {
+ let currentServices: Services | null = null;
+ let isFirstStart = true;
+
+ const { TscWatchClient } = await import("tsc-watch");
+
+ const watch = new TscWatchClient();
+
+ watch.on("started", () => {
+ logger.info("TypeScript compilation started");
+ });
+
+ watch.on("first_success", async () => {
+ logger.success("Initial compilation complete");
+
+ currentServices = startServices();
+
+ logger.info("Waiting for API on localhost:3002");
+ await waitForPort(3002, "localhost");
+ logger.success("API is ready");
+
+ isFirstStart = false;
+ });
+
+ watch.on("success", async () => {
+ if (!isFirstStart && currentServices) {
+ logger.info("Recompilation complete - restarting services");
+
+ await stopServices(currentServices);
+ logger.info("Services stopped");
+
+ currentServices = startServices();
+
+ await waitForPort(3002, "localhost");
+ logger.success("Services restarted");
+ }
+ });
+
+ watch.on("compile_errors", () => {
+ logger.error("Compilation failed - services not restarted");
+ });
+
+ logger.section("Starting development mode");
+ watch.start("--project", ".");
+
+ await new Promise<void>(resolve => {
+ process.on("SIGINT", resolve);
+ process.on("SIGTERM", resolve);
+ });
+
+ if (currentServices) {
+ await stopServices(currentServices);
+ }
+ watch.kill();
+}
+
+async function runProductionMode(command: string[]): Promise<void> {
+ const services = startServices(command);
logger.info("Waiting for API on localhost:3002");
await waitForPort(3002, "localhost");
- logger.success("API is ready");
-
- return {
- api: api.promise,
- worker: worker.promise,
- nuqWorkers: nuqWorkers.map(w => w.promise),
- indexWorker: indexWorker?.promise,
- };
-}
-async function runCommand(command: string[], services: any) {
- logger.section(`Running: ${command.join(" ")}`);
- const cmd = execForward("command", command);
- await Promise.race([
- cmd.promise,
- services.api,
- services.worker,
- ...services.nuqWorkers,
- ...(services.indexWorker ? [services.indexWorker] : []),
- ]);
+ await waitForTermination(services);
}
-async function waitForTermination(services: any) {
+async function waitForTermination(services: Services): Promise<void> {
logger.info("All services running. Press Ctrl+C to stop");
- await Promise.race([
+
+ const promises: Promise<void>[] = [
new Promise<void>(resolve => {
process.on("SIGINT", resolve);
process.on("SIGTERM", resolve);
}),
- services.api,
- services.worker,
- ...services.nuqWorkers,
- ...(services.indexWorker ? [services.indexWorker] : []),
- ]);
+ ];
+
+ if (services.command) promises.push(services.command.promise);
+ if (services.api) promises.push(services.api.promise);
+ if (services.worker) promises.push(services.worker.promise);
+ if (services.indexWorker) promises.push(services.indexWorker.promise);
+
+ promises.push(...services.nuqWorkers.map(w => w.promise));
+
+ await Promise.race(promises);
+}
+
+let shuttingDown = false;
+async function gracefulShutdown() {
+ if (shuttingDown) return;
+ shuttingDown = true;
+
+ logger.section("Shutting down");
+ const terminationPromises = Array.from(childProcesses).map(terminateProcess);
+ await Promise.all(terminationPromises);
+ logger.success("All processes terminated");
}
function printUsage() {
@@ -402,13 +470,11 @@ function printUsage() {
`${colors.bold}Usage:${colors.reset} pnpm harness <command...>\n`,
);
console.error(`${colors.bold}Special commands:${colors.reset}`);
- console.error(` --start Start services and wait for termination`);
- console.error(` --start-built Start services without rebuilding`);
- console.error(` --start-docker Start services (skip build completely)\n`);
console.error(
- `The harness ensures dependencies are installed, everything is built,`,
+ ` --start Start in development mode (auto-restart on changes)`,
);
- console.error(`and services are running before executing your command.`);
+ console.error(` --start-built Start services without rebuilding`);
+ console.error(` --start-docker Start services (skip install, assume built)`);
}
async function main() {
@@ -422,22 +488,16 @@ async function main() {
}
const command = process.argv.slice(2);
- const isStartCommand = [
- "--start",
- "--start-built",
- "--start-docker",
- ].includes(command[0]);
+ const isDev = command[0] === "--start";
if (command[0] !== "--start-docker") {
- await buildDependencies();
+ await installDependencies();
}
- const services = await startServices();
-
- if (isStartCommand) {
- await waitForTermination(services);
+ if (isDev) {
+ await runDevMode();
} else {
- await runCommand(command, services);
+ await runProductionMode(command);
}
} catch (error: any) {
logger.error("Fatal error occurred");
diff --git a/examples/kubernetes/firecrawl-helm/templates/deployment.yaml b/examples/kubernetes/firecrawl-helm/templates/deployment.yaml
index 20d772fb1..ba0be85fe 100644
--- a/examples/kubernetes/firecrawl-helm/templates/deployment.yaml
+++ b/examples/kubernetes/firecrawl-helm/templates/deployment.yaml
@@ -22,7 +22,7 @@ spec:
- name: api
image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
imagePullPolicy: {{ .Values.image.pullPolicy }}
- args: [ "pnpm", "run", "start:production" ]
+ args: [ "pnpm", "run", "start" ]
ports:
- containerPort: {{ .Values.service.api.port }}
env: