Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(api/dx): add test harness

Gergő Móricz634d391b3bd7
diff --git a/.gitignore b/.gitignore
index 37baa733d..fad26481d 100644
--- a/.gitignore
+++ b/.gitignore
@@ -40,4 +40,6 @@ apps/js-sdk/firecrawl/dist
.vscode
llm-links.txt
mapped-links.txt
-gke-key.json
\ No newline at end of file
+gke-key.json
+
+CLAUDE.local.md
diff --git a/apps/api/package.json b/apps/api/package.json
index 5d724dc75..41b794641 100644
--- a/apps/api/package.json
+++ b/apps/api/package.json
@@ -17,6 +17,7 @@
"test:full": "npx jest --detectOpenHandles --forceExit --openHandlesTimeout=120000 --watchAll=false --testPathIgnorePatterns='(src/__tests__/e2e_noAuth|src/__tests__/e2e_withAuth)'",
"test:prod": "npx jest --detectOpenHandles --forceExit --openHandlesTimeout=120000 --watchAll=false --testPathIgnorePatterns='(src/__tests__/e2e_noAuth|src/__tests__/e2e_full_withAuth|src/scraper/scrapeURL)'",
"test:snips": "npx jest --detectOpenHandles --forceExit --openHandlesTimeout=120000 --watchAll=false src/__tests__/snips/*.test.ts",
+ "harness": "tsx src/harness.ts",
"workers": "tsc-watch --onSuccess \"node dist/src/services/queue-worker.js\"",
"worker:production": "node dist/src/services/queue-worker.js",
"index-worker": "tsc-watch --onSuccess \"node dist/src/services/indexing/index-worker.js\"",
@@ -50,6 +51,7 @@
"ts-jest": "^29.1.1",
"ts-node": "^10.9.1",
"tsc-watch": "^7.1.1",
+ "tsx": "^4.20.3",
"typescript": "^5.8.3"
},
"dependencies": {
diff --git a/apps/api/pnpm-lock.yaml b/apps/api/pnpm-lock.yaml
index 1567f780b..4cb562420 100644
--- a/apps/api/pnpm-lock.yaml
+++ b/apps/api/pnpm-lock.yaml
@@ -336,6 +336,9 @@ importers:
tsc-watch:
specifier: ^7.1.1
version: 7.1.1([email protected])
+ tsx:
+ specifier: ^4.20.3
+ version: 4.20.3
typescript:
specifier: ^5.8.3
version: 5.8.3
@@ -907,6 +910,162 @@ packages:
resolution: {integrity: sha512-v2gz0V6DiuR2TsALM32TkBThf6LdjLbxe6HS/nx9/KJxuDX0Z7SGX7N7PvQfqIvRyus42jI9poVUqezc/j/aQw==}
+ '@esbuild/[email protected]':
+ resolution: {integrity: sha512-urAvrUedIqEiFR3FYSLTWQgLu5tb+m0qZw0NBEasUeo6wuqatkMDaRT+1uABiGXEu5vqgPd7FGE1BhsAIy9QVA==}
+ engines: {node: '>=18'}
+ cpu: [ppc64]
+ os: [aix]
+
+ '@esbuild/[email protected]':
+ resolution: {integrity: sha512-OD3p7LYzWpLhZEyATcTSJ67qB5D+20vbtr6vHlHWSQYhKtzUYrETuWThmzFpZtFsBIxRvhO07+UgVA9m0i/O1w==}
+ engines: {node: '>=18'}
+ cpu: [arm64]
+ os: [android]
+
+ '@esbuild/[email protected]':
+ resolution: {integrity: sha512-RONsAvGCz5oWyePVnLdZY/HHwA++nxYWIX1atInlaW6SEkwq6XkP3+cb825EUcRs5Vss/lGh/2YxAb5xqc07Uw==}
+ engines: {node: '>=18'}
+ cpu: [arm]
+ os: [android]
+
+ '@esbuild/[email protected]':
+ resolution: {integrity: sha512-yJAVPklM5+4+9dTeKwHOaA+LQkmrKFX96BM0A/2zQrbS6ENCmxc4OVoBs5dPkCCak2roAD+jKCdnmOqKszPkjA==}
+ engines: {node: '>=18'}
+ cpu: [x64]
+ os: [android]
+
+ '@esbuild/[email protected]':
+ resolution: {integrity: sha512-Jw0mxgIaYX6R8ODrdkLLPwBqHTtYHJSmzzd+QeytSugzQ0Vg4c5rDky5VgkoowbZQahCbsv1rT1KW72MPIkevw==}
+ engines: {node: '>=18'}
+ cpu: [arm64]
+ os: [darwin]
+
+ '@esbuild/[email protected]':
+ resolution: {integrity: sha512-Vh2gLxxHnuoQ+GjPNvDSDRpoBCUzY4Pu0kBqMBDlK4fuWbKgGtmDIeEC081xi26PPjn+1tct+Bh8FjyLlw1Zlg==}
+ engines: {node: '>=18'}
+ cpu: [x64]
+ os: [darwin]
+
+ '@esbuild/[email protected]':
+ resolution: {integrity: sha512-YPJ7hDQ9DnNe5vxOm6jaie9QsTwcKedPvizTVlqWG9GBSq+BuyWEDazlGaDTC5NGU4QJd666V0yqCBL2oWKPfA==}
+ engines: {node: '>=18'}
+ cpu: [arm64]
+ os: [freebsd]
+
+ '@esbuild/[email protected]':
+ resolution: {integrity: sha512-MmaEXxQRdXNFsRN/KcIimLnSJrk2r5H8v+WVafRWz5xdSVmWLoITZQXcgehI2ZE6gioE6HirAEToM/RvFBeuhw==}
+ engines: {node: '>=18'}
+ cpu: [x64]
+ os: [freebsd]
+
+ '@esbuild/[email protected]':
+ resolution: {integrity: sha512-WIgg00ARWv/uYLU7lsuDK00d/hHSfES5BzdWAdAig1ioV5kaFNrtK8EqGcUBJhYqotlUByUKz5Qo6u8tt7iD/w==}
+ engines: {node: '>=18'}
+ cpu: [arm64]
+ os: [linux]
+
+ '@esbuild/[email protected]':
+ resolution: {integrity: sha512-FuzEP9BixzZohl1kLf76KEVOsxtIBFwCaLupVuk4eFVnOZfU+Wsn+x5Ryam7nILV2pkq2TqQM9EZPsOBuMC+kg==}
+ engines: {node: '>=18'}
+ cpu: [arm]
+ os: [linux]
+
+ '@esbuild/[email protected]':
+ resolution: {integrity: sha512-A1D9YzRX1i+1AJZuFFUMP1E9fMaYY+GnSQil9Tlw05utlE86EKTUA7RjwHDkEitmLYiFsRd9HwKBPEftNdBfjg==}
+ engines: {node: '>=18'}
+ cpu: [ia32]
+ os: [linux]
+
+ '@esbuild/[email protected]':
+ resolution: {integrity: sha512-O7k1J/dwHkY1RMVvglFHl1HzutGEFFZ3kNiDMSOyUrB7WcoHGf96Sh+64nTRT26l3GMbCW01Ekh/ThKM5iI7hQ==}
+ engines: {node: '>=18'}
+ cpu: [loong64]
+ os: [linux]
+
+ '@esbuild/[email protected]':
+ resolution: {integrity: sha512-uv+dqfRazte3BzfMp8PAQXmdGHQt2oC/y2ovwpTteqrMx2lwaksiFZ/bdkXJC19ttTvNXBuWH53zy/aTj1FgGw==}
+ engines: {node: '>=18'}
+ cpu: [mips64el]
+ os: [linux]
+
+ '@esbuild/[email protected]':
+ resolution: {integrity: sha512-GyG0KcMi1GBavP5JgAkkstMGyMholMDybAf8wF5A70CALlDM2p/f7YFE7H92eDeH/VBtFJA5MT4nRPDGg4JuzQ==}
+ engines: {node: '>=18'}
+ cpu: [ppc64]
+ os: [linux]
+
+ '@esbuild/[email protected]':
+ resolution: {integrity: sha512-rAqDYFv3yzMrq7GIcen3XP7TUEG/4LK86LUPMIz6RT8A6pRIDn0sDcvjudVZBiiTcZCY9y2SgYX2lgK3AF+1eg==}
+ engines: {node: '>=18'}
+ cpu: [riscv64]
+ os: [linux]
+
+ '@esbuild/[email protected]':
+ resolution: {integrity: sha512-Xutvh6VjlbcHpsIIbwY8GVRbwoviWT19tFhgdA7DlenLGC/mbc3lBoVb7jxj9Z+eyGqvcnSyIltYUrkKzWqSvg==}
+ engines: {node: '>=18'}
+ cpu: [s390x]
+ os: [linux]
+
+ '@esbuild/[email protected]':
+ resolution: {integrity: sha512-ASFQhgY4ElXh3nDcOMTkQero4b1lgubskNlhIfJrsH5OKZXDpUAKBlNS0Kx81jwOBp+HCeZqmoJuihTv57/jvQ==}
+ engines: {node: '>=18'}
+ cpu: [x64]
+ os: [linux]
+
+ '@esbuild/[email protected]':
+ resolution: {integrity: sha512-d1KfruIeohqAi6SA+gENMuObDbEjn22olAR7egqnkCD9DGBG0wsEARotkLgXDu6c4ncgWTZJtN5vcgxzWRMzcw==}
+ engines: {node: '>=18'}
+ cpu: [arm64]
+ os: [netbsd]
+
+ '@esbuild/[email protected]':
+ resolution: {integrity: sha512-nVDCkrvx2ua+XQNyfrujIG38+YGyuy2Ru9kKVNyh5jAys6n+l44tTtToqHjino2My8VAY6Lw9H7RI73XFi66Cg==}
+ engines: {node: '>=18'}
+ cpu: [x64]
+ os: [netbsd]
+
+ '@esbuild/[email protected]':
+ resolution: {integrity: sha512-j8HgrDuSJFAujkivSMSfPQSAa5Fxbvk4rgNAS5i3K+r8s1X0p1uOO2Hl2xNsGFppOeHOLAVgYwDVlmxhq5h+SQ==}
+ engines: {node: '>=18'}
+ cpu: [arm64]
+ os: [openbsd]
+
+ '@esbuild/[email protected]':
+ resolution: {integrity: sha512-1h8MUAwa0VhNCDp6Af0HToI2TJFAn1uqT9Al6DJVzdIBAd21m/G0Yfc77KDM3uF3T/YaOgQq3qTJHPbTOInaIQ==}
+ engines: {node: '>=18'}
+ cpu: [x64]
+ os: [openbsd]
+
+ '@esbuild/[email protected]':
+ resolution: {integrity: sha512-r2nVa5SIK9tSWd0kJd9HCffnDHKchTGikb//9c7HX+r+wHYCpQrSgxhlY6KWV1nFo1l4KFbsMlHk+L6fekLsUg==}
+ engines: {node: '>=18'}
+ cpu: [arm64]
+ os: [openharmony]
+
+ '@esbuild/[email protected]':
+ resolution: {integrity: sha512-zUlaP2S12YhQ2UzUfcCuMDHQFJyKABkAjvO5YSndMiIkMimPmxA+BYSBikWgsRpvyxuRnow4nS5NPnf9fpv41w==}
+ engines: {node: '>=18'}
+ cpu: [x64]
+ os: [sunos]
+
+ '@esbuild/[email protected]':
+ resolution: {integrity: sha512-YEGFFWESlPva8hGL+zvj2z/SaK+pH0SwOM0Nc/d+rVnW7GSTFlLBGzZkuSU9kFIGIo8q9X3ucpZhu8PDN5A2sQ==}
+ engines: {node: '>=18'}
+ cpu: [arm64]
+ os: [win32]
+
+ '@esbuild/[email protected]':
+ resolution: {integrity: sha512-hiGgGC6KZ5LZz58OL/+qVVoZiuZlUYlYHNAmczOm7bs2oE1XriPFi5ZHHrS8ACpV5EjySrnoCKmcbQMN+ojnHg==}
+ engines: {node: '>=18'}
+ cpu: [ia32]
+ os: [win32]
+
+ '@esbuild/[email protected]':
+ resolution: {integrity: sha512-cn3Yr7+OaaZq1c+2pe+8yxC8E144SReCQjN6/2ynubzYjvyqZjTXfQJpAcQpsdJq3My7XADANiYGHoFC69pLQw==}
+ engines: {node: '>=18'}
+ cpu: [x64]
+ os: [win32]
+
'@google-cloud/[email protected]':
resolution: {integrity: sha512-DJS3s0OVH4zFDB1PzjxAsHqJT6sKVbRwwML0ZBP9PbU7Yebtu/7SWMRzvO2J3nUi9pRNITCfu4LJeooM2w4pjg==}
engines: {node: '>=14.0.0'}
@@ -2442,6 +2601,11 @@ packages:
resolution: {integrity: sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA==}
engines: {node: '>= 0.4'}
+ resolution: {integrity: sha512-vVC0USHGtMi8+R4Kz8rt6JhEWLxsv9Rnu/lGYbPR8u47B+DCBksq9JarW0zOO7bs37hyOK1l2/oqtbciutL5+Q==}
+ engines: {node: '>=18'}
+ hasBin: true
+
resolution: {integrity: sha512-ErCHMCae19vR8vQGe50xIsVomy19rg6gFu3+r3jkEO46suLMWBksvVyoGgQV+jOfl84ZSOSlmv6Gxa89PmTGmA==}
engines: {node: '>=6'}
@@ -2660,6 +2824,9 @@ packages:
resolution: {integrity: sha512-ts6Wi+2j3jQjqi70w5AlN8DFnkSwC+MqmxEzdEALB2qXZYV3X/b1CTfgPLGJNMeAWxdPfU8FO1ms3NUfaHCPYg==}
engines: {node: '>=10'}
+ resolution: {integrity: sha512-auHyJ4AgMz7vgS8Hp3N6HXSmlMdUyhSUrfBF16w153rxtLIEOE+HGqaBppczZvnHLqQJfiHotCYpNhl0lUROFQ==}
+
resolution: {integrity: sha512-/Iu4prUrydE3Pb3lCBMbcSNIf81tgGt0W1ZwknnyF62t3tHmtiJTRj0f+1ZIhp3+Rh0ktz1pJVoa7ZXUCskivA==}
engines: {node: '>= 4.8.0'}
@@ -3744,6 +3911,9 @@ packages:
resolution: {integrity: sha512-qYg9KP24dD5qka9J47d0aVky0N+b4fTU89LN9iDnjB5waksiC49rvMB0PrUJQGoTmH50XPiqOvAjDfaijGxYZw==}
engines: {node: '>=8'}
+ resolution: {integrity: sha512-seS2Tj26TBVOC2NIc2rOe2y2ZO7efxITtLZcGSOnHHNOQ7CkiUBfw0Iw2ck6xkIhPwLhKNLS8BO+hEpngQlqzw==}
+
resolution: {integrity: sha512-X2UW6Nw3n/aMgDVy+0rSqgHlv39WZAlZrXCdnbyEiKm17DSqHX4MmQMaST3FbeWR5FTuRcUwYAziZajji0Y7mg==}
engines: {node: '>=10'}
@@ -4130,6 +4300,11 @@ packages:
resolution: {integrity: sha512-xNvxJEOUiWPGhUuUdQgAJPKOOJfGnIyKySOc09XkKsgdUV/3E2zvwZYdejjmRgPCgcym1juLH3226yA7sEFJKQ==}
+ resolution: {integrity: sha512-qjbnuR9Tr+FJOMBqJCW5ehvIo/buZq7vH7qD7JziU98h6l3qGy0a/yPFjwO+y0/T7GFpNgNAvEcPPVfyT8rrPQ==}
+ engines: {node: '>=18.0.0'}
+ hasBin: true
+
resolution: {integrity: sha512-vwz9tfvF7XN/jE0dGoBei3FXWuvll78ohzCZQuOb+ZjWrs3a0XhQVomJEb2Qh4VHTPNRO4GPZh0V7VRbiWwkRg==}
@@ -5368,6 +5543,84 @@ snapshots:
'@dqbd/[email protected]': {}
+ '@esbuild/[email protected]':
+ optional: true
+
+ '@esbuild/[email protected]':
+ optional: true
+
+ '@esbuild/[email protected]':
+ optional: true
+
+ '@esbuild/[email protected]':
+ optional: true
+
+ '@esbuild/[email protected]':
+ optional: true
+
+ '@esbuild/[email protected]':
+ optional: true
+
+ '@esbuild/[email protected]':
+ optional: true
+
+ '@esbuild/[email protected]':
+ optional: true
+
+ '@esbuild/[email protected]':
+ optional: true
+
+ '@esbuild/[email protected]':
+ optional: true
+
+ '@esbuild/[email protected]':
+ optional: true
+
+ '@esbuild/[email protected]':
+ optional: true
+
+ '@esbuild/[email protected]':
+ optional: true
+
+ '@esbuild/[email protected]':
+ optional: true
+
+ '@esbuild/[email protected]':
+ optional: true
+
+ '@esbuild/[email protected]':
+ optional: true
+
+ '@esbuild/[email protected]':
+ optional: true
+
+ '@esbuild/[email protected]':
+ optional: true
+
+ '@esbuild/[email protected]':
+ optional: true
+
+ '@esbuild/[email protected]':
+ optional: true
+
+ '@esbuild/[email protected]':
+ optional: true
+
+ '@esbuild/[email protected]':
+ optional: true
+
+ '@esbuild/[email protected]':
+ optional: true
+
+ '@esbuild/[email protected]':
+ optional: true
+
+ '@esbuild/[email protected]':
+ optional: true
+
+ '@esbuild/[email protected]':
+ optional: true
+
'@google-cloud/[email protected]':
dependencies:
arrify: 2.0.1
@@ -7286,6 +7539,35 @@ snapshots:
has-tostringtag: 1.0.2
hasown: 2.0.2
+ optionalDependencies:
+ '@esbuild/aix-ppc64': 0.25.8
+ '@esbuild/android-arm': 0.25.8
+ '@esbuild/android-arm64': 0.25.8
+ '@esbuild/android-x64': 0.25.8
+ '@esbuild/darwin-arm64': 0.25.8
+ '@esbuild/darwin-x64': 0.25.8
+ '@esbuild/freebsd-arm64': 0.25.8
+ '@esbuild/freebsd-x64': 0.25.8
+ '@esbuild/linux-arm': 0.25.8
+ '@esbuild/linux-arm64': 0.25.8
+ '@esbuild/linux-ia32': 0.25.8
+ '@esbuild/linux-loong64': 0.25.8
+ '@esbuild/linux-mips64el': 0.25.8
+ '@esbuild/linux-ppc64': 0.25.8
+ '@esbuild/linux-riscv64': 0.25.8
+ '@esbuild/linux-s390x': 0.25.8
+ '@esbuild/linux-x64': 0.25.8
+ '@esbuild/netbsd-arm64': 0.25.8
+ '@esbuild/netbsd-x64': 0.25.8
+ '@esbuild/openbsd-arm64': 0.25.8
+ '@esbuild/openbsd-x64': 0.25.8
+ '@esbuild/openharmony-arm64': 0.25.8
+ '@esbuild/sunos-x64': 0.25.8
+ '@esbuild/win32-arm64': 0.25.8
+ '@esbuild/win32-ia32': 0.25.8
+ '@esbuild/win32-x64': 0.25.8
+
@@ -7551,6 +7833,10 @@ snapshots:
+ dependencies:
+ resolve-pkg-maps: 1.0.0
+
dependencies:
chalk: 2.4.2
@@ -8830,6 +9116,8 @@ snapshots:
+
@@ -9236,6 +9524,13 @@ snapshots:
+ dependencies:
+ esbuild: 0.25.8
+ get-tsconfig: 4.10.1
+ optionalDependencies:
+ fsevents: 2.3.3
+
diff --git a/apps/api/src/harness.ts b/apps/api/src/harness.ts
new file mode 100644
index 000000000..1718fbc02
--- /dev/null
+++ b/apps/api/src/harness.ts
@@ -0,0 +1,122 @@
+import { exec } from "child_process";
+import * as net from "net";
+
+function waitForPort(port: number, host: string): Promise<void> {
+ return new Promise((resolve) => {
+ const checkPort = () => {
+ const socket = new net.Socket();
+
+ const onError = () => {
+ socket.destroy();
+ setTimeout(checkPort, 1000); // Try again in 1 second
+ };
+
+ socket.once('error', onError);
+
+ socket.connect(port, host, () => {
+ socket.destroy();
+ resolve();
+ });
+ };
+
+ checkPort();
+ });
+}
+
+if (process.argv.length < 3) {
+ console.error("Usage: pnpm harness <command...>");
+ console.error()
+ console.error("The harness ensures that the dependencies are up to date, everything is built, and the API and Worker are running, before running a command that would require the above.");
+ console.error("It also tears down the API and Worker after the command is run.");
+ process.exit(1);
+}
+
+const command = process.argv.slice(2);
+
+function execForward(fancyName: string, command: string): Promise<void> {
+ return new Promise((resolve, reject) => {
+ const child = exec(command);
+ let stdoutBuffer = "";
+ let stderrBuffer = "";
+ child.stdout?.on("data", (data) => {
+ stdoutBuffer += data;
+ while(stdoutBuffer.includes("\n")) {
+ const split = stdoutBuffer.split("\n");
+ const line = split[0];
+ stdoutBuffer = split.slice(1).join("\n");
+ process.stdout.write(`[${fancyName}] ${line}\n`);
+ }
+ });
+ child.stderr?.on("data", (data) => {
+ stderrBuffer += data;
+ while(stderrBuffer.includes("\n")) {
+ const split = stderrBuffer.split("\n");
+ const line = split[0];
+ stderrBuffer = split.slice(1).join("\n");
+ process.stderr.write(`[${fancyName}] ${line}\n`);
+ }
+ });
+ child.on("close", (code) => {
+ if (code !== 0) {
+ reject(new Error(`Command ${JSON.stringify(command)} failed with code ${code}`));
+ } else {
+ resolve();
+ }
+ });
+ });
+}
+
+(async () => {
+ console.log("=== Installing dependencies and building all components...");
+ await Promise.all([
+ (async () => {
+ const install = execForward("api@install", "pnpm install");
+ await install;
+
+ const build = execForward("api@build", "pnpm build");
+ await build;
+ })(),
+ execForward("sharedLibs/crawler@build", "cd sharedLibs/crawler && cargo build --release"),
+ execForward("sharedLibs/html-transformer@build", "cd sharedLibs/html-transformer && cargo build --release"),
+ execForward("sharedLibs/pdf-parser@build", "cd sharedLibs/pdf-parser && cargo build --release"),
+ (async () => {
+ const install = execForward("sharedLibs/go-html-to-md@install", "cd sharedLibs/go-html-to-md && go mod tidy");
+ await install;
+
+ const build = execForward("sharedLibs/go-html-to-md@build", "cd sharedLibs/go-html-to-md && go build -o html-to-markdown.so -buildmode=c-shared html-to-markdown.go");
+ await build;
+ })(),
+ ]);
+
+ console.log("=== Starting API and Worker...");
+
+ const api = execForward("api", "pnpm start:production:nobuild");
+ const worker = execForward("worker", "pnpm worker:production");
+
+ try {
+ await Promise.race([
+ waitForPort(3002, "localhost"),
+ new Promise((reject) => setTimeout(() => reject(new Error("API did not start in time")), 10000)),
+ ]);
+
+ console.log("=== Running command...");
+
+ const cmd = execForward("command", command.join(" "));
+
+ await Promise.race([
+ cmd,
+ api,
+ worker,
+ ]);
+ } finally {
+ console.log("=== Tearing down API and Worker...");
+ exec("pkill -f 'queue-worker.js'");
+ exec("pkill -f 'index.js'");
+ await Promise.all([
+ api,
+ worker,
+ ]);
+ }
+
+ console.log("=== Goodbye!");
+})();
\ No newline at end of file
diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.ts
index d68659daa..5baa7528b 100644
--- a/apps/api/src/services/queue-worker.ts
+++ b/apps/api/src/services/queue-worker.ts
@@ -458,7 +458,7 @@ app.get("/liveness", (req, res) => {
}
});
-const workerPort = process.env.PORT || 3005;
+const workerPort = process.env.WORKER_PORT || process.env.PORT || 3005;
app.listen(workerPort, () => {
_logger.info(`Liveness endpoint is running on port ${workerPort}`);
});