Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api/dx): add test harness
Gergő Móricz634d391b3bd7
diff --git a/.gitignore b/.gitignoreindex 37baa733d..fad26481d 100644--- a/.gitignore+++ b/.gitignore@@ -40,4 +40,6 @@ apps/js-sdk/firecrawl/dist.vscodellm-links.txtmapped-links.txt-gke-key.json\ No newline at end of file+gke-key.json++CLAUDE.local.mddiff --git a/apps/api/package.json b/apps/api/package.jsonindex 5d724dc75..41b794641 100644--- a/apps/api/package.json+++ b/apps/api/package.json@@ -17,6 +17,7 @@"test:full": "npx jest --detectOpenHandles --forceExit --openHandlesTimeout=120000 --watchAll=false --testPathIgnorePatterns='(src/__tests__/e2e_noAuth|src/__tests__/e2e_withAuth)'","test:prod": "npx jest --detectOpenHandles --forceExit --openHandlesTimeout=120000 --watchAll=false --testPathIgnorePatterns='(src/__tests__/e2e_noAuth|src/__tests__/e2e_full_withAuth|src/scraper/scrapeURL)'","test:snips": "npx jest --detectOpenHandles --forceExit --openHandlesTimeout=120000 --watchAll=false src/__tests__/snips/*.test.ts",+ "harness": "tsx src/harness.ts","workers": "tsc-watch --onSuccess \"node dist/src/services/queue-worker.js\"","worker:production": "node dist/src/services/queue-worker.js","index-worker": "tsc-watch --onSuccess \"node dist/src/services/indexing/index-worker.js\"",@@ -50,6 +51,7 @@"ts-jest": "^29.1.1","ts-node": "^10.9.1","tsc-watch": "^7.1.1",+ "tsx": "^4.20.3","typescript": "^5.8.3"},"dependencies": {diff --git a/apps/api/pnpm-lock.yaml b/apps/api/pnpm-lock.yamlindex 1567f780b..4cb562420 100644--- a/apps/api/pnpm-lock.yaml+++ b/apps/api/pnpm-lock.yaml@@ -336,6 +336,9 @@ importers:tsc-watch:specifier: ^7.1.1version: 7.1.1([email protected])+ tsx:+ specifier: ^4.20.3+ version: 4.20.3typescript:specifier: ^5.8.3version: 5.8.3@@ -907,6 +910,162 @@ packages:'@dqbd/[email protected]':resolution: {integrity: sha512-v2gz0V6DiuR2TsALM32TkBThf6LdjLbxe6HS/nx9/KJxuDX0Z7SGX7N7PvQfqIvRyus42jI9poVUqezc/j/aQw==}+ '@esbuild/[email protected]':+ resolution: {integrity: sha512-urAvrUedIqEiFR3FYSLTWQgLu5tb+m0qZw0NBEasUeo6wuqatkMDaRT+1uABiGXEu5vqgPd7FGE1BhsAIy9QVA==}+ engines: {node: '>=18'}+ cpu: [ppc64]+ os: [aix]++ '@esbuild/[email protected]':+ resolution: {integrity: sha512-OD3p7LYzWpLhZEyATcTSJ67qB5D+20vbtr6vHlHWSQYhKtzUYrETuWThmzFpZtFsBIxRvhO07+UgVA9m0i/O1w==}+ engines: {node: '>=18'}+ cpu: [arm64]+ os: [android]++ '@esbuild/[email protected]':+ resolution: {integrity: sha512-RONsAvGCz5oWyePVnLdZY/HHwA++nxYWIX1atInlaW6SEkwq6XkP3+cb825EUcRs5Vss/lGh/2YxAb5xqc07Uw==}+ engines: {node: '>=18'}+ cpu: [arm]+ os: [android]++ '@esbuild/[email protected]':+ resolution: {integrity: sha512-yJAVPklM5+4+9dTeKwHOaA+LQkmrKFX96BM0A/2zQrbS6ENCmxc4OVoBs5dPkCCak2roAD+jKCdnmOqKszPkjA==}+ engines: {node: '>=18'}+ cpu: [x64]+ os: [android]++ '@esbuild/[email protected]':+ resolution: {integrity: sha512-Jw0mxgIaYX6R8ODrdkLLPwBqHTtYHJSmzzd+QeytSugzQ0Vg4c5rDky5VgkoowbZQahCbsv1rT1KW72MPIkevw==}+ engines: {node: '>=18'}+ cpu: [arm64]+ os: [darwin]++ '@esbuild/[email protected]':+ resolution: {integrity: sha512-Vh2gLxxHnuoQ+GjPNvDSDRpoBCUzY4Pu0kBqMBDlK4fuWbKgGtmDIeEC081xi26PPjn+1tct+Bh8FjyLlw1Zlg==}+ engines: {node: '>=18'}+ cpu: [x64]+ os: [darwin]++ '@esbuild/[email protected]':+ resolution: {integrity: sha512-YPJ7hDQ9DnNe5vxOm6jaie9QsTwcKedPvizTVlqWG9GBSq+BuyWEDazlGaDTC5NGU4QJd666V0yqCBL2oWKPfA==}+ engines: {node: '>=18'}+ cpu: [arm64]+ os: [freebsd]++ '@esbuild/[email protected]':+ resolution: {integrity: sha512-MmaEXxQRdXNFsRN/KcIimLnSJrk2r5H8v+WVafRWz5xdSVmWLoITZQXcgehI2ZE6gioE6HirAEToM/RvFBeuhw==}+ engines: {node: '>=18'}+ cpu: [x64]+ os: [freebsd]++ '@esbuild/[email protected]':+ resolution: {integrity: sha512-WIgg00ARWv/uYLU7lsuDK00d/hHSfES5BzdWAdAig1ioV5kaFNrtK8EqGcUBJhYqotlUByUKz5Qo6u8tt7iD/w==}+ engines: {node: '>=18'}+ cpu: [arm64]+ os: [linux]++ '@esbuild/[email protected]':+ resolution: {integrity: sha512-FuzEP9BixzZohl1kLf76KEVOsxtIBFwCaLupVuk4eFVnOZfU+Wsn+x5Ryam7nILV2pkq2TqQM9EZPsOBuMC+kg==}+ engines: {node: '>=18'}+ cpu: [arm]+ os: [linux]++ '@esbuild/[email protected]':+ resolution: {integrity: sha512-A1D9YzRX1i+1AJZuFFUMP1E9fMaYY+GnSQil9Tlw05utlE86EKTUA7RjwHDkEitmLYiFsRd9HwKBPEftNdBfjg==}+ engines: {node: '>=18'}+ cpu: [ia32]+ os: [linux]++ '@esbuild/[email protected]':+ resolution: {integrity: sha512-O7k1J/dwHkY1RMVvglFHl1HzutGEFFZ3kNiDMSOyUrB7WcoHGf96Sh+64nTRT26l3GMbCW01Ekh/ThKM5iI7hQ==}+ engines: {node: '>=18'}+ cpu: [loong64]+ os: [linux]++ '@esbuild/[email protected]':+ resolution: {integrity: sha512-uv+dqfRazte3BzfMp8PAQXmdGHQt2oC/y2ovwpTteqrMx2lwaksiFZ/bdkXJC19ttTvNXBuWH53zy/aTj1FgGw==}+ engines: {node: '>=18'}+ cpu: [mips64el]+ os: [linux]++ '@esbuild/[email protected]':+ resolution: {integrity: sha512-GyG0KcMi1GBavP5JgAkkstMGyMholMDybAf8wF5A70CALlDM2p/f7YFE7H92eDeH/VBtFJA5MT4nRPDGg4JuzQ==}+ engines: {node: '>=18'}+ cpu: [ppc64]+ os: [linux]++ '@esbuild/[email protected]':+ resolution: {integrity: sha512-rAqDYFv3yzMrq7GIcen3XP7TUEG/4LK86LUPMIz6RT8A6pRIDn0sDcvjudVZBiiTcZCY9y2SgYX2lgK3AF+1eg==}+ engines: {node: '>=18'}+ cpu: [riscv64]+ os: [linux]++ '@esbuild/[email protected]':+ resolution: {integrity: sha512-Xutvh6VjlbcHpsIIbwY8GVRbwoviWT19tFhgdA7DlenLGC/mbc3lBoVb7jxj9Z+eyGqvcnSyIltYUrkKzWqSvg==}+ engines: {node: '>=18'}+ cpu: [s390x]+ os: [linux]++ '@esbuild/[email protected]':+ resolution: {integrity: sha512-ASFQhgY4ElXh3nDcOMTkQero4b1lgubskNlhIfJrsH5OKZXDpUAKBlNS0Kx81jwOBp+HCeZqmoJuihTv57/jvQ==}+ engines: {node: '>=18'}+ cpu: [x64]+ os: [linux]++ '@esbuild/[email protected]':+ resolution: {integrity: sha512-d1KfruIeohqAi6SA+gENMuObDbEjn22olAR7egqnkCD9DGBG0wsEARotkLgXDu6c4ncgWTZJtN5vcgxzWRMzcw==}+ engines: {node: '>=18'}+ cpu: [arm64]+ os: [netbsd]++ '@esbuild/[email protected]':+ resolution: {integrity: sha512-nVDCkrvx2ua+XQNyfrujIG38+YGyuy2Ru9kKVNyh5jAys6n+l44tTtToqHjino2My8VAY6Lw9H7RI73XFi66Cg==}+ engines: {node: '>=18'}+ cpu: [x64]+ os: [netbsd]++ '@esbuild/[email protected]':+ resolution: {integrity: sha512-j8HgrDuSJFAujkivSMSfPQSAa5Fxbvk4rgNAS5i3K+r8s1X0p1uOO2Hl2xNsGFppOeHOLAVgYwDVlmxhq5h+SQ==}+ engines: {node: '>=18'}+ cpu: [arm64]+ os: [openbsd]++ '@esbuild/[email protected]':+ resolution: {integrity: sha512-1h8MUAwa0VhNCDp6Af0HToI2TJFAn1uqT9Al6DJVzdIBAd21m/G0Yfc77KDM3uF3T/YaOgQq3qTJHPbTOInaIQ==}+ engines: {node: '>=18'}+ cpu: [x64]+ os: [openbsd]++ '@esbuild/[email protected]':+ resolution: {integrity: sha512-r2nVa5SIK9tSWd0kJd9HCffnDHKchTGikb//9c7HX+r+wHYCpQrSgxhlY6KWV1nFo1l4KFbsMlHk+L6fekLsUg==}+ engines: {node: '>=18'}+ cpu: [arm64]+ os: [openharmony]++ '@esbuild/[email protected]':+ resolution: {integrity: sha512-zUlaP2S12YhQ2UzUfcCuMDHQFJyKABkAjvO5YSndMiIkMimPmxA+BYSBikWgsRpvyxuRnow4nS5NPnf9fpv41w==}+ engines: {node: '>=18'}+ cpu: [x64]+ os: [sunos]++ '@esbuild/[email protected]':+ resolution: {integrity: sha512-YEGFFWESlPva8hGL+zvj2z/SaK+pH0SwOM0Nc/d+rVnW7GSTFlLBGzZkuSU9kFIGIo8q9X3ucpZhu8PDN5A2sQ==}+ engines: {node: '>=18'}+ cpu: [arm64]+ os: [win32]++ '@esbuild/[email protected]':+ resolution: {integrity: sha512-hiGgGC6KZ5LZz58OL/+qVVoZiuZlUYlYHNAmczOm7bs2oE1XriPFi5ZHHrS8ACpV5EjySrnoCKmcbQMN+ojnHg==}+ engines: {node: '>=18'}+ cpu: [ia32]+ os: [win32]++ '@esbuild/[email protected]':+ resolution: {integrity: sha512-cn3Yr7+OaaZq1c+2pe+8yxC8E144SReCQjN6/2ynubzYjvyqZjTXfQJpAcQpsdJq3My7XADANiYGHoFC69pLQw==}+ engines: {node: '>=18'}+ cpu: [x64]+ os: [win32]+'@google-cloud/[email protected]':resolution: {integrity: sha512-DJS3s0OVH4zFDB1PzjxAsHqJT6sKVbRwwML0ZBP9PbU7Yebtu/7SWMRzvO2J3nUi9pRNITCfu4LJeooM2w4pjg==}engines: {node: '>=14.0.0'}@@ -2442,6 +2601,11 @@ packages:resolution: {integrity: sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA==}engines: {node: '>= 0.4'}+ resolution: {integrity: sha512-vVC0USHGtMi8+R4Kz8rt6JhEWLxsv9Rnu/lGYbPR8u47B+DCBksq9JarW0zOO7bs37hyOK1l2/oqtbciutL5+Q==}+ engines: {node: '>=18'}+ hasBin: true+resolution: {integrity: sha512-ErCHMCae19vR8vQGe50xIsVomy19rg6gFu3+r3jkEO46suLMWBksvVyoGgQV+jOfl84ZSOSlmv6Gxa89PmTGmA==}engines: {node: '>=6'}@@ -2660,6 +2824,9 @@ packages:resolution: {integrity: sha512-ts6Wi+2j3jQjqi70w5AlN8DFnkSwC+MqmxEzdEALB2qXZYV3X/b1CTfgPLGJNMeAWxdPfU8FO1ms3NUfaHCPYg==}engines: {node: '>=10'}+ resolution: {integrity: sha512-auHyJ4AgMz7vgS8Hp3N6HXSmlMdUyhSUrfBF16w153rxtLIEOE+HGqaBppczZvnHLqQJfiHotCYpNhl0lUROFQ==}+resolution: {integrity: sha512-/Iu4prUrydE3Pb3lCBMbcSNIf81tgGt0W1ZwknnyF62t3tHmtiJTRj0f+1ZIhp3+Rh0ktz1pJVoa7ZXUCskivA==}engines: {node: '>= 4.8.0'}@@ -3744,6 +3911,9 @@ packages:resolution: {integrity: sha512-qYg9KP24dD5qka9J47d0aVky0N+b4fTU89LN9iDnjB5waksiC49rvMB0PrUJQGoTmH50XPiqOvAjDfaijGxYZw==}engines: {node: '>=8'}+ resolution: {integrity: sha512-seS2Tj26TBVOC2NIc2rOe2y2ZO7efxITtLZcGSOnHHNOQ7CkiUBfw0Iw2ck6xkIhPwLhKNLS8BO+hEpngQlqzw==}+resolution: {integrity: sha512-X2UW6Nw3n/aMgDVy+0rSqgHlv39WZAlZrXCdnbyEiKm17DSqHX4MmQMaST3FbeWR5FTuRcUwYAziZajji0Y7mg==}engines: {node: '>=10'}@@ -4130,6 +4300,11 @@ packages:resolution: {integrity: sha512-xNvxJEOUiWPGhUuUdQgAJPKOOJfGnIyKySOc09XkKsgdUV/3E2zvwZYdejjmRgPCgcym1juLH3226yA7sEFJKQ==}+ resolution: {integrity: sha512-qjbnuR9Tr+FJOMBqJCW5ehvIo/buZq7vH7qD7JziU98h6l3qGy0a/yPFjwO+y0/T7GFpNgNAvEcPPVfyT8rrPQ==}+ engines: {node: '>=18.0.0'}+ hasBin: true+resolution: {integrity: sha512-vwz9tfvF7XN/jE0dGoBei3FXWuvll78ohzCZQuOb+ZjWrs3a0XhQVomJEb2Qh4VHTPNRO4GPZh0V7VRbiWwkRg==}@@ -5368,6 +5543,84 @@ snapshots:'@dqbd/[email protected]': {}+ '@esbuild/[email protected]':+ optional: true++ '@esbuild/[email protected]':+ optional: true++ '@esbuild/[email protected]':+ optional: true++ '@esbuild/[email protected]':+ optional: true++ '@esbuild/[email protected]':+ optional: true++ '@esbuild/[email protected]':+ optional: true++ '@esbuild/[email protected]':+ optional: true++ '@esbuild/[email protected]':+ optional: true++ '@esbuild/[email protected]':+ optional: true++ '@esbuild/[email protected]':+ optional: true++ '@esbuild/[email protected]':+ optional: true++ '@esbuild/[email protected]':+ optional: true++ '@esbuild/[email protected]':+ optional: true++ '@esbuild/[email protected]':+ optional: true++ '@esbuild/[email protected]':+ optional: true++ '@esbuild/[email protected]':+ optional: true++ '@esbuild/[email protected]':+ optional: true++ '@esbuild/[email protected]':+ optional: true++ '@esbuild/[email protected]':+ optional: true++ '@esbuild/[email protected]':+ optional: true++ '@esbuild/[email protected]':+ optional: true++ '@esbuild/[email protected]':+ optional: true++ '@esbuild/[email protected]':+ optional: true++ '@esbuild/[email protected]':+ optional: true++ '@esbuild/[email protected]':+ optional: true++ '@esbuild/[email protected]':+ optional: true+'@google-cloud/[email protected]':dependencies:arrify: 2.0.1@@ -7286,6 +7539,35 @@ snapshots:has-tostringtag: 1.0.2hasown: 2.0.2+ optionalDependencies:+ '@esbuild/aix-ppc64': 0.25.8+ '@esbuild/android-arm': 0.25.8+ '@esbuild/android-arm64': 0.25.8+ '@esbuild/android-x64': 0.25.8+ '@esbuild/darwin-arm64': 0.25.8+ '@esbuild/darwin-x64': 0.25.8+ '@esbuild/freebsd-arm64': 0.25.8+ '@esbuild/freebsd-x64': 0.25.8+ '@esbuild/linux-arm': 0.25.8+ '@esbuild/linux-arm64': 0.25.8+ '@esbuild/linux-ia32': 0.25.8+ '@esbuild/linux-loong64': 0.25.8+ '@esbuild/linux-mips64el': 0.25.8+ '@esbuild/linux-ppc64': 0.25.8+ '@esbuild/linux-riscv64': 0.25.8+ '@esbuild/linux-s390x': 0.25.8+ '@esbuild/linux-x64': 0.25.8+ '@esbuild/netbsd-arm64': 0.25.8+ '@esbuild/netbsd-x64': 0.25.8+ '@esbuild/openbsd-arm64': 0.25.8+ '@esbuild/openbsd-x64': 0.25.8+ '@esbuild/openharmony-arm64': 0.25.8+ '@esbuild/sunos-x64': 0.25.8+ '@esbuild/win32-arm64': 0.25.8+ '@esbuild/win32-ia32': 0.25.8+ '@esbuild/win32-x64': 0.25.8+@@ -7551,6 +7833,10 @@ snapshots:+ dependencies:+ resolve-pkg-maps: 1.0.0+dependencies:chalk: 2.4.2@@ -8830,6 +9116,8 @@ snapshots:+ [email protected]: {}+@@ -9236,6 +9524,13 @@ snapshots:+ dependencies:+ esbuild: 0.25.8+ get-tsconfig: 4.10.1+ optionalDependencies:+ fsevents: 2.3.3+diff --git a/apps/api/src/harness.ts b/apps/api/src/harness.tsnew file mode 100644index 000000000..1718fbc02--- /dev/null+++ b/apps/api/src/harness.ts@@ -0,0 +1,122 @@+import { exec } from "child_process";+import * as net from "net";++function waitForPort(port: number, host: string): Promise<void> {+ return new Promise((resolve) => {+ const checkPort = () => {+ const socket = new net.Socket();++ const onError = () => {+ socket.destroy();+ setTimeout(checkPort, 1000); // Try again in 1 second+ };++ socket.once('error', onError);++ socket.connect(port, host, () => {+ socket.destroy();+ resolve();+ });+ };++ checkPort();+ });+}++if (process.argv.length < 3) {+ console.error("Usage: pnpm harness <command...>");+ console.error()+ console.error("The harness ensures that the dependencies are up to date, everything is built, and the API and Worker are running, before running a command that would require the above.");+ console.error("It also tears down the API and Worker after the command is run.");+ process.exit(1);+}++const command = process.argv.slice(2);++function execForward(fancyName: string, command: string): Promise<void> {+ return new Promise((resolve, reject) => {+ const child = exec(command);+ let stdoutBuffer = "";+ let stderrBuffer = "";+ child.stdout?.on("data", (data) => {+ stdoutBuffer += data;+ while(stdoutBuffer.includes("\n")) {+ const split = stdoutBuffer.split("\n");+ const line = split[0];+ stdoutBuffer = split.slice(1).join("\n");+ process.stdout.write(`[${fancyName}] ${line}\n`);+ }+ });+ child.stderr?.on("data", (data) => {+ stderrBuffer += data;+ while(stderrBuffer.includes("\n")) {+ const split = stderrBuffer.split("\n");+ const line = split[0];+ stderrBuffer = split.slice(1).join("\n");+ process.stderr.write(`[${fancyName}] ${line}\n`);+ }+ });+ child.on("close", (code) => {+ if (code !== 0) {+ reject(new Error(`Command ${JSON.stringify(command)} failed with code ${code}`));+ } else {+ resolve();+ }+ });+ });+}++(async () => {+ console.log("=== Installing dependencies and building all components...");+ await Promise.all([+ (async () => {+ const install = execForward("api@install", "pnpm install");+ await install;++ const build = execForward("api@build", "pnpm build");+ await build;+ })(),+ execForward("sharedLibs/crawler@build", "cd sharedLibs/crawler && cargo build --release"),+ execForward("sharedLibs/html-transformer@build", "cd sharedLibs/html-transformer && cargo build --release"),+ execForward("sharedLibs/pdf-parser@build", "cd sharedLibs/pdf-parser && cargo build --release"),+ (async () => {+ const install = execForward("sharedLibs/go-html-to-md@install", "cd sharedLibs/go-html-to-md && go mod tidy");+ await install;++ const build = execForward("sharedLibs/go-html-to-md@build", "cd sharedLibs/go-html-to-md && go build -o html-to-markdown.so -buildmode=c-shared html-to-markdown.go");+ await build;+ })(),+ ]);++ console.log("=== Starting API and Worker...");++ const api = execForward("api", "pnpm start:production:nobuild");+ const worker = execForward("worker", "pnpm worker:production");++ try {+ await Promise.race([+ waitForPort(3002, "localhost"),+ new Promise((reject) => setTimeout(() => reject(new Error("API did not start in time")), 10000)),+ ]);++ console.log("=== Running command...");++ const cmd = execForward("command", command.join(" "));++ await Promise.race([+ cmd,+ api,+ worker,+ ]);+ } finally {+ console.log("=== Tearing down API and Worker...");+ exec("pkill -f 'queue-worker.js'");+ exec("pkill -f 'index.js'");+ await Promise.all([+ api,+ worker,+ ]);+ }++ console.log("=== Goodbye!");+})();\ No newline at end of filediff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.tsindex d68659daa..5baa7528b 100644--- a/apps/api/src/services/queue-worker.ts+++ b/apps/api/src/services/queue-worker.ts@@ -458,7 +458,7 @@ app.get("/liveness", (req, res) => {}});-const workerPort = process.env.PORT || 3005;+const workerPort = process.env.WORKER_PORT || process.env.PORT || 3005;app.listen(workerPort, () => {_logger.info(`Liveness endpoint is running on port ${workerPort}`);});