Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
SELFHOST FIXES (#1207)
Gergő Móricz2200f084f35f
diff --git a/.github/workflows/test-server-self-host.yml b/.github/workflows/test-server-self-host.ymlindex 3e13d3d1b..037e96842 100644--- a/.github/workflows/test-server-self-host.yml+++ b/.github/workflows/test-server-self-host.yml@@ -1,22 +1,35 @@name: Self-hosted Server Test Suiteon:- workflow_dispatch:+ pull_request:+ branches:+ - main+ paths:+ - apps/api/**env:PORT: 3002REDIS_URL: redis://localhost:6379+ HOST: 0.0.0.0ENV: ${{ secrets.ENV }}+ TEST_SUITE_SELF_HOSTED: truejobs:test:name: Run tests+ strategy:+ matrix:+ openai: [true, false]+ serper: [true, false]runs-on: ubuntu-latestservices:redis:image: redisports:- 6379:6379+ env:+ OPENAI_API_KEY: ${{ matrix.openai == true && secrets.OPENAI_API_KEY || '' }}+ SERPER_API_KEY: ${{ matrix.serper == true && secrets.SERPER_API_KEY || '' }}steps:- uses: actions/checkout@v3- name: Install pnpm@@ -40,6 +53,8 @@ jobs:run: npm run workers > worker.log 2>&1 &working-directory: ./apps/apiid: start_workers+ - name: Wait for server+ run: pnpx wait-on tcp:3002 -t 15s- name: Run snippet testsrun: |npm run test:snips@@ -50,7 +65,7 @@ jobs:- uses: actions/upload-artifact@v4if: always()with:- name: Logs+ name: Logs (openai ${{ matrix.openai }}, serper ${{ matrix.serper }})path: |./apps/api/api.log./apps/api/worker.logdiff --git a/apps/api/package.json b/apps/api/package.jsonindex 1c5547282..315199980 100644--- a/apps/api/package.json+++ b/apps/api/package.json@@ -56,6 +56,7 @@"typescript": "^5.4.2"},"dependencies": {+ "jsdom": "^26.0.0","@anthropic-ai/sdk": "^0.24.3","@apidevtools/json-schema-ref-parser": "^11.7.3","@brillout/import": "^0.2.2",diff --git a/apps/api/pnpm-lock.yaml b/apps/api/pnpm-lock.yamlindex f66dfe724..85c021407 100644--- a/apps/api/pnpm-lock.yaml+++ b/apps/api/pnpm-lock.yaml@@ -125,6 +125,9 @@ importers:joplin-turndown-plugin-gfm:specifier: ^1.0.12version: 1.0.12+ jsdom:+ specifier: ^26.0.0+ version: 26.0.0json-schema-to-zod:specifier: ^2.3.0version: 2.3.0@@ -136,7 +139,7 @@ importers:version: 2.9.0langchain:specifier: ^0.2.8- version: 0.2.8(@aws-sdk/[email protected](@aws-sdk/[email protected](@aws-sdk/[email protected]))(@aws-sdk/[email protected]))(@pinecone-database/[email protected])(@supabase/[email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected](@aws-sdk/[email protected](@aws-sdk/[email protected](@aws-sdk/[email protected])))([email protected]))([email protected]([email protected])([email protected]))([email protected])([email protected]([email protected]))([email protected])([email protected])+ version: 0.2.8(@aws-sdk/[email protected](@aws-sdk/[email protected](@aws-sdk/[email protected]))(@aws-sdk/[email protected]))(@pinecone-database/[email protected])(@supabase/[email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected](@aws-sdk/[email protected](@aws-sdk/[email protected](@aws-sdk/[email protected])))([email protected]))([email protected]([email protected])([email protected]))([email protected])([email protected]([email protected]))([email protected])([email protected])languagedetect:specifier: ^2.0.0version: 2.0.0@@ -332,6 +335,9 @@ packages:resolution: {integrity: sha512-WApSdLdXEBb/1FUPca2lteASewEfpjEYJ8oXZP+0gExK5qSfsEKBKcA+WjY6Q4wvXwyv0+W6Kvc372pSceib9w==}engines: {node: '>= 16'}+ '@asamuzakjp/[email protected]':+ resolution: {integrity: sha512-GIc76d9UI1hCvOATjZPyHFmE5qhRccp3/zGfMPapK3jBi+yocEzp6BBB0UnfRYP9NP4FANqUZYb0hnfs3TM3hw==}+'@aws-crypto/[email protected]':resolution: {integrity: sha512-IzSgsrxUcsrejQbPVilIKy16kAT52EwB6zSaI+M3xxIhKh5+aldEyvI+z6erM7TCLB2BJsFrtHjp6/4/sr+3dA==}@@ -685,6 +691,34 @@ packages:resolution: {integrity: sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw==}engines: {node: '>=12'}+ '@csstools/[email protected]':+ resolution: {integrity: sha512-MKtmkA0BX87PKaO1NFRTFH+UnkgnmySQOvNxJubsadusqPEC2aJ9MOQiMceZJJ6oitUl/i0L6u0M1IrmAOmgBA==}+ engines: {node: '>=18'}++ '@csstools/[email protected]':+ resolution: {integrity: sha512-rL7kaUnTkL9K+Cvo2pnCieqNpTKgQzy5f+N+5Iuko9HAoasP+xgprVh7KN/MaJVvVL1l0EzQq2MoqBHKSrDrag==}+ engines: {node: '>=18'}+ peerDependencies:+ '@csstools/css-parser-algorithms': ^3.0.4+ '@csstools/css-tokenizer': ^3.0.3++ '@csstools/[email protected]':+ resolution: {integrity: sha512-nkMp2mTICw32uE5NN+EsJ4f5N+IGFeCFu4bGpiKgb2Pq/7J/MpyLBeQ5ry4KKtRFZaYs6sTmcMYrSRIyj5DFKA==}+ engines: {node: '>=18'}+ peerDependencies:+ '@csstools/css-parser-algorithms': ^3.0.4+ '@csstools/css-tokenizer': ^3.0.3++ '@csstools/[email protected]':+ resolution: {integrity: sha512-Up7rBoV77rv29d3uKHUIVubz1BTcgyUK72IvCQAbfbMv584xHcGKCKbWh7i8hPrRJ7qU4Y8IO3IY9m+iTB7P3A==}+ engines: {node: '>=18'}+ peerDependencies:+ '@csstools/css-tokenizer': ^3.0.3++ '@csstools/[email protected]':+ resolution: {integrity: sha512-UJnjoFsmxfKUdNYdWgOB0mWUypuLvAfQPH1+pyvRJs6euowbFkFC6P13w1l8mJyi3vxYMxc9kld5jZEGRQs6bw==}+ engines: {node: '>=18'}+'@dabh/[email protected]':resolution: {integrity: sha512-hrlQOIi7hAfzsMqlGSFyVucrx38O+j6wiGOf//H2ecvIEqYN4ADBSS2iLMh5UFyDunCNniUIPk/q3riFv45xRA==}@@ -1715,6 +1749,10 @@ packages:resolution: {integrity: sha512-H0TSyFNDMomMNJQBn8wFV5YC/2eJ+VXECwOadZJT554xP6cODZHPX3H9QMQECxvrgiSOP1pHjy1sMWQVYJOUOA==}engines: {node: '>= 14'}+ resolution: {integrity: sha512-jRR5wdylq8CkOe6hei19GGZnxM6rBGwFl3Bg0YItGDimvjGtAvdZk4Pu6Cl4u4Igsws4a1fd1Vq3ezrhn4KmFw==}+ engines: {node: '>= 14'}+resolution: {integrity: sha512-5GG/5IbQQpC9FpkRGsSvZI5QYeSCzlJHdpBQntCsuTOxhKD8lqKhrleg2Yi7yvMIf82Ycmmqln9U8V9qwEiJew==}engines: {node: '>= 8.0.0'}@@ -2141,6 +2179,10 @@ packages:resolution: {integrity: sha512-HTUrgRJ7r4dsZKU6GjmpfRK1O76h97Z8MfS1G0FozR+oF2kG6Vfe8JE6zwrkbxigziPHinCJ+gCPjA9EaBDtRw==}engines: {node: '>= 6'}+ resolution: {integrity: sha512-9+vem03dMXG7gDmZ62uqmRiMRNtinIZ9ZyuF6BdxzfOD+FdN5hretzynkn0ReS2DO2GSw76RWHs0UmJPI2zUjw==}+ engines: {node: '>=18'}+resolution: {integrity: sha512-uNpm30m/AGSkLxxy7d9yRXpJQFrZzVWLFBkS+6ngPcZkw/5k3L/jjFuj7tVnEpRn+QgmiXr21nDlhCiUK4ij2A==}@@ -2152,6 +2194,10 @@ packages:resolution: {integrity: sha512-7hvf7/GW8e86rW0ptuwS3OcBGDjIi6SZva7hCyWC0yYry2cOPmLIjXAUHI6DK2HsnwJd9ifmt57i8eV2n4YNpw==}engines: {node: '>= 14'}+ resolution: {integrity: sha512-ZYP5VBHshaDAiVZxjbRVcFJpc+4xGgT0bK3vzy1HLN8jTO975HEbuYzZJcHoQEY5K1a0z8YayJkyVETa08eNTg==}+ engines: {node: '>=18'}+resolution: {integrity: sha512-fRHTG8g/Gif+kSh50gaGEdToemgfj74aRX3swtiouboip5JDLAyDE9F11nHMIcvOaXeOC6D7SpNhi7uFyB7Uww==}@@ -2197,6 +2243,9 @@ packages:resolution: {integrity: sha512-9iE1PgSik9HeIIw2JO94IidnE3eBoQrFJ3w7sFuzSX4DpmZ3v5sZpUiV5Swcf6mQEF+Y0ru8Neo+p+nyh2J+hQ==}engines: {node: '>=10'}+ resolution: {integrity: sha512-8vDa8Qxvr/+d94hSh5P3IJwI5t8/c0KsMp+g8bNw9cY2icONa5aPfvKeieW1WlG0WQYwwhJ7mjui2xtiePQSXw==}+resolution: {integrity: sha512-NHQtfOOW68WD8lgypbLA5oT+Bt0xXJhiYvoR6SmmNXZfpzOGXwdKWmcwG8N7PwVVWV3eF/68nmD9BaJSsTBhyQ==}peerDependencies:@@ -2510,6 +2559,10 @@ packages:resolution: {integrity: sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==}engines: {node: '>= 6'}+ resolution: {integrity: sha512-tzN8e4TX8+kkxGPK8D5u0FNmjPUjw3lwC9lSLxxoB/+GtsJG91CO8bSWy73APlgAZzZbXEYZJuxjkHH2w+Ezhw==}+ engines: {node: '>= 6'}+resolution: {integrity: sha512-0iirZp3uVDjVGt9p49aTaqjk84TrglENEDuqfdlZQ1roC9CWlPk6Avf8EEnZNcAqPonwkG35x4n3ww/1THYAeQ==}engines: {node: '>= 12.20'}@@ -2647,6 +2700,10 @@ packages:resolution: {integrity: sha512-oWv4T4yJ52iKrufjnyZPkrN0CH3QnrUqdB6In1g5Fe1mia8GmF36gnfNySxoZtxD5+NmYw1EElVXiBk93UeskA==}engines: {node: '>=12'}+ resolution: {integrity: sha512-Y22oTqIU4uuPgEemfz7NDJz6OeKf12Lsu+QC+s3BVpda64lTiMYCyGwg5ki4vFxkMwQdeZDl2adZoqUgdFuTgQ==}+ engines: {node: '>=18'}+resolution: {integrity: sha512-H2iMtd0I4Mt5eYiapRdIDjp+XzelXQ0tFE4JS7YFwFevXXMmOp9myNrUvCg0D6ws8iqkRPBfKHgbwig1SmlLfg==}@@ -2686,6 +2743,10 @@ packages:resolution: {integrity: sha512-1e4Wqeblerz+tMKPIq2EMGiiWW1dIjZOksyHWSUm1rmuvw/how9hBHZ38lAGj5ID4Ik6EdkOw7NmWPy6LAwalw==}engines: {node: '>= 14'}+ resolution: {integrity: sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==}+ engines: {node: '>= 14'}+resolution: {integrity: sha512-B4FFZ6q/T2jhhksgkbEW3HBvWIfDW85snkQgawt07S7J5QXTk6BkNV+0yAeZrM5QpMAdYlocGoljn0sJ/WQkFw==}engines: {node: '>=10.17.0'}@@ -2798,6 +2859,9 @@ packages:resolution: {integrity: sha512-YWnfyRwxL/+SsrWYfOpUtz5b3YD+nyfkHvjbcanzk8zgyO4ASD67uVMRt8k5bM4lLMDnXfriRhOpemw+NfT1eA==}engines: {node: '>=8'}+ resolution: {integrity: sha512-bCYeRA2rVibKZd+s2625gGnGF/t7DSqDs4dP7CrLA1m7jKWz6pps0LpYLJN8Q64HtmPKJ1hrN3nzPNKFEKOUiQ==}+resolution: {integrity: sha512-XVm7LOeLpTW4jV19QSH38vkswxoLud8sQ57YwJVTPWdiaI9I8keEhGFpBlslyVsgdQy4Opg8QOLb8YRgsyZiQg==}engines: {node: '>=10'}@@ -3012,6 +3076,15 @@ packages:resolution: {integrity: sha512-4bYVV3aAMtDTTu4+xsDYa6sy9GyJ69/amsu9sYF2zqjiEoZA5xJi3BrfX3uY+/IekIu7MwdObdbDWpoZdBv3/A==}+ resolution: {integrity: sha512-BZYDGVAIriBWTpIxYzrXjv3E/4u8+/pSG5bQdIYCbNCGOvsPkDQfTVLAIXAf9ETdCpduCVTkDe2NNZ8NIwUVzw==}+ engines: {node: '>=18'}+ peerDependencies:+ canvas: ^3.0.0+ peerDependenciesMeta:+ canvas:+ optional: true+resolution: {integrity: sha512-OYu7XEzjkCQ3C5Ps3QIZsQfNpqoJyZZA99wd9aWd05NCtC5pWOkShK2mkL6HXQR6/Cy2lbNdPlZBpuQHXE63gA==}engines: {node: '>=4'}@@ -3298,6 +3371,9 @@ packages:resolution: {integrity: sha512-CQl19J/g+Hbjbv4Y3mFNNXFEL/5t/KCg8POCuUqd4rMKjGG+j1ybER83hxV58zL+dFI1PTkt3GNFSHRt+d8qEQ==}engines: {node: 14 || >=16.14}+ resolution: {integrity: sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==}+resolution: {integrity: sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==}@@ -3588,6 +3664,9 @@ packages:resolution: {integrity: sha512-1MQz1Ed8z2yckoBeSfkQHHO9K1yDRxxtotKSJ9yvcTUUxSvfvzEq5GwBrjjHEpMlq/k5gvXdmJ1SbYxWtpNoVg==}engines: {node: '>=8'}+ resolution: {integrity: sha512-F1I/bimDpj3ncaNDhfyMWuFqmQDBwDB0Fogc2qpL3BWvkQteFD/8BzWuIRl83rq0DXfm8SGt/HFhLXZyljTXcQ==}+resolution: {integrity: sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==}engines: {node: '>=0.10.0'}@@ -3697,6 +3776,9 @@ packages:resolution: {integrity: sha512-Czj1WaSVpaoj0wbhMzLmWD69anp2WH7FXMB9n1Sy8/ZFF9jolSQVMu1Ij5WIyGmcBmhk7EOndpO4mIpihVqAXw==}+ resolution: {integrity: sha512-BuBYQYlv1ckiPdQi/ohiivi9Sagc9JG+Ozs0r7b/0iK3sKmrb0b9FdWdBbOdx6hBCM/F9Ir82ofnBhtZOjCRPQ==}+resolution: {integrity: sha512-e6qHKe3a9HWr0oMRVDTRhKce+bRO8VGQR3NyVwcjwrbhMmFCX9KszEV35+rn4AdilFAq9VPxP/Fe1wC9Qjd2lw==}@@ -4015,6 +4097,9 @@ packages:resolution: {integrity: sha512-s+pyvQeIKIZ0dx5iJiQk1tPLJAWln39+MI5jtM8wnyws+G5azk+dMnMX0qfbqNetKKNgcWWOdi0sfm+FbQbgdQ==}engines: {node: '>=10.0.0'}+ resolution: {integrity: sha512-guoltQEx+9aMf2gDZ0s62EcV8lsXR+0w8915TC3ITdn2YueuNjdAYh/levpU9nFaoChh9RUS5ZdQMrKfVEN9tw==}+resolution: {integrity: sha512-cLgakCUf6PedEu15t8kbsjnwIFFR2D4RfL+W3iWFJ4iac7z4B0ZI8fxy4R3J956kAI68HclCFGL8MPoUVC3qVA==}@@ -4034,6 +4119,10 @@ packages:resolution: {integrity: sha512-+aWOz7yVScEGoKNd4PA10LZ8sk0A/z5+nXQG5giUO5rprX9jgYsTdov9qCchZiPIZezbZH+jRut8nPodFAX4Jg==}+ resolution: {integrity: sha512-xAg7SOnEhrm5zI3puOOKyy1OMcMlIJZYNJY7xLBwSze0UjhPLnWfj2GF2EpT0jmzaJKIWKHLsaSSajf35bcYnA==}+ engines: {node: '>=v12.22.7'}+resolution: {integrity: sha512-UOShsPwz7NrMUqhR6t0hWjFduvOzbtv7toDH1/hIrfRNIDBnnBWd0CwJTGvTpngVlmwGCdP9/Zl/tVrDqcuYzQ==}@@ -4260,6 +4349,9 @@ packages:resolution: {integrity: sha512-SzRP5LQ6Ts2G5NyAa/jg16s8e3R7rfdFjizy1zeoecYWw+nGL+YA1xZvW/+iJmidBGSdLkuvdwTYEyJEb+EiUw==}engines: {node: '>=0.2.6'}+ resolution: {integrity: sha512-9QNk5KwDF+Bvz+PyObkmSYjI5ksVUYtjW7AU22r2NKcfLJcXp96hkDWU3+XndOsUb+AQ9QhfzfCT2O+CNWT5Tw==}+resolution: {integrity: sha512-aLws5yi4KCHTb0BVvbodQY5bY8eW4asMRDTxTW46hqw9lGjACX6TlLdJrkdoHYRB0qs+MekqEq1zG7WDnWE8Ug==}engines: {node: '>=8.0.0'}@@ -4315,6 +4407,10 @@ packages:resolution: {integrity: sha512-r0eojU4bI8MnHr8c5bNo7lJDdI2qXlWWJk6a9EAFG7vbhTjElYhBVS3/miuE0uOuoLdb8Mc/rVfsmm6eo5o9GA==}hasBin: true+ resolution: {integrity: sha512-Ek7HndSVkp10hmHP9V4qZO1u+pn1RU5sI0Fw+jCU3lyvuMZcgqsNgc6CmJJZyByK4Vm/qotGRJlfgAX8q+4JiA==}+ engines: {node: '>=16'}+resolution: {integrity: sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==}@@ -4322,6 +4418,10 @@ packages:resolution: {integrity: sha512-2lv/66T7e5yNyhAAC4NaKe5nVavzuGJQVVtRYLyQ2OI8tsJ61PMLlelehb0wi2Hx6+hT/OJUWZcw8MjlSRnxvw==}engines: {node: '>=14'}+ resolution: {integrity: sha512-tk2G5R2KRwBd+ZN0zaEXpmzdKyOYksXwywulIX95MBODjSzMIuQnQ3m8JxgbhnL1LeVo7lqQKsYa1O3Htl7K5g==}+ engines: {node: '>=18'}+resolution: {integrity: sha512-aZbgViZrg1QNcG+LULa7nhZpJTZSLm/mXnHXnbAbjmN5aSa0y7V+wvv6+4WaBtpISJzThKy+PIPxc1Nq1EJ9mg==}engines: {node: '>= 14.0.0'}@@ -4483,6 +4583,10 @@ packages:resolution: {integrity: sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==}engines: {node: '>= 0.8'}+ resolution: {integrity: sha512-o8qghlI8NZHU1lLPrpi2+Uq7abh4GGPpYANlalzWxyWteJOCsr/P+oPBA49TOLu5FTZO4d3F9MnWJfiMo4BkmA==}+ engines: {node: '>=18'}+resolution: {integrity: sha512-ts/8E8l5b7kY0vlWLewOkDXMmPdLcVV4GmOQLyxuSswIJsweeFZtAsMF7k1Nszz+TYBQrlYRmzOnr398y1JemQ==}@@ -4505,13 +4609,25 @@ packages:resolution: {integrity: sha512-p41ogyeMUrw3jWclHWTQg1k05DSVXPLcVxRTYsXUk+ZooOCZLcoYgPZ/HL/D/N+uQPOtcp1me1WhBEaX02mhWg==}engines: {node: '>=12'}+ resolution: {integrity: sha512-6qN4hJdMwfYBtE3YBTTHhoeuUrDBPZmbQaxWAqSALV/MeEnR5z1xd8UKud2RAkFoPkmB+hli1TZSnyi84xz1vQ==}+ engines: {node: '>=18'}+resolution: {integrity: sha512-EqhiFU6daOA8kpjOWTL0olhVOF3i7OrFzSYiGsEMB8GcXS+RrzauAERX65xMeNWVqxA6HXH2m69Z9LaKKdisfg==}+ resolution: {integrity: sha512-QaKxh0eNIi2mE9p2vEdzfagOKHCcj1pJ56EEHGQOVxp8r9/iszLUUV7v89x9O1p/T+NlTM5W7jW6+cz4Fq1YVg==}+ engines: {node: '>=18'}+resolution: {integrity: sha512-9WWbymnqj57+XEuqADHrCJ2eSXzn8WXIW/YSGaZtb2WKAInQ6CHfaUUcTyyver0p8BDg5StLQq8h1vtZuwmOig==}engines: {node: '>=16'}+ resolution: {integrity: sha512-mDGf9diDad/giZ/Sm9Xi2YcyzaFpbdLpJPr+E9fSkyQ7KpQD4SdFcugkRQYzhmfI4KeV4Qpnn2sKPdo+kmsgRQ==}+ engines: {node: '>=18'}+resolution: {integrity: sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==}@@ -4583,6 +4699,10 @@ packages:utf-8-validate:optional: true+ resolution: {integrity: sha512-EvGK8EJ3DhaHfbRlETOWAS5pO9MZITeauHKJyb8wyajUfQUenkIg2MvLDTZ4T/TgIcm3HU0TFBgWWboAZ30UHg==}+ engines: {node: '>=18'}+resolution: {integrity: sha512-T4rieHaC1EXcES0Kxxj4JWgaUQHDk+qwHcYOCFHfiwKz7tOVPLq7Hjq9dM1WCMhylqMEfP7hMcOIChvotiZegA==}engines: {node: '>=4.0.0'}@@ -4595,6 +4715,9 @@ packages:resolution: {integrity: sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA==}engines: {node: '>=4.0'}+ resolution: {integrity: sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw==}+resolution: {integrity: sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==}engines: {node: '>=0.4'}@@ -4675,6 +4798,14 @@ snapshots:'@types/json-schema': 7.0.15js-yaml: 4.1.0+ '@asamuzakjp/[email protected]':+ dependencies:+ '@csstools/css-calc': 2.1.1(@csstools/[email protected](@csstools/[email protected]))(@csstools/[email protected])+ '@csstools/css-color-parser': 3.0.7(@csstools/[email protected](@csstools/[email protected]))(@csstools/[email protected])+ '@csstools/css-parser-algorithms': 3.0.4(@csstools/[email protected])+ '@csstools/css-tokenizer': 3.0.3+ lru-cache: 10.4.3+'@aws-crypto/[email protected]':dependencies:'@aws-crypto/util': 3.0.0@@ -5413,6 +5544,26 @@ snapshots:dependencies:'@jridgewell/trace-mapping': 0.3.9+ '@csstools/[email protected]': {}++ '@csstools/[email protected](@csstools/[email protected](@csstools/[email protected]))(@csstools/[email protected])':+ dependencies:+ '@csstools/css-parser-algorithms': 3.0.4(@csstools/[email protected])+ '@csstools/css-tokenizer': 3.0.3++ '@csstools/[email protected](@csstools/[email protected](@csstools/[email protected]))(@csstools/[email protected])':+ dependencies:+ '@csstools/color-helpers': 5.0.1+ '@csstools/css-calc': 2.1.1(@csstools/[email protected](@csstools/[email protected]))(@csstools/[email protected])+ '@csstools/css-parser-algorithms': 3.0.4(@csstools/[email protected])+ '@csstools/css-tokenizer': 3.0.3++ '@csstools/[email protected](@csstools/[email protected])':+ dependencies:+ '@csstools/css-tokenizer': 3.0.3++ '@csstools/[email protected]': {}+'@dabh/[email protected]':dependencies:colorspace: 1.1.4@@ -5642,13 +5793,13 @@ snapshots:'@jsdevtools/[email protected]': {}- '@langchain/[email protected]([email protected](@aws-sdk/[email protected](@aws-sdk/[email protected](@aws-sdk/[email protected]))(@aws-sdk/[email protected]))(@pinecone-database/[email protected])(@supabase/[email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected](@aws-sdk/[email protected](@aws-sdk/[email protected](@aws-sdk/[email protected])))([email protected]))([email protected]([email protected])([email protected]))([email protected])([email protected]([email protected]))([email protected])([email protected]))([email protected]([email protected])([email protected]))':+ '@langchain/[email protected]([email protected](@aws-sdk/[email protected](@aws-sdk/[email protected](@aws-sdk/[email protected]))(@aws-sdk/[email protected]))(@pinecone-database/[email protected])(@supabase/[email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected](@aws-sdk/[email protected](@aws-sdk/[email protected](@aws-sdk/[email protected])))([email protected]))([email protected]([email protected])([email protected]))([email protected])([email protected]([email protected]))([email protected])([email protected]))([email protected]([email protected])([email protected]))':dependencies:ansi-styles: 5.2.0camelcase: 6.3.0decamelize: 1.2.0js-tiktoken: 1.0.12- langsmith: 0.1.34(npkyd6f7wyl3urgrzoxaktl5a4)+ langsmith: 0.1.34(7lljbsleilzgkaubvlq4ipicvq)ml-distance: 4.0.1mustache: 4.2.0p-queue: 6.6.2@@ -5660,9 +5811,9 @@ snapshots:- langchain- openai- '@langchain/[email protected]([email protected])([email protected](@aws-sdk/[email protected](@aws-sdk/[email protected](@aws-sdk/[email protected]))(@aws-sdk/[email protected]))(@pinecone-database/[email protected])(@supabase/[email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected](@aws-sdk/[email protected](@aws-sdk/[email protected](@aws-sdk/[email protected])))([email protected]))([email protected]([email protected])([email protected]))([email protected])([email protected]([email protected]))([email protected])([email protected]))':+ '@langchain/[email protected]([email protected])([email protected](@aws-sdk/[email protected](@aws-sdk/[email protected](@aws-sdk/[email protected]))(@aws-sdk/[email protected]))(@pinecone-database/[email protected])(@supabase/[email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected](@aws-sdk/[email protected](@aws-sdk/[email protected](@aws-sdk/[email protected])))([email protected]))([email protected]([email protected])([email protected]))([email protected])([email protected]([email protected]))([email protected])([email protected]))':dependencies:- '@langchain/core': 0.2.12([email protected](@aws-sdk/[email protected](@aws-sdk/[email protected](@aws-sdk/[email protected]))(@aws-sdk/[email protected]))(@pinecone-database/[email protected])(@supabase/[email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected](@aws-sdk/[email protected](@aws-sdk/[email protected](@aws-sdk/[email protected])))([email protected]))([email protected]([email protected])([email protected]))([email protected])([email protected]([email protected]))([email protected])([email protected]))([email protected]([email protected])([email protected]))+ '@langchain/core': 0.2.12([email protected](@aws-sdk/[email protected](@aws-sdk/[email protected](@aws-sdk/[email protected]))(@aws-sdk/[email protected]))(@pinecone-database/[email protected])(@supabase/[email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected](@aws-sdk/[email protected](@aws-sdk/[email protected](@aws-sdk/[email protected])))([email protected]))([email protected]([email protected])([email protected]))([email protected])([email protected]([email protected]))([email protected])([email protected]))([email protected]([email protected])([email protected]))js-tiktoken: 1.0.12openai: 4.57.0([email protected])([email protected])zod: 3.23.8@@ -5671,9 +5822,9 @@ snapshots:- encoding- langchain- '@langchain/[email protected]([email protected](@aws-sdk/[email protected](@aws-sdk/[email protected](@aws-sdk/[email protected]))(@aws-sdk/[email protected]))(@pinecone-database/[email protected])(@supabase/[email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected](@aws-sdk/[email protected](@aws-sdk/[email protected](@aws-sdk/[email protected])))([email protected]))([email protected]([email protected])([email protected]))([email protected])([email protected]([email protected]))([email protected])([email protected]))([email protected]([email protected])([email protected]))':+ '@langchain/[email protected]([email protected](@aws-sdk/[email protected](@aws-sdk/[email protected](@aws-sdk/[email protected]))(@aws-sdk/[email protected]))(@pinecone-database/[email protected])(@supabase/[email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected](@aws-sdk/[email protected](@aws-sdk/[email protected](@aws-sdk/[email protected])))([email protected]))([email protected]([email protected])([email protected]))([email protected])([email protected]([email protected]))([email protected])([email protected]))([email protected]([email protected])([email protected]))':dependencies:- '@langchain/core': 0.2.12([email protected](@aws-sdk/[email protected](@aws-sdk/[email protected](@aws-sdk/[email protected]))(@aws-sdk/[email protected]))(@pinecone-database/[email protected])(@supabase/[email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected](@aws-sdk/[email protected](@aws-sdk/[email protected](@aws-sdk/[email protected])))([email protected]))([email protected]([email protected])([email protected]))([email protected])([email protected]([email protected]))([email protected])([email protected]))([email protected]([email protected])([email protected]))+ '@langchain/core': 0.2.12([email protected](@aws-sdk/[email protected](@aws-sdk/[email protected](@aws-sdk/[email protected]))(@aws-sdk/[email protected]))(@pinecone-database/[email protected])(@supabase/[email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected](@aws-sdk/[email protected](@aws-sdk/[email protected](@aws-sdk/[email protected])))([email protected]))([email protected]([email protected])([email protected]))([email protected])([email protected]([email protected]))([email protected])([email protected]))([email protected]([email protected])([email protected]))js-tiktoken: 1.0.12transitivePeerDependencies:- langchain@@ -6811,6 +6962,8 @@ snapshots:transitivePeerDependencies:- supports-color+ [email protected]: {}+dependencies:humanize-ms: 1.2.1@@ -7321,12 +7474,22 @@ snapshots:+ dependencies:+ '@asamuzakjp/css-color': 2.8.3+ rrweb-cssom: 0.8.0++ dependencies:+ whatwg-mimetype: 4.0.0+ whatwg-url: 14.1.1+@@ -7351,6 +7514,8 @@ snapshots:+ [email protected]: {}+@@ -7661,6 +7826,12 @@ snapshots:combined-stream: 1.0.8mime-types: 2.1.35+ dependencies:+ asynckit: 0.4.0+ combined-stream: 1.0.8+ mime-types: 2.1.35+dependencies:node-domexception: 1.0.0@@ -7795,6 +7966,10 @@ snapshots:dependencies:whatwg-encoding: 2.0.0+ dependencies:+ whatwg-encoding: 3.1.1+@@ -7875,6 +8050,13 @@ snapshots:transitivePeerDependencies:- supports-color+ dependencies:+ agent-base: 7.1.3+ debug: 4.3.5+ transitivePeerDependencies:+ - supports-color+@@ -7984,6 +8166,8 @@ snapshots:+ [email protected]: {}+@@ -8400,6 +8584,34 @@ snapshots:+ dependencies:+ cssstyle: 4.2.1+ data-urls: 5.0.0+ decimal.js: 10.5.0+ form-data: 4.0.1+ html-encoding-sniffer: 4.0.0+ http-proxy-agent: 7.0.2+ https-proxy-agent: 7.0.6+ is-potential-custom-element-name: 1.0.1+ nwsapi: 2.2.16+ parse5: 7.2.1+ rrweb-cssom: 0.8.0+ saxes: 6.0.0+ symbol-tree: 3.2.4+ tough-cookie: 5.1.1+ w3c-xmlserializer: 5.0.0+ webidl-conversions: 7.0.0+ whatwg-encoding: 3.1.1+ whatwg-mimetype: 4.0.0+ whatwg-url: 14.1.1+ ws: 8.18.0+ xml-name-validator: 5.0.0+ transitivePeerDependencies:+ - bufferutil+ - supports-color+ - utf-8-validate+@@ -8435,17 +8647,17 @@ snapshots:- [email protected](@aws-sdk/[email protected](@aws-sdk/[email protected](@aws-sdk/[email protected]))(@aws-sdk/[email protected]))(@pinecone-database/[email protected])(@supabase/[email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected](@aws-sdk/[email protected](@aws-sdk/[email protected](@aws-sdk/[email protected])))([email protected]))([email protected]([email protected])([email protected]))([email protected])([email protected]([email protected]))([email protected])([email protected]):+ [email protected](@aws-sdk/[email protected](@aws-sdk/[email protected](@aws-sdk/[email protected]))(@aws-sdk/[email protected]))(@pinecone-database/[email protected])(@supabase/[email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected](@aws-sdk/[email protected](@aws-sdk/[email protected](@aws-sdk/[email protected])))([email protected]))([email protected]([email protected])([email protected]))([email protected])([email protected]([email protected]))([email protected])([email protected]):dependencies:- '@langchain/core': 0.2.12([email protected](@aws-sdk/[email protected](@aws-sdk/[email protected](@aws-sdk/[email protected]))(@aws-sdk/[email protected]))(@pinecone-database/[email protected])(@supabase/[email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected](@aws-sdk/[email protected](@aws-sdk/[email protected](@aws-sdk/[email protected])))([email protected]))([email protected]([email protected])([email protected]))([email protected])([email protected]([email protected]))([email protected])([email protected]))([email protected]([email protected])([email protected]))- '@langchain/openai': 0.2.1([email protected])([email protected](@aws-sdk/[email protected](@aws-sdk/[email protected](@aws-sdk/[email protected]))(@aws-sdk/[email protected]))(@pinecone-database/[email protected])(@supabase/[email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected](@aws-sdk/[email protected](@aws-sdk/[email protected](@aws-sdk/[email protected])))([email protected]))([email protected]([email protected])([email protected]))([email protected])([email protected]([email protected]))([email protected])([email protected]))- '@langchain/textsplitters': 0.0.3([email protected](@aws-sdk/[email protected](@aws-sdk/[email protected](@aws-sdk/[email protected]))(@aws-sdk/[email protected]))(@pinecone-database/[email protected])(@supabase/[email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected](@aws-sdk/[email protected](@aws-sdk/[email protected](@aws-sdk/[email protected])))([email protected]))([email protected]([email protected])([email protected]))([email protected])([email protected]([email protected]))([email protected])([email protected]))([email protected]([email protected])([email protected]))+ '@langchain/core': 0.2.12([email protected](@aws-sdk/[email protected](@aws-sdk/[email protected](@aws-sdk/[email protected]))(@aws-sdk/[email protected]))(@pinecone-database/[email protected])(@supabase/[email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected](@aws-sdk/[email protected](@aws-sdk/[email protected](@aws-sdk/[email protected])))([email protected]))([email protected]([email protected])([email protected]))([email protected])([email protected]([email protected]))([email protected])([email protected]))([email protected]([email protected])([email protected]))+ '@langchain/openai': 0.2.1([email protected])([email protected](@aws-sdk/[email protected](@aws-sdk/[email protected](@aws-sdk/[email protected]))(@aws-sdk/[email protected]))(@pinecone-database/[email protected])(@supabase/[email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected](@aws-sdk/[email protected](@aws-sdk/[email protected](@aws-sdk/[email protected])))([email protected]))([email protected]([email protected])([email protected]))([email protected])([email protected]([email protected]))([email protected])([email protected]))+ '@langchain/textsplitters': 0.0.3([email protected](@aws-sdk/[email protected](@aws-sdk/[email protected](@aws-sdk/[email protected]))(@aws-sdk/[email protected]))(@pinecone-database/[email protected])(@supabase/[email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected](@aws-sdk/[email protected](@aws-sdk/[email protected](@aws-sdk/[email protected])))([email protected]))([email protected]([email protected])([email protected]))([email protected])([email protected]([email protected]))([email protected])([email protected]))([email protected]([email protected])([email protected]))binary-extensions: 2.3.0js-tiktoken: 1.0.12js-yaml: 4.1.0jsonpointer: 5.0.1langchainhub: 0.0.11- langsmith: 0.1.34(npkyd6f7wyl3urgrzoxaktl5a4)+ langsmith: 0.1.34(7lljbsleilzgkaubvlq4ipicvq)ml-distance: 4.0.1openapi-types: 12.1.3p-retry: 4.6.2@@ -8463,6 +8675,7 @@ snapshots:handlebars: 4.7.8html-to-text: 9.0.5ioredis: 5.4.1+ jsdom: 26.0.0mammoth: 1.7.2mongodb: 6.6.2(@aws-sdk/[email protected](@aws-sdk/[email protected](@aws-sdk/[email protected])))([email protected])pdf-parse: 1.1.1@@ -8475,7 +8688,7 @@ snapshots:- [email protected](npkyd6f7wyl3urgrzoxaktl5a4):+ [email protected](7lljbsleilzgkaubvlq4ipicvq):dependencies:'@types/uuid': 9.0.8commander: 10.0.1@@ -8484,8 +8697,8 @@ snapshots:p-retry: 4.6.2uuid: 9.0.1optionalDependencies:- '@langchain/core': 0.2.12([email protected](@aws-sdk/[email protected](@aws-sdk/[email protected](@aws-sdk/[email protected]))(@aws-sdk/[email protected]))(@pinecone-database/[email protected])(@supabase/[email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected](@aws-sdk/[email protected](@aws-sdk/[email protected](@aws-sdk/[email protected])))([email protected]))([email protected]([email protected])([email protected]))([email protected])([email protected]([email protected]))([email protected])([email protected]))([email protected]([email protected])([email protected]))- langchain: 0.2.8(@aws-sdk/[email protected](@aws-sdk/[email protected](@aws-sdk/[email protected]))(@aws-sdk/[email protected]))(@pinecone-database/[email protected])(@supabase/[email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected](@aws-sdk/[email protected](@aws-sdk/[email protected](@aws-sdk/[email protected])))([email protected]))([email protected]([email protected])([email protected]))([email protected])([email protected]([email protected]))([email protected])([email protected])+ '@langchain/core': 0.2.12([email protected](@aws-sdk/[email protected](@aws-sdk/[email protected](@aws-sdk/[email protected]))(@aws-sdk/[email protected]))(@pinecone-database/[email protected])(@supabase/[email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected](@aws-sdk/[email protected](@aws-sdk/[email protected](@aws-sdk/[email protected])))([email protected]))([email protected]([email protected])([email protected]))([email protected])([email protected]([email protected]))([email protected])([email protected]))([email protected]([email protected])([email protected]))+ langchain: 0.2.8(@aws-sdk/[email protected](@aws-sdk/[email protected](@aws-sdk/[email protected]))(@aws-sdk/[email protected]))(@pinecone-database/[email protected])(@supabase/[email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected](@aws-sdk/[email protected](@aws-sdk/[email protected](@aws-sdk/[email protected])))([email protected]))([email protected]([email protected])([email protected]))([email protected])([email protected]([email protected]))([email protected])([email protected])openai: 4.57.0([email protected])([email protected])@@ -8554,6 +8767,8 @@ snapshots:+ [email protected]: {}+dependencies:yallist: 3.1.1@@ -8849,6 +9064,8 @@ snapshots:+ [email protected]: {}+@@ -8979,6 +9196,10 @@ snapshots:dependencies:entities: 4.5.0+ dependencies:+ entities: 4.5.0+dependencies:leac: 0.6.0@@ -9321,6 +9542,8 @@ snapshots:+ [email protected]: {}+@@ -9333,6 +9556,10 @@ snapshots:+ dependencies:+ xmlchars: 2.2.0+dependencies:loose-envify: 1.4.0@@ -9583,6 +9810,8 @@ snapshots:+ [email protected]: {}+@@ -9640,12 +9869,20 @@ snapshots:+ dependencies:+ tldts: 6.1.75+dependencies:punycode: 2.3.1+ dependencies:+ punycode: 2.3.1+[email protected](@babel/[email protected])(@jest/[email protected])(@jest/[email protected])([email protected](@babel/[email protected]))([email protected](@types/[email protected])([email protected](@types/[email protected])([email protected])))([email protected]):@@ -9777,6 +10014,10 @@ snapshots:+ dependencies:+ xml-name-validator: 5.0.0+dependencies:makeerror: 1.0.12@@ -9793,13 +10034,24 @@ snapshots:dependencies:iconv-lite: 0.6.3+ dependencies:+ iconv-lite: 0.6.3++ [email protected]: {}+dependencies:tr46: 4.1.1webidl-conversions: 7.0.0+ dependencies:+ tr46: 5.0.0+ webidl-conversions: 7.0.0+dependencies:tr46: 0.0.3@@ -9868,6 +10120,8 @@ snapshots:+ [email protected]: {}+dependencies:sax: 1.4.1@@ -9877,6 +10131,8 @@ snapshots:+ [email protected]: {}+diff --git a/apps/api/src/__tests__/snips/batch-scrape.test.ts b/apps/api/src/__tests__/snips/batch-scrape.test.tsindex 59c9da2ec..6dcadae42 100644--- a/apps/api/src/__tests__/snips/batch-scrape.test.ts+++ b/apps/api/src/__tests__/snips/batch-scrape.test.ts@@ -30,7 +30,7 @@ async function batchScrape(body: BatchScrapeRequestInput): ReturnType<typeof batx = await batchScrapeStatus(bss.body.id);expect(x.statusCode).toBe(200);expect(typeof x.body.status).toBe("string");- } while (x.body.status !== "completed")+ } while (x.body.status === "scraping");expectBatchScrapeToSucceed(x);return x;@@ -53,40 +53,51 @@ function expectBatchScrapeToSucceed(response: Awaited<ReturnType<typeof batchScr}describe("Batch scrape tests", () => {- describe("JSON format", () => {it.concurrent("works", async () => {const response = await batchScrape({- urls: ["http://firecrawl.dev"],- formats: ["json"],- jsonOptions: {- prompt: "Based on the information on the page, find what the company's mission is and whether it supports SSO, and whether it is open source.",- schema: {- type: "object",- properties: {- company_mission: {- type: "string",- },- supports_sso: {- type: "boolean",- },- is_open_source: {- type: "boolean",+ urls: ["http://firecrawl.dev"]+ });++ expect(response.body.data[0]).toHaveProperty("markdown");+ expect(response.body.data[0].markdown).toContain("Firecrawl");+ }, 30000);++ if (!process.env.TEST_SUITE_SELF_HOSTED) {+ describe("JSON format", () => {+ it.concurrent("works", async () => {+ const response = await batchScrape({+ urls: ["http://firecrawl.dev"],+ formats: ["json"],+ jsonOptions: {+ prompt: "Based on the information on the page, find what the company's mission is and whether it supports SSO, and whether it is open source.",+ schema: {+ type: "object",+ properties: {+ company_mission: {+ type: "string",+ },+ supports_sso: {+ type: "boolean",+ },+ is_open_source: {+ type: "boolean",+ },+ },+ required: ["company_mission", "supports_sso", "is_open_source"],},},- required: ["company_mission", "supports_sso", "is_open_source"],- },- },+ });++ expect(response.body.data[0]).toHaveProperty("json");+ expect(response.body.data[0].json).toHaveProperty("company_mission");+ expect(typeof response.body.data[0].json.company_mission).toBe("string");+ expect(response.body.data[0].json).toHaveProperty("supports_sso");+ expect(response.body.data[0].json.supports_sso).toBe(false);+ expect(typeof response.body.data[0].json.supports_sso).toBe("boolean");+ expect(response.body.data[0].json).toHaveProperty("is_open_source");+ expect(response.body.data[0].json.is_open_source).toBe(true);+ expect(typeof response.body.data[0].json.is_open_source).toBe("boolean");+ }, 30000);});-- expect(response.body.data[0]).toHaveProperty("json");- expect(response.body.data[0].json).toHaveProperty("company_mission");- expect(typeof response.body.data[0].json.company_mission).toBe("string");- expect(response.body.data[0].json).toHaveProperty("supports_sso");- expect(response.body.data[0].json.supports_sso).toBe(false);- expect(typeof response.body.data[0].json.supports_sso).toBe("boolean");- expect(response.body.data[0].json).toHaveProperty("is_open_source");- expect(response.body.data[0].json.is_open_source).toBe(true);- expect(typeof response.body.data[0].json.is_open_source).toBe("boolean");- }, 30000);- });+ }});diff --git a/apps/api/src/__tests__/snips/crawl.test.ts b/apps/api/src/__tests__/snips/crawl.test.tsindex 6aa47732a..1fb572efb 100644--- a/apps/api/src/__tests__/snips/crawl.test.ts+++ b/apps/api/src/__tests__/snips/crawl.test.ts@@ -30,7 +30,7 @@ async function crawl(body: CrawlRequestInput): ReturnType<typeof crawlStatus> {x = await crawlStatus(cs.body.id);expect(x.statusCode).toBe(200);expect(typeof x.body.status).toBe("string");- } while (x.body.status !== "completed")+ } while (x.body.status === "scraping");expectCrawlToSucceed(x);return x;diff --git a/apps/api/src/__tests__/snips/extract.test.ts b/apps/api/src/__tests__/snips/extract.test.tsindex 7375d85a6..993a17a6d 100644--- a/apps/api/src/__tests__/snips/extract.test.ts+++ b/apps/api/src/__tests__/snips/extract.test.ts@@ -30,7 +30,7 @@ async function extract(body: ExtractRequestInput): Promise<ExtractResponse> {x = await extractStatus(es.body.id);expect(x.statusCode).toBe(200);expect(typeof x.body.status).toBe("string");- } while (x.body.status !== "completed");+ } while (x.body.status === "processing");expectExtractToSucceed(x);return x.body;@@ -51,31 +51,37 @@ function expectExtractToSucceed(response: Awaited<ReturnType<typeof extractStatu}describe("Extract tests", () => {- it.concurrent("works", async () => {- const res = await extract({- urls: ["https://firecrawl.dev"],- schema: {- "type": "object",- "properties": {- "company_mission": {- "type": "string"+ if (!process.env.TEST_SUITE_SELF_HOSTED || process.env.OPENAI_API_KEY) {+ it.concurrent("works", async () => {+ const res = await extract({+ urls: ["https://firecrawl.dev"],+ schema: {+ "type": "object",+ "properties": {+ "company_mission": {+ "type": "string"+ },+ "is_open_source": {+ "type": "boolean"+ }},- "is_open_source": {- "type": "boolean"- }+ "required": [+ "company_mission",+ "is_open_source"+ ]},- "required": [- "company_mission",- "is_open_source"- ]- },- origin: "api-sdk",- });+ origin: "api-sdk",+ });- expect(res.data).toHaveProperty("company_mission");- expect(typeof res.data.company_mission).toBe("string")- expect(res.data).toHaveProperty("is_open_source");- expect(typeof res.data.is_open_source).toBe("boolean");- expect(res.data.is_open_source).toBe(true);- }, 60000);+ expect(res.data).toHaveProperty("company_mission");+ expect(typeof res.data.company_mission).toBe("string")+ expect(res.data).toHaveProperty("is_open_source");+ expect(typeof res.data.is_open_source).toBe("boolean");+ expect(res.data.is_open_source).toBe(true);+ }, 60000);+ } else {+ it.concurrent("dummy test", () => {+ expect(true).toBe(true);+ });+ }});diff --git a/apps/api/src/__tests__/snips/scrape.test.ts b/apps/api/src/__tests__/snips/scrape.test.tsindex 2ab5df9ef..31ce019e4 100644--- a/apps/api/src/__tests__/snips/scrape.test.ts+++ b/apps/api/src/__tests__/snips/scrape.test.ts@@ -41,41 +41,13 @@ describe("Scrape tests", () => {);}, 10000);- describe("Ad blocking (f-e dependant)", () => {- it.concurrent("blocks ads by default", async () => {- const response = await scrape({- url: "https://canyoublockit.com/testing/",- });-- expect(response.markdown).not.toContain(".g.doubleclick.net/");- }, 10000);-- it.concurrent("doesn't block ads if explicitly disabled", async () => {- const response = await scrape({- url: "https://canyoublockit.com/testing/",- blockAds: false,- });-- expect(response.markdown).toContain(".g.doubleclick.net/");- }, 10000);- });-- describe("Location API (f-e dependant)", () => {- it.concurrent("works without specifying an explicit location", async () => {- const response = await scrape({- url: "https://iplocation.com",- });- }, 10000);+ it("works", async () => {+ const response = await scrape({+ url: "http://firecrawl.dev"+ });- it.concurrent("works with country US", async () => {- const response = await scrape({- url: "https://iplocation.com",- location: { country: "US" },- });-- expect(response.markdown).toContain("| Country | United States |");- }, 10000);- });+ expect(response.markdown).toContain("Firecrawl");+ }, 10000);describe("JSON scrape support", () => {it.concurrent("returns parseable JSON", async () => {@@ -89,82 +61,122 @@ describe("Scrape tests", () => {}, 25000); // TODO: mock and shorten});- describe("Screenshot", () => {- it.concurrent("screenshot format works", async () => {- const response = await scrape({- url: "http://firecrawl.dev",- formats: ["screenshot"]- });-- expect(typeof response.screenshot).toBe("string");- }, 15000);+ if (!process.env.TEST_SUITE_SELF_HOSTED) {+ describe("Ad blocking (f-e dependant)", () => {+ it.concurrent("blocks ads by default", async () => {+ const response = await scrape({+ url: "https://canyoublockit.com/testing/",+ });- it.concurrent("screenshot@fullPage format works", async () => {- const response = await scrape({- url: "http://firecrawl.dev",- formats: ["screenshot@fullPage"]- });+ expect(response.markdown).not.toContain(".g.doubleclick.net/");+ }, 10000);++ it.concurrent("doesn't block ads if explicitly disabled", async () => {+ const response = await scrape({+ url: "https://canyoublockit.com/testing/",+ blockAds: false,+ });++ expect(response.markdown).toContain(".g.doubleclick.net/");+ }, 10000);+ });- expect(typeof response.screenshot).toBe("string");- }, 15000);- });+ describe("Location API (f-e dependant)", () => {+ it.concurrent("works without specifying an explicit location", async () => {+ const response = await scrape({+ url: "https://iplocation.com",+ });+ }, 10000);++ it.concurrent("works with country US", async () => {+ const response = await scrape({+ url: "https://iplocation.com",+ location: { country: "US" },+ });++ expect(response.markdown).toContain("| Country | United States |");+ }, 10000);+ });- describe("JSON format", () => {- it.concurrent("works", async () => {- const response = await scrape({- url: "http://firecrawl.dev",- formats: ["json"],- jsonOptions: {- prompt: "Based on the information on the page, find what the company's mission is and whether it supports SSO, and whether it is open source.",- schema: {- type: "object",- properties: {- company_mission: {- type: "string",- },- supports_sso: {- type: "boolean",- },- is_open_source: {- type: "boolean",+ describe("Screenshot (f-e/sb dependant)", () => {+ it.concurrent("screenshot format works", async () => {+ const response = await scrape({+ url: "http://firecrawl.dev",+ formats: ["screenshot"]+ });++ expect(typeof response.screenshot).toBe("string");+ }, 30000);++ it.concurrent("screenshot@fullPage format works", async () => {+ const response = await scrape({+ url: "http://firecrawl.dev",+ formats: ["screenshot@fullPage"]+ });++ expect(typeof response.screenshot).toBe("string");+ }, 30000);+ });++ describe("Proxy API (f-e dependant)", () => {+ it.concurrent("undefined works", async () => {+ await scrape({+ url: "http://firecrawl.dev",+ });+ }, 15000);++ it.concurrent("basic works", async () => {+ await scrape({+ url: "http://firecrawl.dev",+ proxy: "basic",+ });+ }, 15000);++ it.concurrent("stealth works", async () => {+ await scrape({+ url: "http://firecrawl.dev",+ proxy: "stealth",+ });+ }, 15000);+ });+ }++ if (!process.env.TEST_SUITE_SELF_HOSTED || process.env.OPENAI_API_KEY) {+ describe("JSON format", () => {+ it.concurrent("works", async () => {+ const response = await scrape({+ url: "http://firecrawl.dev",+ formats: ["json"],+ jsonOptions: {+ prompt: "Based on the information on the page, find what the company's mission is and whether it supports SSO, and whether it is open source.",+ schema: {+ type: "object",+ properties: {+ company_mission: {+ type: "string",+ },+ supports_sso: {+ type: "boolean",+ },+ is_open_source: {+ type: "boolean",+ },},+ required: ["company_mission", "supports_sso", "is_open_source"],},- required: ["company_mission", "supports_sso", "is_open_source"],},- },- });-- expect(response).toHaveProperty("json");- expect(response.json).toHaveProperty("company_mission");- expect(typeof response.json.company_mission).toBe("string");- expect(response.json).toHaveProperty("supports_sso");- expect(response.json.supports_sso).toBe(false);- expect(typeof response.json.supports_sso).toBe("boolean");- expect(response.json).toHaveProperty("is_open_source");- expect(response.json.is_open_source).toBe(true);- expect(typeof response.json.is_open_source).toBe("boolean");- }, 30000);- });-- describe("Proxy API (f-e dependant)", () => {- it.concurrent("undefined works", async () => {- await scrape({- url: "http://firecrawl.dev",- });- }, 15000);-- it.concurrent("basic works", async () => {- await scrape({- url: "http://firecrawl.dev",- proxy: "basic",- });- }, 15000);-- it.concurrent("stealth works", async () => {- await scrape({- url: "http://firecrawl.dev",- proxy: "stealth",- });- }, 15000);- });+ });++ expect(response).toHaveProperty("json");+ expect(response.json).toHaveProperty("company_mission");+ expect(typeof response.json.company_mission).toBe("string");+ expect(response.json).toHaveProperty("supports_sso");+ expect(response.json.supports_sso).toBe(false);+ expect(typeof response.json.supports_sso).toBe("boolean");+ expect(response.json).toHaveProperty("is_open_source");+ expect(response.json.is_open_source).toBe(true);+ expect(typeof response.json.is_open_source).toBe("boolean");+ }, 30000);+ });+ }});diff --git a/apps/api/src/__tests__/snips/search.test.ts b/apps/api/src/__tests__/snips/search.test.tsindex 5cb5323db..1e191c573 100644--- a/apps/api/src/__tests__/snips/search.test.ts+++ b/apps/api/src/__tests__/snips/search.test.ts@@ -27,8 +27,8 @@ async function search(body: SearchRequestInput): Promise<Document> {return raw.body.data;}-describe("Scrape tests", () => {- it("works", async () => {+describe("Search tests", () => {+ it.concurrent("works", async () => {await search({query: "firecrawl"});diff --git a/apps/api/src/controllers/v1/crawl-status.ts b/apps/api/src/controllers/v1/crawl-status.tsindex 8d0ea1b72..9a5ecaad4 100644--- a/apps/api/src/controllers/v1/crawl-status.ts+++ b/apps/api/src/controllers/v1/crawl-status.ts@@ -13,13 +13,13 @@ import {getDoneJobsOrderedLength,isCrawlKickoffFinished,} from "../../lib/crawl-redis";-import { getScrapeQueue } from "../../services/queue-service";+import { getScrapeQueue, QueueFunction } from "../../services/queue-service";import {supabaseGetJobById,supabaseGetJobsById,} from "../../lib/supabase-jobs";import { configDotenv } from "dotenv";-import type { Job, JobState } from "bullmq";+import type { Job, JobState, Queue } from "bullmq";import { logger } from "../../lib/logger";import { supabase_service } from "../../services/supabase";import { getConcurrencyLimitedJobs } from "../../lib/concurrency-limit";@@ -245,7 +245,7 @@ export async function crawlStatusController(let totalCount = jobIDs.length;- if (totalCount === 0) {+ if (totalCount === 0 && process.env.USE_DB_AUTHENTICATION === "true") {const x = await supabase_service.from('firecrawl_jobs').select('*', { count: 'exact', head: true })diff --git a/apps/api/src/controllers/v1/extract-status.ts b/apps/api/src/controllers/v1/extract-status.tsindex 81477e7c3..c83ab506c 100644--- a/apps/api/src/controllers/v1/extract-status.ts+++ b/apps/api/src/controllers/v1/extract-status.ts@@ -1,7 +1,34 @@import { Response } from "express";-import { supabaseGetJobsById } from "../../lib/supabase-jobs";import { RequestWithAuth } from "./types";import { getExtract, getExtractExpiry } from "../../lib/extract/extract-redis";+import { DBJob, PseudoJob } from "./crawl-status";+import { getExtractQueue } from "../../services/queue-service";+import { ExtractResult } from "../../lib/extract/extraction-service";+import { supabaseGetJobById } from "../../lib/supabase-jobs";++export async function getExtractJob(id: string): Promise<PseudoJob<ExtractResult> | null> {+ const [bullJob, dbJob] = await Promise.all([+ getExtractQueue().getJob(id),+ (process.env.USE_DB_AUTHENTICATION === "true" ? supabaseGetJobById(id) : null) as Promise<DBJob | null>,+ ]);++ if (!bullJob && !dbJob) return null;++ const data = dbJob?.docs ?? bullJob?.returnvalue?.data;++ const job: PseudoJob<any> = {+ id,+ getState: bullJob ? bullJob.getState : (() => dbJob!.success ? "completed" : "failed"),+ returnvalue: data,+ data: {+ scrapeOptions: bullJob ? bullJob.data.scrapeOptions : dbJob!.page_options,+ },+ timestamp: bullJob ? bullJob.timestamp : new Date(dbJob!.date_added).valueOf(),+ failedReason: (bullJob ? bullJob.failedReason : dbJob!.message) || undefined,+ }++ return job;+}export async function extractStatusController(req: RequestWithAuth<{ jobId: string }, any, any>,@@ -16,24 +43,29 @@ export async function extractStatusController(});}- let data: any[] = [];+ let data: ExtractResult | [] = [];if (extract.status === "completed") {- const jobData = await supabaseGetJobsById([req.params.jobId]);- if (!jobData || jobData.length === 0) {+ const jobData = await getExtractJob(req.params.jobId);+ if (!jobData) {return res.status(404).json({success: false,error: "Job not found",});}- data = jobData[0].docs;+ if (!jobData.returnvalue) {+ // if we got in the split-second where the redis is updated but the bull isn't+ // just pretend it's still processing - MG+ extract.status = "processing";+ } else {+ data = jobData.returnvalue ?? [];+ }}- // console.log(extract.sources);return res.status(200).json({success: extract.status === "failed" ? false : true,- data: data,+ data,status: extract.status,error: extract?.error ?? undefined,expiresAt: (await getExtractExpiry(req.params.jobId)).toISOString(),diff --git a/apps/api/src/lib/extract/completions/analyzeSchemaAndPrompt.ts b/apps/api/src/lib/extract/completions/analyzeSchemaAndPrompt.tsindex 1ed7decf5..5b54f3f8a 100644--- a/apps/api/src/lib/extract/completions/analyzeSchemaAndPrompt.ts+++ b/apps/api/src/lib/extract/completions/analyzeSchemaAndPrompt.ts@@ -7,7 +7,6 @@ import {} from "../build-prompts";import OpenAI from "openai";import { logger } from "../../../lib/logger";-const openai = new OpenAI();export async function analyzeSchemaAndPrompt(urls: string[],@@ -40,6 +39,7 @@ export async function analyzeSchemaAndPrompt(const model = "gpt-4o";+ const openai = new OpenAI();const result = await openai.beta.chat.completions.parse({model: model,messages: [diff --git a/apps/api/src/lib/extract/extraction-service.ts b/apps/api/src/lib/extract/extraction-service.tsindex ab4b1aebf..ff43deddf 100644--- a/apps/api/src/lib/extract/extraction-service.ts+++ b/apps/api/src/lib/extract/extraction-service.ts@@ -48,7 +48,7 @@ interface ExtractServiceOptions {cacheKey?: string;}-interface ExtractResult {+export interface ExtractResult {success: boolean;data?: any;extractId: string;diff --git a/apps/api/src/lib/extract/index/pinecone.ts b/apps/api/src/lib/extract/index/pinecone.tsindex df4c7686d..7e5ddf415 100644--- a/apps/api/src/lib/extract/index/pinecone.ts+++ b/apps/api/src/lib/extract/index/pinecone.ts@@ -3,10 +3,6 @@ import { Document } from "../../../controllers/v1/types";import { logger } from "../../logger";import OpenAI from "openai";-const openai = new OpenAI({- apiKey: process.env.OPENAI_API_KEY,-});-const pinecone = new Pinecone({apiKey: process.env.PINECONE_API_KEY!,});@@ -27,6 +23,10 @@ export interface PageMetadata {}async function getEmbedding(text: string) {+ const openai = new OpenAI({+ apiKey: process.env.OPENAI_API_KEY,+ });+const embedding = await openai.embeddings.create({model: "text-embedding-3-small",input: text,diff --git a/apps/api/src/lib/llm/generate.ts b/apps/api/src/lib/llm/generate.tsindex 5249dadd5..6edbda9a0 100644--- a/apps/api/src/lib/llm/generate.ts+++ b/apps/api/src/lib/llm/generate.ts@@ -1,9 +1,5 @@import OpenAI from "openai";-const openai = new OpenAI({- apiKey: process.env.OPENAI_API_KEY,-});-interface Message {role: "system" | "user" | "assistant";content: string;@@ -19,6 +15,10 @@ interface GenerateTextOptions {export async function generateText(options: GenerateTextOptions) {const { model, messages, temperature = 0.7, maxTokens } = options;+ const openai = new OpenAI({+ apiKey: process.env.OPENAI_API_KEY,+ });+const completion = await openai.chat.completions.create({model,messages,diff --git a/apps/api/src/lib/ranker.ts b/apps/api/src/lib/ranker.tsindex 16e497648..63cd6c209 100644--- a/apps/api/src/lib/ranker.ts+++ b/apps/api/src/lib/ranker.ts@@ -1,14 +1,13 @@-import axios from "axios";import { configDotenv } from "dotenv";import OpenAI from "openai";configDotenv();-const openai = new OpenAI({- apiKey: process.env.OPENAI_API_KEY,-});-async function getEmbedding(text: string) {+ const openai = new OpenAI({+ apiKey: process.env.OPENAI_API_KEY,+ });+const embedding = await openai.embeddings.create({model: "text-embedding-3-small",input: text,diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.tsindex 58fc5b3e5..2bf9c5c5e 100644--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts@@ -10,6 +10,7 @@ import {UnsupportedFileError,} from "../../error";import { MockState } from "../../lib/mock";+import { fireEngineURL } from "./scrape";const successSchema = z.object({jobId: z.string(),@@ -85,8 +86,6 @@ export async function fireEngineCheckStatus(jobId: string,mock: MockState | null,): Promise<FireEngineCheckStatusSuccess> {- const fireEngineURL = process.env.FIRE_ENGINE_BETA_URL!;-const status = await Sentry.startSpan({name: "fire-engine: Check status",diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/delete.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/delete.tsindex d20df42eb..d046b738b 100644--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/delete.ts+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/delete.ts@@ -3,14 +3,13 @@ import * as Sentry from "@sentry/node";import { robustFetch } from "../../lib/fetch";import { MockState } from "../../lib/mock";+import { fireEngineURL } from "./scrape";export async function fireEngineDelete(logger: Logger,jobId: string,mock: MockState | null,) {- const fireEngineURL = process.env.FIRE_ENGINE_BETA_URL!;-await Sentry.startSpan({name: "fire-engine: Delete scrape",diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.tsindex 607c0c8d6..1b63dc756 100644--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts@@ -65,6 +65,8 @@ const schema = z.object({processing: z.boolean(),});+export const fireEngineURL = process.env.FIRE_ENGINE_BETA_URL ?? "<mock-fire-engine-url>";+export async function fireEngineScrape<Engine extends| FireEngineScrapeRequestChromeCDP@@ -75,10 +77,6 @@ export async function fireEngineScrape<request: FireEngineScrapeRequestCommon & Engine,mock: MockState | null,): Promise<z.infer<typeof schema>> {- const fireEngineURL = process.env.FIRE_ENGINE_BETA_URL!;-- // TODO: retries-const scrapeRequest = await Sentry.startSpan({name: "fire-engine: Scrape",diff --git a/apps/api/src/scraper/scrapeURL/engines/index.ts b/apps/api/src/scraper/scrapeURL/engines/index.tsindex f32708c0d..0a688fc6a 100644--- a/apps/api/src/scraper/scrapeURL/engines/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/index.ts@@ -310,7 +310,12 @@ export function buildFallbackList(meta: Meta): {engine: Engine;unsupportedFeatures: Set<FeatureFlag>;}[] {- const _engines = [...engines];+ const _engines: Engine[] = [+ ...engines,++ // enable fire-engine in self-hosted testing environment when mocks are supplied+ ...((!useFireEngine && meta.mock !== null) ? ["fire-engine;chrome-cdp", "fire-engine;playwright", "fire-engine;tlsclient"] as Engine[] : [])+ ];if (meta.internalOptions.useCache !== true) {const cacheIndex = _engines.indexOf("cache");diff --git a/apps/api/src/scraper/scrapeURL/lib/fetch.ts b/apps/api/src/scraper/scrapeURL/lib/fetch.tsindex c9513bfe3..c0fc1702f 100644--- a/apps/api/src/scraper/scrapeURL/lib/fetch.ts+++ b/apps/api/src/scraper/scrapeURL/lib/fetch.ts@@ -2,6 +2,7 @@ import { Logger } from "winston";import { z, ZodError } from "zod";import * as Sentry from "@sentry/node";import { MockState, saveMock } from "./mock";+import { fireEngineURL } from "../engines/fire-engine/scrape";export type RobustFetchParams<Schema extends z.Schema<any>> = {url: string;@@ -126,14 +127,13 @@ export async function robustFetch<const makeRequestTypeId = (request: (typeof mock)["requests"][number]["options"],) => {- let trueUrl = (process.env.FIRE_ENGINE_BETA_URL && request.url.startsWith(process.env.FIRE_ENGINE_BETA_URL))- ? request.url.replace(process.env.FIRE_ENGINE_BETA_URL, "<fire-engine>")+ let trueUrl = request.url.startsWith(fireEngineURL)+ ? request.url.replace(fireEngineURL, "<fire-engine>"): request.url;let out = trueUrl + ";" + request.method;if (- process.env.FIRE_ENGINE_BETA_URL &&- (trueUrl.startsWith("<fire-engine>")) &&+ trueUrl.startsWith("<fire-engine>") &&request.method === "POST") {out += "f-e;" + request.body?.engine + ";" + request.body?.url;diff --git a/apps/api/src/search/fireEngine.ts b/apps/api/src/search/fireEngine.tsindex 262775238..8596666c8 100644--- a/apps/api/src/search/fireEngine.ts+++ b/apps/api/src/search/fireEngine.ts@@ -29,9 +29,7 @@ export async function fireEngineMap(});if (!process.env.FIRE_ENGINE_BETA_URL) {- console.warn(- "(v1/map Beta) Results might differ from cloud offering currently.",- );+ logger.warn("(v1/map Beta) Results might differ from cloud offering currently.");return [];}diff --git a/apps/api/src/search/googlesearch.ts b/apps/api/src/search/googlesearch.tsindex 07719ae5c..72ab97d70 100644--- a/apps/api/src/search/googlesearch.ts+++ b/apps/api/src/search/googlesearch.ts@@ -1,21 +1,18 @@import axios from "axios";-import * as cheerio from "cheerio"; // TODO: rustify+import { JSDOM } from 'jsdom';import * as querystring from "querystring";import { SearchResult } from "../../src/lib/entities";import { logger } from "../../src/lib/logger";+import https from 'https';-const _useragent_list = [- "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:66.0) Gecko/20100101 Firefox/66.0",- "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Safari/537.36",- "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Safari/537.36",- "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36",- "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Safari/537.36",- "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Safari/537.36 Edg/111.0.1661.62",- "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/111.0",-];+const getRandomInt = (min: number, max: number): number => Math.floor(Math.random() * (max - min + 1)) + min;-function get_useragent(): string {- return _useragent_list[Math.floor(Math.random() * _useragent_list.length)];+export function get_useragent(): string {+ const lynx_version = `Lynx/${getRandomInt(2, 3)}.${getRandomInt(8, 9)}.${getRandomInt(0, 2)}`;+ const libwww_version = `libwww-FM/${getRandomInt(2, 3)}.${getRandomInt(13, 15)}`;+ const ssl_mm_version = `SSL-MM/${getRandomInt(1, 2)}.${getRandomInt(3, 5)}`;+ const openssl_version = `OpenSSL/${getRandomInt(1, 3)}.${getRandomInt(0, 4)}.${getRandomInt(0, 9)}`;+ return `${lynx_version} ${libwww_version} ${ssl_mm_version} ${openssl_version}`;}async function _req(@@ -31,9 +28,10 @@ async function _req() {const params = {q: term,- num: results, // Number of results to return+ num: results+2, // Number of results to returnhl: lang,gl: country,+ safe: "active",start: start,};if (tbs) {@@ -42,18 +40,25 @@ async function _req(if (filter) {params["filter"] = filter;}+ var agent = get_useragent();try {const resp = await axios.get("https://www.google.com/search", {headers: {- "User-Agent": get_useragent(),+ "User-Agent": agent,+ "Accept": "*/*"},params: params,proxy: proxies,timeout: timeout,+ httpsAgent: new https.Agent({+ rejectUnauthorized: true+ }),+ withCredentials: true});return resp;} catch (error) {if (error.response && error.response.status === 429) {+ logger.warn("Google Search: Too many requests, try again later.", error.response);throw new Error("Google Search: Too many requests, try again later.");}throw error;@@ -100,34 +105,42 @@ export async function googleSearch(tbs,filter,);- const $ = cheerio.load(resp.data);- const result_block = $("div.g");+ const dom = new JSDOM(resp.data);+ const document = dom.window.document;+ const result_block = document.querySelectorAll("div.ezO2md");+ let new_results = 0;+ let unique = true;+ let fetched_results = 0;++ const fetched_links = new Set<string>();if (result_block.length === 0) {start += 1;attempts += 1;} else {- attempts = 0; // Reset attempts if we have results+ attempts = 0;}- result_block.each((index, element) => {- const linkElement = $(element).find("a");- const link =- linkElement && linkElement.attr("href")- ? linkElement.attr("href")- : null;- const title = $(element).find("h3");- const ogImage = $(element).find("img").eq(1).attr("src");- const description_box = $(element).find(- "div[style='-webkit-line-clamp:2']",- );- const answerBox = $(element).find(".mod").text();- if (description_box) {- const description = description_box.text();- if (link && title && description) {- start += 1;- results.push(new SearchResult(link, title.text(), description));++ for (const result of result_block) {+ const link_tag = result.querySelector("a[href]") as HTMLAnchorElement;+ const title_tag = link_tag ? link_tag.querySelector("span.CVA68e") : null;+ const description_tag = result.querySelector("span.FrIlee");++ if (link_tag && title_tag && description_tag) {+ const link = decodeURIComponent(link_tag.href.split("&")[0].replace("/url?q=", ""));+ if (fetched_links.has(link) && unique) continue;+ fetched_links.add(link);+ const title = title_tag.textContent || "";+ const description = description_tag.textContent || "";+ fetched_results++;+ new_results++;+ if (link && title && description) {+ start += 1+ results.push(new SearchResult(link, title, description));+ }+ if (fetched_results >= num_results) break;}- }- });+ }+await new Promise((resolve) =>setTimeout(resolve, sleep_interval * 1000),);diff --git a/apps/api/src/search/index.ts b/apps/api/src/search/index.tsindex d4e6ce9da..e85ee384f 100644--- a/apps/api/src/search/index.ts+++ b/apps/api/src/search/index.ts@@ -64,7 +64,7 @@ export async function search({timeout,);} catch (error) {- logger.error(`Error in search function: ${error}`);+ logger.error(`Error in search function`, { error });return [];}}diff --git a/apps/api/src/services/queue-service.ts b/apps/api/src/services/queue-service.tsindex 9969c4b21..b910fa4b4 100644--- a/apps/api/src/services/queue-service.ts+++ b/apps/api/src/services/queue-service.ts@@ -2,6 +2,8 @@ import { Queue } from "bullmq";import { logger } from "../lib/logger";import IORedis from "ioredis";+export type QueueFunction = () => Queue<any, any, string, any, any, string>;+let scrapeQueue: Queue;let extractQueue: Queue;let loggingQueue: Queue;