Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
chore: ts-prune
Gergő Móricz8beb05c72671
diff --git a/apps/api/package.json b/apps/api/package.jsonindex 5f9b70dd2..248727d08 100644--- a/apps/api/package.json+++ b/apps/api/package.json@@ -59,6 +59,7 @@"supertest": "^6.3.3","ts-jest": "^29.1.1","ts-node": "^10.9.1",+ "ts-prune": "^0.10.3","tsc-watch": "^7.1.1","tsx": "^4.20.3","typescript": "^5.8.3"@@ -79,6 +80,7 @@"@devil7softwares/pos": "^1.0.2","@dqbd/tiktoken": "^1.0.17","@google-cloud/storage": "^7.16.0",+ "@mendable/firecrawl-rs": "workspace:*","@nangohq/node": "^0.40.8","@openrouter/ai-sdk-provider": "^0.4.5","@opentelemetry/auto-instrumentations-node": "^0.62.0",@@ -166,8 +168,7 @@"ws": "^8.18.0","x402-express": "^0.5.3","xml2js": "^0.6.2",- "zod": "^3.24.2",- "@mendable/firecrawl-rs": "workspace:*"+ "zod": "^3.24.2"},"nodemonConfig": {"ignore": [@@ -191,7 +192,8 @@}},"lint-staged": {- "*.{js,jsx,ts,tsx,json,css,md}": "prettier --write"+ "*.{js,jsx,ts,tsx,json,css,md}": "prettier --write",+ "*.{js,tsx,ts,tsx}": "ts-prune --skip \"native/\" --error"},"packageManager": "[email protected]"}diff --git a/apps/api/pnpm-lock.yaml b/apps/api/pnpm-lock.yamlindex 4642bd0ce..bd7b2e423 100644--- a/apps/api/pnpm-lock.yaml+++ b/apps/api/pnpm-lock.yaml@@ -391,6 +391,9 @@ importers:ts-node:specifier: ^10.9.1version: 10.9.2(@types/[email protected])([email protected])+ ts-prune:+ specifier: ^0.10.3+ version: 0.10.3tsc-watch:specifier: ^7.1.1version: 7.1.1([email protected])@@ -1968,6 +1971,18 @@ packages:resolution: {integrity: sha512-jCs9ldd7NwzpgXDIf6P3+NrHh9/sD6CQdxHyjQI+h/6rDNo88ypBxxz45UDuZHz9r3tNz7N/VInSVoVdtXEI4A==}engines: {node: ^14.21.3 || >=16}+ '@nodelib/[email protected]':+ resolution: {integrity: sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==}+ engines: {node: '>= 8'}++ '@nodelib/[email protected]':+ resolution: {integrity: sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A==}+ engines: {node: '>= 8'}++ '@nodelib/[email protected]':+ resolution: {integrity: sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==}+ engines: {node: '>= 8'}+'@octokit/[email protected]':resolution: {integrity: sha512-P4YJBPdPSpWTQ1NU4XYdvHvXJJDxM6YwpS0FZHRgP7YFkdVxsWcpWGy/NVqlAA7PcPCnMacXlRm1y2PFZRWL/w==}engines: {node: '>= 20'}@@ -3347,6 +3362,9 @@ packages:resolution: {integrity: sha512-XCuKFP5PS55gnMVu3dty8KPatLqUoy/ZYzDzAGCQ8JNFCkLXzmI7vNHCR+XpbZaMWQK/vQubr7PkYq8g470J/A==}engines: {node: '>= 10'}+ '@ts-morph/[email protected]':+ resolution: {integrity: sha512-4tUmeLyXJnJWvTFOKtcNJ1yh0a3SsTLi2MUoyj8iUNznFRN1ZquaNe7Oukqrnki2FzZkm0J9adCNLDZxUzvj+w==}+'@tsconfig/[email protected]':resolution: {integrity: sha512-DcRjDCujK/kCk/cUe8Xz8ZSpm8mS3mNNpta+jGCA6USEDfktlNvm1+IuZ9eTcDbNk41BHwpHHeW+N1lKCz4zOw==}@@ -3473,6 +3491,9 @@ packages:'@types/[email protected]':resolution: {integrity: sha512-kK1eBS/Adeyis+3OlBDMeQQuasIDLUYXsi2T15ccNJ0iyUpQ4xDF7svFu3+bGVrI0CMBUclPciz+lsQR3JX3TQ==}+ '@types/[email protected]':+ resolution: {integrity: sha512-dISoDXWWQwUquiKsyZ4Ng+HX2KsPL7LyHKHQwgGFEA3IaKac4Obd+h2a/a6waisAoepJlBcx9paWqjA8/HVjCw==}+'@types/[email protected]':resolution: {integrity: sha512-+gbBHbNCVGGYw1S9lAIIvrHW47UYOhMIFUsJcMkMrzy1Jf0vulBN3XQIjPgnoOXveMuHnF3b57fXROnY/Or7eg==}@@ -4088,6 +4109,9 @@ packages:resolution: {integrity: sha512-QVb0dM5HvG+uaxitm8wONl7jltx8dqhfU33DcqtOZcLSVIKSDDLDi7+0LbAKiyI8hD9u42m2YxXSkMGWThaecQ==}engines: {iojs: '>= 1.0.0', node: '>= 0.12.0'}+ resolution: {integrity: sha512-NiujjUFB4SwScJq2bwbYUtXbZhBSlY6vYzm++3Q6oC+U+injTqfPYFK8wS9COOmb2lueqp0ZRB4nK1VYeHgNyw==}+resolution: {integrity: sha512-hSo2/tFV29whjFFtVtdS7kHmtUsjfMO1sgwE/d5bhOE4O7Vkj5G1R9lLIqkIprp/+rrvCq3HGvEaOgry7xRcDA==}@@ -4141,6 +4165,10 @@ packages:resolution: {integrity: sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==}+ resolution: {integrity: sha512-U7VdrJFnJgo4xjrHpTzu0yrHPGImdsmD95ZlgYSEajAn2JKzDhDTPG9kBTefmObL2w/ngeZnilk+OV9CG3d7UA==}+ engines: {node: '>= 6'}+resolution: {integrity: sha512-T0+barUSQRTUQASh8bx02dl+DhF54GtIDY13Y3m9oWTklKbb3Wv974meRpeZ3lp1JpLVECWWNHC4vaG2XHXouQ==}@@ -4181,6 +4209,10 @@ packages:resolution: {integrity: sha512-KIHbLJqu73RGr/hnbrO9uBeixNGuvSQjul/jdFvS/KFSIH1hWVd1ng7zOHx+YrEfInLG7q4n6GHQ9cDtxv/P6g==}engines: {node: '>= 0.10'}+ resolution: {integrity: sha512-AdmX6xUzdNASswsFtmwSt7Vj8po9IuqXm0UXz7QKPuEUmPB4XyjGfaAr2PSuELMwkRMVH1EpIkX5bTZGRB3eCA==}+ engines: {node: '>=10'}+resolution: {integrity: sha512-nPklKJ5qtmY5MdBKw1NiBAoyx5Sa7p2yPpljZyQ7gyCN1m+eMFs9I6CT37Mxt8zvR5L3VzD3DJBE4WQzX3WF4A==}@@ -4639,6 +4671,10 @@ packages:resolution: {integrity: sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==}+ resolution: {integrity: sha512-7MptL8U0cqcFdzIzwOTHoilX9x5BrNqye7Z/LuC7kCMRio1EMSyqRK3BEAUD7sXRq4iT4AzTVuZdhgQ2TCvYLg==}+ engines: {node: '>=8.6.0'}+resolution: {integrity: sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==}@@ -4659,6 +4695,9 @@ packages:resolution: {integrity: sha512-Pb8d48e+oIuY4MaM64Cd7OW1gt4nxCHs7/ddPPZ/Ic3sg8yVGM7O9wDvZ7us6ScaUupzM+pfBolwtYhN1IxBIw==}+ resolution: {integrity: sha512-GwLTyxkCXjXbxqIhTsMI2Nui8huMPtnxg7krajPJAjnEG/iiOS7i+zCtWGZR9G0NBKbXKh6X9m9UIsYX/N6vvQ==}+resolution: {integrity: sha512-p5161BqbuCaSnB8jIbzQHOlpgsPmK5rJVDfDKO91Axs5NC1uu3HRQm6wt9cd9/+GtQQIO53JdGXXoyDpTAsgYA==}@@ -4824,6 +4863,10 @@ packages:resolution: {integrity: sha512-/Iu4prUrydE3Pb3lCBMbcSNIf81tgGt0W1ZwknnyF62t3tHmtiJTRj0f+1ZIhp3+Rh0ktz1pJVoa7ZXUCskivA==}engines: {node: '>= 4.8.0'}+ resolution: {integrity: sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==}+ engines: {node: '>= 6'}+resolution: {integrity: sha512-GwMlUF6PkPo3Gk21UxkCohOv0PLcIXVtKyLlpEI28R/cO/4eNOdmLk3CMW1wROV/WR/EsZOWAfBbBOqYvs88/w==}engines: {node: '>=16 || 14 >=14.18'}@@ -4976,6 +5019,10 @@ packages:resolution: {integrity: sha512-XXOFtyqDjNDAQxVfYxuF7g9Il/IbWmmlQg2MYKOH8ExIT1qg6xc4zyS3HaEEATgs1btfzxq15ciUiY7gjSXRGQ==}+ resolution: {integrity: sha512-TR3KfrTZTYLPB6jUjfx6MF9WcWrHL9su5TObK4ZkYgBdWKPOFoSoQIdEuTuR82pmtxH2spWG9h6etwfr1pLBqQ==}+ engines: {node: '>=6'}+resolution: {integrity: sha512-5tCuY9BV8ujfOpwtAGgsTx9CGUapcFMEEyByLv1B+v2+6DhAcw+Zr0nhQT7uwaZ7DiourxFEscghOR8e1aPLQw==}@@ -5041,6 +5088,10 @@ packages:resolution: {integrity: sha512-hHrIjvZsftOsvKSn2TRYl63zvxsgE0K+0mYMoH6gD4omR5IWB2KynivBQczo3+wF1cCkjzvptnI9Q0sPU66ilw==}+ resolution: {integrity: sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==}+ engines: {node: '>=0.10.0'}+resolution: {integrity: sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==}engines: {node: '>=8'}@@ -5061,6 +5112,10 @@ packages:resolution: {integrity: sha512-nPUB5km40q9e8UfN/Zc24eLlzdSf9OfKByBw9CIdw4H1giPMeA0OIJvbchsCu4npfI2QcMVBsGEBHKZ7wLTWmQ==}engines: {node: '>= 0.4'}+ resolution: {integrity: sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==}+ engines: {node: '>=0.10.0'}+resolution: {integrity: sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==}engines: {node: '>=0.12.0'}@@ -5582,6 +5637,10 @@ packages:resolution: {integrity: sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w==}+ resolution: {integrity: sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==}+ engines: {node: '>= 8'}+resolution: {integrity: sha512-iclAHeNqNm68zFtnZ0e+1L2yUIdvzNoauKU4WBA3VvH/vPFieF7qfRlwUZU+DA9P9bPXIS90ulxoUoCH23sV2w==}engines: {node: '>= 0.6'}@@ -5658,6 +5717,11 @@ packages:typescript:optional: true+ resolution: {integrity: sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==}+ engines: {node: '>=10'}+ hasBin: true+resolution: {integrity: sha512-+NsyUUAZDmo6YVHzL/stxSu3t9YS1iljliy3BSDrXJ/dkn1KYdmtZODGGjLcc9XLgVVpH4KshHB8XmZgMhaBXg==}engines: {node: '>=10'}@@ -5731,6 +5795,7 @@ packages:resolution: {integrity: sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==}engines: {node: '>=10.5.0'}+ deprecated: Use your platform's native DOMException insteadresolution: {integrity: sha512-DRI60hzo2oKN1ma0ckc6nQWlHU69RH6xN0sjQTjMpChPfTYvKZdcQFfdYK2RWbJcKyUizSIy/l8OTGxMAM1QDw==}@@ -5922,6 +5987,10 @@ packages:resolution: {integrity: sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw==}+ resolution: {integrity: sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==}+ engines: {node: '>=6'}+resolution: {integrity: sha512-Oq4j8LAOPOcssanQkIjxosjATBIEJhCxMCxPhMu+Ci4wdNmAEdx0O+a7gzbR2PyKXgKPvRLIN5g224+dJAsKHA==}@@ -5948,6 +6017,9 @@ packages:resolution: {integrity: sha512-Njv/59hHaokb/hRUjce3Hdv12wd60MtM9Z5Olmn+nehe0QDAsRtRbJPvJ0Z91TusF0SuZRIvnM+S4l6EIP8leA==}+ resolution: {integrity: sha512-b7uo2UCUOYZcnF/3ID0lulOJi/bafxa1xPe7ZPsammBSpjSWQkjNxlt635YGS2MiR9GjvuXCtz2emr3jbsz98g==}+resolution: {integrity: sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==}engines: {node: '>=8'}@@ -5974,6 +6046,10 @@ packages:resolution: {integrity: sha512-RA1GjUVMnvYFxuqovrEqZoxxW5NUZqbwKtYz/Tt7nXerk0LbLblQmrsgdeOxV5SFHf0UDggjS/bSeOZwt1pmEQ==}+ resolution: {integrity: sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw==}+ engines: {node: '>=8'}+resolution: {integrity: sha512-e3FBlXLmN/D1S+zHzanP4E/4Z60oFAa3O051qt1pxa7DEJWKAyil6upYVXCWadEnuoqa4Pkc9oUx9zsxYeRv8A==}@@ -6214,6 +6290,9 @@ packages:resolution: {integrity: sha512-FIqgj2EUvTa7R50u0rGsyTftzjYmv/a3hO345bZNrqabNqjtgiDMgmo4mkUjd+nzU5oF3dClKqFIPUKybUyqoQ==}+ resolution: {integrity: sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==}+resolution: {integrity: sha512-tYC1Q1hgyRuHgloV/YXs2w15unPVh8qfu/qCTfhTYamaw7fyhumKa2yGpdSo87vY32rIclj+4fWYQXUMs9EHvg==}@@ -6318,6 +6397,10 @@ packages:resolution: {integrity: sha512-OrZaX2Mb+rJCpH/6CpSqt9xFVpN++x01XnN2ie9g6P5/3xelLAkXWVADpdz1IHD/KFfEXyE6V0U01OQ3UO2rEg==}engines: {node: '>=8'}+ resolution: {integrity: sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==}+ engines: {node: '>=4'}+resolution: {integrity: sha512-qYg9KP24dD5qka9J47d0aVky0N+b4fTU89LN9iDnjB5waksiC49rvMB0PrUJQGoTmH50XPiqOvAjDfaijGxYZw==}engines: {node: '>=8'}@@ -6345,6 +6428,10 @@ packages:resolution: {integrity: sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==}engines: {node: '>= 4'}+ resolution: {integrity: sha512-g6QUff04oZpHs0eG5p83rFLhHeV00ug/Yf9nZM6fLeUrPguBTkTQOdpAWWspMh55TZfVQDPaN3NQJfbVRAxdIw==}+ engines: {iojs: '>=1.0.0', node: '>=0.10.0'}+resolution: {integrity: sha512-q1b3N5QkRUWUl7iyylaaj3kOpIT0N2i9MqIEQXP73GVsN9cw3fdx8X63cEmWhJGi2PPCF23Ijp7ktmd39rawIA==}@@ -6358,6 +6445,9 @@ packages:resolution: {integrity: sha512-guoltQEx+9aMf2gDZ0s62EcV8lsXR+0w8915TC3ITdn2YueuNjdAYh/levpU9nFaoChh9RUS5ZdQMrKfVEN9tw==}+ resolution: {integrity: sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA==}+resolution: {integrity: sha512-cLgakCUf6PedEu15t8kbsjnwIFFR2D4RfL+W3iWFJ4iac7z4B0ZI8fxy4R3J956kAI68HclCFGL8MPoUVC3qVA==}@@ -6635,7 +6725,7 @@ packages:resolution: {integrity: sha512-6WTxW1EB6yCxV5VFOIPQruWGHqc3yI7hEmZK6h+pyk69Lk/Ut7rLUY6W/ONF2MjBuGjvmMiIpsrVJ2vjrHlslA==}engines: {node: '>=6.4.0 <13 || >=14'}- deprecated: Please upgrade to v9.0.0+ as we have fixed a public vulnerability with formidable dependency. Note that v9.0.0+ requires Node.js v14.18.0+. See https://github.com/ladjs/superagent/pull/1800 for insight. This project is supported and maintained by the team at Forward Email @ https://forwardemail.net+ deprecated: Please upgrade to superagent v10.2.2+, see release notes at https://github.com/forwardemail/superagent/releases/tag/v10.2.2 - maintenance is supported by Forward Email @ https://forwardemail.netresolution: {integrity: sha512-7JpaAoX2NGyoFlI9NBh66BQXGONc+uE+MRS5i2iOBKuS4e+ccgMDjATgZldkah+33DakBxDHiss9kvUcGAO8UQ==}@@ -6648,6 +6738,7 @@ packages:resolution: {integrity: sha512-erY3HFDG0dPnhw4U+udPfrzXa4xhSG+n4rxfRuZWCUvjFWwKl+OxWf/7zk50s84/fAAs7vf5QAb9uRa0cCykxw==}engines: {node: '>=6.4.0'}+ deprecated: Please upgrade to supertest v7.1.3+, see release notes at https://github.com/forwardemail/supertest/releases/tag/v7.1.3 - maintenance is supported by Forward Email @ https://forwardemail.netresolution: {integrity: sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==}@@ -6752,6 +6843,10 @@ packages:resolution: {integrity: sha512-aZbgViZrg1QNcG+LULa7nhZpJTZSLm/mXnHXnbAbjmN5aSa0y7V+wvv6+4WaBtpISJzThKy+PIPxc1Nq1EJ9mg==}engines: {node: '>= 14.0.0'}+ resolution: {integrity: sha512-rqy30BSpxPznbbTcAcci90oZ1YR4DqvKcNXNerG5gQBU2v4jk0cygheiul5J6ExIMrgDVuanv/MkGfqZbKrNNg==}+ engines: {node: 10.* || >= 12.*}+resolution: {integrity: sha512-YiHwDhSvCiItoAgsKtoLFCuakDzDsJ1DLDnSouTaTmdOcOwIkSzbLXduaQ6M5DRVhuZC/NYaaZ/mtHbWMv/S6Q==}engines: {node: ^14.15.0 || ^16.10.0 || ^18.0.0 || >=20.0.0}@@ -6776,6 +6871,9 @@ packages:esbuild:optional: true+ resolution: {integrity: sha512-pSOfUMx8Ld/WUreoSzvMFQG5i9uEiWIsBYjpU9+TTASOeUa89j5HykomeqVULm1oqWtBdleI3KEFRLrlA3zGIw==}+resolution: {integrity: sha512-f0FFpIdcHgn8zcPSbf1dRevwt047YMnaiJM3u2w2RewrB+fob/zePZcrOyQoLMMO7aBIddLcQIEK5dYjkLnGrQ==}hasBin: true@@ -6790,6 +6888,10 @@ packages:'@swc/wasm':optional: true+ resolution: {integrity: sha512-iS47YTbdIcvN8Nh/1BFyziyUqmjXz7GVzWu02RaZXqb+e/3Qe1B7IQ4860krOeCGUeJmterAlaM2FRH0Ue0hjw==}+ hasBin: true+resolution: {integrity: sha512-r6t37Dkk4vK44HwxOe+OzjpE/gDamZAwqXhtcAJD/hPVblcjJK45NxbK0HcDASXG0U4pEnCh640JZbeDVSC6yA==}engines: {node: '>=12.12.0'}@@ -7261,6 +7363,10 @@ packages:resolution: {integrity: sha512-YgvUTfwqyc7UXVMrB+SImsVYSmTS8X/tSrtdNZMImM+n7+QTriRXyXim0mBrTXNeqzVF0KWGgHPeiyViFFrNDw==}engines: {node: '>=18'}+ resolution: {integrity: sha512-r3vXyErRCYJ7wg28yvBY5VSoAF8ZvlcW9/BwUzEtUsjvX/DKs24dIkuwjtuprwJJHsbyUbLApepYTR1BN4uHrg==}+ engines: {node: '>= 6'}+resolution: {integrity: sha512-lcYcMxX2PO9XMGvAJkJ3OsNMw+/7FKes7/hgerGUYWIoWu5j/+YQqcZr5JnPZWzOsEBgMbSbiSTn/dv/69Mkpw==}engines: {node: '>= 14.6'}@@ -9360,6 +9466,18 @@ snapshots:'@noble/[email protected]': {}+ '@nodelib/[email protected]':+ dependencies:+ '@nodelib/fs.stat': 2.0.5+ run-parallel: 1.2.0++ '@nodelib/[email protected]': {}++ '@nodelib/[email protected]':+ dependencies:+ '@nodelib/fs.scandir': 2.1.5+ fastq: 1.19.1+'@octokit/[email protected]': {}'@octokit/[email protected]':@@ -11477,6 +11595,13 @@ snapshots:'@tootallnate/[email protected]': {}+ '@ts-morph/[email protected]':+ dependencies:+ fast-glob: 3.3.3+ minimatch: 3.1.2+ mkdirp: 1.0.4+ path-browserify: 1.0.1+'@tsconfig/[email protected]': {}'@tsconfig/[email protected]': {}@@ -11625,6 +11750,8 @@ snapshots:dependencies:'@types/node': 20.14.1+ '@types/[email protected]': {}+'@types/[email protected]': {}'@types/[email protected]':@@ -12768,6 +12895,8 @@ snapshots:+ [email protected]: {}+dependencies:'@aws-sdk/client-sagemaker': 3.679.0@@ -12832,6 +12961,8 @@ snapshots:+ [email protected]: {}+@@ -12864,6 +12995,14 @@ snapshots:object-assign: 4.1.1vary: 1.1.2+ dependencies:+ '@types/parse-json': 4.0.2+ import-fresh: 3.3.1+ parse-json: 5.2.0+ path-type: 4.0.0+ yaml: 1.10.2+@@ -13350,6 +13489,14 @@ snapshots:+ dependencies:+ '@nodelib/fs.stat': 2.0.5+ '@nodelib/fs.walk': 1.2.8+ glob-parent: 5.1.2+ merge2: 1.4.1+ micromatch: 4.0.8+@@ -13364,6 +13511,10 @@ snapshots:+ dependencies:+ reusify: 1.1.0+dependencies:bser: 2.1.1@@ -13556,6 +13707,10 @@ snapshots:shelljs: 0.8.5shelljs.exec: 1.1.8+ dependencies:+ is-glob: 4.0.3+dependencies:foreground-child: 3.2.1@@ -13735,6 +13890,11 @@ snapshots:+ dependencies:+ parent-module: 1.0.1+ resolve-from: 4.0.0+dependencies:acorn: 8.15.0@@ -13807,6 +13967,8 @@ snapshots:dependencies:hasown: 2.0.2+ [email protected]: {}+@@ -13824,6 +13986,10 @@ snapshots:has-tostringtag: 1.0.2safe-regex-test: 1.1.0+ dependencies:+ is-extglob: 2.1.1+@@ -14587,6 +14753,8 @@ snapshots:+ [email protected]: {}+@@ -14640,6 +14808,8 @@ snapshots:optionalDependencies:typescript: 5.8.3+ [email protected]: {}+@@ -14920,6 +15090,10 @@ snapshots:+ dependencies:+ callsites: 3.1.0+@@ -14951,6 +15125,8 @@ snapshots:+ [email protected]: {}+@@ -14968,6 +15144,8 @@ snapshots:+ [email protected]: {}+dependencies:through: 2.3.8@@ -15204,6 +15382,8 @@ snapshots:+ [email protected]: {}+@@ -15312,6 +15492,8 @@ snapshots:dependencies:resolve-from: 5.0.0+ [email protected]: {}+@@ -15340,6 +15522,8 @@ snapshots:+ [email protected]: {}+@@ -15359,6 +15543,10 @@ snapshots:+ dependencies:+ queue-microtask: 1.2.3+@@ -15783,6 +15971,8 @@ snapshots:+ [email protected]: {}+[email protected](@babel/[email protected])(@jest/[email protected])(@jest/[email protected])([email protected](@babel/[email protected]))([email protected](@types/[email protected])([email protected](@types/[email protected])([email protected])))([email protected]):dependencies:bs-logger: 0.2.6@@ -15801,6 +15991,11 @@ snapshots:'@jest/types': 29.6.3babel-jest: 29.7.0(@babel/[email protected])+ dependencies:+ '@ts-morph/common': 0.12.3+ code-block-writer: 11.0.3+dependencies:'@cspotcode/source-map-support': 0.8.1@@ -15819,6 +16014,15 @@ snapshots:v8-compile-cache-lib: 3.0.1yn: 3.1.1+ dependencies:+ commander: 6.2.1+ cosmiconfig: 7.1.0+ json5: 2.2.3+ lodash: 4.17.21+ true-myth: 4.1.1+ ts-morph: 13.0.3+dependencies:cross-spawn: 7.0.6@@ -16328,6 +16532,8 @@ snapshots:+ [email protected]: {}+diff --git a/apps/api/src/__tests__/snips/v1/lib.ts b/apps/api/src/__tests__/snips/v1/lib.tsindex 4a8d9906e..0d770234d 100644--- a/apps/api/src/__tests__/snips/v1/lib.ts+++ b/apps/api/src/__tests__/snips/v1/lib.ts@@ -25,14 +25,7 @@ import {} from "../lib";// Re-export shared utilities for backwards compatibility-export {- TEST_URL,- scrapeTimeout,- indexCooldown,- Identity,- IdmuxRequest,- idmux,-};+export { scrapeTimeout, indexCooldown, Identity, idmux };export async function scrapeRaw(body: ScrapeRequestInput, identity: Identity) {return await request(TEST_URL)@@ -207,7 +200,7 @@ export async function asyncCrawlWaitForFinish(return x.body;}-export async function crawlErrors(+async function crawlErrors(id: string,identity: Identity,): Promise<Exclude<CrawlErrorsResponse, ErrorResponse>> {@@ -473,7 +466,7 @@ export async function tokenUsage(// Concurrency API// =========================================-export async function concurrencyCheck(+async function concurrencyCheck(identity: Identity,): Promise<{ concurrency: number; maxConcurrency: number }> {const x = await request(TEST_URL)diff --git a/apps/api/src/__tests__/snips/v2/lib.ts b/apps/api/src/__tests__/snips/v2/lib.tsindex ef4103e9a..6ab7c96f1 100644--- a/apps/api/src/__tests__/snips/v2/lib.ts+++ b/apps/api/src/__tests__/snips/v2/lib.ts@@ -27,14 +27,7 @@ import {import { SearchV2Response } from "../../../lib/entities";// Re-export shared utilities for backwards compatibility-export {- TEST_URL,- scrapeTimeout,- indexCooldown,- Identity,- IdmuxRequest,- idmux,-};+export { scrapeTimeout, indexCooldown, Identity, idmux };// =========================================// Scrape API@@ -208,7 +201,7 @@ export async function asyncCrawlWaitForFinish(return x.body;}-export async function crawlErrors(+async function crawlErrors(id: string,identity: Identity,): Promise<Exclude<CrawlErrorsResponse, ErrorResponse>> {@@ -409,7 +402,7 @@ export async function creditUsage(// Concurrency API// =========================================-export async function concurrencyCheck(+async function concurrencyCheck(identity: Identity,): Promise<{ concurrency: number; maxConcurrency: number }> {const x = await request(TEST_URL)diff --git a/apps/api/src/control.ts b/apps/api/src/control.tsdeleted file mode 100644index ba42542ea..000000000--- a/apps/api/src/control.ts+++ /dev/null@@ -1,2 +0,0 @@-// ! IN CASE OPENAI goes down, then activate the fallback -> true-export const is_fallback = false;diff --git a/apps/api/src/controllers/auth.ts b/apps/api/src/controllers/auth.tsindex ad2fd5256..c5c116fd6 100644--- a/apps/api/src/controllers/auth.ts+++ b/apps/api/src/controllers/auth.ts@@ -398,7 +398,7 @@ export async function authenticateUser(})(req, res, mode);}-export async function supaAuthenticateUser(+async function supaAuthenticateUser(req,res,mode?: RateLimiterMode,diff --git a/apps/api/src/controllers/v0/crawl-status.ts b/apps/api/src/controllers/v0/crawl-status.tsindex 446e20f0c..5a17c08c8 100644--- a/apps/api/src/controllers/v0/crawl-status.ts+++ b/apps/api/src/controllers/v0/crawl-status.ts@@ -13,7 +13,7 @@ import { getJobFromGCS } from "../../lib/gcs-jobs";import { scrapeQueue, NuQJob } from "../../services/worker/nuq";configDotenv();-export async function getJobs(+async function getJobs(crawlId: string,ids: string[],): Promise<PseudoJob<any>[]> {diff --git a/apps/api/src/controllers/v0/search.ts b/apps/api/src/controllers/v0/search.tsindex 065c54c54..820bef824 100644--- a/apps/api/src/controllers/v0/search.ts+++ b/apps/api/src/controllers/v0/search.ts@@ -24,7 +24,7 @@ import { fromV0Combo } from "../v2/types";import { ScrapeJobTimeoutError } from "../../lib/error";import { scrapeQueue } from "../../services/worker/nuq";-export async function searchHelper(+async function searchHelper(jobId: string,req: Request,team_id: string,diff --git a/apps/api/src/controllers/v1/deep-research.ts b/apps/api/src/controllers/v1/deep-research.tsindex 947e1aa22..682b95de3 100644--- a/apps/api/src/controllers/v1/deep-research.ts+++ b/apps/api/src/controllers/v1/deep-research.ts@@ -5,7 +5,7 @@ import * as Sentry from "@sentry/node";import { saveDeepResearch } from "../../lib/deep-research/deep-research-redis";import { z } from "zod";-export const deepResearchRequestSchema = z+const deepResearchRequestSchema = z.object({query: z.string().describe("The query or topic to search for").optional(),maxDepth: z@@ -60,9 +60,9 @@ export const deepResearchRequestSchema = zquery: data.topic || data.query, // Use topic as query if provided}));-export type DeepResearchRequest = z.infer<typeof deepResearchRequestSchema>;+type DeepResearchRequest = z.infer<typeof deepResearchRequestSchema>;-export type DeepResearchResponse =+type DeepResearchResponse =| ErrorResponse| {success: boolean;diff --git a/apps/api/src/controllers/v1/extract-status.ts b/apps/api/src/controllers/v1/extract-status.tsindex cea0cc3bf..32fad784a 100644--- a/apps/api/src/controllers/v1/extract-status.ts+++ b/apps/api/src/controllers/v1/extract-status.ts@@ -19,7 +19,7 @@ type ExtractPseudoJob<T> = {failedReason?: string;};-export async function getExtractJob(+async function getExtractJob(id: string,): Promise<ExtractPseudoJob<ExtractResult> | null> {const [bullJob, dbJob] = await Promise.all([diff --git a/apps/api/src/controllers/v1/extract.ts b/apps/api/src/controllers/v1/extract.tsindex 2e9e39972..0400763a8 100644--- a/apps/api/src/controllers/v1/extract.ts+++ b/apps/api/src/controllers/v1/extract.ts@@ -19,7 +19,7 @@ import { logger as _logger } from "../../lib/logger";import { fromV1ScrapeOptions } from "../v2/types";import { createWebhookSender, WebhookEvent } from "../../services/webhook";-export async function oldExtract(+async function oldExtract(req: RequestWithAuth<{}, ExtractResponse, ExtractRequest>,res: Response<ExtractResponse>,extractId: string,diff --git a/apps/api/src/controllers/v1/generate-llmstxt.ts b/apps/api/src/controllers/v1/generate-llmstxt.tsindex ccc2145d7..2e6464e73 100644--- a/apps/api/src/controllers/v1/generate-llmstxt.ts+++ b/apps/api/src/controllers/v1/generate-llmstxt.ts@@ -9,7 +9,7 @@ import { getGenerateLlmsTxtQueue } from "../../services/queue-service";import * as Sentry from "@sentry/node";import { saveGeneratedLlmsTxt } from "../../lib/generate-llmstxt/generate-llmstxt-redis";-export type GenerateLLMsTextResponse =+type GenerateLLMsTextResponse =| ErrorResponse| {success: boolean;diff --git a/apps/api/src/controllers/v1/liveness.ts b/apps/api/src/controllers/v1/liveness.tsdeleted file mode 100644index 8ff1a96f9..000000000--- a/apps/api/src/controllers/v1/liveness.ts+++ /dev/null@@ -1,6 +0,0 @@-import { Request, Response } from "express";--export async function livenessController(req: Request, res: Response) {- //TODO: add checks if the application is live and healthy like checking the redis connection- res.status(200).json({ status: "ok" });-}diff --git a/apps/api/src/controllers/v1/queue-status.ts b/apps/api/src/controllers/v1/queue-status.tsindex 64d30be1f..63e0157fc 100644--- a/apps/api/src/controllers/v1/queue-status.ts+++ b/apps/api/src/controllers/v1/queue-status.ts@@ -10,7 +10,7 @@ import {getConcurrencyQueueJobsCount,} from "../../lib/concurrency-limit";-export type QueueStatusResponse = {+type QueueStatusResponse = {success: boolean;jobsInQueue: number;activeJobsInQueue: number;diff --git a/apps/api/src/controllers/v1/readiness.ts b/apps/api/src/controllers/v1/readiness.tsdeleted file mode 100644index cdb1f02c3..000000000--- a/apps/api/src/controllers/v1/readiness.ts+++ /dev/null@@ -1,6 +0,0 @@-import { Request, Response } from "express";--export async function readinessController(req: Request, res: Response) {- // TODO: add checks when the application is ready to serve traffic- res.status(200).json({ status: "ok" });-}diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.tsindex 0c3cac160..2a1e5c7c7 100644--- a/apps/api/src/controllers/v1/types.ts+++ b/apps/api/src/controllers/v1/types.ts@@ -15,7 +15,7 @@ import { ErrorCodes } from "../../lib/error";import { integrationSchema } from "../../utils/integration";import { webhookSchema } from "../../services/webhook/schema";-export type Format =+type Format =| "markdown"| "html"| "rawHtml"@@ -70,7 +70,7 @@ export const url = z.preprocess(const strictMessage ="Unrecognized key in body -- please review the v1 API documentation for request body changes";-export const agentExtractModelValue = "fire-1";+const agentExtractModelValue = "fire-1";export const isAgentExtractModelValid = (x: string | undefined) =>x?.toLowerCase() === agentExtractModelValue;@@ -80,8 +80,6 @@ export const agentOptionsExtract = z}).strict(strictMessage);-export type AgentOptions = z.infer<typeof agentOptionsExtract>;-export const extractOptions = z.object({mode: z.enum(["llm"]).default("llm"),@@ -113,7 +111,7 @@ export const extractOptions = z"Based on the information on the page, extract all the information from the schema in JSON format. Try to extract all the fields even those that might not be marked as required.",}));-export const extractOptionsWithAgent = z+const extractOptionsWithAgent = z.object({mode: z.enum(["llm"]).default("llm"),schema: z@@ -188,7 +186,7 @@ function calculateTotalWaitTime(return waitFor + actionWaitTime;}-export const actionSchema = z.union([+const actionSchema = z.union([z.object({type: z.literal("wait"),@@ -258,7 +256,7 @@ export const actionSchema = z.union([export type Action = z.infer<typeof actionSchema>;-export const actionsSchema = z+const actionsSchema = z.array(actionSchema).refine(actions => actions.length <= MAX_ACTIONS, {message: `Number of actions cannot exceed ${MAX_ACTIONS}`,@@ -272,7 +270,7 @@ export const actionsSchema = zconst SPECIAL_COUNTRIES = ["us-generic", "us-whitelist"];-export const locationSchema = z+const locationSchema = z.object({country: z.string()@@ -536,7 +534,7 @@ export const scrapeOptions = baseScrapeOptionsreturn extractTransform(obj) as typeof obj;});-export type BaseScrapeOptions = z.infer<typeof baseScrapeOptions>;+type BaseScrapeOptions = z.infer<typeof baseScrapeOptions>;export type ScrapeOptions = BaseScrapeOptions & {extract?: z.infer<typeof extractOptionsWithAgent>;@@ -551,7 +549,7 @@ export type ScrapeOptions = BaseScrapeOptions & {const ajv = new Ajv();-export const extractV1Options = z+const extractV1Options = z.object({urls: url.array()@@ -629,7 +627,6 @@ export const extractV1Options = z: x.scrapeOptions,}));-export type ExtractV1Options = z.infer<typeof extractV1Options>;export const extractRequestSchema = extractV1Options;export type ExtractRequest = z.infer<typeof extractRequestSchema>;export type ExtractRequestInput = z.input<typeof extractRequestSchema>;@@ -732,7 +729,7 @@ const crawlerOptions = z// ignoreSitemap?: boolean;// };-export type CrawlerOptions = z.infer<typeof crawlerOptions>;+type CrawlerOptions = z.infer<typeof crawlerOptions>;export const crawlRequestSchema = crawlerOptions.extend({@@ -935,12 +932,6 @@ export type ScrapeResponse =scrape_id?: string;};-export interface ScrapeResponseRequestTest {- statusCode: number;- body: ScrapeResponse;- error?: string;-}-export interface URLTrace {url: string;status: "mapped" | "scraped" | "error";@@ -975,12 +966,6 @@ export interface ExtractResponse {tokensUsed?: number;}-export interface ExtractResponseRequestTest {- statusCode: number;- body: ExtractResponse;- error?: string;-}-export type CrawlResponse =| ErrorResponse| {@@ -1207,7 +1192,7 @@ export function toNewCrawlerOptions(x: any): CrawlerOptions {};}-export function fromLegacyCrawlerOptions(+function fromLegacyCrawlerOptions(x: any,teamId: string,): {@@ -1305,26 +1290,6 @@ export function fromLegacyScrapeOptions(};}-export function fromLegacyCombo(- pageOptions: PageOptions,- extractorOptions: ExtractorOptions | undefined,- timeout: number | undefined,- crawlerOptions: any,- teamId: string,-): { scrapeOptions: ScrapeOptions; internalOptions: InternalOptions } {- const { scrapeOptions, internalOptions: i1 } = fromLegacyScrapeOptions(- pageOptions,- extractorOptions,- timeout,- teamId,- );- const { internalOptions: i2 } = fromLegacyCrawlerOptions(- crawlerOptions,- teamId,- );- return { scrapeOptions, internalOptions: Object.assign(i1, i2) };-}-export function toLegacyDocument(document: Document,internalOptions: InternalOptions,diff --git a/apps/api/src/controllers/v2/extract-status.ts b/apps/api/src/controllers/v2/extract-status.tsindex bec3b62bc..e3ef33d9a 100644--- a/apps/api/src/controllers/v2/extract-status.ts+++ b/apps/api/src/controllers/v2/extract-status.ts@@ -19,7 +19,7 @@ type ExtractPseudoJob<T> = {failedReason?: string;};-export async function getExtractJob(+async function getExtractJob(id: string,): Promise<ExtractPseudoJob<ExtractResult> | null> {const [bullJob, dbJob] = await Promise.all([diff --git a/apps/api/src/controllers/v2/map.ts b/apps/api/src/controllers/v2/map.tsindex 849250c5c..aaf966b1a 100644--- a/apps/api/src/controllers/v2/map.ts+++ b/apps/api/src/controllers/v2/map.ts@@ -86,7 +86,7 @@ async function queryIndex(}}-export async function getMapResults({+async function getMapResults({url,search,limit = MAX_MAP_LIMIT,diff --git a/apps/api/src/controllers/v2/queue-status.ts b/apps/api/src/controllers/v2/queue-status.tsindex bf03dbc29..e52542dce 100644--- a/apps/api/src/controllers/v2/queue-status.ts+++ b/apps/api/src/controllers/v2/queue-status.ts@@ -11,7 +11,7 @@ import {getConcurrencyQueueJobsCount,} from "../../lib/concurrency-limit";-export type QueueStatusResponse = {+type QueueStatusResponse = {success: boolean;jobsInQueue: number;activeJobsInQueue: number;diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.tsindex 12fc05b8f..dc67932cf 100644--- a/apps/api/src/controllers/v2/types.ts+++ b/apps/api/src/controllers/v2/types.ts@@ -20,19 +20,6 @@ import Ajv from "ajv";import { integrationSchema } from "../../utils/integration";import { webhookSchema } from "../../services/webhook/schema";-export type Format =- | "markdown"- | "html"- | "rawHtml"- | "links"- | "images"- | "screenshot"- | "screenshot@fullPage"- | "extract"- | "json"- | "summary"- | "changeTracking";-export const url = z.preprocess(x => {if (!protocolIncluded(x as string)) {@@ -98,7 +85,7 @@ function calculateTotalWaitTime(return waitFor + actionWaitTime;}-export const actionSchema = z.union([+const actionSchema = z.union([z.object({type: z.literal("wait"),@@ -172,9 +159,7 @@ export const actionSchema = z.union([}),]);-export type Action = z.infer<typeof actionSchema>;--export const actionsSchema = z+const actionsSchema = z.array(actionSchema).refine(actions => actions.length <= MAX_ACTIONS, {message: `Number of actions cannot exceed ${MAX_ACTIONS}`,@@ -186,7 +171,7 @@ export const actionsSchema = z},);-export const jsonFormatWithOptions = z+const jsonFormatWithOptions = z.object({type: z.literal("json"),schema: z.any().optional(),@@ -196,7 +181,7 @@ export const jsonFormatWithOptions = zexport type JsonFormatWithOptions = z.output<typeof jsonFormatWithOptions>;-export const changeTrackingFormatWithOptions = z+const changeTrackingFormatWithOptions = z.object({type: z.literal("changeTracking"),prompt: z.string().optional(),@@ -206,11 +191,11 @@ export const changeTrackingFormatWithOptions = z}).strict();-export type ChangeTrackingFormatWithOptions = z.output<+type ChangeTrackingFormatWithOptions = z.output<typeof changeTrackingFormatWithOptions>;-export const screenshotFormatWithOptions = z.object({+const screenshotFormatWithOptions = z.object({type: z.literal("screenshot"),fullPage: z.boolean().default(false),quality: z.number().min(1).max(100).optional(),@@ -222,11 +207,9 @@ export const screenshotFormatWithOptions = z.object({.optional(),});-export type ScreenshotFormatWithOptions = z.output<- typeof screenshotFormatWithOptions->;+type ScreenshotFormatWithOptions = z.output<typeof screenshotFormatWithOptions>;-export const attributesFormatWithOptions = z+const attributesFormatWithOptions = z.object({type: z.literal("attributes"),selectors: z@@ -244,9 +227,7 @@ export const attributesFormatWithOptions = z}).strict();-export type AttributesFormatWithOptions = z.output<- typeof attributesFormatWithOptions->;+type AttributesFormatWithOptions = z.output<typeof attributesFormatWithOptions>;export type FormatObject =| { type: "markdown" }@@ -260,18 +241,18 @@ export type FormatObject =| ScreenshotFormatWithOptions| AttributesFormatWithOptions;-export const pdfParserWithOptions = z+const pdfParserWithOptions = z.object({type: z.literal("pdf"),maxPages: z.number().int().positive().finite().max(10000).optional(),}).strict();-export const parsersSchema = z+const parsersSchema = z.array(z.union([z.literal("pdf"), pdfParserWithOptions])).default(["pdf"]);-export type Parsers = z.infer<typeof parsersSchema>;+type Parsers = z.infer<typeof parsersSchema>;export function shouldParsePDF(parsers?: Parsers): boolean {if (!parsers) return true;@@ -307,7 +288,7 @@ function transformIframeSelector(selector: string): string {const SPECIAL_COUNTRIES = ["us-generic", "us-whitelist"];-export const locationSchema = z+const locationSchema = z.object({country: z.string()@@ -473,7 +454,7 @@ export type ScrapeOptions = BaseScrapeOptions;const ajv = new Ajv();-export const extractOptions = z+const extractOptions = z.object({urls: url.array()@@ -543,7 +524,6 @@ export const extractOptions = z: x.scrapeOptions,}));-export type ExtractOptions = z.infer<typeof extractOptions>;export const extractRequestSchema = extractOptions;export type ExtractRequest = z.infer<typeof extractRequestSchema>;export type ExtractRequestInput = z.input<typeof extractRequestSchema>;@@ -623,7 +603,7 @@ const crawlerOptions = z// ignoreSitemap?: boolean;// };-export type CrawlerOptions = z.infer<typeof crawlerOptions>;+type CrawlerOptions = z.infer<typeof crawlerOptions>;export const crawlRequestSchema = crawlerOptions.extend({@@ -811,12 +791,6 @@ export type ScrapeResponse =scrape_id?: string;};-export interface ScrapeResponseRequestTest {- statusCode: number;- body: ScrapeResponse;- error?: string;-}-export interface URLTrace {url: string;status: "mapped" | "scraped" | "error";@@ -851,12 +825,6 @@ export interface ExtractResponse {tokensUsed?: number;}-export interface ExtractResponseRequestTest {- statusCode: number;- body: ExtractResponse;- error?: string;-}-export type CrawlResponse =| ErrorResponse| {@@ -964,7 +932,7 @@ export type TeamFlags = {ipWhitelist?: boolean;} | null;-export interface RequestWithMaybeACUC<+interface RequestWithMaybeACUC<ReqParams = {},ReqBody = undefined,ResBody = undefined,@@ -972,15 +940,6 @@ export interface RequestWithMaybeACUC<acuc?: AuthCreditUsageChunk;}-export interface RequestWithMaybeAuth<- ReqParams = {},- ReqBody = undefined,- ResBody = undefined,-> extends RequestWithMaybeACUC<ReqParams, ReqBody, ResBody> {- auth?: AuthObject;- account?: Account;-}-export interface RequestWithAuth<ReqParams = {},ReqBody = undefined,@@ -990,11 +949,6 @@ export interface RequestWithAuth<account?: Account;}-export interface ResponseWithSentry<ResBody = undefined>- extends Response<ResBody> {- sentry?: string;-}-export function toV0CrawlerOptions(x: CrawlerOptions) {return {includes: x.includePaths,@@ -1035,7 +989,7 @@ export function toV2CrawlerOptions(x: any): CrawlerOptions {};}-export function fromV0CrawlerOptions(+function fromV0CrawlerOptions(x: any,teamId: string,): {@@ -1257,39 +1211,11 @@ export function fromV0Combo(return { scrapeOptions, internalOptions: Object.assign(i1, i2) };}-export function toLegacyDocument(- document: Document,- internalOptions: InternalOptions,-): V0Document | { url: string } {- if (internalOptions.v0CrawlOnlyUrls) {- return { url: document.metadata.sourceURL! };- }-- return {- content: document.markdown!,- markdown: document.markdown!,- html: document.html,- rawHtml: document.rawHtml,- linksOnPage: document.links,- llm_extraction: document.extract,- metadata: {- ...document.metadata,- error: undefined,- statusCode: undefined,- pageError: document.metadata.error,- pageStatusCode: document.metadata.statusCode,- screenshot: document.screenshot,- },- actions: document.actions,- warning: document.warning,- };-}-// Search source type definitions// These allow fine-grained control over each search source type// Similar to how scrape formats work with jsonFormatWithOptions, etc.-export const webSearchSourceOptions = z+const webSearchSourceOptions = z.object({type: z.literal("web"),tbs: z.string().optional(), // Time-based search (e.g., "qdr:d" for past day)@@ -1300,40 +1226,31 @@ export const webSearchSourceOptions = z}).strict();-export const imagesSearchSourceOptions = z+const imagesSearchSourceOptions = z.object({type: z.literal("images"),}).strict();-export const newsSearchSourceOptions = z+const newsSearchSourceOptions = z.object({type: z.literal("news"),}).strict();-export type WebSearchSourceOptions = z.infer<typeof webSearchSourceOptions>;-export type ImagesSearchSourceOptions = z.infer<- typeof imagesSearchSourceOptions->;-export type NewsSearchSourceOptions = z.infer<typeof newsSearchSourceOptions>;-// Category source type definitions-export const githubCategoryOptions = z+const githubCategoryOptions = z.object({type: z.literal("github"),}).strict();-export const researchCategoryOptions = z+const researchCategoryOptions = z.object({type: z.literal("research"),}).strict();-export type GithubCategoryOptions = z.infer<typeof githubCategoryOptions>;-export type ResearchCategoryOptions = z.infer<typeof researchCategoryOptions>;-export const searchRequestSchema = z.object({query: z.string(),@@ -1526,7 +1443,7 @@ export type TokenUsage = {model?: string;};-export const generateLLMsTextRequestSchema = z.object({+const generateLLMsTextRequestSchema = z.object({url: url.describe("The URL to generate text from"),maxUrls: z.number()@@ -1545,6 +1462,4 @@ export const generateLLMsTextRequestSchema = z.object({__experimental_stream: z.boolean().optional(),});-export type GenerateLLMsTextRequest = z.infer<- typeof generateLLMsTextRequestSchema->;+type GenerateLLMsTextRequest = z.infer<typeof generateLLMsTextRequestSchema>;diff --git a/apps/api/src/lib/LLM-extraction/helpers.ts b/apps/api/src/lib/LLM-extraction/helpers.tsdeleted file mode 100644index 2143a32db..000000000--- a/apps/api/src/lib/LLM-extraction/helpers.ts+++ /dev/null@@ -1,22 +0,0 @@-import { encoding_for_model } from "@dqbd/tiktoken";-import { TiktokenModel } from "@dqbd/tiktoken";--// This function calculates the number of tokens in a text string using GPT-3.5-turbo model-export function numTokensFromString(message: string, model: string): number {- const encoder = encoding_for_model(model as TiktokenModel);-- // Encode the message into tokens- let tokens: Uint32Array;- try {- tokens = encoder.encode(message);- } catch (error) {- message = message.replace("<|endoftext|>", "");- tokens = encoder.encode(message);- }-- // Free the encoder resources after use- encoder.free();-- // Return the number of tokens- return tokens.length;-}diff --git a/apps/api/src/lib/batch-process.ts b/apps/api/src/lib/batch-process.tsdeleted file mode 100644index 1e4ac7be2..000000000--- a/apps/api/src/lib/batch-process.ts+++ /dev/null@@ -1,15 +0,0 @@-export async function batchProcess<T>(- array: T[],- batchSize: number,- asyncFunction: (item: T, index: number) => Promise<void>,-): Promise<void> {- const batches: T[][] = [];- for (let i = 0; i < array.length; i += batchSize) {- const batch = array.slice(i, i + batchSize);- batches.push(batch);- }-- for (const batch of batches) {- await Promise.all(batch.map((item, i) => asyncFunction(item, i)));- }-}diff --git a/apps/api/src/lib/concurrency-limit.ts b/apps/api/src/lib/concurrency-limit.tsindex e56721769..e8bbb122c 100644--- a/apps/api/src/lib/concurrency-limit.ts+++ b/apps/api/src/lib/concurrency-limit.ts@@ -54,14 +54,11 @@ export async function pushConcurrencyLimitActiveJob(await redisEvictConnection.zadd(constructKey(team_id), now + timeout, id);}-export async function removeConcurrencyLimitActiveJob(- team_id: string,- id: string,-) {+async function removeConcurrencyLimitActiveJob(team_id: string, id: string) {await redisEvictConnection.zrem(constructKey(team_id), id);}-export type ConcurrencyLimitedJob = {+type ConcurrencyLimitedJob = {id: string;data: any;priority: number;@@ -78,22 +75,6 @@ export async function cleanOldConcurrencyLimitedJobs();}-export async function takeConcurrencyLimitedJob(- team_id: string,-): Promise<ConcurrencyLimitedJob | null> {- await cleanOldConcurrencyLimitedJobs(team_id);- const res = await redisEvictConnection.zmpop(- 1,- constructQueueKey(team_id),- "MIN",- );- if (res === null || res === undefined) {- return null;- }-- return JSON.parse(res[1][0][0]);-}-export async function pushConcurrencyLimitedJob(team_id: string,job: ConcurrencyLimitedJob,@@ -123,7 +104,7 @@ export async function getConcurrencyQueueJobsCount();}-export async function cleanOldCrawlConcurrencyLimitEntries(+async function cleanOldCrawlConcurrencyLimitEntries(crawl_id: string,now: number = Date.now(),) {@@ -158,7 +139,7 @@ export async function pushCrawlConcurrencyLimitActiveJob();}-export async function removeCrawlConcurrencyLimitActiveJob(+async function removeCrawlConcurrencyLimitActiveJob(crawl_id: string,id: string,) {diff --git a/apps/api/src/lib/crawl-redis.ts b/apps/api/src/lib/crawl-redis.tsindex 0ffc6a97a..78ba0cda2 100644--- a/apps/api/src/lib/crawl-redis.ts+++ b/apps/api/src/lib/crawl-redis.ts@@ -189,7 +189,7 @@ export async function getDoneJobsOrderedUntil();}-export async function isCrawlFinished(id: string) {+async function isCrawlFinished(id: string) {await redisEvictConnection.expire("crawl:" + id + ":kickoff:finish",24 * 60 * 60,@@ -211,10 +211,6 @@ export async function isCrawlKickoffFinished(id: string) {);}-export async function isCrawlFinishedLocked(id: string) {- return await redisEvictConnection.exists("crawl:" + id + ":finish");-}-export async function finishCrawlKickoff(id: string) {await redisEvictConnection.set("crawl:" + id + ":kickoff:finish",@@ -279,10 +275,6 @@ export async function getCrawlJobs(id: string): Promise<string[]> {return await redisEvictConnection.smembers("crawl:" + id + ":jobs");}-export async function getCrawlJobCount(id: string): Promise<number> {- return await redisEvictConnection.scard("crawl:" + id + ":jobs");-}-export async function getCrawlQualifiedJobCount(id: string): Promise<number> {return await redisEvictConnection.scard("crawl:" + id + ":jobs_qualified");}diff --git a/apps/api/src/lib/deep-research/deep-research-redis.ts b/apps/api/src/lib/deep-research/deep-research-redis.tsindex d53bf5e9b..0135a0f04 100644--- a/apps/api/src/lib/deep-research/deep-research-redis.ts+++ b/apps/api/src/lib/deep-research/deep-research-redis.ts@@ -1,15 +1,6 @@import { redisEvictConnection } from "../../services/redis";import { logger as _logger } from "../logger";-export enum DeepResearchStep {- INITIAL = "initial",- SEARCH = "search",- EXTRACT = "extract",- ANALYZE = "analyze",- SYNTHESIS = "synthesis",- COMPLETE = "complete",-}-export type DeepResearchActivity = {type:| "search"diff --git a/apps/api/src/lib/entities.ts b/apps/api/src/lib/entities.tsindex 6a22db560..bd76e17aa 100644--- a/apps/api/src/lib/entities.ts+++ b/apps/api/src/lib/entities.ts@@ -1,16 +1,4 @@-import type { Document as V1Document, Action } from "../controllers/v1/types";--export interface Progress {- current: number;- total: number;- status: string;- metadata?: {- sourceURL?: string;- [key: string]: any;- };- currentDocumentUrl?: string;- currentDocument?: Document;-}+import type { Action } from "../controllers/v1/types";export type PageOptions = {includeMarkdown?: boolean;@@ -61,38 +49,6 @@ export type SearchOptions = {location?: string;};-export type CrawlerOptions = {- returnOnlyUrls?: boolean;- includes?: string | string[];- excludes?: string | string[];- maxCrawledLinks?: number;- maxDepth?: number;- limit?: number;- generateImgAltText?: boolean;- replaceAllPathsWithAbsolutePaths?: boolean;- ignoreSitemap?: boolean;- mode?: "default" | "fast"; // have a mode of some sort- allowBackwardCrawling?: boolean;- allowExternalContentLinks?: boolean;-};--export type WebScraperOptions = {- jobId: string;- urls: string[];- mode: "single_urls" | "sitemap" | "crawl";- crawlerOptions?: CrawlerOptions;- pageOptions?: PageOptions;- extractorOptions?: ExtractorOptions;- concurrentRequests?: number;- bullJobId?: string;- priority?: number;- teamId?: string;-};--export interface DocumentUrl {- url: string;-}-export class Document {id?: string;url?: string; // Used only in /search for now@@ -151,7 +107,7 @@ export class SearchResult {}}-export interface ImageSearchResult {+interface ImageSearchResult {title?: string;imageUrl?: string;imageWidth?: number;@@ -160,7 +116,7 @@ export interface ImageSearchResult {position?: number;}-export interface NewsSearchResult {+interface NewsSearchResult {title?: string;url?: string;snippet?: string;@@ -204,21 +160,3 @@ export interface ScrapeActionContent {url: string;html: string;}--export interface FireEngineResponse {- html: string;- screenshots?: string[];- pageStatusCode?: number;- pageError?: string;- scrapeActionContent?: ScrapeActionContent[];-}--export interface FireEngineOptions {- mobileProxy?: boolean;- method?: string;- engine?: string;- blockMedia?: boolean;- blockAds?: boolean;- disableJsDom?: boolean;- atsv?: boolean; // beta-}diff --git a/apps/api/src/lib/extract/build-prompts.ts b/apps/api/src/lib/extract/build-prompts.tsindex 655b87edb..a6b8d00ce 100644--- a/apps/api/src/lib/extract/build-prompts.ts+++ b/apps/api/src/lib/extract/build-prompts.ts@@ -30,15 +30,6 @@ Website to get content from: ${url}Return only a concise sentece or 2 focused on the essential data points that the user wants to extract. This will be used by an LLM to determine how releavant the links that are present are to the user's request.`;}-export function buildRerankerSystemPrompt(): string {- return `You are a relevance expert scoring links from a website the user is trying to extract information from. Analyze the provided URLs and their content-to determine their relevance to the user's query and intent.- For each URL, assign a relevance score between 0 and 1, where 1- means highly relevant and we should extract the content from it and 0 means not relevant at all, we should not extract the content from it.- Always return all the links scored that you are giving. Do not omit links.- Always return the links in the same order they were provided. If the user wants the content from all the links, all links should be scored 1.`;-}-export function buildRerankerUserPrompt(searchQuery: string): string {return `Given these URLs and their content, analyze their relevance to this extraction request: "${searchQuery}".@@ -89,19 +80,6 @@ export function buildAnalyzeSchemaUserPrompt(Schema: ${schemaString}\nPrompt: ${prompt}\n URLs: ${urls}`;}-// Should Extract--export function buildShouldExtractSystemPrompt(): string {- return `You are a content relevance checker. Your job is to determine if the provided content is very relevant to extract information from based on the user's prompt. Return true only if the content appears relevant and contains information that could help answer the prompt. Return false if the content seems irrelevant or unlikely to contain useful information for the prompt.`;-}--export function buildShouldExtractUserPrompt(- prompt: string,- schema: any,-): string {- return `Should the following content be used to extract information for this prompt: "${prompt}" User schema is: ${JSON.stringify(schema)}\nReturn only true or false.`;-}-// Batch extractexport function buildBatchExtractSystemPrompt(systemPrompt: string,@@ -123,25 +101,3 @@ export function buildRephraseToSerpPrompt(prompt: string): string {Original Prompt: "${prompt}"`;}--export function buildCrawlerOptionsPrompt(prompt: string): string {- return `You are a web crawler configuration expert. Based on the user's natural language instruction, generate appropriate crawler options.--User instruction: "${prompt}"--Generate a JSON object with crawler configuration options. Available options:-- includePaths: string[] - URL patterns to include (regex patterns)-- excludePaths: string[] - URL patterns to exclude (regex patterns)-- maxDepth: number - Maximum crawl depth (default: 10)-- crawlEntireDomain: boolean - Whether to crawl the entire domain-- allowExternalLinks: boolean - Whether to follow external links-- allowSubdomains: boolean - Whether to crawl subdomains-- ignoreRobotsTxt: boolean - Whether to ignore robots.txt-- sitemap: "skip" | "include" - Whether to ignore sitemap-- deduplicateSimilarURLs: boolean - Whether to deduplicate similar URLs-- ignoreQueryParameters: boolean - Whether to ignore query parameters-- regexOnFullURL: boolean - Whether regex patterns apply to full URL-- delay: number - Delay between requests in milliseconds--Return only a valid JSON object with the appropriate options based on the user's intent. If the instruction doesn't clearly map to specific options, use reasonable defaults.`;-}diff --git a/apps/api/src/lib/extract/completions/checkShouldExtract.ts b/apps/api/src/lib/extract/completions/checkShouldExtract.tsdeleted file mode 100644index 7120a7675..000000000--- a/apps/api/src/lib/extract/completions/checkShouldExtract.ts+++ /dev/null@@ -1,62 +0,0 @@-import { logger } from "../../../lib/logger";-import { buildDocument } from "../build-document";-import { Document, TokenUsage } from "../../../controllers/v1/types";-import { generateCompletions } from "../../../scraper/scrapeURL/transformers/llmExtract";-import {- buildShouldExtractSystemPrompt,- buildShouldExtractUserPrompt,-} from "../build-prompts";-import { getModel } from "../../../lib/generic-ai";-import { CostTracking } from "../../cost-tracking";--export async function checkShouldExtract(- prompt: string,- multiEntitySchema: any,- doc: Document,- costTracking: CostTracking,- metadata: {- teamId: string;- functionId?: string;- extractId?: string;- scrapeId?: string;- deepResearchId?: string;- },-): Promise<{ tokenUsage: TokenUsage; extract: boolean }> {- const shouldExtractCheck = await generateCompletions({- logger: logger.child({ method: "extractService/checkShouldExtract" }),- options: {- systemPrompt: buildShouldExtractSystemPrompt(),- prompt: buildShouldExtractUserPrompt(prompt, multiEntitySchema),- schema: {- type: "object",- properties: {- extract: {- type: "boolean",- },- },- required: ["extract"],- },- },- markdown: buildDocument(doc),- isExtractEndpoint: true,- model: getModel("gpt-4o-mini", "openai"),- costTrackingOptions: {- costTracking,- metadata: {- module: "extract",- method: "checkShouldExtract",- },- },- metadata: {- ...metadata,- functionId: metadata.functionId- ? metadata.functionId + "/checkShouldExtract"- : "checkShouldExtract",- },- });-- return {- tokenUsage: shouldExtractCheck.totalUsage,- extract: shouldExtractCheck.extract["extract"],- };-}diff --git a/apps/api/src/lib/extract/extract-redis.ts b/apps/api/src/lib/extract/extract-redis.tsindex be9ac4a6b..a33a061d4 100644--- a/apps/api/src/lib/extract/extract-redis.ts+++ b/apps/api/src/lib/extract/extract-redis.ts@@ -15,7 +15,7 @@ export enum ExtractStep {COMPLETE = "complete",}-export type ExtractedStep = {+type ExtractedStep = {step: ExtractStep;startedAt: number;finishedAt: number | null;@@ -23,7 +23,7 @@ export type ExtractedStep = {discoveredLinks?: string[];};-export type StoredExtract = {+type StoredExtract = {id: string;team_id: string;createdAt: number;diff --git a/apps/api/src/lib/extract/fire-0/build-prompts-f0.ts b/apps/api/src/lib/extract/fire-0/build-prompts-f0.tsindex 0e072a302..9fc3f3252 100644--- a/apps/api/src/lib/extract/fire-0/build-prompts-f0.ts+++ b/apps/api/src/lib/extract/fire-0/build-prompts-f0.ts@@ -1,35 +1,3 @@-export function buildRefrasedPrompt_F0(prompt: string, url: string): string {- return `You are a search query optimizer. Your task is to rephrase the following prompt into an effective search query that will find relevant results about this topic on ${url}.-- Original prompt: "${prompt}"-- Provide a rephrased search query that:- 1. Maintains the core intent of the original prompt with ONLY the keywords- 2. Uses relevant keywords- 3. Is optimized for search engine results- 4. Is concise and focused- 5. Short is better than long- 6. It is a search engine, not a chatbot- 7. Concise-- Return only the rephrased search query, without any explanation or additional text.`;-}--export function buildPreRerankPrompt_F0(- prompt: string | undefined,- schema: any,- url: string,-): string {- const schemaString = JSON.stringify(schema, null, 2);- return `Create a concise search query that combines the key data points from both the schema and prompt. Focus on the core information needed while keeping it general enough to find relevant matches.-- Schema: ${schemaString}- Prompt: ${prompt}- Website to get content from: ${url}-- Return only a concise sentece or 2 focused on the essential data points that the user wants to extract. This will be used by an LLM to determine how releavant the links that are present are to the user's request.`;-}-export function buildRerankerSystemPrompt_F0(): string {return `You are a relevance expert scoring links from a website the user is trying to extract information from. Analyze the provided URLs and their contentto determine their relevance to the user's query and intent.@@ -43,39 +11,6 @@ export function buildRerankerUserPrompt_F0(searchQuery: string): string {return `Given these URLs and their content, identify which ones are relevant to the user's extraction request: "${searchQuery}". Return an array of relevant links with their relevance scores (0-1). Higher scores should be given to URLs that directly address the user's extraction request. Be very mindful with the links you select, as if they are not that relevant it may affect the quality of the extraction. Only include URLs that have a relevancy score of 0.6+.`;}-// Multi entity schema anlayzer-export function buildAnalyzeSchemaPrompt_F0(): string {- return `You are a query classifier for a web scraping system. Classify the data extraction query as either:- A) Single-Answer: One answer across a few pages, possibly containing small arrays.- B) Multi-Entity: Many items across many pages, often involving large arrays.-- Consider:- 1. Answer Cardinality: Single or multiple items?- 2. Page Distribution: Found on 1-3 pages or many?- 3. Verification Needs: Cross-page verification or independent extraction?-- Provide:- - Method: [Single-Answer/Multi-Entity]- - Confidence: [0-100%]- - Reasoning: Why this classification?- - Key Indicators: Specific aspects leading to this decision.-- Examples:- - "Is this company a non-profit?" -> Single-Answer- - "Extract all product prices" -> Multi-Entity-- For Single-Answer, arrays may be present but are typically small. For Multi-Entity, if arrays have multiple items not from a single page, return keys with large arrays. If nested, return the full key (e.g., 'ecommerce.products').`;-}--export function buildAnalyzeSchemaUserPrompt_F0(- schemaString: string,- prompt: string,- urls: string[],-): string {- return `Classify the query as Single-Answer or Multi-Entity. For Multi-Entity, return keys with large arrays; otherwise, return none:- Schema: ${schemaString}\nPrompt: ${prompt}\nRelevant URLs: ${urls}`;-}-// Should Extractexport function buildShouldExtractSystemPrompt_F0(): string {diff --git a/apps/api/src/lib/extract/fire-0/extraction-service-f0.ts b/apps/api/src/lib/extract/fire-0/extraction-service-f0.tsindex c6a2d92be..8de1cb842 100644--- a/apps/api/src/lib/extract/fire-0/extraction-service-f0.ts+++ b/apps/api/src/lib/extract/fire-0/extraction-service-f0.ts@@ -50,7 +50,7 @@ interface ExtractServiceOptions {apiKeyId: number | null;}-export interface ExtractResult {+interface ExtractResult {success: boolean;data?: any;extractId: string;diff --git a/apps/api/src/lib/extract/fire-0/llmExtract-f0.ts b/apps/api/src/lib/extract/fire-0/llmExtract-f0.tsindex 54b0d4a73..ecd761672 100644--- a/apps/api/src/lib/extract/fire-0/llmExtract-f0.ts+++ b/apps/api/src/lib/extract/fire-0/llmExtract-f0.ts@@ -31,7 +31,7 @@ const getModelLimits_F0 = (model: string) => {};};-export class LLMRefusalError extends Error {+class LLMRefusalError extends Error {public refusal: string;constructor(refusal: string) {@@ -96,7 +96,7 @@ interface TrimResult {warning?: string;}-export function trimToTokenLimit_F0(+function trimToTokenLimit_F0(text: string,maxTokens: number,modelId: string = "gpt-4o",@@ -446,7 +446,7 @@ export async function generateCompletions_F0({}}-export function removeDefaultProperty_F0(schema: any): any {+function removeDefaultProperty_F0(schema: any): any {if (typeof schema !== "object" || schema === null) return schema;const rest = { ...schema };diff --git a/apps/api/src/lib/extract/fire-0/ranker-f0.ts b/apps/api/src/lib/extract/fire-0/ranker-f0.tsindex 470969343..0ef2eff8f 100644--- a/apps/api/src/lib/extract/fire-0/ranker-f0.ts+++ b/apps/api/src/lib/extract/fire-0/ranker-f0.ts@@ -98,5 +98,3 @@ async function performRanking_F0(return [];}}--export { performRanking_F0 };diff --git a/apps/api/src/lib/extract/fire-0/reranker-f0.ts b/apps/api/src/lib/extract/fire-0/reranker-f0.tsindex d9e8cb5c0..58e33e8fd 100644--- a/apps/api/src/lib/extract/fire-0/reranker-f0.ts+++ b/apps/api/src/lib/extract/fire-0/reranker-f0.ts@@ -1,181 +1,18 @@-import {- MapDocument,- TeamFlags,- URLTrace,-} from "../../../controllers/v1/types";-import { isUrlBlocked } from "../../../scraper/WebScraper/utils/blocklist";+import { MapDocument, URLTrace } from "../../../controllers/v1/types";import { logger } from "../../logger";-import { CohereClient } from "cohere-ai";-import { extractConfig } from "../config";import { generateCompletions } from "../../../scraper/scrapeURL/transformers/llmExtract";-import { performRanking_F0 } from "./ranker-f0";import {buildRerankerSystemPrompt_F0,buildRerankerUserPrompt_F0,} from "./build-prompts-f0";import { CostTracking } from "../../cost-tracking";-const cohere = new CohereClient({- token: process.env.COHERE_API_KEY,-});--interface RankingResult {- mappedLinks: MapDocument[];- linksAndScores: {- link: string;- linkWithContext: string;- score: number;- originalIndex: number;- }[];-}--export async function rerankDocuments_FO(- documents: (string | Record<string, string>)[],- query: string,- topN = 3,- model = "rerank-english-v3.0",-) {- const rerank = await cohere.v2.rerank({- documents,- query,- topN,- model,- returnDocuments: true,- });-- return rerank.results- .sort((a, b) => b.relevanceScore - a.relevanceScore)- .map(x => ({- document: x.document,- index: x.index,- relevanceScore: x.relevanceScore,- }));-}--export async function rerankLinks_F0(- mappedLinks: MapDocument[],- searchQuery: string,- urlTraces: URLTrace[],- flags: TeamFlags,- metadata: { teamId: string; extractId?: string },-): Promise<MapDocument[]> {- // console.log("Going to rerank links");- const mappedLinksRerank = mappedLinks.map(- x => `url: ${x.url}, title: ${x.title}, description: ${x.description}`,- );-- const linksAndScores = await performRanking_F0(- mappedLinksRerank,- mappedLinks.map(l => l.url),- searchQuery,- metadata,- );-- // First try with high threshold- let filteredLinks = filterAndProcessLinks_F0(- mappedLinks,- linksAndScores,- extractConfig.RERANKING.INITIAL_SCORE_THRESHOLD_FOR_RELEVANCE,- flags,- );-- // If we don't have enough high-quality links, try with lower threshold- if (filteredLinks.length < extractConfig.RERANKING.MIN_REQUIRED_LINKS) {- logger.info(- `Only found ${filteredLinks.length} links with score > ${extractConfig.RERANKING.INITIAL_SCORE_THRESHOLD_FOR_RELEVANCE}. Trying lower threshold...`,- );- filteredLinks = filterAndProcessLinks_F0(- mappedLinks,- linksAndScores,- extractConfig.RERANKING.FALLBACK_SCORE_THRESHOLD_FOR_RELEVANCE,- flags,- );-- if (filteredLinks.length === 0) {- // If still no results, take top N results regardless of score- logger.warn(- `No links found with score > ${extractConfig.RERANKING.FALLBACK_SCORE_THRESHOLD_FOR_RELEVANCE}. Taking top ${extractConfig.RERANKING.MIN_REQUIRED_LINKS} results.`,- );- filteredLinks = linksAndScores- .sort((a, b) => b.score - a.score)- .slice(0, extractConfig.RERANKING.MIN_REQUIRED_LINKS)- .map(x => mappedLinks.find(link => link.url === x.link))- .filter(- (x): x is MapDocument =>- x !== undefined &&- x.url !== undefined &&- !isUrlBlocked(x.url, flags),- );- }- }-- // Update URL traces with relevance scores and mark filtered out URLs- linksAndScores.forEach(score => {- const trace = urlTraces.find(t => t.url === score.link);- if (trace) {- trace.relevanceScore = score.score;- // If URL didn't make it through filtering, mark it as filtered out- if (!filteredLinks.some(link => link.url === score.link)) {- trace.warning = `Relevance score ${score.score} below threshold`;- trace.usedInCompletion = false;- }- }- });-- const rankedLinks = filteredLinks.slice(- 0,- extractConfig.RERANKING.MAX_RANKING_LIMIT_FOR_RELEVANCE,- );-- // Mark URLs that will be used in completion- rankedLinks.forEach(link => {- const trace = urlTraces.find(t => t.url === link.url);- if (trace) {- trace.usedInCompletion = true;- }- });-- // Mark URLs that were dropped due to ranking limit- filteredLinks- .slice(extractConfig.RERANKING.MAX_RANKING_LIMIT_FOR_RELEVANCE)- .forEach(link => {- const trace = urlTraces.find(t => t.url === link.url);- if (trace) {- trace.warning = "Excluded due to ranking limit";- trace.usedInCompletion = false;- }- });-- // console.log("Reranked links: ", rankedLinks.length);-- return rankedLinks;-}--function filterAndProcessLinks_F0(- mappedLinks: MapDocument[],- linksAndScores: {- link: string;- linkWithContext: string;- score: number;- originalIndex: number;- }[],- threshold: number,- flags: TeamFlags,-): MapDocument[] {- return linksAndScores- .filter(x => x.score > threshold)- .map(x => mappedLinks.find(link => link.url === x.link))- .filter(- (x): x is MapDocument =>- x !== undefined && x.url !== undefined && !isUrlBlocked(x.url, flags),- );-}-export type RerankerResult = {+type RerankerResult = {mapDocument: (MapDocument & { relevanceScore?: number; reason?: string })[];tokensUsed: number;};-export type RerankerOptions = {+type RerankerOptions = {links: MapDocument[];searchQuery: string;urlTraces: URLTrace[];diff --git a/apps/api/src/lib/extract/fire-0/usage/llm-cost-f0.ts b/apps/api/src/lib/extract/fire-0/usage/llm-cost-f0.tsindex ab6db63f6..b41a021f5 100644--- a/apps/api/src/lib/extract/fire-0/usage/llm-cost-f0.ts+++ b/apps/api/src/lib/extract/fire-0/usage/llm-cost-f0.ts@@ -23,7 +23,7 @@ export function estimateTotalCost_F0(tokenUsage: TokenUsage[]): number {}, 0);}-export function estimateCost_F0(tokenUsage: TokenUsage): number {+function estimateCost_F0(tokenUsage: TokenUsage): number {let totalCost = 0;try {let model = tokenUsage.model ?? (process.env.MODEL_NAME || "gpt-4o-mini");diff --git a/apps/api/src/lib/extract/helpers/dump-to-file.ts b/apps/api/src/lib/extract/helpers/dump-to-file.tsdeleted file mode 100644index f52b924c1..000000000--- a/apps/api/src/lib/extract/helpers/dump-to-file.ts+++ /dev/null@@ -1,28 +0,0 @@-import * as fs from "fs";-import * as path from "path";--/**- * Helper function to dump data to a file for debugging/logging purposes- * @param filename The name of the file to write to (will be created in __dirname)- * @param data The data to write to the file- * @param formatter Optional function to format each item in the data array- */-export function dumpToFile<T>(- filename: string,- data: T[],- formatter?: (item: T, index: number) => string,-) {- const filePath = path.join(__dirname, filename);-- let fileContent: string;- if (formatter) {- fileContent = data.map((item, index) => formatter(item, index)).join("\n");- } else {- fileContent = data- .map((item, index) => `${index + 1}. ${JSON.stringify(item)}`)- .join("\n");- }-- fs.writeFileSync(filePath, fileContent, "utf8");- console.log(`Dumped data to ${filename}`);-}diff --git a/apps/api/src/lib/extract/reranker.ts b/apps/api/src/lib/extract/reranker.tsindex c06949ab4..44a1b8b30 100644--- a/apps/api/src/lib/extract/reranker.ts+++ b/apps/api/src/lib/extract/reranker.ts@@ -1,183 +1,20 @@-import { MapDocument, TeamFlags, URLTrace } from "../../controllers/v1/types";-import { performRanking } from "../ranker";-import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist";+import { MapDocument, URLTrace } from "../../controllers/v1/types";import { logger } from "../logger";-import { CohereClient } from "cohere-ai";-import { extractConfig } from "./config";import { generateCompletions } from "../../scraper/scrapeURL/transformers/llmExtract";import { buildRerankerUserPrompt } from "./build-prompts";-import { buildRerankerSystemPrompt } from "./build-prompts";-import { dumpToFile } from "./helpers/dump-to-file";import { getModel } from "../generic-ai";-import fs from "fs/promises";import { CostTracking } from "../cost-tracking";const THRESHOLD_FOR_SINGLEPAGE = 0.6;const THRESHOLD_FOR_MULTIENTITY = 0.45;-const cohere = new CohereClient({- token: process.env.COHERE_API_KEY,-});--interface RankingResult {- mappedLinks: MapDocument[];- linksAndScores: {- link: string;- linkWithContext: string;- score: number;- originalIndex: number;- }[];-}--export async function rerankDocuments(- documents: (string | Record<string, string>)[],- query: string,- topN = 3,- model = "rerank-english-v3.0",-) {- const rerank = await cohere.v2.rerank({- documents,- query,- topN,- model,- returnDocuments: true,- });-- return rerank.results- .sort((a, b) => b.relevanceScore - a.relevanceScore)- .map(x => ({- document: x.document,- index: x.index,- relevanceScore: x.relevanceScore,- }));-}--export async function rerankLinks(- mappedLinks: MapDocument[],- searchQuery: string,- urlTraces: URLTrace[],- flags: TeamFlags,- metadata: { teamId: string; extractId?: string },-): Promise<MapDocument[]> {- // console.log("Going to rerank links");- const mappedLinksRerank = mappedLinks.map(- x => `url: ${x.url}, title: ${x.title}, description: ${x.description}`,- );-- const linksAndScores = await performRanking(- mappedLinksRerank,- mappedLinks.map(l => l.url),- searchQuery,- metadata,- );-- // First try with high threshold- let filteredLinks = filterAndProcessLinks(- mappedLinks,- linksAndScores,- extractConfig.RERANKING.INITIAL_SCORE_THRESHOLD_FOR_RELEVANCE,- flags,- );-- // If we don't have enough high-quality links, try with lower threshold- if (filteredLinks.length < extractConfig.RERANKING.MIN_REQUIRED_LINKS) {- logger.info(- `Only found ${filteredLinks.length} links with score > ${extractConfig.RERANKING.INITIAL_SCORE_THRESHOLD_FOR_RELEVANCE}. Trying lower threshold...`,- );- filteredLinks = filterAndProcessLinks(- mappedLinks,- linksAndScores,- extractConfig.RERANKING.FALLBACK_SCORE_THRESHOLD_FOR_RELEVANCE,- flags,- );-- if (filteredLinks.length === 0) {- // If still no results, take top N results regardless of score- logger.warn(- `No links found with score > ${extractConfig.RERANKING.FALLBACK_SCORE_THRESHOLD_FOR_RELEVANCE}. Taking top ${extractConfig.RERANKING.MIN_REQUIRED_LINKS} results.`,- );- filteredLinks = linksAndScores- .sort((a, b) => b.score - a.score)- .slice(0, extractConfig.RERANKING.MIN_REQUIRED_LINKS)- .map(x => mappedLinks.find(link => link.url === x.link))- .filter(- (x): x is MapDocument =>- x !== undefined &&- x.url !== undefined &&- !isUrlBlocked(x.url, flags),- );- }- }-- // Update URL traces with relevance scores and mark filtered out URLs- linksAndScores.forEach(score => {- const trace = urlTraces.find(t => t.url === score.link);- if (trace) {- trace.relevanceScore = score.score;- // If URL didn't make it through filtering, mark it as filtered out- if (!filteredLinks.some(link => link.url === score.link)) {- trace.warning = `Relevance score ${score.score} below threshold`;- trace.usedInCompletion = false;- }- }- });-- const rankedLinks = filteredLinks.slice(- 0,- extractConfig.RERANKING.MAX_RANKING_LIMIT_FOR_RELEVANCE,- );-- // Mark URLs that will be used in completion- rankedLinks.forEach(link => {- const trace = urlTraces.find(t => t.url === link.url);- if (trace) {- trace.usedInCompletion = true;- }- });-- // Mark URLs that were dropped due to ranking limit- filteredLinks- .slice(extractConfig.RERANKING.MAX_RANKING_LIMIT_FOR_RELEVANCE)- .forEach(link => {- const trace = urlTraces.find(t => t.url === link.url);- if (trace) {- trace.warning = "Excluded due to ranking limit";- trace.usedInCompletion = false;- }- });-- // console.log("Reranked links: ", rankedLinks.length);-- return rankedLinks;-}--function filterAndProcessLinks(- mappedLinks: MapDocument[],- linksAndScores: {- link: string;- linkWithContext: string;- score: number;- originalIndex: number;- }[],- threshold: number,- flags: TeamFlags,-): MapDocument[] {- return linksAndScores- .filter(x => x.score > threshold)- .map(x => mappedLinks.find(link => link.url === x.link))- .filter(- (x): x is MapDocument =>- x !== undefined && x.url !== undefined && !isUrlBlocked(x.url, flags),- );-}--export type RerankerResult = {+type RerankerResult = {mapDocument: (MapDocument & { relevanceScore?: number; reason?: string })[];tokensUsed: number;cost: number;};-export type RerankerOptions = {+type RerankerOptions = {links: MapDocument[];searchQuery: string;urlTraces: URLTrace[];diff --git a/apps/api/src/lib/extract/usage/llm-cost.ts b/apps/api/src/lib/extract/usage/llm-cost.tsindex 4e675acef..4a9fd3a95 100644--- a/apps/api/src/lib/extract/usage/llm-cost.ts+++ b/apps/api/src/lib/extract/usage/llm-cost.ts@@ -28,7 +28,7 @@ export function estimateTotalCost(tokenUsage: TokenUsage[]): number {}, 0);}-export function estimateCost(tokenUsage: TokenUsage): number {+function estimateCost(tokenUsage: TokenUsage): number {let totalCost = 0;try {let model = tokenUsage.model ?? (process.env.MODEL_NAME || "gpt-4o-mini");diff --git a/apps/api/src/lib/format-utils.ts b/apps/api/src/lib/format-utils.tsindex fa438be81..119fbd2b9 100644--- a/apps/api/src/lib/format-utils.ts+++ b/apps/api/src/lib/format-utils.ts@@ -41,42 +41,3 @@ export function hasAnyFormatOfTypes(return formats.some(f => types.includes(f.type));}--/**- * Gets all format objects of a specific type from the formats array.- * This is useful when multiple formats of the same type might exist.- *- * @param formats - Array of format objects- * @param type - The format type to filter by- * @returns Array of format objects matching the type- */-export function getFormatsOfType<T extends FormatObject["type"]>(- formats: FormatObject[] | undefined,- type: T,-): Extract<FormatObject, { type: T }>[] {- if (!formats) {- return [];- }-- return formats.filter(f => f.type === type) as Extract<- FormatObject,- { type: T }- >[];-}--/**- * Extracts just the format types from an array of format objects.- * Useful for checking which formats are present without their options.- *- * @param formats - Array of format objects- * @returns Array of format type strings- */-export function getFormatTypes(- formats: FormatObject[] | undefined,-): FormatObject["type"][] {- if (!formats) {- return [];- }-- return formats.map(f => f.type);-}diff --git a/apps/api/src/lib/gcs-pdf-cache.ts b/apps/api/src/lib/gcs-pdf-cache.tsindex 70eaa099e..5f42a5734 100644--- a/apps/api/src/lib/gcs-pdf-cache.ts+++ b/apps/api/src/lib/gcs-pdf-cache.ts@@ -11,7 +11,7 @@ const PDF_CACHE_PREFIX = "pdf-cache-v2/";* Creates a SHA-256 hash of the PDF content to use as a cache key* Directly hashes the content without any conversion*/-export function createPdfCacheKey(pdfContent: string | Buffer): string {+function createPdfCacheKey(pdfContent: string | Buffer): string {return crypto.createHash("sha256").update(pdfContent).digest("hex");}diff --git a/apps/api/src/lib/generate-llmstxt/generate-llmstxt-redis.ts b/apps/api/src/lib/generate-llmstxt/generate-llmstxt-redis.tsindex 3a7f1aaee..8eab4e4aa 100644--- a/apps/api/src/lib/generate-llmstxt/generate-llmstxt-redis.ts+++ b/apps/api/src/lib/generate-llmstxt/generate-llmstxt-redis.ts@@ -1,7 +1,7 @@import { redisEvictConnection } from "../../services/redis";import { logger as _logger } from "../logger";-export interface GenerationData {+interface GenerationData {id: string;team_id: string;createdAt: number;@@ -62,7 +62,7 @@ export async function getGeneratedLlmsTxtExpiry(id: string): Promise<Date> {}// Convenience method for status updates-export async function updateGeneratedLlmsTxtStatus(+async function updateGeneratedLlmsTxtStatus(id: string,status: "processing" | "completed" | "failed",generatedText?: string,diff --git a/apps/api/src/lib/parse-mode.ts b/apps/api/src/lib/parse-mode.tsdeleted file mode 100644index 91c04ef14..000000000--- a/apps/api/src/lib/parse-mode.ts+++ /dev/null@@ -1,12 +0,0 @@-export function parseMode(mode: string) {- switch (mode) {- case "single_urls":- return "single_urls";- case "sitemap":- return "sitemap";- case "crawl":- return "crawl";- default:- return "single_urls";- }-}diff --git a/apps/api/src/lib/parseApi.ts b/apps/api/src/lib/parseApi.tsindex e29135fd3..4479d44d1 100644--- a/apps/api/src/lib/parseApi.ts+++ b/apps/api/src/lib/parseApi.ts@@ -12,8 +12,3 @@ export function parseApi(api: string) {.replace(/(.{8})(.{4})(.{4})(.{4})(.{12})/, "$1-$2-$3-$4-$5");return uuid;}--export function uuidToFcUuid(uuid: string) {- const uuidWithoutDashes = uuid.replace(/-/g, "");- return `fc-${uuidWithoutDashes}`;-}diff --git a/apps/api/src/lib/robots-txt.ts b/apps/api/src/lib/robots-txt.tsindex 656299e39..1a67051d2 100644--- a/apps/api/src/lib/robots-txt.ts+++ b/apps/api/src/lib/robots-txt.ts@@ -9,7 +9,7 @@ const useFireEngine =process.env.FIRE_ENGINE_BETA_URL !== "" &&process.env.FIRE_ENGINE_BETA_URL !== undefined;-export interface RobotsTxtChecker {+interface RobotsTxtChecker {robotsTxtUrl: string;robotsTxt: string;robots: Robot;diff --git a/apps/api/src/lib/search-query-builder.ts b/apps/api/src/lib/search-query-builder.tsindex ec1bbe339..613d0fd86 100644--- a/apps/api/src/lib/search-query-builder.ts+++ b/apps/api/src/lib/search-query-builder.ts@@ -3,14 +3,14 @@* Builds search queries with category filters for the search API*/-export interface CategoryInput {+interface CategoryInput {type: "github" | "research";sites?: string[];}export type CategoryOption = string | CategoryInput;-export interface QueryBuilderResult {+interface QueryBuilderResult {query: string;categoryMap: Map<string, string>;}diff --git a/apps/api/src/lib/timeout.ts b/apps/api/src/lib/timeout.tsdeleted file mode 100644index f913817a1..000000000--- a/apps/api/src/lib/timeout.ts+++ /dev/null@@ -1 +0,0 @@-export const axiosTimeout = 5000;diff --git a/apps/api/src/main/runWebScraper.ts b/apps/api/src/main/runWebScraper.tsindex 7d90faffe..0ef187f87 100644--- a/apps/api/src/main/runWebScraper.ts+++ b/apps/api/src/main/runWebScraper.ts@@ -42,7 +42,7 @@ export async function startWebScraperPipeline({});}-export async function runWebScraper({+async function runWebScraper({url,mode,scrapeOptions,diff --git a/apps/api/src/scraper/WebScraper/crawler.ts b/apps/api/src/scraper/WebScraper/crawler.tsindex be91be585..e34335134 100644--- a/apps/api/src/scraper/WebScraper/crawler.ts+++ b/apps/api/src/scraper/WebScraper/crawler.ts@@ -20,13 +20,13 @@ import { filterLinks } from "@mendable/firecrawl-rs";export const SITEMAP_LIMIT = 100;-export interface FilterResult {+interface FilterResult {allowed: boolean;url?: string;denialReason?: string;}-export enum DenialReason {+enum DenialReason {DEPTH_LIMIT = "URL exceeds maximum crawl depth",EXCLUDE_PATTERN = "URL matches exclude pattern",INCLUDE_PATTERN = "URL does not match required include pattern",@@ -39,7 +39,7 @@ export enum DenialReason {SECTION_LINK = "URL contains section anchor (#)",}-export interface FilterLinksResult {+interface FilterLinksResult {links: string[];denialReasons: Map<string, string>;}diff --git a/apps/api/src/scraper/WebScraper/custom/handleCustomScraping.ts b/apps/api/src/scraper/WebScraper/custom/handleCustomScraping.tsdeleted file mode 100644index 01c40de9b..000000000--- a/apps/api/src/scraper/WebScraper/custom/handleCustomScraping.ts+++ /dev/null@@ -1,67 +0,0 @@-import { logger } from "../../../lib/logger";--export async function handleCustomScraping(- text: string,- url: string,-): Promise<{- scraper: string;- url: string;- waitAfterLoad?: number;- pageOptions?: { scrollXPaths?: string[] };-} | null> {- // Check for Readme Docs special case- if (- text.includes('<meta name="readme-deploy"') &&- !url.includes("developers.notion.com")- ) {- logger.debug(- `Special use case detected for ${url}, using Fire Engine with wait time 1000ms`,- );- return {- scraper: "fire-engine",- url: url,- waitAfterLoad: 1000,- pageOptions: {- scrollXPaths: [- '//*[@id="ReferencePlayground"]/section[3]/div/pre/div/div/div[5]',- ],- },- };- }-- // Check for Vanta security portals- if (text.includes('<link href="https://static.vanta.com')) {- logger.debug(- `Vanta link detected for ${url}, using Fire Engine with wait time 3000ms`,- );- return {- scraper: "fire-engine",- url: url,- waitAfterLoad: 3000,- };- }-- // Check for Google Drive PDF links in meta tags- const googleDriveMetaPattern =- /<meta itemprop="url" content="(https:\/\/drive\.google\.com\/file\/d\/[^"]+)"/;- const googleDriveMetaMatch = text.match(googleDriveMetaPattern);- if (googleDriveMetaMatch) {- const url = googleDriveMetaMatch[1];- logger.debug(`Google Drive PDF link detected: ${url}`);-- const fileIdMatch = url.match(- /https:\/\/drive\.google\.com\/file\/d\/([^\/]+)\/view/,- );- if (fileIdMatch) {- const fileId = fileIdMatch[1];- const pdfUrl = `https://drive.google.com/uc?export=download&id=${fileId}`;-- return {- scraper: "pdf",- url: pdfUrl,- };- }- }-- return null;-}diff --git a/apps/api/src/scraper/WebScraper/utils/blocklist.ts b/apps/api/src/scraper/WebScraper/utils/blocklist.tsindex 1299b6353..a69bf2404 100644--- a/apps/api/src/scraper/WebScraper/utils/blocklist.ts+++ b/apps/api/src/scraper/WebScraper/utils/blocklist.ts@@ -8,15 +8,6 @@ configDotenv();const hashKey = Buffer.from(process.env.HASH_KEY || "", "utf-8");const algorithm = "aes-256-ecb";-export function encryptAES(plaintext: string, key: Buffer): string {- const cipher = crypto.createCipheriv(algorithm, key, null);- const encrypted = Buffer.concat([- cipher.update(plaintext, "utf-8"),- cipher.final(),- ]);- return encrypted.toString("base64");-}-export function decryptAES(ciphertext: string, key: Buffer): string {const decipher = crypto.createDecipheriv(algorithm, key, null);const decrypted = Buffer.concat([@@ -102,7 +93,7 @@ const allowedKeywords = ["://www.aa.org",];-export function decryptedBlocklist(list: string[]): string[] {+function decryptedBlocklist(list: string[]): string[] {return hashKey.length > 0? list.map(ciphertext => decryptAES(ciphertext, hashKey)): [];diff --git a/apps/api/src/scraper/WebScraper/utils/removeBase64Images.ts b/apps/api/src/scraper/WebScraper/utils/removeBase64Images.tsdeleted file mode 100644index 73452c428..000000000--- a/apps/api/src/scraper/WebScraper/utils/removeBase64Images.ts+++ /dev/null@@ -1,5 +0,0 @@-export const removeBase64Images = async (markdown: string) => {- const regex = /(!\[.*?\])\(data:image\/.*?;base64,.*?\)/g;- markdown = markdown.replace(regex, "$1(<Base64-Image-Removed>)");- return markdown;-};diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.tsindex fc67828ea..93b4e429a 100644--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts@@ -152,7 +152,7 @@ const successSchema = z.object({usedMobileProxy: z.boolean().optional(),});-export type FireEngineCheckStatusSuccess = z.infer<typeof successSchema>;+type FireEngineCheckStatusSuccess = z.infer<typeof successSchema>;const processingSchema = z.object({jobId: z.string(),diff --git a/apps/api/src/scraper/scrapeURL/engines/index.ts b/apps/api/src/scraper/scrapeURL/engines/index.tsindex 24028270d..246e27a7e 100644--- a/apps/api/src/scraper/scrapeURL/engines/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/index.ts@@ -42,7 +42,7 @@ const usePlaywright =process.env.PLAYWRIGHT_MICROSERVICE_URL !== "" &&process.env.PLAYWRIGHT_MICROSERVICE_URL !== undefined;-export const engines: Engine[] = [+const engines: Engine[] = [...(useIndex ? ["index" as const, "index;documents" as const] : []),...(useFireEngine? [@@ -62,7 +62,7 @@ export const engines: Engine[] = ["docx",];-export const featureFlags = [+const featureFlags = ["actions","waitFor","screenshot",@@ -80,7 +80,7 @@ export const featureFlags = [export type FeatureFlag = (typeof featureFlags)[number];-export const featureFlagOptions: {+const featureFlagOptions: {[F in FeatureFlag]: {priority: number;};@@ -176,7 +176,7 @@ const engineMRTs: {docx: docxMaxReasonableTime,};-export const engineOptions: {+const engineOptions: {[E in Engine]: {// A list of feature flags the engine supports.features: { [F in FeatureFlag]: boolean };diff --git a/apps/api/src/scraper/scrapeURL/engines/utils/safeFetch.ts b/apps/api/src/scraper/scrapeURL/engines/utils/safeFetch.tsindex cad336b43..031e3beb3 100644--- a/apps/api/src/scraper/scrapeURL/engines/utils/safeFetch.ts+++ b/apps/api/src/scraper/scrapeURL/engines/utils/safeFetch.ts@@ -18,7 +18,7 @@ export function isIPPrivate(address: string): boolean {return addr.range() !== "unicast";}-export function makeSecureDispatcher(skipTlsVerification: boolean) {+function makeSecureDispatcher(skipTlsVerification: boolean) {const agentOpts: undici.Agent.Options = {maxRedirections: 5000,};diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.tsindex 2be357363..e2937ae44 100644--- a/apps/api/src/scraper/scrapeURL/index.ts+++ b/apps/api/src/scraper/scrapeURL/index.ts@@ -292,7 +292,7 @@ export type InternalOptions = {v1OriginalFormat?: "extract" | "json"; // Track original v1 format for backward compatibility};-export type EngineScrapeResultWithContext = {+type EngineScrapeResultWithContext = {engine: Engine;unsupportedFeatures: Set<FeatureFlag>;result: EngineScrapeResult;diff --git a/apps/api/src/scraper/scrapeURL/lib/extractAttributes.ts b/apps/api/src/scraper/scrapeURL/lib/extractAttributes.tsindex 3b6d75e89..d909db311 100644--- a/apps/api/src/scraper/scrapeURL/lib/extractAttributes.ts+++ b/apps/api/src/scraper/scrapeURL/lib/extractAttributes.ts@@ -2,13 +2,13 @@ import { load } from "cheerio";import { logger } from "../../../lib/logger";import { extractAttributes as _extractAttributes } from "@mendable/firecrawl-rs";-export type AttributeResult = {+type AttributeResult = {selector: string;attribute: string;values: string[];};-export type AttributeSelector = {+type AttributeSelector = {selector: string;attribute: string;};diff --git a/apps/api/src/scraper/scrapeURL/lib/extractMetadata.ts b/apps/api/src/scraper/scrapeURL/lib/extractMetadata.tsindex 9a891122e..1d4bc253f 100644--- a/apps/api/src/scraper/scrapeURL/lib/extractMetadata.ts+++ b/apps/api/src/scraper/scrapeURL/lib/extractMetadata.ts@@ -3,7 +3,7 @@ import { Document } from "../../../controllers/v1/types";import { Meta } from "..";import { extractMetadata as _extractMetadata } from "@mendable/firecrawl-rs";-export async function extractMetadataRust(+async function extractMetadataRust(meta: Meta,html: string,): Promise<Partial<Document["metadata"]>> {diff --git a/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.ts b/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.tsindex e04a93f46..3088a46c2 100644--- a/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.ts+++ b/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.ts@@ -81,69 +81,6 @@ const multiSmartScrapeWrapperSchemaDefinition = {required: ["extractedData", "shouldUseSmartscrape"],};-//TODO: go over and check-// should add null to all types-// type:string should be type:["string","null"]-export function makeSchemaNullable(schema: any): any {- if (typeof schema !== "object" || schema === null) {- return schema; // Base case: not an object/array or is null- }-- if (Array.isArray(schema)) {- return schema.map(makeSchemaNullable); // Recurse for array items- }-- // Process object properties- const newSchema: { [key: string]: any } = {};- let isObject = false; // Flag to track if this level is an object type-- for (const key in schema) {- if (key === "additionalProperties") {- continue; // Skip existing additionalProperties, we'll set it later if needed- }-- if (key === "type") {- const currentType = schema[key];- let finalType: string | string[];-- if (typeof currentType === "string") {- if (currentType === "object") isObject = true;- finalType =- currentType === "null" ? currentType : [currentType, "null"];- } else if (Array.isArray(currentType)) {- if (currentType.includes("object")) isObject = true;- finalType = currentType.includes("null")- ? currentType- : [...currentType, "null"];- } else {- finalType = currentType; // Handle unexpected types?- }- newSchema[key] = finalType;- } else if (typeof schema[key] === "object" && schema[key] !== null) {- // Recurse for nested objects (properties, items, definitions, etc.)- newSchema[key] = makeSchemaNullable(schema[key]);- if (key === "properties") {- // Having a 'properties' key strongly implies an object type- isObject = true;- }- } else {- // Copy other properties directly (like required, description, etc.)- newSchema[key] = schema[key];- }- }-- // **Crucial Fix:** If this schema represents an object type, add additionalProperties: false- if (isObject) {- // Ensure 'properties' exists if 'type' was 'object' but 'properties' wasn't defined- if (!newSchema.properties) {- newSchema.properties = {};- }- newSchema.additionalProperties = false;- }-- return newSchema;-}-/*** Wraps the original schema with SmartScrape fields if an original schema exists.*@@ -151,7 +88,7 @@ export function makeSchemaNullable(schema: any): any {* @param logger Winston logger instance.* @returns An object containing the schema to use for the LLM call and whether wrapping occurred.*/-export function prepareSmartScrapeSchema(+function prepareSmartScrapeSchema(originalSchema: any | z.ZodTypeAny | undefined,logger: Logger,isSingleUrl: boolean,diff --git a/apps/api/src/scraper/scrapeURL/lib/fetch.ts b/apps/api/src/scraper/scrapeURL/lib/fetch.tsindex 04022a701..da9866a05 100644--- a/apps/api/src/scraper/scrapeURL/lib/fetch.ts+++ b/apps/api/src/scraper/scrapeURL/lib/fetch.ts@@ -8,7 +8,7 @@ import { cacheableLookup } from "./cacheableLookup";import dns from "dns";import { AbortManagerThrownError } from "./abortManager";-export type RobustFetchParams<Schema extends z.Schema<any>> = {+type RobustFetchParams<Schema extends z.Schema<any>> = {url: string;logger: Logger;method: "GET" | "POST" | "DELETE" | "PUT";diff --git a/apps/api/src/scraper/scrapeURL/lib/urlSpecificParams.ts b/apps/api/src/scraper/scrapeURL/lib/urlSpecificParams.tsindex 8a3d6c3e5..fca2526e2 100644--- a/apps/api/src/scraper/scrapeURL/lib/urlSpecificParams.ts+++ b/apps/api/src/scraper/scrapeURL/lib/urlSpecificParams.ts@@ -1,7 +1,7 @@import { InternalOptions } from "..";import { ScrapeOptions } from "../../../controllers/v1/types";-export type UrlSpecificParams = {+type UrlSpecificParams = {scrapeOptions: Partial<ScrapeOptions>;internalOptions: Partial<InternalOptions>;};diff --git a/apps/api/src/scraper/scrapeURL/transformers/index.ts b/apps/api/src/scraper/scrapeURL/transformers/index.tsindex 06bf44c4f..e45b24a51 100644--- a/apps/api/src/scraper/scrapeURL/transformers/index.ts+++ b/apps/api/src/scraper/scrapeURL/transformers/index.ts@@ -19,12 +19,12 @@ import {hasAnyFormatOfTypes,} from "../../../lib/format-utils";-export type Transformer = (+type Transformer = (meta: Meta,document: Document,) => Document | Promise<Document>;-export async function deriveMetadataFromRawHTML(+async function deriveMetadataFromRawHTML(meta: Meta,document: Document,): Promise<Document> {@@ -41,7 +41,7 @@ export async function deriveMetadataFromRawHTML(return document;}-export async function deriveHTMLFromRawHTML(+async function deriveHTMLFromRawHTML(meta: Meta,document: Document,): Promise<Document> {@@ -62,7 +62,7 @@ export async function deriveHTMLFromRawHTML(return document;}-export async function deriveMarkdownFromHTML(+async function deriveMarkdownFromHTML(meta: Meta,document: Document,): Promise<Document> {@@ -112,7 +112,7 @@ export async function deriveMarkdownFromHTML(return document;}-export async function deriveLinksFromHTML(+async function deriveLinksFromHTML(meta: Meta,document: Document,): Promise<Document> {@@ -136,7 +136,7 @@ export async function deriveLinksFromHTML(return document;}-export async function deriveImagesFromHTML(+async function deriveImagesFromHTML(meta: Meta,document: Document,): Promise<Document> {@@ -160,10 +160,7 @@ export async function deriveImagesFromHTML(return document;}-export function coerceFieldsToFormats(- meta: Meta,- document: Document,-): Document {+function coerceFieldsToFormats(meta: Meta, document: Document): Document {const hasMarkdown = hasFormatOfType(meta.options.formats, "markdown");const hasRawHtml = hasFormatOfType(meta.options.formats, "rawHtml");const hasHtml = hasFormatOfType(meta.options.formats, "html");@@ -331,7 +328,7 @@ export function coerceFieldsToFormats(}// TODO: allow some of these to run in parallel-export const transformerStack: Transformer[] = [+const transformerStack: Transformer[] = [deriveHTMLFromRawHTML,deriveMarkdownFromHTML,deriveLinksFromHTML,diff --git a/apps/api/src/search/googlesearch.ts b/apps/api/src/search/googlesearch.tsindex a43883e66..2344b7786 100644--- a/apps/api/src/search/googlesearch.ts+++ b/apps/api/src/search/googlesearch.ts@@ -7,7 +7,7 @@ import https from "https";const getRandomInt = (min: number, max: number): number =>Math.floor(Math.random() * (max - min + 1)) + min;-export function get_useragent(): string {+function get_useragent(): string {const lynx_version = `Lynx/${getRandomInt(2, 3)}.${getRandomInt(8, 9)}.${getRandomInt(0, 2)}`;const libwww_version = `libwww-FM/${getRandomInt(2, 3)}.${getRandomInt(13, 15)}`;const ssl_mm_version = `SSL-MM/${getRandomInt(1, 2)}.${getRandomInt(3, 5)}`;diff --git a/apps/api/src/search/v2/fireEngine-v2.ts b/apps/api/src/search/v2/fireEngine-v2.tsindex 351209d46..32417376c 100644--- a/apps/api/src/search/v2/fireEngine-v2.ts+++ b/apps/api/src/search/v2/fireEngine-v2.ts@@ -64,86 +64,3 @@ export async function fire_engine_search_v2(return {};}}--// Backward compatibility function-export async function fire_engine_search(- q: string,- options: {- tbs?: string;- filter?: string;- lang?: string;- country?: string;- location?: string;- numResults: number;- page?: number;- type?: SearchResultType | SearchResultType[];- },- abort?: AbortSignal,-): Promise<SearchResult[]> {- const result = await fire_engine_search_v2(q, options, abort);-- // Handle backward compatibility - convert to SearchResult array- if (Array.isArray(result)) {- return result;- } else if (result.web) {- return result.web.map(- item => new SearchResult(item.url, item.title, item.description),- );- }-- return [];-}--export async function fireEngineMap(- q: string,- options: {- tbs?: string;- filter?: string;- lang?: string;- country?: string;- location?: string;- numResults: number;- page?: number;- },- abort?: AbortSignal,-): Promise<SearchResult[]> {- try {- let data = JSON.stringify({- query: q,- lang: options.lang,- country: options.country,- location: options.location,- tbs: options.tbs,- numResults: options.numResults,- page: options.page ?? 1,- });-- if (!process.env.FIRE_ENGINE_BETA_URL) {- logger.warn(- "(v1/map Beta) Results might differ from cloud offering currently.",- );- return [];- }-- const response = await fetch(`${process.env.FIRE_ENGINE_BETA_URL}/map`, {- method: "POST",- headers: {- "Content-Type": "application/json",- "X-Disable-Cache": "true",- },- body: data,- signal: abort,- });-- if (response.ok) {- const responseData = await response.json();- return responseData;- } else {- return [];- }- } catch (error) {- logger.error(error);- Sentry.captureException(error);- return [];- }-}diff --git a/apps/api/src/search/v2/googlesearch.ts b/apps/api/src/search/v2/googlesearch.tsindex 9b74c33e4..76de00389 100644--- a/apps/api/src/search/v2/googlesearch.ts+++ b/apps/api/src/search/v2/googlesearch.ts@@ -7,7 +7,7 @@ import https from "https";const getRandomInt = (min: number, max: number): number =>Math.floor(Math.random() * (max - min + 1)) + min;-export function get_useragent(): string {+function get_useragent(): string {const lynx_version = `Lynx/${getRandomInt(2, 3)}.${getRandomInt(8, 9)}.${getRandomInt(0, 2)}`;const libwww_version = `libwww-FM/${getRandomInt(2, 3)}.${getRandomInt(13, 15)}`;const ssl_mm_version = `SSL-MM/${getRandomInt(1, 2)}.${getRandomInt(3, 5)}`;diff --git a/apps/api/src/services/billing/credit_billing.ts b/apps/api/src/services/billing/credit_billing.tsindex 0b616e335..76f408fdd 100644--- a/apps/api/src/services/billing/credit_billing.ts+++ b/apps/api/src/services/billing/credit_billing.ts@@ -47,45 +47,7 @@ export async function billTeam()(team_id, subscription_id, credits, api_key_id, logger, is_extract);}-export async function supaBillTeam(- team_id: string,- subscription_id: string | null | undefined,- credits: number,- api_key_id: number | null,- __logger?: Logger,- is_extract: boolean = false,-) {- // This function should no longer be called directly- // It has been moved to batch_billing.ts- const _logger = (__logger ?? logger).child({- module: "credit_billing",- method: "supaBillTeam",- teamId: team_id,- subscriptionId: subscription_id,- credits,- });-- _logger.warn(- "supaBillTeam was called directly. This function is deprecated and should only be called from batch_billing.ts",- );- queueBillingOperation(- team_id,- subscription_id,- credits,- api_key_id,- is_extract,- ).catch(err => {- _logger.error("Error queuing billing operation", { err });- Sentry.captureException(err);- });- // Forward to the batch billing system- return {- success: true,- message: "Billing operation queued",- };-}--export type CheckTeamCreditsResponse = {+type CheckTeamCreditsResponse = {success: boolean;message: string;remainingCredits: number;@@ -105,7 +67,7 @@ export async function checkTeamCredits(}// if team has enough credits for the operation, return true, else return false-export async function supaCheckTeamCredits(+async function supaCheckTeamCredits(chunk: AuthCreditUsageChunk | null,team_id: string,credits: number,@@ -227,96 +189,3 @@ export async function supaCheckTeamCredits(chunk,};}--// Count the total credits used by a team within the current billing period and return the remaining credits.-export async function countCreditsAndRemainingForCurrentBillingPeriod(- team_id: string,-) {- // 1. Retrieve the team's active subscription based on the team_id.- const { data: subscription, error: subscriptionError } =- await supabase_service- .from("subscriptions")- .select("id, price_id, current_period_start, current_period_end")- .eq("team_id", team_id)- .single();-- const { data: coupons } = await supabase_service- .from("coupons")- .select("credits")- .eq("team_id", team_id)- .eq("status", "active");-- let couponCredits = 0;- if (coupons && coupons.length > 0) {- couponCredits = coupons.reduce(- (total, coupon) => total + coupon.credits,- 0,- );- }-- if (subscriptionError || !subscription) {- // Free- const { data: creditUsages, error: creditUsageError } =- await supabase_service- .from("credit_usage")- .select("credits_used")- .is("subscription_id", null)- .eq("team_id", team_id);-- if (creditUsageError || !creditUsages) {- throw new Error(- `Failed to retrieve credit usage for team_id: ${team_id}`,- );- }-- const totalCreditsUsed = creditUsages.reduce(- (acc, usage) => acc + usage.credits_used,- 0,- );-- const remainingCredits = FREE_CREDITS + couponCredits - totalCreditsUsed;- return {- totalCreditsUsed: totalCreditsUsed,- remainingCredits,- totalCredits: FREE_CREDITS + couponCredits,- };- }-- const { data: creditUsages, error: creditUsageError } = await supabase_service- .from("credit_usage")- .select("credits_used")- .eq("subscription_id", subscription.id)- .gte("created_at", subscription.current_period_start)- .lte("created_at", subscription.current_period_end);-- if (creditUsageError || !creditUsages) {- throw new Error(- `Failed to retrieve credit usage for subscription_id: ${subscription.id}`,- );- }-- const totalCreditsUsed = creditUsages.reduce(- (acc, usage) => acc + usage.credits_used,- 0,- );-- const { data: price, error: priceError } = await supabase_service- .from("prices")- .select("credits")- .eq("id", subscription.price_id)- .single();-- if (priceError || !price) {- throw new Error(- `Failed to retrieve price for price_id: ${subscription.price_id}`,- );- }-- const remainingCredits = price.credits + couponCredits - totalCreditsUsed;-- return {- totalCreditsUsed,- remainingCredits,- totalCredits: price.credits,- };-}diff --git a/apps/api/src/services/billing/queue-billing.ts b/apps/api/src/services/billing/queue-billing.tsdeleted file mode 100644index 5fe109996..000000000--- a/apps/api/src/services/billing/queue-billing.ts+++ /dev/null@@ -1,49 +0,0 @@-import { logger } from "../../lib/logger";-import { getBillingQueue } from "../queue-service";-import { v4 as uuidv4 } from "uuid";-import * as Sentry from "@sentry/node";--/**- * Adds a job to the billing queue to trigger batch processing- * This can be used when we want to ensure billing is processed without waiting for the next interval- */-export async function addBillingBatchJob() {- try {- const jobId = uuidv4();- logger.info("Adding billing batch job to queue", { jobId });-- await getBillingQueue().add(- "process-batch",- {- timestamp: new Date().toISOString(),- },- {- jobId,- priority: 10,- },- );-- return { success: true, jobId };- } catch (error) {- logger.error("Error adding billing batch job", { error });- Sentry.captureException(error, {- data: {- operation: "add_billing_batch_job",- },- });- return { success: false, error };- }-}--/**- * Trigger immediate processing of any pending billing operations- * This is useful for ensuring billing operations are processed without delay- */-export async function triggerImmediateBillingProcess() {- try {- return await addBillingBatchJob();- } catch (error) {- logger.error("Error triggering immediate billing process", { error });- return { success: false, error };- }-}diff --git a/apps/api/src/services/index.ts b/apps/api/src/services/index.tsindex ddaf50392..b41814a4f 100644--- a/apps/api/src/services/index.ts+++ b/apps/api/src/services/index.ts@@ -297,7 +297,7 @@ export async function addIndexInsertJob(data: any) {);}-export async function getIndexInsertJobs(): Promise<any[]> {+async function getIndexInsertJobs(): Promise<any[]> {const jobs =(await redisEvictConnection.lpop(INDEX_INSERT_QUEUE_KEY,@@ -345,7 +345,7 @@ export async function addIndexRFInsertJob(data: any) {);}-export async function getIndexRFInsertJobs(): Promise<any[]> {+async function getIndexRFInsertJobs(): Promise<any[]> {const jobs =(await redisEvictConnection.lpop(INDEX_RF_INSERT_QUEUE_KEY,@@ -392,7 +392,7 @@ export async function addOMCEJob(data: [number, string]) {await redisEvictConnection.sadd(OMCE_JOB_QUEUE_KEY, JSON.stringify(data));}-export async function getOMCEJobs(): Promise<[number, string][]> {+async function getOMCEJobs(): Promise<[number, string][]> {const jobs =(await redisEvictConnection.spop(OMCE_JOB_QUEUE_KEY,@@ -445,7 +445,7 @@ export async function getOMCEQueueLength(): Promise<number> {const DOMAIN_FREQUENCY_QUEUE_KEY = "domain-frequency-queue";const DOMAIN_FREQUENCY_BATCH_SIZE = 100;-export function extractDomainFromUrl(url: string): string | null {+function extractDomainFromUrl(url: string): string | null {try {const urlObj = new URL(url);// Remove www. prefix for consistency@@ -469,7 +469,7 @@ export async function addDomainFrequencyJob(url: string) {await redisEvictConnection.rpush(DOMAIN_FREQUENCY_QUEUE_KEY, domain);}-export async function getDomainFrequencyJobs(): Promise<Map<string, number>> {+async function getDomainFrequencyJobs(): Promise<Map<string, number>> {const domains =(await redisEvictConnection.lpop(DOMAIN_FREQUENCY_QUEUE_KEY,diff --git a/apps/api/src/services/ledger/data-schemas.ts b/apps/api/src/services/ledger/data-schemas.tsindex dee50ec2b..3eec1a331 100644--- a/apps/api/src/services/ledger/data-schemas.ts+++ b/apps/api/src/services/ledger/data-schemas.ts@@ -6,7 +6,7 @@/*** Common properties shared across all events*/-export interface BaseEventData {+/* export */ interface BaseEventData {user_id?: string; // User ID if availableteam_id?: string; // Team ID if available}@@ -15,7 +15,8 @@ export interface BaseEventData {* Concurrent browser limit reached event data* Tracks when a user reaches the concurrent browser limit*/-export interface ConcurrentBrowserLimitReachedEventData extends BaseEventData {+/* export */ interface ConcurrentBrowserLimitReachedEventData+ extends BaseEventData {team_id: string; // The team ID}/**@@ -30,19 +31,19 @@ export interface EventDataMap {*/export type EventDefinitionSlug = keyof EventDataMap;-/**- * Helper type to extract the data type for a specific event- */-export type EventData<T extends EventDefinitionSlug> = EventDataMap[T];+// /**+// * Helper type to extract the data type for a specific event+// */+// export type EventData<T extends EventDefinitionSlug> = EventDataMap[T];-/**- * Creates a properly typed event data object with current timestamp- */-export function createEventData<T extends EventDefinitionSlug>(- eventType: T,- data: EventDataMap[T],-): EventDataMap[T] {- return {- ...data,- } as EventDataMap[T];-}+// /**+// * Creates a properly typed event data object with current timestamp+// */+// export function createEventData<T extends EventDefinitionSlug>(+// eventType: T,+// data: EventDataMap[T],+// ): EventDataMap[T] {+// return {+// ...data,+// } as EventDataMap[T];+// }diff --git a/apps/api/src/services/posthog.ts b/apps/api/src/services/posthog.tsindex 3f56123ca..02c9f7907 100644--- a/apps/api/src/services/posthog.ts+++ b/apps/api/src/services/posthog.ts@@ -2,7 +2,7 @@ import { PostHog } from "posthog-node";import "dotenv/config";import { logger } from "../../src/lib/logger";-export default function PostHogClient(apiKey: string) {+function PostHogClient(apiKey: string) {const posthogClient = new PostHog(apiKey, {host: process.env.POSTHOG_HOST,flushAt: 1,diff --git a/apps/api/src/services/queue-service.ts b/apps/api/src/services/queue-service.tsindex 5e0f68f47..c726ffb54 100644--- a/apps/api/src/services/queue-service.ts+++ b/apps/api/src/services/queue-service.ts@@ -4,8 +4,6 @@ import IORedis from "ioredis";import { BullMQOtel } from "bullmq-otel";import type { DeepResearchServiceOptions } from "../lib/deep-research/deep-research-service";-export type QueueFunction = () => Queue<any, any, string, any, any, string>;-let extractQueue: Queue;let loggingQueue: Queue;let indexQueue: Queue;@@ -27,12 +25,10 @@ export function getRedisConnection(): IORedis {return redisConnection;}-export const extractQueueName = "{extractQueue}";-export const loggingQueueName = "{loggingQueue}";-export const indexQueueName = "{indexQueue}";-export const generateLlmsTxtQueueName = "{generateLlmsTxtQueue}";-export const deepResearchQueueName = "{deepResearchQueue}";-export const billingQueueName = "{billingQueue}";+const extractQueueName = "{extractQueue}";+const generateLlmsTxtQueueName = "{generateLlmsTxtQueue}";+const deepResearchQueueName = "{deepResearchQueue}";+const billingQueueName = "{billingQueue}";export const precrawlQueueName = "{precrawlQueue}";export function getExtractQueue() {diff --git a/apps/api/src/services/rate-limiter.ts b/apps/api/src/services/rate-limiter.tsindex b9dfea7b3..2b6bbda05 100644--- a/apps/api/src/services/rate-limiter.ts+++ b/apps/api/src/services/rate-limiter.ts@@ -15,13 +15,6 @@ const createRateLimiter = (keyPrefix, points) =>duration: 60, // Duration in seconds});-export const testSuiteRateLimiter = new RateLimiterRedis({- storeClient: redisRateLimitClient,- keyPrefix: "test-suite",- points: 10000,- duration: 60, // Duration in seconds-});-const fallbackRateLimits: AuthCreditUsageChunk["rate_limits"] = {crawl: 15,scrape: 100,diff --git a/apps/api/src/services/webhook/delivery.ts b/apps/api/src/services/webhook/delivery.tsindex 422b91106..4edccaa7d 100644--- a/apps/api/src/services/webhook/delivery.ts+++ b/apps/api/src/services/webhook/delivery.ts@@ -174,7 +174,7 @@ export async function processWebhookInsertJobs() {}}-export async function logWebhook(data: {+async function logWebhook(data: {success: boolean;error?: string;teamId: string;diff --git a/apps/api/src/services/webhook/index.ts b/apps/api/src/services/webhook/index.tsindex 6b2ef3bef..f3369ffe8 100644--- a/apps/api/src/services/webhook/index.ts+++ b/apps/api/src/services/webhook/index.ts@@ -29,4 +29,4 @@ export {getWebhookInsertQueueLength,processWebhookInsertJobs,} from "./delivery";-export * from "./types";+export { WebhookEvent } from "./types";diff --git a/apps/api/src/services/webhook/types.ts b/apps/api/src/services/webhook/types.tsindex 8582a4b67..92684cce0 100644--- a/apps/api/src/services/webhook/types.ts+++ b/apps/api/src/services/webhook/types.ts@@ -29,13 +29,13 @@ export type WebhookEventDataMap = {export type WebhookConfig = z.infer<typeof webhookSchema>;-export interface WebhookDocument {+interface WebhookDocument {content?: string;markdown: string;metadata: Record<string, any>;}-export interface WebhookDocumentLink {+interface WebhookDocumentLink {content: WebhookDocument;source: string;}@@ -46,55 +46,50 @@ interface BaseWebhookData {}// crawl-export interface CrawlStartedData extends BaseWebhookData {+interface CrawlStartedData extends BaseWebhookData {success: true;}-export interface CrawlPageData extends BaseWebhookData {+interface CrawlPageData extends BaseWebhookData {success: boolean;data: Document[] | WebhookDocumentLink[]; // links or documents (v0 compatible)scrapeId: string;error?: string;}-export interface CrawlCompletedData extends BaseWebhookData {+interface CrawlCompletedData extends BaseWebhookData {success: true;data: Document[] | WebhookDocumentLink[]; // empty array or links (v0 compatible)}-export interface CrawlFailedData extends BaseWebhookData {- success: false;- error: string;-}-// batch scrape-export interface BatchScrapeStartedData extends BaseWebhookData {+interface BatchScrapeStartedData extends BaseWebhookData {success: true;}-export interface BatchScrapePageData extends BaseWebhookData {+interface BatchScrapePageData extends BaseWebhookData {success: boolean;data: Document[];scrapeId: string;error?: string; // more v0 tomfoolery}-export interface BatchScrapeCompletedData extends BaseWebhookData {+interface BatchScrapeCompletedData extends BaseWebhookData {success: true;data: WebhookDocumentLink[];}// extract-export interface ExtractStartedData extends BaseWebhookData {+interface ExtractStartedData extends BaseWebhookData {success: true;}-export interface ExtractCompletedData extends BaseWebhookData {+interface ExtractCompletedData extends BaseWebhookData {success: true;data: ExtractResult[];}-export interface ExtractFailedData extends BaseWebhookData {+interface ExtractFailedData extends BaseWebhookData {success: false;error: string;}diff --git a/apps/api/src/strings.ts b/apps/api/src/strings.tsdeleted file mode 100644index d5672b82e..000000000--- a/apps/api/src/strings.ts+++ /dev/null@@ -1,4 +0,0 @@-export const errorNoResults =- "No results found, please check the URL or contact us at [email protected] to file a ticket.";--export const clientSideError = "client-side exception has occurred";diff --git a/apps/api/src/supabase_types.ts b/apps/api/src/supabase_types.tsdeleted file mode 100644index 00b2efbba..000000000--- a/apps/api/src/supabase_types.ts+++ /dev/null@@ -1,1514 +0,0 @@-export type Json =- | string- | number- | boolean- | null- | { [key: string]: Json | undefined }- | Json[];--export interface Database {- public: {- Tables: {- api_keys: {- Row: {- created_at: string | null;- id: number;- key: string | null;- keyType: string;- name: string | null;- project_id: number | null;- };- Insert: {- created_at?: string | null;- id?: number;- key?: string | null;- keyType?: string;- name?: string | null;- project_id?: number | null;- };- Update: {- created_at?: string | null;- id?: number;- key?: string | null;- keyType?: string;- name?: string | null;- project_id?: number | null;- };- Relationships: [- {- foreignKeyName: "api_keys_project_id_fkey";- columns: ["project_id"];- referencedRelation: "mendable_project";- referencedColumns: ["id"];- },- ];- };- company: {- Row: {- company_id: number;- company_name: string | null;- display_name: string | null;- is_white_label: boolean | null;- name: string;- playground_type: string | null;- pricing_plan_id: number;- };- Insert: {- company_id?: number;- company_name?: string | null;- display_name?: string | null;- is_white_label?: boolean | null;- name: string;- playground_type?: string | null;- pricing_plan_id?: number;- };- Update: {- company_id?: number;- company_name?: string | null;- display_name?: string | null;- is_white_label?: boolean | null;- name?: string;- playground_type?: string | null;- pricing_plan_id?: number;- };- Relationships: [- {- foreignKeyName: "company_pricing_plan_id_fkey";- columns: ["pricing_plan_id"];- referencedRelation: "pricing_plan";- referencedColumns: ["id"];- },- ];- };- constants: {- Row: {- created_at: string | null;- default_prompt: string;- id: number;- };- Insert: {- created_at?: string | null;- default_prompt: string;- id?: number;- };- Update: {- created_at?: string | null;- default_prompt?: string;- id?: number;- };- Relationships: [];- };- conversation: {- Row: {- conversation_id: number;- end_time: string | null;- experiment_id: string | null;- project_id: number;- start_time: string;- };- Insert: {- conversation_id?: number;- end_time?: string | null;- experiment_id?: string | null;- project_id: number;- start_time: string;- };- Update: {- conversation_id?: number;- end_time?: string | null;- experiment_id?: string | null;- project_id?: number;- start_time?: string;- };- Relationships: [- {- foreignKeyName: "conversation_project_id_fkey";- columns: ["project_id"];- referencedRelation: "mendable_project";- referencedColumns: ["id"];- },- ];- };- customers: {- Row: {- id: string;- stripe_customer_id: string | null;- user_id: string | null;- };- Insert: {- id: string;- stripe_customer_id?: string | null;- user_id?: string | null;- };- Update: {- id?: string;- stripe_customer_id?: string | null;- user_id?: string | null;- };- Relationships: [- {- foreignKeyName: "customers_id_fkey";- columns: ["id"];- referencedRelation: "users";- referencedColumns: ["id"];- },- {- foreignKeyName: "customers_user_id_fkey";- columns: ["user_id"];- referencedRelation: "users";- referencedColumns: ["id"];- },- ];- };- data: {- Row: {- company_id: number | null;- content: string | null;- data_id: string | null;- date_added: string | null;- date_modified: string | null;- embedding: string | null;- id: number;- manual_add: boolean | null;- message_id: number | null;- project_id: number | null;- search_index: unknown | null;- source: string | null;- source_name: string | null;- source_rank: number | null;- source_text: string | null;- };- Insert: {- company_id?: number | null;- content?: string | null;- data_id?: string | null;- date_added?: string | null;- date_modified?: string | null;- embedding?: string | null;- id?: number;- manual_add?: boolean | null;- message_id?: number | null;- project_id?: number | null;- search_index?: unknown | null;- source?: string | null;- source_name?: string | null;- source_rank?: number | null;- source_text?: string | null;- };- Update: {- company_id?: number | null;- content?: string | null;- data_id?: string | null;- date_added?: string | null;- date_modified?: string | null;- embedding?: string | null;- id?: number;- manual_add?: boolean | null;- message_id?: number | null;- project_id?: number | null;- search_index?: unknown | null;- source?: string | null;- source_name?: string | null;- source_rank?: number | null;- source_text?: string | null;- };- Relationships: [- {- foreignKeyName: "data_company_id_fkey";- columns: ["company_id"];- referencedRelation: "company";- referencedColumns: ["company_id"];- },- {- foreignKeyName: "data_data_id_fkey";- columns: ["data_id"];- referencedRelation: "data_sources";- referencedColumns: ["id"];- },- {- foreignKeyName: "data_message_id_fkey";- columns: ["message_id"];- referencedRelation: "message";- referencedColumns: ["message_id"];- },- {- foreignKeyName: "data_project_id_fkey";- columns: ["project_id"];- referencedRelation: "mendable_project";- referencedColumns: ["id"];- },- ];- };- data_partitioned: {- Row: {- company_id: number | null;- company_name: string;- content: string | null;- data_id: string | null;- date_added: string | null;- date_modified: string | null;- embedding: string | null;- id: number;- manual_add: boolean | null;- message_id: number | null;- project_id: number | null;- search_index: unknown | null;- source: string | null;- source_name: string | null;- source_rank: number | null;- source_text: string | null;- };- Insert: {- company_id?: number | null;- company_name: string;- content?: string | null;- data_id?: string | null;- date_added?: string | null;- date_modified?: string | null;- embedding?: string | null;- id: number;- manual_add?: boolean | null;- message_id?: number | null;- project_id?: number | null;- search_index?: unknown | null;- source?: string | null;- source_name?: string | null;- source_rank?: number | null;- source_text?: string | null;- };- Update: {- company_id?: number | null;- company_name?: string;- content?: string | null;- data_id?: string | null;- date_added?: string | null;- date_modified?: string | null;- embedding?: string | null;- id?: number;- manual_add?: boolean | null;- message_id?: number | null;- project_id?: number | null;- search_index?: unknown | null;- source?: string | null;- source_name?: string | null;- source_rank?: number | null;- source_text?: string | null;- };- Relationships: [];- };- data_sources: {- Row: {- created_at: string | null;- description: string | null;- id: string;- name: string | null;- placeholder: string | null;- type: string | null;- };- Insert: {- created_at?: string | null;- description?: string | null;- id?: string;- name?: string | null;- placeholder?: string | null;- type?: string | null;- };- Update: {- created_at?: string | null;- description?: string | null;- id?: string;- name?: string | null;- placeholder?: string | null;- type?: string | null;- };- Relationships: [];- };- marketing: {- Row: {- created_at: string | null;- id: number;- message: string | null;- };- Insert: {- created_at?: string | null;- id?: number;- message?: string | null;- };- Update: {- created_at?: string | null;- id?: number;- message?: string | null;- };- Relationships: [];- };- mendable_project: {- Row: {- company_id: number | null;- created_at: string | null;- display_id: string;- enforce_whitelist: boolean | null;- id: number;- isFaqPublic: boolean;- max_messages_per_month: number | null;- max_req_per_ip_per_minute: number | null;- name: string | null;- prompt_id: number | null;- support_url: string | null;- whitelisted_domains: string[] | null;- };- Insert: {- company_id?: number | null;- created_at?: string | null;- display_id?: string;- enforce_whitelist?: boolean | null;- id?: number;- isFaqPublic?: boolean;- max_messages_per_month?: number | null;- max_req_per_ip_per_minute?: number | null;- name?: string | null;- prompt_id?: number | null;- support_url?: string | null;- whitelisted_domains?: string[] | null;- };- Update: {- company_id?: number | null;- created_at?: string | null;- display_id?: string;- enforce_whitelist?: boolean | null;- id?: number;- isFaqPublic?: boolean;- max_messages_per_month?: number | null;- max_req_per_ip_per_minute?: number | null;- name?: string | null;- prompt_id?: number | null;- support_url?: string | null;- whitelisted_domains?: string[] | null;- };- Relationships: [- {- foreignKeyName: "mendable_project_company_id_fkey";- columns: ["company_id"];- referencedRelation: "company";- referencedColumns: ["company_id"];- },- ];- };- message: {- Row: {- conversation_id: number;- embedding: string | null;- is_taught: boolean | null;- message: string;- message_id: number;- model_configuration: Json | null;- prompt_text: string | null;- rating_value: number | null;- rephrased_text: string | null;- sender: string;- timestamp: string;- };- Insert: {- conversation_id: number;- embedding?: string | null;- is_taught?: boolean | null;- message: string;- message_id?: number;- model_configuration?: Json | null;- prompt_text?: string | null;- rating_value?: number | null;- rephrased_text?: string | null;- sender: string;- timestamp: string;- };- Update: {- conversation_id?: number;- embedding?: string | null;- is_taught?: boolean | null;- message?: string;- message_id?: number;- model_configuration?: Json | null;- prompt_text?: string | null;- rating_value?: number | null;- rephrased_text?: string | null;- sender?: string;- timestamp?: string;- };- Relationships: [- {- foreignKeyName: "message_conversation_id_fkey";- columns: ["conversation_id"];- referencedRelation: "conversation";- referencedColumns: ["conversation_id"];- },- ];- };- model_configuration: {- Row: {- created_at: string | null;- custom_prompt: string | null;- id: number;- model_name: string | null;- project_id: number;- suggested_questions: string | null;- support_link: string | null;- temperature: number;- };- Insert: {- created_at?: string | null;- custom_prompt?: string | null;- id?: number;- model_name?: string | null;- project_id: number;- suggested_questions?: string | null;- support_link?: string | null;- temperature?: number;- };- Update: {- created_at?: string | null;- custom_prompt?: string | null;- id?: number;- model_name?: string | null;- project_id?: number;- suggested_questions?: string | null;- support_link?: string | null;- temperature?: number;- };- Relationships: [- {- foreignKeyName: "model_configuration_project_id_fkey";- columns: ["project_id"];- referencedRelation: "mendable_project";- referencedColumns: ["id"];- },- ];- };- monthly_message_counts: {- Row: {- message_count: number | null;- month: number;- project_id: number;- year: number;- };- Insert: {- message_count?: number | null;- month: number;- project_id: number;- year: number;- };- Update: {- message_count?: number | null;- month?: number;- project_id?: number;- year?: number;- };- Relationships: [- {- foreignKeyName: "monthly_message_counts_project_id_fkey";- columns: ["project_id"];- referencedRelation: "mendable_project";- referencedColumns: ["id"];- },- ];- };- prices: {- Row: {- active: boolean | null;- currency: string | null;- description: string | null;- id: string;- interval: Database["public"]["Enums"]["pricing_plan_interval"] | null;- interval_count: number | null;- metadata: Json | null;- product_id: string | null;- trial_period_days: number | null;- type: Database["public"]["Enums"]["pricing_type"] | null;- unit_amount: number | null;- };- Insert: {- active?: boolean | null;- currency?: string | null;- description?: string | null;- id: string;- interval?:- | Database["public"]["Enums"]["pricing_plan_interval"]- | null;- interval_count?: number | null;- metadata?: Json | null;- product_id?: string | null;- trial_period_days?: number | null;- type?: Database["public"]["Enums"]["pricing_type"] | null;- unit_amount?: number | null;- };- Update: {- active?: boolean | null;- currency?: string | null;- description?: string | null;- id?: string;- interval?:- | Database["public"]["Enums"]["pricing_plan_interval"]- | null;- interval_count?: number | null;- metadata?: Json | null;- product_id?: string | null;- trial_period_days?: number | null;- type?: Database["public"]["Enums"]["pricing_type"] | null;- unit_amount?: number | null;- };- Relationships: [- {- foreignKeyName: "prices_product_id_fkey";- columns: ["product_id"];- referencedRelation: "products";- referencedColumns: ["id"];- },- ];- };- pricing_plan: {- Row: {- base_price: number;- created_at: string | null;- gpt_35_turbo_price: number | null;- gpt_4_price: number | null;- id: number;- max_messages_per_month: number | null;- name: string | null;- price_per_message: number | null;- stripe_subscription: boolean;- };- Insert: {- base_price?: number;- created_at?: string | null;- gpt_35_turbo_price?: number | null;- gpt_4_price?: number | null;- id?: number;- max_messages_per_month?: number | null;- name?: string | null;- price_per_message?: number | null;- stripe_subscription?: boolean;- };- Update: {- base_price?: number;- created_at?: string | null;- gpt_35_turbo_price?: number | null;- gpt_4_price?: number | null;- id?: number;- max_messages_per_month?: number | null;- name?: string | null;- price_per_message?: number | null;- stripe_subscription?: boolean;- };- Relationships: [];- };- products: {- Row: {- active: boolean | null;- description: string | null;- id: string;- image: string | null;- metadata: Json | null;- name: string | null;- };- Insert: {- active?: boolean | null;- description?: string | null;- id: string;- image?: string | null;- metadata?: Json | null;- name?: string | null;- };- Update: {- active?: boolean | null;- description?: string | null;- id?: string;- image?: string | null;- metadata?: Json | null;- name?: string | null;- };- Relationships: [];- };- prompt: {- Row: {- name: string | null;- prompt: string | null;- prompt_id: number;- };- Insert: {- name?: string | null;- prompt?: string | null;- prompt_id?: number;- };- Update: {- name?: string | null;- prompt?: string | null;- prompt_id?: number;- };- Relationships: [];- };- source: {- Row: {- content: string | null;- created_at: string | null;- id: number;- link: string | null;- message_id: number;- };- Insert: {- content?: string | null;- created_at?: string | null;- id?: number;- link?: string | null;- message_id: number;- };- Update: {- content?: string | null;- created_at?: string | null;- id?: number;- link?: string | null;- message_id?: number;- };- Relationships: [];- };- subscriptions: {- Row: {- cancel_at: string | null;- cancel_at_period_end: boolean | null;- canceled_at: string | null;- company_id: number | null;- created: string;- current_period_end: string;- current_period_start: string;- ended_at: string | null;- id: string;- metadata: Json | null;- price_id: string | null;- quantity: number | null;- status: Database["public"]["Enums"]["subscription_status"] | null;- sub_item_id: string | null;- trial_end: string | null;- trial_start: string | null;- usage_pricing_id: string | null;- user_id: string;- };- Insert: {- cancel_at?: string | null;- cancel_at_period_end?: boolean | null;- canceled_at?: string | null;- company_id?: number | null;- created?: string;- current_period_end?: string;- current_period_start?: string;- ended_at?: string | null;- id: string;- metadata?: Json | null;- price_id?: string | null;- quantity?: number | null;- status?: Database["public"]["Enums"]["subscription_status"] | null;- sub_item_id?: string | null;- trial_end?: string | null;- trial_start?: string | null;- usage_pricing_id?: string | null;- user_id: string;- };- Update: {- cancel_at?: string | null;- cancel_at_period_end?: boolean | null;- canceled_at?: string | null;- company_id?: number | null;- created?: string;- current_period_end?: string;- current_period_start?: string;- ended_at?: string | null;- id?: string;- metadata?: Json | null;- price_id?: string | null;- quantity?: number | null;- status?: Database["public"]["Enums"]["subscription_status"] | null;- sub_item_id?: string | null;- trial_end?: string | null;- trial_start?: string | null;- usage_pricing_id?: string | null;- user_id?: string;- };- Relationships: [- {- foreignKeyName: "subscriptions_company_id_fkey";- columns: ["company_id"];- referencedRelation: "company";- referencedColumns: ["company_id"];- },- {- foreignKeyName: "subscriptions_price_id_fkey";- columns: ["price_id"];- referencedRelation: "prices";- referencedColumns: ["id"];- },- {- foreignKeyName: "subscriptions_user_id_fkey";- columns: ["user_id"];- referencedRelation: "users";- referencedColumns: ["id"];- },- ];- };- suggested_questions: {- Row: {- created_at: string | null;- id: number;- project_id: number | null;- question: string | null;- };- Insert: {- created_at?: string | null;- id?: number;- project_id?: number | null;- question?: string | null;- };- Update: {- created_at?: string | null;- id?: number;- project_id?: number | null;- question?: string | null;- };- Relationships: [- {- foreignKeyName: "suggested_questions_project_id_fkey";- columns: ["project_id"];- referencedRelation: "mendable_project";- referencedColumns: ["id"];- },- ];- };- user_notifications: {- Row: {- company_id: number;- id: number;- notification_type: string;- project_id: number;- sent_date: string;- user_id: string;- };- Insert: {- company_id: number;- id?: number;- notification_type: string;- project_id: number;- sent_date: string;- user_id: string;- };- Update: {- company_id?: number;- id?: number;- notification_type?: string;- project_id?: number;- sent_date?: string;- user_id?: string;- };- Relationships: [- {- foreignKeyName: "user_notifications_company_id_fkey";- columns: ["company_id"];- referencedRelation: "company";- referencedColumns: ["company_id"];- },- {- foreignKeyName: "user_notifications_project_id_fkey";- columns: ["project_id"];- referencedRelation: "mendable_project";- referencedColumns: ["id"];- },- {- foreignKeyName: "user_notifications_user_id_fkey";- columns: ["user_id"];- referencedRelation: "users";- referencedColumns: ["id"];- },- ];- };- users: {- Row: {- avatar_url: string | null;- billing_address: Json | null;- company_id: number | null;- email: string | null;- full_name: string | null;- id: string;- payment_method: Json | null;- };- Insert: {- avatar_url?: string | null;- billing_address?: Json | null;- company_id?: number | null;- email?: string | null;- full_name?: string | null;- id: string;- payment_method?: Json | null;- };- Update: {- avatar_url?: string | null;- billing_address?: Json | null;- company_id?: number | null;- email?: string | null;- full_name?: string | null;- id?: string;- payment_method?: Json | null;- };- Relationships: [- {- foreignKeyName: "users_company_id_fkey";- columns: ["company_id"];- referencedRelation: "company";- referencedColumns: ["company_id"];- },- {- foreignKeyName: "users_id_fkey";- columns: ["id"];- referencedRelation: "users";- referencedColumns: ["id"];- },- ];- };- z_testcomp_92511: {- Row: {- content: string | null;- data_id: string | null;- date_added: string | null;- date_modified: string | null;- embedding: string | null;- id: number;- manual_add: boolean | null;- message_id: number | null;- project_id: number | null;- search_index: unknown | null;- source: string | null;- source_name: string | null;- source_rank: number | null;- source_text: string | null;- };- Insert: {- content?: string | null;- data_id?: string | null;- date_added?: string | null;- date_modified?: string | null;- embedding?: string | null;- id?: number;- manual_add?: boolean | null;- message_id?: number | null;- project_id?: number | null;- search_index?: unknown | null;- source?: string | null;- source_name?: string | null;- source_rank?: number | null;- source_text?: string | null;- };- Update: {- content?: string | null;- data_id?: string | null;- date_added?: string | null;- date_modified?: string | null;- embedding?: string | null;- id?: number;- manual_add?: boolean | null;- message_id?: number | null;- project_id?: number | null;- search_index?: unknown | null;- source?: string | null;- source_name?: string | null;- source_rank?: number | null;- source_text?: string | null;- };- Relationships: [- {- foreignKeyName: "z_testcomp_92511_data_id_fkey";- columns: ["data_id"];- referencedRelation: "data_sources";- referencedColumns: ["id"];- },- {- foreignKeyName: "z_testcomp_92511_message_id_fkey";- columns: ["message_id"];- referencedRelation: "message";- referencedColumns: ["message_id"];- },- {- foreignKeyName: "z_testcomp_92511_project_id_fkey";- columns: ["project_id"];- referencedRelation: "mendable_project";- referencedColumns: ["id"];- },- ];- };- };- Views: {- [_ in never]: never;- };- Functions: {- combine_search_29: {- Args: {- query_embedding: string;- query_text: string;- table_name: string;- project_id: number;- similarity_threshold: number;- search_thres: number;- k?: number;- };- Returns: {- id: number;- content: string;- source: string;- similarity: number;- rank: number;- similarity_rank: number;- rank_rank: number;- rrf_score: number;- }[];- };- create_company_table_2: {- Args: {- company: string;- };- Returns: undefined;- };- create_indexs: {- Args: {- tbl_name: string;- };- Returns: undefined;- };- create_mendable_project_3: {- Args: {- _company_name: string;- _pricing_plan_id: number;- _project_name: string;- };- Returns: Json;- };- create_new_company_table_5: {- Args: {- _company_name: string;- _company_display_name: string;- };- Returns: number;- };- create_new_project_2: {- Args: {- _company_id: number;- _project_name: string;- };- Returns: Json;- };- create_project_with_api_key_10: {- Args: {- _company_name: string;- _pricing_plan_id: number;- _project_name: string;- };- Returns: Json;- };- create_project_with_api_key_8: {- Args: {- _company_name: string;- _pricing_plan_id: number;- _project_name: string;- };- Returns: boolean;- };- create_table: {- Args: {- table_name: string;- };- Returns: undefined;- };- dmetaphone: {- Args: {- "": string;- };- Returns: string;- };- dmetaphone_alt: {- Args: {- "": string;- };- Returns: string;- };- get_all_conversation_with_messages_by_project_id: {- Args: {- _project_id: number;- page: number;- conversations_per_page: number;- };- Returns: {- conversation_id: number;- end_time: string | null;- experiment_id: string | null;- project_id: number;- start_time: string;- }[];- };- get_index_types: {- Args: {- _company_name: string;- _project_id: number;- };- Returns: unknown;- };- get_message_count_by_project: {- Args: {- p_project_id: number;- };- Returns: {- message_day: string;- message_count: number;- }[];- };- get_messages_by_project_and_rating: {- Args: {- _project_id: number;- _rating_value: number;- current_page: number;- pages_per_set: number;- };- Returns: {- message_id: number;- conversation_id: number;- message: string;- timestamp: string;- sender: string;- rating_value: number;- prev_message_id: number;- prev_message: string;- prev_timestamp: string;- prev_sender: string;- }[];- };- get_messages_without_sources_by_project_with_prev: {- Args: {- _project_id: number;- current_page: number;- pages_per_set: number;- };- Returns: {- message_id: number;- conversation_id: number;- message: string;- timestamp: string;- sender: string;- rating_value: number;- prev_message_id: number;- prev_message: string;- prev_timestamp: string;- prev_sender: string;- }[];- };- get_messages_without_sources_count_by_project: {- Args: {- _project_id: number;- };- Returns: number;- };- get_negative_messages_count_by_project_id: {- Args: {- _project_id: number;- };- Returns: number;- };- get_positive_messages_count_by_project: {- Args: {- _project_id: number;- };- Returns: number;- };- get_project_and_company_with_model_configuration: {- Args: {- p_project_id: number;- };- Returns: {- project: unknown;- company: unknown;- model_configuration: unknown;- }[];- };- get_project_company_model: {- Args: {- api_key_in: string;- };- Returns: {- company: unknown;- project: unknown;- configuration: unknown;- }[];- };- get_project_message_count: {- Args: {- p_project_id: number;- p_start_date: string;- p_end_date: string;- };- Returns: number;- };- get_sources_count: {- Args: {- company_name: string;- project_id: number;- };- Returns: number;- };- get_sources_from_message: {- Args: {- mid: number;- };- Returns: {- content: string | null;- created_at: string | null;- id: number;- link: string | null;- message_id: number;- }[];- };- get_sources_info_new: {- Args: {- company_name: string;- project_id: number;- pages_per_set: number;- current_page: number;- };- Returns: {- source: string;- source_rank: number;- data_id: string;- source_text: string;- source_name: string;- }[];- };- get_subscription: {- Args: {- p_project_id: number;- };- Returns: {- cancel_at: string | null;- cancel_at_period_end: boolean | null;- canceled_at: string | null;- company_id: number | null;- created: string;- current_period_end: string;- current_period_start: string;- ended_at: string | null;- id: string;- metadata: Json | null;- price_id: string | null;- quantity: number | null;- status: Database["public"]["Enums"]["subscription_status"] | null;- sub_item_id: string | null;- trial_end: string | null;- trial_start: string | null;- usage_pricing_id: string | null;- user_id: string;- };- };- get_total_conversations_count_by_project: {- Args: {- _project_id: number;- };- Returns: number;- };- get_trained_messages_from_project: {- Args: {- p_project_id: number;- current_page: number;- pages_per_set: number;- };- Returns: {- current_message: Json;- preceding_message: Json;- }[];- };- get_trained_messages_from_project_count: {- Args: {- p_project_id: number;- };- Returns: number;- };- getPricingPlanByName: {- Args: {- company_name: string;- };- Returns: {- base_price: number;- created_at: string | null;- gpt_35_turbo_price: number | null;- gpt_4_price: number | null;- id: number;- max_messages_per_month: number | null;- name: string | null;- price_per_message: number | null;- stripe_subscription: boolean;- };- };- gtrgm_compress: {- Args: {- "": unknown;- };- Returns: unknown;- };- gtrgm_decompress: {- Args: {- "": unknown;- };- Returns: unknown;- };- gtrgm_in: {- Args: {- "": unknown;- };- Returns: unknown;- };- gtrgm_options: {- Args: {- "": unknown;- };- Returns: undefined;- };- gtrgm_out: {- Args: {- "": unknown;- };- Returns: unknown;- };- increment_message_count: {- Args: {- projectid: number;- month: number;- year: number;- };- Returns: undefined;- };- increment_message_count_2: {- Args: {- p_projectid: number;- p_month: number;- p_year: number;- };- Returns: undefined;- };- ivfflathandler: {- Args: {- "": unknown;- };- Returns: unknown;- };- keyword_search_trigram22: {- Args: {- k: number;- query_text: string;- table_name: string;- project_id: number;- limit_results: number;- };- Returns: {- id: number;- content: string;- source: string;- similarity_score: number;- }[];- };- match_documents: {- Args: {- table_name: string;- query_embedding: string;- similarity_threshold: number;- match_count: number;- };- Returns: {- id: number;- content: string;- similarity: number;- source: string;- }[];- };- messagesSentThisMonthByName: {- Args: {- company_name: string;- };- Returns: number;- };- search_content_priority_by_project_2: {- Args: {- tbl_name: string;- query_text: string;- search_thres: number;- num_results: number;- project_id: number;- };- Returns: {- id: number;- content: string;- source: string;- rank: number;- }[];- };- search_content_priority30: {- Args: {- query_text: string;- tbl_name: string;- num_results: number;- search_thres: number;- };- Returns: {- id: number;- content: string;- source: string;- rank: number;- }[];- };- search_documents: {- Args: {- k: number;- query_embedding: string;- query_text: string;- table_name: string;- project_id: number;- limit_results: number;- };- Returns: {- id: number;- content: string;- source: string;- similarity: number;- rank: number;- similarity_rank: number;- rank_rank: number;- rrf_score: number;- }[];- };- search_documents_return_all5: {- Args: {- k: number;- query_embedding: string;- query_text: string;- table_name: string;- project_id: number;- limit_results: number;- };- Returns: {- id: number;- content: string;- source: string;- similarity: number;- rank: number;- similarity_rank: number;- rank_rank: number;- rrf_score: number;- source_name: string;- date_added: string;- date_modified: string;- }[];- };- set_limit: {- Args: {- "": number;- };- Returns: number;- };- show_limit: {- Args: Record<PropertyKey, never>;- Returns: number;- };- show_trgm: {- Args: {- "": string;- };- Returns: unknown;- };- soundex: {- Args: {- "": string;- };- Returns: string;- };- text_soundex: {- Args: {- "": string;- };- Returns: string;- };- update_search_index: {- Args: {- table_name: string;- };- Returns: undefined;- };- update_search_index_2: {- Args: {- table_name: string;- };- Returns: Json;- };- update_search_index_single_row: {- Args: {- table_name: string;- row_id: number;- };- Returns: Json;- };- vector_avg: {- Args: {- "": number[];- };- Returns: string;- };- vector_dims: {- Args: {- "": string;- };- Returns: number;- };- vector_norm: {- Args: {- "": string;- };- Returns: number;- };- vector_out: {- Args: {- "": string;- };- Returns: unknown;- };- vector_send: {- Args: {- "": string;- };- Returns: string;- };- vector_typmod_in: {- Args: {- "": unknown[];- };- Returns: number;- };- get_documents: {- Args: {- c_company_id;- page_size;- z_offset;- };- };- };- Enums: {- pricing_plan_interval: "day" | "week" | "month" | "year";- pricing_type: "one_time" | "recurring";- subscription_status:- | "trialing"- | "active"- | "canceled"- | "incomplete"- | "incomplete_expired"- | "past_due"- | "unpaid"- | "paused";- };- CompositeTypes: {- [_ in never]: never;- };- };-}diff --git a/apps/api/src/types.ts b/apps/api/src/types.tsindex b5f99a7e9..28aced098 100644--- a/apps/api/src/types.ts+++ b/apps/api/src/types.ts@@ -15,24 +15,6 @@ type Mode = "crawl" | "single_urls" | "sitemap" | "kickoff";export { Mode };-export interface CrawlResult {- source: string;- content: string;- options?: {- summarize?: boolean;- summarize_max_chars?: number;- };- metadata?: any;- raw_context_id?: number | string;- permissions?: any[];-}--export interface IngestResult {- success: boolean;- error: string;- data: CrawlResult[];-}-export interface WebScraperOptions {url: string;mode: Mode;@@ -77,16 +59,6 @@ export interface RunWebScraperParams {costTracking: CostTracking;}-export type RunWebScraperResult =- | {- success: false;- error: Error;- }- | {- success: true;- document: V2Document;- };-export interface FirecrawlJob {job_id?: string;success: boolean;@@ -142,15 +114,6 @@ export interface FirecrawlCrawlStatusResponse {error?: string;}-export interface FirecrawlExtractResponse {- statusCode: number;- body: {- success: boolean;- data: any[];- };- error?: string;-}-export enum RateLimiterMode {Crawl = "crawl",CrawlStatus = "crawlStatus",@@ -186,18 +149,3 @@ export enum NotificationType {CONCURRENCY_LIMIT_REACHED = "concurrencyLimitReached",AUTO_RECHARGE_FREQUENT = "autoRechargeFrequent",}--export type ScrapeLog = {- url: string;- scraper: string;- success?: boolean;- response_code?: number;- time_taken_seconds?: number;- proxy?: string;- retried?: boolean;- error_message?: string;- date_added?: string; // ISO 8601 format- html?: string;- ipv4_support?: boolean | null;- ipv6_support?: boolean | null;-};diff --git a/apps/api/src/utils/integration.ts b/apps/api/src/utils/integration.tsindex 8263a0a81..d44b66520 100644--- a/apps/api/src/utils/integration.ts+++ b/apps/api/src/utils/integration.ts@@ -1,6 +1,6 @@import { z } from "zod";-export enum IntegrationEnum {+enum IntegrationEnum {DIFY = "dify",ZAPIER = "zapier",PIPEDREAM = "pipedream",